1 | /* |
2 | * Copyright (c) 2022 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <mach/exclaves.h> |
30 | #include <mach/mach_traps.h> |
31 | #include <kern/misc_protos.h> |
32 | #include <kern/assert.h> |
33 | #include <kern/recount.h> |
34 | #include <kern/startup.h> |
35 | |
36 | #if CONFIG_EXCLAVES |
37 | |
38 | #if CONFIG_SPTM |
39 | #include <arm64/sptm/sptm.h> |
40 | #else |
41 | #error Invalid configuration |
42 | #endif /* CONFIG_SPTM */ |
43 | |
44 | #include <arm/cpu_data_internal.h> |
45 | #include <kern/epoch_sync.h> |
46 | #include <kern/ipc_kobject.h> |
47 | #include <kern/kalloc.h> |
48 | #include <kern/locks.h> |
49 | #include <kern/percpu.h> |
50 | #include <kern/task.h> |
51 | #include <kern/thread.h> |
52 | #include <kern/zalloc.h> |
53 | #include <kern/exclaves_stackshot.h> |
54 | #include <kern/exclaves_test_stackshot.h> |
55 | #include <vm/pmap.h> |
56 | #include <pexpert/pexpert.h> |
57 | #include <pexpert/device_tree.h> |
58 | |
59 | #include <mach/exclaves_l4.h> |
60 | #include <mach/mach_port.h> |
61 | |
62 | #include <Exclaves/Exclaves.h> |
63 | |
64 | #include <IOKit/IOBSD.h> |
65 | |
66 | #include "exclaves_debug.h" |
67 | #include "exclaves_panic.h" |
68 | |
69 | /* External & generated headers */ |
70 | #include <xrt_hosted_types/types.h> |
71 | #include <xnuproxy/messages.h> |
72 | |
73 | /* Use the new version of xnuproxy_msg_t. */ |
74 | #define xnuproxy_msg_t xnuproxy_msg_new_t |
75 | |
76 | #if __has_include(<Tightbeam/tightbeam.h>) |
77 | #include <Tightbeam/tightbeam.h> |
78 | #include <Tightbeam/tightbeam_private.h> |
79 | #endif |
80 | |
81 | #include "exclaves_resource.h" |
82 | #include "exclaves_upcalls.h" |
83 | #include "exclaves_boot.h" |
84 | #include "exclaves_inspection.h" |
85 | #include "exclaves_memory.h" |
86 | |
87 | /* Unslid pointers defining the range of code which switches threads into |
88 | * secure world */ |
89 | uintptr_t exclaves_enter_range_start; |
90 | uintptr_t exclaves_enter_range_end; |
91 | |
92 | /* Unslid pointers defining the range of code which triggers upcall handlers */ |
93 | uintptr_t exclaves_upcall_range_start; |
94 | uintptr_t exclaves_upcall_range_end; |
95 | |
96 | /* Number of allocated ipcb buffers, estimate of active exclave threads */ |
97 | static _Atomic size_t exclaves_ipcb_cnt; |
98 | |
99 | LCK_GRP_DECLARE(exclaves_lck_grp, "exclaves" ); |
100 | |
101 | /* Lock around communication with singleton xnu proxy server thread */ |
102 | LCK_MTX_DECLARE(exclaves_xnu_proxy_lock, &exclaves_lck_grp); |
103 | |
104 | /* Boot lock - only used here for assertions. */ |
105 | extern lck_mtx_t exclaves_boot_lock; |
106 | |
107 | /* |
108 | * Control access to exclaves. Multicore support is learned at runtime. |
109 | */ |
110 | static LCK_MTX_DECLARE(exclaves_scheduler_lock, &exclaves_lck_grp); |
111 | static bool exclaves_multicore; |
112 | #if DEVELOPMENT || DEBUG |
113 | /* boot-arg to control use of the exclaves_scheduler_lock independently of |
114 | * whether exclaves multicore support is enabled */ |
115 | static TUNABLE(bool, exclaves_smp_enabled, "exclaves_smp" , true); |
116 | #else |
117 | #define exclaves_smp_enabled true |
118 | #endif |
119 | |
120 | static xnuproxy_msg_t *exclaves_xnu_proxy_msg_buffer; |
121 | static uint64_t exclaves_xnu_proxy_scid; |
122 | #if XNUPROXY_MSG_VERSION >= 3 |
123 | static pmap_paddr_t exclaves_xnu_proxy_upcall_ipcb_paddr; |
124 | #endif /* XNUPROXY_MSG_VERSION >= 3 */ |
125 | static Exclaves_L4_IpcBuffer_t *exclaves_xnu_proxy_upcall_ipcb; |
126 | |
127 | |
128 | /* |
129 | * Sent/latest offset for updating exclaves clocks |
130 | */ |
131 | typedef struct { |
132 | union { |
133 | /* atomic fields are used via atomic primitives */ |
134 | struct { _Atomic uint64_t sent_offset, latest_offset; } a_u64; |
135 | _Atomic unsigned __int128 a_u128; |
136 | /* non-atomic fields are used via local variable. this is needed to |
137 | * avoid undefined behavior with an atomic struct or accessing atomic |
138 | * fields non-atomically */ |
139 | struct { uint64_t sent_offset, latest_offset; } u64; |
140 | unsigned __int128 u128; |
141 | }; |
142 | } exclaves_clock_t; |
143 | |
144 | static exclaves_clock_t exclaves_absolute_clock, exclaves_continuous_clock; |
145 | |
146 | /* |
147 | * boot-arg to control the service lookup fallback. |
148 | * When set, it allows services in the com.apple.kernel and com.apple.darwin |
149 | * domains to be found when the service can't be found in the attached conclave |
150 | * domain. |
151 | */ |
152 | static TUNABLE(bool, exclaves_service_fallback, "exclaves_service_fallback" , true); |
153 | |
154 | static kern_return_t |
155 | exclaves_acquire_ipc_buffer(Exclaves_L4_IpcBuffer_t **ipcb_out, |
156 | Exclaves_L4_Word_t *scid_out); |
157 | static kern_return_t |
158 | exclaves_relinquish_ipc_buffer(Exclaves_L4_IpcBuffer_t *ipcb, |
159 | Exclaves_L4_Word_t scid); |
160 | static kern_return_t |
161 | exclaves_endpoint_call_internal(ipc_port_t port, exclaves_id_t endpoint_id); |
162 | |
163 | static kern_return_t |
164 | exclaves_enter(void); |
165 | static kern_return_t |
166 | exclaves_bootinfo(uint64_t *out_boot_info, bool *early_enter); |
167 | |
168 | static kern_return_t |
169 | exclaves_scheduler_init(uint64_t boot_info); |
170 | OS_NORETURN OS_NOINLINE |
171 | static void |
172 | exclaves_wait_for_panic(void); |
173 | |
174 | static bool |
175 | exclaves_clock_needs_update(const exclaves_clock_t *clock); |
176 | static kern_return_t |
177 | exclaves_clock_update(exclaves_clock_t *clock, XrtHosted_Buffer_t *save_out_ptr, XrtHosted_Buffer_t *save_in_ptr); |
178 | |
179 | kern_return_t |
180 | exclaves_scheduler_resume_scheduling_context(Exclaves_L4_Word_t scid, |
181 | Exclaves_L4_Word_t *spawned_scid, bool interrupted); |
182 | static kern_return_t |
183 | exclaves_scheduler_boot(void); |
184 | |
185 | static kern_return_t |
186 | exclaves_xnu_proxy_init(uint64_t xnu_proxy_boot_info); |
187 | static kern_return_t |
188 | exclaves_xnu_proxy_allocate_context(Exclaves_L4_Word_t *out_scid, |
189 | Exclaves_L4_IpcBuffer_t **out_ipcb); |
190 | static kern_return_t |
191 | exclaves_xnu_proxy_free_context(Exclaves_L4_Word_t scid); |
192 | static kern_return_t |
193 | exclaves_xnu_proxy_endpoint_call(Exclaves_L4_Word_t endpoint_id); |
194 | static kern_return_t |
195 | exclaves_hosted_error(bool success, XrtHosted_Error_t *error); |
196 | |
197 | /* |
198 | * A static set of exclave epoch counters. |
199 | */ |
200 | static os_atomic(uint64_t) epoch_counter[XrtHosted_Counter_limit] = {}; |
201 | |
202 | static inline os_atomic(uint64_t) * |
203 | exclaves_get_queue_counter(const uint64_t id) |
204 | { |
205 | return &epoch_counter[XrtHosted_Counter_fromQueueId(id)]; |
206 | } |
207 | |
208 | static inline os_atomic(uint64_t) * |
209 | exclaves_get_thread_counter(const uint64_t id) |
210 | { |
211 | return &epoch_counter[XrtHosted_Counter_fromThreadId(id)]; |
212 | } |
213 | |
214 | /* |
215 | * A (simple, for now...) cache of IPC buffers for communicating with XNU-Proxy. |
216 | * Limited in size by the same value as XNU-Proxy's EC limit. |
217 | * Must be realtime-safe. |
218 | */ |
219 | |
220 | static kern_return_t |
221 | exclaves_ipc_buffer_cache_init(void); |
222 | |
223 | /* Intrusive linked list within the unused IPC buffer */ |
224 | struct exclaves_ipc_buffer_cache_item { |
225 | struct exclaves_ipc_buffer_cache_item *next; |
226 | Exclaves_L4_Word_t scid; |
227 | } __attribute__((__packed__)); |
228 | |
229 | _Static_assert(Exclaves_L4_IpcBuffer_Size >= sizeof(struct exclaves_ipc_buffer_cache_item), |
230 | "Invalid Exclaves_L4_IpcBuffer_Size" ); |
231 | |
232 | LCK_SPIN_DECLARE(exclaves_ipc_buffer_cache_lock, &exclaves_lck_grp); |
233 | static struct exclaves_ipc_buffer_cache_item *exclaves_ipc_buffer_cache; |
234 | |
235 | /* -------------------------------------------------------------------------- */ |
236 | #pragma mark exclaves debug configuration |
237 | |
238 | #if DEVELOPMENT || DEBUG |
239 | TUNABLE_WRITEABLE(unsigned int, exclaves_debug, "exclaves_debug" , |
240 | exclaves_debug_show_errors); |
241 | #endif /* DEVELOPMENT || DEBUG */ |
242 | |
243 | #if DEVELOPMENT || DEBUG |
244 | TUNABLE_WRITEABLE(unsigned int, exclaves_ipc_buffer_cache_enabled, "exclaves_ipcb_cache" , 1); |
245 | #else |
246 | #define exclaves_ipc_buffer_cache_enabled 1 |
247 | #endif |
248 | #endif /* CONFIG_EXCLAVES */ |
249 | |
250 | /* -------------------------------------------------------------------------- */ |
251 | #pragma mark userspace entry point |
252 | |
253 | kern_return_t |
254 | _exclaves_ctl_trap(struct exclaves_ctl_trap_args *uap) |
255 | { |
256 | #if CONFIG_EXCLAVES |
257 | kern_return_t kr = KERN_SUCCESS; |
258 | int error = 0; |
259 | |
260 | mach_port_name_t name = uap->name; |
261 | exclaves_id_t identifier = uap->identifier; |
262 | mach_vm_address_t ubuffer = uap->buffer; |
263 | mach_vm_size_t usize = uap->size; |
264 | mach_vm_size_t uoffset = (mach_vm_size_t)uap->identifier; |
265 | mach_vm_size_t usize2 = uap->size2; |
266 | mach_vm_size_t uoffset2 = uap->offset; |
267 | task_t task = current_task(); |
268 | |
269 | /* |
270 | * EXCLAVES_XNU_PROXY_CR_RETVAL comes from ExclavePlatform and is shared |
271 | * with xnu. That header is not shared with userspace. Make sure that |
272 | * the retval userspace picks up is the same as the one |
273 | * xnu/ExclavePlatform thinks it is. |
274 | */ |
275 | assert3p(&EXCLAVES_XNU_PROXY_CR_RETVAL((Exclaves_L4_IpcBuffer_t *)0), ==, |
276 | &XNUPROXY_CR_RETVAL((Exclaves_L4_IpcBuffer_t *)0)); |
277 | |
278 | uint8_t operation = EXCLAVES_CTL_OP(uap->operation_and_flags); |
279 | uint32_t flags = EXCLAVES_CTL_FLAGS(uap->operation_and_flags); |
280 | if (flags != 0) { |
281 | return KERN_INVALID_ARGUMENT; |
282 | } |
283 | |
284 | /* |
285 | * All operations other than OP_BOOT are restricted to properly entitled |
286 | * tasks which can operation in the kernel domain, or those which have |
287 | * joined conclaves (which has its own entitlement check). |
288 | */ |
289 | if (operation != EXCLAVES_CTL_OP_BOOT && |
290 | task_get_conclave(task) == NULL && |
291 | !exclaves_has_priv(task, EXCLAVES_PRIV_KERNEL_DOMAIN)) { |
292 | return KERN_DENIED; |
293 | } |
294 | |
295 | /* |
296 | * As the boot operation itself happens outside the context of any |
297 | * conclave, it requires special privilege. |
298 | */ |
299 | if (operation == EXCLAVES_CTL_OP_BOOT && |
300 | !exclaves_has_priv(current_task(), EXCLAVES_PRIV_BOOT)) { |
301 | return KERN_DENIED; |
302 | } |
303 | |
304 | /* |
305 | * The only valid operation if exclaves are not booted to |
306 | * EXCLAVES_BOOT_STAGE_EXCLAVEKIT, is the BOOT op. |
307 | */ |
308 | if (operation != EXCLAVES_CTL_OP_BOOT) { |
309 | /* |
310 | * Make this EXCLAVES_BOOT_STAGE_2 until userspace is actually |
311 | * triggering the EXCLAVESKIT boot stage. |
312 | */ |
313 | kr = exclaves_boot_wait(EXCLAVES_BOOT_STAGE_2); |
314 | if (kr != KERN_SUCCESS) { |
315 | return kr; |
316 | } |
317 | } |
318 | |
319 | switch (operation) { |
320 | case EXCLAVES_CTL_OP_ENDPOINT_CALL: { |
321 | if (name != MACH_PORT_NULL) { |
322 | /* Only accept MACH_PORT_NULL for now */ |
323 | return KERN_INVALID_CAPABILITY; |
324 | } |
325 | if (ubuffer == USER_ADDR_NULL || usize == 0 || |
326 | usize != Exclaves_L4_IpcBuffer_Size) { |
327 | return KERN_INVALID_ARGUMENT; |
328 | } |
329 | |
330 | Exclaves_L4_IpcBuffer_t *ipcb; |
331 | if ((error = exclaves_allocate_ipc_buffer((void**)&ipcb))) { |
332 | return error; |
333 | } |
334 | assert(ipcb != NULL); |
335 | if ((error = copyin(ubuffer, ipcb, usize))) { |
336 | return error; |
337 | } |
338 | |
339 | if (identifier >= CONCLAVE_SERVICE_MAX) { |
340 | return KERN_INVALID_ARGUMENT; |
341 | } |
342 | |
343 | /* |
344 | * Verify that the service actually exists in the current |
345 | * domain (only when the fallbacks are not enabled). |
346 | */ |
347 | if (!exclaves_service_fallback && |
348 | !exclaves_conclave_has_service(task_get_conclave(task), |
349 | identifier)) { |
350 | return KERN_INVALID_ARGUMENT; |
351 | } |
352 | |
353 | kr = exclaves_endpoint_call_internal(IPC_PORT_NULL, identifier); |
354 | error = copyout(ipcb, ubuffer, usize); |
355 | /* |
356 | * Endpoint call to conclave may have trigger a stop upcall, |
357 | * check if stop upcall completion handler needs to run. |
358 | */ |
359 | task_stop_conclave_upcall_complete(); |
360 | if (error) { |
361 | return error; |
362 | } |
363 | break; |
364 | } |
365 | |
366 | case EXCLAVES_CTL_OP_NAMED_BUFFER_CREATE: { |
367 | if (name != MACH_PORT_NULL) { |
368 | /* Only accept MACH_PORT_NULL for now */ |
369 | return KERN_INVALID_CAPABILITY; |
370 | } |
371 | |
372 | size_t len = 0; |
373 | char id_name[XNUPROXY_RESOURCE_NAME_MAX] = "" ; |
374 | if (copyinstr(identifier, id_name, XNUPROXY_RESOURCE_NAME_MAX, |
375 | &len) != 0 || id_name[0] == '\0') { |
376 | return KERN_INVALID_ARGUMENT; |
377 | } |
378 | |
379 | exclaves_buffer_perm_t perm = (exclaves_buffer_perm_t)usize2; |
380 | const exclaves_buffer_perm_t supported = |
381 | EXCLAVES_BUFFER_PERM_READ | EXCLAVES_BUFFER_PERM_WRITE; |
382 | if ((perm & supported) == 0 || (perm & ~supported) != 0) { |
383 | return KERN_INVALID_ARGUMENT; |
384 | } |
385 | |
386 | const char *domain = exclaves_conclave_get_domain(task_get_conclave(task)); |
387 | const bool new_api = |
388 | (perm == EXCLAVES_BUFFER_PERM_READ) || |
389 | (perm == EXCLAVES_BUFFER_PERM_WRITE); |
390 | const bool shared_mem_available = |
391 | exclaves_resource_lookup_by_name(domain, id_name, |
392 | XNUPROXY_RESOURCE_SHARED_MEMORY) != NULL; |
393 | const bool use_shared_mem = new_api && shared_mem_available; |
394 | |
395 | exclaves_resource_t *resource = NULL; |
396 | kr = use_shared_mem ? |
397 | exclaves_resource_shared_memory_map(domain, id_name, usize, perm, &resource) : |
398 | exclaves_named_buffer_map(domain, id_name, usize, perm, &resource); |
399 | if (kr != KERN_SUCCESS) { |
400 | return kr; |
401 | } |
402 | |
403 | kr = exclaves_resource_create_port_name(resource, |
404 | current_space(), &name); |
405 | if (kr != KERN_SUCCESS) { |
406 | return kr; |
407 | } |
408 | |
409 | kr = copyout(&name, ubuffer, sizeof(mach_port_name_t)); |
410 | if (kr != KERN_SUCCESS) { |
411 | mach_port_deallocate(current_space(), name); |
412 | return kr; |
413 | } |
414 | |
415 | break; |
416 | } |
417 | |
418 | case EXCLAVES_CTL_OP_NAMED_BUFFER_COPYIN: { |
419 | exclaves_resource_t *resource = NULL; |
420 | kr = exclaves_resource_from_port_name(current_space(), name, |
421 | &resource); |
422 | if (kr != KERN_SUCCESS) { |
423 | return kr; |
424 | } |
425 | |
426 | switch (resource->r_type) { |
427 | case XNUPROXY_RESOURCE_NAMED_BUFFER: |
428 | kr = exclaves_named_buffer_copyin(resource, ubuffer, |
429 | usize, uoffset, usize2, uoffset2); |
430 | break; |
431 | |
432 | case XNUPROXY_RESOURCE_SHARED_MEMORY: |
433 | kr = exclaves_resource_shared_memory_copyin(resource, |
434 | ubuffer, usize, uoffset, usize2, uoffset2); |
435 | break; |
436 | |
437 | default: |
438 | exclaves_resource_release(resource); |
439 | return KERN_INVALID_CAPABILITY; |
440 | } |
441 | |
442 | exclaves_resource_release(resource); |
443 | |
444 | if (kr != KERN_SUCCESS) { |
445 | return kr; |
446 | } |
447 | break; |
448 | } |
449 | |
450 | case EXCLAVES_CTL_OP_NAMED_BUFFER_COPYOUT: { |
451 | exclaves_resource_t *resource = NULL; |
452 | kr = exclaves_resource_from_port_name(current_space(), name, |
453 | &resource); |
454 | if (kr != KERN_SUCCESS) { |
455 | return kr; |
456 | } |
457 | |
458 | switch (resource->r_type) { |
459 | case XNUPROXY_RESOURCE_NAMED_BUFFER: |
460 | kr = exclaves_named_buffer_copyout(resource, ubuffer, |
461 | usize, uoffset, usize2, uoffset2); |
462 | break; |
463 | |
464 | case XNUPROXY_RESOURCE_SHARED_MEMORY: |
465 | kr = exclaves_resource_shared_memory_copyout(resource, |
466 | ubuffer, usize, uoffset, usize2, uoffset2); |
467 | break; |
468 | |
469 | default: |
470 | exclaves_resource_release(resource); |
471 | return KERN_INVALID_CAPABILITY; |
472 | } |
473 | |
474 | exclaves_resource_release(resource); |
475 | |
476 | if (kr != KERN_SUCCESS) { |
477 | return kr; |
478 | } |
479 | break; |
480 | } |
481 | |
482 | case EXCLAVES_CTL_OP_BOOT: |
483 | if (name != MACH_PORT_NULL) { |
484 | /* Only accept MACH_PORT_NULL for now */ |
485 | return KERN_INVALID_CAPABILITY; |
486 | } |
487 | kr = exclaves_boot((uint32_t)identifier); |
488 | break; |
489 | |
490 | case EXCLAVES_CTL_OP_LAUNCH_CONCLAVE: |
491 | if (name != MACH_PORT_NULL) { |
492 | /* Only accept MACH_PORT_NULL for now */ |
493 | return KERN_INVALID_CAPABILITY; |
494 | } |
495 | kr = task_launch_conclave(name); |
496 | |
497 | /* |
498 | * Conclave launch call to may have trigger a stop upcall, |
499 | * check if stop upcall completion handler needs to run. |
500 | */ |
501 | task_stop_conclave_upcall_complete(); |
502 | break; |
503 | |
504 | case EXCLAVES_CTL_OP_LOOKUP_SERVICES: { |
505 | if (name != MACH_PORT_NULL) { |
506 | /* Only accept MACH_PORT_NULL for now */ |
507 | return KERN_INVALID_CAPABILITY; |
508 | } |
509 | struct exclaves_resource_user uresource = {}; |
510 | |
511 | if (usize > (MAX_CONCLAVE_RESOURCE_NUM * sizeof(struct exclaves_resource_user)) || |
512 | (usize % sizeof(struct exclaves_resource_user) != 0)) { |
513 | return KERN_INVALID_ARGUMENT; |
514 | } |
515 | |
516 | if ((ubuffer == USER_ADDR_NULL && usize != 0) || |
517 | (usize == 0 && ubuffer != USER_ADDR_NULL)) { |
518 | return KERN_INVALID_ARGUMENT; |
519 | } |
520 | |
521 | if (ubuffer == USER_ADDR_NULL) { |
522 | return KERN_INVALID_ARGUMENT; |
523 | } |
524 | |
525 | /* For the moment we only ever have to deal with one request. */ |
526 | if (usize != sizeof(struct exclaves_resource_user)) { |
527 | return KERN_INVALID_ARGUMENT; |
528 | } |
529 | error = copyin(ubuffer, &uresource, usize); |
530 | if (error) { |
531 | return KERN_INVALID_ARGUMENT; |
532 | } |
533 | |
534 | const size_t name_buf_len = sizeof(uresource.r_name); |
535 | if (strnlen(uresource.r_name, name_buf_len) == name_buf_len) { |
536 | return KERN_INVALID_ARGUMENT; |
537 | } |
538 | |
539 | /* |
540 | * Do the regular lookup first. If that fails, fallback to the |
541 | * DARWIN domain, finally fallback to the KERNEL domain. |
542 | */ |
543 | const char *domain = exclaves_conclave_get_domain(task_get_conclave(task)); |
544 | uint64_t id = exclaves_service_lookup(domain, uresource.r_name); |
545 | |
546 | /* Disable fallbacks via boot-arg. */ |
547 | if (exclaves_service_fallback) { |
548 | if (id == UINT64_C(~0)) { |
549 | id = exclaves_service_lookup(EXCLAVES_DOMAIN_DARWIN, |
550 | uresource.r_name); |
551 | } |
552 | if (id == UINT64_C(~0)) { |
553 | id = exclaves_service_lookup(EXCLAVES_DOMAIN_KERNEL, |
554 | uresource.r_name); |
555 | } |
556 | } |
557 | |
558 | if (id == UINT64_C(~0)) { |
559 | return KERN_NOT_FOUND; |
560 | } |
561 | |
562 | uresource.r_id = id; |
563 | uresource.r_port = MACH_PORT_NULL; |
564 | |
565 | error = copyout(&uresource, ubuffer, usize); |
566 | if (error) { |
567 | return KERN_INVALID_ADDRESS; |
568 | } |
569 | |
570 | kr = KERN_SUCCESS; |
571 | break; |
572 | } |
573 | |
574 | case EXCLAVES_CTL_OP_AUDIO_BUFFER_CREATE: { |
575 | if (identifier == 0) { |
576 | return KERN_INVALID_ARGUMENT; |
577 | } |
578 | |
579 | /* copy in string name */ |
580 | char id_name[XNUPROXY_RESOURCE_NAME_MAX] = "" ; |
581 | size_t done = 0; |
582 | if (copyinstr(identifier, id_name, XNUPROXY_RESOURCE_NAME_MAX, &done) != 0) { |
583 | return KERN_INVALID_ARGUMENT; |
584 | } |
585 | |
586 | const char *domain = exclaves_conclave_get_domain(task_get_conclave(task)); |
587 | const bool use_audio_memory = |
588 | exclaves_resource_lookup_by_name(domain, id_name, |
589 | XNUPROXY_RESOURCE_ARBITRATED_AUDIO_MEMORY) != NULL; |
590 | exclaves_resource_t *resource = NULL; |
591 | kr = use_audio_memory ? |
592 | exclaves_resource_audio_memory_map(domain, id_name, usize, &resource) : |
593 | exclaves_audio_buffer_map(domain, id_name, usize, &resource); |
594 | if (kr != KERN_SUCCESS) { |
595 | return kr; |
596 | } |
597 | |
598 | kr = exclaves_resource_create_port_name(resource, current_space(), |
599 | &name); |
600 | if (kr != KERN_SUCCESS) { |
601 | return kr; |
602 | } |
603 | |
604 | kr = copyout(&name, ubuffer, sizeof(mach_port_name_t)); |
605 | if (kr != KERN_SUCCESS) { |
606 | mach_port_deallocate(current_space(), name); |
607 | return kr; |
608 | } |
609 | |
610 | break; |
611 | } |
612 | |
613 | case EXCLAVES_CTL_OP_AUDIO_BUFFER_COPYOUT: { |
614 | exclaves_resource_t *resource; |
615 | |
616 | kr = exclaves_resource_from_port_name(current_space(), name, &resource); |
617 | if (kr != KERN_SUCCESS) { |
618 | return kr; |
619 | } |
620 | |
621 | switch (resource->r_type) { |
622 | case XNUPROXY_RESOURCE_ARBITRATED_AUDIO_BUFFER: |
623 | kr = exclaves_audio_buffer_copyout(resource, ubuffer, |
624 | usize, uoffset, usize2, uoffset2); |
625 | break; |
626 | |
627 | case XNUPROXY_RESOURCE_ARBITRATED_AUDIO_MEMORY: |
628 | kr = exclaves_resource_audio_memory_copyout(resource, |
629 | ubuffer, usize, uoffset, usize2, uoffset2); |
630 | break; |
631 | |
632 | default: |
633 | exclaves_resource_release(resource); |
634 | return KERN_INVALID_CAPABILITY; |
635 | } |
636 | |
637 | exclaves_resource_release(resource); |
638 | |
639 | if (kr != KERN_SUCCESS) { |
640 | return kr; |
641 | } |
642 | |
643 | break; |
644 | } |
645 | |
646 | case EXCLAVES_CTL_OP_SENSOR_CREATE: { |
647 | if (identifier == 0) { |
648 | return KERN_INVALID_ARGUMENT; |
649 | } |
650 | |
651 | /* copy in string name */ |
652 | char id_name[XNUPROXY_RESOURCE_NAME_MAX] = "" ; |
653 | size_t done = 0; |
654 | if (copyinstr(identifier, id_name, XNUPROXY_RESOURCE_NAME_MAX, &done) != 0) { |
655 | return KERN_INVALID_ARGUMENT; |
656 | } |
657 | |
658 | const char *domain = exclaves_conclave_get_domain(task_get_conclave(task)); |
659 | exclaves_resource_t *resource = NULL; |
660 | kr = exclaves_resource_sensor_open(domain, id_name, &resource); |
661 | if (kr != KERN_SUCCESS) { |
662 | return kr; |
663 | } |
664 | |
665 | kr = exclaves_resource_create_port_name(resource, current_space(), |
666 | &name); |
667 | if (kr != KERN_SUCCESS) { |
668 | return kr; |
669 | } |
670 | |
671 | kr = copyout(&name, ubuffer, sizeof(mach_port_name_t)); |
672 | if (kr != KERN_SUCCESS) { |
673 | /* No senders drops the reference. */ |
674 | mach_port_deallocate(current_space(), name); |
675 | return kr; |
676 | } |
677 | |
678 | break; |
679 | } |
680 | |
681 | case EXCLAVES_CTL_OP_SENSOR_START: { |
682 | exclaves_resource_t *resource; |
683 | kr = exclaves_resource_from_port_name(current_space(), name, &resource); |
684 | if (kr != KERN_SUCCESS) { |
685 | return kr; |
686 | } |
687 | |
688 | if (resource->r_type != XNUPROXY_RESOURCE_SENSOR) { |
689 | exclaves_resource_release(resource); |
690 | return KERN_FAILURE; |
691 | } |
692 | |
693 | exclaves_sensor_status_t status; |
694 | kr = exclaves_resource_sensor_start(resource, identifier, &status); |
695 | |
696 | exclaves_resource_release(resource); |
697 | |
698 | if (kr != KERN_SUCCESS) { |
699 | return kr; |
700 | } |
701 | |
702 | kr = copyout(&status, ubuffer, sizeof(exclaves_sensor_status_t)); |
703 | |
704 | break; |
705 | } |
706 | case EXCLAVES_CTL_OP_SENSOR_STOP: { |
707 | exclaves_resource_t *resource; |
708 | kr = exclaves_resource_from_port_name(current_space(), name, &resource); |
709 | if (kr != KERN_SUCCESS) { |
710 | return kr; |
711 | } |
712 | |
713 | if (resource->r_type != XNUPROXY_RESOURCE_SENSOR) { |
714 | exclaves_resource_release(resource); |
715 | return KERN_FAILURE; |
716 | } |
717 | |
718 | exclaves_sensor_status_t status; |
719 | kr = exclaves_resource_sensor_stop(resource, identifier, &status); |
720 | |
721 | exclaves_resource_release(resource); |
722 | |
723 | if (kr != KERN_SUCCESS) { |
724 | return kr; |
725 | } |
726 | |
727 | kr = copyout(&status, ubuffer, sizeof(exclaves_sensor_status_t)); |
728 | |
729 | break; |
730 | } |
731 | case EXCLAVES_CTL_OP_SENSOR_STATUS: { |
732 | exclaves_resource_t *resource; |
733 | kr = exclaves_resource_from_port_name(current_space(), name, &resource); |
734 | if (kr != KERN_SUCCESS) { |
735 | return kr; |
736 | } |
737 | |
738 | if (resource->r_type != XNUPROXY_RESOURCE_SENSOR) { |
739 | exclaves_resource_release(resource); |
740 | return KERN_FAILURE; |
741 | } |
742 | |
743 | |
744 | exclaves_sensor_status_t status; |
745 | kr = exclaves_resource_sensor_status(resource, identifier, &status); |
746 | |
747 | exclaves_resource_release(resource); |
748 | |
749 | if (kr != KERN_SUCCESS) { |
750 | return kr; |
751 | } |
752 | |
753 | kr = copyout(&status, ubuffer, sizeof(exclaves_sensor_status_t)); |
754 | break; |
755 | } |
756 | case EXCLAVES_CTL_OP_NOTIFICATION_RESOURCE_LOOKUP: { |
757 | exclaves_resource_t *notification_resource = NULL; |
758 | mach_port_name_t port_name = MACH_PORT_NULL; |
759 | |
760 | struct exclaves_resource_user *notification_resource_user = NULL; |
761 | if (usize != sizeof(struct exclaves_resource_user)) { |
762 | return KERN_INVALID_ARGUMENT; |
763 | } |
764 | |
765 | if (ubuffer == USER_ADDR_NULL) { |
766 | return KERN_INVALID_ARGUMENT; |
767 | } |
768 | |
769 | notification_resource_user = (struct exclaves_resource_user *) |
770 | kalloc_data(usize, Z_WAITOK | Z_ZERO | Z_NOFAIL); |
771 | |
772 | error = copyin(ubuffer, notification_resource_user, usize); |
773 | if (error) { |
774 | kr = KERN_INVALID_ARGUMENT; |
775 | goto notification_resource_lookup_out; |
776 | } |
777 | |
778 | const size_t name_buf_len = sizeof(notification_resource_user->r_name); |
779 | if (strnlen(notification_resource_user->r_name, name_buf_len) |
780 | == name_buf_len) { |
781 | kr = KERN_INVALID_ARGUMENT; |
782 | goto notification_resource_lookup_out; |
783 | } |
784 | |
785 | const char *domain = exclaves_conclave_get_domain(task_get_conclave(task)); |
786 | kr = exclaves_notification_create(domain, |
787 | notification_resource_user->r_name, ¬ification_resource); |
788 | if (kr != KERN_SUCCESS) { |
789 | goto notification_resource_lookup_out; |
790 | } |
791 | |
792 | kr = exclaves_resource_create_port_name(notification_resource, |
793 | current_space(), &port_name); |
794 | if (kr != KERN_SUCCESS) { |
795 | goto notification_resource_lookup_out; |
796 | } |
797 | notification_resource_user->r_type = notification_resource->r_type; |
798 | notification_resource_user->r_id = notification_resource->r_id; |
799 | notification_resource_user->r_port = port_name; |
800 | error = copyout(notification_resource_user, ubuffer, usize); |
801 | if (error) { |
802 | kr = KERN_INVALID_ADDRESS; |
803 | goto notification_resource_lookup_out; |
804 | } |
805 | |
806 | notification_resource_lookup_out: |
807 | if (notification_resource_user != NULL) { |
808 | kfree_data(notification_resource_user, usize); |
809 | } |
810 | if (kr != KERN_SUCCESS && port_name != MACH_PORT_NULL) { |
811 | mach_port_deallocate(current_space(), port_name); |
812 | } |
813 | break; |
814 | } |
815 | |
816 | default: |
817 | kr = KERN_INVALID_ARGUMENT; |
818 | break; |
819 | } |
820 | |
821 | return kr; |
822 | #else /* CONFIG_EXCLAVES */ |
823 | #pragma unused(uap) |
824 | return KERN_NOT_SUPPORTED; |
825 | #endif /* CONFIG_EXCLAVES */ |
826 | } |
827 | |
828 | /* -------------------------------------------------------------------------- */ |
829 | #pragma mark kernel entry points |
830 | |
831 | kern_return_t |
832 | exclaves_endpoint_call(ipc_port_t port, exclaves_id_t endpoint_id, |
833 | exclaves_tag_t *tag, exclaves_error_t *error) |
834 | { |
835 | #if CONFIG_EXCLAVES |
836 | kern_return_t kr = KERN_SUCCESS; |
837 | assert(port == IPC_PORT_NULL); |
838 | |
839 | Exclaves_L4_IpcBuffer_t *ipcb = Exclaves_L4_IpcBuffer(); |
840 | assert(ipcb != NULL); |
841 | |
842 | exclaves_debug_printf(show_progress, |
843 | "exclaves: endpoint call:\tendpoint id %lld tag 0x%llx\n" , |
844 | endpoint_id, *tag); |
845 | |
846 | ipcb->mr[Exclaves_L4_Ipc_Mr_Tag] = *tag; |
847 | kr = exclaves_endpoint_call_internal(port, endpoint_id); |
848 | *tag = ipcb->mr[Exclaves_L4_Ipc_Mr_Tag]; |
849 | *error = XNUPROXY_CR_RETVAL(ipcb); |
850 | |
851 | exclaves_debug_printf(show_progress, |
852 | "exclaves: endpoint call return:\tendpoint id %lld tag 0x%llx " |
853 | "error 0x%llx\n" , endpoint_id, *tag, *error); |
854 | |
855 | return kr; |
856 | #else /* CONFIG_EXCLAVES */ |
857 | #pragma unused(port, endpoint_id, tag, error) |
858 | return KERN_NOT_SUPPORTED; |
859 | #endif /* CONFIG_EXCLAVES */ |
860 | } |
861 | |
862 | /* Realtime-safe acquisition of an IPC buffer */ |
863 | kern_return_t |
864 | exclaves_allocate_ipc_buffer(void **out_ipc_buffer) |
865 | { |
866 | #if CONFIG_EXCLAVES |
867 | kern_return_t kr = KERN_SUCCESS; |
868 | thread_t thread = current_thread(); |
869 | Exclaves_L4_IpcBuffer_t *ipcb = thread->th_exclaves_ipc_buffer; |
870 | Exclaves_L4_Word_t scid = thread->th_exclaves_scheduling_context_id; |
871 | |
872 | if (ipcb == NULL) { |
873 | assert(scid == 0); |
874 | if ((kr = exclaves_acquire_ipc_buffer(&ipcb, &scid))) { |
875 | return kr; |
876 | } |
877 | thread->th_exclaves_ipc_buffer = ipcb; |
878 | thread->th_exclaves_scheduling_context_id = scid; |
879 | } |
880 | if (out_ipc_buffer) { |
881 | *out_ipc_buffer = (void*)ipcb; |
882 | } |
883 | |
884 | return kr; |
885 | #else /* CONFIG_EXCLAVES */ |
886 | #pragma unused(out_ipc_buffer) |
887 | return KERN_NOT_SUPPORTED; |
888 | #endif /* CONFIG_EXCLAVES */ |
889 | } |
890 | |
891 | #if CONFIG_EXCLAVES |
892 | static kern_return_t |
893 | exclaves_thread_free_ipc_buffer(thread_t thread) |
894 | { |
895 | kern_return_t kr = KERN_SUCCESS; |
896 | Exclaves_L4_IpcBuffer_t *ipcb = thread->th_exclaves_ipc_buffer; |
897 | Exclaves_L4_Word_t scid = thread->th_exclaves_scheduling_context_id; |
898 | |
899 | if (ipcb != NULL) { |
900 | assert(scid != 0); |
901 | thread->th_exclaves_ipc_buffer = NULL; |
902 | thread->th_exclaves_scheduling_context_id = 0; |
903 | |
904 | kr = exclaves_relinquish_ipc_buffer(ipcb, scid); |
905 | } else { |
906 | assert(scid == 0); |
907 | } |
908 | |
909 | return kr; |
910 | } |
911 | #endif /* CONFIG_EXCLAVES */ |
912 | |
913 | kern_return_t |
914 | exclaves_free_ipc_buffer(void) |
915 | { |
916 | #if CONFIG_EXCLAVES |
917 | thread_t thread = current_thread(); |
918 | |
919 | /* The inspection thread's cached buffer should never be freed */ |
920 | if ((os_atomic_load(&thread->th_exclaves_inspection_state, relaxed) & TH_EXCLAVES_INSPECTION_NOINSPECT) != 0) { |
921 | return KERN_SUCCESS; |
922 | } |
923 | |
924 | return exclaves_thread_free_ipc_buffer(thread); |
925 | #else /* CONFIG_EXCLAVES */ |
926 | return KERN_NOT_SUPPORTED; |
927 | #endif /* CONFIG_EXCLAVES */ |
928 | } |
929 | |
930 | kern_return_t |
931 | exclaves_thread_terminate(__unused thread_t thread) |
932 | { |
933 | kern_return_t kr = KERN_SUCCESS; |
934 | |
935 | #if CONFIG_EXCLAVES |
936 | assert(thread == current_thread()); |
937 | assert(thread->th_exclaves_intstate == 0); |
938 | assert(thread->th_exclaves_state == 0); |
939 | if (thread->th_exclaves_ipc_buffer) { |
940 | exclaves_debug_printf(show_progress, |
941 | "exclaves: thread_terminate freeing abandoned exclaves " |
942 | "ipc buffer\n" ); |
943 | kr = exclaves_thread_free_ipc_buffer(thread); |
944 | assert(kr == KERN_SUCCESS); |
945 | } |
946 | #else |
947 | #pragma unused(thread) |
948 | #endif /* CONFIG_EXCLAVES */ |
949 | |
950 | return kr; |
951 | } |
952 | |
953 | OS_CONST |
954 | void* |
955 | exclaves_get_ipc_buffer(void) |
956 | { |
957 | #if CONFIG_EXCLAVES |
958 | thread_t thread = current_thread(); |
959 | Exclaves_L4_IpcBuffer_t *ipcb = thread->th_exclaves_ipc_buffer; |
960 | assert(ipcb != NULL); |
961 | |
962 | return ipcb; |
963 | #else /* CONFIG_EXCLAVES */ |
964 | return NULL; |
965 | #endif /* CONFIG_EXCLAVES */ |
966 | } |
967 | |
968 | #if CONFIG_EXCLAVES |
969 | |
970 | __startup_func |
971 | static void |
972 | initialize_exclaves_call_range(void) |
973 | { |
974 | exclaves_enter_range_start = VM_KERNEL_UNSLIDE(&exclaves_enter_start_label); |
975 | assert3u(exclaves_enter_range_start, !=, 0); |
976 | exclaves_enter_range_end = VM_KERNEL_UNSLIDE(&exclaves_enter_end_label); |
977 | assert3u(exclaves_enter_range_end, !=, 0); |
978 | exclaves_upcall_range_start = VM_KERNEL_UNSLIDE(&exclaves_upcall_start_label); |
979 | assert3u(exclaves_upcall_range_start, !=, 0); |
980 | exclaves_upcall_range_end = VM_KERNEL_UNSLIDE(&exclaves_upcall_end_label); |
981 | assert3u(exclaves_upcall_range_end, !=, 0); |
982 | } |
983 | STARTUP(EARLY_BOOT, STARTUP_RANK_MIDDLE, initialize_exclaves_call_range); |
984 | |
985 | static void |
986 | bind_to_boot_core(void) |
987 | { |
988 | /* |
989 | * First ensure the boot cluster isn't powered down preventing the |
990 | * thread from running at all. |
991 | */ |
992 | suspend_cluster_powerdown(); |
993 | const int cpu = ml_get_boot_cpu_number(); |
994 | processor_t processor = cpu_to_processor(cpu); |
995 | assert3p(processor, !=, NULL); |
996 | __assert_only processor_t old = thread_bind(processor); |
997 | assert3p(old, ==, PROCESSOR_NULL); |
998 | thread_block(THREAD_CONTINUE_NULL); |
999 | } |
1000 | |
1001 | static void |
1002 | unbind_from_boot_core(void) |
1003 | { |
1004 | /* Unbind the thread from the boot CPU. */ |
1005 | thread_bind(PROCESSOR_NULL); |
1006 | thread_block(THREAD_CONTINUE_NULL); |
1007 | resume_cluster_powerdown(); |
1008 | } |
1009 | |
1010 | extern kern_return_t exclaves_boot_early(void); |
1011 | kern_return_t |
1012 | exclaves_boot_early(void) |
1013 | { |
1014 | kern_return_t kr = KERN_FAILURE; |
1015 | uint64_t boot_info = 0; |
1016 | bool early_enter = false; |
1017 | |
1018 | lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED); |
1019 | |
1020 | kr = exclaves_bootinfo(&boot_info, &early_enter); |
1021 | if (kr != KERN_SUCCESS) { |
1022 | exclaves_debug_printf(show_errors, |
1023 | "exclaves: Get bootinfo failed\n" ); |
1024 | return kr; |
1025 | } |
1026 | |
1027 | if (early_enter) { |
1028 | thread_t thread = current_thread(); |
1029 | assert3u(thread->th_exclaves_state & TH_EXCLAVES_STATE_ANY, ==, 0); |
1030 | |
1031 | bind_to_boot_core(); |
1032 | |
1033 | disable_preemption_without_measurements(); |
1034 | thread->th_exclaves_state |= TH_EXCLAVES_SCHEDULER_CALL; |
1035 | |
1036 | kr = exclaves_enter(); |
1037 | |
1038 | thread->th_exclaves_state &= ~TH_EXCLAVES_SCHEDULER_CALL; |
1039 | enable_preemption(); |
1040 | |
1041 | unbind_from_boot_core(); |
1042 | |
1043 | if (kr != KERN_SUCCESS) { |
1044 | exclaves_debug_printf(show_errors, |
1045 | "exclaves: early exclaves enter failed\n" ); |
1046 | if (kr == KERN_ABORTED) { |
1047 | panic("Unexpected ringgate panic status" ); |
1048 | } |
1049 | return kr; |
1050 | } |
1051 | } |
1052 | |
1053 | kr = exclaves_scheduler_init(boot_info); |
1054 | if (kr != KERN_SUCCESS) { |
1055 | exclaves_debug_printf(show_errors, |
1056 | "exclaves: Init scheduler failed\n" ); |
1057 | return kr; |
1058 | } |
1059 | |
1060 | kr = exclaves_ipc_buffer_cache_init(); |
1061 | if (kr != KERN_SUCCESS) { |
1062 | exclaves_debug_printf(show_errors, |
1063 | "exclaves: failed to initialize IPC buffer cache\n" ); |
1064 | return kr; |
1065 | } |
1066 | |
1067 | kr = exclaves_resource_init(); |
1068 | if (kr != KERN_SUCCESS) { |
1069 | exclaves_debug_printf(show_errors, |
1070 | "exclaves: failed to initialize resources\n" ); |
1071 | return kr; |
1072 | } |
1073 | |
1074 | return KERN_SUCCESS; |
1075 | } |
1076 | #endif /* CONFIG_EXCLAVES */ |
1077 | |
1078 | #if CONFIG_EXCLAVES |
1079 | static struct XrtHosted_Callbacks *exclaves_callbacks = NULL; |
1080 | #endif /* CONFIG_EXCLAVES */ |
1081 | |
1082 | void |
1083 | exclaves_register_xrt_hosted_callbacks(struct XrtHosted_Callbacks *callbacks) |
1084 | { |
1085 | #if CONFIG_EXCLAVES |
1086 | if (exclaves_callbacks == NULL) { |
1087 | exclaves_callbacks = callbacks; |
1088 | } |
1089 | #else /* CONFIG_EXCLAVES */ |
1090 | #pragma unused(callbacks) |
1091 | #endif /* CONFIG_EXCLAVES */ |
1092 | } |
1093 | |
1094 | void |
1095 | exclaves_update_timebase(exclaves_clock_type_t type, uint64_t offset) |
1096 | { |
1097 | #if CONFIG_EXCLAVES |
1098 | exclaves_clock_t *clock = (type == EXCLAVES_CLOCK_ABSOLUTE ? |
1099 | &exclaves_absolute_clock : &exclaves_continuous_clock); |
1100 | uint64_t latest_offset = os_atomic_load(&clock->a_u64.latest_offset, relaxed); |
1101 | while (latest_offset < offset) { |
1102 | /* Update the latest offset with the new offset. If this fails, then a |
1103 | * concurrent update occurred and our offset may be stale. */ |
1104 | if (os_atomic_cmpxchgv(&clock->a_u64.latest_offset, latest_offset, |
1105 | offset, &latest_offset, relaxed)) { |
1106 | break; |
1107 | } |
1108 | } |
1109 | #else |
1110 | #pragma unused(type, offset) |
1111 | #endif /* CONFIG_EXCLAVES */ |
1112 | } |
1113 | |
1114 | /* -------------------------------------------------------------------------- */ |
1115 | |
1116 | #pragma mark exclaves ipc internals |
1117 | |
1118 | #if CONFIG_EXCLAVES |
1119 | |
1120 | static kern_return_t |
1121 | exclaves_acquire_ipc_buffer(Exclaves_L4_IpcBuffer_t **out_ipcb, |
1122 | Exclaves_L4_Word_t *out_scid) |
1123 | { |
1124 | kern_return_t kr = KERN_SUCCESS; |
1125 | Exclaves_L4_IpcBuffer_t *ipcb = NULL; |
1126 | Exclaves_L4_Word_t scid = 0; |
1127 | struct exclaves_ipc_buffer_cache_item *cached_buffer = NULL; |
1128 | |
1129 | |
1130 | _Static_assert(Exclaves_L4_IpcBuffer_Size < PAGE_SIZE, |
1131 | "Invalid Exclaves_L4_IpcBuffer_Size" ); |
1132 | |
1133 | if (exclaves_ipc_buffer_cache_enabled) { |
1134 | lck_spin_lock(&exclaves_ipc_buffer_cache_lock); |
1135 | if (exclaves_ipc_buffer_cache != NULL) { |
1136 | cached_buffer = exclaves_ipc_buffer_cache; |
1137 | exclaves_ipc_buffer_cache = cached_buffer->next; |
1138 | } |
1139 | lck_spin_unlock(&exclaves_ipc_buffer_cache_lock); |
1140 | } |
1141 | |
1142 | if (cached_buffer) { |
1143 | scid = cached_buffer->scid; |
1144 | |
1145 | /* zero out this usage of the buffer to avoid any confusion in xnuproxy */ |
1146 | cached_buffer->next = NULL; |
1147 | cached_buffer->scid = 0; |
1148 | |
1149 | ipcb = (Exclaves_L4_IpcBuffer_t*)cached_buffer; |
1150 | } else { |
1151 | kr = exclaves_xnu_proxy_allocate_context(&scid, &ipcb); |
1152 | if (kr == KERN_NO_SPACE) { |
1153 | panic("Exclaves IPC buffer allocation failed" ); |
1154 | } |
1155 | } |
1156 | |
1157 | *out_ipcb = ipcb; |
1158 | *out_scid = scid; |
1159 | |
1160 | return kr; |
1161 | } |
1162 | |
1163 | size_t |
1164 | exclaves_ipc_buffer_count(void) |
1165 | { |
1166 | return os_atomic_load(&exclaves_ipcb_cnt, relaxed); |
1167 | } |
1168 | |
1169 | static kern_return_t |
1170 | exclaves_relinquish_ipc_buffer(Exclaves_L4_IpcBuffer_t *ipcb, |
1171 | Exclaves_L4_Word_t scid) |
1172 | { |
1173 | kern_return_t kr = KERN_SUCCESS; |
1174 | struct exclaves_ipc_buffer_cache_item *cached_buffer; |
1175 | |
1176 | if (!exclaves_ipc_buffer_cache_enabled) { |
1177 | kr = exclaves_xnu_proxy_free_context(scid); |
1178 | } else { |
1179 | cached_buffer = (struct exclaves_ipc_buffer_cache_item*)ipcb; |
1180 | cached_buffer->scid = scid; |
1181 | |
1182 | lck_spin_lock(&exclaves_ipc_buffer_cache_lock); |
1183 | cached_buffer->next = exclaves_ipc_buffer_cache; |
1184 | exclaves_ipc_buffer_cache = cached_buffer; |
1185 | lck_spin_unlock(&exclaves_ipc_buffer_cache_lock); |
1186 | } |
1187 | |
1188 | return kr; |
1189 | } |
1190 | |
1191 | static kern_return_t |
1192 | exclaves_endpoint_call_internal(__unused ipc_port_t port, |
1193 | exclaves_id_t endpoint_id) |
1194 | { |
1195 | kern_return_t kr = KERN_SUCCESS; |
1196 | |
1197 | assert(port == IPC_PORT_NULL); |
1198 | |
1199 | kr = exclaves_xnu_proxy_endpoint_call(endpoint_id); |
1200 | |
1201 | return kr; |
1202 | } |
1203 | |
1204 | /* -------------------------------------------------------------------------- */ |
1205 | #pragma mark secure kernel communication |
1206 | |
1207 | /* ringgate entry endpoints */ |
1208 | enum { |
1209 | RINGGATE_EP_ENTER, |
1210 | RINGGATE_EP_INFO |
1211 | }; |
1212 | |
1213 | /* ringgate entry status codes */ |
1214 | enum { |
1215 | RINGGATE_STATUS_SUCCESS, |
1216 | RINGGATE_STATUS_ERROR, |
1217 | RINGGATE_STATUS_PANIC, /* RINGGATE_EP_ENTER: Another core paniced */ |
1218 | }; |
1219 | |
1220 | OS_NOINLINE |
1221 | static kern_return_t |
1222 | exclaves_enter(void) |
1223 | { |
1224 | uint32_t endpoint = RINGGATE_EP_ENTER; |
1225 | uint64_t result = RINGGATE_STATUS_ERROR; |
1226 | |
1227 | sptm_call_regs_t regs = { }; |
1228 | |
1229 | __assert_only thread_t thread = current_thread(); |
1230 | |
1231 | /* |
1232 | * Should never re-enter exclaves. |
1233 | */ |
1234 | if ((thread->th_exclaves_state & TH_EXCLAVES_UPCALL) != 0 || |
1235 | (thread->th_exclaves_state & TH_EXCLAVES_SCHEDULER_REQUEST) != 0) { |
1236 | panic("attempt to re-enter exclaves" ); |
1237 | } |
1238 | |
1239 | /* |
1240 | * Must have one (and only one) of the flags set to enter exclaves. |
1241 | */ |
1242 | __assert_only const thread_exclaves_state_flags_t mask = ( |
1243 | TH_EXCLAVES_RPC | |
1244 | TH_EXCLAVES_XNUPROXY | |
1245 | TH_EXCLAVES_SCHEDULER_CALL); |
1246 | assert3u(thread->th_exclaves_state & mask, !=, 0); |
1247 | assert3u(thread->th_exclaves_intstate & TH_EXCLAVES_EXECUTION, ==, 0); |
1248 | |
1249 | #if MACH_ASSERT |
1250 | /* |
1251 | * Set the ast to check that the thread doesn't return to userspace |
1252 | * while in an RPC or XNUPROXY call. |
1253 | */ |
1254 | act_set_debug_assert(); |
1255 | #endif /* MACH_ASSERT */ |
1256 | |
1257 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_SWITCH) |
1258 | | DBG_FUNC_START); |
1259 | recount_enter_secure(); |
1260 | |
1261 | /* xnu_return_to_gl2 relies on this flag being present to correctly return |
1262 | * to SK from interrupts xnu handles on behalf of SK. */ |
1263 | thread->th_exclaves_intstate |= TH_EXCLAVES_EXECUTION; |
1264 | |
1265 | /* |
1266 | * Bracket with labels so stackshot can determine where exclaves are |
1267 | * entered from xnu. |
1268 | */ |
1269 | __asm__ volatile ( |
1270 | "EXCLAVES_ENTRY_START: nop\n\t" |
1271 | ); |
1272 | result = sk_enter(endpoint, ®s); |
1273 | __asm__ volatile ( |
1274 | "EXCLAVES_ENTRY_END: nop\n\t" |
1275 | ); |
1276 | |
1277 | thread->th_exclaves_intstate &= ~TH_EXCLAVES_EXECUTION; |
1278 | |
1279 | recount_leave_secure(); |
1280 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_SWITCH) |
1281 | | DBG_FUNC_END); |
1282 | |
1283 | switch (result) { |
1284 | case RINGGATE_STATUS_SUCCESS: |
1285 | return KERN_SUCCESS; |
1286 | case RINGGATE_STATUS_ERROR: |
1287 | return KERN_FAILURE; |
1288 | case RINGGATE_STATUS_PANIC: |
1289 | return KERN_ABORTED; |
1290 | default: |
1291 | assertf(false, "Unknown ringgate status %llu" , result); |
1292 | __builtin_trap(); |
1293 | } |
1294 | } |
1295 | |
1296 | |
1297 | /* |
1298 | * A bit in the lower byte of the value returned by RINGGATE_EP_INFO. If set, |
1299 | * it in indicates that we should immediately enter the ringgate once in order |
1300 | * to allow the scheduler to perform early boot initialisation. |
1301 | */ |
1302 | #define EARLY_RINGGATE_ENTER 2 |
1303 | |
1304 | OS_NOINLINE |
1305 | static kern_return_t |
1306 | exclaves_bootinfo(uint64_t *out_boot_info, bool *early_enter) |
1307 | { |
1308 | uint32_t endpoint = RINGGATE_EP_INFO; |
1309 | uint64_t result = RINGGATE_STATUS_ERROR; |
1310 | |
1311 | sptm_call_regs_t regs = { }; |
1312 | |
1313 | recount_enter_secure(); |
1314 | result = sk_enter(endpoint, ®s); |
1315 | recount_leave_secure(); |
1316 | if (result == RINGGATE_STATUS_ERROR) { |
1317 | return KERN_FAILURE; |
1318 | } |
1319 | |
1320 | *early_enter = (result & EARLY_RINGGATE_ENTER) != 0; |
1321 | *out_boot_info = result & ~EARLY_RINGGATE_ENTER; |
1322 | |
1323 | return KERN_SUCCESS; |
1324 | } |
1325 | |
1326 | /* -------------------------------------------------------------------------- */ |
1327 | |
1328 | #pragma mark exclaves scheduler communication |
1329 | |
1330 | static XrtHosted_Buffer_t * PERCPU_DATA(exclaves_request); |
1331 | static XrtHosted_Buffer_t * PERCPU_DATA(exclaves_response); |
1332 | |
1333 | static void |
1334 | exclaves_init_multicore(void) |
1335 | { |
1336 | assert(exclaves_multicore); |
1337 | |
1338 | XrtHosted_Buffer_t **req, **res; |
1339 | |
1340 | exclaves_wait_for_cpu_init(); |
1341 | |
1342 | DTEntry entry, child; |
1343 | OpaqueDTEntryIterator iter; |
1344 | int err = SecureDTLookupEntry(NULL, "/cpus" , &entry); |
1345 | assert(err == kSuccess); |
1346 | err = SecureDTInitEntryIterator(entry, &iter); |
1347 | assert(err == kSuccess); |
1348 | |
1349 | bool exclaves_uses_mpidr = (exclaves_callbacks->v1.global()->v2.smpStatus == XrtHosted_SmpStatus_MulticoreMpidr); |
1350 | if (exclaves_uses_mpidr) { |
1351 | exclaves_debug_printf(show_progress, "Using MPIDR for exclave scheduler core IDs\n" ); |
1352 | } else { |
1353 | // TODO(rdar://120679733) - clean up non-MPIDR identification logic. |
1354 | exclaves_debug_printf(show_progress, "Not using MPIDR for exclave scheduler core IDs\n" ); |
1355 | } |
1356 | |
1357 | /* |
1358 | * Match the hardwareID to the physical ID and stash the pointers to the |
1359 | * request/response buffers in per-cpu data for quick access. |
1360 | */ |
1361 | size_t core_count = exclaves_callbacks->v1.cores(); |
1362 | for (size_t i = 0; i < core_count; i++) { |
1363 | const XrtHosted_Core_t *core = exclaves_callbacks->v1.core(i); |
1364 | uint32_t dt_phys_id = 0; |
1365 | if (exclaves_uses_mpidr) { |
1366 | dt_phys_id = (uint32_t)core->v2.hardwareId; |
1367 | } else { |
1368 | /* Find the physical ID of the entry at position hardwareId in the |
1369 | * DeviceTree "cpus" array */ |
1370 | uint32_t dt_index = 0; |
1371 | bool dt_entry_found = false; |
1372 | err = SecureDTRestartEntryIteration(&iter); |
1373 | assert(err == kSuccess); |
1374 | while (kSuccess == SecureDTIterateEntries(&iter, &child)) { |
1375 | if (core->v2.hardwareId == dt_index) { |
1376 | void const *dt_prop; |
1377 | unsigned int dt_prop_sz; |
1378 | err = SecureDTGetProperty(child, "reg" , &dt_prop, &dt_prop_sz); |
1379 | assert(err == kSuccess); |
1380 | assert(dt_prop_sz == sizeof(uint32_t)); |
1381 | dt_phys_id = *((uint32_t const *)dt_prop); |
1382 | dt_entry_found = true; |
1383 | break; |
1384 | } |
1385 | dt_index++; |
1386 | } |
1387 | if (!dt_entry_found) { |
1388 | continue; |
1389 | } |
1390 | } |
1391 | percpu_foreach(cpu_data, cpu_data) { |
1392 | if (cpu_data->cpu_phys_id != dt_phys_id) { |
1393 | continue; |
1394 | } |
1395 | req = PERCPU_GET_RELATIVE(exclaves_request, cpu_data, cpu_data); |
1396 | *req = exclaves_callbacks->v1.Core.request(i); |
1397 | |
1398 | res = PERCPU_GET_RELATIVE(exclaves_response, cpu_data, cpu_data); |
1399 | *res = exclaves_callbacks->v1.Core.response(i); |
1400 | |
1401 | break; |
1402 | } |
1403 | } |
1404 | } |
1405 | |
1406 | static void |
1407 | exclaves_init_unicore(void) |
1408 | { |
1409 | assert(!exclaves_multicore); |
1410 | |
1411 | XrtHosted_Buffer_t *breq, *bres, **req, **res; |
1412 | |
1413 | exclaves_wait_for_cpu_init(); |
1414 | |
1415 | breq = exclaves_callbacks->v1.Core.request(XrtHosted_Core_bootIndex); |
1416 | bres = exclaves_callbacks->v1.Core.response(XrtHosted_Core_bootIndex); |
1417 | |
1418 | /* Always use the boot request/response buffers. */ |
1419 | percpu_foreach(cpu_data, cpu_data) { |
1420 | req = PERCPU_GET_RELATIVE(exclaves_request, cpu_data, cpu_data); |
1421 | *req = breq; |
1422 | |
1423 | res = PERCPU_GET_RELATIVE(exclaves_response, cpu_data, cpu_data); |
1424 | *res = bres; |
1425 | } |
1426 | } |
1427 | |
1428 | static kern_return_t |
1429 | exclaves_scheduler_init(uint64_t boot_info) |
1430 | { |
1431 | kern_return_t kr = KERN_SUCCESS; |
1432 | XrtHosted_Error_t hosted_error; |
1433 | |
1434 | lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED); |
1435 | |
1436 | if (!pmap_valid_address(boot_info)) { |
1437 | exclaves_debug_printf(show_errors, |
1438 | "exclaves: %s: 0x%012llx\n" , |
1439 | "Invalid root physical address" , |
1440 | boot_info); |
1441 | return KERN_FAILURE; |
1442 | } |
1443 | |
1444 | if (exclaves_callbacks == NULL) { |
1445 | exclaves_debug_printf(show_errors, |
1446 | "exclaves: Callbacks not registered\n" ); |
1447 | return KERN_FAILURE; |
1448 | } |
1449 | |
1450 | /* Initialise XrtHostedXnu kext */ |
1451 | kr = exclaves_hosted_error( |
1452 | exclaves_callbacks->v1.init( |
1453 | XrtHosted_Version_current, |
1454 | phystokv(boot_info), |
1455 | &hosted_error), |
1456 | &hosted_error); |
1457 | if (kr != KERN_SUCCESS) { |
1458 | return kr; |
1459 | } |
1460 | |
1461 | /* Record aperture addresses in buffer */ |
1462 | size_t frames = exclaves_callbacks->v1.frames(); |
1463 | XrtHosted_Mapped_t **pages = zalloc_permanent( |
1464 | frames * sizeof(XrtHosted_Mapped_t *), |
1465 | ZALIGN(XrtHosted_Mapped_t *)); |
1466 | size_t index = 0; |
1467 | uint64_t phys = boot_info; |
1468 | while (index < frames) { |
1469 | if (!pmap_valid_address(phys)) { |
1470 | exclaves_debug_printf(show_errors, |
1471 | "exclaves: %s: 0x%012llx\n" , |
1472 | "Invalid shared physical address" , |
1473 | phys); |
1474 | return KERN_FAILURE; |
1475 | } |
1476 | pages[index] = (XrtHosted_Mapped_t *)phystokv(phys); |
1477 | kr = exclaves_hosted_error( |
1478 | exclaves_callbacks->v1.nextPhys( |
1479 | pages[index], |
1480 | &index, |
1481 | &phys, |
1482 | &hosted_error), |
1483 | &hosted_error); |
1484 | if (kr != KERN_SUCCESS) { |
1485 | return kr; |
1486 | } |
1487 | } |
1488 | |
1489 | /* Initialise the mapped region */ |
1490 | exclaves_callbacks->v1.setMapping( |
1491 | XrtHosted_Region_scattered(frames, pages)); |
1492 | |
1493 | /* Boot the scheduler. */ |
1494 | kr = exclaves_scheduler_boot(); |
1495 | if (kr != KERN_SUCCESS) { |
1496 | return kr; |
1497 | } |
1498 | |
1499 | /* Initialise the XNU proxy */ |
1500 | XrtHosted_Global_t *global = exclaves_callbacks->v1.global(); |
1501 | |
1502 | exclaves_multicore = (global->v2.smpStatus == XrtHosted_SmpStatus_Multicore || global->v2.smpStatus == XrtHosted_SmpStatus_MulticoreMpidr); |
1503 | exclaves_multicore ? exclaves_init_multicore() : exclaves_init_unicore(); |
1504 | |
1505 | uint64_t xnu_proxy_boot_info = global->v1.proxyInit; |
1506 | kr = exclaves_xnu_proxy_init(xnu_proxy_boot_info); |
1507 | |
1508 | return kr; |
1509 | } |
1510 | |
1511 | #if EXCLAVES_ENABLE_SHOW_SCHEDULER_REQUEST_RESPONSE |
1512 | #define exclaves_scheduler_debug_save_buffer(_buf_in, _buf_out) \ |
1513 | *(_buf_out) = *(_buf_in) |
1514 | #define exclaves_scheduler_debug_show_request_response(_request_buf, \ |
1515 | _response_buf) ({ \ |
1516 | if (exclaves_debug_enabled(show_scheduler_request_response)) { \ |
1517 | printf("exclaves: Scheduler request = %p\n", _request_buf); \ |
1518 | printf("exclaves: Scheduler request.tag = 0x%04llx\n", \ |
1519 | (_request_buf)->tag); \ |
1520 | for (size_t arg = 0; arg < XrtHosted_Buffer_args; arg += 1) { \ |
1521 | printf("exclaves: Scheduler request.arguments[%02zu] = " \ |
1522 | "0x%04llx\n", arg, \ |
1523 | (_request_buf)->arguments[arg]); \ |
1524 | } \ |
1525 | printf("exclaves: Scheduler response = %p\n", _response_buf); \ |
1526 | printf("exclaves: Scheduler response.tag = 0x%04llx\n", \ |
1527 | (_response_buf)->tag); \ |
1528 | for (size_t arg = 0; arg < XrtHosted_Buffer_args; arg += 1) { \ |
1529 | printf("exclaves: Scheduler response.arguments[%02zu] = " \ |
1530 | "0x%04llx\n", arg, \ |
1531 | (_response_buf)->arguments[arg]); \ |
1532 | } \ |
1533 | }}) |
1534 | #else // EXCLAVES_SHOW_SCHEDULER_REQUEST_RESPONSE |
1535 | #define exclaves_scheduler_debug_save_buffer(_buf_in, _buf_out) (void)_buf_out |
1536 | #define exclaves_scheduler_debug_show_request_response(_request_buf, \ |
1537 | _response_buf) ({ }) |
1538 | #endif // EXCLAVES_SHOW_SCHEDULER_REQUEST_RESPONSE |
1539 | |
1540 | __attribute__((always_inline)) |
1541 | static kern_return_t |
1542 | exclaves_scheduler_send(const XrtHosted_Request_t *request, |
1543 | XrtHosted_Response_t *response, XrtHosted_Buffer_t *save_out_ptr, XrtHosted_Buffer_t *save_in_ptr) |
1544 | { |
1545 | /* Must be called with preemption and interrupts disabled */ |
1546 | kern_return_t kr; |
1547 | |
1548 | XrtHosted_Buffer_t *request_buf = *PERCPU_GET(exclaves_request); |
1549 | assert3p(request_buf, !=, NULL); |
1550 | |
1551 | exclaves_callbacks->v1.Request.encode(request_buf, request); |
1552 | exclaves_scheduler_debug_save_buffer(request_buf, save_out_ptr); |
1553 | |
1554 | kr = exclaves_enter(); |
1555 | |
1556 | /* The response may have come back on a different core. */ |
1557 | XrtHosted_Buffer_t *response_buf = *PERCPU_GET(exclaves_response); |
1558 | assert3p(response_buf, !=, NULL); |
1559 | |
1560 | exclaves_scheduler_debug_save_buffer(response_buf, save_in_ptr); |
1561 | exclaves_callbacks->v1.Response.decode(response_buf, response); |
1562 | |
1563 | return kr; |
1564 | } |
1565 | |
1566 | __attribute__((always_inline)) |
1567 | static kern_return_t |
1568 | exclaves_scheduler_request(const XrtHosted_Request_t *request, |
1569 | XrtHosted_Response_t *response) |
1570 | { |
1571 | #if EXCLAVES_ENABLE_SHOW_SCHEDULER_REQUEST_RESPONSE |
1572 | XrtHosted_Buffer_t save_in[3], save_out[3] = {{ .tag = XrtHosted_Message_Invalid }, { .tag = XrtHosted_Message_Invalid }, { .tag = XrtHosted_Message_Invalid }}; |
1573 | XrtHosted_Buffer_t *save_out_ptr = save_out, *save_in_ptr = save_in; |
1574 | #else |
1575 | XrtHosted_Buffer_t *save_out_ptr = NULL, *save_in_ptr = NULL; |
1576 | #endif // EXCLAVES_SHOW_SCHEDULER_REQUEST_RESPONSE |
1577 | |
1578 | assert3u(request->tag, >, XrtHosted_Request_Invalid); |
1579 | assert3u(request->tag, <, XrtHosted_Request_Limit); |
1580 | |
1581 | kern_return_t kr = KERN_SUCCESS; |
1582 | bool istate; |
1583 | |
1584 | if (!exclaves_multicore || !exclaves_smp_enabled) { |
1585 | lck_mtx_lock(&exclaves_scheduler_lock); |
1586 | } |
1587 | |
1588 | /* |
1589 | * Disable preemption and interrupts as the xrt hosted scheduler data |
1590 | * structures are per-core. |
1591 | * Preemption disabled and interrupt disabled timeouts are disabled for |
1592 | * now until we can co-ordinate the measurements with the exclaves side of |
1593 | * things. |
1594 | */ |
1595 | istate = ml_set_interrupts_enabled_with_debug(false, false); |
1596 | |
1597 | /* |
1598 | * This needs to be done with interrupts disabled, otherwise stackshot could |
1599 | * mark the thread blocked just after this function exits and a thread marked |
1600 | * as AST blocked would go into exclaves. |
1601 | */ |
1602 | |
1603 | while ((os_atomic_load(¤t_thread()->th_exclaves_inspection_state, relaxed) & ~TH_EXCLAVES_INSPECTION_NOINSPECT) != 0) { |
1604 | /* Enable interrupts */ |
1605 | (void) ml_set_interrupts_enabled_with_debug(true, false); |
1606 | |
1607 | if (!exclaves_multicore || !exclaves_smp_enabled) { |
1608 | lck_mtx_unlock(&exclaves_scheduler_lock); |
1609 | } |
1610 | |
1611 | /* Wait until the thread is collected on exclaves side */ |
1612 | exclaves_inspection_check_ast(); |
1613 | |
1614 | if (!exclaves_multicore || !exclaves_smp_enabled) { |
1615 | lck_mtx_lock(&exclaves_scheduler_lock); |
1616 | } |
1617 | |
1618 | /* Disable interrupts and preemption before next AST check */ |
1619 | ml_set_interrupts_enabled_with_debug(false, false); |
1620 | } |
1621 | /* Interrupts are disabled and exclaves_stackshot_ast is clean */ |
1622 | |
1623 | disable_preemption_without_measurements(); |
1624 | |
1625 | /* Update clock offsets before any other scheduler operation */ |
1626 | exclaves_clock_t *clocks[] = { &exclaves_absolute_clock, |
1627 | &exclaves_continuous_clock }; |
1628 | for (unsigned i = 0; i < ARRAY_COUNT(clocks); ++i) { |
1629 | if (exclaves_clock_needs_update(clocks[i])) { |
1630 | kr = exclaves_clock_update(clocks[i], &save_out_ptr[i], &save_in_ptr[i]); |
1631 | if (kr != KERN_SUCCESS) { |
1632 | break; |
1633 | } |
1634 | } |
1635 | } |
1636 | |
1637 | if (kr == KERN_SUCCESS) { |
1638 | kr = exclaves_scheduler_send(request, response, &save_out_ptr[2], &save_in_ptr[2]); |
1639 | } |
1640 | |
1641 | enable_preemption(); |
1642 | (void) ml_set_interrupts_enabled_with_debug(istate, false); |
1643 | |
1644 | #if EXCLAVES_ENABLE_SHOW_SCHEDULER_REQUEST_RESPONSE |
1645 | for (unsigned i = 0; i < ARRAY_COUNT(save_out); ++i) { |
1646 | if (save_out_ptr[i].tag != XrtHosted_Message_Invalid) { |
1647 | exclaves_scheduler_debug_show_request_response(&save_out_ptr[i], &save_in_ptr[i]); |
1648 | } |
1649 | } |
1650 | #endif // EXCLAVES_ENABLE_SHOW_SCHEDULER_REQUEST_RESPONSE |
1651 | |
1652 | if (!exclaves_multicore || !exclaves_smp_enabled) { |
1653 | lck_mtx_unlock(&exclaves_scheduler_lock); |
1654 | } |
1655 | |
1656 | if (kr == KERN_ABORTED) { |
1657 | /* RINGGATE_EP_ENTER returned RINGGATE_STATUS_PANIC indicating that |
1658 | * another core has paniced in exclaves and is on the way to call xnu |
1659 | * panic() via SPTM, so wait here for that to happen. */ |
1660 | exclaves_wait_for_panic(); |
1661 | } |
1662 | |
1663 | return kr; |
1664 | } |
1665 | |
1666 | OS_NORETURN OS_NOINLINE |
1667 | static void |
1668 | exclaves_wait_for_panic(void) |
1669 | { |
1670 | assert_wait_timeout((event_t)exclaves_wait_for_panic, THREAD_UNINT, 1, |
1671 | NSEC_PER_SEC); |
1672 | wait_result_t wr = thread_block(THREAD_CONTINUE_NULL); |
1673 | panic("Unexpected wait for panic result: %d" , wr); |
1674 | } |
1675 | |
1676 | static kern_return_t |
1677 | handle_response_yield(bool early, __assert_only Exclaves_L4_Word_t scid, |
1678 | const XrtHosted_Yield_t *yield) |
1679 | { |
1680 | Exclaves_L4_Word_t responding_scid = yield->thread; |
1681 | Exclaves_L4_Word_t yielded_to_scid = yield->yieldTo; |
1682 | __assert_only ctid_t ctid = thread_get_ctid(current_thread()); |
1683 | |
1684 | exclaves_debug_printf(show_progress, |
1685 | "exclaves: Scheduler: %s scid 0x%lx yielded to scid 0x%lx\n" , |
1686 | early ? "(early yield)" : "" , responding_scid, yielded_to_scid); |
1687 | /* TODO: 1. remember yielding scid if it isn't the xnu proxy's |
1688 | * th_exclaves_scheduling_context_id so we know to resume it later |
1689 | * 2. translate yield_to to thread_switch()-style handoff. |
1690 | */ |
1691 | if (!early) { |
1692 | assert3u(responding_scid, ==, scid); |
1693 | assert3u(yield->threadHostId, ==, ctid); |
1694 | } |
1695 | |
1696 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1697 | MACH_EXCLAVES_SCHEDULER_YIELD), yielded_to_scid, early); |
1698 | |
1699 | return KERN_SUCCESS; |
1700 | } |
1701 | |
1702 | static kern_return_t |
1703 | handle_response_spawned(__assert_only Exclaves_L4_Word_t scid, |
1704 | const XrtHosted_Spawned_t *spawned, Exclaves_L4_Word_t *spawned_scid) |
1705 | { |
1706 | Exclaves_L4_Word_t responding_scid = spawned->thread; |
1707 | __assert_only ctid_t ctid = thread_get_ctid(current_thread()); |
1708 | |
1709 | if (spawned_scid == NULL) { |
1710 | exclaves_debug_printf(show_errors, |
1711 | "exclaves: Scheduler: Unexpected thread spawn: " |
1712 | "scid 0x%lx spawned scid 0x%llx\n" , |
1713 | responding_scid, spawned->spawned); |
1714 | return KERN_FAILURE; |
1715 | } |
1716 | |
1717 | *spawned_scid = spawned->spawned; |
1718 | exclaves_debug_printf(show_progress, |
1719 | "exclaves: Scheduler: scid 0x%lx spawned scid 0x%lx\n" , |
1720 | responding_scid, *spawned_scid); |
1721 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1722 | MACH_EXCLAVES_SCHEDULER_SPAWNED), *spawned_scid); |
1723 | |
1724 | /* TODO: remember yielding scid if it isn't the xnu proxy's |
1725 | * th_exclaves_scheduling_context_id so we know to resume it later |
1726 | */ |
1727 | if (0) { |
1728 | // FIXME: reenable when exclaves scheduler is fixed |
1729 | assert3u(responding_scid, ==, scid); |
1730 | assert3u(spawned->threadHostId, ==, ctid); |
1731 | } |
1732 | |
1733 | return KERN_SUCCESS; |
1734 | } |
1735 | |
1736 | static kern_return_t |
1737 | handle_response_terminated(const XrtHosted_Terminated_t *terminated) |
1738 | { |
1739 | Exclaves_L4_Word_t responding_scid = terminated->thread; |
1740 | __assert_only ctid_t ctid = thread_get_ctid(current_thread()); |
1741 | |
1742 | exclaves_debug_printf(show_errors, |
1743 | "exclaves: Scheduler: Unexpected thread terminate: " |
1744 | "scid 0x%lx terminated scid 0x%llx\n" , responding_scid, |
1745 | terminated->terminated); |
1746 | assert3u(terminated->threadHostId, ==, ctid); |
1747 | |
1748 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1749 | MACH_EXCLAVES_SCHEDULER_TERMINATED), |
1750 | terminated->terminated); |
1751 | |
1752 | return KERN_TERMINATED; |
1753 | } |
1754 | |
1755 | static kern_return_t |
1756 | handle_response_wait(const XrtHosted_Wait_t *wait) |
1757 | { |
1758 | Exclaves_L4_Word_t responding_scid = wait->waiter; |
1759 | __assert_only ctid_t ctid = thread_get_ctid(current_thread()); |
1760 | |
1761 | exclaves_debug_printf(show_progress, |
1762 | "exclaves: Scheduler: Wait: " |
1763 | "scid 0x%lx wait on owner scid 0x%llx, queue id 0x%llx, " |
1764 | "epoch 0x%llx\n" , responding_scid, wait->owner, |
1765 | wait->queueId, wait->epoch); |
1766 | assert3u(wait->waiterHostId, ==, ctid); |
1767 | |
1768 | /* |
1769 | * Note, "owner" may not be safe to access directly, for example |
1770 | * the thread may have exited and been freed. esync_wait will |
1771 | * only access it under a lock if the epoch is fresh thus |
1772 | * ensuring safety. |
1773 | */ |
1774 | const ctid_t owner = (ctid_t)wait->ownerHostId; |
1775 | const XrtHosted_Word_t id = wait->queueId; |
1776 | const uint64_t epoch = wait->epoch; |
1777 | |
1778 | wait_interrupt_t interruptible; |
1779 | esync_policy_t policy; |
1780 | |
1781 | switch (wait->interruptible) { |
1782 | case XrtHosted_Interruptibility_None: |
1783 | interruptible = THREAD_UNINT; |
1784 | policy = ESYNC_POLICY_KERNEL; |
1785 | break; |
1786 | |
1787 | case XrtHosted_Interruptibility_Voluntary: |
1788 | interruptible = THREAD_INTERRUPTIBLE; |
1789 | policy = ESYNC_POLICY_KERNEL; |
1790 | break; |
1791 | |
1792 | case XrtHosted_Interruptibility_DynamicQueue: |
1793 | interruptible = THREAD_INTERRUPTIBLE; |
1794 | policy = ESYNC_POLICY_USER; |
1795 | break; |
1796 | |
1797 | default: |
1798 | panic("Unknown exclaves interruptibility: %llu" , |
1799 | wait->interruptible); |
1800 | } |
1801 | |
1802 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1803 | MACH_EXCLAVES_SCHEDULER_WAIT) | DBG_FUNC_START, id, epoch, owner, |
1804 | wait->interruptible); |
1805 | const wait_result_t wr = esync_wait(&esync_queue_ht, id, epoch, |
1806 | exclaves_get_queue_counter(id), owner, policy, interruptible); |
1807 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1808 | MACH_EXCLAVES_SCHEDULER_WAIT) | DBG_FUNC_END, wr); |
1809 | |
1810 | switch (wr) { |
1811 | case THREAD_INTERRUPTED: |
1812 | return KERN_ABORTED; |
1813 | |
1814 | case THREAD_NOT_WAITING: |
1815 | case THREAD_AWAKENED: |
1816 | return KERN_SUCCESS; |
1817 | |
1818 | default: |
1819 | panic("Unexpected wait result from esync_wait: %d" , wr); |
1820 | } |
1821 | } |
1822 | |
1823 | static kern_return_t |
1824 | handle_response_wake(const XrtHosted_Wake_t *wake) |
1825 | { |
1826 | Exclaves_L4_Word_t responding_scid = wake->waker; |
1827 | __assert_only ctid_t ctid = thread_get_ctid(current_thread()); |
1828 | |
1829 | exclaves_debug_printf(show_progress, |
1830 | "exclaves: Scheduler: Wake: " |
1831 | "scid 0x%lx wake of queue id 0x%llx, " |
1832 | "epoch 0x%llx, all 0x%llx\n" , responding_scid, |
1833 | wake->queueId, wake->epoch, wake->all); |
1834 | assert3u(wake->wakerHostId, ==, ctid); |
1835 | |
1836 | const XrtHosted_Word_t id = wake->queueId; |
1837 | const uint64_t epoch = wake->epoch; |
1838 | const esync_wake_mode_t mode = wake->all != 0 ? |
1839 | ESYNC_WAKE_ALL : ESYNC_WAKE_ONE; |
1840 | |
1841 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1842 | MACH_EXCLAVES_SCHEDULER_WAKE) | DBG_FUNC_START, id, epoch, 0, mode); |
1843 | |
1844 | kern_return_t kr = esync_wake(&esync_queue_ht, id, epoch, |
1845 | exclaves_get_queue_counter(id), mode, 0); |
1846 | |
1847 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1848 | MACH_EXCLAVES_SCHEDULER_WAKE) | DBG_FUNC_END, |
1849 | kr == KERN_SUCCESS ? THREAD_AWAKENED : THREAD_NOT_WAITING); |
1850 | |
1851 | return KERN_SUCCESS; |
1852 | } |
1853 | |
1854 | static kern_return_t |
1855 | handle_response_wake_with_owner(const XrtHosted_WakeWithOwner_t *wake) |
1856 | { |
1857 | Exclaves_L4_Word_t responding_scid = wake->waker; |
1858 | __assert_only ctid_t ctid = thread_get_ctid(current_thread()); |
1859 | |
1860 | exclaves_debug_printf(show_progress, |
1861 | "exclaves: Scheduler: WakeWithOwner: " |
1862 | "scid 0x%lx wake of queue id 0x%llx, " |
1863 | "epoch 0x%llx, owner 0x%llx\n" , responding_scid, |
1864 | wake->queueId, wake->epoch, |
1865 | wake->owner); |
1866 | |
1867 | assert3u(wake->wakerHostId, ==, ctid); |
1868 | |
1869 | const ctid_t owner = (ctid_t)wake->ownerHostId; |
1870 | const XrtHosted_Word_t id = wake->queueId; |
1871 | const uint64_t epoch = wake->epoch; |
1872 | |
1873 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1874 | MACH_EXCLAVES_SCHEDULER_WAKE) | DBG_FUNC_START, id, epoch, owner, |
1875 | ESYNC_WAKE_ONE); |
1876 | |
1877 | kern_return_t kr = esync_wake(&esync_queue_ht, id, epoch, |
1878 | exclaves_get_queue_counter(id), ESYNC_WAKE_ONE_WITH_OWNER, owner); |
1879 | |
1880 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, |
1881 | MACH_EXCLAVES_SCHEDULER_WAKE) | DBG_FUNC_END, |
1882 | kr == KERN_SUCCESS ? THREAD_AWAKENED : THREAD_NOT_WAITING); |
1883 | |
1884 | return KERN_SUCCESS; |
1885 | } |
1886 | |
1887 | static kern_return_t |
1888 | handle_response_panic_wait(const XrtHosted_PanicWait_t *panic_wait) |
1889 | { |
1890 | Exclaves_L4_Word_t panic_thread_scid = panic_wait->handler; |
1891 | __assert_only thread_t thread = current_thread(); |
1892 | |
1893 | exclaves_debug_printf(show_progress, |
1894 | "exclaves: Scheduler: PanicWait: " |
1895 | "Panic thread SCID %lx\n" , |
1896 | panic_thread_scid); |
1897 | |
1898 | assert3u(panic_thread_scid, ==, thread->th_exclaves_scheduling_context_id); |
1899 | |
1900 | exclaves_panic_thread_wait(); |
1901 | |
1902 | /* NOT REACHABLE */ |
1903 | return KERN_SUCCESS; |
1904 | } |
1905 | |
1906 | static kern_return_t |
1907 | handle_response_suspended(const XrtHosted_Suspended_t *suspended) |
1908 | { |
1909 | Exclaves_L4_Word_t responding_scid = suspended->suspended; |
1910 | __assert_only ctid_t ctid = thread_get_ctid(current_thread()); |
1911 | |
1912 | exclaves_debug_printf(show_progress, |
1913 | "exclaves: Scheduler: Suspended: " |
1914 | "scid 0x%lx epoch 0x%llx\n" , responding_scid, suspended->epoch); |
1915 | assert3u(suspended->suspendedHostId, ==, ctid); |
1916 | |
1917 | const uint64_t id = suspended->suspended; |
1918 | const uint64_t epoch = suspended->epoch; |
1919 | |
1920 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1921 | MACH_EXCLAVES_SCHEDULER_SUSPENDED) | DBG_FUNC_START, id, epoch); |
1922 | |
1923 | const wait_result_t wr = esync_wait(&esync_thread_ht, id, epoch, |
1924 | exclaves_get_thread_counter(id), 0, ESYNC_POLICY_KERNEL, THREAD_UNINT); |
1925 | |
1926 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1927 | MACH_EXCLAVES_SCHEDULER_SUSPENDED) | DBG_FUNC_END, wr); |
1928 | |
1929 | switch (wr) { |
1930 | case THREAD_INTERRUPTED: |
1931 | return KERN_ABORTED; |
1932 | |
1933 | case THREAD_NOT_WAITING: |
1934 | case THREAD_AWAKENED: |
1935 | return KERN_SUCCESS; |
1936 | |
1937 | default: |
1938 | panic("Unexpected wait result from esync_wait: %d" , wr); |
1939 | } |
1940 | } |
1941 | |
1942 | static kern_return_t |
1943 | handle_response_resumed(const XrtHosted_Resumed_t *resumed) |
1944 | { |
1945 | Exclaves_L4_Word_t responding_scid = resumed->thread; |
1946 | __assert_only ctid_t ctid = thread_get_ctid(current_thread()); |
1947 | |
1948 | exclaves_debug_printf(show_progress, |
1949 | "exclaves: Scheduler: Resumed: scid 0x%lx resume of scid 0x%llx " |
1950 | "(ctid: 0x%llx), epoch 0x%llx\n" , responding_scid, resumed->resumed, |
1951 | resumed->resumedHostId, resumed->epoch); |
1952 | assert3u(resumed->threadHostId, ==, ctid); |
1953 | |
1954 | const ctid_t target = (ctid_t)resumed->resumedHostId; |
1955 | const XrtHosted_Word_t id = resumed->resumed; |
1956 | const uint64_t epoch = resumed->epoch; |
1957 | |
1958 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1959 | MACH_EXCLAVES_SCHEDULER_RESUMED) | DBG_FUNC_START, id, epoch, |
1960 | target); |
1961 | |
1962 | kern_return_t kr = esync_wake(&esync_thread_ht, id, epoch, |
1963 | exclaves_get_thread_counter(id), ESYNC_WAKE_THREAD, target); |
1964 | |
1965 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1966 | MACH_EXCLAVES_SCHEDULER_RESUMED) | DBG_FUNC_END, |
1967 | kr == KERN_SUCCESS ? THREAD_AWAKENED : THREAD_NOT_WAITING); |
1968 | |
1969 | return KERN_SUCCESS; |
1970 | } |
1971 | |
1972 | static kern_return_t |
1973 | handle_response_interrupted(const XrtHosted_Interrupted_t *interrupted) |
1974 | { |
1975 | Exclaves_L4_Word_t responding_scid = interrupted->thread; |
1976 | __assert_only ctid_t ctid = thread_get_ctid(current_thread()); |
1977 | |
1978 | exclaves_debug_printf(show_progress, |
1979 | "exclaves: Scheduler: Interrupted: " |
1980 | "scid 0x%lx interrupt on queue id 0x%llx, " |
1981 | "epoch 0x%llx, target 0x%llx\n" , responding_scid, |
1982 | interrupted->queueId, interrupted->epoch, |
1983 | interrupted->interruptedHostId); |
1984 | assert3u(interrupted->threadHostId, ==, ctid); |
1985 | |
1986 | const ctid_t target = (ctid_t)interrupted->interruptedHostId; |
1987 | const XrtHosted_Word_t id = interrupted->queueId; |
1988 | const uint64_t epoch = interrupted->epoch; |
1989 | |
1990 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
1991 | MACH_EXCLAVES_SCHEDULER_INTERRUPTED) | DBG_FUNC_START, id, epoch, |
1992 | target); |
1993 | |
1994 | kern_return_t kr = esync_wake(&esync_queue_ht, id, epoch, |
1995 | exclaves_get_queue_counter(id), ESYNC_WAKE_THREAD, target); |
1996 | |
1997 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, |
1998 | MACH_EXCLAVES_SCHEDULER_INTERRUPTED) | DBG_FUNC_END, |
1999 | kr == KERN_SUCCESS ? THREAD_AWAKENED : THREAD_NOT_WAITING); |
2000 | |
2001 | return KERN_SUCCESS; |
2002 | } |
2003 | |
2004 | static kern_return_t |
2005 | handle_response_nothing_scheduled( |
2006 | __unused const XrtHosted_NothingScheduled_t *nothing_scheduled) |
2007 | { |
2008 | exclaves_debug_printf(show_progress, |
2009 | "exclaves: Scheduler: nothing scheduled\n" ); |
2010 | |
2011 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
2012 | MACH_EXCLAVES_SCHEDULER_NOTHING_SCHEDULED)); |
2013 | |
2014 | return KERN_SUCCESS; |
2015 | } |
2016 | |
2017 | static kern_return_t |
2018 | handle_response_all_exclaves_booted( |
2019 | __unused const XrtHosted_AllExclavesBooted_t *all_exclaves_booted) |
2020 | { |
2021 | exclaves_debug_printf(show_progress, |
2022 | "exclaves: scheduler: all exclaves booted\n" ); |
2023 | |
2024 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
2025 | MACH_EXCLAVES_SCHEDULER_ALL_EXCLAVES_BOOTED)); |
2026 | |
2027 | return KERN_SUCCESS; |
2028 | } |
2029 | |
2030 | /* |
2031 | * The Early Alloc response asks for npages to be allocated. The list of |
2032 | * allocated pages is written into the first allocated page in the form of 32bit |
2033 | * page numbers. The physical address of the first page is passed back to the |
2034 | * exclaves scheduler as part of the next request. |
2035 | */ |
2036 | static kern_return_t |
2037 | handle_response_pmm_early_alloc(const XrtHosted_PmmEarlyAlloc_t *pmm_early_alloc, |
2038 | uint64_t *pagelist_pa) |
2039 | { |
2040 | const uint32_t npages = (uint32_t)pmm_early_alloc->a; |
2041 | const uint64_t flags = pmm_early_alloc->b; |
2042 | |
2043 | exclaves_debug_printf(show_progress, |
2044 | "exclaves: scheduler: pmm early alloc, npages: %u, flags: %llu\n" , |
2045 | npages, flags); |
2046 | |
2047 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER, |
2048 | MACH_EXCLAVES_SCHEDULER_EARLY_ALLOC), npages, flags); |
2049 | |
2050 | if (npages == 0) { |
2051 | return KERN_SUCCESS; |
2052 | } |
2053 | |
2054 | if (npages > EXCLAVES_MEMORY_MAX_REQUEST) { |
2055 | exclaves_debug_printf(show_errors, |
2056 | "exclaves: request to allocate too many pages: %u\n" , |
2057 | npages); |
2058 | return KERN_NO_SPACE; |
2059 | } |
2060 | |
2061 | /* |
2062 | * As npages must be relatively small (<= EXCLAVES_MEMORY_MAX_REQUEST), |
2063 | * stack allocation is sufficient and fast. If |
2064 | * EXCLAVES_MEMORY_MAX_REQUEST gets large, this should probably be moved |
2065 | * to the heap. |
2066 | */ |
2067 | uint32_t page[npages]; |
2068 | exclaves_memory_alloc(npages, page, XNUUPCALLS_PAGEKIND_ROOTDOMAIN); |
2069 | |
2070 | /* Now copy the list of pages into the first page. */ |
2071 | uint64_t first_page_pa = ptoa(page[0]); |
2072 | #if 0 |
2073 | // move to before sptm retype |
2074 | uint32_t *first_page = (uint32_t *)phystokv(first_page_pa); |
2075 | for (int i = 0; i < npages; i++) { |
2076 | first_page[i] = page[i]; |
2077 | } |
2078 | #endif |
2079 | |
2080 | *pagelist_pa = first_page_pa; |
2081 | return KERN_SUCCESS; |
2082 | } |
2083 | |
2084 | static inline bool |
2085 | exclaves_clock_needs_update(const exclaves_clock_t *clock) |
2086 | { |
2087 | exclaves_clock_t local = { |
2088 | .u128 = os_atomic_load(&clock->a_u128, relaxed), |
2089 | }; |
2090 | |
2091 | return local.u64.sent_offset != local.u64.latest_offset; |
2092 | } |
2093 | |
2094 | OS_NOINLINE |
2095 | static kern_return_t |
2096 | exclaves_clock_update(exclaves_clock_t *clock, XrtHosted_Buffer_t *save_out_ptr, XrtHosted_Buffer_t *save_in_ptr) |
2097 | { |
2098 | XrtHosted_Response_t response = { .tag = XrtHosted_Response_NothingScheduled, }; |
2099 | kern_return_t kr = KERN_SUCCESS; |
2100 | exclaves_clock_t local; |
2101 | |
2102 | local.u128 = os_atomic_load(&clock->a_u128, relaxed); |
2103 | while (local.u64.sent_offset != local.u64.latest_offset) { |
2104 | XrtHosted_Request_t request = XrtHosted_Request_UpdateTimerOffsetMsg( |
2105 | .timer = |
2106 | (clock == &exclaves_absolute_clock ? |
2107 | XrtHosted_Timer_Absolute : XrtHosted_Timer_Continuous), |
2108 | .offset = local.u64.latest_offset, |
2109 | ); |
2110 | |
2111 | kr = exclaves_scheduler_send(&request, &response, save_out_ptr, save_in_ptr); |
2112 | if (kr) { |
2113 | return kr; |
2114 | } |
2115 | |
2116 | /* Swap the sent offset with the local latest offset. If it fails, |
2117 | * the sent offset will be reloaded. */ |
2118 | os_atomic_cmpxchgv(&clock->a_u64.sent_offset, local.u64.sent_offset, |
2119 | local.u64.latest_offset, &local.u64.sent_offset, relaxed); |
2120 | |
2121 | /* Fetch the latest offset again, in case we are stale. */ |
2122 | local.u64.latest_offset = os_atomic_load(&clock->a_u64.latest_offset, |
2123 | relaxed); |
2124 | } |
2125 | |
2126 | if (response.tag != XrtHosted_Response_NothingScheduled) { |
2127 | kr = KERN_FAILURE; |
2128 | } |
2129 | |
2130 | return kr; |
2131 | } |
2132 | |
2133 | static kern_return_t |
2134 | exclaves_scheduler_boot(void) |
2135 | { |
2136 | kern_return_t kr = KERN_FAILURE; |
2137 | thread_t thread = current_thread(); |
2138 | |
2139 | exclaves_debug_printf(show_progress, |
2140 | "exclaves: Scheduler: Request to boot exclave\n" ); |
2141 | |
2142 | /* This must happen on the boot CPU - bind the thread. */ |
2143 | bind_to_boot_core(); |
2144 | |
2145 | assert3u(thread->th_exclaves_state & TH_EXCLAVES_STATE_ANY, ==, 0); |
2146 | thread->th_exclaves_state |= TH_EXCLAVES_SCHEDULER_CALL; |
2147 | |
2148 | /* |
2149 | * Set the request/response buffers. These may be overriden later when |
2150 | * doing multicore setup. |
2151 | */ |
2152 | *PERCPU_GET(exclaves_request) = |
2153 | exclaves_callbacks->v1.Core.request(XrtHosted_Core_bootIndex); |
2154 | *PERCPU_GET(exclaves_response) = |
2155 | exclaves_callbacks->v1.Core.response(XrtHosted_Core_bootIndex); |
2156 | |
2157 | XrtHosted_Response_t response = {.tag = XrtHosted_Response_Invalid}; |
2158 | uint64_t pagelist_pa = 0; |
2159 | |
2160 | while (response.tag != XrtHosted_Response_AllExclavesBooted) { |
2161 | const XrtHosted_Request_t request = pagelist_pa != 0 ? |
2162 | XrtHosted_Request_PmmEarlyAllocResponseMsg(.a = pagelist_pa): |
2163 | XrtHosted_Request_BootExclavesMsg(); |
2164 | pagelist_pa = 0; |
2165 | |
2166 | kr = exclaves_scheduler_request(&request, &response); |
2167 | if (kr != KERN_SUCCESS) { |
2168 | exclaves_debug_printf(show_errors, |
2169 | "exclaves: Enter failed\n" ); |
2170 | break; |
2171 | } |
2172 | |
2173 | thread->th_exclaves_state |= TH_EXCLAVES_SCHEDULER_REQUEST; |
2174 | |
2175 | switch (response.tag) { |
2176 | case XrtHosted_Response_Yield: |
2177 | kr = handle_response_yield(true, 0, &response.Yield); |
2178 | break; |
2179 | |
2180 | case XrtHosted_Response_NothingScheduled: |
2181 | kr = handle_response_nothing_scheduled(&response.NothingScheduled); |
2182 | break; |
2183 | |
2184 | case XrtHosted_Response_AllExclavesBooted: |
2185 | kr = handle_response_all_exclaves_booted(&response.AllExclavesBooted); |
2186 | break; |
2187 | |
2188 | case XrtHosted_Response_PmmEarlyAlloc: |
2189 | kr = handle_response_pmm_early_alloc(&response.PmmEarlyAlloc, &pagelist_pa); |
2190 | break; |
2191 | |
2192 | case XrtHosted_Response_PanicBufferAddress: |
2193 | handle_response_panic_buffer_address(response.PanicBufferAddress.physical); |
2194 | break; |
2195 | |
2196 | default: |
2197 | exclaves_debug_printf(show_errors, |
2198 | "exclaves: Scheduler: Unexpected response: tag 0x%x\n" , |
2199 | response.tag); |
2200 | kr = KERN_FAILURE; |
2201 | break; |
2202 | } |
2203 | |
2204 | thread->th_exclaves_state &= ~TH_EXCLAVES_SCHEDULER_REQUEST; |
2205 | |
2206 | /* Bail out if an error is hit. */ |
2207 | if (kr != KERN_SUCCESS) { |
2208 | break; |
2209 | } |
2210 | } |
2211 | |
2212 | thread->th_exclaves_state &= ~TH_EXCLAVES_SCHEDULER_CALL; |
2213 | |
2214 | unbind_from_boot_core(); |
2215 | |
2216 | return kr; |
2217 | } |
2218 | |
2219 | kern_return_t |
2220 | exclaves_scheduler_resume_scheduling_context(Exclaves_L4_Word_t scid, |
2221 | Exclaves_L4_Word_t *spawned_scid, bool interrupted) |
2222 | { |
2223 | kern_return_t kr = KERN_SUCCESS; |
2224 | thread_t thread = current_thread(); |
2225 | const ctid_t ctid = thread_get_ctid(thread); |
2226 | |
2227 | exclaves_debug_printf(show_progress, |
2228 | "exclaves: Scheduler: Request to resume scid 0x%lx\n" , scid); |
2229 | |
2230 | XrtHosted_Response_t response = {}; |
2231 | const XrtHosted_Request_t request = interrupted ? |
2232 | XrtHosted_Request_InterruptWithHostIdMsg( |
2233 | .thread = scid, |
2234 | .hostId = ctid, |
2235 | ) : |
2236 | XrtHosted_Request_ResumeWithHostIdMsg( |
2237 | .thread = scid, |
2238 | .hostId = ctid, |
2239 | ); |
2240 | kr = exclaves_scheduler_request(&request, &response); |
2241 | if (kr) { |
2242 | exclaves_debug_printf(show_errors, "exclaves: Enter failed\n" ); |
2243 | return kr; |
2244 | } |
2245 | |
2246 | thread->th_exclaves_state |= TH_EXCLAVES_SCHEDULER_REQUEST; |
2247 | |
2248 | switch (response.tag) { |
2249 | case XrtHosted_Response_Wait: |
2250 | kr = handle_response_wait(&response.Wait); |
2251 | goto out; |
2252 | |
2253 | case XrtHosted_Response_Wake: |
2254 | kr = handle_response_wake(&response.Wake); |
2255 | goto out; |
2256 | |
2257 | case XrtHosted_Response_Yield: |
2258 | kr = handle_response_yield(false, scid, &response.Yield); |
2259 | goto out; |
2260 | |
2261 | case XrtHosted_Response_Spawned: |
2262 | kr = handle_response_spawned(scid, &response.Spawned, spawned_scid); |
2263 | goto out; |
2264 | |
2265 | case XrtHosted_Response_Terminated: |
2266 | kr = handle_response_terminated(&response.Terminated); |
2267 | goto out; |
2268 | |
2269 | case XrtHosted_Response_WakeWithOwner: |
2270 | kr = handle_response_wake_with_owner(&response.WakeWithOwner); |
2271 | goto out; |
2272 | |
2273 | case XrtHosted_Response_PanicWait: |
2274 | kr = handle_response_panic_wait(&response.PanicWait); |
2275 | goto out; |
2276 | |
2277 | case XrtHosted_Response_Suspended: |
2278 | kr = handle_response_suspended(&response.Suspended); |
2279 | goto out; |
2280 | |
2281 | case XrtHosted_Response_Resumed: |
2282 | kr = handle_response_resumed(&response.Resumed); |
2283 | goto out; |
2284 | |
2285 | case XrtHosted_Response_Interrupted: |
2286 | kr = handle_response_interrupted(&response.Interrupted); |
2287 | goto out; |
2288 | |
2289 | case XrtHosted_Response_Invalid: |
2290 | case XrtHosted_Response_Failure: |
2291 | case XrtHosted_Response_Pong: |
2292 | case XrtHosted_Response_SleepUntil: |
2293 | case XrtHosted_Response_Awaken: |
2294 | default: |
2295 | exclaves_debug_printf(show_errors, |
2296 | "exclaves: Scheduler: Unexpected response: tag 0x%x\n" , |
2297 | response.tag); |
2298 | kr = KERN_FAILURE; |
2299 | goto out; |
2300 | } |
2301 | |
2302 | out: |
2303 | thread->th_exclaves_state &= ~TH_EXCLAVES_SCHEDULER_REQUEST; |
2304 | return kr; |
2305 | } |
2306 | |
2307 | /* -------------------------------------------------------------------------- */ |
2308 | |
2309 | #pragma mark exclaves xnu proxy communication |
2310 | static const char * |
2311 | cmd_to_str(xnuproxy_cmd_t cmd) |
2312 | { |
2313 | switch (cmd) { |
2314 | case XNUPROXY_CMD_UNDEFINED: return "undefined" ; |
2315 | case XNUPROXY_CMD_SETUP: return "setup" ; |
2316 | case XNUPROXY_CMD_CONTEXT_ALLOCATE: return "allocate context" ; |
2317 | case XNUPROXY_CMD_CONTEXT_FREE: return "free context" ; |
2318 | case XNUPROXY_CMD_NAMED_BUFFER_CREATE: return "named buffer create" ; |
2319 | case XNUPROXY_CMD_NAMED_BUFFER_DELETE: return "named buffer delete" ; |
2320 | case XNUPROXY_CMD_RESOURCE_INFO: return "resource info" ; |
2321 | case XNUPROXY_CMD_AUDIO_BUFFER_CREATE: return "audio buffer create" ; |
2322 | case XNUPROXY_CMD_AUDIO_BUFFER_COPYOUT: return "audio buffer copyout" ; |
2323 | case XNUPROXY_CMD_AUDIO_BUFFER_DELETE: return "audio buffer delete" ; |
2324 | case XNUPROXY_CMD_SENSOR_START: return "sensor start" ; |
2325 | case XNUPROXY_CMD_SENSOR_STOP: return "sensor stop" ; |
2326 | case XNUPROXY_CMD_SENSOR_STATUS: return "sensor status" ; |
2327 | case XNUPROXY_CMD_DISPLAY_HEALTHCHECK_RATE: return "display healthcheck rate" ; |
2328 | case XNUPROXY_CMD_NAMED_BUFFER_MAP: return "named buffer map" ; |
2329 | case XNUPROXY_CMD_NAMED_BUFFER_LAYOUT: return "named buffer layout" ; |
2330 | case XNUPROXY_CMD_AUDIO_BUFFER_MAP: return "audio buffer map" ; |
2331 | case XNUPROXY_CMD_AUDIO_BUFFER_LAYOUT: return "audio buffer layout" ; |
2332 | case XNUPROXY_CMD_REPORT_MEMORY_USAGE: return "memory usage" ; |
2333 | case XNUPROXY_CMD_UPCALL_READY: return "upcall ready" ; |
2334 | default: return "<unknown>" ; |
2335 | } |
2336 | } |
2337 | #define exclaves_xnu_proxy_debug(flag, step, msg) \ |
2338 | exclaves_debug_printf(flag, \ |
2339 | "exclaves: xnu proxy %s " #step ":\t" \ |
2340 | "msg %p server_id 0x%lx cmd %u status %u\n", \ |
2341 | cmd_to_str((msg)->cmd), (msg), (msg)->server_id, (msg)->cmd, \ |
2342 | os_atomic_load(&(msg)->status, relaxed)) |
2343 | #define exclaves_xnu_proxy_show_progress(step, msg) \ |
2344 | exclaves_xnu_proxy_debug(show_progress, step, msg) |
2345 | #define exclaves_xnu_proxy_show_error(msg) \ |
2346 | exclaves_xnu_proxy_debug(show_errors, failed, msg) |
2347 | #define exclaves_xnu_proxy_endpoint_call_show_progress(operation, step, \ |
2348 | eid, scid, status) \ |
2349 | exclaves_debug_printf(show_progress, \ |
2350 | "exclaves: xnu proxy endpoint " #operation " " #step ":\t" \ |
2351 | "endpoint id %ld scid 0x%lx status %u\n", \ |
2352 | (eid), (scid), (status)) |
2353 | |
2354 | |
2355 | static kern_return_t |
2356 | exclaves_handle_upcall(thread_t thread, Exclaves_L4_IpcBuffer_t *ipcb, |
2357 | Exclaves_L4_Word_t scid, xnuproxy_msg_status_t status) |
2358 | { |
2359 | kern_return_t kr; |
2360 | Exclaves_L4_Word_t endpoint_id; |
2361 | |
2362 | uint64_t oldscid = thread->th_exclaves_scheduling_context_id; |
2363 | void *oldipcb = thread->th_exclaves_ipc_buffer; |
2364 | |
2365 | thread->th_exclaves_scheduling_context_id = scid; |
2366 | thread->th_exclaves_ipc_buffer = ipcb; |
2367 | |
2368 | thread->th_exclaves_state |= TH_EXCLAVES_UPCALL; |
2369 | endpoint_id = XNUPROXY_CR_ENDPOINT_ID(ipcb); |
2370 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_UPCALL) |
2371 | | DBG_FUNC_START, scid, endpoint_id); |
2372 | exclaves_xnu_proxy_endpoint_call_show_progress(upcall, entry, |
2373 | endpoint_id, scid, status); |
2374 | __asm__ volatile ( |
2375 | "EXCLAVES_UPCALL_START: nop\n\t" |
2376 | ); |
2377 | kr = exclaves_call_upcall_handler(endpoint_id); |
2378 | __asm__ volatile ( |
2379 | "EXCLAVES_UPCALL_END: nop\n\t" |
2380 | ); |
2381 | XNUPROXY_CR_STATUS(ipcb) = |
2382 | XNUPROXY_MSG_STATUS_PROCESSING; |
2383 | /* TODO: More state returned than Success or OperationInvalid? */ |
2384 | XNUPROXY_CR_RETVAL(ipcb) = |
2385 | (kr == KERN_SUCCESS) ? Exclaves_L4_Success : |
2386 | Exclaves_L4_ErrorOperationInvalid; |
2387 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_UPCALL) |
2388 | | DBG_FUNC_END); |
2389 | thread->th_exclaves_state &= ~TH_EXCLAVES_UPCALL; |
2390 | exclaves_xnu_proxy_endpoint_call_show_progress(upcall, returned, |
2391 | endpoint_id, scid, |
2392 | (unsigned int)XNUPROXY_CR_RETVAL(ipcb)); |
2393 | |
2394 | thread->th_exclaves_scheduling_context_id = oldscid; |
2395 | thread->th_exclaves_ipc_buffer = oldipcb; |
2396 | |
2397 | return kr; |
2398 | } |
2399 | |
2400 | extern kern_return_t exclaves_xnu_proxy_send(xnuproxy_msg_t *, Exclaves_L4_Word_t *); |
2401 | kern_return_t |
2402 | exclaves_xnu_proxy_send(xnuproxy_msg_t *_msg, Exclaves_L4_Word_t *spawned) |
2403 | { |
2404 | assert3p(_msg, !=, NULL); |
2405 | |
2406 | thread_t thread = current_thread(); |
2407 | |
2408 | if (exclaves_xnu_proxy_msg_buffer == NULL) { |
2409 | return KERN_FAILURE; |
2410 | } |
2411 | |
2412 | kern_return_t kr = KERN_SUCCESS; |
2413 | xnuproxy_msg_t *msg = exclaves_xnu_proxy_msg_buffer; |
2414 | bool interrupted = false; |
2415 | |
2416 | lck_mtx_lock(&exclaves_xnu_proxy_lock); |
2417 | |
2418 | assert3u(thread->th_exclaves_state & TH_EXCLAVES_STATE_ANY, ==, 0); |
2419 | thread->th_exclaves_state |= TH_EXCLAVES_XNUPROXY; |
2420 | |
2421 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_XNUPROXY) |
2422 | | DBG_FUNC_START, exclaves_xnu_proxy_scid, _msg->cmd); |
2423 | |
2424 | *msg = *_msg; |
2425 | msg->server_id = exclaves_xnu_proxy_scid; |
2426 | |
2427 | os_atomic_store(&msg->status, XNUPROXY_MSG_STATUS_PROCESSING, |
2428 | release); |
2429 | |
2430 | while (os_atomic_load(&msg->status, relaxed) == |
2431 | XNUPROXY_MSG_STATUS_PROCESSING) { |
2432 | exclaves_xnu_proxy_show_progress(in progress, msg); |
2433 | kr = exclaves_scheduler_resume_scheduling_context(msg->server_id, |
2434 | spawned, interrupted); |
2435 | assert(kr == KERN_SUCCESS || kr == KERN_ABORTED); |
2436 | |
2437 | /* A wait was interrupted. */ |
2438 | interrupted = kr == KERN_ABORTED; |
2439 | |
2440 | if (NULL != exclaves_xnu_proxy_upcall_ipcb) { |
2441 | if (XNUPROXY_MSG_STATUS_UPCALL == XNUPROXY_CR_STATUS(exclaves_xnu_proxy_upcall_ipcb)) { |
2442 | xnuproxy_msg_status_t status = (xnuproxy_msg_status_t) |
2443 | XNUPROXY_CR_STATUS(exclaves_xnu_proxy_upcall_ipcb); |
2444 | (void) exclaves_handle_upcall(thread, exclaves_xnu_proxy_upcall_ipcb, |
2445 | exclaves_xnu_proxy_scid, status); |
2446 | } |
2447 | } |
2448 | } |
2449 | |
2450 | if (os_atomic_load(&msg->status, acquire) == |
2451 | XNUPROXY_MSG_STATUS_NONE) { |
2452 | exclaves_xnu_proxy_show_progress(complete, msg); |
2453 | } else { |
2454 | kr = KERN_FAILURE; |
2455 | exclaves_xnu_proxy_show_error(msg); |
2456 | } |
2457 | |
2458 | *_msg = *msg; |
2459 | |
2460 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_XNUPROXY) |
2461 | | DBG_FUNC_END); |
2462 | |
2463 | thread->th_exclaves_state &= ~TH_EXCLAVES_XNUPROXY; |
2464 | lck_mtx_unlock(&exclaves_xnu_proxy_lock); |
2465 | |
2466 | return kr; |
2467 | } |
2468 | |
2469 | static kern_return_t |
2470 | exclaves_xnu_proxy_init(uint64_t xnu_proxy_boot_info) |
2471 | { |
2472 | kern_return_t kr = KERN_SUCCESS; |
2473 | pmap_paddr_t msg_buffer_paddr = xnu_proxy_boot_info; |
2474 | |
2475 | lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED); |
2476 | |
2477 | if (msg_buffer_paddr && pmap_valid_address(msg_buffer_paddr)) { |
2478 | lck_mtx_lock(&exclaves_xnu_proxy_lock); |
2479 | exclaves_xnu_proxy_msg_buffer = |
2480 | (xnuproxy_msg_t*)phystokv(msg_buffer_paddr); |
2481 | exclaves_xnu_proxy_scid = |
2482 | exclaves_xnu_proxy_msg_buffer->server_id; |
2483 | |
2484 | #if XNUPROXY_MSG_VERSION >= 3 |
2485 | exclaves_xnu_proxy_upcall_ipcb_paddr = |
2486 | exclaves_xnu_proxy_msg_buffer->upcall_ipc_buffer_paddr; |
2487 | if (exclaves_xnu_proxy_upcall_ipcb_paddr != 0) { |
2488 | exclaves_xnu_proxy_upcall_ipcb = (Exclaves_L4_IpcBuffer_t *) |
2489 | phystokv(exclaves_xnu_proxy_upcall_ipcb_paddr); |
2490 | } |
2491 | #endif /* XNUPROXY_MSG_VERSION >= 3 */ |
2492 | lck_mtx_unlock(&exclaves_xnu_proxy_lock); |
2493 | } else { |
2494 | exclaves_debug_printf(show_errors, |
2495 | "exclaves: %s: 0x%012llx\n" , |
2496 | "Invalid xnu proxy boot info physical address" , |
2497 | xnu_proxy_boot_info); |
2498 | return KERN_FAILURE; |
2499 | } |
2500 | |
2501 | xnuproxy_msg_t msg = { |
2502 | .cmd = XNUPROXY_CMD_SETUP, |
2503 | }; |
2504 | |
2505 | kr = exclaves_xnu_proxy_send(&msg, NULL); |
2506 | if (kr != KERN_SUCCESS) { |
2507 | return kr; |
2508 | } |
2509 | |
2510 | if (msg.cmd_setup.response.version != XNUPROXY_MSG_VERSION) { |
2511 | exclaves_debug_printf(show_errors, |
2512 | "exclaves: mismatched xnuproxy message version, " |
2513 | "xnuproxy: %u, xnu: %u " , msg.cmd_setup.response.version, |
2514 | XNUPROXY_MSG_VERSION); |
2515 | return KERN_FAILURE; |
2516 | } |
2517 | |
2518 | exclaves_debug_printf(show_progress, |
2519 | "exclaves: xnuproxy message version: 0x%u\n" , XNUPROXY_MSG_VERSION); |
2520 | |
2521 | kr = exclaves_panic_thread_setup(); |
2522 | if (kr != KERN_SUCCESS) { |
2523 | exclaves_debug_printf(show_errors, |
2524 | "XNU proxy panic thread setup failed\n" ); |
2525 | return KERN_FAILURE; |
2526 | } |
2527 | |
2528 | return KERN_SUCCESS; |
2529 | } |
2530 | |
2531 | static kern_return_t |
2532 | exclaves_xnu_proxy_allocate_context(Exclaves_L4_Word_t *scid, |
2533 | Exclaves_L4_IpcBuffer_t **ipcb) |
2534 | { |
2535 | kern_return_t kr = KERN_FAILURE; |
2536 | Exclaves_L4_Word_t spawned_scid = 0; |
2537 | |
2538 | xnuproxy_msg_t msg = { |
2539 | .cmd = XNUPROXY_CMD_CONTEXT_ALLOCATE, |
2540 | }; |
2541 | |
2542 | kr = exclaves_xnu_proxy_send(&msg, &spawned_scid); |
2543 | if (kr != KERN_SUCCESS) { |
2544 | return kr; |
2545 | } |
2546 | |
2547 | if (msg.cmd_ctx_alloc.response.ipc_paddr == 0) { |
2548 | return KERN_NO_SPACE; |
2549 | } |
2550 | |
2551 | if (spawned_scid != 0) { |
2552 | assert3u(msg.cmd_ctx_alloc.response.sched_id, ==, spawned_scid); |
2553 | } |
2554 | |
2555 | *scid = msg.cmd_ctx_alloc.response.sched_id; |
2556 | *ipcb = (Exclaves_L4_IpcBuffer_t *) |
2557 | phystokv(msg.cmd_ctx_alloc.response.ipc_paddr); |
2558 | os_atomic_inc(&exclaves_ipcb_cnt, relaxed); |
2559 | |
2560 | return KERN_SUCCESS; |
2561 | } |
2562 | |
2563 | static kern_return_t |
2564 | exclaves_xnu_proxy_free_context(Exclaves_L4_Word_t scid) |
2565 | { |
2566 | kern_return_t kr = KERN_FAILURE; |
2567 | xnuproxy_msg_t msg = { |
2568 | .cmd = XNUPROXY_CMD_CONTEXT_FREE, |
2569 | .cmd_ctx_free = (xnuproxy_cmd_ctx_free_t) { |
2570 | .request.sched_id = scid, |
2571 | .request.destroy = false, |
2572 | }, |
2573 | }; |
2574 | |
2575 | kr = exclaves_xnu_proxy_send(&msg, NULL); |
2576 | if (kr == KERN_SUCCESS) { |
2577 | size_t orig_ipcb_cnt = os_atomic_dec_orig(&exclaves_ipcb_cnt, relaxed); |
2578 | assert3u(orig_ipcb_cnt, >=, 1); |
2579 | if (orig_ipcb_cnt == 0) { /* This is just to avoid unused variable warning */ |
2580 | kr = KERN_FAILURE; |
2581 | } |
2582 | } |
2583 | return kr; |
2584 | } |
2585 | |
2586 | OS_NOINLINE |
2587 | static kern_return_t |
2588 | exclaves_xnu_proxy_endpoint_call(Exclaves_L4_Word_t endpoint_id) |
2589 | { |
2590 | kern_return_t kr = KERN_SUCCESS; |
2591 | thread_t thread = current_thread(); |
2592 | bool interrupted = false; |
2593 | |
2594 | Exclaves_L4_Word_t scid = thread->th_exclaves_scheduling_context_id; |
2595 | Exclaves_L4_IpcBuffer_t *ipcb = thread->th_exclaves_ipc_buffer; |
2596 | xnuproxy_msg_status_t status = |
2597 | XNUPROXY_MSG_STATUS_PROCESSING; |
2598 | |
2599 | XNUPROXY_CR_ENDPOINT_ID(ipcb) = endpoint_id; |
2600 | XNUPROXY_CR_STATUS(ipcb) = status; |
2601 | |
2602 | exclaves_xnu_proxy_endpoint_call_show_progress(call, entry, |
2603 | endpoint_id, scid, status); |
2604 | |
2605 | assert3u(thread->th_exclaves_state & TH_EXCLAVES_STATE_ANY, ==, 0); |
2606 | thread->th_exclaves_state |= TH_EXCLAVES_RPC; |
2607 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_RPC) |
2608 | | DBG_FUNC_START, scid, endpoint_id); |
2609 | |
2610 | while (1) { |
2611 | kr = exclaves_scheduler_resume_scheduling_context(scid, NULL, |
2612 | interrupted); |
2613 | assert(kr == KERN_SUCCESS || kr == KERN_ABORTED); |
2614 | |
2615 | /* A wait was interrupted. */ |
2616 | interrupted = kr == KERN_ABORTED; |
2617 | |
2618 | status = (xnuproxy_msg_status_t) |
2619 | XNUPROXY_CR_STATUS(ipcb); |
2620 | |
2621 | switch (status) { |
2622 | case XNUPROXY_MSG_STATUS_PROCESSING: |
2623 | exclaves_xnu_proxy_endpoint_call_show_progress(call, yielded, |
2624 | endpoint_id, scid, status); |
2625 | continue; |
2626 | |
2627 | case XNUPROXY_MSG_STATUS_REPLY: |
2628 | exclaves_xnu_proxy_endpoint_call_show_progress(call, returned, |
2629 | endpoint_id, scid, status); |
2630 | kr = KERN_SUCCESS; |
2631 | break; |
2632 | |
2633 | case XNUPROXY_MSG_STATUS_UPCALL: |
2634 | kr = exclaves_handle_upcall(thread, ipcb, scid, status); |
2635 | continue; |
2636 | |
2637 | default: |
2638 | // Should we have an assert(valid return) here? |
2639 | exclaves_xnu_proxy_endpoint_call_show_progress(call, failed, |
2640 | endpoint_id, scid, status); |
2641 | kr = KERN_FAILURE; |
2642 | break; |
2643 | } |
2644 | break; |
2645 | } |
2646 | |
2647 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_RPC) |
2648 | | DBG_FUNC_END); |
2649 | thread->th_exclaves_state &= ~TH_EXCLAVES_RPC; |
2650 | |
2651 | return kr; |
2652 | } |
2653 | |
2654 | static kern_return_t |
2655 | exclaves_hosted_error(bool success, XrtHosted_Error_t *error) |
2656 | { |
2657 | if (success) { |
2658 | return KERN_SUCCESS; |
2659 | } else { |
2660 | exclaves_debug_printf(show_errors, |
2661 | "exclaves: XrtHosted: %s[%d] (%s): %s\n" , |
2662 | error->file, |
2663 | error->line, |
2664 | error->function, |
2665 | error->expression |
2666 | ); |
2667 | return KERN_FAILURE; |
2668 | } |
2669 | } |
2670 | |
2671 | kern_return_t |
2672 | exclaves_ipc_buffer_cache_init(void) |
2673 | { |
2674 | kern_return_t kr = KERN_SUCCESS; |
2675 | Exclaves_L4_IpcBuffer_t *ipcb = NULL; |
2676 | Exclaves_L4_Word_t scid = 0; |
2677 | |
2678 | LCK_MTX_ASSERT(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED); |
2679 | assert(exclaves_ipc_buffer_cache == NULL); |
2680 | |
2681 | if (exclaves_ipc_buffer_cache_enabled) { |
2682 | if ((kr = exclaves_xnu_proxy_allocate_context(&scid, &ipcb))) { |
2683 | return kr; |
2684 | } |
2685 | |
2686 | /* relinquish the new buffer into the cache */ |
2687 | exclaves_relinquish_ipc_buffer(ipcb, scid); |
2688 | } |
2689 | return kr; |
2690 | } |
2691 | |
2692 | #pragma mark exclaves privilege management |
2693 | |
2694 | /* |
2695 | * All entitlement checking enabled by default. |
2696 | */ |
2697 | #define DEFAULT_ENTITLEMENT_FLAGS (~(0)) |
2698 | |
2699 | /* |
2700 | * boot-arg to control the use of entitlements. |
2701 | */ |
2702 | static TUNABLE(unsigned int, exclaves_entitlement_flags, |
2703 | "exclaves_entitlement_flags" , DEFAULT_ENTITLEMENT_FLAGS); |
2704 | |
2705 | static bool |
2706 | has_entitlement(task_t task, const exclaves_priv_t priv, |
2707 | const char *entitlement) |
2708 | { |
2709 | /* Skip the entitlement if not enabled. */ |
2710 | if ((exclaves_entitlement_flags & priv) == 0) { |
2711 | return true; |
2712 | } |
2713 | |
2714 | return IOTaskHasEntitlement(task, entitlement); |
2715 | } |
2716 | |
2717 | static bool |
2718 | has_entitlement_vnode(void *vnode, const int64_t off, |
2719 | const exclaves_priv_t priv, const char *entitlement) |
2720 | { |
2721 | /* Skip the entitlement if not enabled. */ |
2722 | if ((exclaves_entitlement_flags & priv) == 0) { |
2723 | return true; |
2724 | } |
2725 | |
2726 | return IOVnodeHasEntitlement(vnode, off, entitlement); |
2727 | } |
2728 | |
2729 | bool |
2730 | exclaves_has_priv(task_t task, exclaves_priv_t priv) |
2731 | { |
2732 | const bool is_kernel = task == kernel_task; |
2733 | const bool is_launchd = task_pid(task) == 1; |
2734 | |
2735 | switch (priv) { |
2736 | case EXCLAVES_PRIV_CONCLAVE_SPAWN: |
2737 | /* Both launchd and entitled tasks can spawn new conclaves. */ |
2738 | if (is_launchd) { |
2739 | return true; |
2740 | } |
2741 | return has_entitlement(task, priv, |
2742 | "com.apple.private.exclaves.conclave-spawn" ); |
2743 | |
2744 | case EXCLAVES_PRIV_KERNEL_DOMAIN: |
2745 | /* |
2746 | * Both the kernel itself and user tasks with the right |
2747 | * privilege can access exclaves resources in the kernel domain. |
2748 | */ |
2749 | if (is_kernel) { |
2750 | return true; |
2751 | } |
2752 | |
2753 | /* |
2754 | * If the task was entitled and has been through this path |
2755 | * before, it will have set the TFRO_HAS_KD_ACCESS flag. |
2756 | */ |
2757 | if ((task_ro_flags_get(task) & TFRO_HAS_KD_ACCESS) != 0) { |
2758 | return true; |
2759 | } |
2760 | |
2761 | if (has_entitlement(task, priv, |
2762 | "com.apple.private.exclaves.kernel-domain" )) { |
2763 | task_ro_flags_set(task, TFRO_HAS_KD_ACCESS); |
2764 | return true; |
2765 | } |
2766 | |
2767 | return false; |
2768 | |
2769 | case EXCLAVES_PRIV_BOOT: |
2770 | /* Both launchd and entitled tasks can boot exclaves. */ |
2771 | if (is_launchd) { |
2772 | return true; |
2773 | } |
2774 | return has_entitlement(task, priv, |
2775 | "com.apple.private.exclaves.boot" ); |
2776 | |
2777 | /* The CONCLAVE HOST priv is always checked by vnode. */ |
2778 | case EXCLAVES_PRIV_CONCLAVE_HOST: |
2779 | default: |
2780 | panic("bad exclaves privilege (%u)" , priv); |
2781 | } |
2782 | } |
2783 | |
2784 | bool |
2785 | exclaves_has_priv_vnode(void *vnode, int64_t off, exclaves_priv_t priv) |
2786 | { |
2787 | switch (priv) { |
2788 | case EXCLAVES_PRIV_CONCLAVE_HOST: |
2789 | return has_entitlement_vnode(vnode, off, priv, |
2790 | "com.apple.private.exclaves.conclave-host" ); |
2791 | |
2792 | case EXCLAVES_PRIV_CONCLAVE_SPAWN: |
2793 | return has_entitlement_vnode(vnode, off, priv, |
2794 | "com.apple.private.exclaves.conclave-spawn" ); |
2795 | |
2796 | default: |
2797 | panic("bad exclaves privilege (%u)" , priv); |
2798 | } |
2799 | } |
2800 | |
2801 | uint32_t |
2802 | exclaves_stack_offset(uintptr_t * out_addr, size_t nframes, bool slid_addresses) |
2803 | { |
2804 | size_t i = 0; |
2805 | uintptr_t enter_range_start = 0; |
2806 | uintptr_t enter_range_end = 0; |
2807 | uintptr_t upcall_range_start = 0; |
2808 | uintptr_t upcall_range_end = 0; |
2809 | |
2810 | if (slid_addresses) { |
2811 | enter_range_start = (uintptr_t)&exclaves_enter_start_label; |
2812 | enter_range_end = (uintptr_t)&exclaves_enter_end_label; |
2813 | upcall_range_start = (uintptr_t)&exclaves_upcall_start_label; |
2814 | upcall_range_end = (uintptr_t)&exclaves_upcall_end_label; |
2815 | } else { |
2816 | enter_range_start = exclaves_enter_range_start; |
2817 | enter_range_end = exclaves_enter_range_end; |
2818 | upcall_range_start = exclaves_upcall_range_start; |
2819 | upcall_range_end = exclaves_upcall_range_end; |
2820 | } |
2821 | |
2822 | while (i < nframes && |
2823 | !((enter_range_start < out_addr[i]) && (out_addr[i] <= enter_range_end)) |
2824 | && !((upcall_range_start < out_addr[i]) && (out_addr[i] <= upcall_range_end)) |
2825 | ) { |
2826 | i++; |
2827 | } |
2828 | |
2829 | return (uint32_t)i; |
2830 | } |
2831 | |
2832 | #endif /* CONFIG_EXCLAVES */ |
2833 | |
2834 | #ifndef CONFIG_EXCLAVES |
2835 | /* stubs for sensor functions which are not compiled in from exclaves.c when |
2836 | * CONFIG_EXCLAVE is disabled */ |
2837 | |
2838 | kern_return_t |
2839 | exclaves_sensor_start(exclaves_sensor_type_t sensor_type, uint64_t flags, |
2840 | exclaves_sensor_status_t *status) |
2841 | { |
2842 | #pragma unused(sensor_type, flags, status) |
2843 | return KERN_NOT_SUPPORTED; |
2844 | } |
2845 | |
2846 | kern_return_t |
2847 | exclaves_sensor_stop(exclaves_sensor_type_t sensor_type, uint64_t flags, |
2848 | exclaves_sensor_status_t *status) |
2849 | { |
2850 | #pragma unused(sensor_type, flags, status) |
2851 | return KERN_NOT_SUPPORTED; |
2852 | } |
2853 | |
2854 | kern_return_t |
2855 | exclaves_sensor_status(exclaves_sensor_type_t sensor_type, uint64_t flags, |
2856 | exclaves_sensor_status_t *status) |
2857 | { |
2858 | #pragma unused(sensor_type, flags, status) |
2859 | return KERN_NOT_SUPPORTED; |
2860 | } |
2861 | |
2862 | #endif /* ! CONFIG_EXCLAVES */ |
2863 | |