1/*
2 * Copyright (c) 2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <mach/exclaves.h>
30#include <mach/mach_traps.h>
31#include <kern/misc_protos.h>
32#include <kern/assert.h>
33#include <kern/recount.h>
34#include <kern/startup.h>
35
36#if CONFIG_EXCLAVES
37
38#if CONFIG_SPTM
39#include <arm64/sptm/sptm.h>
40#else
41#error Invalid configuration
42#endif /* CONFIG_SPTM */
43
44#include <arm/cpu_data_internal.h>
45#include <kern/epoch_sync.h>
46#include <kern/ipc_kobject.h>
47#include <kern/kalloc.h>
48#include <kern/locks.h>
49#include <kern/percpu.h>
50#include <kern/task.h>
51#include <kern/thread.h>
52#include <kern/zalloc.h>
53#include <kern/exclaves_stackshot.h>
54#include <kern/exclaves_test_stackshot.h>
55#include <vm/pmap.h>
56#include <pexpert/pexpert.h>
57#include <pexpert/device_tree.h>
58
59#include <mach/exclaves_l4.h>
60#include <mach/mach_port.h>
61
62#include <Exclaves/Exclaves.h>
63
64#include <IOKit/IOBSD.h>
65
66#include "exclaves_debug.h"
67#include "exclaves_panic.h"
68
69/* External & generated headers */
70#include <xrt_hosted_types/types.h>
71#include <xnuproxy/messages.h>
72
73/* Use the new version of xnuproxy_msg_t. */
74#define xnuproxy_msg_t xnuproxy_msg_new_t
75
76#if __has_include(<Tightbeam/tightbeam.h>)
77#include <Tightbeam/tightbeam.h>
78#include <Tightbeam/tightbeam_private.h>
79#endif
80
81#include "exclaves_resource.h"
82#include "exclaves_upcalls.h"
83#include "exclaves_boot.h"
84#include "exclaves_inspection.h"
85#include "exclaves_memory.h"
86
87/* Unslid pointers defining the range of code which switches threads into
88 * secure world */
89uintptr_t exclaves_enter_range_start;
90uintptr_t exclaves_enter_range_end;
91
92/* Unslid pointers defining the range of code which triggers upcall handlers */
93uintptr_t exclaves_upcall_range_start;
94uintptr_t exclaves_upcall_range_end;
95
96/* Number of allocated ipcb buffers, estimate of active exclave threads */
97static _Atomic size_t exclaves_ipcb_cnt;
98
99LCK_GRP_DECLARE(exclaves_lck_grp, "exclaves");
100
101/* Lock around communication with singleton xnu proxy server thread */
102LCK_MTX_DECLARE(exclaves_xnu_proxy_lock, &exclaves_lck_grp);
103
104/* Boot lock - only used here for assertions. */
105extern lck_mtx_t exclaves_boot_lock;
106
107/*
108 * Control access to exclaves. Multicore support is learned at runtime.
109 */
110static LCK_MTX_DECLARE(exclaves_scheduler_lock, &exclaves_lck_grp);
111static bool exclaves_multicore;
112#if DEVELOPMENT || DEBUG
113/* boot-arg to control use of the exclaves_scheduler_lock independently of
114 * whether exclaves multicore support is enabled */
115static TUNABLE(bool, exclaves_smp_enabled, "exclaves_smp", true);
116#else
117#define exclaves_smp_enabled true
118#endif
119
120static xnuproxy_msg_t *exclaves_xnu_proxy_msg_buffer;
121static uint64_t exclaves_xnu_proxy_scid;
122#if XNUPROXY_MSG_VERSION >= 3
123static pmap_paddr_t exclaves_xnu_proxy_upcall_ipcb_paddr;
124#endif /* XNUPROXY_MSG_VERSION >= 3 */
125static Exclaves_L4_IpcBuffer_t *exclaves_xnu_proxy_upcall_ipcb;
126
127
128/*
129 * Sent/latest offset for updating exclaves clocks
130 */
131typedef struct {
132 union {
133 /* atomic fields are used via atomic primitives */
134 struct { _Atomic uint64_t sent_offset, latest_offset; } a_u64;
135 _Atomic unsigned __int128 a_u128;
136 /* non-atomic fields are used via local variable. this is needed to
137 * avoid undefined behavior with an atomic struct or accessing atomic
138 * fields non-atomically */
139 struct { uint64_t sent_offset, latest_offset; } u64;
140 unsigned __int128 u128;
141 };
142} exclaves_clock_t;
143
144static exclaves_clock_t exclaves_absolute_clock, exclaves_continuous_clock;
145
146/*
147 * boot-arg to control the service lookup fallback.
148 * When set, it allows services in the com.apple.kernel and com.apple.darwin
149 * domains to be found when the service can't be found in the attached conclave
150 * domain.
151 */
152static TUNABLE(bool, exclaves_service_fallback, "exclaves_service_fallback", true);
153
154static kern_return_t
155exclaves_acquire_ipc_buffer(Exclaves_L4_IpcBuffer_t **ipcb_out,
156 Exclaves_L4_Word_t *scid_out);
157static kern_return_t
158exclaves_relinquish_ipc_buffer(Exclaves_L4_IpcBuffer_t *ipcb,
159 Exclaves_L4_Word_t scid);
160static kern_return_t
161exclaves_endpoint_call_internal(ipc_port_t port, exclaves_id_t endpoint_id);
162
163static kern_return_t
164exclaves_enter(void);
165static kern_return_t
166exclaves_bootinfo(uint64_t *out_boot_info, bool *early_enter);
167
168static kern_return_t
169exclaves_scheduler_init(uint64_t boot_info);
170OS_NORETURN OS_NOINLINE
171static void
172exclaves_wait_for_panic(void);
173
174static bool
175exclaves_clock_needs_update(const exclaves_clock_t *clock);
176static kern_return_t
177exclaves_clock_update(exclaves_clock_t *clock, XrtHosted_Buffer_t *save_out_ptr, XrtHosted_Buffer_t *save_in_ptr);
178
179kern_return_t
180exclaves_scheduler_resume_scheduling_context(Exclaves_L4_Word_t scid,
181 Exclaves_L4_Word_t *spawned_scid, bool interrupted);
182static kern_return_t
183exclaves_scheduler_boot(void);
184
185static kern_return_t
186exclaves_xnu_proxy_init(uint64_t xnu_proxy_boot_info);
187static kern_return_t
188exclaves_xnu_proxy_allocate_context(Exclaves_L4_Word_t *out_scid,
189 Exclaves_L4_IpcBuffer_t **out_ipcb);
190static kern_return_t
191exclaves_xnu_proxy_free_context(Exclaves_L4_Word_t scid);
192static kern_return_t
193exclaves_xnu_proxy_endpoint_call(Exclaves_L4_Word_t endpoint_id);
194static kern_return_t
195exclaves_hosted_error(bool success, XrtHosted_Error_t *error);
196
197/*
198 * A static set of exclave epoch counters.
199 */
200static os_atomic(uint64_t) epoch_counter[XrtHosted_Counter_limit] = {};
201
202static inline os_atomic(uint64_t) *
203exclaves_get_queue_counter(const uint64_t id)
204{
205 return &epoch_counter[XrtHosted_Counter_fromQueueId(id)];
206}
207
208static inline os_atomic(uint64_t) *
209exclaves_get_thread_counter(const uint64_t id)
210{
211 return &epoch_counter[XrtHosted_Counter_fromThreadId(id)];
212}
213
214/*
215 * A (simple, for now...) cache of IPC buffers for communicating with XNU-Proxy.
216 * Limited in size by the same value as XNU-Proxy's EC limit.
217 * Must be realtime-safe.
218 */
219
220static kern_return_t
221exclaves_ipc_buffer_cache_init(void);
222
223/* Intrusive linked list within the unused IPC buffer */
224struct exclaves_ipc_buffer_cache_item {
225 struct exclaves_ipc_buffer_cache_item *next;
226 Exclaves_L4_Word_t scid;
227} __attribute__((__packed__));
228
229_Static_assert(Exclaves_L4_IpcBuffer_Size >= sizeof(struct exclaves_ipc_buffer_cache_item),
230 "Invalid Exclaves_L4_IpcBuffer_Size");
231
232LCK_SPIN_DECLARE(exclaves_ipc_buffer_cache_lock, &exclaves_lck_grp);
233static struct exclaves_ipc_buffer_cache_item *exclaves_ipc_buffer_cache;
234
235/* -------------------------------------------------------------------------- */
236#pragma mark exclaves debug configuration
237
238#if DEVELOPMENT || DEBUG
239TUNABLE_WRITEABLE(unsigned int, exclaves_debug, "exclaves_debug",
240 exclaves_debug_show_errors);
241#endif /* DEVELOPMENT || DEBUG */
242
243#if DEVELOPMENT || DEBUG
244TUNABLE_WRITEABLE(unsigned int, exclaves_ipc_buffer_cache_enabled, "exclaves_ipcb_cache", 1);
245#else
246#define exclaves_ipc_buffer_cache_enabled 1
247#endif
248#endif /* CONFIG_EXCLAVES */
249
250/* -------------------------------------------------------------------------- */
251#pragma mark userspace entry point
252
253kern_return_t
254_exclaves_ctl_trap(struct exclaves_ctl_trap_args *uap)
255{
256#if CONFIG_EXCLAVES
257 kern_return_t kr = KERN_SUCCESS;
258 int error = 0;
259
260 mach_port_name_t name = uap->name;
261 exclaves_id_t identifier = uap->identifier;
262 mach_vm_address_t ubuffer = uap->buffer;
263 mach_vm_size_t usize = uap->size;
264 mach_vm_size_t uoffset = (mach_vm_size_t)uap->identifier;
265 mach_vm_size_t usize2 = uap->size2;
266 mach_vm_size_t uoffset2 = uap->offset;
267 task_t task = current_task();
268
269 /*
270 * EXCLAVES_XNU_PROXY_CR_RETVAL comes from ExclavePlatform and is shared
271 * with xnu. That header is not shared with userspace. Make sure that
272 * the retval userspace picks up is the same as the one
273 * xnu/ExclavePlatform thinks it is.
274 */
275 assert3p(&EXCLAVES_XNU_PROXY_CR_RETVAL((Exclaves_L4_IpcBuffer_t *)0), ==,
276 &XNUPROXY_CR_RETVAL((Exclaves_L4_IpcBuffer_t *)0));
277
278 uint8_t operation = EXCLAVES_CTL_OP(uap->operation_and_flags);
279 uint32_t flags = EXCLAVES_CTL_FLAGS(uap->operation_and_flags);
280 if (flags != 0) {
281 return KERN_INVALID_ARGUMENT;
282 }
283
284 /*
285 * All operations other than OP_BOOT are restricted to properly entitled
286 * tasks which can operation in the kernel domain, or those which have
287 * joined conclaves (which has its own entitlement check).
288 */
289 if (operation != EXCLAVES_CTL_OP_BOOT &&
290 task_get_conclave(task) == NULL &&
291 !exclaves_has_priv(task, EXCLAVES_PRIV_KERNEL_DOMAIN)) {
292 return KERN_DENIED;
293 }
294
295 /*
296 * As the boot operation itself happens outside the context of any
297 * conclave, it requires special privilege.
298 */
299 if (operation == EXCLAVES_CTL_OP_BOOT &&
300 !exclaves_has_priv(current_task(), EXCLAVES_PRIV_BOOT)) {
301 return KERN_DENIED;
302 }
303
304 /*
305 * The only valid operation if exclaves are not booted to
306 * EXCLAVES_BOOT_STAGE_EXCLAVEKIT, is the BOOT op.
307 */
308 if (operation != EXCLAVES_CTL_OP_BOOT) {
309 /*
310 * Make this EXCLAVES_BOOT_STAGE_2 until userspace is actually
311 * triggering the EXCLAVESKIT boot stage.
312 */
313 kr = exclaves_boot_wait(EXCLAVES_BOOT_STAGE_2);
314 if (kr != KERN_SUCCESS) {
315 return kr;
316 }
317 }
318
319 switch (operation) {
320 case EXCLAVES_CTL_OP_ENDPOINT_CALL: {
321 if (name != MACH_PORT_NULL) {
322 /* Only accept MACH_PORT_NULL for now */
323 return KERN_INVALID_CAPABILITY;
324 }
325 if (ubuffer == USER_ADDR_NULL || usize == 0 ||
326 usize != Exclaves_L4_IpcBuffer_Size) {
327 return KERN_INVALID_ARGUMENT;
328 }
329
330 Exclaves_L4_IpcBuffer_t *ipcb;
331 if ((error = exclaves_allocate_ipc_buffer((void**)&ipcb))) {
332 return error;
333 }
334 assert(ipcb != NULL);
335 if ((error = copyin(ubuffer, ipcb, usize))) {
336 return error;
337 }
338
339 if (identifier >= CONCLAVE_SERVICE_MAX) {
340 return KERN_INVALID_ARGUMENT;
341 }
342
343 /*
344 * Verify that the service actually exists in the current
345 * domain (only when the fallbacks are not enabled).
346 */
347 if (!exclaves_service_fallback &&
348 !exclaves_conclave_has_service(task_get_conclave(task),
349 identifier)) {
350 return KERN_INVALID_ARGUMENT;
351 }
352
353 kr = exclaves_endpoint_call_internal(IPC_PORT_NULL, identifier);
354 error = copyout(ipcb, ubuffer, usize);
355 /*
356 * Endpoint call to conclave may have trigger a stop upcall,
357 * check if stop upcall completion handler needs to run.
358 */
359 task_stop_conclave_upcall_complete();
360 if (error) {
361 return error;
362 }
363 break;
364 }
365
366 case EXCLAVES_CTL_OP_NAMED_BUFFER_CREATE: {
367 if (name != MACH_PORT_NULL) {
368 /* Only accept MACH_PORT_NULL for now */
369 return KERN_INVALID_CAPABILITY;
370 }
371
372 size_t len = 0;
373 char id_name[XNUPROXY_RESOURCE_NAME_MAX] = "";
374 if (copyinstr(identifier, id_name, XNUPROXY_RESOURCE_NAME_MAX,
375 &len) != 0 || id_name[0] == '\0') {
376 return KERN_INVALID_ARGUMENT;
377 }
378
379 exclaves_buffer_perm_t perm = (exclaves_buffer_perm_t)usize2;
380 const exclaves_buffer_perm_t supported =
381 EXCLAVES_BUFFER_PERM_READ | EXCLAVES_BUFFER_PERM_WRITE;
382 if ((perm & supported) == 0 || (perm & ~supported) != 0) {
383 return KERN_INVALID_ARGUMENT;
384 }
385
386 const char *domain = exclaves_conclave_get_domain(task_get_conclave(task));
387 const bool new_api =
388 (perm == EXCLAVES_BUFFER_PERM_READ) ||
389 (perm == EXCLAVES_BUFFER_PERM_WRITE);
390 const bool shared_mem_available =
391 exclaves_resource_lookup_by_name(domain, id_name,
392 XNUPROXY_RESOURCE_SHARED_MEMORY) != NULL;
393 const bool use_shared_mem = new_api && shared_mem_available;
394
395 exclaves_resource_t *resource = NULL;
396 kr = use_shared_mem ?
397 exclaves_resource_shared_memory_map(domain, id_name, usize, perm, &resource) :
398 exclaves_named_buffer_map(domain, id_name, usize, perm, &resource);
399 if (kr != KERN_SUCCESS) {
400 return kr;
401 }
402
403 kr = exclaves_resource_create_port_name(resource,
404 current_space(), &name);
405 if (kr != KERN_SUCCESS) {
406 return kr;
407 }
408
409 kr = copyout(&name, ubuffer, sizeof(mach_port_name_t));
410 if (kr != KERN_SUCCESS) {
411 mach_port_deallocate(current_space(), name);
412 return kr;
413 }
414
415 break;
416 }
417
418 case EXCLAVES_CTL_OP_NAMED_BUFFER_COPYIN: {
419 exclaves_resource_t *resource = NULL;
420 kr = exclaves_resource_from_port_name(current_space(), name,
421 &resource);
422 if (kr != KERN_SUCCESS) {
423 return kr;
424 }
425
426 switch (resource->r_type) {
427 case XNUPROXY_RESOURCE_NAMED_BUFFER:
428 kr = exclaves_named_buffer_copyin(resource, ubuffer,
429 usize, uoffset, usize2, uoffset2);
430 break;
431
432 case XNUPROXY_RESOURCE_SHARED_MEMORY:
433 kr = exclaves_resource_shared_memory_copyin(resource,
434 ubuffer, usize, uoffset, usize2, uoffset2);
435 break;
436
437 default:
438 exclaves_resource_release(resource);
439 return KERN_INVALID_CAPABILITY;
440 }
441
442 exclaves_resource_release(resource);
443
444 if (kr != KERN_SUCCESS) {
445 return kr;
446 }
447 break;
448 }
449
450 case EXCLAVES_CTL_OP_NAMED_BUFFER_COPYOUT: {
451 exclaves_resource_t *resource = NULL;
452 kr = exclaves_resource_from_port_name(current_space(), name,
453 &resource);
454 if (kr != KERN_SUCCESS) {
455 return kr;
456 }
457
458 switch (resource->r_type) {
459 case XNUPROXY_RESOURCE_NAMED_BUFFER:
460 kr = exclaves_named_buffer_copyout(resource, ubuffer,
461 usize, uoffset, usize2, uoffset2);
462 break;
463
464 case XNUPROXY_RESOURCE_SHARED_MEMORY:
465 kr = exclaves_resource_shared_memory_copyout(resource,
466 ubuffer, usize, uoffset, usize2, uoffset2);
467 break;
468
469 default:
470 exclaves_resource_release(resource);
471 return KERN_INVALID_CAPABILITY;
472 }
473
474 exclaves_resource_release(resource);
475
476 if (kr != KERN_SUCCESS) {
477 return kr;
478 }
479 break;
480 }
481
482 case EXCLAVES_CTL_OP_BOOT:
483 if (name != MACH_PORT_NULL) {
484 /* Only accept MACH_PORT_NULL for now */
485 return KERN_INVALID_CAPABILITY;
486 }
487 kr = exclaves_boot((uint32_t)identifier);
488 break;
489
490 case EXCLAVES_CTL_OP_LAUNCH_CONCLAVE:
491 if (name != MACH_PORT_NULL) {
492 /* Only accept MACH_PORT_NULL for now */
493 return KERN_INVALID_CAPABILITY;
494 }
495 kr = task_launch_conclave(name);
496
497 /*
498 * Conclave launch call to may have trigger a stop upcall,
499 * check if stop upcall completion handler needs to run.
500 */
501 task_stop_conclave_upcall_complete();
502 break;
503
504 case EXCLAVES_CTL_OP_LOOKUP_SERVICES: {
505 if (name != MACH_PORT_NULL) {
506 /* Only accept MACH_PORT_NULL for now */
507 return KERN_INVALID_CAPABILITY;
508 }
509 struct exclaves_resource_user uresource = {};
510
511 if (usize > (MAX_CONCLAVE_RESOURCE_NUM * sizeof(struct exclaves_resource_user)) ||
512 (usize % sizeof(struct exclaves_resource_user) != 0)) {
513 return KERN_INVALID_ARGUMENT;
514 }
515
516 if ((ubuffer == USER_ADDR_NULL && usize != 0) ||
517 (usize == 0 && ubuffer != USER_ADDR_NULL)) {
518 return KERN_INVALID_ARGUMENT;
519 }
520
521 if (ubuffer == USER_ADDR_NULL) {
522 return KERN_INVALID_ARGUMENT;
523 }
524
525 /* For the moment we only ever have to deal with one request. */
526 if (usize != sizeof(struct exclaves_resource_user)) {
527 return KERN_INVALID_ARGUMENT;
528 }
529 error = copyin(ubuffer, &uresource, usize);
530 if (error) {
531 return KERN_INVALID_ARGUMENT;
532 }
533
534 const size_t name_buf_len = sizeof(uresource.r_name);
535 if (strnlen(uresource.r_name, name_buf_len) == name_buf_len) {
536 return KERN_INVALID_ARGUMENT;
537 }
538
539 /*
540 * Do the regular lookup first. If that fails, fallback to the
541 * DARWIN domain, finally fallback to the KERNEL domain.
542 */
543 const char *domain = exclaves_conclave_get_domain(task_get_conclave(task));
544 uint64_t id = exclaves_service_lookup(domain, uresource.r_name);
545
546 /* Disable fallbacks via boot-arg. */
547 if (exclaves_service_fallback) {
548 if (id == UINT64_C(~0)) {
549 id = exclaves_service_lookup(EXCLAVES_DOMAIN_DARWIN,
550 uresource.r_name);
551 }
552 if (id == UINT64_C(~0)) {
553 id = exclaves_service_lookup(EXCLAVES_DOMAIN_KERNEL,
554 uresource.r_name);
555 }
556 }
557
558 if (id == UINT64_C(~0)) {
559 return KERN_NOT_FOUND;
560 }
561
562 uresource.r_id = id;
563 uresource.r_port = MACH_PORT_NULL;
564
565 error = copyout(&uresource, ubuffer, usize);
566 if (error) {
567 return KERN_INVALID_ADDRESS;
568 }
569
570 kr = KERN_SUCCESS;
571 break;
572 }
573
574 case EXCLAVES_CTL_OP_AUDIO_BUFFER_CREATE: {
575 if (identifier == 0) {
576 return KERN_INVALID_ARGUMENT;
577 }
578
579 /* copy in string name */
580 char id_name[XNUPROXY_RESOURCE_NAME_MAX] = "";
581 size_t done = 0;
582 if (copyinstr(identifier, id_name, XNUPROXY_RESOURCE_NAME_MAX, &done) != 0) {
583 return KERN_INVALID_ARGUMENT;
584 }
585
586 const char *domain = exclaves_conclave_get_domain(task_get_conclave(task));
587 const bool use_audio_memory =
588 exclaves_resource_lookup_by_name(domain, id_name,
589 XNUPROXY_RESOURCE_ARBITRATED_AUDIO_MEMORY) != NULL;
590 exclaves_resource_t *resource = NULL;
591 kr = use_audio_memory ?
592 exclaves_resource_audio_memory_map(domain, id_name, usize, &resource) :
593 exclaves_audio_buffer_map(domain, id_name, usize, &resource);
594 if (kr != KERN_SUCCESS) {
595 return kr;
596 }
597
598 kr = exclaves_resource_create_port_name(resource, current_space(),
599 &name);
600 if (kr != KERN_SUCCESS) {
601 return kr;
602 }
603
604 kr = copyout(&name, ubuffer, sizeof(mach_port_name_t));
605 if (kr != KERN_SUCCESS) {
606 mach_port_deallocate(current_space(), name);
607 return kr;
608 }
609
610 break;
611 }
612
613 case EXCLAVES_CTL_OP_AUDIO_BUFFER_COPYOUT: {
614 exclaves_resource_t *resource;
615
616 kr = exclaves_resource_from_port_name(current_space(), name, &resource);
617 if (kr != KERN_SUCCESS) {
618 return kr;
619 }
620
621 switch (resource->r_type) {
622 case XNUPROXY_RESOURCE_ARBITRATED_AUDIO_BUFFER:
623 kr = exclaves_audio_buffer_copyout(resource, ubuffer,
624 usize, uoffset, usize2, uoffset2);
625 break;
626
627 case XNUPROXY_RESOURCE_ARBITRATED_AUDIO_MEMORY:
628 kr = exclaves_resource_audio_memory_copyout(resource,
629 ubuffer, usize, uoffset, usize2, uoffset2);
630 break;
631
632 default:
633 exclaves_resource_release(resource);
634 return KERN_INVALID_CAPABILITY;
635 }
636
637 exclaves_resource_release(resource);
638
639 if (kr != KERN_SUCCESS) {
640 return kr;
641 }
642
643 break;
644 }
645
646 case EXCLAVES_CTL_OP_SENSOR_CREATE: {
647 if (identifier == 0) {
648 return KERN_INVALID_ARGUMENT;
649 }
650
651 /* copy in string name */
652 char id_name[XNUPROXY_RESOURCE_NAME_MAX] = "";
653 size_t done = 0;
654 if (copyinstr(identifier, id_name, XNUPROXY_RESOURCE_NAME_MAX, &done) != 0) {
655 return KERN_INVALID_ARGUMENT;
656 }
657
658 const char *domain = exclaves_conclave_get_domain(task_get_conclave(task));
659 exclaves_resource_t *resource = NULL;
660 kr = exclaves_resource_sensor_open(domain, id_name, &resource);
661 if (kr != KERN_SUCCESS) {
662 return kr;
663 }
664
665 kr = exclaves_resource_create_port_name(resource, current_space(),
666 &name);
667 if (kr != KERN_SUCCESS) {
668 return kr;
669 }
670
671 kr = copyout(&name, ubuffer, sizeof(mach_port_name_t));
672 if (kr != KERN_SUCCESS) {
673 /* No senders drops the reference. */
674 mach_port_deallocate(current_space(), name);
675 return kr;
676 }
677
678 break;
679 }
680
681 case EXCLAVES_CTL_OP_SENSOR_START: {
682 exclaves_resource_t *resource;
683 kr = exclaves_resource_from_port_name(current_space(), name, &resource);
684 if (kr != KERN_SUCCESS) {
685 return kr;
686 }
687
688 if (resource->r_type != XNUPROXY_RESOURCE_SENSOR) {
689 exclaves_resource_release(resource);
690 return KERN_FAILURE;
691 }
692
693 exclaves_sensor_status_t status;
694 kr = exclaves_resource_sensor_start(resource, identifier, &status);
695
696 exclaves_resource_release(resource);
697
698 if (kr != KERN_SUCCESS) {
699 return kr;
700 }
701
702 kr = copyout(&status, ubuffer, sizeof(exclaves_sensor_status_t));
703
704 break;
705 }
706 case EXCLAVES_CTL_OP_SENSOR_STOP: {
707 exclaves_resource_t *resource;
708 kr = exclaves_resource_from_port_name(current_space(), name, &resource);
709 if (kr != KERN_SUCCESS) {
710 return kr;
711 }
712
713 if (resource->r_type != XNUPROXY_RESOURCE_SENSOR) {
714 exclaves_resource_release(resource);
715 return KERN_FAILURE;
716 }
717
718 exclaves_sensor_status_t status;
719 kr = exclaves_resource_sensor_stop(resource, identifier, &status);
720
721 exclaves_resource_release(resource);
722
723 if (kr != KERN_SUCCESS) {
724 return kr;
725 }
726
727 kr = copyout(&status, ubuffer, sizeof(exclaves_sensor_status_t));
728
729 break;
730 }
731 case EXCLAVES_CTL_OP_SENSOR_STATUS: {
732 exclaves_resource_t *resource;
733 kr = exclaves_resource_from_port_name(current_space(), name, &resource);
734 if (kr != KERN_SUCCESS) {
735 return kr;
736 }
737
738 if (resource->r_type != XNUPROXY_RESOURCE_SENSOR) {
739 exclaves_resource_release(resource);
740 return KERN_FAILURE;
741 }
742
743
744 exclaves_sensor_status_t status;
745 kr = exclaves_resource_sensor_status(resource, identifier, &status);
746
747 exclaves_resource_release(resource);
748
749 if (kr != KERN_SUCCESS) {
750 return kr;
751 }
752
753 kr = copyout(&status, ubuffer, sizeof(exclaves_sensor_status_t));
754 break;
755 }
756 case EXCLAVES_CTL_OP_NOTIFICATION_RESOURCE_LOOKUP: {
757 exclaves_resource_t *notification_resource = NULL;
758 mach_port_name_t port_name = MACH_PORT_NULL;
759
760 struct exclaves_resource_user *notification_resource_user = NULL;
761 if (usize != sizeof(struct exclaves_resource_user)) {
762 return KERN_INVALID_ARGUMENT;
763 }
764
765 if (ubuffer == USER_ADDR_NULL) {
766 return KERN_INVALID_ARGUMENT;
767 }
768
769 notification_resource_user = (struct exclaves_resource_user *)
770 kalloc_data(usize, Z_WAITOK | Z_ZERO | Z_NOFAIL);
771
772 error = copyin(ubuffer, notification_resource_user, usize);
773 if (error) {
774 kr = KERN_INVALID_ARGUMENT;
775 goto notification_resource_lookup_out;
776 }
777
778 const size_t name_buf_len = sizeof(notification_resource_user->r_name);
779 if (strnlen(notification_resource_user->r_name, name_buf_len)
780 == name_buf_len) {
781 kr = KERN_INVALID_ARGUMENT;
782 goto notification_resource_lookup_out;
783 }
784
785 const char *domain = exclaves_conclave_get_domain(task_get_conclave(task));
786 kr = exclaves_notification_create(domain,
787 notification_resource_user->r_name, &notification_resource);
788 if (kr != KERN_SUCCESS) {
789 goto notification_resource_lookup_out;
790 }
791
792 kr = exclaves_resource_create_port_name(notification_resource,
793 current_space(), &port_name);
794 if (kr != KERN_SUCCESS) {
795 goto notification_resource_lookup_out;
796 }
797 notification_resource_user->r_type = notification_resource->r_type;
798 notification_resource_user->r_id = notification_resource->r_id;
799 notification_resource_user->r_port = port_name;
800 error = copyout(notification_resource_user, ubuffer, usize);
801 if (error) {
802 kr = KERN_INVALID_ADDRESS;
803 goto notification_resource_lookup_out;
804 }
805
806notification_resource_lookup_out:
807 if (notification_resource_user != NULL) {
808 kfree_data(notification_resource_user, usize);
809 }
810 if (kr != KERN_SUCCESS && port_name != MACH_PORT_NULL) {
811 mach_port_deallocate(current_space(), port_name);
812 }
813 break;
814 }
815
816 default:
817 kr = KERN_INVALID_ARGUMENT;
818 break;
819 }
820
821 return kr;
822#else /* CONFIG_EXCLAVES */
823#pragma unused(uap)
824 return KERN_NOT_SUPPORTED;
825#endif /* CONFIG_EXCLAVES */
826}
827
828/* -------------------------------------------------------------------------- */
829#pragma mark kernel entry points
830
831kern_return_t
832exclaves_endpoint_call(ipc_port_t port, exclaves_id_t endpoint_id,
833 exclaves_tag_t *tag, exclaves_error_t *error)
834{
835#if CONFIG_EXCLAVES
836 kern_return_t kr = KERN_SUCCESS;
837 assert(port == IPC_PORT_NULL);
838
839 Exclaves_L4_IpcBuffer_t *ipcb = Exclaves_L4_IpcBuffer();
840 assert(ipcb != NULL);
841
842 exclaves_debug_printf(show_progress,
843 "exclaves: endpoint call:\tendpoint id %lld tag 0x%llx\n",
844 endpoint_id, *tag);
845
846 ipcb->mr[Exclaves_L4_Ipc_Mr_Tag] = *tag;
847 kr = exclaves_endpoint_call_internal(port, endpoint_id);
848 *tag = ipcb->mr[Exclaves_L4_Ipc_Mr_Tag];
849 *error = XNUPROXY_CR_RETVAL(ipcb);
850
851 exclaves_debug_printf(show_progress,
852 "exclaves: endpoint call return:\tendpoint id %lld tag 0x%llx "
853 "error 0x%llx\n", endpoint_id, *tag, *error);
854
855 return kr;
856#else /* CONFIG_EXCLAVES */
857#pragma unused(port, endpoint_id, tag, error)
858 return KERN_NOT_SUPPORTED;
859#endif /* CONFIG_EXCLAVES */
860}
861
862/* Realtime-safe acquisition of an IPC buffer */
863kern_return_t
864exclaves_allocate_ipc_buffer(void **out_ipc_buffer)
865{
866#if CONFIG_EXCLAVES
867 kern_return_t kr = KERN_SUCCESS;
868 thread_t thread = current_thread();
869 Exclaves_L4_IpcBuffer_t *ipcb = thread->th_exclaves_ipc_buffer;
870 Exclaves_L4_Word_t scid = thread->th_exclaves_scheduling_context_id;
871
872 if (ipcb == NULL) {
873 assert(scid == 0);
874 if ((kr = exclaves_acquire_ipc_buffer(&ipcb, &scid))) {
875 return kr;
876 }
877 thread->th_exclaves_ipc_buffer = ipcb;
878 thread->th_exclaves_scheduling_context_id = scid;
879 }
880 if (out_ipc_buffer) {
881 *out_ipc_buffer = (void*)ipcb;
882 }
883
884 return kr;
885#else /* CONFIG_EXCLAVES */
886#pragma unused(out_ipc_buffer)
887 return KERN_NOT_SUPPORTED;
888#endif /* CONFIG_EXCLAVES */
889}
890
891#if CONFIG_EXCLAVES
892static kern_return_t
893exclaves_thread_free_ipc_buffer(thread_t thread)
894{
895 kern_return_t kr = KERN_SUCCESS;
896 Exclaves_L4_IpcBuffer_t *ipcb = thread->th_exclaves_ipc_buffer;
897 Exclaves_L4_Word_t scid = thread->th_exclaves_scheduling_context_id;
898
899 if (ipcb != NULL) {
900 assert(scid != 0);
901 thread->th_exclaves_ipc_buffer = NULL;
902 thread->th_exclaves_scheduling_context_id = 0;
903
904 kr = exclaves_relinquish_ipc_buffer(ipcb, scid);
905 } else {
906 assert(scid == 0);
907 }
908
909 return kr;
910}
911#endif /* CONFIG_EXCLAVES */
912
913kern_return_t
914exclaves_free_ipc_buffer(void)
915{
916#if CONFIG_EXCLAVES
917 thread_t thread = current_thread();
918
919 /* The inspection thread's cached buffer should never be freed */
920 if ((os_atomic_load(&thread->th_exclaves_inspection_state, relaxed) & TH_EXCLAVES_INSPECTION_NOINSPECT) != 0) {
921 return KERN_SUCCESS;
922 }
923
924 return exclaves_thread_free_ipc_buffer(thread);
925#else /* CONFIG_EXCLAVES */
926 return KERN_NOT_SUPPORTED;
927#endif /* CONFIG_EXCLAVES */
928}
929
930kern_return_t
931exclaves_thread_terminate(__unused thread_t thread)
932{
933 kern_return_t kr = KERN_SUCCESS;
934
935#if CONFIG_EXCLAVES
936 assert(thread == current_thread());
937 assert(thread->th_exclaves_intstate == 0);
938 assert(thread->th_exclaves_state == 0);
939 if (thread->th_exclaves_ipc_buffer) {
940 exclaves_debug_printf(show_progress,
941 "exclaves: thread_terminate freeing abandoned exclaves "
942 "ipc buffer\n");
943 kr = exclaves_thread_free_ipc_buffer(thread);
944 assert(kr == KERN_SUCCESS);
945 }
946#else
947#pragma unused(thread)
948#endif /* CONFIG_EXCLAVES */
949
950 return kr;
951}
952
953OS_CONST
954void*
955exclaves_get_ipc_buffer(void)
956{
957#if CONFIG_EXCLAVES
958 thread_t thread = current_thread();
959 Exclaves_L4_IpcBuffer_t *ipcb = thread->th_exclaves_ipc_buffer;
960 assert(ipcb != NULL);
961
962 return ipcb;
963#else /* CONFIG_EXCLAVES */
964 return NULL;
965#endif /* CONFIG_EXCLAVES */
966}
967
968#if CONFIG_EXCLAVES
969
970__startup_func
971static void
972initialize_exclaves_call_range(void)
973{
974 exclaves_enter_range_start = VM_KERNEL_UNSLIDE(&exclaves_enter_start_label);
975 assert3u(exclaves_enter_range_start, !=, 0);
976 exclaves_enter_range_end = VM_KERNEL_UNSLIDE(&exclaves_enter_end_label);
977 assert3u(exclaves_enter_range_end, !=, 0);
978 exclaves_upcall_range_start = VM_KERNEL_UNSLIDE(&exclaves_upcall_start_label);
979 assert3u(exclaves_upcall_range_start, !=, 0);
980 exclaves_upcall_range_end = VM_KERNEL_UNSLIDE(&exclaves_upcall_end_label);
981 assert3u(exclaves_upcall_range_end, !=, 0);
982}
983STARTUP(EARLY_BOOT, STARTUP_RANK_MIDDLE, initialize_exclaves_call_range);
984
985static void
986bind_to_boot_core(void)
987{
988 /*
989 * First ensure the boot cluster isn't powered down preventing the
990 * thread from running at all.
991 */
992 suspend_cluster_powerdown();
993 const int cpu = ml_get_boot_cpu_number();
994 processor_t processor = cpu_to_processor(cpu);
995 assert3p(processor, !=, NULL);
996 __assert_only processor_t old = thread_bind(processor);
997 assert3p(old, ==, PROCESSOR_NULL);
998 thread_block(THREAD_CONTINUE_NULL);
999}
1000
1001static void
1002unbind_from_boot_core(void)
1003{
1004 /* Unbind the thread from the boot CPU. */
1005 thread_bind(PROCESSOR_NULL);
1006 thread_block(THREAD_CONTINUE_NULL);
1007 resume_cluster_powerdown();
1008}
1009
1010extern kern_return_t exclaves_boot_early(void);
1011kern_return_t
1012exclaves_boot_early(void)
1013{
1014 kern_return_t kr = KERN_FAILURE;
1015 uint64_t boot_info = 0;
1016 bool early_enter = false;
1017
1018 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
1019
1020 kr = exclaves_bootinfo(&boot_info, &early_enter);
1021 if (kr != KERN_SUCCESS) {
1022 exclaves_debug_printf(show_errors,
1023 "exclaves: Get bootinfo failed\n");
1024 return kr;
1025 }
1026
1027 if (early_enter) {
1028 thread_t thread = current_thread();
1029 assert3u(thread->th_exclaves_state & TH_EXCLAVES_STATE_ANY, ==, 0);
1030
1031 bind_to_boot_core();
1032
1033 disable_preemption_without_measurements();
1034 thread->th_exclaves_state |= TH_EXCLAVES_SCHEDULER_CALL;
1035
1036 kr = exclaves_enter();
1037
1038 thread->th_exclaves_state &= ~TH_EXCLAVES_SCHEDULER_CALL;
1039 enable_preemption();
1040
1041 unbind_from_boot_core();
1042
1043 if (kr != KERN_SUCCESS) {
1044 exclaves_debug_printf(show_errors,
1045 "exclaves: early exclaves enter failed\n");
1046 if (kr == KERN_ABORTED) {
1047 panic("Unexpected ringgate panic status");
1048 }
1049 return kr;
1050 }
1051 }
1052
1053 kr = exclaves_scheduler_init(boot_info);
1054 if (kr != KERN_SUCCESS) {
1055 exclaves_debug_printf(show_errors,
1056 "exclaves: Init scheduler failed\n");
1057 return kr;
1058 }
1059
1060 kr = exclaves_ipc_buffer_cache_init();
1061 if (kr != KERN_SUCCESS) {
1062 exclaves_debug_printf(show_errors,
1063 "exclaves: failed to initialize IPC buffer cache\n");
1064 return kr;
1065 }
1066
1067 kr = exclaves_resource_init();
1068 if (kr != KERN_SUCCESS) {
1069 exclaves_debug_printf(show_errors,
1070 "exclaves: failed to initialize resources\n");
1071 return kr;
1072 }
1073
1074 return KERN_SUCCESS;
1075}
1076#endif /* CONFIG_EXCLAVES */
1077
1078#if CONFIG_EXCLAVES
1079static struct XrtHosted_Callbacks *exclaves_callbacks = NULL;
1080#endif /* CONFIG_EXCLAVES */
1081
1082void
1083exclaves_register_xrt_hosted_callbacks(struct XrtHosted_Callbacks *callbacks)
1084{
1085#if CONFIG_EXCLAVES
1086 if (exclaves_callbacks == NULL) {
1087 exclaves_callbacks = callbacks;
1088 }
1089#else /* CONFIG_EXCLAVES */
1090#pragma unused(callbacks)
1091#endif /* CONFIG_EXCLAVES */
1092}
1093
1094void
1095exclaves_update_timebase(exclaves_clock_type_t type, uint64_t offset)
1096{
1097#if CONFIG_EXCLAVES
1098 exclaves_clock_t *clock = (type == EXCLAVES_CLOCK_ABSOLUTE ?
1099 &exclaves_absolute_clock : &exclaves_continuous_clock);
1100 uint64_t latest_offset = os_atomic_load(&clock->a_u64.latest_offset, relaxed);
1101 while (latest_offset < offset) {
1102 /* Update the latest offset with the new offset. If this fails, then a
1103 * concurrent update occurred and our offset may be stale. */
1104 if (os_atomic_cmpxchgv(&clock->a_u64.latest_offset, latest_offset,
1105 offset, &latest_offset, relaxed)) {
1106 break;
1107 }
1108 }
1109#else
1110#pragma unused(type, offset)
1111#endif /* CONFIG_EXCLAVES */
1112}
1113
1114/* -------------------------------------------------------------------------- */
1115
1116#pragma mark exclaves ipc internals
1117
1118#if CONFIG_EXCLAVES
1119
1120static kern_return_t
1121exclaves_acquire_ipc_buffer(Exclaves_L4_IpcBuffer_t **out_ipcb,
1122 Exclaves_L4_Word_t *out_scid)
1123{
1124 kern_return_t kr = KERN_SUCCESS;
1125 Exclaves_L4_IpcBuffer_t *ipcb = NULL;
1126 Exclaves_L4_Word_t scid = 0;
1127 struct exclaves_ipc_buffer_cache_item *cached_buffer = NULL;
1128
1129
1130 _Static_assert(Exclaves_L4_IpcBuffer_Size < PAGE_SIZE,
1131 "Invalid Exclaves_L4_IpcBuffer_Size");
1132
1133 if (exclaves_ipc_buffer_cache_enabled) {
1134 lck_spin_lock(&exclaves_ipc_buffer_cache_lock);
1135 if (exclaves_ipc_buffer_cache != NULL) {
1136 cached_buffer = exclaves_ipc_buffer_cache;
1137 exclaves_ipc_buffer_cache = cached_buffer->next;
1138 }
1139 lck_spin_unlock(&exclaves_ipc_buffer_cache_lock);
1140 }
1141
1142 if (cached_buffer) {
1143 scid = cached_buffer->scid;
1144
1145 /* zero out this usage of the buffer to avoid any confusion in xnuproxy */
1146 cached_buffer->next = NULL;
1147 cached_buffer->scid = 0;
1148
1149 ipcb = (Exclaves_L4_IpcBuffer_t*)cached_buffer;
1150 } else {
1151 kr = exclaves_xnu_proxy_allocate_context(&scid, &ipcb);
1152 if (kr == KERN_NO_SPACE) {
1153 panic("Exclaves IPC buffer allocation failed");
1154 }
1155 }
1156
1157 *out_ipcb = ipcb;
1158 *out_scid = scid;
1159
1160 return kr;
1161}
1162
1163size_t
1164exclaves_ipc_buffer_count(void)
1165{
1166 return os_atomic_load(&exclaves_ipcb_cnt, relaxed);
1167}
1168
1169static kern_return_t
1170exclaves_relinquish_ipc_buffer(Exclaves_L4_IpcBuffer_t *ipcb,
1171 Exclaves_L4_Word_t scid)
1172{
1173 kern_return_t kr = KERN_SUCCESS;
1174 struct exclaves_ipc_buffer_cache_item *cached_buffer;
1175
1176 if (!exclaves_ipc_buffer_cache_enabled) {
1177 kr = exclaves_xnu_proxy_free_context(scid);
1178 } else {
1179 cached_buffer = (struct exclaves_ipc_buffer_cache_item*)ipcb;
1180 cached_buffer->scid = scid;
1181
1182 lck_spin_lock(&exclaves_ipc_buffer_cache_lock);
1183 cached_buffer->next = exclaves_ipc_buffer_cache;
1184 exclaves_ipc_buffer_cache = cached_buffer;
1185 lck_spin_unlock(&exclaves_ipc_buffer_cache_lock);
1186 }
1187
1188 return kr;
1189}
1190
1191static kern_return_t
1192exclaves_endpoint_call_internal(__unused ipc_port_t port,
1193 exclaves_id_t endpoint_id)
1194{
1195 kern_return_t kr = KERN_SUCCESS;
1196
1197 assert(port == IPC_PORT_NULL);
1198
1199 kr = exclaves_xnu_proxy_endpoint_call(endpoint_id);
1200
1201 return kr;
1202}
1203
1204/* -------------------------------------------------------------------------- */
1205#pragma mark secure kernel communication
1206
1207/* ringgate entry endpoints */
1208enum {
1209 RINGGATE_EP_ENTER,
1210 RINGGATE_EP_INFO
1211};
1212
1213/* ringgate entry status codes */
1214enum {
1215 RINGGATE_STATUS_SUCCESS,
1216 RINGGATE_STATUS_ERROR,
1217 RINGGATE_STATUS_PANIC, /* RINGGATE_EP_ENTER: Another core paniced */
1218};
1219
1220OS_NOINLINE
1221static kern_return_t
1222exclaves_enter(void)
1223{
1224 uint32_t endpoint = RINGGATE_EP_ENTER;
1225 uint64_t result = RINGGATE_STATUS_ERROR;
1226
1227 sptm_call_regs_t regs = { };
1228
1229 __assert_only thread_t thread = current_thread();
1230
1231 /*
1232 * Should never re-enter exclaves.
1233 */
1234 if ((thread->th_exclaves_state & TH_EXCLAVES_UPCALL) != 0 ||
1235 (thread->th_exclaves_state & TH_EXCLAVES_SCHEDULER_REQUEST) != 0) {
1236 panic("attempt to re-enter exclaves");
1237 }
1238
1239 /*
1240 * Must have one (and only one) of the flags set to enter exclaves.
1241 */
1242 __assert_only const thread_exclaves_state_flags_t mask = (
1243 TH_EXCLAVES_RPC |
1244 TH_EXCLAVES_XNUPROXY |
1245 TH_EXCLAVES_SCHEDULER_CALL);
1246 assert3u(thread->th_exclaves_state & mask, !=, 0);
1247 assert3u(thread->th_exclaves_intstate & TH_EXCLAVES_EXECUTION, ==, 0);
1248
1249#if MACH_ASSERT
1250 /*
1251 * Set the ast to check that the thread doesn't return to userspace
1252 * while in an RPC or XNUPROXY call.
1253 */
1254 act_set_debug_assert();
1255#endif /* MACH_ASSERT */
1256
1257 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_SWITCH)
1258 | DBG_FUNC_START);
1259 recount_enter_secure();
1260
1261 /* xnu_return_to_gl2 relies on this flag being present to correctly return
1262 * to SK from interrupts xnu handles on behalf of SK. */
1263 thread->th_exclaves_intstate |= TH_EXCLAVES_EXECUTION;
1264
1265 /*
1266 * Bracket with labels so stackshot can determine where exclaves are
1267 * entered from xnu.
1268 */
1269 __asm__ volatile (
1270 "EXCLAVES_ENTRY_START: nop\n\t"
1271 );
1272 result = sk_enter(endpoint, &regs);
1273 __asm__ volatile (
1274 "EXCLAVES_ENTRY_END: nop\n\t"
1275 );
1276
1277 thread->th_exclaves_intstate &= ~TH_EXCLAVES_EXECUTION;
1278
1279 recount_leave_secure();
1280 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_SWITCH)
1281 | DBG_FUNC_END);
1282
1283 switch (result) {
1284 case RINGGATE_STATUS_SUCCESS:
1285 return KERN_SUCCESS;
1286 case RINGGATE_STATUS_ERROR:
1287 return KERN_FAILURE;
1288 case RINGGATE_STATUS_PANIC:
1289 return KERN_ABORTED;
1290 default:
1291 assertf(false, "Unknown ringgate status %llu", result);
1292 __builtin_trap();
1293 }
1294}
1295
1296
1297/*
1298 * A bit in the lower byte of the value returned by RINGGATE_EP_INFO. If set,
1299 * it in indicates that we should immediately enter the ringgate once in order
1300 * to allow the scheduler to perform early boot initialisation.
1301 */
1302#define EARLY_RINGGATE_ENTER 2
1303
1304OS_NOINLINE
1305static kern_return_t
1306exclaves_bootinfo(uint64_t *out_boot_info, bool *early_enter)
1307{
1308 uint32_t endpoint = RINGGATE_EP_INFO;
1309 uint64_t result = RINGGATE_STATUS_ERROR;
1310
1311 sptm_call_regs_t regs = { };
1312
1313 recount_enter_secure();
1314 result = sk_enter(endpoint, &regs);
1315 recount_leave_secure();
1316 if (result == RINGGATE_STATUS_ERROR) {
1317 return KERN_FAILURE;
1318 }
1319
1320 *early_enter = (result & EARLY_RINGGATE_ENTER) != 0;
1321 *out_boot_info = result & ~EARLY_RINGGATE_ENTER;
1322
1323 return KERN_SUCCESS;
1324}
1325
1326/* -------------------------------------------------------------------------- */
1327
1328#pragma mark exclaves scheduler communication
1329
1330static XrtHosted_Buffer_t * PERCPU_DATA(exclaves_request);
1331static XrtHosted_Buffer_t * PERCPU_DATA(exclaves_response);
1332
1333static void
1334exclaves_init_multicore(void)
1335{
1336 assert(exclaves_multicore);
1337
1338 XrtHosted_Buffer_t **req, **res;
1339
1340 exclaves_wait_for_cpu_init();
1341
1342 DTEntry entry, child;
1343 OpaqueDTEntryIterator iter;
1344 int err = SecureDTLookupEntry(NULL, "/cpus", &entry);
1345 assert(err == kSuccess);
1346 err = SecureDTInitEntryIterator(entry, &iter);
1347 assert(err == kSuccess);
1348
1349 bool exclaves_uses_mpidr = (exclaves_callbacks->v1.global()->v2.smpStatus == XrtHosted_SmpStatus_MulticoreMpidr);
1350 if (exclaves_uses_mpidr) {
1351 exclaves_debug_printf(show_progress, "Using MPIDR for exclave scheduler core IDs\n");
1352 } else {
1353 // TODO(rdar://120679733) - clean up non-MPIDR identification logic.
1354 exclaves_debug_printf(show_progress, "Not using MPIDR for exclave scheduler core IDs\n");
1355 }
1356
1357 /*
1358 * Match the hardwareID to the physical ID and stash the pointers to the
1359 * request/response buffers in per-cpu data for quick access.
1360 */
1361 size_t core_count = exclaves_callbacks->v1.cores();
1362 for (size_t i = 0; i < core_count; i++) {
1363 const XrtHosted_Core_t *core = exclaves_callbacks->v1.core(i);
1364 uint32_t dt_phys_id = 0;
1365 if (exclaves_uses_mpidr) {
1366 dt_phys_id = (uint32_t)core->v2.hardwareId;
1367 } else {
1368 /* Find the physical ID of the entry at position hardwareId in the
1369 * DeviceTree "cpus" array */
1370 uint32_t dt_index = 0;
1371 bool dt_entry_found = false;
1372 err = SecureDTRestartEntryIteration(&iter);
1373 assert(err == kSuccess);
1374 while (kSuccess == SecureDTIterateEntries(&iter, &child)) {
1375 if (core->v2.hardwareId == dt_index) {
1376 void const *dt_prop;
1377 unsigned int dt_prop_sz;
1378 err = SecureDTGetProperty(child, "reg", &dt_prop, &dt_prop_sz);
1379 assert(err == kSuccess);
1380 assert(dt_prop_sz == sizeof(uint32_t));
1381 dt_phys_id = *((uint32_t const *)dt_prop);
1382 dt_entry_found = true;
1383 break;
1384 }
1385 dt_index++;
1386 }
1387 if (!dt_entry_found) {
1388 continue;
1389 }
1390 }
1391 percpu_foreach(cpu_data, cpu_data) {
1392 if (cpu_data->cpu_phys_id != dt_phys_id) {
1393 continue;
1394 }
1395 req = PERCPU_GET_RELATIVE(exclaves_request, cpu_data, cpu_data);
1396 *req = exclaves_callbacks->v1.Core.request(i);
1397
1398 res = PERCPU_GET_RELATIVE(exclaves_response, cpu_data, cpu_data);
1399 *res = exclaves_callbacks->v1.Core.response(i);
1400
1401 break;
1402 }
1403 }
1404}
1405
1406static void
1407exclaves_init_unicore(void)
1408{
1409 assert(!exclaves_multicore);
1410
1411 XrtHosted_Buffer_t *breq, *bres, **req, **res;
1412
1413 exclaves_wait_for_cpu_init();
1414
1415 breq = exclaves_callbacks->v1.Core.request(XrtHosted_Core_bootIndex);
1416 bres = exclaves_callbacks->v1.Core.response(XrtHosted_Core_bootIndex);
1417
1418 /* Always use the boot request/response buffers. */
1419 percpu_foreach(cpu_data, cpu_data) {
1420 req = PERCPU_GET_RELATIVE(exclaves_request, cpu_data, cpu_data);
1421 *req = breq;
1422
1423 res = PERCPU_GET_RELATIVE(exclaves_response, cpu_data, cpu_data);
1424 *res = bres;
1425 }
1426}
1427
1428static kern_return_t
1429exclaves_scheduler_init(uint64_t boot_info)
1430{
1431 kern_return_t kr = KERN_SUCCESS;
1432 XrtHosted_Error_t hosted_error;
1433
1434 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
1435
1436 if (!pmap_valid_address(boot_info)) {
1437 exclaves_debug_printf(show_errors,
1438 "exclaves: %s: 0x%012llx\n",
1439 "Invalid root physical address",
1440 boot_info);
1441 return KERN_FAILURE;
1442 }
1443
1444 if (exclaves_callbacks == NULL) {
1445 exclaves_debug_printf(show_errors,
1446 "exclaves: Callbacks not registered\n");
1447 return KERN_FAILURE;
1448 }
1449
1450 /* Initialise XrtHostedXnu kext */
1451 kr = exclaves_hosted_error(
1452 exclaves_callbacks->v1.init(
1453 XrtHosted_Version_current,
1454 phystokv(boot_info),
1455 &hosted_error),
1456 &hosted_error);
1457 if (kr != KERN_SUCCESS) {
1458 return kr;
1459 }
1460
1461 /* Record aperture addresses in buffer */
1462 size_t frames = exclaves_callbacks->v1.frames();
1463 XrtHosted_Mapped_t **pages = zalloc_permanent(
1464 frames * sizeof(XrtHosted_Mapped_t *),
1465 ZALIGN(XrtHosted_Mapped_t *));
1466 size_t index = 0;
1467 uint64_t phys = boot_info;
1468 while (index < frames) {
1469 if (!pmap_valid_address(phys)) {
1470 exclaves_debug_printf(show_errors,
1471 "exclaves: %s: 0x%012llx\n",
1472 "Invalid shared physical address",
1473 phys);
1474 return KERN_FAILURE;
1475 }
1476 pages[index] = (XrtHosted_Mapped_t *)phystokv(phys);
1477 kr = exclaves_hosted_error(
1478 exclaves_callbacks->v1.nextPhys(
1479 pages[index],
1480 &index,
1481 &phys,
1482 &hosted_error),
1483 &hosted_error);
1484 if (kr != KERN_SUCCESS) {
1485 return kr;
1486 }
1487 }
1488
1489 /* Initialise the mapped region */
1490 exclaves_callbacks->v1.setMapping(
1491 XrtHosted_Region_scattered(frames, pages));
1492
1493 /* Boot the scheduler. */
1494 kr = exclaves_scheduler_boot();
1495 if (kr != KERN_SUCCESS) {
1496 return kr;
1497 }
1498
1499 /* Initialise the XNU proxy */
1500 XrtHosted_Global_t *global = exclaves_callbacks->v1.global();
1501
1502 exclaves_multicore = (global->v2.smpStatus == XrtHosted_SmpStatus_Multicore || global->v2.smpStatus == XrtHosted_SmpStatus_MulticoreMpidr);
1503 exclaves_multicore ? exclaves_init_multicore() : exclaves_init_unicore();
1504
1505 uint64_t xnu_proxy_boot_info = global->v1.proxyInit;
1506 kr = exclaves_xnu_proxy_init(xnu_proxy_boot_info);
1507
1508 return kr;
1509}
1510
1511#if EXCLAVES_ENABLE_SHOW_SCHEDULER_REQUEST_RESPONSE
1512#define exclaves_scheduler_debug_save_buffer(_buf_in, _buf_out) \
1513 *(_buf_out) = *(_buf_in)
1514#define exclaves_scheduler_debug_show_request_response(_request_buf, \
1515 _response_buf) ({ \
1516 if (exclaves_debug_enabled(show_scheduler_request_response)) { \
1517 printf("exclaves: Scheduler request = %p\n", _request_buf); \
1518 printf("exclaves: Scheduler request.tag = 0x%04llx\n", \
1519 (_request_buf)->tag); \
1520 for (size_t arg = 0; arg < XrtHosted_Buffer_args; arg += 1) { \
1521 printf("exclaves: Scheduler request.arguments[%02zu] = " \
1522 "0x%04llx\n", arg, \
1523 (_request_buf)->arguments[arg]); \
1524 } \
1525 printf("exclaves: Scheduler response = %p\n", _response_buf); \
1526 printf("exclaves: Scheduler response.tag = 0x%04llx\n", \
1527 (_response_buf)->tag); \
1528 for (size_t arg = 0; arg < XrtHosted_Buffer_args; arg += 1) { \
1529 printf("exclaves: Scheduler response.arguments[%02zu] = " \
1530 "0x%04llx\n", arg, \
1531 (_response_buf)->arguments[arg]); \
1532 } \
1533 }})
1534#else // EXCLAVES_SHOW_SCHEDULER_REQUEST_RESPONSE
1535#define exclaves_scheduler_debug_save_buffer(_buf_in, _buf_out) (void)_buf_out
1536#define exclaves_scheduler_debug_show_request_response(_request_buf, \
1537 _response_buf) ({ })
1538#endif // EXCLAVES_SHOW_SCHEDULER_REQUEST_RESPONSE
1539
1540__attribute__((always_inline))
1541static kern_return_t
1542exclaves_scheduler_send(const XrtHosted_Request_t *request,
1543 XrtHosted_Response_t *response, XrtHosted_Buffer_t *save_out_ptr, XrtHosted_Buffer_t *save_in_ptr)
1544{
1545 /* Must be called with preemption and interrupts disabled */
1546 kern_return_t kr;
1547
1548 XrtHosted_Buffer_t *request_buf = *PERCPU_GET(exclaves_request);
1549 assert3p(request_buf, !=, NULL);
1550
1551 exclaves_callbacks->v1.Request.encode(request_buf, request);
1552 exclaves_scheduler_debug_save_buffer(request_buf, save_out_ptr);
1553
1554 kr = exclaves_enter();
1555
1556 /* The response may have come back on a different core. */
1557 XrtHosted_Buffer_t *response_buf = *PERCPU_GET(exclaves_response);
1558 assert3p(response_buf, !=, NULL);
1559
1560 exclaves_scheduler_debug_save_buffer(response_buf, save_in_ptr);
1561 exclaves_callbacks->v1.Response.decode(response_buf, response);
1562
1563 return kr;
1564}
1565
1566__attribute__((always_inline))
1567static kern_return_t
1568exclaves_scheduler_request(const XrtHosted_Request_t *request,
1569 XrtHosted_Response_t *response)
1570{
1571#if EXCLAVES_ENABLE_SHOW_SCHEDULER_REQUEST_RESPONSE
1572 XrtHosted_Buffer_t save_in[3], save_out[3] = {{ .tag = XrtHosted_Message_Invalid }, { .tag = XrtHosted_Message_Invalid }, { .tag = XrtHosted_Message_Invalid }};
1573 XrtHosted_Buffer_t *save_out_ptr = save_out, *save_in_ptr = save_in;
1574#else
1575 XrtHosted_Buffer_t *save_out_ptr = NULL, *save_in_ptr = NULL;
1576#endif // EXCLAVES_SHOW_SCHEDULER_REQUEST_RESPONSE
1577
1578 assert3u(request->tag, >, XrtHosted_Request_Invalid);
1579 assert3u(request->tag, <, XrtHosted_Request_Limit);
1580
1581 kern_return_t kr = KERN_SUCCESS;
1582 bool istate;
1583
1584 if (!exclaves_multicore || !exclaves_smp_enabled) {
1585 lck_mtx_lock(&exclaves_scheduler_lock);
1586 }
1587
1588 /*
1589 * Disable preemption and interrupts as the xrt hosted scheduler data
1590 * structures are per-core.
1591 * Preemption disabled and interrupt disabled timeouts are disabled for
1592 * now until we can co-ordinate the measurements with the exclaves side of
1593 * things.
1594 */
1595 istate = ml_set_interrupts_enabled_with_debug(false, false);
1596
1597 /*
1598 * This needs to be done with interrupts disabled, otherwise stackshot could
1599 * mark the thread blocked just after this function exits and a thread marked
1600 * as AST blocked would go into exclaves.
1601 */
1602
1603 while ((os_atomic_load(&current_thread()->th_exclaves_inspection_state, relaxed) & ~TH_EXCLAVES_INSPECTION_NOINSPECT) != 0) {
1604 /* Enable interrupts */
1605 (void) ml_set_interrupts_enabled_with_debug(true, false);
1606
1607 if (!exclaves_multicore || !exclaves_smp_enabled) {
1608 lck_mtx_unlock(&exclaves_scheduler_lock);
1609 }
1610
1611 /* Wait until the thread is collected on exclaves side */
1612 exclaves_inspection_check_ast();
1613
1614 if (!exclaves_multicore || !exclaves_smp_enabled) {
1615 lck_mtx_lock(&exclaves_scheduler_lock);
1616 }
1617
1618 /* Disable interrupts and preemption before next AST check */
1619 ml_set_interrupts_enabled_with_debug(false, false);
1620 }
1621 /* Interrupts are disabled and exclaves_stackshot_ast is clean */
1622
1623 disable_preemption_without_measurements();
1624
1625 /* Update clock offsets before any other scheduler operation */
1626 exclaves_clock_t *clocks[] = { &exclaves_absolute_clock,
1627 &exclaves_continuous_clock };
1628 for (unsigned i = 0; i < ARRAY_COUNT(clocks); ++i) {
1629 if (exclaves_clock_needs_update(clocks[i])) {
1630 kr = exclaves_clock_update(clocks[i], &save_out_ptr[i], &save_in_ptr[i]);
1631 if (kr != KERN_SUCCESS) {
1632 break;
1633 }
1634 }
1635 }
1636
1637 if (kr == KERN_SUCCESS) {
1638 kr = exclaves_scheduler_send(request, response, &save_out_ptr[2], &save_in_ptr[2]);
1639 }
1640
1641 enable_preemption();
1642 (void) ml_set_interrupts_enabled_with_debug(istate, false);
1643
1644#if EXCLAVES_ENABLE_SHOW_SCHEDULER_REQUEST_RESPONSE
1645 for (unsigned i = 0; i < ARRAY_COUNT(save_out); ++i) {
1646 if (save_out_ptr[i].tag != XrtHosted_Message_Invalid) {
1647 exclaves_scheduler_debug_show_request_response(&save_out_ptr[i], &save_in_ptr[i]);
1648 }
1649 }
1650#endif // EXCLAVES_ENABLE_SHOW_SCHEDULER_REQUEST_RESPONSE
1651
1652 if (!exclaves_multicore || !exclaves_smp_enabled) {
1653 lck_mtx_unlock(&exclaves_scheduler_lock);
1654 }
1655
1656 if (kr == KERN_ABORTED) {
1657 /* RINGGATE_EP_ENTER returned RINGGATE_STATUS_PANIC indicating that
1658 * another core has paniced in exclaves and is on the way to call xnu
1659 * panic() via SPTM, so wait here for that to happen. */
1660 exclaves_wait_for_panic();
1661 }
1662
1663 return kr;
1664}
1665
1666OS_NORETURN OS_NOINLINE
1667static void
1668exclaves_wait_for_panic(void)
1669{
1670 assert_wait_timeout((event_t)exclaves_wait_for_panic, THREAD_UNINT, 1,
1671 NSEC_PER_SEC);
1672 wait_result_t wr = thread_block(THREAD_CONTINUE_NULL);
1673 panic("Unexpected wait for panic result: %d", wr);
1674}
1675
1676static kern_return_t
1677handle_response_yield(bool early, __assert_only Exclaves_L4_Word_t scid,
1678 const XrtHosted_Yield_t *yield)
1679{
1680 Exclaves_L4_Word_t responding_scid = yield->thread;
1681 Exclaves_L4_Word_t yielded_to_scid = yield->yieldTo;
1682 __assert_only ctid_t ctid = thread_get_ctid(current_thread());
1683
1684 exclaves_debug_printf(show_progress,
1685 "exclaves: Scheduler: %s scid 0x%lx yielded to scid 0x%lx\n",
1686 early ? "(early yield)" : "", responding_scid, yielded_to_scid);
1687 /* TODO: 1. remember yielding scid if it isn't the xnu proxy's
1688 * th_exclaves_scheduling_context_id so we know to resume it later
1689 * 2. translate yield_to to thread_switch()-style handoff.
1690 */
1691 if (!early) {
1692 assert3u(responding_scid, ==, scid);
1693 assert3u(yield->threadHostId, ==, ctid);
1694 }
1695
1696 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1697 MACH_EXCLAVES_SCHEDULER_YIELD), yielded_to_scid, early);
1698
1699 return KERN_SUCCESS;
1700}
1701
1702static kern_return_t
1703handle_response_spawned(__assert_only Exclaves_L4_Word_t scid,
1704 const XrtHosted_Spawned_t *spawned, Exclaves_L4_Word_t *spawned_scid)
1705{
1706 Exclaves_L4_Word_t responding_scid = spawned->thread;
1707 __assert_only ctid_t ctid = thread_get_ctid(current_thread());
1708
1709 if (spawned_scid == NULL) {
1710 exclaves_debug_printf(show_errors,
1711 "exclaves: Scheduler: Unexpected thread spawn: "
1712 "scid 0x%lx spawned scid 0x%llx\n",
1713 responding_scid, spawned->spawned);
1714 return KERN_FAILURE;
1715 }
1716
1717 *spawned_scid = spawned->spawned;
1718 exclaves_debug_printf(show_progress,
1719 "exclaves: Scheduler: scid 0x%lx spawned scid 0x%lx\n",
1720 responding_scid, *spawned_scid);
1721 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1722 MACH_EXCLAVES_SCHEDULER_SPAWNED), *spawned_scid);
1723
1724 /* TODO: remember yielding scid if it isn't the xnu proxy's
1725 * th_exclaves_scheduling_context_id so we know to resume it later
1726 */
1727 if (0) {
1728 // FIXME: reenable when exclaves scheduler is fixed
1729 assert3u(responding_scid, ==, scid);
1730 assert3u(spawned->threadHostId, ==, ctid);
1731 }
1732
1733 return KERN_SUCCESS;
1734}
1735
1736static kern_return_t
1737handle_response_terminated(const XrtHosted_Terminated_t *terminated)
1738{
1739 Exclaves_L4_Word_t responding_scid = terminated->thread;
1740 __assert_only ctid_t ctid = thread_get_ctid(current_thread());
1741
1742 exclaves_debug_printf(show_errors,
1743 "exclaves: Scheduler: Unexpected thread terminate: "
1744 "scid 0x%lx terminated scid 0x%llx\n", responding_scid,
1745 terminated->terminated);
1746 assert3u(terminated->threadHostId, ==, ctid);
1747
1748 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1749 MACH_EXCLAVES_SCHEDULER_TERMINATED),
1750 terminated->terminated);
1751
1752 return KERN_TERMINATED;
1753}
1754
1755static kern_return_t
1756handle_response_wait(const XrtHosted_Wait_t *wait)
1757{
1758 Exclaves_L4_Word_t responding_scid = wait->waiter;
1759 __assert_only ctid_t ctid = thread_get_ctid(current_thread());
1760
1761 exclaves_debug_printf(show_progress,
1762 "exclaves: Scheduler: Wait: "
1763 "scid 0x%lx wait on owner scid 0x%llx, queue id 0x%llx, "
1764 "epoch 0x%llx\n", responding_scid, wait->owner,
1765 wait->queueId, wait->epoch);
1766 assert3u(wait->waiterHostId, ==, ctid);
1767
1768 /*
1769 * Note, "owner" may not be safe to access directly, for example
1770 * the thread may have exited and been freed. esync_wait will
1771 * only access it under a lock if the epoch is fresh thus
1772 * ensuring safety.
1773 */
1774 const ctid_t owner = (ctid_t)wait->ownerHostId;
1775 const XrtHosted_Word_t id = wait->queueId;
1776 const uint64_t epoch = wait->epoch;
1777
1778 wait_interrupt_t interruptible;
1779 esync_policy_t policy;
1780
1781 switch (wait->interruptible) {
1782 case XrtHosted_Interruptibility_None:
1783 interruptible = THREAD_UNINT;
1784 policy = ESYNC_POLICY_KERNEL;
1785 break;
1786
1787 case XrtHosted_Interruptibility_Voluntary:
1788 interruptible = THREAD_INTERRUPTIBLE;
1789 policy = ESYNC_POLICY_KERNEL;
1790 break;
1791
1792 case XrtHosted_Interruptibility_DynamicQueue:
1793 interruptible = THREAD_INTERRUPTIBLE;
1794 policy = ESYNC_POLICY_USER;
1795 break;
1796
1797 default:
1798 panic("Unknown exclaves interruptibility: %llu",
1799 wait->interruptible);
1800 }
1801
1802 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1803 MACH_EXCLAVES_SCHEDULER_WAIT) | DBG_FUNC_START, id, epoch, owner,
1804 wait->interruptible);
1805 const wait_result_t wr = esync_wait(&esync_queue_ht, id, epoch,
1806 exclaves_get_queue_counter(id), owner, policy, interruptible);
1807 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1808 MACH_EXCLAVES_SCHEDULER_WAIT) | DBG_FUNC_END, wr);
1809
1810 switch (wr) {
1811 case THREAD_INTERRUPTED:
1812 return KERN_ABORTED;
1813
1814 case THREAD_NOT_WAITING:
1815 case THREAD_AWAKENED:
1816 return KERN_SUCCESS;
1817
1818 default:
1819 panic("Unexpected wait result from esync_wait: %d", wr);
1820 }
1821}
1822
1823static kern_return_t
1824handle_response_wake(const XrtHosted_Wake_t *wake)
1825{
1826 Exclaves_L4_Word_t responding_scid = wake->waker;
1827 __assert_only ctid_t ctid = thread_get_ctid(current_thread());
1828
1829 exclaves_debug_printf(show_progress,
1830 "exclaves: Scheduler: Wake: "
1831 "scid 0x%lx wake of queue id 0x%llx, "
1832 "epoch 0x%llx, all 0x%llx\n", responding_scid,
1833 wake->queueId, wake->epoch, wake->all);
1834 assert3u(wake->wakerHostId, ==, ctid);
1835
1836 const XrtHosted_Word_t id = wake->queueId;
1837 const uint64_t epoch = wake->epoch;
1838 const esync_wake_mode_t mode = wake->all != 0 ?
1839 ESYNC_WAKE_ALL : ESYNC_WAKE_ONE;
1840
1841 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1842 MACH_EXCLAVES_SCHEDULER_WAKE) | DBG_FUNC_START, id, epoch, 0, mode);
1843
1844 kern_return_t kr = esync_wake(&esync_queue_ht, id, epoch,
1845 exclaves_get_queue_counter(id), mode, 0);
1846
1847 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1848 MACH_EXCLAVES_SCHEDULER_WAKE) | DBG_FUNC_END,
1849 kr == KERN_SUCCESS ? THREAD_AWAKENED : THREAD_NOT_WAITING);
1850
1851 return KERN_SUCCESS;
1852}
1853
1854static kern_return_t
1855handle_response_wake_with_owner(const XrtHosted_WakeWithOwner_t *wake)
1856{
1857 Exclaves_L4_Word_t responding_scid = wake->waker;
1858 __assert_only ctid_t ctid = thread_get_ctid(current_thread());
1859
1860 exclaves_debug_printf(show_progress,
1861 "exclaves: Scheduler: WakeWithOwner: "
1862 "scid 0x%lx wake of queue id 0x%llx, "
1863 "epoch 0x%llx, owner 0x%llx\n", responding_scid,
1864 wake->queueId, wake->epoch,
1865 wake->owner);
1866
1867 assert3u(wake->wakerHostId, ==, ctid);
1868
1869 const ctid_t owner = (ctid_t)wake->ownerHostId;
1870 const XrtHosted_Word_t id = wake->queueId;
1871 const uint64_t epoch = wake->epoch;
1872
1873 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1874 MACH_EXCLAVES_SCHEDULER_WAKE) | DBG_FUNC_START, id, epoch, owner,
1875 ESYNC_WAKE_ONE);
1876
1877 kern_return_t kr = esync_wake(&esync_queue_ht, id, epoch,
1878 exclaves_get_queue_counter(id), ESYNC_WAKE_ONE_WITH_OWNER, owner);
1879
1880 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES,
1881 MACH_EXCLAVES_SCHEDULER_WAKE) | DBG_FUNC_END,
1882 kr == KERN_SUCCESS ? THREAD_AWAKENED : THREAD_NOT_WAITING);
1883
1884 return KERN_SUCCESS;
1885}
1886
1887static kern_return_t
1888handle_response_panic_wait(const XrtHosted_PanicWait_t *panic_wait)
1889{
1890 Exclaves_L4_Word_t panic_thread_scid = panic_wait->handler;
1891 __assert_only thread_t thread = current_thread();
1892
1893 exclaves_debug_printf(show_progress,
1894 "exclaves: Scheduler: PanicWait: "
1895 "Panic thread SCID %lx\n",
1896 panic_thread_scid);
1897
1898 assert3u(panic_thread_scid, ==, thread->th_exclaves_scheduling_context_id);
1899
1900 exclaves_panic_thread_wait();
1901
1902 /* NOT REACHABLE */
1903 return KERN_SUCCESS;
1904}
1905
1906static kern_return_t
1907handle_response_suspended(const XrtHosted_Suspended_t *suspended)
1908{
1909 Exclaves_L4_Word_t responding_scid = suspended->suspended;
1910 __assert_only ctid_t ctid = thread_get_ctid(current_thread());
1911
1912 exclaves_debug_printf(show_progress,
1913 "exclaves: Scheduler: Suspended: "
1914 "scid 0x%lx epoch 0x%llx\n", responding_scid, suspended->epoch);
1915 assert3u(suspended->suspendedHostId, ==, ctid);
1916
1917 const uint64_t id = suspended->suspended;
1918 const uint64_t epoch = suspended->epoch;
1919
1920 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1921 MACH_EXCLAVES_SCHEDULER_SUSPENDED) | DBG_FUNC_START, id, epoch);
1922
1923 const wait_result_t wr = esync_wait(&esync_thread_ht, id, epoch,
1924 exclaves_get_thread_counter(id), 0, ESYNC_POLICY_KERNEL, THREAD_UNINT);
1925
1926 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1927 MACH_EXCLAVES_SCHEDULER_SUSPENDED) | DBG_FUNC_END, wr);
1928
1929 switch (wr) {
1930 case THREAD_INTERRUPTED:
1931 return KERN_ABORTED;
1932
1933 case THREAD_NOT_WAITING:
1934 case THREAD_AWAKENED:
1935 return KERN_SUCCESS;
1936
1937 default:
1938 panic("Unexpected wait result from esync_wait: %d", wr);
1939 }
1940}
1941
1942static kern_return_t
1943handle_response_resumed(const XrtHosted_Resumed_t *resumed)
1944{
1945 Exclaves_L4_Word_t responding_scid = resumed->thread;
1946 __assert_only ctid_t ctid = thread_get_ctid(current_thread());
1947
1948 exclaves_debug_printf(show_progress,
1949 "exclaves: Scheduler: Resumed: scid 0x%lx resume of scid 0x%llx "
1950 "(ctid: 0x%llx), epoch 0x%llx\n", responding_scid, resumed->resumed,
1951 resumed->resumedHostId, resumed->epoch);
1952 assert3u(resumed->threadHostId, ==, ctid);
1953
1954 const ctid_t target = (ctid_t)resumed->resumedHostId;
1955 const XrtHosted_Word_t id = resumed->resumed;
1956 const uint64_t epoch = resumed->epoch;
1957
1958 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1959 MACH_EXCLAVES_SCHEDULER_RESUMED) | DBG_FUNC_START, id, epoch,
1960 target);
1961
1962 kern_return_t kr = esync_wake(&esync_thread_ht, id, epoch,
1963 exclaves_get_thread_counter(id), ESYNC_WAKE_THREAD, target);
1964
1965 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1966 MACH_EXCLAVES_SCHEDULER_RESUMED) | DBG_FUNC_END,
1967 kr == KERN_SUCCESS ? THREAD_AWAKENED : THREAD_NOT_WAITING);
1968
1969 return KERN_SUCCESS;
1970}
1971
1972static kern_return_t
1973handle_response_interrupted(const XrtHosted_Interrupted_t *interrupted)
1974{
1975 Exclaves_L4_Word_t responding_scid = interrupted->thread;
1976 __assert_only ctid_t ctid = thread_get_ctid(current_thread());
1977
1978 exclaves_debug_printf(show_progress,
1979 "exclaves: Scheduler: Interrupted: "
1980 "scid 0x%lx interrupt on queue id 0x%llx, "
1981 "epoch 0x%llx, target 0x%llx\n", responding_scid,
1982 interrupted->queueId, interrupted->epoch,
1983 interrupted->interruptedHostId);
1984 assert3u(interrupted->threadHostId, ==, ctid);
1985
1986 const ctid_t target = (ctid_t)interrupted->interruptedHostId;
1987 const XrtHosted_Word_t id = interrupted->queueId;
1988 const uint64_t epoch = interrupted->epoch;
1989
1990 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
1991 MACH_EXCLAVES_SCHEDULER_INTERRUPTED) | DBG_FUNC_START, id, epoch,
1992 target);
1993
1994 kern_return_t kr = esync_wake(&esync_queue_ht, id, epoch,
1995 exclaves_get_queue_counter(id), ESYNC_WAKE_THREAD, target);
1996
1997 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES,
1998 MACH_EXCLAVES_SCHEDULER_INTERRUPTED) | DBG_FUNC_END,
1999 kr == KERN_SUCCESS ? THREAD_AWAKENED : THREAD_NOT_WAITING);
2000
2001 return KERN_SUCCESS;
2002}
2003
2004static kern_return_t
2005handle_response_nothing_scheduled(
2006 __unused const XrtHosted_NothingScheduled_t *nothing_scheduled)
2007{
2008 exclaves_debug_printf(show_progress,
2009 "exclaves: Scheduler: nothing scheduled\n");
2010
2011 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
2012 MACH_EXCLAVES_SCHEDULER_NOTHING_SCHEDULED));
2013
2014 return KERN_SUCCESS;
2015}
2016
2017static kern_return_t
2018handle_response_all_exclaves_booted(
2019 __unused const XrtHosted_AllExclavesBooted_t *all_exclaves_booted)
2020{
2021 exclaves_debug_printf(show_progress,
2022 "exclaves: scheduler: all exclaves booted\n");
2023
2024 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
2025 MACH_EXCLAVES_SCHEDULER_ALL_EXCLAVES_BOOTED));
2026
2027 return KERN_SUCCESS;
2028}
2029
2030/*
2031 * The Early Alloc response asks for npages to be allocated. The list of
2032 * allocated pages is written into the first allocated page in the form of 32bit
2033 * page numbers. The physical address of the first page is passed back to the
2034 * exclaves scheduler as part of the next request.
2035 */
2036static kern_return_t
2037handle_response_pmm_early_alloc(const XrtHosted_PmmEarlyAlloc_t *pmm_early_alloc,
2038 uint64_t *pagelist_pa)
2039{
2040 const uint32_t npages = (uint32_t)pmm_early_alloc->a;
2041 const uint64_t flags = pmm_early_alloc->b;
2042
2043 exclaves_debug_printf(show_progress,
2044 "exclaves: scheduler: pmm early alloc, npages: %u, flags: %llu\n",
2045 npages, flags);
2046
2047 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES_SCHEDULER,
2048 MACH_EXCLAVES_SCHEDULER_EARLY_ALLOC), npages, flags);
2049
2050 if (npages == 0) {
2051 return KERN_SUCCESS;
2052 }
2053
2054 if (npages > EXCLAVES_MEMORY_MAX_REQUEST) {
2055 exclaves_debug_printf(show_errors,
2056 "exclaves: request to allocate too many pages: %u\n",
2057 npages);
2058 return KERN_NO_SPACE;
2059 }
2060
2061 /*
2062 * As npages must be relatively small (<= EXCLAVES_MEMORY_MAX_REQUEST),
2063 * stack allocation is sufficient and fast. If
2064 * EXCLAVES_MEMORY_MAX_REQUEST gets large, this should probably be moved
2065 * to the heap.
2066 */
2067 uint32_t page[npages];
2068 exclaves_memory_alloc(npages, page, XNUUPCALLS_PAGEKIND_ROOTDOMAIN);
2069
2070 /* Now copy the list of pages into the first page. */
2071 uint64_t first_page_pa = ptoa(page[0]);
2072#if 0
2073 // move to before sptm retype
2074 uint32_t *first_page = (uint32_t *)phystokv(first_page_pa);
2075 for (int i = 0; i < npages; i++) {
2076 first_page[i] = page[i];
2077 }
2078#endif
2079
2080 *pagelist_pa = first_page_pa;
2081 return KERN_SUCCESS;
2082}
2083
2084static inline bool
2085exclaves_clock_needs_update(const exclaves_clock_t *clock)
2086{
2087 exclaves_clock_t local = {
2088 .u128 = os_atomic_load(&clock->a_u128, relaxed),
2089 };
2090
2091 return local.u64.sent_offset != local.u64.latest_offset;
2092}
2093
2094OS_NOINLINE
2095static kern_return_t
2096exclaves_clock_update(exclaves_clock_t *clock, XrtHosted_Buffer_t *save_out_ptr, XrtHosted_Buffer_t *save_in_ptr)
2097{
2098 XrtHosted_Response_t response = { .tag = XrtHosted_Response_NothingScheduled, };
2099 kern_return_t kr = KERN_SUCCESS;
2100 exclaves_clock_t local;
2101
2102 local.u128 = os_atomic_load(&clock->a_u128, relaxed);
2103 while (local.u64.sent_offset != local.u64.latest_offset) {
2104 XrtHosted_Request_t request = XrtHosted_Request_UpdateTimerOffsetMsg(
2105 .timer =
2106 (clock == &exclaves_absolute_clock ?
2107 XrtHosted_Timer_Absolute : XrtHosted_Timer_Continuous),
2108 .offset = local.u64.latest_offset,
2109 );
2110
2111 kr = exclaves_scheduler_send(&request, &response, save_out_ptr, save_in_ptr);
2112 if (kr) {
2113 return kr;
2114 }
2115
2116 /* Swap the sent offset with the local latest offset. If it fails,
2117 * the sent offset will be reloaded. */
2118 os_atomic_cmpxchgv(&clock->a_u64.sent_offset, local.u64.sent_offset,
2119 local.u64.latest_offset, &local.u64.sent_offset, relaxed);
2120
2121 /* Fetch the latest offset again, in case we are stale. */
2122 local.u64.latest_offset = os_atomic_load(&clock->a_u64.latest_offset,
2123 relaxed);
2124 }
2125
2126 if (response.tag != XrtHosted_Response_NothingScheduled) {
2127 kr = KERN_FAILURE;
2128 }
2129
2130 return kr;
2131}
2132
2133static kern_return_t
2134exclaves_scheduler_boot(void)
2135{
2136 kern_return_t kr = KERN_FAILURE;
2137 thread_t thread = current_thread();
2138
2139 exclaves_debug_printf(show_progress,
2140 "exclaves: Scheduler: Request to boot exclave\n");
2141
2142 /* This must happen on the boot CPU - bind the thread. */
2143 bind_to_boot_core();
2144
2145 assert3u(thread->th_exclaves_state & TH_EXCLAVES_STATE_ANY, ==, 0);
2146 thread->th_exclaves_state |= TH_EXCLAVES_SCHEDULER_CALL;
2147
2148 /*
2149 * Set the request/response buffers. These may be overriden later when
2150 * doing multicore setup.
2151 */
2152 *PERCPU_GET(exclaves_request) =
2153 exclaves_callbacks->v1.Core.request(XrtHosted_Core_bootIndex);
2154 *PERCPU_GET(exclaves_response) =
2155 exclaves_callbacks->v1.Core.response(XrtHosted_Core_bootIndex);
2156
2157 XrtHosted_Response_t response = {.tag = XrtHosted_Response_Invalid};
2158 uint64_t pagelist_pa = 0;
2159
2160 while (response.tag != XrtHosted_Response_AllExclavesBooted) {
2161 const XrtHosted_Request_t request = pagelist_pa != 0 ?
2162 XrtHosted_Request_PmmEarlyAllocResponseMsg(.a = pagelist_pa):
2163 XrtHosted_Request_BootExclavesMsg();
2164 pagelist_pa = 0;
2165
2166 kr = exclaves_scheduler_request(&request, &response);
2167 if (kr != KERN_SUCCESS) {
2168 exclaves_debug_printf(show_errors,
2169 "exclaves: Enter failed\n");
2170 break;
2171 }
2172
2173 thread->th_exclaves_state |= TH_EXCLAVES_SCHEDULER_REQUEST;
2174
2175 switch (response.tag) {
2176 case XrtHosted_Response_Yield:
2177 kr = handle_response_yield(true, 0, &response.Yield);
2178 break;
2179
2180 case XrtHosted_Response_NothingScheduled:
2181 kr = handle_response_nothing_scheduled(&response.NothingScheduled);
2182 break;
2183
2184 case XrtHosted_Response_AllExclavesBooted:
2185 kr = handle_response_all_exclaves_booted(&response.AllExclavesBooted);
2186 break;
2187
2188 case XrtHosted_Response_PmmEarlyAlloc:
2189 kr = handle_response_pmm_early_alloc(&response.PmmEarlyAlloc, &pagelist_pa);
2190 break;
2191
2192 case XrtHosted_Response_PanicBufferAddress:
2193 handle_response_panic_buffer_address(response.PanicBufferAddress.physical);
2194 break;
2195
2196 default:
2197 exclaves_debug_printf(show_errors,
2198 "exclaves: Scheduler: Unexpected response: tag 0x%x\n",
2199 response.tag);
2200 kr = KERN_FAILURE;
2201 break;
2202 }
2203
2204 thread->th_exclaves_state &= ~TH_EXCLAVES_SCHEDULER_REQUEST;
2205
2206 /* Bail out if an error is hit. */
2207 if (kr != KERN_SUCCESS) {
2208 break;
2209 }
2210 }
2211
2212 thread->th_exclaves_state &= ~TH_EXCLAVES_SCHEDULER_CALL;
2213
2214 unbind_from_boot_core();
2215
2216 return kr;
2217}
2218
2219kern_return_t
2220exclaves_scheduler_resume_scheduling_context(Exclaves_L4_Word_t scid,
2221 Exclaves_L4_Word_t *spawned_scid, bool interrupted)
2222{
2223 kern_return_t kr = KERN_SUCCESS;
2224 thread_t thread = current_thread();
2225 const ctid_t ctid = thread_get_ctid(thread);
2226
2227 exclaves_debug_printf(show_progress,
2228 "exclaves: Scheduler: Request to resume scid 0x%lx\n", scid);
2229
2230 XrtHosted_Response_t response = {};
2231 const XrtHosted_Request_t request = interrupted ?
2232 XrtHosted_Request_InterruptWithHostIdMsg(
2233 .thread = scid,
2234 .hostId = ctid,
2235 ) :
2236 XrtHosted_Request_ResumeWithHostIdMsg(
2237 .thread = scid,
2238 .hostId = ctid,
2239 );
2240 kr = exclaves_scheduler_request(&request, &response);
2241 if (kr) {
2242 exclaves_debug_printf(show_errors, "exclaves: Enter failed\n");
2243 return kr;
2244 }
2245
2246 thread->th_exclaves_state |= TH_EXCLAVES_SCHEDULER_REQUEST;
2247
2248 switch (response.tag) {
2249 case XrtHosted_Response_Wait:
2250 kr = handle_response_wait(&response.Wait);
2251 goto out;
2252
2253 case XrtHosted_Response_Wake:
2254 kr = handle_response_wake(&response.Wake);
2255 goto out;
2256
2257 case XrtHosted_Response_Yield:
2258 kr = handle_response_yield(false, scid, &response.Yield);
2259 goto out;
2260
2261 case XrtHosted_Response_Spawned:
2262 kr = handle_response_spawned(scid, &response.Spawned, spawned_scid);
2263 goto out;
2264
2265 case XrtHosted_Response_Terminated:
2266 kr = handle_response_terminated(&response.Terminated);
2267 goto out;
2268
2269 case XrtHosted_Response_WakeWithOwner:
2270 kr = handle_response_wake_with_owner(&response.WakeWithOwner);
2271 goto out;
2272
2273 case XrtHosted_Response_PanicWait:
2274 kr = handle_response_panic_wait(&response.PanicWait);
2275 goto out;
2276
2277 case XrtHosted_Response_Suspended:
2278 kr = handle_response_suspended(&response.Suspended);
2279 goto out;
2280
2281 case XrtHosted_Response_Resumed:
2282 kr = handle_response_resumed(&response.Resumed);
2283 goto out;
2284
2285 case XrtHosted_Response_Interrupted:
2286 kr = handle_response_interrupted(&response.Interrupted);
2287 goto out;
2288
2289 case XrtHosted_Response_Invalid:
2290 case XrtHosted_Response_Failure:
2291 case XrtHosted_Response_Pong:
2292 case XrtHosted_Response_SleepUntil:
2293 case XrtHosted_Response_Awaken:
2294 default:
2295 exclaves_debug_printf(show_errors,
2296 "exclaves: Scheduler: Unexpected response: tag 0x%x\n",
2297 response.tag);
2298 kr = KERN_FAILURE;
2299 goto out;
2300 }
2301
2302out:
2303 thread->th_exclaves_state &= ~TH_EXCLAVES_SCHEDULER_REQUEST;
2304 return kr;
2305}
2306
2307/* -------------------------------------------------------------------------- */
2308
2309#pragma mark exclaves xnu proxy communication
2310static const char *
2311cmd_to_str(xnuproxy_cmd_t cmd)
2312{
2313 switch (cmd) {
2314 case XNUPROXY_CMD_UNDEFINED: return "undefined";
2315 case XNUPROXY_CMD_SETUP: return "setup";
2316 case XNUPROXY_CMD_CONTEXT_ALLOCATE: return "allocate context";
2317 case XNUPROXY_CMD_CONTEXT_FREE: return "free context";
2318 case XNUPROXY_CMD_NAMED_BUFFER_CREATE: return "named buffer create";
2319 case XNUPROXY_CMD_NAMED_BUFFER_DELETE: return "named buffer delete";
2320 case XNUPROXY_CMD_RESOURCE_INFO: return "resource info";
2321 case XNUPROXY_CMD_AUDIO_BUFFER_CREATE: return "audio buffer create";
2322 case XNUPROXY_CMD_AUDIO_BUFFER_COPYOUT: return "audio buffer copyout";
2323 case XNUPROXY_CMD_AUDIO_BUFFER_DELETE: return "audio buffer delete";
2324 case XNUPROXY_CMD_SENSOR_START: return "sensor start";
2325 case XNUPROXY_CMD_SENSOR_STOP: return "sensor stop";
2326 case XNUPROXY_CMD_SENSOR_STATUS: return "sensor status";
2327 case XNUPROXY_CMD_DISPLAY_HEALTHCHECK_RATE: return "display healthcheck rate";
2328 case XNUPROXY_CMD_NAMED_BUFFER_MAP: return "named buffer map";
2329 case XNUPROXY_CMD_NAMED_BUFFER_LAYOUT: return "named buffer layout";
2330 case XNUPROXY_CMD_AUDIO_BUFFER_MAP: return "audio buffer map";
2331 case XNUPROXY_CMD_AUDIO_BUFFER_LAYOUT: return "audio buffer layout";
2332 case XNUPROXY_CMD_REPORT_MEMORY_USAGE: return "memory usage";
2333 case XNUPROXY_CMD_UPCALL_READY: return "upcall ready";
2334 default: return "<unknown>";
2335 }
2336}
2337#define exclaves_xnu_proxy_debug(flag, step, msg) \
2338 exclaves_debug_printf(flag, \
2339 "exclaves: xnu proxy %s " #step ":\t" \
2340 "msg %p server_id 0x%lx cmd %u status %u\n", \
2341 cmd_to_str((msg)->cmd), (msg), (msg)->server_id, (msg)->cmd, \
2342 os_atomic_load(&(msg)->status, relaxed))
2343#define exclaves_xnu_proxy_show_progress(step, msg) \
2344 exclaves_xnu_proxy_debug(show_progress, step, msg)
2345#define exclaves_xnu_proxy_show_error(msg) \
2346 exclaves_xnu_proxy_debug(show_errors, failed, msg)
2347#define exclaves_xnu_proxy_endpoint_call_show_progress(operation, step, \
2348 eid, scid, status) \
2349 exclaves_debug_printf(show_progress, \
2350 "exclaves: xnu proxy endpoint " #operation " " #step ":\t" \
2351 "endpoint id %ld scid 0x%lx status %u\n", \
2352 (eid), (scid), (status))
2353
2354
2355static kern_return_t
2356exclaves_handle_upcall(thread_t thread, Exclaves_L4_IpcBuffer_t *ipcb,
2357 Exclaves_L4_Word_t scid, xnuproxy_msg_status_t status)
2358{
2359 kern_return_t kr;
2360 Exclaves_L4_Word_t endpoint_id;
2361
2362 uint64_t oldscid = thread->th_exclaves_scheduling_context_id;
2363 void *oldipcb = thread->th_exclaves_ipc_buffer;
2364
2365 thread->th_exclaves_scheduling_context_id = scid;
2366 thread->th_exclaves_ipc_buffer = ipcb;
2367
2368 thread->th_exclaves_state |= TH_EXCLAVES_UPCALL;
2369 endpoint_id = XNUPROXY_CR_ENDPOINT_ID(ipcb);
2370 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_UPCALL)
2371 | DBG_FUNC_START, scid, endpoint_id);
2372 exclaves_xnu_proxy_endpoint_call_show_progress(upcall, entry,
2373 endpoint_id, scid, status);
2374 __asm__ volatile (
2375 "EXCLAVES_UPCALL_START: nop\n\t"
2376 );
2377 kr = exclaves_call_upcall_handler(endpoint_id);
2378 __asm__ volatile (
2379 "EXCLAVES_UPCALL_END: nop\n\t"
2380 );
2381 XNUPROXY_CR_STATUS(ipcb) =
2382 XNUPROXY_MSG_STATUS_PROCESSING;
2383 /* TODO: More state returned than Success or OperationInvalid? */
2384 XNUPROXY_CR_RETVAL(ipcb) =
2385 (kr == KERN_SUCCESS) ? Exclaves_L4_Success :
2386 Exclaves_L4_ErrorOperationInvalid;
2387 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_UPCALL)
2388 | DBG_FUNC_END);
2389 thread->th_exclaves_state &= ~TH_EXCLAVES_UPCALL;
2390 exclaves_xnu_proxy_endpoint_call_show_progress(upcall, returned,
2391 endpoint_id, scid,
2392 (unsigned int)XNUPROXY_CR_RETVAL(ipcb));
2393
2394 thread->th_exclaves_scheduling_context_id = oldscid;
2395 thread->th_exclaves_ipc_buffer = oldipcb;
2396
2397 return kr;
2398}
2399
2400extern kern_return_t exclaves_xnu_proxy_send(xnuproxy_msg_t *, Exclaves_L4_Word_t *);
2401kern_return_t
2402exclaves_xnu_proxy_send(xnuproxy_msg_t *_msg, Exclaves_L4_Word_t *spawned)
2403{
2404 assert3p(_msg, !=, NULL);
2405
2406 thread_t thread = current_thread();
2407
2408 if (exclaves_xnu_proxy_msg_buffer == NULL) {
2409 return KERN_FAILURE;
2410 }
2411
2412 kern_return_t kr = KERN_SUCCESS;
2413 xnuproxy_msg_t *msg = exclaves_xnu_proxy_msg_buffer;
2414 bool interrupted = false;
2415
2416 lck_mtx_lock(&exclaves_xnu_proxy_lock);
2417
2418 assert3u(thread->th_exclaves_state & TH_EXCLAVES_STATE_ANY, ==, 0);
2419 thread->th_exclaves_state |= TH_EXCLAVES_XNUPROXY;
2420
2421 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_XNUPROXY)
2422 | DBG_FUNC_START, exclaves_xnu_proxy_scid, _msg->cmd);
2423
2424 *msg = *_msg;
2425 msg->server_id = exclaves_xnu_proxy_scid;
2426
2427 os_atomic_store(&msg->status, XNUPROXY_MSG_STATUS_PROCESSING,
2428 release);
2429
2430 while (os_atomic_load(&msg->status, relaxed) ==
2431 XNUPROXY_MSG_STATUS_PROCESSING) {
2432 exclaves_xnu_proxy_show_progress(in progress, msg);
2433 kr = exclaves_scheduler_resume_scheduling_context(msg->server_id,
2434 spawned, interrupted);
2435 assert(kr == KERN_SUCCESS || kr == KERN_ABORTED);
2436
2437 /* A wait was interrupted. */
2438 interrupted = kr == KERN_ABORTED;
2439
2440 if (NULL != exclaves_xnu_proxy_upcall_ipcb) {
2441 if (XNUPROXY_MSG_STATUS_UPCALL == XNUPROXY_CR_STATUS(exclaves_xnu_proxy_upcall_ipcb)) {
2442 xnuproxy_msg_status_t status = (xnuproxy_msg_status_t)
2443 XNUPROXY_CR_STATUS(exclaves_xnu_proxy_upcall_ipcb);
2444 (void) exclaves_handle_upcall(thread, exclaves_xnu_proxy_upcall_ipcb,
2445 exclaves_xnu_proxy_scid, status);
2446 }
2447 }
2448 }
2449
2450 if (os_atomic_load(&msg->status, acquire) ==
2451 XNUPROXY_MSG_STATUS_NONE) {
2452 exclaves_xnu_proxy_show_progress(complete, msg);
2453 } else {
2454 kr = KERN_FAILURE;
2455 exclaves_xnu_proxy_show_error(msg);
2456 }
2457
2458 *_msg = *msg;
2459
2460 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_XNUPROXY)
2461 | DBG_FUNC_END);
2462
2463 thread->th_exclaves_state &= ~TH_EXCLAVES_XNUPROXY;
2464 lck_mtx_unlock(&exclaves_xnu_proxy_lock);
2465
2466 return kr;
2467}
2468
2469static kern_return_t
2470exclaves_xnu_proxy_init(uint64_t xnu_proxy_boot_info)
2471{
2472 kern_return_t kr = KERN_SUCCESS;
2473 pmap_paddr_t msg_buffer_paddr = xnu_proxy_boot_info;
2474
2475 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
2476
2477 if (msg_buffer_paddr && pmap_valid_address(msg_buffer_paddr)) {
2478 lck_mtx_lock(&exclaves_xnu_proxy_lock);
2479 exclaves_xnu_proxy_msg_buffer =
2480 (xnuproxy_msg_t*)phystokv(msg_buffer_paddr);
2481 exclaves_xnu_proxy_scid =
2482 exclaves_xnu_proxy_msg_buffer->server_id;
2483
2484#if XNUPROXY_MSG_VERSION >= 3
2485 exclaves_xnu_proxy_upcall_ipcb_paddr =
2486 exclaves_xnu_proxy_msg_buffer->upcall_ipc_buffer_paddr;
2487 if (exclaves_xnu_proxy_upcall_ipcb_paddr != 0) {
2488 exclaves_xnu_proxy_upcall_ipcb = (Exclaves_L4_IpcBuffer_t *)
2489 phystokv(exclaves_xnu_proxy_upcall_ipcb_paddr);
2490 }
2491#endif /* XNUPROXY_MSG_VERSION >= 3 */
2492 lck_mtx_unlock(&exclaves_xnu_proxy_lock);
2493 } else {
2494 exclaves_debug_printf(show_errors,
2495 "exclaves: %s: 0x%012llx\n",
2496 "Invalid xnu proxy boot info physical address",
2497 xnu_proxy_boot_info);
2498 return KERN_FAILURE;
2499 }
2500
2501 xnuproxy_msg_t msg = {
2502 .cmd = XNUPROXY_CMD_SETUP,
2503 };
2504
2505 kr = exclaves_xnu_proxy_send(&msg, NULL);
2506 if (kr != KERN_SUCCESS) {
2507 return kr;
2508 }
2509
2510 if (msg.cmd_setup.response.version != XNUPROXY_MSG_VERSION) {
2511 exclaves_debug_printf(show_errors,
2512 "exclaves: mismatched xnuproxy message version, "
2513 "xnuproxy: %u, xnu: %u ", msg.cmd_setup.response.version,
2514 XNUPROXY_MSG_VERSION);
2515 return KERN_FAILURE;
2516 }
2517
2518 exclaves_debug_printf(show_progress,
2519 "exclaves: xnuproxy message version: 0x%u\n", XNUPROXY_MSG_VERSION);
2520
2521 kr = exclaves_panic_thread_setup();
2522 if (kr != KERN_SUCCESS) {
2523 exclaves_debug_printf(show_errors,
2524 "XNU proxy panic thread setup failed\n");
2525 return KERN_FAILURE;
2526 }
2527
2528 return KERN_SUCCESS;
2529}
2530
2531static kern_return_t
2532exclaves_xnu_proxy_allocate_context(Exclaves_L4_Word_t *scid,
2533 Exclaves_L4_IpcBuffer_t **ipcb)
2534{
2535 kern_return_t kr = KERN_FAILURE;
2536 Exclaves_L4_Word_t spawned_scid = 0;
2537
2538 xnuproxy_msg_t msg = {
2539 .cmd = XNUPROXY_CMD_CONTEXT_ALLOCATE,
2540 };
2541
2542 kr = exclaves_xnu_proxy_send(&msg, &spawned_scid);
2543 if (kr != KERN_SUCCESS) {
2544 return kr;
2545 }
2546
2547 if (msg.cmd_ctx_alloc.response.ipc_paddr == 0) {
2548 return KERN_NO_SPACE;
2549 }
2550
2551 if (spawned_scid != 0) {
2552 assert3u(msg.cmd_ctx_alloc.response.sched_id, ==, spawned_scid);
2553 }
2554
2555 *scid = msg.cmd_ctx_alloc.response.sched_id;
2556 *ipcb = (Exclaves_L4_IpcBuffer_t *)
2557 phystokv(msg.cmd_ctx_alloc.response.ipc_paddr);
2558 os_atomic_inc(&exclaves_ipcb_cnt, relaxed);
2559
2560 return KERN_SUCCESS;
2561}
2562
2563static kern_return_t
2564exclaves_xnu_proxy_free_context(Exclaves_L4_Word_t scid)
2565{
2566 kern_return_t kr = KERN_FAILURE;
2567 xnuproxy_msg_t msg = {
2568 .cmd = XNUPROXY_CMD_CONTEXT_FREE,
2569 .cmd_ctx_free = (xnuproxy_cmd_ctx_free_t) {
2570 .request.sched_id = scid,
2571 .request.destroy = false,
2572 },
2573 };
2574
2575 kr = exclaves_xnu_proxy_send(&msg, NULL);
2576 if (kr == KERN_SUCCESS) {
2577 size_t orig_ipcb_cnt = os_atomic_dec_orig(&exclaves_ipcb_cnt, relaxed);
2578 assert3u(orig_ipcb_cnt, >=, 1);
2579 if (orig_ipcb_cnt == 0) { /* This is just to avoid unused variable warning */
2580 kr = KERN_FAILURE;
2581 }
2582 }
2583 return kr;
2584}
2585
2586OS_NOINLINE
2587static kern_return_t
2588exclaves_xnu_proxy_endpoint_call(Exclaves_L4_Word_t endpoint_id)
2589{
2590 kern_return_t kr = KERN_SUCCESS;
2591 thread_t thread = current_thread();
2592 bool interrupted = false;
2593
2594 Exclaves_L4_Word_t scid = thread->th_exclaves_scheduling_context_id;
2595 Exclaves_L4_IpcBuffer_t *ipcb = thread->th_exclaves_ipc_buffer;
2596 xnuproxy_msg_status_t status =
2597 XNUPROXY_MSG_STATUS_PROCESSING;
2598
2599 XNUPROXY_CR_ENDPOINT_ID(ipcb) = endpoint_id;
2600 XNUPROXY_CR_STATUS(ipcb) = status;
2601
2602 exclaves_xnu_proxy_endpoint_call_show_progress(call, entry,
2603 endpoint_id, scid, status);
2604
2605 assert3u(thread->th_exclaves_state & TH_EXCLAVES_STATE_ANY, ==, 0);
2606 thread->th_exclaves_state |= TH_EXCLAVES_RPC;
2607 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_RPC)
2608 | DBG_FUNC_START, scid, endpoint_id);
2609
2610 while (1) {
2611 kr = exclaves_scheduler_resume_scheduling_context(scid, NULL,
2612 interrupted);
2613 assert(kr == KERN_SUCCESS || kr == KERN_ABORTED);
2614
2615 /* A wait was interrupted. */
2616 interrupted = kr == KERN_ABORTED;
2617
2618 status = (xnuproxy_msg_status_t)
2619 XNUPROXY_CR_STATUS(ipcb);
2620
2621 switch (status) {
2622 case XNUPROXY_MSG_STATUS_PROCESSING:
2623 exclaves_xnu_proxy_endpoint_call_show_progress(call, yielded,
2624 endpoint_id, scid, status);
2625 continue;
2626
2627 case XNUPROXY_MSG_STATUS_REPLY:
2628 exclaves_xnu_proxy_endpoint_call_show_progress(call, returned,
2629 endpoint_id, scid, status);
2630 kr = KERN_SUCCESS;
2631 break;
2632
2633 case XNUPROXY_MSG_STATUS_UPCALL:
2634 kr = exclaves_handle_upcall(thread, ipcb, scid, status);
2635 continue;
2636
2637 default:
2638 // Should we have an assert(valid return) here?
2639 exclaves_xnu_proxy_endpoint_call_show_progress(call, failed,
2640 endpoint_id, scid, status);
2641 kr = KERN_FAILURE;
2642 break;
2643 }
2644 break;
2645 }
2646
2647 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, MACH_EXCLAVES_RPC)
2648 | DBG_FUNC_END);
2649 thread->th_exclaves_state &= ~TH_EXCLAVES_RPC;
2650
2651 return kr;
2652}
2653
2654static kern_return_t
2655exclaves_hosted_error(bool success, XrtHosted_Error_t *error)
2656{
2657 if (success) {
2658 return KERN_SUCCESS;
2659 } else {
2660 exclaves_debug_printf(show_errors,
2661 "exclaves: XrtHosted: %s[%d] (%s): %s\n",
2662 error->file,
2663 error->line,
2664 error->function,
2665 error->expression
2666 );
2667 return KERN_FAILURE;
2668 }
2669}
2670
2671kern_return_t
2672exclaves_ipc_buffer_cache_init(void)
2673{
2674 kern_return_t kr = KERN_SUCCESS;
2675 Exclaves_L4_IpcBuffer_t *ipcb = NULL;
2676 Exclaves_L4_Word_t scid = 0;
2677
2678 LCK_MTX_ASSERT(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
2679 assert(exclaves_ipc_buffer_cache == NULL);
2680
2681 if (exclaves_ipc_buffer_cache_enabled) {
2682 if ((kr = exclaves_xnu_proxy_allocate_context(&scid, &ipcb))) {
2683 return kr;
2684 }
2685
2686 /* relinquish the new buffer into the cache */
2687 exclaves_relinquish_ipc_buffer(ipcb, scid);
2688 }
2689 return kr;
2690}
2691
2692#pragma mark exclaves privilege management
2693
2694/*
2695 * All entitlement checking enabled by default.
2696 */
2697#define DEFAULT_ENTITLEMENT_FLAGS (~(0))
2698
2699/*
2700 * boot-arg to control the use of entitlements.
2701 */
2702static TUNABLE(unsigned int, exclaves_entitlement_flags,
2703 "exclaves_entitlement_flags", DEFAULT_ENTITLEMENT_FLAGS);
2704
2705static bool
2706has_entitlement(task_t task, const exclaves_priv_t priv,
2707 const char *entitlement)
2708{
2709 /* Skip the entitlement if not enabled. */
2710 if ((exclaves_entitlement_flags & priv) == 0) {
2711 return true;
2712 }
2713
2714 return IOTaskHasEntitlement(task, entitlement);
2715}
2716
2717static bool
2718has_entitlement_vnode(void *vnode, const int64_t off,
2719 const exclaves_priv_t priv, const char *entitlement)
2720{
2721 /* Skip the entitlement if not enabled. */
2722 if ((exclaves_entitlement_flags & priv) == 0) {
2723 return true;
2724 }
2725
2726 return IOVnodeHasEntitlement(vnode, off, entitlement);
2727}
2728
2729bool
2730exclaves_has_priv(task_t task, exclaves_priv_t priv)
2731{
2732 const bool is_kernel = task == kernel_task;
2733 const bool is_launchd = task_pid(task) == 1;
2734
2735 switch (priv) {
2736 case EXCLAVES_PRIV_CONCLAVE_SPAWN:
2737 /* Both launchd and entitled tasks can spawn new conclaves. */
2738 if (is_launchd) {
2739 return true;
2740 }
2741 return has_entitlement(task, priv,
2742 "com.apple.private.exclaves.conclave-spawn");
2743
2744 case EXCLAVES_PRIV_KERNEL_DOMAIN:
2745 /*
2746 * Both the kernel itself and user tasks with the right
2747 * privilege can access exclaves resources in the kernel domain.
2748 */
2749 if (is_kernel) {
2750 return true;
2751 }
2752
2753 /*
2754 * If the task was entitled and has been through this path
2755 * before, it will have set the TFRO_HAS_KD_ACCESS flag.
2756 */
2757 if ((task_ro_flags_get(task) & TFRO_HAS_KD_ACCESS) != 0) {
2758 return true;
2759 }
2760
2761 if (has_entitlement(task, priv,
2762 "com.apple.private.exclaves.kernel-domain")) {
2763 task_ro_flags_set(task, TFRO_HAS_KD_ACCESS);
2764 return true;
2765 }
2766
2767 return false;
2768
2769 case EXCLAVES_PRIV_BOOT:
2770 /* Both launchd and entitled tasks can boot exclaves. */
2771 if (is_launchd) {
2772 return true;
2773 }
2774 return has_entitlement(task, priv,
2775 "com.apple.private.exclaves.boot");
2776
2777 /* The CONCLAVE HOST priv is always checked by vnode. */
2778 case EXCLAVES_PRIV_CONCLAVE_HOST:
2779 default:
2780 panic("bad exclaves privilege (%u)", priv);
2781 }
2782}
2783
2784bool
2785exclaves_has_priv_vnode(void *vnode, int64_t off, exclaves_priv_t priv)
2786{
2787 switch (priv) {
2788 case EXCLAVES_PRIV_CONCLAVE_HOST:
2789 return has_entitlement_vnode(vnode, off, priv,
2790 "com.apple.private.exclaves.conclave-host");
2791
2792 case EXCLAVES_PRIV_CONCLAVE_SPAWN:
2793 return has_entitlement_vnode(vnode, off, priv,
2794 "com.apple.private.exclaves.conclave-spawn");
2795
2796 default:
2797 panic("bad exclaves privilege (%u)", priv);
2798 }
2799}
2800
2801uint32_t
2802exclaves_stack_offset(uintptr_t * out_addr, size_t nframes, bool slid_addresses)
2803{
2804 size_t i = 0;
2805 uintptr_t enter_range_start = 0;
2806 uintptr_t enter_range_end = 0;
2807 uintptr_t upcall_range_start = 0;
2808 uintptr_t upcall_range_end = 0;
2809
2810 if (slid_addresses) {
2811 enter_range_start = (uintptr_t)&exclaves_enter_start_label;
2812 enter_range_end = (uintptr_t)&exclaves_enter_end_label;
2813 upcall_range_start = (uintptr_t)&exclaves_upcall_start_label;
2814 upcall_range_end = (uintptr_t)&exclaves_upcall_end_label;
2815 } else {
2816 enter_range_start = exclaves_enter_range_start;
2817 enter_range_end = exclaves_enter_range_end;
2818 upcall_range_start = exclaves_upcall_range_start;
2819 upcall_range_end = exclaves_upcall_range_end;
2820 }
2821
2822 while (i < nframes &&
2823 !((enter_range_start < out_addr[i]) && (out_addr[i] <= enter_range_end))
2824 && !((upcall_range_start < out_addr[i]) && (out_addr[i] <= upcall_range_end))
2825 ) {
2826 i++;
2827 }
2828
2829 return (uint32_t)i;
2830}
2831
2832#endif /* CONFIG_EXCLAVES */
2833
2834#ifndef CONFIG_EXCLAVES
2835/* stubs for sensor functions which are not compiled in from exclaves.c when
2836 * CONFIG_EXCLAVE is disabled */
2837
2838kern_return_t
2839exclaves_sensor_start(exclaves_sensor_type_t sensor_type, uint64_t flags,
2840 exclaves_sensor_status_t *status)
2841{
2842#pragma unused(sensor_type, flags, status)
2843 return KERN_NOT_SUPPORTED;
2844}
2845
2846kern_return_t
2847exclaves_sensor_stop(exclaves_sensor_type_t sensor_type, uint64_t flags,
2848 exclaves_sensor_status_t *status)
2849{
2850#pragma unused(sensor_type, flags, status)
2851 return KERN_NOT_SUPPORTED;
2852}
2853
2854kern_return_t
2855exclaves_sensor_status(exclaves_sensor_type_t sensor_type, uint64_t flags,
2856 exclaves_sensor_status_t *status)
2857{
2858#pragma unused(sensor_type, flags, status)
2859 return KERN_NOT_SUPPORTED;
2860}
2861
2862#endif /* ! CONFIG_EXCLAVES */
2863