1/*
2 * Copyright (c) 2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <mach/exclaves.h>
30
31#if CONFIG_EXCLAVES
32
33#if CONFIG_SPTM
34#include <arm64/sptm/sptm.h>
35#else
36#error Invalid configuration
37#endif /* CONFIG_SPTM */
38
39#include <libkern/section_keywords.h>
40#include <kern/assert.h>
41#include <kern/locks.h>
42#include <kern/thread.h>
43#include <stddef.h>
44
45#include "exclaves_boot.h"
46#include "exclaves_debug.h"
47#include "exclaves_frame_mint.h"
48#include "exclaves_upcalls.h"
49
50extern lck_grp_t exclaves_lck_grp;
51
52/* Lock around single-threaded exclaves boot */
53LCK_MTX_DECLARE(exclaves_boot_lock, &exclaves_lck_grp);
54
55/* Boot status. */
56__enum_closed_decl(exclaves_boot_status_t, uint32_t, {
57 EXCLAVES_BS_NOT_SUPPORTED = 0,
58 EXCLAVES_BS_NOT_STARTED = 1,
59 EXCLAVES_BS_BOOTED_STAGE_2 = 2,
60 EXCLAVES_BS_BOOTED_EXCLAVEKIT = 3,
61 EXCLAVES_BS_BOOTED_FAILURE = 4,
62});
63
64/* Atomic so that it can be safely checked outside the boot lock. */
65static os_atomic(exclaves_boot_status_t) exclaves_boot_status = EXCLAVES_BS_NOT_SUPPORTED;
66
67static thread_t exclaves_boot_thread = THREAD_NULL;
68
69extern exclaves_boot_task_entry_t exclaves_boot_task_entries[]
70__SECTION_START_SYM(EXCLAVES_BOOT_TASK_SEGMENT, EXCLAVES_BOOT_TASK_SECTION);
71
72extern exclaves_boot_task_entry_t exclaves_boot_task_entries_end[]
73__SECTION_END_SYM(EXCLAVES_BOOT_TASK_SEGMENT, EXCLAVES_BOOT_TASK_SECTION);
74
75static int
76ebt_cmp(const void *e1, const void *e2)
77{
78 const struct exclaves_boot_task_entry *a = e1;
79 const struct exclaves_boot_task_entry *b = e2;
80
81 if (a->ebt_rank > b->ebt_rank) {
82 return 1;
83 }
84
85 if (a->ebt_rank < b->ebt_rank) {
86 return -1;
87 }
88
89 return 0;
90}
91
92static void
93exclaves_boot_tasks(void)
94{
95 const size_t count =
96 exclaves_boot_task_entries_end - exclaves_boot_task_entries;
97 assert3u(count, >, 0);
98
99 __assert_only const size_t size =
100 ((uintptr_t)exclaves_boot_task_entries_end -
101 (uintptr_t)exclaves_boot_task_entries);
102 assert3u(size % sizeof(exclaves_boot_task_entry_t), ==, 0);
103
104 /*
105 * exclaves_boot_task_entries is in _DATA_CONST, make a stack copy so it
106 * can be sorted.
107 */
108 exclaves_boot_task_entry_t boot_tasks[count];
109 memcpy(boot_tasks, exclaves_boot_task_entries, count * sizeof(boot_tasks[0]));
110
111 extern void qsort(void *, size_t, size_t,
112 int (*)(const void *, const void *));
113 qsort(boot_tasks, count, sizeof(boot_tasks[0]), ebt_cmp);
114
115 for (size_t i = 0; i < count; i++) {
116 exclaves_debug_printf(show_progress,
117 "exclaves: boot task started, %s\n", boot_tasks[i].ebt_name);
118
119 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES,
120 MACH_EXCLAVES_BOOT_TASK) | DBG_FUNC_START, i);
121 kern_return_t ret = boot_tasks[i].ebt_func();
122 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES,
123 MACH_EXCLAVES_BOOT_TASK) | DBG_FUNC_END);
124
125 exclaves_debug_printf(show_progress,
126 "exclaves: boot task done, %s\n", boot_tasks[i].ebt_name);
127
128 if (ret != KERN_SUCCESS) {
129 panic("exclaves: boot task failed: %s (%p)",
130 boot_tasks[i].ebt_name, &boot_tasks[i]);
131 }
132 }
133}
134
135/*
136 * Check early in boot if the secure world has bootstrapped, and update the
137 * boot status if not.
138 */
139__startup_func
140static void
141exclaves_check_sk(void)
142{
143 const bool sk_bootstrapped = SPTMArgs->sk_bootstrapped;
144
145 if (sk_bootstrapped) {
146 os_atomic_store(&exclaves_boot_status,
147 EXCLAVES_BS_NOT_STARTED, relaxed);
148 }
149}
150STARTUP(TUNABLES, STARTUP_RANK_MIDDLE, exclaves_check_sk);
151
152static void
153exclaves_boot_status_wait(const exclaves_boot_status_t status)
154{
155 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
156
157 while (os_atomic_load(&exclaves_boot_status, relaxed) < status) {
158 lck_mtx_sleep(&exclaves_boot_lock, LCK_SLEEP_DEFAULT,
159 (event_t)(uintptr_t)&exclaves_boot_status, THREAD_UNINT);
160 }
161}
162
163static void
164exclaves_boot_status_wake(void)
165{
166 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
167 thread_wakeup((event_t)(uintptr_t)&exclaves_boot_status);
168}
169
170static void
171exclaves_boot_status_set(const exclaves_boot_status_t status)
172{
173 assert3u(status, >, os_atomic_load(&exclaves_boot_status, relaxed));
174
175 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
176
177 /*
178 * 'release' here to ensure that the status update is only seen after
179 * any boot update. exclaves_boot_status is loaded with 'acquire' in
180 * exclaves_boot_wait() without holding the lock so the mutex alone
181 * isn't sufficient for ordering.
182 */
183 os_atomic_store(&exclaves_boot_status, status, release);
184 exclaves_boot_status_wake();
185}
186
187static kern_return_t
188exclaves_boot_stage_2(void)
189{
190 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
191
192 kern_return_t kr = KERN_FAILURE;
193 const exclaves_boot_status_t status =
194 os_atomic_load(&exclaves_boot_status, relaxed);
195
196 if (status == EXCLAVES_BS_NOT_SUPPORTED) {
197 return KERN_NOT_SUPPORTED;
198 }
199
200 /*
201 * This should only ever happen if there's a userspace bug which causes
202 * boot to be called twice and cause a race.
203 */
204 if (status != EXCLAVES_BS_NOT_STARTED) {
205 return KERN_INVALID_ARGUMENT;
206 }
207
208 /* Initialize xnu upcall server. */
209 kr = exclaves_upcall_early_init();
210 if (kr != KERN_SUCCESS) {
211 panic("Exclaves upcall early init failed");
212 }
213
214 /* Early boot. */
215 extern kern_return_t exclaves_boot_early(void);
216 kr = exclaves_boot_early();
217 if (kr != KERN_SUCCESS) {
218 /*
219 * If exclaves failed to boot, there's not much that can be done other
220 * than panic.
221 */
222 panic("Exclaves stage2 boot failed");
223 }
224
225 /*
226 * At this point it should be possible to make tightbeam calls/configure
227 * endpoints etc.
228 */
229 exclaves_boot_thread = current_thread();
230 exclaves_boot_tasks();
231 exclaves_boot_thread = THREAD_NULL;
232
233 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_STAGE_2);
234
235 return kr;
236}
237
238static kern_return_t
239exclaves_boot_exclavekit(void)
240{
241 lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED);
242
243 /* This should require an entitlement at some point. */
244
245 const exclaves_boot_status_t status =
246 os_atomic_load(&exclaves_boot_status, relaxed);
247
248 if (status == EXCLAVES_BS_NOT_SUPPORTED) {
249 return KERN_NOT_SUPPORTED;
250 }
251
252 /* Should only be called after stage2 boot. */
253 if (status != EXCLAVES_BS_BOOTED_STAGE_2) {
254 return KERN_INVALID_ARGUMENT;
255 }
256
257
258 /*
259 * Treat a failure to boot exclavekit as a transition to the
260 * EXCLAVES_BS_BOOTED_FAILURE state and return a failure.
261 */
262 kern_return_t kr = exclaves_frame_mint_populate();
263 if (kr != KERN_SUCCESS) {
264 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_FAILURE);
265 return KERN_FAILURE;
266 }
267
268 exclaves_boot_status_set(EXCLAVES_BS_BOOTED_EXCLAVEKIT);
269
270 return KERN_SUCCESS;
271}
272
273kern_return_t
274exclaves_boot(exclaves_boot_stage_t boot_stage)
275{
276 lck_mtx_lock(&exclaves_boot_lock);
277
278 kern_return_t kr = KERN_FAILURE;
279
280 switch (boot_stage) {
281 case EXCLAVES_BOOT_STAGE_2:
282 kr = exclaves_boot_stage_2();
283 break;
284
285 case EXCLAVES_BOOT_STAGE_EXCLAVEKIT:
286 kr = exclaves_boot_exclavekit();
287 break;
288
289 default:
290 kr = KERN_INVALID_ARGUMENT;
291 break;
292 }
293
294 lck_mtx_unlock(&exclaves_boot_lock);
295
296 return kr;
297}
298
299/*
300 * This returns once the specified boot stage has been reached. If exclaves are
301 * unavailable, it returns immediately with KERN_NOT_SUPPORTED.
302 * Make it NOINLINE so it shows up in backtraces.
303 */
304kern_return_t OS_NOINLINE
305exclaves_boot_wait(const exclaves_boot_stage_t desired_boot_stage)
306{
307 assert(desired_boot_stage == EXCLAVES_BOOT_STAGE_2 ||
308 desired_boot_stage == EXCLAVES_BOOT_STAGE_EXCLAVEKIT);
309
310 /* Look up the equivalent status for the specified boot stage. */
311 const exclaves_boot_status_t desired_boot_status =
312 desired_boot_stage == EXCLAVES_BOOT_STAGE_2 ?
313 EXCLAVES_BS_BOOTED_STAGE_2 : EXCLAVES_BS_BOOTED_EXCLAVEKIT;
314
315 /*
316 * See comment in exclaves_boot_status_set() for why this is an acquire.
317 */
318 const exclaves_boot_status_t current_boot_status =
319 os_atomic_load(&exclaves_boot_status, acquire);
320
321 if (current_boot_status >= desired_boot_status) {
322 /*
323 * Special-case the situation where the request is to wait for
324 * EXCLAVES_BOOT_STAGE_EXCLAVEKIT. EXCLAVEKIT boot can fail
325 * (unlike STAGE_2 boot which will panic on failure). If
326 * EXCLAVEKIT has failed, just return KERN_NOT_SUPPORTED.
327 */
328 if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVEKIT &&
329 current_boot_status == EXCLAVES_BS_BOOTED_FAILURE) {
330 return KERN_NOT_SUPPORTED;
331 }
332
333 return KERN_SUCCESS;
334 }
335
336 if (current_boot_status == EXCLAVES_BS_NOT_SUPPORTED) {
337 return KERN_NOT_SUPPORTED;
338 }
339
340 /*
341 * Allow the exclaves boot thread to pass this check during stage2
342 * boot. This allows exclaves boot tasks to make TB calls etc during
343 * stage2 boot.
344 */
345 if (desired_boot_status == EXCLAVES_BS_BOOTED_STAGE_2 &&
346 current_thread() == exclaves_boot_thread) {
347 return KERN_SUCCESS;
348 }
349
350 /*
351 * Otherwise, wait until exclaves has booted to the requested stage or
352 * failed to boot.
353 */
354 lck_mtx_lock(&exclaves_boot_lock);
355 exclaves_boot_status_wait(desired_boot_status);
356 lck_mtx_unlock(&exclaves_boot_lock);
357
358 /*
359 * At this point there are two possibilities. Success or EXCLAVEKIT has
360 * failed (STAGE_2 can never fail as it panics on failure).
361 */
362 if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVEKIT &&
363 current_boot_status == EXCLAVES_BS_BOOTED_FAILURE) {
364 return KERN_NOT_SUPPORTED;
365 }
366
367 return KERN_SUCCESS;
368}
369
370/*
371 * This returns AVAILABLE once EXCLAVES_BOOT_STAGE_2 has completed. This is to
372 * maintain backwards compatibility with existing code.
373 */
374exclaves_status_t
375exclaves_get_status(void)
376{
377 kern_return_t kr = exclaves_boot_wait(EXCLAVES_BOOT_STAGE_2);
378 assert(kr == KERN_SUCCESS || kr == KERN_NOT_SUPPORTED);
379
380 if (kr == KERN_SUCCESS) {
381 return EXCLAVES_STATUS_AVAILABLE;
382 }
383
384 return EXCLAVES_STATUS_NOT_SUPPORTED;
385}
386
387exclaves_boot_stage_t
388exclaves_get_boot_stage(void)
389{
390 exclaves_boot_status_t status =
391 os_atomic_load(&exclaves_boot_status, relaxed);
392
393 switch (status) {
394 case EXCLAVES_BS_NOT_STARTED:
395 case EXCLAVES_BS_NOT_SUPPORTED:
396 return EXCLAVES_BOOT_STAGE_NONE;
397
398 case EXCLAVES_BS_BOOTED_STAGE_2:
399 return EXCLAVES_BOOT_STAGE_2;
400
401 case EXCLAVES_BS_BOOTED_EXCLAVEKIT:
402 return EXCLAVES_BOOT_STAGE_EXCLAVEKIT;
403
404 case EXCLAVES_BS_BOOTED_FAILURE:
405 return EXCLAVES_BOOT_STAGE_FAILED;
406
407 default:
408 panic("unknown boot status: %u", status);
409 }
410}
411
412#else /* CONFIG_EXCLAVES */
413
414exclaves_status_t
415exclaves_get_status(void)
416{
417 return EXCLAVES_STATUS_NOT_SUPPORTED;
418}
419
420
421exclaves_boot_stage_t
422exclaves_get_boot_stage(void)
423{
424 return EXCLAVES_BOOT_STAGE_NONE;
425}
426
427#endif /* CONFIG_EXCLAVES */
428