1 | /* |
2 | * Copyright (c) 2023 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <mach/exclaves.h> |
30 | |
31 | #if CONFIG_EXCLAVES |
32 | |
33 | #if CONFIG_SPTM |
34 | #include <arm64/sptm/sptm.h> |
35 | #else |
36 | #error Invalid configuration |
37 | #endif /* CONFIG_SPTM */ |
38 | |
39 | #include <libkern/section_keywords.h> |
40 | #include <kern/assert.h> |
41 | #include <kern/locks.h> |
42 | #include <kern/thread.h> |
43 | #include <stddef.h> |
44 | |
45 | #include "exclaves_boot.h" |
46 | #include "exclaves_debug.h" |
47 | #include "exclaves_frame_mint.h" |
48 | #include "exclaves_upcalls.h" |
49 | |
50 | extern lck_grp_t exclaves_lck_grp; |
51 | |
52 | /* Lock around single-threaded exclaves boot */ |
53 | LCK_MTX_DECLARE(exclaves_boot_lock, &exclaves_lck_grp); |
54 | |
55 | /* Boot status. */ |
56 | __enum_closed_decl(exclaves_boot_status_t, uint32_t, { |
57 | EXCLAVES_BS_NOT_SUPPORTED = 0, |
58 | EXCLAVES_BS_NOT_STARTED = 1, |
59 | EXCLAVES_BS_BOOTED_STAGE_2 = 2, |
60 | EXCLAVES_BS_BOOTED_EXCLAVEKIT = 3, |
61 | EXCLAVES_BS_BOOTED_FAILURE = 4, |
62 | }); |
63 | |
64 | /* Atomic so that it can be safely checked outside the boot lock. */ |
65 | static os_atomic(exclaves_boot_status_t) exclaves_boot_status = EXCLAVES_BS_NOT_SUPPORTED; |
66 | |
67 | static thread_t exclaves_boot_thread = THREAD_NULL; |
68 | |
69 | extern exclaves_boot_task_entry_t exclaves_boot_task_entries[] |
70 | __SECTION_START_SYM(EXCLAVES_BOOT_TASK_SEGMENT, EXCLAVES_BOOT_TASK_SECTION); |
71 | |
72 | extern exclaves_boot_task_entry_t exclaves_boot_task_entries_end[] |
73 | __SECTION_END_SYM(EXCLAVES_BOOT_TASK_SEGMENT, EXCLAVES_BOOT_TASK_SECTION); |
74 | |
75 | static int |
76 | ebt_cmp(const void *e1, const void *e2) |
77 | { |
78 | const struct exclaves_boot_task_entry *a = e1; |
79 | const struct exclaves_boot_task_entry *b = e2; |
80 | |
81 | if (a->ebt_rank > b->ebt_rank) { |
82 | return 1; |
83 | } |
84 | |
85 | if (a->ebt_rank < b->ebt_rank) { |
86 | return -1; |
87 | } |
88 | |
89 | return 0; |
90 | } |
91 | |
92 | static void |
93 | exclaves_boot_tasks(void) |
94 | { |
95 | const size_t count = |
96 | exclaves_boot_task_entries_end - exclaves_boot_task_entries; |
97 | assert3u(count, >, 0); |
98 | |
99 | __assert_only const size_t size = |
100 | ((uintptr_t)exclaves_boot_task_entries_end - |
101 | (uintptr_t)exclaves_boot_task_entries); |
102 | assert3u(size % sizeof(exclaves_boot_task_entry_t), ==, 0); |
103 | |
104 | /* |
105 | * exclaves_boot_task_entries is in _DATA_CONST, make a stack copy so it |
106 | * can be sorted. |
107 | */ |
108 | exclaves_boot_task_entry_t boot_tasks[count]; |
109 | memcpy(boot_tasks, exclaves_boot_task_entries, count * sizeof(boot_tasks[0])); |
110 | |
111 | extern void qsort(void *, size_t, size_t, |
112 | int (*)(const void *, const void *)); |
113 | qsort(boot_tasks, count, sizeof(boot_tasks[0]), ebt_cmp); |
114 | |
115 | for (size_t i = 0; i < count; i++) { |
116 | exclaves_debug_printf(show_progress, |
117 | "exclaves: boot task started, %s\n" , boot_tasks[i].ebt_name); |
118 | |
119 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, |
120 | MACH_EXCLAVES_BOOT_TASK) | DBG_FUNC_START, i); |
121 | kern_return_t ret = boot_tasks[i].ebt_func(); |
122 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCLAVES, |
123 | MACH_EXCLAVES_BOOT_TASK) | DBG_FUNC_END); |
124 | |
125 | exclaves_debug_printf(show_progress, |
126 | "exclaves: boot task done, %s\n" , boot_tasks[i].ebt_name); |
127 | |
128 | if (ret != KERN_SUCCESS) { |
129 | panic("exclaves: boot task failed: %s (%p)" , |
130 | boot_tasks[i].ebt_name, &boot_tasks[i]); |
131 | } |
132 | } |
133 | } |
134 | |
135 | /* |
136 | * Check early in boot if the secure world has bootstrapped, and update the |
137 | * boot status if not. |
138 | */ |
139 | __startup_func |
140 | static void |
141 | exclaves_check_sk(void) |
142 | { |
143 | const bool sk_bootstrapped = SPTMArgs->sk_bootstrapped; |
144 | |
145 | if (sk_bootstrapped) { |
146 | os_atomic_store(&exclaves_boot_status, |
147 | EXCLAVES_BS_NOT_STARTED, relaxed); |
148 | } |
149 | } |
150 | STARTUP(TUNABLES, STARTUP_RANK_MIDDLE, exclaves_check_sk); |
151 | |
152 | static void |
153 | exclaves_boot_status_wait(const exclaves_boot_status_t status) |
154 | { |
155 | lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED); |
156 | |
157 | while (os_atomic_load(&exclaves_boot_status, relaxed) < status) { |
158 | lck_mtx_sleep(&exclaves_boot_lock, LCK_SLEEP_DEFAULT, |
159 | (event_t)(uintptr_t)&exclaves_boot_status, THREAD_UNINT); |
160 | } |
161 | } |
162 | |
163 | static void |
164 | exclaves_boot_status_wake(void) |
165 | { |
166 | lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED); |
167 | thread_wakeup((event_t)(uintptr_t)&exclaves_boot_status); |
168 | } |
169 | |
170 | static void |
171 | exclaves_boot_status_set(const exclaves_boot_status_t status) |
172 | { |
173 | assert3u(status, >, os_atomic_load(&exclaves_boot_status, relaxed)); |
174 | |
175 | lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED); |
176 | |
177 | /* |
178 | * 'release' here to ensure that the status update is only seen after |
179 | * any boot update. exclaves_boot_status is loaded with 'acquire' in |
180 | * exclaves_boot_wait() without holding the lock so the mutex alone |
181 | * isn't sufficient for ordering. |
182 | */ |
183 | os_atomic_store(&exclaves_boot_status, status, release); |
184 | exclaves_boot_status_wake(); |
185 | } |
186 | |
187 | static kern_return_t |
188 | exclaves_boot_stage_2(void) |
189 | { |
190 | lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED); |
191 | |
192 | kern_return_t kr = KERN_FAILURE; |
193 | const exclaves_boot_status_t status = |
194 | os_atomic_load(&exclaves_boot_status, relaxed); |
195 | |
196 | if (status == EXCLAVES_BS_NOT_SUPPORTED) { |
197 | return KERN_NOT_SUPPORTED; |
198 | } |
199 | |
200 | /* |
201 | * This should only ever happen if there's a userspace bug which causes |
202 | * boot to be called twice and cause a race. |
203 | */ |
204 | if (status != EXCLAVES_BS_NOT_STARTED) { |
205 | return KERN_INVALID_ARGUMENT; |
206 | } |
207 | |
208 | /* Initialize xnu upcall server. */ |
209 | kr = exclaves_upcall_early_init(); |
210 | if (kr != KERN_SUCCESS) { |
211 | panic("Exclaves upcall early init failed" ); |
212 | } |
213 | |
214 | /* Early boot. */ |
215 | extern kern_return_t exclaves_boot_early(void); |
216 | kr = exclaves_boot_early(); |
217 | if (kr != KERN_SUCCESS) { |
218 | /* |
219 | * If exclaves failed to boot, there's not much that can be done other |
220 | * than panic. |
221 | */ |
222 | panic("Exclaves stage2 boot failed" ); |
223 | } |
224 | |
225 | /* |
226 | * At this point it should be possible to make tightbeam calls/configure |
227 | * endpoints etc. |
228 | */ |
229 | exclaves_boot_thread = current_thread(); |
230 | exclaves_boot_tasks(); |
231 | exclaves_boot_thread = THREAD_NULL; |
232 | |
233 | exclaves_boot_status_set(EXCLAVES_BS_BOOTED_STAGE_2); |
234 | |
235 | return kr; |
236 | } |
237 | |
238 | static kern_return_t |
239 | exclaves_boot_exclavekit(void) |
240 | { |
241 | lck_mtx_assert(&exclaves_boot_lock, LCK_MTX_ASSERT_OWNED); |
242 | |
243 | /* This should require an entitlement at some point. */ |
244 | |
245 | const exclaves_boot_status_t status = |
246 | os_atomic_load(&exclaves_boot_status, relaxed); |
247 | |
248 | if (status == EXCLAVES_BS_NOT_SUPPORTED) { |
249 | return KERN_NOT_SUPPORTED; |
250 | } |
251 | |
252 | /* Should only be called after stage2 boot. */ |
253 | if (status != EXCLAVES_BS_BOOTED_STAGE_2) { |
254 | return KERN_INVALID_ARGUMENT; |
255 | } |
256 | |
257 | |
258 | /* |
259 | * Treat a failure to boot exclavekit as a transition to the |
260 | * EXCLAVES_BS_BOOTED_FAILURE state and return a failure. |
261 | */ |
262 | kern_return_t kr = exclaves_frame_mint_populate(); |
263 | if (kr != KERN_SUCCESS) { |
264 | exclaves_boot_status_set(EXCLAVES_BS_BOOTED_FAILURE); |
265 | return KERN_FAILURE; |
266 | } |
267 | |
268 | exclaves_boot_status_set(EXCLAVES_BS_BOOTED_EXCLAVEKIT); |
269 | |
270 | return KERN_SUCCESS; |
271 | } |
272 | |
273 | kern_return_t |
274 | exclaves_boot(exclaves_boot_stage_t boot_stage) |
275 | { |
276 | lck_mtx_lock(&exclaves_boot_lock); |
277 | |
278 | kern_return_t kr = KERN_FAILURE; |
279 | |
280 | switch (boot_stage) { |
281 | case EXCLAVES_BOOT_STAGE_2: |
282 | kr = exclaves_boot_stage_2(); |
283 | break; |
284 | |
285 | case EXCLAVES_BOOT_STAGE_EXCLAVEKIT: |
286 | kr = exclaves_boot_exclavekit(); |
287 | break; |
288 | |
289 | default: |
290 | kr = KERN_INVALID_ARGUMENT; |
291 | break; |
292 | } |
293 | |
294 | lck_mtx_unlock(&exclaves_boot_lock); |
295 | |
296 | return kr; |
297 | } |
298 | |
299 | /* |
300 | * This returns once the specified boot stage has been reached. If exclaves are |
301 | * unavailable, it returns immediately with KERN_NOT_SUPPORTED. |
302 | * Make it NOINLINE so it shows up in backtraces. |
303 | */ |
304 | kern_return_t OS_NOINLINE |
305 | exclaves_boot_wait(const exclaves_boot_stage_t desired_boot_stage) |
306 | { |
307 | assert(desired_boot_stage == EXCLAVES_BOOT_STAGE_2 || |
308 | desired_boot_stage == EXCLAVES_BOOT_STAGE_EXCLAVEKIT); |
309 | |
310 | /* Look up the equivalent status for the specified boot stage. */ |
311 | const exclaves_boot_status_t desired_boot_status = |
312 | desired_boot_stage == EXCLAVES_BOOT_STAGE_2 ? |
313 | EXCLAVES_BS_BOOTED_STAGE_2 : EXCLAVES_BS_BOOTED_EXCLAVEKIT; |
314 | |
315 | /* |
316 | * See comment in exclaves_boot_status_set() for why this is an acquire. |
317 | */ |
318 | const exclaves_boot_status_t current_boot_status = |
319 | os_atomic_load(&exclaves_boot_status, acquire); |
320 | |
321 | if (current_boot_status >= desired_boot_status) { |
322 | /* |
323 | * Special-case the situation where the request is to wait for |
324 | * EXCLAVES_BOOT_STAGE_EXCLAVEKIT. EXCLAVEKIT boot can fail |
325 | * (unlike STAGE_2 boot which will panic on failure). If |
326 | * EXCLAVEKIT has failed, just return KERN_NOT_SUPPORTED. |
327 | */ |
328 | if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVEKIT && |
329 | current_boot_status == EXCLAVES_BS_BOOTED_FAILURE) { |
330 | return KERN_NOT_SUPPORTED; |
331 | } |
332 | |
333 | return KERN_SUCCESS; |
334 | } |
335 | |
336 | if (current_boot_status == EXCLAVES_BS_NOT_SUPPORTED) { |
337 | return KERN_NOT_SUPPORTED; |
338 | } |
339 | |
340 | /* |
341 | * Allow the exclaves boot thread to pass this check during stage2 |
342 | * boot. This allows exclaves boot tasks to make TB calls etc during |
343 | * stage2 boot. |
344 | */ |
345 | if (desired_boot_status == EXCLAVES_BS_BOOTED_STAGE_2 && |
346 | current_thread() == exclaves_boot_thread) { |
347 | return KERN_SUCCESS; |
348 | } |
349 | |
350 | /* |
351 | * Otherwise, wait until exclaves has booted to the requested stage or |
352 | * failed to boot. |
353 | */ |
354 | lck_mtx_lock(&exclaves_boot_lock); |
355 | exclaves_boot_status_wait(desired_boot_status); |
356 | lck_mtx_unlock(&exclaves_boot_lock); |
357 | |
358 | /* |
359 | * At this point there are two possibilities. Success or EXCLAVEKIT has |
360 | * failed (STAGE_2 can never fail as it panics on failure). |
361 | */ |
362 | if (desired_boot_status == EXCLAVES_BS_BOOTED_EXCLAVEKIT && |
363 | current_boot_status == EXCLAVES_BS_BOOTED_FAILURE) { |
364 | return KERN_NOT_SUPPORTED; |
365 | } |
366 | |
367 | return KERN_SUCCESS; |
368 | } |
369 | |
370 | /* |
371 | * This returns AVAILABLE once EXCLAVES_BOOT_STAGE_2 has completed. This is to |
372 | * maintain backwards compatibility with existing code. |
373 | */ |
374 | exclaves_status_t |
375 | exclaves_get_status(void) |
376 | { |
377 | kern_return_t kr = exclaves_boot_wait(EXCLAVES_BOOT_STAGE_2); |
378 | assert(kr == KERN_SUCCESS || kr == KERN_NOT_SUPPORTED); |
379 | |
380 | if (kr == KERN_SUCCESS) { |
381 | return EXCLAVES_STATUS_AVAILABLE; |
382 | } |
383 | |
384 | return EXCLAVES_STATUS_NOT_SUPPORTED; |
385 | } |
386 | |
387 | exclaves_boot_stage_t |
388 | exclaves_get_boot_stage(void) |
389 | { |
390 | exclaves_boot_status_t status = |
391 | os_atomic_load(&exclaves_boot_status, relaxed); |
392 | |
393 | switch (status) { |
394 | case EXCLAVES_BS_NOT_STARTED: |
395 | case EXCLAVES_BS_NOT_SUPPORTED: |
396 | return EXCLAVES_BOOT_STAGE_NONE; |
397 | |
398 | case EXCLAVES_BS_BOOTED_STAGE_2: |
399 | return EXCLAVES_BOOT_STAGE_2; |
400 | |
401 | case EXCLAVES_BS_BOOTED_EXCLAVEKIT: |
402 | return EXCLAVES_BOOT_STAGE_EXCLAVEKIT; |
403 | |
404 | case EXCLAVES_BS_BOOTED_FAILURE: |
405 | return EXCLAVES_BOOT_STAGE_FAILED; |
406 | |
407 | default: |
408 | panic("unknown boot status: %u" , status); |
409 | } |
410 | } |
411 | |
412 | #else /* CONFIG_EXCLAVES */ |
413 | |
414 | exclaves_status_t |
415 | exclaves_get_status(void) |
416 | { |
417 | return EXCLAVES_STATUS_NOT_SUPPORTED; |
418 | } |
419 | |
420 | |
421 | exclaves_boot_stage_t |
422 | exclaves_get_boot_stage(void) |
423 | { |
424 | return EXCLAVES_BOOT_STAGE_NONE; |
425 | } |
426 | |
427 | #endif /* CONFIG_EXCLAVES */ |
428 | |