1/*
2 * Copyright (c) 2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#if CONFIG_EXCLAVES
30
31#include <vm/vm_page.h>
32#include <vm/vm_pageout.h>
33#include <libkern/coreanalytics/coreanalytics.h>
34#include <kern/ledger.h>
35
36#include "exclaves_memory.h"
37
38/* -------------------------------------------------------------------------- */
39#pragma mark Accounting
40
41typedef struct {
42 _Atomic uint64_t pages_alloced;
43 _Atomic uint64_t pages_freed;
44 _Atomic uint64_t time_allocating;
45 _Atomic uint64_t max_alloc_latency;
46 _Atomic uint64_t alloc_latency_byhighbit[16];// highbit(MCT end - MCT start)/4
47} exclaves_allocation_statistics_t;
48
49exclaves_allocation_statistics_t exclaves_allocation_statistics;
50
51CA_EVENT(ca_exclaves_allocation_statistics,
52 CA_INT, pages_alloced,
53 CA_INT, pages_freed,
54 CA_INT, time_allocating,
55 CA_INT, max_alloc_latency,
56 CA_INT, alloc_latency_highbit0,
57 CA_INT, alloc_latency_highbit1,
58 CA_INT, alloc_latency_highbit2,
59 CA_INT, alloc_latency_highbit3,
60 CA_INT, alloc_latency_highbit4,
61 CA_INT, alloc_latency_highbit5,
62 CA_INT, alloc_latency_highbit6,
63 CA_INT, alloc_latency_highbit7,
64 CA_INT, alloc_latency_highbit8,
65 CA_INT, alloc_latency_highbit9,
66 CA_INT, alloc_latency_highbit10,
67 CA_INT, alloc_latency_highbit11,
68 CA_INT, alloc_latency_highbit12,
69 CA_INT, alloc_latency_highbit13,
70 CA_INT, alloc_latency_highbit14,
71 CA_INT, alloc_latency_highbit15);
72
73void
74exclaves_memory_report_accounting(void)
75{
76 ca_event_t event = CA_EVENT_ALLOCATE(ca_exclaves_allocation_statistics);
77 CA_EVENT_TYPE(ca_exclaves_allocation_statistics) * e = event->data;
78
79 e->pages_alloced = os_atomic_load(&exclaves_allocation_statistics.pages_alloced, relaxed);
80 e->pages_freed = os_atomic_load(&exclaves_allocation_statistics.pages_freed, relaxed);
81 e->time_allocating = os_atomic_load(&exclaves_allocation_statistics.time_allocating, relaxed);
82 e->max_alloc_latency = os_atomic_load(&exclaves_allocation_statistics.max_alloc_latency, relaxed);
83 e->alloc_latency_highbit0 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[0], relaxed);
84 e->alloc_latency_highbit1 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[1], relaxed);
85 e->alloc_latency_highbit2 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[2], relaxed);
86 e->alloc_latency_highbit3 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[3], relaxed);
87 e->alloc_latency_highbit4 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[4], relaxed);
88 e->alloc_latency_highbit5 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[5], relaxed);
89 e->alloc_latency_highbit6 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[6], relaxed);
90 e->alloc_latency_highbit7 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[7], relaxed);
91 e->alloc_latency_highbit8 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[8], relaxed);
92 e->alloc_latency_highbit9 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[9], relaxed);
93 e->alloc_latency_highbit10 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[10], relaxed);
94 e->alloc_latency_highbit11 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[11], relaxed);
95 e->alloc_latency_highbit12 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[12], relaxed);
96 e->alloc_latency_highbit13 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[13], relaxed);
97 e->alloc_latency_highbit14 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[14], relaxed);
98 e->alloc_latency_highbit15 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[15], relaxed);
99
100 CA_EVENT_SEND(event);
101}
102
103static ledger_t
104get_conclave_mem_ledger(xnuupcalls_pagekind_s kind)
105{
106 ledger_t ledger;
107 switch (kind) {
108 case XNUUPCALLS_PAGEKIND_ROOTDOMAIN:
109 ledger = kernel_task->ledger;
110 break;
111 case XNUUPCALLS_PAGEKIND_CONCLAVE:
112 if (current_thread()->conclave_stop_task != NULL) {
113 ledger = current_thread()->conclave_stop_task->ledger;
114 } else {
115 ledger = current_task()->ledger;
116 }
117 break;
118 default:
119 panic("Conclave Memory ledger doesn't recognize pagekind");
120 break;
121 }
122 return ledger;
123}
124
125
126/* -------------------------------------------------------------------------- */
127#pragma mark Allocation/Free
128
129void
130exclaves_memory_alloc(const uint32_t npages, uint32_t *pages, const xnuupcalls_pagekind_s kind)
131{
132 uint32_t pages_left = npages;
133 vm_page_t page_list = NULL;
134 vm_page_t sequestered = NULL;
135 unsigned p = 0;
136
137 uint64_t start_time = mach_continuous_approximate_time();
138
139 while (pages_left) {
140 vm_page_t next;
141 vm_page_alloc_list(npages, KMA_ZERO | KMA_NOFAIL, &page_list);
142
143 vm_object_lock(exclaves_object);
144 for (vm_page_t mem = page_list; mem != VM_PAGE_NULL; mem = next) {
145 next = mem->vmp_snext;
146 if (vm_page_created(mem)) {
147 // avoid ml_static_mfree() pages due to 117505258
148 mem->vmp_snext = sequestered;
149 sequestered = mem;
150 continue;
151 }
152 mem->vmp_snext = NULL;
153
154 vm_page_lock_queues();
155 vm_page_wire(mem, VM_KERN_MEMORY_EXCLAVES, FALSE);
156 vm_page_unlock_queues();
157 /* Insert the page into the exclaves object */
158 vm_page_insert_wired(mem, exclaves_object,
159 ptoa(VM_PAGE_GET_PHYS_PAGE(mem)),
160 VM_KERN_MEMORY_EXCLAVES);
161
162 /* Retype via SPTM to SK owned */
163 sptm_retype_params_t retype_params = {
164 .raw = SPTM_RETYPE_PARAMS_NULL
165 };
166 sptm_retype(ptoa(VM_PAGE_GET_PHYS_PAGE(mem)),
167 XNU_DEFAULT, SK_DEFAULT, retype_params);
168
169 pages[p++] = VM_PAGE_GET_PHYS_PAGE(mem);
170 pages_left--;
171 }
172 vm_object_unlock(exclaves_object);
173 }
174
175 vm_page_free_list(sequestered, FALSE);
176
177 uint64_t elapsed_time = mach_continuous_approximate_time() - start_time;
178
179 os_atomic_add(&exclaves_allocation_statistics.pages_alloced, npages, relaxed);
180 os_atomic_add(&exclaves_allocation_statistics.time_allocating, elapsed_time, relaxed);
181 os_atomic_max(&exclaves_allocation_statistics.max_alloc_latency, elapsed_time, relaxed);
182 os_atomic_add(&exclaves_allocation_statistics.alloc_latency_byhighbit[ffsll(elapsed_time) / 4], elapsed_time, relaxed);
183
184 ledger_t ledger = get_conclave_mem_ledger(kind);
185 kern_return_t ledger_ret = ledger_credit(ledger,
186 task_ledgers.conclave_mem,
187 (ledger_amount_t) npages);
188 if (ledger_ret != KERN_SUCCESS) {
189 panic("Ledger credit failed. count %u error code %d",
190 npages,
191 ledger_ret);
192 }
193}
194
195void
196exclaves_memory_free(const uint32_t npages, const uint32_t *pages, const xnuupcalls_pagekind_s kind)
197{
198 vm_object_lock(exclaves_object);
199 for (size_t p = 0; p < npages; p++) {
200 /* Find the page in the exclaves object. */
201 vm_page_t m;
202 m = vm_page_lookup(exclaves_object, ptoa(pages[p]));
203
204 /* Assert we found the page */
205 assert(m != VM_PAGE_NULL);
206
207 /* Via SPTM, verify the page type is something ownable by xnu. */
208 assert3u(sptm_get_frame_type(ptoa(VM_PAGE_GET_PHYS_PAGE(m))),
209 ==, XNU_DEFAULT);
210
211 /* Free the page */
212 vm_page_lock_queues();
213 vm_page_free(m);
214 vm_page_unlock_queues();
215 }
216 vm_object_unlock(exclaves_object);
217
218 os_atomic_add(&exclaves_allocation_statistics.pages_freed, npages, relaxed);
219
220 ledger_t ledger = get_conclave_mem_ledger(kind);
221 kern_return_t ledger_ret = ledger_debit(ledger,
222 task_ledgers.conclave_mem,
223 (ledger_amount_t) npages);
224 if (ledger_ret != KERN_SUCCESS) {
225 panic("Ledger debit failed. count %u error code %d",
226 npages,
227 ledger_ret);
228 }
229}
230
231
232/* -------------------------------------------------------------------------- */
233#pragma mark Upcalls
234
235tb_error_t
236exclaves_memory_upcall_alloc(uint32_t npages, xnuupcalls_pagekind_s kind,
237 tb_error_t (^completion)(xnuupcalls_pagelist_s))
238{
239 xnuupcalls_pagelist_s pagelist = {};
240
241 assert3u(npages, <=, ARRAY_COUNT(pagelist.pages));
242 if (npages > ARRAY_COUNT(pagelist.pages)) {
243 panic("npages");
244 }
245
246 exclaves_memory_alloc(npages, pagelist.pages, kind);
247 return completion(pagelist);
248}
249
250
251tb_error_t
252exclaves_memory_upcall_free(const uint32_t pages[EXCLAVES_MEMORY_MAX_REQUEST],
253 uint32_t npages, const xnuupcalls_pagekind_s kind,
254 tb_error_t (^completion)(void))
255{
256 /* Get pointer for page list paddr */
257 assert(npages <= EXCLAVES_MEMORY_MAX_REQUEST);
258 if (npages > EXCLAVES_MEMORY_MAX_REQUEST) {
259 panic("npages");
260 }
261
262 exclaves_memory_free(npages, pages, kind);
263
264 return completion();
265}
266
267#endif /* CONFIG_EXCLAVES */
268