1 | /* |
2 | * Copyright (c) 2024 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #if CONFIG_EXCLAVES |
30 | |
31 | #include <vm/vm_page.h> |
32 | #include <vm/vm_pageout.h> |
33 | #include <libkern/coreanalytics/coreanalytics.h> |
34 | #include <kern/ledger.h> |
35 | |
36 | #include "exclaves_memory.h" |
37 | |
38 | /* -------------------------------------------------------------------------- */ |
39 | #pragma mark Accounting |
40 | |
41 | typedef struct { |
42 | _Atomic uint64_t pages_alloced; |
43 | _Atomic uint64_t pages_freed; |
44 | _Atomic uint64_t time_allocating; |
45 | _Atomic uint64_t max_alloc_latency; |
46 | _Atomic uint64_t alloc_latency_byhighbit[16];// highbit(MCT end - MCT start)/4 |
47 | } exclaves_allocation_statistics_t; |
48 | |
49 | exclaves_allocation_statistics_t exclaves_allocation_statistics; |
50 | |
51 | CA_EVENT(ca_exclaves_allocation_statistics, |
52 | CA_INT, pages_alloced, |
53 | CA_INT, pages_freed, |
54 | CA_INT, time_allocating, |
55 | CA_INT, max_alloc_latency, |
56 | CA_INT, alloc_latency_highbit0, |
57 | CA_INT, alloc_latency_highbit1, |
58 | CA_INT, alloc_latency_highbit2, |
59 | CA_INT, alloc_latency_highbit3, |
60 | CA_INT, alloc_latency_highbit4, |
61 | CA_INT, alloc_latency_highbit5, |
62 | CA_INT, alloc_latency_highbit6, |
63 | CA_INT, alloc_latency_highbit7, |
64 | CA_INT, alloc_latency_highbit8, |
65 | CA_INT, alloc_latency_highbit9, |
66 | CA_INT, alloc_latency_highbit10, |
67 | CA_INT, alloc_latency_highbit11, |
68 | CA_INT, alloc_latency_highbit12, |
69 | CA_INT, alloc_latency_highbit13, |
70 | CA_INT, alloc_latency_highbit14, |
71 | CA_INT, alloc_latency_highbit15); |
72 | |
73 | void |
74 | exclaves_memory_report_accounting(void) |
75 | { |
76 | ca_event_t event = CA_EVENT_ALLOCATE(ca_exclaves_allocation_statistics); |
77 | CA_EVENT_TYPE(ca_exclaves_allocation_statistics) * e = event->data; |
78 | |
79 | e->pages_alloced = os_atomic_load(&exclaves_allocation_statistics.pages_alloced, relaxed); |
80 | e->pages_freed = os_atomic_load(&exclaves_allocation_statistics.pages_freed, relaxed); |
81 | e->time_allocating = os_atomic_load(&exclaves_allocation_statistics.time_allocating, relaxed); |
82 | e->max_alloc_latency = os_atomic_load(&exclaves_allocation_statistics.max_alloc_latency, relaxed); |
83 | e->alloc_latency_highbit0 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[0], relaxed); |
84 | e->alloc_latency_highbit1 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[1], relaxed); |
85 | e->alloc_latency_highbit2 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[2], relaxed); |
86 | e->alloc_latency_highbit3 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[3], relaxed); |
87 | e->alloc_latency_highbit4 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[4], relaxed); |
88 | e->alloc_latency_highbit5 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[5], relaxed); |
89 | e->alloc_latency_highbit6 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[6], relaxed); |
90 | e->alloc_latency_highbit7 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[7], relaxed); |
91 | e->alloc_latency_highbit8 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[8], relaxed); |
92 | e->alloc_latency_highbit9 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[9], relaxed); |
93 | e->alloc_latency_highbit10 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[10], relaxed); |
94 | e->alloc_latency_highbit11 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[11], relaxed); |
95 | e->alloc_latency_highbit12 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[12], relaxed); |
96 | e->alloc_latency_highbit13 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[13], relaxed); |
97 | e->alloc_latency_highbit14 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[14], relaxed); |
98 | e->alloc_latency_highbit15 = os_atomic_load(&exclaves_allocation_statistics.alloc_latency_byhighbit[15], relaxed); |
99 | |
100 | CA_EVENT_SEND(event); |
101 | } |
102 | |
103 | static ledger_t |
104 | get_conclave_mem_ledger(xnuupcalls_pagekind_s kind) |
105 | { |
106 | ledger_t ledger; |
107 | switch (kind) { |
108 | case XNUUPCALLS_PAGEKIND_ROOTDOMAIN: |
109 | ledger = kernel_task->ledger; |
110 | break; |
111 | case XNUUPCALLS_PAGEKIND_CONCLAVE: |
112 | if (current_thread()->conclave_stop_task != NULL) { |
113 | ledger = current_thread()->conclave_stop_task->ledger; |
114 | } else { |
115 | ledger = current_task()->ledger; |
116 | } |
117 | break; |
118 | default: |
119 | panic("Conclave Memory ledger doesn't recognize pagekind" ); |
120 | break; |
121 | } |
122 | return ledger; |
123 | } |
124 | |
125 | |
126 | /* -------------------------------------------------------------------------- */ |
127 | #pragma mark Allocation/Free |
128 | |
129 | void |
130 | exclaves_memory_alloc(const uint32_t npages, uint32_t *pages, const xnuupcalls_pagekind_s kind) |
131 | { |
132 | uint32_t pages_left = npages; |
133 | vm_page_t page_list = NULL; |
134 | vm_page_t sequestered = NULL; |
135 | unsigned p = 0; |
136 | |
137 | uint64_t start_time = mach_continuous_approximate_time(); |
138 | |
139 | while (pages_left) { |
140 | vm_page_t next; |
141 | vm_page_alloc_list(npages, KMA_ZERO | KMA_NOFAIL, &page_list); |
142 | |
143 | vm_object_lock(exclaves_object); |
144 | for (vm_page_t mem = page_list; mem != VM_PAGE_NULL; mem = next) { |
145 | next = mem->vmp_snext; |
146 | if (vm_page_created(mem)) { |
147 | // avoid ml_static_mfree() pages due to 117505258 |
148 | mem->vmp_snext = sequestered; |
149 | sequestered = mem; |
150 | continue; |
151 | } |
152 | mem->vmp_snext = NULL; |
153 | |
154 | vm_page_lock_queues(); |
155 | vm_page_wire(mem, VM_KERN_MEMORY_EXCLAVES, FALSE); |
156 | vm_page_unlock_queues(); |
157 | /* Insert the page into the exclaves object */ |
158 | vm_page_insert_wired(mem, exclaves_object, |
159 | ptoa(VM_PAGE_GET_PHYS_PAGE(mem)), |
160 | VM_KERN_MEMORY_EXCLAVES); |
161 | |
162 | /* Retype via SPTM to SK owned */ |
163 | sptm_retype_params_t retype_params = { |
164 | .raw = SPTM_RETYPE_PARAMS_NULL |
165 | }; |
166 | sptm_retype(ptoa(VM_PAGE_GET_PHYS_PAGE(mem)), |
167 | XNU_DEFAULT, SK_DEFAULT, retype_params); |
168 | |
169 | pages[p++] = VM_PAGE_GET_PHYS_PAGE(mem); |
170 | pages_left--; |
171 | } |
172 | vm_object_unlock(exclaves_object); |
173 | } |
174 | |
175 | vm_page_free_list(sequestered, FALSE); |
176 | |
177 | uint64_t elapsed_time = mach_continuous_approximate_time() - start_time; |
178 | |
179 | os_atomic_add(&exclaves_allocation_statistics.pages_alloced, npages, relaxed); |
180 | os_atomic_add(&exclaves_allocation_statistics.time_allocating, elapsed_time, relaxed); |
181 | os_atomic_max(&exclaves_allocation_statistics.max_alloc_latency, elapsed_time, relaxed); |
182 | os_atomic_add(&exclaves_allocation_statistics.alloc_latency_byhighbit[ffsll(elapsed_time) / 4], elapsed_time, relaxed); |
183 | |
184 | ledger_t ledger = get_conclave_mem_ledger(kind); |
185 | kern_return_t ledger_ret = ledger_credit(ledger, |
186 | task_ledgers.conclave_mem, |
187 | (ledger_amount_t) npages); |
188 | if (ledger_ret != KERN_SUCCESS) { |
189 | panic("Ledger credit failed. count %u error code %d" , |
190 | npages, |
191 | ledger_ret); |
192 | } |
193 | } |
194 | |
195 | void |
196 | exclaves_memory_free(const uint32_t npages, const uint32_t *pages, const xnuupcalls_pagekind_s kind) |
197 | { |
198 | vm_object_lock(exclaves_object); |
199 | for (size_t p = 0; p < npages; p++) { |
200 | /* Find the page in the exclaves object. */ |
201 | vm_page_t m; |
202 | m = vm_page_lookup(exclaves_object, ptoa(pages[p])); |
203 | |
204 | /* Assert we found the page */ |
205 | assert(m != VM_PAGE_NULL); |
206 | |
207 | /* Via SPTM, verify the page type is something ownable by xnu. */ |
208 | assert3u(sptm_get_frame_type(ptoa(VM_PAGE_GET_PHYS_PAGE(m))), |
209 | ==, XNU_DEFAULT); |
210 | |
211 | /* Free the page */ |
212 | vm_page_lock_queues(); |
213 | vm_page_free(m); |
214 | vm_page_unlock_queues(); |
215 | } |
216 | vm_object_unlock(exclaves_object); |
217 | |
218 | os_atomic_add(&exclaves_allocation_statistics.pages_freed, npages, relaxed); |
219 | |
220 | ledger_t ledger = get_conclave_mem_ledger(kind); |
221 | kern_return_t ledger_ret = ledger_debit(ledger, |
222 | task_ledgers.conclave_mem, |
223 | (ledger_amount_t) npages); |
224 | if (ledger_ret != KERN_SUCCESS) { |
225 | panic("Ledger debit failed. count %u error code %d" , |
226 | npages, |
227 | ledger_ret); |
228 | } |
229 | } |
230 | |
231 | |
232 | /* -------------------------------------------------------------------------- */ |
233 | #pragma mark Upcalls |
234 | |
235 | tb_error_t |
236 | exclaves_memory_upcall_alloc(uint32_t npages, xnuupcalls_pagekind_s kind, |
237 | tb_error_t (^completion)(xnuupcalls_pagelist_s)) |
238 | { |
239 | xnuupcalls_pagelist_s pagelist = {}; |
240 | |
241 | assert3u(npages, <=, ARRAY_COUNT(pagelist.pages)); |
242 | if (npages > ARRAY_COUNT(pagelist.pages)) { |
243 | panic("npages" ); |
244 | } |
245 | |
246 | exclaves_memory_alloc(npages, pagelist.pages, kind); |
247 | return completion(pagelist); |
248 | } |
249 | |
250 | |
251 | tb_error_t |
252 | exclaves_memory_upcall_free(const uint32_t pages[EXCLAVES_MEMORY_MAX_REQUEST], |
253 | uint32_t npages, const xnuupcalls_pagekind_s kind, |
254 | tb_error_t (^completion)(void)) |
255 | { |
256 | /* Get pointer for page list paddr */ |
257 | assert(npages <= EXCLAVES_MEMORY_MAX_REQUEST); |
258 | if (npages > EXCLAVES_MEMORY_MAX_REQUEST) { |
259 | panic("npages" ); |
260 | } |
261 | |
262 | exclaves_memory_free(npages, pages, kind); |
263 | |
264 | return completion(); |
265 | } |
266 | |
267 | #endif /* CONFIG_EXCLAVES */ |
268 | |