1 | /* |
2 | * Copyright (c) 2007 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. Please obtain a copy of the License at |
10 | * http://www.opensource.apple.com/apsl/ and read it before using this |
11 | * file. |
12 | * |
13 | * The Original Code and all software distributed under the License are |
14 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
15 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
16 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
18 | * Please see the License for the specific language governing rights and |
19 | * limitations under the License. |
20 | * |
21 | * @APPLE_LICENSE_HEADER_END@ |
22 | */ |
23 | /* |
24 | * |
25 | * File: vm/vm_shared_region.h |
26 | * |
27 | * protos and struct definitions for shared region |
28 | */ |
29 | |
30 | #ifndef _VM_SHARED_REGION_H_ |
31 | #define _VM_SHARED_REGION_H_ |
32 | |
33 | #ifdef KERNEL_PRIVATE |
34 | |
35 | #include <mach/vm_prot.h> |
36 | #include <mach/mach_types.h> |
37 | #include <mach/shared_region.h> |
38 | |
39 | #include <kern/kern_types.h> |
40 | #include <kern/macro_help.h> |
41 | |
42 | #include <vm/vm_map.h> |
43 | |
44 | extern int shared_region_version; |
45 | extern int shared_region_persistence; |
46 | |
47 | #if DEBUG |
48 | extern int shared_region_debug; |
49 | #define SHARED_REGION_DEBUG(args) \ |
50 | MACRO_BEGIN \ |
51 | if (shared_region_debug) { \ |
52 | kprintf args; \ |
53 | } \ |
54 | MACRO_END |
55 | #else /* DEBUG */ |
56 | #define SHARED_REGION_DEBUG(args) |
57 | #endif /* DEBUG */ |
58 | |
59 | extern int shared_region_trace_level; |
60 | |
61 | extern struct vm_shared_region *primary_system_shared_region; |
62 | |
63 | #define SHARED_REGION_TRACE_NONE_LVL 0 /* no trace */ |
64 | #define SHARED_REGION_TRACE_ERROR_LVL 1 /* trace abnormal events */ |
65 | #define SHARED_REGION_TRACE_INFO_LVL 2 /* trace all events */ |
66 | #define SHARED_REGION_TRACE_DEBUG_LVL 3 /* extra traces for debug */ |
67 | #define SHARED_REGION_TRACE(level, args) \ |
68 | MACRO_BEGIN \ |
69 | if (shared_region_trace_level >= level) { \ |
70 | printf args; \ |
71 | } \ |
72 | MACRO_END |
73 | #define SHARED_REGION_TRACE_NONE(args) |
74 | #define SHARED_REGION_TRACE_ERROR(args) \ |
75 | MACRO_BEGIN \ |
76 | SHARED_REGION_TRACE(SHARED_REGION_TRACE_ERROR_LVL, \ |
77 | args); \ |
78 | MACRO_END |
79 | #define SHARED_REGION_TRACE_INFO(args) \ |
80 | MACRO_BEGIN \ |
81 | SHARED_REGION_TRACE(SHARED_REGION_TRACE_INFO_LVL, \ |
82 | args); \ |
83 | MACRO_END |
84 | #define SHARED_REGION_TRACE_DEBUG(args) \ |
85 | MACRO_BEGIN \ |
86 | SHARED_REGION_TRACE(SHARED_REGION_TRACE_DEBUG_LVL, \ |
87 | args); \ |
88 | MACRO_END |
89 | |
90 | typedef struct vm_shared_region *vm_shared_region_t; |
91 | |
92 | #ifdef MACH_KERNEL_PRIVATE |
93 | |
94 | #include <kern/queue.h> |
95 | #include <vm/vm_object.h> |
96 | #include <vm/memory_object.h> |
97 | |
98 | #define PAGE_SIZE_FOR_SR_SLIDE 4096 |
99 | #define PAGE_SIZE_FOR_SR_SLIDE_16KB 16384 |
100 | |
101 | /* |
102 | * Documentation for the slide info format can be found in the dyld project in |
103 | * the file 'launch-cache/dyld_cache_format.h'. |
104 | */ |
105 | |
106 | typedef struct vm_shared_region_slide_info_entry_v1 *vm_shared_region_slide_info_entry_v1_t; |
107 | struct vm_shared_region_slide_info_entry_v1 { |
108 | uint32_t version; |
109 | uint32_t toc_offset; // offset from start of header to table-of-contents |
110 | uint32_t toc_count; // number of entries in toc (same as number of pages in r/w mapping) |
111 | uint32_t entry_offset; |
112 | uint32_t entry_count; |
113 | uint32_t entries_size; |
114 | // uint16_t toc[toc_count]; |
115 | // entrybitmap entries[entries_count]; |
116 | }; |
117 | |
118 | #define NUM_SLIDING_BITMAPS_PER_PAGE (0x1000/sizeof(int)/8) /*128*/ |
119 | typedef struct slide_info_entry_toc *slide_info_entry_toc_t; |
120 | struct slide_info_entry_toc { |
121 | uint8_t entry[NUM_SLIDING_BITMAPS_PER_PAGE]; |
122 | }; |
123 | |
124 | typedef struct vm_shared_region_slide_info_entry_v2 *vm_shared_region_slide_info_entry_v2_t; |
125 | struct vm_shared_region_slide_info_entry_v2 { |
126 | uint32_t version; |
127 | uint32_t page_size; |
128 | uint32_t page_starts_offset; |
129 | uint32_t page_starts_count; |
130 | uint32_t page_extras_offset; |
131 | uint32_t page_extras_count; |
132 | uint64_t delta_mask; // which (contiguous) set of bits contains the delta to the next rebase location |
133 | uint64_t value_add; |
134 | // uint16_t page_starts[page_starts_count]; |
135 | // uint16_t page_extras[page_extras_count]; |
136 | }; |
137 | |
138 | #define DYLD_CACHE_SLIDE_PAGE_ATTRS 0xC000 // high bits of uint16_t are flags |
139 | #define DYLD_CACHE_SLIDE_PAGE_ATTR_EXTRA 0x8000 // index is into extras array (not starts array) |
140 | #define DYLD_CACHE_SLIDE_PAGE_ATTR_NO_REBASE 0x4000 // page has no rebasing |
141 | #define DYLD_CACHE_SLIDE_PAGE_ATTR_END 0x8000 // last chain entry for page |
142 | #define DYLD_CACHE_SLIDE_PAGE_VALUE 0x3FFF // bitwise negation of DYLD_CACHE_SLIDE_PAGE_ATTRS |
143 | #define DYLD_CACHE_SLIDE_PAGE_OFFSET_SHIFT 2 |
144 | |
145 | typedef struct vm_shared_region_slide_info_entry_v3 *vm_shared_region_slide_info_entry_v3_t; |
146 | struct vm_shared_region_slide_info_entry_v3 { |
147 | uint32_t version; // currently 3 |
148 | uint32_t page_size; // currently 4096 (may also be 16384) |
149 | uint32_t page_starts_count; |
150 | uint64_t value_add; |
151 | uint16_t page_starts[] /* page_starts_count */; |
152 | }; |
153 | |
154 | #define DYLD_CACHE_SLIDE_V3_PAGE_ATTR_NO_REBASE 0xFFFF // page has no rebasing |
155 | |
156 | |
157 | typedef struct vm_shared_region_slide_info_entry_v4 *vm_shared_region_slide_info_entry_v4_t; |
158 | struct vm_shared_region_slide_info_entry_v4 { |
159 | uint32_t version; // currently 4 |
160 | uint32_t page_size; // currently 4096 (may also be 16384) |
161 | uint32_t page_starts_offset; |
162 | uint32_t page_starts_count; |
163 | uint32_t page_extras_offset; |
164 | uint32_t page_extras_count; |
165 | uint64_t delta_mask; // which (contiguous) set of bits contains the delta to the next rebase location (0xC0000000) |
166 | uint64_t value_add; // base address of cache |
167 | // uint16_t page_starts[page_starts_count]; |
168 | // uint16_t page_extras[page_extras_count]; |
169 | }; |
170 | |
171 | #define DYLD_CACHE_SLIDE4_PAGE_NO_REBASE 0xFFFF // page has no rebasing |
172 | #define DYLD_CACHE_SLIDE4_PAGE_INDEX 0x7FFF // index into starts or extras |
173 | #define DYLD_CACHE_SLIDE4_PAGE_USE_EXTRA 0x8000 // index is into extras array (not starts array) |
174 | #define DYLD_CACHE_SLIDE4_PAGE_EXTRA_END 0x8000 // last chain entry for page |
175 | |
176 | |
177 | typedef struct vm_shared_region_slide_info_entry_v5 *vm_shared_region_slide_info_entry_v5_t; |
178 | struct vm_shared_region_slide_info_entry_v5 { |
179 | uint32_t version; // currently 5 |
180 | uint32_t page_size; // 16384 |
181 | uint32_t page_starts_count; |
182 | uint64_t value_add; |
183 | uint16_t page_starts[] /* page_starts_count */; |
184 | }; |
185 | |
186 | #define DYLD_CACHE_SLIDE_V5_PAGE_ATTR_NO_REBASE 0xFFFF // page has no rebasing |
187 | |
188 | |
189 | typedef union vm_shared_region_slide_info_entry *vm_shared_region_slide_info_entry_t; |
190 | union vm_shared_region_slide_info_entry { |
191 | struct { |
192 | uint32_t version; |
193 | uint32_t page_size; // only valid to use this on version >= 2 |
194 | }; |
195 | struct vm_shared_region_slide_info_entry_v1 v1; |
196 | struct vm_shared_region_slide_info_entry_v2 v2; |
197 | struct vm_shared_region_slide_info_entry_v3 v3; |
198 | struct vm_shared_region_slide_info_entry_v4 v4; |
199 | struct vm_shared_region_slide_info_entry_v5 v5; |
200 | }; |
201 | |
202 | #define MIN_SLIDE_INFO_SIZE \ |
203 | MIN(sizeof(struct vm_shared_region_slide_info_entry_v1), \ |
204 | MIN(sizeof(struct vm_shared_region_slide_info_entry_v2), \ |
205 | MIN(sizeof(struct vm_shared_region_slide_info_entry_v3), \ |
206 | MIN(sizeof(struct vm_shared_region_slide_info_entry_v4), \ |
207 | sizeof(struct vm_shared_region_slide_info_entry_v5))))) |
208 | |
209 | /* |
210 | * This is the information used by the shared cache pager for sub-sections |
211 | * which must be modified for relocations and/or pointer authentications |
212 | * before it can be used. The shared_region_pager gets source pages from |
213 | * the shared cache file and modifies them -- see shared_region_pager_data_request(). |
214 | * |
215 | * A single pager may be used from multiple shared regions provided: |
216 | * - same si_slide_object, si_start, si_end, si_slide, si_ptrauth and si_jop_key |
217 | * - The size and contents of si_slide_info_entry are the same. |
218 | */ |
219 | typedef struct vm_shared_region_slide_info { |
220 | uint32_t si_slide; /* the distance that the file data is relocated */ |
221 | bool si_slid; |
222 | #if __has_feature(ptrauth_calls) |
223 | bool si_ptrauth; |
224 | uint64_t si_jop_key; |
225 | struct vm_shared_region *si_shared_region; /* so we can ref/dealloc for authenticated slide info */ |
226 | #endif /* __has_feature(ptrauth_calls) */ |
227 | mach_vm_address_t si_slid_address __kernel_data_semantics; |
228 | mach_vm_offset_t si_start __kernel_data_semantics; /* start offset in si_slide_object */ |
229 | mach_vm_offset_t si_end __kernel_data_semantics; |
230 | vm_object_t si_slide_object; /* The source object for the pages to be modified */ |
231 | mach_vm_size_t si_slide_info_size; /* size of dyld provided relocation information */ |
232 | vm_shared_region_slide_info_entry_t si_slide_info_entry; /* dyld provided relocation information */ |
233 | } *vm_shared_region_slide_info_t; |
234 | |
235 | /* |
236 | * Data structure that represents a unique shared cache region. |
237 | */ |
238 | struct vm_shared_region { |
239 | uint32_t sr_ref_count; |
240 | uint32_t sr_slide; |
241 | queue_chain_t sr_q; |
242 | void *sr_root_dir; |
243 | cpu_type_t sr_cpu_type; |
244 | cpu_subtype_t sr_cpu_subtype; |
245 | ipc_port_t sr_mem_entry; |
246 | mach_vm_offset_t sr_first_mapping; |
247 | mach_vm_offset_t sr_base_address; |
248 | mach_vm_size_t sr_size; |
249 | mach_vm_offset_t sr_pmap_nesting_start; |
250 | mach_vm_size_t sr_pmap_nesting_size; |
251 | thread_call_t sr_timer_call; |
252 | uuid_t sr_uuid; |
253 | |
254 | #if __ARM_MIXED_PAGE_SIZE__ |
255 | uint8_t sr_page_shift; |
256 | #endif /* __ARM_MIXED_PAGE_SIZE__ */ |
257 | thread_t sr_mapping_in_progress; /* sr_first_mapping will be != -1 when done */ |
258 | thread_t sr_slide_in_progress; |
259 | bool sr_64bit; |
260 | bool sr_persists; |
261 | bool sr_uuid_copied; |
262 | bool sr_stale; /* This region should never be used again. */ |
263 | bool sr_driverkit; |
264 | |
265 | #if __has_feature(ptrauth_calls) |
266 | bool sr_reslide; /* Special shared region for suspected attacked processes */ |
267 | uint_t sr_num_auth_section; /* num entries in sr_auth_section */ |
268 | uint_t sr_next_auth_section; /* used while filling in sr_auth_section */ |
269 | vm_shared_region_slide_info_t *sr_auth_section; |
270 | #endif /* __has_feature(ptrauth_calls) */ |
271 | |
272 | uint32_t sr_rsr_version; |
273 | |
274 | uint64_t sr_install_time; /* mach_absolute_time() of installation into global list */ |
275 | uint32_t sr_id; /* identifier for shared cache */ |
276 | uint32_t sr_images_count; |
277 | struct dyld_uuid_info_64 *sr_images; |
278 | }; |
279 | |
280 | extern kern_return_t vm_shared_region_slide_page( |
281 | vm_shared_region_slide_info_t si, |
282 | vm_offset_t vaddr, |
283 | mach_vm_offset_t uservaddr, |
284 | uint32_t pageIndex, |
285 | uint64_t jop_key); |
286 | extern uint64_t shared_region_find_key(char *shared_region_id); |
287 | #else /* !MACH_KERNEL_PRIVATE */ |
288 | |
289 | struct vm_shared_region; |
290 | struct vm_shared_region_slide_info; |
291 | struct vm_shared_region_slide_info_entry; |
292 | struct slide_info_entry_toc; |
293 | |
294 | #endif /* MACH_KERNEL_PRIVATE */ |
295 | |
296 | struct _sr_file_mappings { |
297 | int fd; |
298 | uint32_t mappings_count; |
299 | struct shared_file_mapping_slide_np *mappings; |
300 | uint32_t slide; |
301 | struct fileproc *fp; |
302 | struct vnode *vp; |
303 | memory_object_size_t file_size; |
304 | memory_object_control_t file_control; |
305 | }; |
306 | |
307 | extern void vm_shared_region_init(void); |
308 | extern kern_return_t vm_shared_region_enter( |
309 | struct _vm_map *map, |
310 | struct task *task, |
311 | boolean_t is_64bit, |
312 | void *fsroot, |
313 | cpu_type_t cpu, |
314 | cpu_subtype_t cpu_subtype, |
315 | boolean_t reslide, |
316 | boolean_t is_driverkit, |
317 | uint32_t rsr_version); |
318 | extern void vm_shared_region_remove( |
319 | struct task *task, |
320 | struct vm_shared_region *sr); |
321 | extern vm_shared_region_t vm_shared_region_get( |
322 | struct task *task); |
323 | extern vm_shared_region_t vm_shared_region_trim_and_get( |
324 | struct task *task); |
325 | extern void vm_shared_region_deallocate( |
326 | struct vm_shared_region *shared_region); |
327 | extern vm_map_t vm_shared_region_vm_map( |
328 | struct vm_shared_region *shared_region); |
329 | extern void vm_shared_region_set( |
330 | struct task *task, |
331 | struct vm_shared_region *new_shared_region); |
332 | extern vm_shared_region_t vm_shared_region_lookup( |
333 | void *root_dir, |
334 | cpu_type_t cpu, |
335 | cpu_subtype_t cpu_subtype, |
336 | boolean_t is_64bit, |
337 | int target_page_shift, |
338 | boolean_t reslide, |
339 | boolean_t is_driverkit, |
340 | uint32_t rsr_version); |
341 | extern kern_return_t vm_shared_region_start_address( |
342 | struct vm_shared_region *shared_region, |
343 | mach_vm_offset_t *start_address, |
344 | task_t task); |
345 | extern void vm_shared_region_undo_mappings( |
346 | vm_map_t sr_map, |
347 | mach_vm_offset_t sr_base_address, |
348 | struct _sr_file_mappings *srf_mappings, |
349 | struct _sr_file_mappings *srf_mappings_count, |
350 | unsigned int mappings_count); |
351 | __attribute__((noinline)) |
352 | extern kern_return_t vm_shared_region_map_file( |
353 | struct vm_shared_region *shared_region, |
354 | int sr_mappings_count, |
355 | struct _sr_file_mappings *sr_mappings); |
356 | extern void *vm_shared_region_root_dir( |
357 | struct vm_shared_region *shared_region); |
358 | extern kern_return_t vm_shared_region_sliding_valid(uint32_t slide); |
359 | extern void vm_commpage_init(void); |
360 | extern void vm_commpage_text_init(void); |
361 | extern kern_return_t vm_commpage_enter( |
362 | struct _vm_map *map, |
363 | struct task *task, |
364 | boolean_t is64bit); |
365 | extern kern_return_t vm_commpage_remove( |
366 | struct _vm_map *map, |
367 | struct task *task); |
368 | int vm_shared_region_slide(uint32_t, |
369 | mach_vm_offset_t, |
370 | mach_vm_size_t, |
371 | mach_vm_offset_t, |
372 | mach_vm_size_t, |
373 | mach_vm_offset_t, |
374 | memory_object_control_t, |
375 | vm_prot_t); |
376 | extern void vm_shared_region_pivot(void); |
377 | extern void vm_shared_region_reslide_stale(boolean_t driverkit); |
378 | #if __has_feature(ptrauth_calls) |
379 | __attribute__((noinline)) |
380 | extern kern_return_t vm_shared_region_auth_remap(vm_shared_region_t sr); |
381 | #endif /* __has_feature(ptrauth_calls) */ |
382 | extern void vm_shared_region_reference(vm_shared_region_t sr); |
383 | |
384 | #endif /* KERNEL_PRIVATE */ |
385 | |
386 | #endif /* _VM_SHARED_REGION_H_ */ |
387 | |