1/*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23/*
24 *
25 * File: vm/vm_shared_region.h
26 *
27 * protos and struct definitions for shared region
28 */
29
30#ifndef _VM_SHARED_REGION_H_
31#define _VM_SHARED_REGION_H_
32
33#ifdef KERNEL_PRIVATE
34
35#include <mach/vm_prot.h>
36#include <mach/mach_types.h>
37#include <mach/shared_region.h>
38
39#include <kern/kern_types.h>
40#include <kern/macro_help.h>
41
42#include <vm/vm_map.h>
43
44extern int shared_region_version;
45extern int shared_region_persistence;
46
47#if DEBUG
48extern int shared_region_debug;
49#define SHARED_REGION_DEBUG(args) \
50 MACRO_BEGIN \
51 if (shared_region_debug) { \
52 kprintf args; \
53 } \
54 MACRO_END
55#else /* DEBUG */
56#define SHARED_REGION_DEBUG(args)
57#endif /* DEBUG */
58
59extern int shared_region_trace_level;
60
61extern struct vm_shared_region *primary_system_shared_region;
62
63#define SHARED_REGION_TRACE_NONE_LVL 0 /* no trace */
64#define SHARED_REGION_TRACE_ERROR_LVL 1 /* trace abnormal events */
65#define SHARED_REGION_TRACE_INFO_LVL 2 /* trace all events */
66#define SHARED_REGION_TRACE_DEBUG_LVL 3 /* extra traces for debug */
67#define SHARED_REGION_TRACE(level, args) \
68 MACRO_BEGIN \
69 if (shared_region_trace_level >= level) { \
70 printf args; \
71 } \
72 MACRO_END
73#define SHARED_REGION_TRACE_NONE(args)
74#define SHARED_REGION_TRACE_ERROR(args) \
75 MACRO_BEGIN \
76 SHARED_REGION_TRACE(SHARED_REGION_TRACE_ERROR_LVL, \
77 args); \
78 MACRO_END
79#define SHARED_REGION_TRACE_INFO(args) \
80 MACRO_BEGIN \
81 SHARED_REGION_TRACE(SHARED_REGION_TRACE_INFO_LVL, \
82 args); \
83 MACRO_END
84#define SHARED_REGION_TRACE_DEBUG(args) \
85 MACRO_BEGIN \
86 SHARED_REGION_TRACE(SHARED_REGION_TRACE_DEBUG_LVL, \
87 args); \
88 MACRO_END
89
90typedef struct vm_shared_region *vm_shared_region_t;
91
92#ifdef MACH_KERNEL_PRIVATE
93
94#include <kern/queue.h>
95#include <vm/vm_object.h>
96#include <vm/memory_object.h>
97
98#define PAGE_SIZE_FOR_SR_SLIDE 4096
99#define PAGE_SIZE_FOR_SR_SLIDE_16KB 16384
100
101/*
102 * Documentation for the slide info format can be found in the dyld project in
103 * the file 'launch-cache/dyld_cache_format.h'.
104 */
105
106typedef struct vm_shared_region_slide_info_entry_v1 *vm_shared_region_slide_info_entry_v1_t;
107struct vm_shared_region_slide_info_entry_v1 {
108 uint32_t version;
109 uint32_t toc_offset; // offset from start of header to table-of-contents
110 uint32_t toc_count; // number of entries in toc (same as number of pages in r/w mapping)
111 uint32_t entry_offset;
112 uint32_t entry_count;
113 uint32_t entries_size;
114 // uint16_t toc[toc_count];
115 // entrybitmap entries[entries_count];
116};
117
118#define NUM_SLIDING_BITMAPS_PER_PAGE (0x1000/sizeof(int)/8) /*128*/
119typedef struct slide_info_entry_toc *slide_info_entry_toc_t;
120struct slide_info_entry_toc {
121 uint8_t entry[NUM_SLIDING_BITMAPS_PER_PAGE];
122};
123
124typedef struct vm_shared_region_slide_info_entry_v2 *vm_shared_region_slide_info_entry_v2_t;
125struct vm_shared_region_slide_info_entry_v2 {
126 uint32_t version;
127 uint32_t page_size;
128 uint32_t page_starts_offset;
129 uint32_t page_starts_count;
130 uint32_t page_extras_offset;
131 uint32_t page_extras_count;
132 uint64_t delta_mask; // which (contiguous) set of bits contains the delta to the next rebase location
133 uint64_t value_add;
134 // uint16_t page_starts[page_starts_count];
135 // uint16_t page_extras[page_extras_count];
136};
137
138#define DYLD_CACHE_SLIDE_PAGE_ATTRS 0xC000 // high bits of uint16_t are flags
139#define DYLD_CACHE_SLIDE_PAGE_ATTR_EXTRA 0x8000 // index is into extras array (not starts array)
140#define DYLD_CACHE_SLIDE_PAGE_ATTR_NO_REBASE 0x4000 // page has no rebasing
141#define DYLD_CACHE_SLIDE_PAGE_ATTR_END 0x8000 // last chain entry for page
142#define DYLD_CACHE_SLIDE_PAGE_VALUE 0x3FFF // bitwise negation of DYLD_CACHE_SLIDE_PAGE_ATTRS
143#define DYLD_CACHE_SLIDE_PAGE_OFFSET_SHIFT 2
144
145typedef struct vm_shared_region_slide_info_entry_v3 *vm_shared_region_slide_info_entry_v3_t;
146struct vm_shared_region_slide_info_entry_v3 {
147 uint32_t version; // currently 3
148 uint32_t page_size; // currently 4096 (may also be 16384)
149 uint32_t page_starts_count;
150 uint64_t value_add;
151 uint16_t page_starts[] /* page_starts_count */;
152};
153
154#define DYLD_CACHE_SLIDE_V3_PAGE_ATTR_NO_REBASE 0xFFFF // page has no rebasing
155
156
157typedef struct vm_shared_region_slide_info_entry_v4 *vm_shared_region_slide_info_entry_v4_t;
158struct vm_shared_region_slide_info_entry_v4 {
159 uint32_t version; // currently 4
160 uint32_t page_size; // currently 4096 (may also be 16384)
161 uint32_t page_starts_offset;
162 uint32_t page_starts_count;
163 uint32_t page_extras_offset;
164 uint32_t page_extras_count;
165 uint64_t delta_mask; // which (contiguous) set of bits contains the delta to the next rebase location (0xC0000000)
166 uint64_t value_add; // base address of cache
167 // uint16_t page_starts[page_starts_count];
168 // uint16_t page_extras[page_extras_count];
169};
170
171#define DYLD_CACHE_SLIDE4_PAGE_NO_REBASE 0xFFFF // page has no rebasing
172#define DYLD_CACHE_SLIDE4_PAGE_INDEX 0x7FFF // index into starts or extras
173#define DYLD_CACHE_SLIDE4_PAGE_USE_EXTRA 0x8000 // index is into extras array (not starts array)
174#define DYLD_CACHE_SLIDE4_PAGE_EXTRA_END 0x8000 // last chain entry for page
175
176
177typedef struct vm_shared_region_slide_info_entry_v5 *vm_shared_region_slide_info_entry_v5_t;
178struct vm_shared_region_slide_info_entry_v5 {
179 uint32_t version; // currently 5
180 uint32_t page_size; // 16384
181 uint32_t page_starts_count;
182 uint64_t value_add;
183 uint16_t page_starts[] /* page_starts_count */;
184};
185
186#define DYLD_CACHE_SLIDE_V5_PAGE_ATTR_NO_REBASE 0xFFFF // page has no rebasing
187
188
189typedef union vm_shared_region_slide_info_entry *vm_shared_region_slide_info_entry_t;
190union vm_shared_region_slide_info_entry {
191 struct {
192 uint32_t version;
193 uint32_t page_size; // only valid to use this on version >= 2
194 };
195 struct vm_shared_region_slide_info_entry_v1 v1;
196 struct vm_shared_region_slide_info_entry_v2 v2;
197 struct vm_shared_region_slide_info_entry_v3 v3;
198 struct vm_shared_region_slide_info_entry_v4 v4;
199 struct vm_shared_region_slide_info_entry_v5 v5;
200};
201
202#define MIN_SLIDE_INFO_SIZE \
203 MIN(sizeof(struct vm_shared_region_slide_info_entry_v1), \
204 MIN(sizeof(struct vm_shared_region_slide_info_entry_v2), \
205 MIN(sizeof(struct vm_shared_region_slide_info_entry_v3), \
206 MIN(sizeof(struct vm_shared_region_slide_info_entry_v4), \
207 sizeof(struct vm_shared_region_slide_info_entry_v5)))))
208
209/*
210 * This is the information used by the shared cache pager for sub-sections
211 * which must be modified for relocations and/or pointer authentications
212 * before it can be used. The shared_region_pager gets source pages from
213 * the shared cache file and modifies them -- see shared_region_pager_data_request().
214 *
215 * A single pager may be used from multiple shared regions provided:
216 * - same si_slide_object, si_start, si_end, si_slide, si_ptrauth and si_jop_key
217 * - The size and contents of si_slide_info_entry are the same.
218 */
219typedef struct vm_shared_region_slide_info {
220 uint32_t si_slide; /* the distance that the file data is relocated */
221 bool si_slid;
222#if __has_feature(ptrauth_calls)
223 bool si_ptrauth;
224 uint64_t si_jop_key;
225 struct vm_shared_region *si_shared_region; /* so we can ref/dealloc for authenticated slide info */
226#endif /* __has_feature(ptrauth_calls) */
227 mach_vm_address_t si_slid_address __kernel_data_semantics;
228 mach_vm_offset_t si_start __kernel_data_semantics; /* start offset in si_slide_object */
229 mach_vm_offset_t si_end __kernel_data_semantics;
230 vm_object_t si_slide_object; /* The source object for the pages to be modified */
231 mach_vm_size_t si_slide_info_size; /* size of dyld provided relocation information */
232 vm_shared_region_slide_info_entry_t si_slide_info_entry; /* dyld provided relocation information */
233} *vm_shared_region_slide_info_t;
234
235/*
236 * Data structure that represents a unique shared cache region.
237 */
238struct vm_shared_region {
239 uint32_t sr_ref_count;
240 uint32_t sr_slide;
241 queue_chain_t sr_q;
242 void *sr_root_dir;
243 cpu_type_t sr_cpu_type;
244 cpu_subtype_t sr_cpu_subtype;
245 ipc_port_t sr_mem_entry;
246 mach_vm_offset_t sr_first_mapping;
247 mach_vm_offset_t sr_base_address;
248 mach_vm_size_t sr_size;
249 mach_vm_offset_t sr_pmap_nesting_start;
250 mach_vm_size_t sr_pmap_nesting_size;
251 thread_call_t sr_timer_call;
252 uuid_t sr_uuid;
253
254#if __ARM_MIXED_PAGE_SIZE__
255 uint8_t sr_page_shift;
256#endif /* __ARM_MIXED_PAGE_SIZE__ */
257 thread_t sr_mapping_in_progress; /* sr_first_mapping will be != -1 when done */
258 thread_t sr_slide_in_progress;
259 bool sr_64bit;
260 bool sr_persists;
261 bool sr_uuid_copied;
262 bool sr_stale; /* This region should never be used again. */
263 bool sr_driverkit;
264
265#if __has_feature(ptrauth_calls)
266 bool sr_reslide; /* Special shared region for suspected attacked processes */
267 uint_t sr_num_auth_section; /* num entries in sr_auth_section */
268 uint_t sr_next_auth_section; /* used while filling in sr_auth_section */
269 vm_shared_region_slide_info_t *sr_auth_section;
270#endif /* __has_feature(ptrauth_calls) */
271
272 uint32_t sr_rsr_version;
273
274 uint64_t sr_install_time; /* mach_absolute_time() of installation into global list */
275 uint32_t sr_id; /* identifier for shared cache */
276 uint32_t sr_images_count;
277 struct dyld_uuid_info_64 *sr_images;
278};
279
280extern kern_return_t vm_shared_region_slide_page(
281 vm_shared_region_slide_info_t si,
282 vm_offset_t vaddr,
283 mach_vm_offset_t uservaddr,
284 uint32_t pageIndex,
285 uint64_t jop_key);
286extern uint64_t shared_region_find_key(char *shared_region_id);
287#else /* !MACH_KERNEL_PRIVATE */
288
289struct vm_shared_region;
290struct vm_shared_region_slide_info;
291struct vm_shared_region_slide_info_entry;
292struct slide_info_entry_toc;
293
294#endif /* MACH_KERNEL_PRIVATE */
295
296struct _sr_file_mappings {
297 int fd;
298 uint32_t mappings_count;
299 struct shared_file_mapping_slide_np *mappings;
300 uint32_t slide;
301 struct fileproc *fp;
302 struct vnode *vp;
303 memory_object_size_t file_size;
304 memory_object_control_t file_control;
305};
306
307extern void vm_shared_region_init(void);
308extern kern_return_t vm_shared_region_enter(
309 struct _vm_map *map,
310 struct task *task,
311 boolean_t is_64bit,
312 void *fsroot,
313 cpu_type_t cpu,
314 cpu_subtype_t cpu_subtype,
315 boolean_t reslide,
316 boolean_t is_driverkit,
317 uint32_t rsr_version);
318extern void vm_shared_region_remove(
319 struct task *task,
320 struct vm_shared_region *sr);
321extern vm_shared_region_t vm_shared_region_get(
322 struct task *task);
323extern vm_shared_region_t vm_shared_region_trim_and_get(
324 struct task *task);
325extern void vm_shared_region_deallocate(
326 struct vm_shared_region *shared_region);
327extern vm_map_t vm_shared_region_vm_map(
328 struct vm_shared_region *shared_region);
329extern void vm_shared_region_set(
330 struct task *task,
331 struct vm_shared_region *new_shared_region);
332extern vm_shared_region_t vm_shared_region_lookup(
333 void *root_dir,
334 cpu_type_t cpu,
335 cpu_subtype_t cpu_subtype,
336 boolean_t is_64bit,
337 int target_page_shift,
338 boolean_t reslide,
339 boolean_t is_driverkit,
340 uint32_t rsr_version);
341extern kern_return_t vm_shared_region_start_address(
342 struct vm_shared_region *shared_region,
343 mach_vm_offset_t *start_address,
344 task_t task);
345extern void vm_shared_region_undo_mappings(
346 vm_map_t sr_map,
347 mach_vm_offset_t sr_base_address,
348 struct _sr_file_mappings *srf_mappings,
349 struct _sr_file_mappings *srf_mappings_count,
350 unsigned int mappings_count);
351__attribute__((noinline))
352extern kern_return_t vm_shared_region_map_file(
353 struct vm_shared_region *shared_region,
354 int sr_mappings_count,
355 struct _sr_file_mappings *sr_mappings);
356extern void *vm_shared_region_root_dir(
357 struct vm_shared_region *shared_region);
358extern kern_return_t vm_shared_region_sliding_valid(uint32_t slide);
359extern void vm_commpage_init(void);
360extern void vm_commpage_text_init(void);
361extern kern_return_t vm_commpage_enter(
362 struct _vm_map *map,
363 struct task *task,
364 boolean_t is64bit);
365extern kern_return_t vm_commpage_remove(
366 struct _vm_map *map,
367 struct task *task);
368int vm_shared_region_slide(uint32_t,
369 mach_vm_offset_t,
370 mach_vm_size_t,
371 mach_vm_offset_t,
372 mach_vm_size_t,
373 mach_vm_offset_t,
374 memory_object_control_t,
375 vm_prot_t);
376extern void vm_shared_region_pivot(void);
377extern void vm_shared_region_reslide_stale(boolean_t driverkit);
378#if __has_feature(ptrauth_calls)
379__attribute__((noinline))
380extern kern_return_t vm_shared_region_auth_remap(vm_shared_region_t sr);
381#endif /* __has_feature(ptrauth_calls) */
382extern void vm_shared_region_reference(vm_shared_region_t sr);
383
384#endif /* KERNEL_PRIVATE */
385
386#endif /* _VM_SHARED_REGION_H_ */
387