1 | /* |
2 | * Copyright (c) 2018 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <sys/errno.h> |
30 | |
31 | #include <mach/mach_types.h> |
32 | #include <mach/mach_traps.h> |
33 | #include <mach/host_priv.h> |
34 | #include <mach/kern_return.h> |
35 | #include <mach/memory_object_control.h> |
36 | #include <mach/memory_object_types.h> |
37 | #include <mach/port.h> |
38 | #include <mach/policy.h> |
39 | #include <mach/upl.h> |
40 | #include <mach/thread_act.h> |
41 | #include <mach/mach_vm.h> |
42 | |
43 | #include <kern/host.h> |
44 | #include <kern/kalloc.h> |
45 | #include <kern/queue.h> |
46 | #include <kern/thread.h> |
47 | #include <kern/ipc_kobject.h> |
48 | |
49 | #include <ipc/ipc_port.h> |
50 | #include <ipc/ipc_space.h> |
51 | |
52 | #include <vm/memory_object.h> |
53 | #include <vm/vm_kern.h> |
54 | #include <vm/vm_fault.h> |
55 | #include <vm/vm_map.h> |
56 | #include <vm/vm_pageout.h> |
57 | #include <vm/vm_pageout.h> |
58 | #include <vm/vm_protos.h> |
59 | #include <vm/vm_shared_region.h> |
60 | |
61 | |
62 | /* |
63 | * SHARED REGION MEMORY PAGER |
64 | * |
65 | * This external memory manager (EMM) handles mappings of a dyld shared cache |
66 | * in shared regions, applying any necessary modifications (sliding, |
67 | * pointer signing, ...). |
68 | * |
69 | * It mostly handles page-in requests (from memory_object_data_request()) by |
70 | * getting the original data from its backing VM object, itself backed by |
71 | * the dyld shared cache file, modifying it if needed and providing it to VM. |
72 | * |
73 | * The modified pages will never be dirtied, so the memory manager doesn't |
74 | * need to handle page-out requests (from memory_object_data_return()). The |
75 | * pages need to be mapped copy-on-write, so that the originals stay clean. |
76 | * |
77 | * We don't expect to have to handle a large number of shared cache files, |
78 | * so the data structures are very simple (simple linked list) for now. |
79 | */ |
80 | |
81 | /* forward declarations */ |
82 | void shared_region_pager_reference(memory_object_t mem_obj); |
83 | void shared_region_pager_deallocate(memory_object_t mem_obj); |
84 | kern_return_t shared_region_pager_init(memory_object_t mem_obj, |
85 | memory_object_control_t control, |
86 | memory_object_cluster_size_t pg_size); |
87 | kern_return_t shared_region_pager_terminate(memory_object_t mem_obj); |
88 | kern_return_t shared_region_pager_data_request(memory_object_t mem_obj, |
89 | memory_object_offset_t offset, |
90 | memory_object_cluster_size_t length, |
91 | vm_prot_t protection_required, |
92 | memory_object_fault_info_t fault_info); |
93 | kern_return_t shared_region_pager_data_return(memory_object_t mem_obj, |
94 | memory_object_offset_t offset, |
95 | memory_object_cluster_size_t data_cnt, |
96 | memory_object_offset_t *resid_offset, |
97 | int *io_error, |
98 | boolean_t dirty, |
99 | boolean_t kernel_copy, |
100 | int upl_flags); |
101 | kern_return_t shared_region_pager_data_initialize(memory_object_t mem_obj, |
102 | memory_object_offset_t offset, |
103 | memory_object_cluster_size_t data_cnt); |
104 | kern_return_t shared_region_pager_data_unlock(memory_object_t mem_obj, |
105 | memory_object_offset_t offset, |
106 | memory_object_size_t size, |
107 | vm_prot_t desired_access); |
108 | kern_return_t shared_region_pager_synchronize(memory_object_t mem_obj, |
109 | memory_object_offset_t offset, |
110 | memory_object_size_t length, |
111 | vm_sync_t sync_flags); |
112 | kern_return_t shared_region_pager_map(memory_object_t mem_obj, |
113 | vm_prot_t prot); |
114 | kern_return_t shared_region_pager_last_unmap(memory_object_t mem_obj); |
115 | |
116 | /* |
117 | * Vector of VM operations for this EMM. |
118 | * These routines are invoked by VM via the memory_object_*() interfaces. |
119 | */ |
120 | const struct memory_object_pager_ops = { |
121 | shared_region_pager_reference, |
122 | shared_region_pager_deallocate, |
123 | shared_region_pager_init, |
124 | shared_region_pager_terminate, |
125 | shared_region_pager_data_request, |
126 | shared_region_pager_data_return, |
127 | shared_region_pager_data_initialize, |
128 | shared_region_pager_data_unlock, |
129 | shared_region_pager_synchronize, |
130 | shared_region_pager_map, |
131 | shared_region_pager_last_unmap, |
132 | NULL, /* data_reclaim */ |
133 | "shared_region" |
134 | }; |
135 | |
136 | /* |
137 | * The "shared_region_pager" describes a memory object backed by |
138 | * the "shared_region" EMM. |
139 | */ |
140 | typedef struct { |
141 | /* mandatory generic header */ |
142 | struct memory_object ; |
143 | |
144 | /* pager-specific data */ |
145 | queue_chain_t ; /* next & prev pagers */ |
146 | unsigned int ; /* reference count */ |
147 | boolean_t ; /* is this pager ready ? */ |
148 | boolean_t ; /* is this mem_obj mapped ? */ |
149 | vm_object_t ; /* VM obj for shared cache */ |
150 | vm_object_offset_t ; |
151 | struct vm_shared_region_slide_info *; |
152 | } *; |
153 | #define ((shared_region_pager_t) NULL) |
154 | |
155 | /* |
156 | * List of memory objects managed by this EMM. |
157 | * The list is protected by the "shared_region_pager_lock" lock. |
158 | */ |
159 | int = 0; /* number of pagers */ |
160 | int = 0; /* number of unmapped pagers */ |
161 | queue_head_t ; |
162 | decl_lck_mtx_data(,) |
163 | |
164 | /* |
165 | * Maximum number of unmapped pagers we're willing to keep around. |
166 | */ |
167 | int = 0; |
168 | |
169 | /* |
170 | * Statistics & counters. |
171 | */ |
172 | int = 0; |
173 | int = 0; |
174 | int = 0; |
175 | int = 0; |
176 | |
177 | |
178 | lck_grp_t ; |
179 | lck_grp_attr_t ; |
180 | lck_attr_t ; |
181 | |
182 | uint64_t = 0; |
183 | uint64_t = 0; |
184 | uint64_t = 0; |
185 | uint64_t = 0; |
186 | |
187 | /* internal prototypes */ |
188 | shared_region_pager_t shared_region_pager_create( |
189 | vm_object_t backing_object, |
190 | vm_object_offset_t backing_offset, |
191 | struct vm_shared_region_slide_info *slide_info); |
192 | shared_region_pager_t shared_region_pager_lookup(memory_object_t mem_obj); |
193 | void shared_region_pager_dequeue(shared_region_pager_t ); |
194 | void shared_region_pager_deallocate_internal(shared_region_pager_t , |
195 | boolean_t locked); |
196 | void shared_region_pager_terminate_internal(shared_region_pager_t ); |
197 | void shared_region_pager_trim(void); |
198 | |
199 | |
200 | #if DEBUG |
201 | int shared_region_pagerdebug = 0; |
202 | #define PAGER_ALL 0xffffffff |
203 | #define PAGER_INIT 0x00000001 |
204 | #define PAGER_PAGEIN 0x00000002 |
205 | |
206 | #define PAGER_DEBUG(LEVEL, A) \ |
207 | MACRO_BEGIN \ |
208 | if ((shared_region_pagerdebug & (LEVEL)) == (LEVEL)) { \ |
209 | printf A; \ |
210 | } \ |
211 | MACRO_END |
212 | #else |
213 | #define (LEVEL, A) |
214 | #endif |
215 | |
216 | |
217 | void |
218 | (void) |
219 | { |
220 | lck_grp_attr_setdefault(&shared_region_pager_lck_grp_attr); |
221 | lck_grp_init(&shared_region_pager_lck_grp, "shared_region" , &shared_region_pager_lck_grp_attr); |
222 | lck_attr_setdefault(&shared_region_pager_lck_attr); |
223 | lck_mtx_init(&shared_region_pager_lock, &shared_region_pager_lck_grp, &shared_region_pager_lck_attr); |
224 | queue_init(&shared_region_pager_queue); |
225 | } |
226 | |
227 | /* |
228 | * shared_region_pager_init() |
229 | * |
230 | * Initialize the memory object and makes it ready to be used and mapped. |
231 | */ |
232 | kern_return_t |
233 | ( |
234 | memory_object_t mem_obj, |
235 | memory_object_control_t control, |
236 | #if !DEBUG |
237 | __unused |
238 | #endif |
239 | memory_object_cluster_size_t pg_size) |
240 | { |
241 | shared_region_pager_t ; |
242 | kern_return_t kr; |
243 | memory_object_attr_info_data_t attributes; |
244 | |
245 | PAGER_DEBUG(PAGER_ALL, |
246 | ("shared_region_pager_init: %p, %p, %x\n" , |
247 | mem_obj, control, pg_size)); |
248 | |
249 | if (control == MEMORY_OBJECT_CONTROL_NULL) |
250 | return KERN_INVALID_ARGUMENT; |
251 | |
252 | pager = shared_region_pager_lookup(mem_obj); |
253 | |
254 | memory_object_control_reference(control); |
255 | |
256 | pager->sc_pgr_hdr.mo_control = control; |
257 | |
258 | attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; |
259 | /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/ |
260 | attributes.cluster_size = (1 << (PAGE_SHIFT)); |
261 | attributes.may_cache_object = FALSE; |
262 | attributes.temporary = TRUE; |
263 | |
264 | kr = memory_object_change_attributes( |
265 | control, |
266 | MEMORY_OBJECT_ATTRIBUTE_INFO, |
267 | (memory_object_info_t) &attributes, |
268 | MEMORY_OBJECT_ATTR_INFO_COUNT); |
269 | if (kr != KERN_SUCCESS) |
270 | panic("shared_region_pager_init: " |
271 | "memory_object_change_attributes() failed" ); |
272 | |
273 | #if CONFIG_SECLUDED_MEMORY |
274 | if (secluded_for_filecache) { |
275 | #if 00 |
276 | /* |
277 | * XXX FBDP do we want this in the secluded pool? |
278 | * Ideally, we'd want the shared region used by Camera to |
279 | * NOT be in the secluded pool, but all other shared regions |
280 | * in the secluded pool... |
281 | */ |
282 | memory_object_mark_eligible_for_secluded(control, TRUE); |
283 | #endif /* 00 */ |
284 | } |
285 | #endif /* CONFIG_SECLUDED_MEMORY */ |
286 | |
287 | return KERN_SUCCESS; |
288 | } |
289 | |
290 | /* |
291 | * shared_region_data_return() |
292 | * |
293 | * Handles page-out requests from VM. This should never happen since |
294 | * the pages provided by this EMM are not supposed to be dirty or dirtied |
295 | * and VM should simply discard the contents and reclaim the pages if it |
296 | * needs to. |
297 | */ |
298 | kern_return_t |
299 | ( |
300 | __unused memory_object_t mem_obj, |
301 | __unused memory_object_offset_t offset, |
302 | __unused memory_object_cluster_size_t data_cnt, |
303 | __unused memory_object_offset_t *resid_offset, |
304 | __unused int *io_error, |
305 | __unused boolean_t dirty, |
306 | __unused boolean_t kernel_copy, |
307 | __unused int upl_flags) |
308 | { |
309 | panic("shared_region_pager_data_return: should never get called" ); |
310 | return KERN_FAILURE; |
311 | } |
312 | |
313 | kern_return_t |
314 | ( |
315 | __unused memory_object_t mem_obj, |
316 | __unused memory_object_offset_t offset, |
317 | __unused memory_object_cluster_size_t data_cnt) |
318 | { |
319 | panic("shared_region_pager_data_initialize: should never get called" ); |
320 | return KERN_FAILURE; |
321 | } |
322 | |
323 | kern_return_t |
324 | ( |
325 | __unused memory_object_t mem_obj, |
326 | __unused memory_object_offset_t offset, |
327 | __unused memory_object_size_t size, |
328 | __unused vm_prot_t desired_access) |
329 | { |
330 | return KERN_FAILURE; |
331 | } |
332 | |
333 | /* |
334 | * shared_region_pager_data_request() |
335 | * |
336 | * Handles page-in requests from VM. |
337 | */ |
338 | int = 0; |
339 | kern_return_t |
340 | ( |
341 | memory_object_t mem_obj, |
342 | memory_object_offset_t offset, |
343 | memory_object_cluster_size_t length, |
344 | #if !DEBUG |
345 | __unused |
346 | #endif |
347 | vm_prot_t protection_required, |
348 | memory_object_fault_info_t mo_fault_info) |
349 | { |
350 | shared_region_pager_t ; |
351 | memory_object_control_t mo_control; |
352 | upl_t upl; |
353 | int upl_flags; |
354 | upl_size_t upl_size; |
355 | upl_page_info_t *upl_pl; |
356 | unsigned int pl_count; |
357 | vm_object_t src_top_object, src_page_object, dst_object; |
358 | kern_return_t kr, retval; |
359 | vm_offset_t src_vaddr, dst_vaddr; |
360 | vm_offset_t cur_offset; |
361 | vm_offset_t offset_in_page; |
362 | kern_return_t error_code; |
363 | vm_prot_t prot; |
364 | vm_page_t src_page, top_page; |
365 | int interruptible; |
366 | struct vm_object_fault_info fault_info; |
367 | mach_vm_offset_t slide_start_address; |
368 | |
369 | PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_data_request: %p, %llx, %x, %x\n" , mem_obj, offset, length, protection_required)); |
370 | |
371 | retval = KERN_SUCCESS; |
372 | src_top_object = VM_OBJECT_NULL; |
373 | src_page_object = VM_OBJECT_NULL; |
374 | upl = NULL; |
375 | upl_pl = NULL; |
376 | fault_info = *((struct vm_object_fault_info *)(uintptr_t)mo_fault_info); |
377 | fault_info.stealth = TRUE; |
378 | fault_info.io_sync = FALSE; |
379 | fault_info.mark_zf_absent = FALSE; |
380 | fault_info.batch_pmap_op = FALSE; |
381 | interruptible = fault_info.interruptible; |
382 | |
383 | pager = shared_region_pager_lookup(mem_obj); |
384 | assert(pager->is_ready); |
385 | assert(pager->ref_count > 1); /* pager is alive and mapped */ |
386 | |
387 | PAGER_DEBUG(PAGER_PAGEIN, ("shared_region_pager_data_request: %p, %llx, %x, %x, pager %p\n" , mem_obj, offset, length, protection_required, pager)); |
388 | |
389 | /* |
390 | * Gather in a UPL all the VM pages requested by VM. |
391 | */ |
392 | mo_control = pager->sc_pgr_hdr.mo_control; |
393 | |
394 | upl_size = length; |
395 | upl_flags = |
396 | UPL_RET_ONLY_ABSENT | |
397 | UPL_SET_LITE | |
398 | UPL_NO_SYNC | |
399 | UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ |
400 | UPL_SET_INTERNAL; |
401 | pl_count = 0; |
402 | kr = memory_object_upl_request(mo_control, |
403 | offset, upl_size, |
404 | &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_SECURITY); |
405 | if (kr != KERN_SUCCESS) { |
406 | retval = kr; |
407 | goto done; |
408 | } |
409 | dst_object = mo_control->moc_object; |
410 | assert(dst_object != VM_OBJECT_NULL); |
411 | |
412 | /* |
413 | * We'll map the original data in the kernel address space from the |
414 | * backing VM object (itself backed by the shared cache file via |
415 | * the vnode pager). |
416 | */ |
417 | src_top_object = pager->backing_object; |
418 | assert(src_top_object != VM_OBJECT_NULL); |
419 | vm_object_reference(src_top_object); /* keep the source object alive */ |
420 | |
421 | slide_start_address = pager->scp_slide_info->slid_address; |
422 | |
423 | fault_info.lo_offset += pager->backing_offset; |
424 | fault_info.hi_offset += pager->backing_offset; |
425 | |
426 | /* |
427 | * Fill in the contents of the pages requested by VM. |
428 | */ |
429 | upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl); |
430 | pl_count = length / PAGE_SIZE; |
431 | for (cur_offset = 0; |
432 | retval == KERN_SUCCESS && cur_offset < length; |
433 | cur_offset += PAGE_SIZE) { |
434 | ppnum_t dst_pnum; |
435 | |
436 | if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) { |
437 | /* this page is not in the UPL: skip it */ |
438 | continue; |
439 | } |
440 | |
441 | /* |
442 | * Map the source (dyld shared cache) page in the kernel's |
443 | * virtual address space. |
444 | * We already hold a reference on the src_top_object. |
445 | */ |
446 | retry_src_fault: |
447 | vm_object_lock(src_top_object); |
448 | vm_object_paging_begin(src_top_object); |
449 | error_code = 0; |
450 | prot = VM_PROT_READ; |
451 | src_page = VM_PAGE_NULL; |
452 | kr = vm_fault_page(src_top_object, |
453 | pager->backing_offset + offset + cur_offset, |
454 | VM_PROT_READ, |
455 | FALSE, |
456 | FALSE, /* src_page not looked up */ |
457 | &prot, |
458 | &src_page, |
459 | &top_page, |
460 | NULL, |
461 | &error_code, |
462 | FALSE, |
463 | FALSE, |
464 | &fault_info); |
465 | switch (kr) { |
466 | case VM_FAULT_SUCCESS: |
467 | break; |
468 | case VM_FAULT_RETRY: |
469 | goto retry_src_fault; |
470 | case VM_FAULT_MEMORY_SHORTAGE: |
471 | if (vm_page_wait(interruptible)) { |
472 | goto retry_src_fault; |
473 | } |
474 | /* fall thru */ |
475 | case VM_FAULT_INTERRUPTED: |
476 | retval = MACH_SEND_INTERRUPTED; |
477 | goto done; |
478 | case VM_FAULT_SUCCESS_NO_VM_PAGE: |
479 | /* success but no VM page: fail */ |
480 | vm_object_paging_end(src_top_object); |
481 | vm_object_unlock(src_top_object); |
482 | /*FALLTHROUGH*/ |
483 | case VM_FAULT_MEMORY_ERROR: |
484 | /* the page is not there ! */ |
485 | if (error_code) { |
486 | retval = error_code; |
487 | } else { |
488 | retval = KERN_MEMORY_ERROR; |
489 | } |
490 | goto done; |
491 | default: |
492 | panic("shared_region_pager_data_request: " |
493 | "vm_fault_page() unexpected error 0x%x\n" , |
494 | kr); |
495 | } |
496 | assert(src_page != VM_PAGE_NULL); |
497 | assert(src_page->vmp_busy); |
498 | |
499 | if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) { |
500 | vm_page_lockspin_queues(); |
501 | if (src_page->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q) { |
502 | vm_page_speculate(src_page, FALSE); |
503 | } |
504 | vm_page_unlock_queues(); |
505 | } |
506 | |
507 | /* |
508 | * Establish pointers to the source |
509 | * and destination physical pages. |
510 | */ |
511 | dst_pnum = (ppnum_t) |
512 | upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); |
513 | assert(dst_pnum != 0); |
514 | #if __x86_64__ |
515 | src_vaddr = (vm_map_offset_t) |
516 | PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) |
517 | << PAGE_SHIFT); |
518 | dst_vaddr = (vm_map_offset_t) |
519 | PHYSMAP_PTOV((pmap_paddr_t)dst_pnum << PAGE_SHIFT); |
520 | |
521 | #elif __arm__ || __arm64__ |
522 | src_vaddr = (vm_map_offset_t) |
523 | phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(src_page) |
524 | << PAGE_SHIFT); |
525 | dst_vaddr = (vm_map_offset_t) |
526 | phystokv((pmap_paddr_t)dst_pnum << PAGE_SHIFT); |
527 | #else |
528 | #error "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..." |
529 | src_vaddr = 0; |
530 | dst_vaddr = 0; |
531 | #endif |
532 | src_page_object = VM_PAGE_OBJECT(src_page); |
533 | |
534 | /* |
535 | * Validate the original page... |
536 | */ |
537 | if (src_page_object->code_signed) { |
538 | vm_page_validate_cs_mapped( |
539 | src_page, |
540 | (const void *) src_vaddr); |
541 | } |
542 | /* |
543 | * ... and transfer the results to the destination page. |
544 | */ |
545 | UPL_SET_CS_VALIDATED(upl_pl, cur_offset / PAGE_SIZE, |
546 | src_page->vmp_cs_validated); |
547 | UPL_SET_CS_TAINTED(upl_pl, cur_offset / PAGE_SIZE, |
548 | src_page->vmp_cs_tainted); |
549 | UPL_SET_CS_NX(upl_pl, cur_offset / PAGE_SIZE, |
550 | src_page->vmp_cs_nx); |
551 | |
552 | /* |
553 | * The page provider might access a mapped file, so let's |
554 | * release the object lock for the source page to avoid a |
555 | * potential deadlock. |
556 | * The source page is kept busy and we have a |
557 | * "paging_in_progress" reference on its object, so it's safe |
558 | * to unlock the object here. |
559 | */ |
560 | assert(src_page->vmp_busy); |
561 | assert(src_page_object->paging_in_progress > 0); |
562 | vm_object_unlock(src_page_object); |
563 | |
564 | /* |
565 | * Process the original contents of the source page |
566 | * into the destination page. |
567 | */ |
568 | for (offset_in_page = 0; |
569 | offset_in_page < PAGE_SIZE; |
570 | offset_in_page += PAGE_SIZE_FOR_SR_SLIDE) { |
571 | vm_object_offset_t chunk_offset; |
572 | vm_object_offset_t offset_in_backing_object; |
573 | vm_object_offset_t offset_in_sliding_range; |
574 | |
575 | chunk_offset = offset + cur_offset + offset_in_page; |
576 | |
577 | bcopy((const char *)(src_vaddr + |
578 | offset_in_page), |
579 | (char *)(dst_vaddr + offset_in_page), |
580 | PAGE_SIZE_FOR_SR_SLIDE); |
581 | |
582 | offset_in_backing_object = (chunk_offset + |
583 | pager->backing_offset); |
584 | if ((offset_in_backing_object < pager->scp_slide_info->start) || |
585 | (offset_in_backing_object >= pager->scp_slide_info->end)) { |
586 | /* chunk is outside of sliding range: done */ |
587 | shared_region_pager_copied++; |
588 | continue; |
589 | } |
590 | |
591 | offset_in_sliding_range = |
592 | (offset_in_backing_object - |
593 | pager->scp_slide_info->start); |
594 | kr = vm_shared_region_slide_page( |
595 | pager->scp_slide_info, |
596 | dst_vaddr + offset_in_page, |
597 | (mach_vm_offset_t) (offset_in_sliding_range + |
598 | slide_start_address), |
599 | (uint32_t) (offset_in_sliding_range / |
600 | PAGE_SIZE_FOR_SR_SLIDE)); |
601 | if (shared_region_pager_data_request_debug) { |
602 | printf("shared_region_data_request" |
603 | "(%p,0x%llx+0x%llx+0x%04llx): 0x%llx " |
604 | "in sliding range [0x%llx:0x%llx]: " |
605 | "SLIDE offset 0x%llx=" |
606 | "(0x%llx+0x%llx+0x%llx+0x%04llx)" |
607 | "[0x%016llx 0x%016llx] " |
608 | "code_signed=%d " |
609 | "cs_validated=%d " |
610 | "cs_tainted=%d " |
611 | "cs_nx=%d " |
612 | "kr=0x%x\n" , |
613 | pager, |
614 | offset, |
615 | (uint64_t) cur_offset, |
616 | (uint64_t) offset_in_page, |
617 | chunk_offset, |
618 | pager->scp_slide_info->start, |
619 | pager->scp_slide_info->end, |
620 | (pager->backing_offset + |
621 | offset + |
622 | cur_offset + |
623 | offset_in_page), |
624 | pager->backing_offset, |
625 | offset, |
626 | (uint64_t) cur_offset, |
627 | (uint64_t) offset_in_page, |
628 | *(uint64_t *)(dst_vaddr+offset_in_page), |
629 | *(uint64_t *)(dst_vaddr+offset_in_page+8), |
630 | src_page_object->code_signed, |
631 | src_page->vmp_cs_validated, |
632 | src_page->vmp_cs_tainted, |
633 | src_page->vmp_cs_nx, |
634 | kr); |
635 | } |
636 | if (kr != KERN_SUCCESS) { |
637 | shared_region_pager_slid_error++; |
638 | break; |
639 | } |
640 | shared_region_pager_slid++; |
641 | } |
642 | |
643 | assert(VM_PAGE_OBJECT(src_page) == src_page_object); |
644 | assert(src_page->vmp_busy); |
645 | assert(src_page_object->paging_in_progress > 0); |
646 | vm_object_lock(src_page_object); |
647 | |
648 | /* |
649 | * Cleanup the result of vm_fault_page() of the source page. |
650 | */ |
651 | PAGE_WAKEUP_DONE(src_page); |
652 | src_page = VM_PAGE_NULL; |
653 | vm_object_paging_end(src_page_object); |
654 | vm_object_unlock(src_page_object); |
655 | |
656 | if (top_page != VM_PAGE_NULL) { |
657 | assert(VM_PAGE_OBJECT(top_page) == src_top_object); |
658 | vm_object_lock(src_top_object); |
659 | VM_PAGE_FREE(top_page); |
660 | vm_object_paging_end(src_top_object); |
661 | vm_object_unlock(src_top_object); |
662 | } |
663 | } |
664 | |
665 | done: |
666 | if (upl != NULL) { |
667 | /* clean up the UPL */ |
668 | |
669 | /* |
670 | * The pages are currently dirty because we've just been |
671 | * writing on them, but as far as we're concerned, they're |
672 | * clean since they contain their "original" contents as |
673 | * provided by us, the pager. |
674 | * Tell the UPL to mark them "clean". |
675 | */ |
676 | upl_clear_dirty(upl, TRUE); |
677 | |
678 | /* abort or commit the UPL */ |
679 | if (retval != KERN_SUCCESS) { |
680 | upl_abort(upl, 0); |
681 | } else { |
682 | boolean_t empty; |
683 | upl_commit_range(upl, 0, upl->size, |
684 | UPL_COMMIT_CS_VALIDATED | UPL_COMMIT_WRITTEN_BY_KERNEL, |
685 | upl_pl, pl_count, &empty); |
686 | } |
687 | |
688 | /* and deallocate the UPL */ |
689 | upl_deallocate(upl); |
690 | upl = NULL; |
691 | } |
692 | if (src_top_object != VM_OBJECT_NULL) { |
693 | vm_object_deallocate(src_top_object); |
694 | } |
695 | return retval; |
696 | } |
697 | |
698 | /* |
699 | * shared_region_pager_reference() |
700 | * |
701 | * Get a reference on this memory object. |
702 | * For external usage only. Assumes that the initial reference count is not 0, |
703 | * i.e one should not "revive" a dead pager this way. |
704 | */ |
705 | void |
706 | ( |
707 | memory_object_t mem_obj) |
708 | { |
709 | shared_region_pager_t ; |
710 | |
711 | pager = shared_region_pager_lookup(mem_obj); |
712 | |
713 | lck_mtx_lock(&shared_region_pager_lock); |
714 | assert(pager->ref_count > 0); |
715 | pager->ref_count++; |
716 | lck_mtx_unlock(&shared_region_pager_lock); |
717 | } |
718 | |
719 | |
720 | /* |
721 | * shared_region_pager_dequeue: |
722 | * |
723 | * Removes a pager from the list of pagers. |
724 | * |
725 | * The caller must hold "shared_region_pager_lock". |
726 | */ |
727 | void |
728 | ( |
729 | shared_region_pager_t ) |
730 | { |
731 | assert(!pager->is_mapped); |
732 | |
733 | queue_remove(&shared_region_pager_queue, |
734 | pager, |
735 | shared_region_pager_t, |
736 | pager_queue); |
737 | pager->pager_queue.next = NULL; |
738 | pager->pager_queue.prev = NULL; |
739 | |
740 | shared_region_pager_count--; |
741 | } |
742 | |
743 | /* |
744 | * shared_region_pager_terminate_internal: |
745 | * |
746 | * Trigger the asynchronous termination of the memory object associated |
747 | * with this pager. |
748 | * When the memory object is terminated, there will be one more call |
749 | * to memory_object_deallocate() (i.e. shared_region_pager_deallocate()) |
750 | * to finish the clean up. |
751 | * |
752 | * "shared_region_pager_lock" should not be held by the caller. |
753 | * We don't need the lock because the pager has already been removed from |
754 | * the pagers' list and is now ours exclusively. |
755 | */ |
756 | void |
757 | ( |
758 | shared_region_pager_t ) |
759 | { |
760 | assert(pager->is_ready); |
761 | assert(!pager->is_mapped); |
762 | |
763 | if (pager->backing_object != VM_OBJECT_NULL) { |
764 | vm_object_deallocate(pager->backing_object); |
765 | pager->backing_object = VM_OBJECT_NULL; |
766 | } |
767 | /* trigger the destruction of the memory object */ |
768 | memory_object_destroy(pager->sc_pgr_hdr.mo_control, 0); |
769 | } |
770 | |
771 | /* |
772 | * shared_region_pager_deallocate_internal() |
773 | * |
774 | * Release a reference on this pager and free it when the last |
775 | * reference goes away. |
776 | * Can be called with shared_region_pager_lock held or not but always returns |
777 | * with it unlocked. |
778 | */ |
779 | void |
780 | ( |
781 | shared_region_pager_t , |
782 | boolean_t locked) |
783 | { |
784 | boolean_t needs_trimming; |
785 | int count_unmapped; |
786 | |
787 | if (! locked) { |
788 | lck_mtx_lock(&shared_region_pager_lock); |
789 | } |
790 | |
791 | count_unmapped = (shared_region_pager_count - |
792 | shared_region_pager_count_mapped); |
793 | if (count_unmapped > shared_region_pager_cache_limit) { |
794 | /* we have too many unmapped pagers: trim some */ |
795 | needs_trimming = TRUE; |
796 | } else { |
797 | needs_trimming = FALSE; |
798 | } |
799 | |
800 | /* drop a reference on this pager */ |
801 | pager->ref_count--; |
802 | |
803 | if (pager->ref_count == 1) { |
804 | /* |
805 | * Only the "named" reference is left, which means that |
806 | * no one is really holding on to this pager anymore. |
807 | * Terminate it. |
808 | */ |
809 | shared_region_pager_dequeue(pager); |
810 | /* the pager is all ours: no need for the lock now */ |
811 | lck_mtx_unlock(&shared_region_pager_lock); |
812 | shared_region_pager_terminate_internal(pager); |
813 | } else if (pager->ref_count == 0) { |
814 | /* |
815 | * Dropped the existence reference; the memory object has |
816 | * been terminated. Do some final cleanup and release the |
817 | * pager structure. |
818 | */ |
819 | lck_mtx_unlock(&shared_region_pager_lock); |
820 | if (pager->sc_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { |
821 | memory_object_control_deallocate(pager->sc_pgr_hdr.mo_control); |
822 | pager->sc_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; |
823 | } |
824 | kfree(pager, sizeof (*pager)); |
825 | pager = SHARED_REGION_PAGER_NULL; |
826 | } else { |
827 | /* there are still plenty of references: keep going... */ |
828 | lck_mtx_unlock(&shared_region_pager_lock); |
829 | } |
830 | |
831 | if (needs_trimming) { |
832 | shared_region_pager_trim(); |
833 | } |
834 | /* caution: lock is not held on return... */ |
835 | } |
836 | |
837 | /* |
838 | * shared_region_pager_deallocate() |
839 | * |
840 | * Release a reference on this pager and free it when the last |
841 | * reference goes away. |
842 | */ |
843 | void |
844 | ( |
845 | memory_object_t mem_obj) |
846 | { |
847 | shared_region_pager_t ; |
848 | |
849 | PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_deallocate: %p\n" , mem_obj)); |
850 | pager = shared_region_pager_lookup(mem_obj); |
851 | shared_region_pager_deallocate_internal(pager, FALSE); |
852 | } |
853 | |
854 | /* |
855 | * |
856 | */ |
857 | kern_return_t |
858 | ( |
859 | #if !DEBUG |
860 | __unused |
861 | #endif |
862 | memory_object_t mem_obj) |
863 | { |
864 | PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_terminate: %p\n" , mem_obj)); |
865 | |
866 | return KERN_SUCCESS; |
867 | } |
868 | |
869 | /* |
870 | * |
871 | */ |
872 | kern_return_t |
873 | ( |
874 | __unused memory_object_t mem_obj, |
875 | __unused memory_object_offset_t offset, |
876 | __unused memory_object_size_t length, |
877 | __unused vm_sync_t sync_flags) |
878 | { |
879 | panic("shared_region_pager_synchronize: memory_object_synchronize no longer supported\n" ); |
880 | return KERN_FAILURE; |
881 | } |
882 | |
883 | /* |
884 | * shared_region_pager_map() |
885 | * |
886 | * This allows VM to let us, the EMM, know that this memory object |
887 | * is currently mapped one or more times. This is called by VM each time |
888 | * the memory object gets mapped and we take one extra reference on the |
889 | * memory object to account for all its mappings. |
890 | */ |
891 | kern_return_t |
892 | ( |
893 | memory_object_t mem_obj, |
894 | __unused vm_prot_t prot) |
895 | { |
896 | shared_region_pager_t ; |
897 | |
898 | PAGER_DEBUG(PAGER_ALL, ("shared_region_pager_map: %p\n" , mem_obj)); |
899 | |
900 | pager = shared_region_pager_lookup(mem_obj); |
901 | |
902 | lck_mtx_lock(&shared_region_pager_lock); |
903 | assert(pager->is_ready); |
904 | assert(pager->ref_count > 0); /* pager is alive */ |
905 | if (pager->is_mapped == FALSE) { |
906 | /* |
907 | * First mapping of this pager: take an extra reference |
908 | * that will remain until all the mappings of this pager |
909 | * are removed. |
910 | */ |
911 | pager->is_mapped = TRUE; |
912 | pager->ref_count++; |
913 | shared_region_pager_count_mapped++; |
914 | } |
915 | lck_mtx_unlock(&shared_region_pager_lock); |
916 | |
917 | return KERN_SUCCESS; |
918 | } |
919 | |
920 | /* |
921 | * shared_region_pager_last_unmap() |
922 | * |
923 | * This is called by VM when this memory object is no longer mapped anywhere. |
924 | */ |
925 | kern_return_t |
926 | ( |
927 | memory_object_t mem_obj) |
928 | { |
929 | shared_region_pager_t ; |
930 | int count_unmapped; |
931 | |
932 | PAGER_DEBUG(PAGER_ALL, |
933 | ("shared_region_pager_last_unmap: %p\n" , mem_obj)); |
934 | |
935 | pager = shared_region_pager_lookup(mem_obj); |
936 | |
937 | lck_mtx_lock(&shared_region_pager_lock); |
938 | if (pager->is_mapped) { |
939 | /* |
940 | * All the mappings are gone, so let go of the one extra |
941 | * reference that represents all the mappings of this pager. |
942 | */ |
943 | shared_region_pager_count_mapped--; |
944 | count_unmapped = (shared_region_pager_count - |
945 | shared_region_pager_count_mapped); |
946 | if (count_unmapped > shared_region_pager_count_unmapped_max) { |
947 | shared_region_pager_count_unmapped_max = count_unmapped; |
948 | } |
949 | pager->is_mapped = FALSE; |
950 | shared_region_pager_deallocate_internal(pager, TRUE); |
951 | /* caution: deallocate_internal() released the lock ! */ |
952 | } else { |
953 | lck_mtx_unlock(&shared_region_pager_lock); |
954 | } |
955 | |
956 | return KERN_SUCCESS; |
957 | } |
958 | |
959 | |
960 | /* |
961 | * |
962 | */ |
963 | shared_region_pager_t |
964 | ( |
965 | memory_object_t mem_obj) |
966 | { |
967 | shared_region_pager_t ; |
968 | |
969 | assert(mem_obj->mo_pager_ops == &shared_region_pager_ops); |
970 | pager = (shared_region_pager_t)(uintptr_t) mem_obj; |
971 | assert(pager->ref_count > 0); |
972 | return pager; |
973 | } |
974 | |
975 | shared_region_pager_t |
976 | ( |
977 | vm_object_t backing_object, |
978 | vm_object_offset_t backing_offset, |
979 | struct vm_shared_region_slide_info *slide_info) |
980 | { |
981 | shared_region_pager_t ; |
982 | memory_object_control_t control; |
983 | kern_return_t kr; |
984 | |
985 | pager = (shared_region_pager_t) kalloc(sizeof (*pager)); |
986 | if (pager == SHARED_REGION_PAGER_NULL) { |
987 | return SHARED_REGION_PAGER_NULL; |
988 | } |
989 | |
990 | /* |
991 | * The vm_map call takes both named entry ports and raw memory |
992 | * objects in the same parameter. We need to make sure that |
993 | * vm_map does not see this object as a named entry port. So, |
994 | * we reserve the first word in the object for a fake ip_kotype |
995 | * setting - that will tell vm_map to use it as a memory object. |
996 | */ |
997 | pager->sc_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; |
998 | pager->sc_pgr_hdr.mo_pager_ops = &shared_region_pager_ops; |
999 | pager->sc_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; |
1000 | |
1001 | pager->is_ready = FALSE;/* not ready until it has a "name" */ |
1002 | pager->ref_count = 1; /* existence reference (for the cache) */ |
1003 | pager->ref_count++; /* for the caller */ |
1004 | pager->is_mapped = FALSE; |
1005 | pager->backing_object = backing_object; |
1006 | pager->backing_offset = backing_offset; |
1007 | pager->scp_slide_info = slide_info; |
1008 | |
1009 | vm_object_reference(backing_object); |
1010 | |
1011 | lck_mtx_lock(&shared_region_pager_lock); |
1012 | /* enter new pager at the head of our list of pagers */ |
1013 | queue_enter_first(&shared_region_pager_queue, |
1014 | pager, |
1015 | shared_region_pager_t, |
1016 | pager_queue); |
1017 | shared_region_pager_count++; |
1018 | if (shared_region_pager_count > shared_region_pager_count_max) { |
1019 | shared_region_pager_count_max = shared_region_pager_count; |
1020 | } |
1021 | lck_mtx_unlock(&shared_region_pager_lock); |
1022 | |
1023 | kr = memory_object_create_named((memory_object_t) pager, |
1024 | 0, |
1025 | &control); |
1026 | assert(kr == KERN_SUCCESS); |
1027 | |
1028 | lck_mtx_lock(&shared_region_pager_lock); |
1029 | /* the new pager is now ready to be used */ |
1030 | pager->is_ready = TRUE; |
1031 | lck_mtx_unlock(&shared_region_pager_lock); |
1032 | |
1033 | /* wakeup anyone waiting for this pager to be ready */ |
1034 | thread_wakeup(&pager->is_ready); |
1035 | |
1036 | return pager; |
1037 | } |
1038 | |
1039 | /* |
1040 | * shared_region_pager_setup() |
1041 | * |
1042 | * Provide the caller with a memory object backed by the provided |
1043 | * "backing_object" VM object. |
1044 | */ |
1045 | memory_object_t |
1046 | ( |
1047 | vm_object_t backing_object, |
1048 | vm_object_offset_t backing_offset, |
1049 | struct vm_shared_region_slide_info *slide_info) |
1050 | { |
1051 | shared_region_pager_t ; |
1052 | |
1053 | /* create new pager */ |
1054 | pager = shared_region_pager_create( |
1055 | backing_object, |
1056 | backing_offset, |
1057 | slide_info); |
1058 | if (pager == SHARED_REGION_PAGER_NULL) { |
1059 | /* could not create a new pager */ |
1060 | return MEMORY_OBJECT_NULL; |
1061 | } |
1062 | |
1063 | lck_mtx_lock(&shared_region_pager_lock); |
1064 | while (!pager->is_ready) { |
1065 | lck_mtx_sleep(&shared_region_pager_lock, |
1066 | LCK_SLEEP_DEFAULT, |
1067 | &pager->is_ready, |
1068 | THREAD_UNINT); |
1069 | } |
1070 | lck_mtx_unlock(&shared_region_pager_lock); |
1071 | |
1072 | return (memory_object_t) pager; |
1073 | } |
1074 | |
1075 | void |
1076 | (void) |
1077 | { |
1078 | shared_region_pager_t , ; |
1079 | queue_head_t trim_queue; |
1080 | int num_trim; |
1081 | int count_unmapped; |
1082 | |
1083 | lck_mtx_lock(&shared_region_pager_lock); |
1084 | |
1085 | /* |
1086 | * We have too many pagers, try and trim some unused ones, |
1087 | * starting with the oldest pager at the end of the queue. |
1088 | */ |
1089 | queue_init(&trim_queue); |
1090 | num_trim = 0; |
1091 | |
1092 | for (pager = (shared_region_pager_t) |
1093 | queue_last(&shared_region_pager_queue); |
1094 | !queue_end(&shared_region_pager_queue, |
1095 | (queue_entry_t) pager); |
1096 | pager = prev_pager) { |
1097 | /* get prev elt before we dequeue */ |
1098 | prev_pager = (shared_region_pager_t) |
1099 | queue_prev(&pager->pager_queue); |
1100 | |
1101 | if (pager->ref_count == 2 && |
1102 | pager->is_ready && |
1103 | !pager->is_mapped) { |
1104 | /* this pager can be trimmed */ |
1105 | num_trim++; |
1106 | /* remove this pager from the main list ... */ |
1107 | shared_region_pager_dequeue(pager); |
1108 | /* ... and add it to our trim queue */ |
1109 | queue_enter_first(&trim_queue, |
1110 | pager, |
1111 | shared_region_pager_t, |
1112 | pager_queue); |
1113 | |
1114 | count_unmapped = (shared_region_pager_count - |
1115 | shared_region_pager_count_mapped); |
1116 | if (count_unmapped <= shared_region_pager_cache_limit) { |
1117 | /* we have enough pagers to trim */ |
1118 | break; |
1119 | } |
1120 | } |
1121 | } |
1122 | if (num_trim > shared_region_pager_num_trim_max) { |
1123 | shared_region_pager_num_trim_max = num_trim; |
1124 | } |
1125 | shared_region_pager_num_trim_total += num_trim; |
1126 | |
1127 | lck_mtx_unlock(&shared_region_pager_lock); |
1128 | |
1129 | /* terminate the trimmed pagers */ |
1130 | while (!queue_empty(&trim_queue)) { |
1131 | queue_remove_first(&trim_queue, |
1132 | pager, |
1133 | shared_region_pager_t, |
1134 | pager_queue); |
1135 | pager->pager_queue.next = NULL; |
1136 | pager->pager_queue.prev = NULL; |
1137 | assert(pager->ref_count == 2); |
1138 | /* |
1139 | * We can't call deallocate_internal() because the pager |
1140 | * has already been dequeued, but we still need to remove |
1141 | * a reference. |
1142 | */ |
1143 | pager->ref_count--; |
1144 | shared_region_pager_terminate_internal(pager); |
1145 | } |
1146 | } |
1147 | |