1 | /* |
2 | * Copyright (c) 2008-2020 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <mach/kern_return.h> |
30 | #include <mach/memory_object_control.h> |
31 | #include <mach/upl.h> |
32 | |
33 | #include <kern/ipc_kobject.h> |
34 | #include <kern/kalloc.h> |
35 | #include <kern/queue.h> |
36 | |
37 | #include <vm/memory_object.h> |
38 | #include <vm/vm_kern.h> |
39 | #include <vm/vm_map.h> |
40 | #include <vm/vm_pageout.h> |
41 | #include <vm/vm_protos.h> |
42 | |
43 | |
44 | /* |
45 | * APPLE SWAPFILE MEMORY PAGER |
46 | * |
47 | * This external memory manager (EMM) handles mappings of the swap files. |
48 | * Swap files are not regular files and are used solely to store contents of |
49 | * anonymous memory mappings while not resident in memory. |
50 | * There's no valid reason to map a swap file. This just puts extra burden |
51 | * on the system, is potentially a security issue and is not reliable since |
52 | * the contents can change at any time with pageout operations. |
53 | * Here are some of the issues with mapping a swap file. |
54 | * * PERFORMANCE: |
55 | * Each page in the swap file belong to an anonymous memory object. Mapping |
56 | * the swap file makes those pages also accessible via a vnode memory |
57 | * object and each page can now be resident twice. |
58 | * * SECURITY: |
59 | * Mapping a swap file allows access to other processes' memory. Swap files |
60 | * are only accessible by the "root" super-user, who can already access any |
61 | * process's memory, so this is not a real issue but if permissions on the |
62 | * swap file got changed, it could become one. |
63 | * Swap files are not "zero-filled" on creation, so until their contents are |
64 | * overwritten with pageout operations, they still contain whatever was on |
65 | * the disk blocks they were allocated. The "super-user" could see the |
66 | * contents of free blocks anyway, so this is not a new security issue but |
67 | * it may be perceive as one. |
68 | * |
69 | * We can't legitimately prevent a user process with appropriate privileges |
70 | * from mapping a swap file, but we can prevent it from accessing its actual |
71 | * contents. |
72 | * This pager mostly handles page-in request (from memory_object_data_request()) |
73 | * for swap file mappings and just returns bogus data. |
74 | * Pageouts are not handled, so mmap() has to make sure it does not allow |
75 | * writable (i.e. MAP_SHARED and PROT_WRITE) mappings of swap files. |
76 | */ |
77 | |
78 | /* forward declarations */ |
79 | void swapfile_pager_reference(memory_object_t mem_obj); |
80 | void swapfile_pager_deallocate(memory_object_t mem_obj); |
81 | kern_return_t swapfile_pager_init(memory_object_t mem_obj, |
82 | memory_object_control_t control, |
83 | memory_object_cluster_size_t pg_size); |
84 | kern_return_t swapfile_pager_terminate(memory_object_t mem_obj); |
85 | kern_return_t swapfile_pager_data_request(memory_object_t mem_obj, |
86 | memory_object_offset_t offset, |
87 | memory_object_cluster_size_t length, |
88 | vm_prot_t protection_required, |
89 | memory_object_fault_info_t fault_info); |
90 | kern_return_t swapfile_pager_data_return(memory_object_t mem_obj, |
91 | memory_object_offset_t offset, |
92 | memory_object_cluster_size_t data_cnt, |
93 | memory_object_offset_t *resid_offset, |
94 | int *io_error, |
95 | boolean_t dirty, |
96 | boolean_t kernel_copy, |
97 | int upl_flags); |
98 | kern_return_t swapfile_pager_data_initialize(memory_object_t mem_obj, |
99 | memory_object_offset_t offset, |
100 | memory_object_cluster_size_t data_cnt); |
101 | kern_return_t swapfile_pager_map(memory_object_t mem_obj, |
102 | vm_prot_t prot); |
103 | kern_return_t swapfile_pager_last_unmap(memory_object_t mem_obj); |
104 | |
105 | /* |
106 | * Vector of VM operations for this EMM. |
107 | * These routines are invoked by VM via the memory_object_*() interfaces. |
108 | */ |
109 | const struct memory_object_pager_ops = { |
110 | .memory_object_reference = swapfile_pager_reference, |
111 | .memory_object_deallocate = swapfile_pager_deallocate, |
112 | .memory_object_init = swapfile_pager_init, |
113 | .memory_object_terminate = swapfile_pager_terminate, |
114 | .memory_object_data_request = swapfile_pager_data_request, |
115 | .memory_object_data_return = swapfile_pager_data_return, |
116 | .memory_object_data_initialize = swapfile_pager_data_initialize, |
117 | .memory_object_map = swapfile_pager_map, |
118 | .memory_object_last_unmap = swapfile_pager_last_unmap, |
119 | .memory_object_backing_object = NULL, |
120 | .memory_object_pager_name = "swapfile pager" |
121 | }; |
122 | |
123 | /* |
124 | * The "swapfile_pager" describes a memory object backed by |
125 | * the "swapfile" EMM. |
126 | */ |
127 | typedef struct { |
128 | /* mandatory generic header */ |
129 | struct memory_object ; |
130 | |
131 | /* pager-specific data */ |
132 | queue_chain_t ; /* next & prev pagers */ |
133 | #if MEMORY_OBJECT_HAS_REFCOUNT |
134 | #define swp_pgr_hdr_ref swp_pgr_hdr.mo_ref |
135 | #else |
136 | os_ref_atomic_t swp_pgr_hdr_ref; /* reference count */ |
137 | #endif |
138 | bool ; /* is this pager ready ? */ |
139 | bool ; /* is this pager mapped ? */ |
140 | struct vnode *;/* the swapfile's vnode */ |
141 | } *; |
142 | #define ((swapfile_pager_t) NULL) |
143 | |
144 | /* |
145 | * List of memory objects managed by this EMM. |
146 | * The list is protected by the "swapfile_pager_lock" lock. |
147 | */ |
148 | int = 0; /* number of pagers */ |
149 | queue_head_t = QUEUE_HEAD_INITIALIZER(swapfile_pager_queue); |
150 | LCK_GRP_DECLARE(, "swapfile pager" ); |
151 | LCK_MTX_DECLARE(, &swapfile_pager_lck_grp); |
152 | |
153 | /* |
154 | * Statistics & counters. |
155 | */ |
156 | int = 0; |
157 | |
158 | /* internal prototypes */ |
159 | swapfile_pager_t swapfile_pager_create(struct vnode *vp); |
160 | swapfile_pager_t swapfile_pager_lookup(memory_object_t mem_obj); |
161 | void swapfile_pager_dequeue(swapfile_pager_t ); |
162 | void swapfile_pager_deallocate_internal(swapfile_pager_t , |
163 | boolean_t locked); |
164 | void swapfile_pager_terminate_internal(swapfile_pager_t ); |
165 | |
166 | |
167 | #if DEBUG |
168 | int swapfile_pagerdebug = 0; |
169 | #define PAGER_ALL 0xffffffff |
170 | #define PAGER_INIT 0x00000001 |
171 | #define PAGER_PAGEIN 0x00000002 |
172 | |
173 | #define PAGER_DEBUG(LEVEL, A) \ |
174 | MACRO_BEGIN \ |
175 | if ((swapfile_pagerdebug & LEVEL)==LEVEL) { \ |
176 | printf A; \ |
177 | } \ |
178 | MACRO_END |
179 | #else |
180 | #define (LEVEL, A) |
181 | #endif |
182 | |
183 | |
184 | /* |
185 | * swapfile_pager_init() |
186 | * |
187 | * Initialize the memory object and makes it ready to be used and mapped. |
188 | */ |
189 | kern_return_t |
190 | ( |
191 | memory_object_t mem_obj, |
192 | memory_object_control_t control, |
193 | #if !DEBUG |
194 | __unused |
195 | #endif |
196 | memory_object_cluster_size_t pg_size) |
197 | { |
198 | swapfile_pager_t ; |
199 | kern_return_t kr; |
200 | memory_object_attr_info_data_t attributes; |
201 | |
202 | PAGER_DEBUG(PAGER_ALL, |
203 | ("swapfile_pager_init: %p, %p, %x\n" , |
204 | mem_obj, control, pg_size)); |
205 | |
206 | if (control == MEMORY_OBJECT_CONTROL_NULL) { |
207 | return KERN_INVALID_ARGUMENT; |
208 | } |
209 | |
210 | pager = swapfile_pager_lookup(mem_obj); |
211 | |
212 | memory_object_control_reference(control); |
213 | |
214 | pager->swp_pgr_hdr.mo_control = control; |
215 | |
216 | attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; |
217 | attributes.cluster_size = (1 << (PAGE_SHIFT)); |
218 | attributes.may_cache_object = FALSE; |
219 | attributes.temporary = TRUE; |
220 | |
221 | kr = memory_object_change_attributes( |
222 | memory_control: control, |
223 | MEMORY_OBJECT_ATTRIBUTE_INFO, |
224 | attributes: (memory_object_info_t) &attributes, |
225 | MEMORY_OBJECT_ATTR_INFO_COUNT); |
226 | if (kr != KERN_SUCCESS) { |
227 | panic("swapfile_pager_init: " |
228 | "memory_object_change_attributes() failed" ); |
229 | } |
230 | |
231 | return KERN_SUCCESS; |
232 | } |
233 | |
234 | /* |
235 | * swapfile_data_return() |
236 | * |
237 | * Handles page-out requests from VM. This should never happen since |
238 | * the pages provided by this EMM are not supposed to be dirty or dirtied |
239 | * and VM should simply discard the contents and reclaim the pages if it |
240 | * needs to. |
241 | */ |
242 | kern_return_t |
243 | ( |
244 | __unused memory_object_t mem_obj, |
245 | __unused memory_object_offset_t offset, |
246 | __unused memory_object_cluster_size_t data_cnt, |
247 | __unused memory_object_offset_t *resid_offset, |
248 | __unused int *io_error, |
249 | __unused boolean_t dirty, |
250 | __unused boolean_t kernel_copy, |
251 | __unused int upl_flags) |
252 | { |
253 | panic("swapfile_pager_data_return: should never get called" ); |
254 | return KERN_FAILURE; |
255 | } |
256 | |
257 | kern_return_t |
258 | ( |
259 | __unused memory_object_t mem_obj, |
260 | __unused memory_object_offset_t offset, |
261 | __unused memory_object_cluster_size_t data_cnt) |
262 | { |
263 | panic("swapfile_pager_data_initialize: should never get called" ); |
264 | return KERN_FAILURE; |
265 | } |
266 | |
267 | /* |
268 | * swapfile_pager_data_request() |
269 | * |
270 | * Handles page-in requests from VM. |
271 | */ |
272 | kern_return_t |
273 | ( |
274 | memory_object_t mem_obj, |
275 | memory_object_offset_t offset, |
276 | memory_object_cluster_size_t length, |
277 | #if !DEBUG |
278 | __unused |
279 | #endif |
280 | vm_prot_t protection_required, |
281 | __unused memory_object_fault_info_t mo_fault_info) |
282 | { |
283 | swapfile_pager_t ; |
284 | memory_object_control_t mo_control; |
285 | upl_t upl; |
286 | int upl_flags; |
287 | upl_size_t upl_size; |
288 | upl_page_info_t *upl_pl = NULL; |
289 | unsigned int pl_count; |
290 | vm_object_t dst_object; |
291 | kern_return_t kr, retval; |
292 | vm_offset_t kernel_mapping; |
293 | char *dst_ptr; |
294 | vm_offset_t cur_offset; |
295 | |
296 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_data_request: %p, %llx, %x, %x\n" , mem_obj, offset, length, protection_required)); |
297 | |
298 | kernel_mapping = 0; |
299 | upl = NULL; |
300 | upl_pl = NULL; |
301 | |
302 | pager = swapfile_pager_lookup(mem_obj); |
303 | assert(pager->is_ready); |
304 | assert(os_ref_get_count_raw(&pager->swp_pgr_hdr_ref) > 1); /* pager is alive and mapped */ |
305 | |
306 | PAGER_DEBUG(PAGER_PAGEIN, ("swapfile_pager_data_request: %p, %llx, %x, %x, pager %p\n" , mem_obj, offset, length, protection_required, pager)); |
307 | |
308 | /* |
309 | * Gather in a UPL all the VM pages requested by VM. |
310 | */ |
311 | mo_control = pager->swp_pgr_hdr.mo_control; |
312 | |
313 | upl_size = length; |
314 | upl_flags = |
315 | UPL_RET_ONLY_ABSENT | |
316 | UPL_SET_LITE | |
317 | UPL_NO_SYNC | |
318 | UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ |
319 | UPL_SET_INTERNAL; |
320 | pl_count = 0; |
321 | kr = memory_object_upl_request(memory_control: mo_control, |
322 | offset, size: upl_size, |
323 | upl: &upl, NULL, NULL, cntrl_flags: upl_flags, VM_KERN_MEMORY_OSFMK); |
324 | if (kr != KERN_SUCCESS) { |
325 | retval = kr; |
326 | goto done; |
327 | } |
328 | dst_object = memory_object_control_to_vm_object(control: mo_control); |
329 | assert(dst_object != VM_OBJECT_NULL); |
330 | |
331 | |
332 | /* |
333 | * Reserve a virtual page in the kernel address space to map each |
334 | * destination physical page when it's its turn to be processed. |
335 | */ |
336 | kr = kmem_alloc(map: kernel_map, addrp: &kernel_mapping, PAGE_SIZE, |
337 | flags: KMA_DATA | KMA_KOBJECT | KMA_PAGEABLE, VM_KERN_MEMORY_NONE); |
338 | if (kr != KERN_SUCCESS) { |
339 | retval = kr; |
340 | goto done; |
341 | } |
342 | dst_ptr = (char *)kernel_mapping; |
343 | |
344 | /* |
345 | * Fill in the contents of the pages requested by VM. |
346 | */ |
347 | upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl); |
348 | pl_count = length / PAGE_SIZE; |
349 | for (cur_offset = 0; cur_offset < length; cur_offset += PAGE_SIZE) { |
350 | ppnum_t dst_pnum; |
351 | |
352 | if (!upl_page_present(upl: upl_pl, index: (int)(cur_offset / PAGE_SIZE))) { |
353 | /* this page is not in the UPL: skip it */ |
354 | continue; |
355 | } |
356 | |
357 | /* |
358 | * Establish an explicit pmap mapping of the destination |
359 | * physical page. |
360 | * We can't do a regular VM mapping because the VM page |
361 | * is "busy". |
362 | */ |
363 | dst_pnum = (ppnum_t) |
364 | upl_phys_page(upl: upl_pl, index: (int)(cur_offset / PAGE_SIZE)); |
365 | assert(dst_pnum != 0); |
366 | retval = pmap_enter(pmap: kernel_pmap, |
367 | v: kernel_mapping, |
368 | pn: dst_pnum, |
369 | VM_PROT_READ | VM_PROT_WRITE, |
370 | VM_PROT_NONE, |
371 | flags: 0, |
372 | TRUE, |
373 | mapping_type: PMAP_MAPPING_TYPE_INFER); |
374 | |
375 | assert(retval == KERN_SUCCESS); |
376 | |
377 | if (retval != KERN_SUCCESS) { |
378 | goto done; |
379 | } |
380 | |
381 | memset(s: dst_ptr, c: '\0', PAGE_SIZE); |
382 | /* add an end-of-line to keep line counters happy */ |
383 | dst_ptr[PAGE_SIZE - 1] = '\n'; |
384 | |
385 | /* |
386 | * Remove the pmap mapping of the destination page |
387 | * in the kernel. |
388 | */ |
389 | pmap_remove(map: kernel_pmap, |
390 | s: (addr64_t) kernel_mapping, |
391 | e: (addr64_t) (kernel_mapping + PAGE_SIZE_64)); |
392 | } |
393 | |
394 | retval = KERN_SUCCESS; |
395 | done: |
396 | if (upl != NULL) { |
397 | /* clean up the UPL */ |
398 | |
399 | /* |
400 | * The pages are currently dirty because we've just been |
401 | * writing on them, but as far as we're concerned, they're |
402 | * clean since they contain their "original" contents as |
403 | * provided by us, the pager. |
404 | * Tell the UPL to mark them "clean". |
405 | */ |
406 | upl_clear_dirty(upl, TRUE); |
407 | |
408 | /* abort or commit the UPL */ |
409 | if (retval != KERN_SUCCESS) { |
410 | upl_abort(upl_object: upl, abort_cond: 0); |
411 | } else { |
412 | boolean_t empty; |
413 | assertf(page_aligned(upl->u_offset) && page_aligned(upl->u_size), |
414 | "upl %p offset 0x%llx size 0x%x" , |
415 | upl, upl->u_offset, upl->u_size); |
416 | upl_commit_range(upl_object: upl, offset: 0, size: upl->u_size, |
417 | UPL_COMMIT_CS_VALIDATED, |
418 | page_list: upl_pl, page_listCnt: pl_count, empty: &empty); |
419 | } |
420 | |
421 | /* and deallocate the UPL */ |
422 | upl_deallocate(upl); |
423 | upl = NULL; |
424 | } |
425 | |
426 | if (kernel_mapping != 0) { |
427 | /* clean up the mapping of the source and destination pages */ |
428 | kmem_free(map: kernel_map, addr: kernel_mapping, PAGE_SIZE); |
429 | kernel_mapping = 0; |
430 | } |
431 | |
432 | return retval; |
433 | } |
434 | |
435 | /* |
436 | * swapfile_pager_reference() |
437 | * |
438 | * Get a reference on this memory object. |
439 | * For external usage only. Assumes that the initial reference count is not 0, |
440 | * i.e one should not "revive" a dead pager this way. |
441 | */ |
442 | void |
443 | ( |
444 | memory_object_t mem_obj) |
445 | { |
446 | swapfile_pager_t ; |
447 | |
448 | pager = swapfile_pager_lookup(mem_obj); |
449 | |
450 | lck_mtx_lock(lck: &swapfile_pager_lock); |
451 | os_ref_retain_locked_raw(&pager->swp_pgr_hdr_ref, NULL); |
452 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
453 | } |
454 | |
455 | |
456 | /* |
457 | * swapfile_pager_dequeue: |
458 | * |
459 | * Removes a pager from the list of pagers. |
460 | * |
461 | * The caller must hold "swapfile_pager_lock". |
462 | */ |
463 | void |
464 | ( |
465 | swapfile_pager_t ) |
466 | { |
467 | assert(!pager->is_mapped); |
468 | |
469 | queue_remove(&swapfile_pager_queue, |
470 | pager, |
471 | swapfile_pager_t, |
472 | pager_queue); |
473 | pager->pager_queue.next = NULL; |
474 | pager->pager_queue.prev = NULL; |
475 | |
476 | swapfile_pager_count--; |
477 | } |
478 | |
479 | /* |
480 | * swapfile_pager_terminate_internal: |
481 | * |
482 | * Trigger the asynchronous termination of the memory object associated |
483 | * with this pager. |
484 | * When the memory object is terminated, there will be one more call |
485 | * to memory_object_deallocate() (i.e. swapfile_pager_deallocate()) |
486 | * to finish the clean up. |
487 | * |
488 | * "swapfile_pager_lock" should not be held by the caller. |
489 | * We don't need the lock because the pager has already been removed from |
490 | * the pagers' list and is now ours exclusively. |
491 | */ |
492 | void |
493 | ( |
494 | swapfile_pager_t ) |
495 | { |
496 | assert(pager->is_ready); |
497 | assert(!pager->is_mapped); |
498 | |
499 | if (pager->swapfile_vnode != NULL) { |
500 | pager->swapfile_vnode = NULL; |
501 | } |
502 | |
503 | /* trigger the destruction of the memory object */ |
504 | memory_object_destroy(memory_control: pager->swp_pgr_hdr.mo_control, reason: VM_OBJECT_DESTROY_UNKNOWN_REASON); |
505 | } |
506 | |
507 | /* |
508 | * swapfile_pager_deallocate_internal() |
509 | * |
510 | * Release a reference on this pager and free it when the last |
511 | * reference goes away. |
512 | * Can be called with swapfile_pager_lock held or not but always returns |
513 | * with it unlocked. |
514 | */ |
515 | void |
516 | ( |
517 | swapfile_pager_t , |
518 | boolean_t locked) |
519 | { |
520 | os_ref_count_t ref_count; |
521 | |
522 | if (!locked) { |
523 | lck_mtx_lock(lck: &swapfile_pager_lock); |
524 | } |
525 | |
526 | /* drop a reference on this pager */ |
527 | ref_count = os_ref_release_locked_raw(&pager->swp_pgr_hdr_ref, NULL); |
528 | |
529 | if (ref_count == 1) { |
530 | /* |
531 | * Only the "named" reference is left, which means that |
532 | * no one is really holding on to this pager anymore. |
533 | * Terminate it. |
534 | */ |
535 | swapfile_pager_dequeue(pager); |
536 | /* the pager is all ours: no need for the lock now */ |
537 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
538 | swapfile_pager_terminate_internal(pager); |
539 | } else if (ref_count == 0) { |
540 | /* |
541 | * Dropped the existence reference; the memory object has |
542 | * been terminated. Do some final cleanup and release the |
543 | * pager structure. |
544 | */ |
545 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
546 | if (pager->swp_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { |
547 | memory_object_control_deallocate(control: pager->swp_pgr_hdr.mo_control); |
548 | pager->swp_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; |
549 | } |
550 | kfree_type(struct swapfile_pager, pager); |
551 | pager = SWAPFILE_PAGER_NULL; |
552 | } else { |
553 | /* there are still plenty of references: keep going... */ |
554 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
555 | } |
556 | |
557 | /* caution: lock is not held on return... */ |
558 | } |
559 | |
560 | /* |
561 | * swapfile_pager_deallocate() |
562 | * |
563 | * Release a reference on this pager and free it when the last |
564 | * reference goes away. |
565 | */ |
566 | void |
567 | ( |
568 | memory_object_t mem_obj) |
569 | { |
570 | swapfile_pager_t ; |
571 | |
572 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_deallocate: %p\n" , mem_obj)); |
573 | pager = swapfile_pager_lookup(mem_obj); |
574 | swapfile_pager_deallocate_internal(pager, FALSE); |
575 | } |
576 | |
577 | /* |
578 | * |
579 | */ |
580 | kern_return_t |
581 | ( |
582 | #if !DEBUG |
583 | __unused |
584 | #endif |
585 | memory_object_t mem_obj) |
586 | { |
587 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_terminate: %p\n" , mem_obj)); |
588 | |
589 | return KERN_SUCCESS; |
590 | } |
591 | |
592 | /* |
593 | * swapfile_pager_map() |
594 | * |
595 | * This allows VM to let us, the EMM, know that this memory object |
596 | * is currently mapped one or more times. This is called by VM each time |
597 | * the memory object gets mapped and we take one extra reference on the |
598 | * memory object to account for all its mappings. |
599 | */ |
600 | kern_return_t |
601 | ( |
602 | memory_object_t mem_obj, |
603 | __unused vm_prot_t prot) |
604 | { |
605 | swapfile_pager_t ; |
606 | |
607 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_map: %p\n" , mem_obj)); |
608 | |
609 | pager = swapfile_pager_lookup(mem_obj); |
610 | |
611 | lck_mtx_lock(lck: &swapfile_pager_lock); |
612 | assert(pager->is_ready); |
613 | assert(os_ref_get_count_raw(&pager->swp_pgr_hdr_ref) > 0); /* pager is alive */ |
614 | if (pager->is_mapped == FALSE) { |
615 | /* |
616 | * First mapping of this pager: take an extra reference |
617 | * that will remain until all the mappings of this pager |
618 | * are removed. |
619 | */ |
620 | pager->is_mapped = TRUE; |
621 | os_ref_retain_locked_raw(&pager->swp_pgr_hdr_ref, NULL); |
622 | } |
623 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
624 | |
625 | return KERN_SUCCESS; |
626 | } |
627 | |
628 | /* |
629 | * swapfile_pager_last_unmap() |
630 | * |
631 | * This is called by VM when this memory object is no longer mapped anywhere. |
632 | */ |
633 | kern_return_t |
634 | ( |
635 | memory_object_t mem_obj) |
636 | { |
637 | swapfile_pager_t ; |
638 | |
639 | PAGER_DEBUG(PAGER_ALL, |
640 | ("swapfile_pager_last_unmap: %p\n" , mem_obj)); |
641 | |
642 | pager = swapfile_pager_lookup(mem_obj); |
643 | |
644 | lck_mtx_lock(lck: &swapfile_pager_lock); |
645 | if (pager->is_mapped) { |
646 | /* |
647 | * All the mappings are gone, so let go of the one extra |
648 | * reference that represents all the mappings of this pager. |
649 | */ |
650 | pager->is_mapped = FALSE; |
651 | swapfile_pager_deallocate_internal(pager, TRUE); |
652 | /* caution: deallocate_internal() released the lock ! */ |
653 | } else { |
654 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
655 | } |
656 | |
657 | return KERN_SUCCESS; |
658 | } |
659 | |
660 | |
661 | /* |
662 | * |
663 | */ |
664 | swapfile_pager_t |
665 | ( |
666 | memory_object_t mem_obj) |
667 | { |
668 | swapfile_pager_t ; |
669 | |
670 | assert(mem_obj->mo_pager_ops == &swapfile_pager_ops); |
671 | __IGNORE_WCASTALIGN(pager = (swapfile_pager_t) mem_obj); |
672 | assert(os_ref_get_count_raw(&pager->swp_pgr_hdr_ref) > 0); |
673 | return pager; |
674 | } |
675 | |
676 | swapfile_pager_t |
677 | ( |
678 | struct vnode *vp) |
679 | { |
680 | swapfile_pager_t , ; |
681 | memory_object_control_t control; |
682 | kern_return_t kr; |
683 | |
684 | pager = kalloc_type(struct swapfile_pager, Z_WAITOK | Z_NOFAIL); |
685 | |
686 | /* |
687 | * The vm_map call takes both named entry ports and raw memory |
688 | * objects in the same parameter. We need to make sure that |
689 | * vm_map does not see this object as a named entry port. So, |
690 | * we reserve the second word in the object for a fake ip_kotype |
691 | * setting - that will tell vm_map to use it as a memory object. |
692 | */ |
693 | pager->swp_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; |
694 | pager->swp_pgr_hdr.mo_pager_ops = &swapfile_pager_ops; |
695 | pager->swp_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; |
696 | |
697 | pager->is_ready = FALSE;/* not ready until it has a "name" */ |
698 | os_ref_init_raw(&pager->swp_pgr_hdr_ref, NULL); /* setup reference */ |
699 | pager->is_mapped = FALSE; |
700 | pager->swapfile_vnode = vp; |
701 | |
702 | lck_mtx_lock(lck: &swapfile_pager_lock); |
703 | /* see if anyone raced us to create a pager for the same object */ |
704 | queue_iterate(&swapfile_pager_queue, |
705 | pager2, |
706 | swapfile_pager_t, |
707 | pager_queue) { |
708 | if (pager2->swapfile_vnode == vp) { |
709 | break; |
710 | } |
711 | } |
712 | if (!queue_end(&swapfile_pager_queue, |
713 | (queue_entry_t) pager2)) { |
714 | /* while we hold the lock, transfer our setup ref to winner */ |
715 | os_ref_retain_locked_raw(&pager2->swp_pgr_hdr_ref, NULL); |
716 | /* we lost the race, down with the loser... */ |
717 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
718 | pager->swapfile_vnode = NULL; |
719 | kfree_type(struct swapfile_pager, pager); |
720 | /* ... and go with the winner */ |
721 | pager = pager2; |
722 | /* let the winner make sure the pager gets ready */ |
723 | return pager; |
724 | } |
725 | |
726 | /* enter new pager at the head of our list of pagers */ |
727 | queue_enter_first(&swapfile_pager_queue, |
728 | pager, |
729 | swapfile_pager_t, |
730 | pager_queue); |
731 | swapfile_pager_count++; |
732 | if (swapfile_pager_count > swapfile_pager_count_max) { |
733 | swapfile_pager_count_max = swapfile_pager_count; |
734 | } |
735 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
736 | |
737 | kr = memory_object_create_named(pager: (memory_object_t) pager, |
738 | size: 0, |
739 | control: &control); |
740 | assert(kr == KERN_SUCCESS); |
741 | |
742 | memory_object_mark_trusted(control); |
743 | |
744 | lck_mtx_lock(lck: &swapfile_pager_lock); |
745 | /* the new pager is now ready to be used */ |
746 | pager->is_ready = TRUE; |
747 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
748 | |
749 | /* wakeup anyone waiting for this pager to be ready */ |
750 | thread_wakeup(&pager->is_ready); |
751 | |
752 | return pager; |
753 | } |
754 | |
755 | /* |
756 | * swapfile_pager_setup() |
757 | * |
758 | * Provide the caller with a memory object backed by the provided |
759 | * "backing_object" VM object. If such a memory object already exists, |
760 | * re-use it, otherwise create a new memory object. |
761 | */ |
762 | memory_object_t |
763 | ( |
764 | struct vnode *vp) |
765 | { |
766 | swapfile_pager_t ; |
767 | |
768 | lck_mtx_lock(lck: &swapfile_pager_lock); |
769 | |
770 | queue_iterate(&swapfile_pager_queue, |
771 | pager, |
772 | swapfile_pager_t, |
773 | pager_queue) { |
774 | if (pager->swapfile_vnode == vp) { |
775 | break; |
776 | } |
777 | } |
778 | if (queue_end(&swapfile_pager_queue, |
779 | (queue_entry_t) pager)) { |
780 | /* no existing pager for this backing object */ |
781 | pager = SWAPFILE_PAGER_NULL; |
782 | } else { |
783 | /* make sure pager doesn't disappear */ |
784 | os_ref_retain_raw(&pager->swp_pgr_hdr_ref, NULL); |
785 | } |
786 | |
787 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
788 | |
789 | if (pager == SWAPFILE_PAGER_NULL) { |
790 | pager = swapfile_pager_create(vp); |
791 | if (pager == SWAPFILE_PAGER_NULL) { |
792 | return MEMORY_OBJECT_NULL; |
793 | } |
794 | } |
795 | |
796 | lck_mtx_lock(lck: &swapfile_pager_lock); |
797 | while (!pager->is_ready) { |
798 | lck_mtx_sleep(lck: &swapfile_pager_lock, |
799 | lck_sleep_action: LCK_SLEEP_DEFAULT, |
800 | event: &pager->is_ready, |
801 | THREAD_UNINT); |
802 | } |
803 | lck_mtx_unlock(lck: &swapfile_pager_lock); |
804 | |
805 | return (memory_object_t) pager; |
806 | } |
807 | |
808 | memory_object_control_t |
809 | ( |
810 | memory_object_t mem_obj) |
811 | { |
812 | swapfile_pager_t ; |
813 | |
814 | if (mem_obj == MEMORY_OBJECT_NULL || |
815 | mem_obj->mo_pager_ops != &swapfile_pager_ops) { |
816 | return MEMORY_OBJECT_CONTROL_NULL; |
817 | } |
818 | pager = swapfile_pager_lookup(mem_obj); |
819 | return pager->swp_pgr_hdr.mo_control; |
820 | } |
821 | |