1 | /* |
2 | * Copyright (c) 2008 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <mach/kern_return.h> |
30 | #include <mach/memory_object_control.h> |
31 | #include <mach/upl.h> |
32 | |
33 | #include <kern/ipc_kobject.h> |
34 | #include <kern/kalloc.h> |
35 | #include <kern/queue.h> |
36 | |
37 | #include <vm/vm_kern.h> |
38 | #include <vm/vm_map.h> |
39 | #include <vm/vm_pageout.h> |
40 | #include <vm/vm_protos.h> |
41 | |
42 | |
43 | /* |
44 | * APPLE SWAPFILE MEMORY PAGER |
45 | * |
46 | * This external memory manager (EMM) handles mappings of the swap files. |
47 | * Swap files are not regular files and are used solely to store contents of |
48 | * anonymous memory mappings while not resident in memory. |
49 | * There's no valid reason to map a swap file. This just puts extra burden |
50 | * on the system, is potentially a security issue and is not reliable since |
51 | * the contents can change at any time with pageout operations. |
52 | * Here are some of the issues with mapping a swap file. |
53 | * * PERFORMANCE: |
54 | * Each page in the swap file belong to an anonymous memory object. Mapping |
55 | * the swap file makes those pages also accessible via a vnode memory |
56 | * object and each page can now be resident twice. |
57 | * * SECURITY: |
58 | * Mapping a swap file allows access to other processes' memory. Swap files |
59 | * are only accessible by the "root" super-user, who can already access any |
60 | * process's memory, so this is not a real issue but if permissions on the |
61 | * swap file got changed, it could become one. |
62 | * Swap files are not "zero-filled" on creation, so until their contents are |
63 | * overwritten with pageout operations, they still contain whatever was on |
64 | * the disk blocks they were allocated. The "super-user" could see the |
65 | * contents of free blocks anyway, so this is not a new security issue but |
66 | * it may be perceive as one. |
67 | * |
68 | * We can't legitimately prevent a user process with appropriate privileges |
69 | * from mapping a swap file, but we can prevent it from accessing its actual |
70 | * contents. |
71 | * This pager mostly handles page-in request (from memory_object_data_request()) |
72 | * for swap file mappings and just returns bogus data. |
73 | * Pageouts are not handled, so mmap() has to make sure it does not allow |
74 | * writable (i.e. MAP_SHARED and PROT_WRITE) mappings of swap files. |
75 | */ |
76 | |
77 | /* forward declarations */ |
78 | void swapfile_pager_reference(memory_object_t mem_obj); |
79 | void swapfile_pager_deallocate(memory_object_t mem_obj); |
80 | kern_return_t swapfile_pager_init(memory_object_t mem_obj, |
81 | memory_object_control_t control, |
82 | memory_object_cluster_size_t pg_size); |
83 | kern_return_t swapfile_pager_terminate(memory_object_t mem_obj); |
84 | kern_return_t swapfile_pager_data_request(memory_object_t mem_obj, |
85 | memory_object_offset_t offset, |
86 | memory_object_cluster_size_t length, |
87 | vm_prot_t protection_required, |
88 | memory_object_fault_info_t fault_info); |
89 | kern_return_t swapfile_pager_data_return(memory_object_t mem_obj, |
90 | memory_object_offset_t offset, |
91 | memory_object_cluster_size_t data_cnt, |
92 | memory_object_offset_t *resid_offset, |
93 | int *io_error, |
94 | boolean_t dirty, |
95 | boolean_t kernel_copy, |
96 | int upl_flags); |
97 | kern_return_t swapfile_pager_data_initialize(memory_object_t mem_obj, |
98 | memory_object_offset_t offset, |
99 | memory_object_cluster_size_t data_cnt); |
100 | kern_return_t swapfile_pager_data_unlock(memory_object_t mem_obj, |
101 | memory_object_offset_t offset, |
102 | memory_object_size_t size, |
103 | vm_prot_t desired_access); |
104 | kern_return_t swapfile_pager_synchronize(memory_object_t mem_obj, |
105 | memory_object_offset_t offset, |
106 | memory_object_size_t length, |
107 | vm_sync_t sync_flags); |
108 | kern_return_t swapfile_pager_map(memory_object_t mem_obj, |
109 | vm_prot_t prot); |
110 | kern_return_t swapfile_pager_last_unmap(memory_object_t mem_obj); |
111 | |
112 | /* |
113 | * Vector of VM operations for this EMM. |
114 | * These routines are invoked by VM via the memory_object_*() interfaces. |
115 | */ |
116 | const struct memory_object_pager_ops = { |
117 | swapfile_pager_reference, |
118 | swapfile_pager_deallocate, |
119 | swapfile_pager_init, |
120 | swapfile_pager_terminate, |
121 | swapfile_pager_data_request, |
122 | swapfile_pager_data_return, |
123 | swapfile_pager_data_initialize, |
124 | swapfile_pager_data_unlock, |
125 | swapfile_pager_synchronize, |
126 | swapfile_pager_map, |
127 | swapfile_pager_last_unmap, |
128 | NULL, /* data_reclaim */ |
129 | "swapfile pager" |
130 | }; |
131 | |
132 | /* |
133 | * The "swapfile_pager" describes a memory object backed by |
134 | * the "swapfile" EMM. |
135 | */ |
136 | typedef struct { |
137 | /* mandatory generic header */ |
138 | struct memory_object ; |
139 | |
140 | /* pager-specific data */ |
141 | queue_chain_t ; /* next & prev pagers */ |
142 | unsigned int ; /* reference count */ |
143 | boolean_t ; /* is this pager ready ? */ |
144 | boolean_t ; /* is this pager mapped ? */ |
145 | struct vnode *;/* the swapfile's vnode */ |
146 | } *; |
147 | #define ((swapfile_pager_t) NULL) |
148 | |
149 | /* |
150 | * List of memory objects managed by this EMM. |
151 | * The list is protected by the "swapfile_pager_lock" lock. |
152 | */ |
153 | int = 0; /* number of pagers */ |
154 | queue_head_t ; |
155 | decl_lck_mtx_data(,) |
156 | |
157 | /* |
158 | * Statistics & counters. |
159 | */ |
160 | int = 0; |
161 | |
162 | |
163 | lck_grp_t ; |
164 | lck_grp_attr_t ; |
165 | lck_attr_t ; |
166 | |
167 | |
168 | /* internal prototypes */ |
169 | swapfile_pager_t swapfile_pager_create(struct vnode *vp); |
170 | swapfile_pager_t swapfile_pager_lookup(memory_object_t mem_obj); |
171 | void swapfile_pager_dequeue(swapfile_pager_t ); |
172 | void swapfile_pager_deallocate_internal(swapfile_pager_t , |
173 | boolean_t locked); |
174 | void swapfile_pager_terminate_internal(swapfile_pager_t ); |
175 | |
176 | |
177 | #if DEBUG |
178 | int swapfile_pagerdebug = 0; |
179 | #define PAGER_ALL 0xffffffff |
180 | #define PAGER_INIT 0x00000001 |
181 | #define PAGER_PAGEIN 0x00000002 |
182 | |
183 | #define PAGER_DEBUG(LEVEL, A) \ |
184 | MACRO_BEGIN \ |
185 | if ((swapfile_pagerdebug & LEVEL)==LEVEL) { \ |
186 | printf A; \ |
187 | } \ |
188 | MACRO_END |
189 | #else |
190 | #define (LEVEL, A) |
191 | #endif |
192 | |
193 | |
194 | void |
195 | (void) |
196 | { |
197 | lck_grp_attr_setdefault(&swapfile_pager_lck_grp_attr); |
198 | lck_grp_init(&swapfile_pager_lck_grp, "swapfile pager" , &swapfile_pager_lck_grp_attr); |
199 | lck_attr_setdefault(&swapfile_pager_lck_attr); |
200 | lck_mtx_init(&swapfile_pager_lock, &swapfile_pager_lck_grp, &swapfile_pager_lck_attr); |
201 | queue_init(&swapfile_pager_queue); |
202 | } |
203 | |
204 | /* |
205 | * swapfile_pager_init() |
206 | * |
207 | * Initialize the memory object and makes it ready to be used and mapped. |
208 | */ |
209 | kern_return_t |
210 | ( |
211 | memory_object_t mem_obj, |
212 | memory_object_control_t control, |
213 | #if !DEBUG |
214 | __unused |
215 | #endif |
216 | memory_object_cluster_size_t pg_size) |
217 | { |
218 | swapfile_pager_t ; |
219 | kern_return_t kr; |
220 | memory_object_attr_info_data_t attributes; |
221 | |
222 | PAGER_DEBUG(PAGER_ALL, |
223 | ("swapfile_pager_init: %p, %p, %x\n" , |
224 | mem_obj, control, pg_size)); |
225 | |
226 | if (control == MEMORY_OBJECT_CONTROL_NULL) |
227 | return KERN_INVALID_ARGUMENT; |
228 | |
229 | pager = swapfile_pager_lookup(mem_obj); |
230 | |
231 | memory_object_control_reference(control); |
232 | |
233 | pager->swp_pgr_hdr.mo_control = control; |
234 | |
235 | attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY; |
236 | attributes.cluster_size = (1 << (PAGE_SHIFT)); |
237 | attributes.may_cache_object = FALSE; |
238 | attributes.temporary = TRUE; |
239 | |
240 | kr = memory_object_change_attributes( |
241 | control, |
242 | MEMORY_OBJECT_ATTRIBUTE_INFO, |
243 | (memory_object_info_t) &attributes, |
244 | MEMORY_OBJECT_ATTR_INFO_COUNT); |
245 | if (kr != KERN_SUCCESS) |
246 | panic("swapfile_pager_init: " |
247 | "memory_object_change_attributes() failed" ); |
248 | |
249 | return KERN_SUCCESS; |
250 | } |
251 | |
252 | /* |
253 | * swapfile_data_return() |
254 | * |
255 | * Handles page-out requests from VM. This should never happen since |
256 | * the pages provided by this EMM are not supposed to be dirty or dirtied |
257 | * and VM should simply discard the contents and reclaim the pages if it |
258 | * needs to. |
259 | */ |
260 | kern_return_t |
261 | ( |
262 | __unused memory_object_t mem_obj, |
263 | __unused memory_object_offset_t offset, |
264 | __unused memory_object_cluster_size_t data_cnt, |
265 | __unused memory_object_offset_t *resid_offset, |
266 | __unused int *io_error, |
267 | __unused boolean_t dirty, |
268 | __unused boolean_t kernel_copy, |
269 | __unused int upl_flags) |
270 | { |
271 | panic("swapfile_pager_data_return: should never get called" ); |
272 | return KERN_FAILURE; |
273 | } |
274 | |
275 | kern_return_t |
276 | ( |
277 | __unused memory_object_t mem_obj, |
278 | __unused memory_object_offset_t offset, |
279 | __unused memory_object_cluster_size_t data_cnt) |
280 | { |
281 | panic("swapfile_pager_data_initialize: should never get called" ); |
282 | return KERN_FAILURE; |
283 | } |
284 | |
285 | kern_return_t |
286 | ( |
287 | __unused memory_object_t mem_obj, |
288 | __unused memory_object_offset_t offset, |
289 | __unused memory_object_size_t size, |
290 | __unused vm_prot_t desired_access) |
291 | { |
292 | return KERN_FAILURE; |
293 | } |
294 | |
295 | /* |
296 | * swapfile_pager_data_request() |
297 | * |
298 | * Handles page-in requests from VM. |
299 | */ |
300 | kern_return_t |
301 | ( |
302 | memory_object_t mem_obj, |
303 | memory_object_offset_t offset, |
304 | memory_object_cluster_size_t length, |
305 | #if !DEBUG |
306 | __unused |
307 | #endif |
308 | vm_prot_t protection_required, |
309 | __unused memory_object_fault_info_t mo_fault_info) |
310 | { |
311 | swapfile_pager_t ; |
312 | memory_object_control_t mo_control; |
313 | upl_t upl; |
314 | int upl_flags; |
315 | upl_size_t upl_size; |
316 | upl_page_info_t *upl_pl = NULL; |
317 | unsigned int pl_count; |
318 | vm_object_t dst_object; |
319 | kern_return_t kr, retval; |
320 | vm_map_offset_t kernel_mapping; |
321 | vm_offset_t dst_vaddr; |
322 | char *dst_ptr; |
323 | vm_offset_t cur_offset; |
324 | vm_map_entry_t map_entry; |
325 | |
326 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_data_request: %p, %llx, %x, %x\n" , mem_obj, offset, length, protection_required)); |
327 | |
328 | kernel_mapping = 0; |
329 | upl = NULL; |
330 | upl_pl = NULL; |
331 | |
332 | pager = swapfile_pager_lookup(mem_obj); |
333 | assert(pager->is_ready); |
334 | assert(pager->ref_count > 1); /* pager is alive and mapped */ |
335 | |
336 | PAGER_DEBUG(PAGER_PAGEIN, ("swapfile_pager_data_request: %p, %llx, %x, %x, pager %p\n" , mem_obj, offset, length, protection_required, pager)); |
337 | |
338 | /* |
339 | * Gather in a UPL all the VM pages requested by VM. |
340 | */ |
341 | mo_control = pager->swp_pgr_hdr.mo_control; |
342 | |
343 | upl_size = length; |
344 | upl_flags = |
345 | UPL_RET_ONLY_ABSENT | |
346 | UPL_SET_LITE | |
347 | UPL_NO_SYNC | |
348 | UPL_CLEAN_IN_PLACE | /* triggers UPL_CLEAR_DIRTY */ |
349 | UPL_SET_INTERNAL; |
350 | pl_count = 0; |
351 | kr = memory_object_upl_request(mo_control, |
352 | offset, upl_size, |
353 | &upl, NULL, NULL, upl_flags, VM_KERN_MEMORY_OSFMK); |
354 | if (kr != KERN_SUCCESS) { |
355 | retval = kr; |
356 | goto done; |
357 | } |
358 | dst_object = mo_control->moc_object; |
359 | assert(dst_object != VM_OBJECT_NULL); |
360 | |
361 | |
362 | /* |
363 | * Reserve a virtual page in the kernel address space to map each |
364 | * destination physical page when it's its turn to be processed. |
365 | */ |
366 | vm_object_reference(kernel_object); /* ref. for mapping */ |
367 | kr = vm_map_find_space(kernel_map, |
368 | &kernel_mapping, |
369 | PAGE_SIZE_64, |
370 | 0, |
371 | 0, |
372 | VM_MAP_KERNEL_FLAGS_NONE, |
373 | VM_KERN_MEMORY_NONE, |
374 | &map_entry); |
375 | if (kr != KERN_SUCCESS) { |
376 | vm_object_deallocate(kernel_object); |
377 | retval = kr; |
378 | goto done; |
379 | } |
380 | VME_OBJECT_SET(map_entry, kernel_object); |
381 | VME_OFFSET_SET(map_entry, kernel_mapping - VM_MIN_KERNEL_ADDRESS); |
382 | vm_map_unlock(kernel_map); |
383 | dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping); |
384 | dst_ptr = (char *) dst_vaddr; |
385 | |
386 | /* |
387 | * Fill in the contents of the pages requested by VM. |
388 | */ |
389 | upl_pl = UPL_GET_INTERNAL_PAGE_LIST(upl); |
390 | pl_count = length / PAGE_SIZE; |
391 | for (cur_offset = 0; cur_offset < length; cur_offset += PAGE_SIZE) { |
392 | ppnum_t dst_pnum; |
393 | |
394 | if (!upl_page_present(upl_pl, (int)(cur_offset / PAGE_SIZE))) { |
395 | /* this page is not in the UPL: skip it */ |
396 | continue; |
397 | } |
398 | |
399 | /* |
400 | * Establish an explicit pmap mapping of the destination |
401 | * physical page. |
402 | * We can't do a regular VM mapping because the VM page |
403 | * is "busy". |
404 | */ |
405 | dst_pnum = (ppnum_t) |
406 | upl_phys_page(upl_pl, (int)(cur_offset / PAGE_SIZE)); |
407 | assert(dst_pnum != 0); |
408 | retval = pmap_enter(kernel_pmap, |
409 | kernel_mapping, |
410 | dst_pnum, |
411 | VM_PROT_READ | VM_PROT_WRITE, |
412 | VM_PROT_NONE, |
413 | 0, |
414 | TRUE); |
415 | |
416 | assert(retval == KERN_SUCCESS); |
417 | |
418 | if (retval != KERN_SUCCESS) { |
419 | goto done; |
420 | } |
421 | |
422 | memset(dst_ptr, '\0', PAGE_SIZE); |
423 | /* add an end-of-line to keep line counters happy */ |
424 | dst_ptr[PAGE_SIZE-1] = '\n'; |
425 | |
426 | /* |
427 | * Remove the pmap mapping of the destination page |
428 | * in the kernel. |
429 | */ |
430 | pmap_remove(kernel_pmap, |
431 | (addr64_t) kernel_mapping, |
432 | (addr64_t) (kernel_mapping + PAGE_SIZE_64)); |
433 | |
434 | } |
435 | |
436 | retval = KERN_SUCCESS; |
437 | done: |
438 | if (upl != NULL) { |
439 | /* clean up the UPL */ |
440 | |
441 | /* |
442 | * The pages are currently dirty because we've just been |
443 | * writing on them, but as far as we're concerned, they're |
444 | * clean since they contain their "original" contents as |
445 | * provided by us, the pager. |
446 | * Tell the UPL to mark them "clean". |
447 | */ |
448 | upl_clear_dirty(upl, TRUE); |
449 | |
450 | /* abort or commit the UPL */ |
451 | if (retval != KERN_SUCCESS) { |
452 | upl_abort(upl, 0); |
453 | } else { |
454 | boolean_t empty; |
455 | upl_commit_range(upl, 0, upl->size, |
456 | UPL_COMMIT_CS_VALIDATED, |
457 | upl_pl, pl_count, &empty); |
458 | } |
459 | |
460 | /* and deallocate the UPL */ |
461 | upl_deallocate(upl); |
462 | upl = NULL; |
463 | } |
464 | if (kernel_mapping != 0) { |
465 | /* clean up the mapping of the source and destination pages */ |
466 | kr = vm_map_remove(kernel_map, |
467 | kernel_mapping, |
468 | kernel_mapping + PAGE_SIZE_64, |
469 | VM_MAP_REMOVE_NO_FLAGS); |
470 | assert(kr == KERN_SUCCESS); |
471 | kernel_mapping = 0; |
472 | dst_vaddr = 0; |
473 | } |
474 | |
475 | return retval; |
476 | } |
477 | |
478 | /* |
479 | * swapfile_pager_reference() |
480 | * |
481 | * Get a reference on this memory object. |
482 | * For external usage only. Assumes that the initial reference count is not 0, |
483 | * i.e one should not "revive" a dead pager this way. |
484 | */ |
485 | void |
486 | ( |
487 | memory_object_t mem_obj) |
488 | { |
489 | swapfile_pager_t ; |
490 | |
491 | pager = swapfile_pager_lookup(mem_obj); |
492 | |
493 | lck_mtx_lock(&swapfile_pager_lock); |
494 | assert(pager->ref_count > 0); |
495 | pager->ref_count++; |
496 | lck_mtx_unlock(&swapfile_pager_lock); |
497 | } |
498 | |
499 | |
500 | /* |
501 | * swapfile_pager_dequeue: |
502 | * |
503 | * Removes a pager from the list of pagers. |
504 | * |
505 | * The caller must hold "swapfile_pager_lock". |
506 | */ |
507 | void |
508 | ( |
509 | swapfile_pager_t ) |
510 | { |
511 | assert(!pager->is_mapped); |
512 | |
513 | queue_remove(&swapfile_pager_queue, |
514 | pager, |
515 | swapfile_pager_t, |
516 | pager_queue); |
517 | pager->pager_queue.next = NULL; |
518 | pager->pager_queue.prev = NULL; |
519 | |
520 | swapfile_pager_count--; |
521 | } |
522 | |
523 | /* |
524 | * swapfile_pager_terminate_internal: |
525 | * |
526 | * Trigger the asynchronous termination of the memory object associated |
527 | * with this pager. |
528 | * When the memory object is terminated, there will be one more call |
529 | * to memory_object_deallocate() (i.e. swapfile_pager_deallocate()) |
530 | * to finish the clean up. |
531 | * |
532 | * "swapfile_pager_lock" should not be held by the caller. |
533 | * We don't need the lock because the pager has already been removed from |
534 | * the pagers' list and is now ours exclusively. |
535 | */ |
536 | void |
537 | ( |
538 | swapfile_pager_t ) |
539 | { |
540 | assert(pager->is_ready); |
541 | assert(!pager->is_mapped); |
542 | |
543 | if (pager->swapfile_vnode != NULL) { |
544 | pager->swapfile_vnode = NULL; |
545 | } |
546 | |
547 | /* trigger the destruction of the memory object */ |
548 | memory_object_destroy(pager->swp_pgr_hdr.mo_control, 0); |
549 | } |
550 | |
551 | /* |
552 | * swapfile_pager_deallocate_internal() |
553 | * |
554 | * Release a reference on this pager and free it when the last |
555 | * reference goes away. |
556 | * Can be called with swapfile_pager_lock held or not but always returns |
557 | * with it unlocked. |
558 | */ |
559 | void |
560 | ( |
561 | swapfile_pager_t , |
562 | boolean_t locked) |
563 | { |
564 | if (! locked) { |
565 | lck_mtx_lock(&swapfile_pager_lock); |
566 | } |
567 | |
568 | /* drop a reference on this pager */ |
569 | pager->ref_count--; |
570 | |
571 | if (pager->ref_count == 1) { |
572 | /* |
573 | * Only the "named" reference is left, which means that |
574 | * no one is really holding on to this pager anymore. |
575 | * Terminate it. |
576 | */ |
577 | swapfile_pager_dequeue(pager); |
578 | /* the pager is all ours: no need for the lock now */ |
579 | lck_mtx_unlock(&swapfile_pager_lock); |
580 | swapfile_pager_terminate_internal(pager); |
581 | } else if (pager->ref_count == 0) { |
582 | /* |
583 | * Dropped the existence reference; the memory object has |
584 | * been terminated. Do some final cleanup and release the |
585 | * pager structure. |
586 | */ |
587 | lck_mtx_unlock(&swapfile_pager_lock); |
588 | if (pager->swp_pgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { |
589 | memory_object_control_deallocate(pager->swp_pgr_hdr.mo_control); |
590 | pager->swp_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; |
591 | } |
592 | kfree(pager, sizeof (*pager)); |
593 | pager = SWAPFILE_PAGER_NULL; |
594 | } else { |
595 | /* there are still plenty of references: keep going... */ |
596 | lck_mtx_unlock(&swapfile_pager_lock); |
597 | } |
598 | |
599 | /* caution: lock is not held on return... */ |
600 | } |
601 | |
602 | /* |
603 | * swapfile_pager_deallocate() |
604 | * |
605 | * Release a reference on this pager and free it when the last |
606 | * reference goes away. |
607 | */ |
608 | void |
609 | ( |
610 | memory_object_t mem_obj) |
611 | { |
612 | swapfile_pager_t ; |
613 | |
614 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_deallocate: %p\n" , mem_obj)); |
615 | pager = swapfile_pager_lookup(mem_obj); |
616 | swapfile_pager_deallocate_internal(pager, FALSE); |
617 | } |
618 | |
619 | /* |
620 | * |
621 | */ |
622 | kern_return_t |
623 | ( |
624 | #if !DEBUG |
625 | __unused |
626 | #endif |
627 | memory_object_t mem_obj) |
628 | { |
629 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_terminate: %p\n" , mem_obj)); |
630 | |
631 | return KERN_SUCCESS; |
632 | } |
633 | |
634 | /* |
635 | * |
636 | */ |
637 | kern_return_t |
638 | ( |
639 | __unused memory_object_t mem_obbj, |
640 | __unused memory_object_offset_t offset, |
641 | __unused memory_object_size_t length, |
642 | __unused vm_sync_t sync_flags) |
643 | { |
644 | panic("swapfile_pager_synchronize: memory_object_synchronize no longer supported\n" ); |
645 | return (KERN_FAILURE); |
646 | } |
647 | |
648 | /* |
649 | * swapfile_pager_map() |
650 | * |
651 | * This allows VM to let us, the EMM, know that this memory object |
652 | * is currently mapped one or more times. This is called by VM each time |
653 | * the memory object gets mapped and we take one extra reference on the |
654 | * memory object to account for all its mappings. |
655 | */ |
656 | kern_return_t |
657 | ( |
658 | memory_object_t mem_obj, |
659 | __unused vm_prot_t prot) |
660 | { |
661 | swapfile_pager_t ; |
662 | |
663 | PAGER_DEBUG(PAGER_ALL, ("swapfile_pager_map: %p\n" , mem_obj)); |
664 | |
665 | pager = swapfile_pager_lookup(mem_obj); |
666 | |
667 | lck_mtx_lock(&swapfile_pager_lock); |
668 | assert(pager->is_ready); |
669 | assert(pager->ref_count > 0); /* pager is alive */ |
670 | if (pager->is_mapped == FALSE) { |
671 | /* |
672 | * First mapping of this pager: take an extra reference |
673 | * that will remain until all the mappings of this pager |
674 | * are removed. |
675 | */ |
676 | pager->is_mapped = TRUE; |
677 | pager->ref_count++; |
678 | } |
679 | lck_mtx_unlock(&swapfile_pager_lock); |
680 | |
681 | return KERN_SUCCESS; |
682 | } |
683 | |
684 | /* |
685 | * swapfile_pager_last_unmap() |
686 | * |
687 | * This is called by VM when this memory object is no longer mapped anywhere. |
688 | */ |
689 | kern_return_t |
690 | ( |
691 | memory_object_t mem_obj) |
692 | { |
693 | swapfile_pager_t ; |
694 | |
695 | PAGER_DEBUG(PAGER_ALL, |
696 | ("swapfile_pager_last_unmap: %p\n" , mem_obj)); |
697 | |
698 | pager = swapfile_pager_lookup(mem_obj); |
699 | |
700 | lck_mtx_lock(&swapfile_pager_lock); |
701 | if (pager->is_mapped) { |
702 | /* |
703 | * All the mappings are gone, so let go of the one extra |
704 | * reference that represents all the mappings of this pager. |
705 | */ |
706 | pager->is_mapped = FALSE; |
707 | swapfile_pager_deallocate_internal(pager, TRUE); |
708 | /* caution: deallocate_internal() released the lock ! */ |
709 | } else { |
710 | lck_mtx_unlock(&swapfile_pager_lock); |
711 | } |
712 | |
713 | return KERN_SUCCESS; |
714 | } |
715 | |
716 | |
717 | /* |
718 | * |
719 | */ |
720 | swapfile_pager_t |
721 | ( |
722 | memory_object_t mem_obj) |
723 | { |
724 | swapfile_pager_t ; |
725 | |
726 | assert(mem_obj->mo_pager_ops == &swapfile_pager_ops); |
727 | __IGNORE_WCASTALIGN(pager = (swapfile_pager_t) mem_obj); |
728 | assert(pager->ref_count > 0); |
729 | return pager; |
730 | } |
731 | |
732 | swapfile_pager_t |
733 | ( |
734 | struct vnode *vp) |
735 | { |
736 | swapfile_pager_t , ; |
737 | memory_object_control_t control; |
738 | kern_return_t kr; |
739 | |
740 | pager = (swapfile_pager_t) kalloc(sizeof (*pager)); |
741 | if (pager == SWAPFILE_PAGER_NULL) { |
742 | return SWAPFILE_PAGER_NULL; |
743 | } |
744 | |
745 | /* |
746 | * The vm_map call takes both named entry ports and raw memory |
747 | * objects in the same parameter. We need to make sure that |
748 | * vm_map does not see this object as a named entry port. So, |
749 | * we reserve the second word in the object for a fake ip_kotype |
750 | * setting - that will tell vm_map to use it as a memory object. |
751 | */ |
752 | pager->swp_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; |
753 | pager->swp_pgr_hdr.mo_pager_ops = &swapfile_pager_ops; |
754 | pager->swp_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; |
755 | |
756 | pager->is_ready = FALSE;/* not ready until it has a "name" */ |
757 | pager->ref_count = 1; /* setup reference */ |
758 | pager->is_mapped = FALSE; |
759 | pager->swapfile_vnode = vp; |
760 | |
761 | lck_mtx_lock(&swapfile_pager_lock); |
762 | /* see if anyone raced us to create a pager for the same object */ |
763 | queue_iterate(&swapfile_pager_queue, |
764 | pager2, |
765 | swapfile_pager_t, |
766 | pager_queue) { |
767 | if (pager2->swapfile_vnode == vp) { |
768 | break; |
769 | } |
770 | } |
771 | if (! queue_end(&swapfile_pager_queue, |
772 | (queue_entry_t) pager2)) { |
773 | /* while we hold the lock, transfer our setup ref to winner */ |
774 | pager2->ref_count++; |
775 | /* we lost the race, down with the loser... */ |
776 | lck_mtx_unlock(&swapfile_pager_lock); |
777 | pager->swapfile_vnode = NULL; |
778 | kfree(pager, sizeof (*pager)); |
779 | /* ... and go with the winner */ |
780 | pager = pager2; |
781 | /* let the winner make sure the pager gets ready */ |
782 | return pager; |
783 | } |
784 | |
785 | /* enter new pager at the head of our list of pagers */ |
786 | queue_enter_first(&swapfile_pager_queue, |
787 | pager, |
788 | swapfile_pager_t, |
789 | pager_queue); |
790 | swapfile_pager_count++; |
791 | if (swapfile_pager_count > swapfile_pager_count_max) { |
792 | swapfile_pager_count_max = swapfile_pager_count; |
793 | } |
794 | lck_mtx_unlock(&swapfile_pager_lock); |
795 | |
796 | kr = memory_object_create_named((memory_object_t) pager, |
797 | 0, |
798 | &control); |
799 | assert(kr == KERN_SUCCESS); |
800 | |
801 | lck_mtx_lock(&swapfile_pager_lock); |
802 | /* the new pager is now ready to be used */ |
803 | pager->is_ready = TRUE; |
804 | lck_mtx_unlock(&swapfile_pager_lock); |
805 | |
806 | /* wakeup anyone waiting for this pager to be ready */ |
807 | thread_wakeup(&pager->is_ready); |
808 | |
809 | return pager; |
810 | } |
811 | |
812 | /* |
813 | * swapfile_pager_setup() |
814 | * |
815 | * Provide the caller with a memory object backed by the provided |
816 | * "backing_object" VM object. If such a memory object already exists, |
817 | * re-use it, otherwise create a new memory object. |
818 | */ |
819 | memory_object_t |
820 | ( |
821 | struct vnode *vp) |
822 | { |
823 | swapfile_pager_t ; |
824 | |
825 | lck_mtx_lock(&swapfile_pager_lock); |
826 | |
827 | queue_iterate(&swapfile_pager_queue, |
828 | pager, |
829 | swapfile_pager_t, |
830 | pager_queue) { |
831 | if (pager->swapfile_vnode == vp) { |
832 | break; |
833 | } |
834 | } |
835 | if (queue_end(&swapfile_pager_queue, |
836 | (queue_entry_t) pager)) { |
837 | /* no existing pager for this backing object */ |
838 | pager = SWAPFILE_PAGER_NULL; |
839 | } else { |
840 | /* make sure pager doesn't disappear */ |
841 | pager->ref_count++; |
842 | } |
843 | |
844 | lck_mtx_unlock(&swapfile_pager_lock); |
845 | |
846 | if (pager == SWAPFILE_PAGER_NULL) { |
847 | pager = swapfile_pager_create(vp); |
848 | if (pager == SWAPFILE_PAGER_NULL) { |
849 | return MEMORY_OBJECT_NULL; |
850 | } |
851 | } |
852 | |
853 | lck_mtx_lock(&swapfile_pager_lock); |
854 | while (!pager->is_ready) { |
855 | lck_mtx_sleep(&swapfile_pager_lock, |
856 | LCK_SLEEP_DEFAULT, |
857 | &pager->is_ready, |
858 | THREAD_UNINT); |
859 | } |
860 | lck_mtx_unlock(&swapfile_pager_lock); |
861 | |
862 | return (memory_object_t) pager; |
863 | } |
864 | |
865 | memory_object_control_t |
866 | ( |
867 | memory_object_t mem_obj) |
868 | { |
869 | swapfile_pager_t ; |
870 | |
871 | if (mem_obj == MEMORY_OBJECT_NULL || |
872 | mem_obj->mo_pager_ops != &swapfile_pager_ops) { |
873 | return MEMORY_OBJECT_CONTROL_NULL; |
874 | } |
875 | pager = swapfile_pager_lookup(mem_obj); |
876 | return pager->swp_pgr_hdr.mo_control; |
877 | } |
878 | |