1 | /* |
2 | * Copyright (c) 2000-2009 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | /* |
57 | */ |
58 | /* |
59 | * File: vm/vm_pageout.h |
60 | * Author: Avadis Tevanian, Jr. |
61 | * Date: 1986 |
62 | * |
63 | * Declarations for the pageout daemon interface. |
64 | */ |
65 | |
66 | #ifndef _VM_VM_PAGEOUT_H_ |
67 | #define _VM_VM_PAGEOUT_H_ |
68 | |
69 | #ifdef KERNEL_PRIVATE |
70 | |
71 | #include <mach/mach_types.h> |
72 | #include <mach/boolean.h> |
73 | #include <mach/machine/vm_types.h> |
74 | #include <mach/memory_object_types.h> |
75 | |
76 | #include <kern/kern_types.h> |
77 | #include <kern/locks.h> |
78 | |
79 | #include <libkern/OSAtomic.h> |
80 | |
81 | |
82 | #include <vm/vm_options.h> |
83 | |
84 | #ifdef MACH_KERNEL_PRIVATE |
85 | #include <vm/vm_page.h> |
86 | #endif |
87 | |
88 | #include <sys/kdebug.h> |
89 | |
90 | #define VM_PAGE_AVAILABLE_COUNT() ((unsigned int)(vm_page_cleaned_count)) |
91 | |
92 | /* externally manipulated counters */ |
93 | extern unsigned int vm_pageout_cleaned_fault_reactivated; |
94 | |
95 | #if CONFIG_FREEZE |
96 | extern boolean_t memorystatus_freeze_enabled; |
97 | #endif |
98 | |
99 | #define VM_DYNAMIC_PAGING_ENABLED() (VM_CONFIG_COMPRESSOR_IS_ACTIVE) |
100 | |
101 | #if VM_PRESSURE_EVENTS |
102 | extern boolean_t vm_pressure_events_enabled; |
103 | #endif /* VM_PRESSURE_EVENTS */ |
104 | |
105 | |
106 | /* |
107 | * the following codes are used in the DBG_MACH_WORKINGSET subclass |
108 | * of the DBG_MACH class |
109 | */ |
110 | #define VM_DISCONNECT_ALL_PAGE_MAPPINGS 0x00 |
111 | #define VM_DISCONNECT_TASK_PAGE_MAPPINGS 0x01 |
112 | #define VM_REAL_FAULT_ADDR_INTERNAL 0x02 |
113 | #define VM_REAL_FAULT_ADDR_PURGABLE 0x03 |
114 | #define VM_REAL_FAULT_ADDR_EXTERNAL 0x04 |
115 | #define VM_REAL_FAULT_ADDR_SHAREDCACHE 0x05 |
116 | |
117 | |
118 | |
119 | extern int vm_debug_events; |
120 | |
121 | #define VMF_CHECK_ZFDELAY 0x100 |
122 | #define VMF_COWDELAY 0x101 |
123 | #define VMF_ZFDELAY 0x102 |
124 | #define VMF_COMPRESSORDELAY 0x103 |
125 | |
126 | #define VM_PAGEOUT_SCAN 0x104 |
127 | #define VM_PAGEOUT_BALANCE 0x105 |
128 | #define VM_PAGEOUT_FREELIST 0x106 |
129 | #define VM_PAGEOUT_PURGEONE 0x107 |
130 | #define VM_PAGEOUT_CACHE_EVICT 0x108 |
131 | #define VM_PAGEOUT_THREAD_BLOCK 0x109 |
132 | #define VM_PAGEOUT_JETSAM 0x10A |
133 | #define VM_INFO1 0x10B |
134 | #define VM_INFO2 0x10C |
135 | #define VM_INFO3 0x10D |
136 | #define VM_INFO4 0x10E |
137 | #define VM_INFO5 0x10F |
138 | #define VM_INFO6 0x110 |
139 | #define VM_INFO7 0x111 |
140 | #define VM_INFO8 0x112 |
141 | #define VM_INFO9 0x113 |
142 | |
143 | #define VM_UPL_PAGE_WAIT 0x120 |
144 | #define VM_IOPL_PAGE_WAIT 0x121 |
145 | #define VM_PAGE_WAIT_BLOCK 0x122 |
146 | |
147 | #if CONFIG_IOSCHED |
148 | #define VM_PAGE_SLEEP 0x123 |
149 | #define VM_PAGE_EXPEDITE 0x124 |
150 | #define VM_PAGE_EXPEDITE_NO_MEMORY 0x125 |
151 | #endif |
152 | |
153 | #define VM_PAGE_GRAB 0x126 |
154 | #define VM_PAGE_RELEASE 0x127 |
155 | |
156 | #define VM_PRESSURE_EVENT 0x130 |
157 | #define VM_EXECVE 0x131 |
158 | #define VM_WAKEUP_COMPACTOR_SWAPPER 0x132 |
159 | #define VM_UPL_REQUEST 0x133 |
160 | #define VM_IOPL_REQUEST 0x134 |
161 | #define VM_KERN_REQUEST 0x135 |
162 | |
163 | #define VM_DATA_WRITE 0x140 |
164 | |
165 | #define VM_PRESSURE_LEVEL_CHANGE 0x141 |
166 | |
167 | #define VM_DEBUG_EVENT(name, event, control, arg1, arg2, arg3, arg4) \ |
168 | MACRO_BEGIN \ |
169 | if (__improbable(vm_debug_events)) { \ |
170 | KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \ |
171 | } \ |
172 | MACRO_END |
173 | |
174 | #define VM_DEBUG_CONSTANT_EVENT(name, event, control, arg1, arg2, arg3, arg4) \ |
175 | MACRO_BEGIN \ |
176 | KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, event)) | control, arg1, arg2, arg3, arg4, 0); \ |
177 | MACRO_END |
178 | |
179 | extern void memoryshot(unsigned int event, unsigned int control); |
180 | |
181 | extern void update_vm_info(void); |
182 | |
183 | #if CONFIG_IOSCHED |
184 | extern int upl_get_cached_tier( |
185 | upl_t upl); |
186 | #endif |
187 | |
188 | extern void upl_set_iodone(upl_t, void *); |
189 | extern void upl_set_iodone_error(upl_t, int); |
190 | extern void upl_callout_iodone(upl_t); |
191 | |
192 | extern ppnum_t upl_get_highest_page( |
193 | upl_t upl); |
194 | |
195 | extern upl_size_t upl_get_size( |
196 | upl_t upl); |
197 | |
198 | extern upl_t upl_associated_upl(upl_t upl); |
199 | extern void upl_set_associated_upl(upl_t upl, upl_t associated_upl); |
200 | |
201 | #ifdef XNU_KERNEL_PRIVATE |
202 | |
203 | extern kern_return_t vm_map_create_upl( |
204 | vm_map_t map, |
205 | vm_map_address_t offset, |
206 | upl_size_t *upl_size, |
207 | upl_t *upl, |
208 | upl_page_info_array_t page_list, |
209 | unsigned int *count, |
210 | upl_control_flags_t *flags, |
211 | vm_tag_t tag); |
212 | |
213 | extern void iopl_valid_data( |
214 | upl_t upl_ptr, |
215 | vm_tag_t tag); |
216 | |
217 | #endif /* XNU_KERNEL_PRIVATE */ |
218 | |
219 | extern struct vnode * upl_lookup_vnode(upl_t upl); |
220 | |
221 | #ifndef MACH_KERNEL_PRIVATE |
222 | typedef struct vm_page *vm_page_t; |
223 | #endif |
224 | |
225 | extern void vm_page_free_list( |
226 | vm_page_t mem, |
227 | boolean_t prepare_object); |
228 | |
229 | extern kern_return_t vm_page_alloc_list( |
230 | int page_count, |
231 | int flags, |
232 | vm_page_t * list); |
233 | |
234 | extern void vm_page_set_offset(vm_page_t page, vm_object_offset_t offset); |
235 | extern vm_object_offset_t vm_page_get_offset(vm_page_t page); |
236 | extern ppnum_t vm_page_get_phys_page(vm_page_t page); |
237 | extern vm_page_t vm_page_get_next(vm_page_t page); |
238 | |
239 | extern kern_return_t mach_vm_pressure_level_monitor(boolean_t wait_for_pressure, unsigned int *pressure_level); |
240 | |
241 | #if !CONFIG_EMBEDDED |
242 | extern kern_return_t vm_pageout_wait(uint64_t deadline); |
243 | #endif |
244 | |
245 | #ifdef MACH_KERNEL_PRIVATE |
246 | |
247 | #include <vm/vm_page.h> |
248 | |
249 | extern unsigned int vm_pageout_scan_event_counter; |
250 | extern unsigned int vm_page_anonymous_count; |
251 | |
252 | |
253 | /* |
254 | * must hold the page queues lock to |
255 | * manipulate this structure |
256 | */ |
257 | struct vm_pageout_queue { |
258 | vm_page_queue_head_t pgo_pending; /* laundry pages to be processed by pager's iothread */ |
259 | unsigned int pgo_laundry; /* current count of laundry pages on queue or in flight */ |
260 | unsigned int pgo_maxlaundry; |
261 | uint64_t pgo_tid; /* thread ID of I/O thread that services this queue */ |
262 | uint8_t pgo_lowpriority; /* iothread is set to use low priority I/O */ |
263 | |
264 | unsigned int pgo_idle:1, /* iothread is blocked waiting for work to do */ |
265 | pgo_busy:1, /* iothread is currently processing request from pgo_pending */ |
266 | pgo_throttled:1,/* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */ |
267 | pgo_draining:1, |
268 | pgo_inited:1, |
269 | :0; |
270 | }; |
271 | |
272 | #define VM_PAGE_Q_THROTTLED(q) \ |
273 | ((q)->pgo_laundry >= (q)->pgo_maxlaundry) |
274 | |
275 | extern struct vm_pageout_queue vm_pageout_queue_internal; |
276 | extern struct vm_pageout_queue vm_pageout_queue_external; |
277 | |
278 | |
279 | /* |
280 | * Routines exported to Mach. |
281 | */ |
282 | extern void vm_pageout(void); |
283 | |
284 | extern kern_return_t vm_pageout_internal_start(void); |
285 | |
286 | extern void vm_pageout_object_terminate( |
287 | vm_object_t object); |
288 | |
289 | extern void vm_pageout_cluster( |
290 | vm_page_t m); |
291 | |
292 | extern void vm_pageout_initialize_page( |
293 | vm_page_t m); |
294 | |
295 | /* UPL exported routines and structures */ |
296 | |
297 | #define upl_lock_init(object) lck_mtx_init(&(object)->Lock, &vm_object_lck_grp, &vm_object_lck_attr) |
298 | #define upl_lock_destroy(object) lck_mtx_destroy(&(object)->Lock, &vm_object_lck_grp) |
299 | #define upl_lock(object) lck_mtx_lock(&(object)->Lock) |
300 | #define upl_unlock(object) lck_mtx_unlock(&(object)->Lock) |
301 | #define upl_try_lock(object) lck_mtx_try_lock(&(object)->Lock) |
302 | |
303 | #define MAX_VECTOR_UPL_ELEMENTS 8 |
304 | |
305 | struct _vector_upl_iostates{ |
306 | upl_offset_t offset; |
307 | upl_size_t size; |
308 | }; |
309 | |
310 | typedef struct _vector_upl_iostates vector_upl_iostates_t; |
311 | |
312 | struct _vector_upl { |
313 | upl_size_t size; |
314 | uint32_t num_upls; |
315 | uint32_t invalid_upls; |
316 | uint32_t _reserved; |
317 | vm_map_t submap; |
318 | vm_offset_t submap_dst_addr; |
319 | vm_object_offset_t offset; |
320 | upl_t upl_elems[MAX_VECTOR_UPL_ELEMENTS]; |
321 | upl_page_info_array_t pagelist; |
322 | vector_upl_iostates_t upl_iostates[MAX_VECTOR_UPL_ELEMENTS]; |
323 | }; |
324 | |
325 | typedef struct _vector_upl* vector_upl_t; |
326 | |
327 | /* universal page list structure */ |
328 | |
329 | #if UPL_DEBUG |
330 | #define UPL_DEBUG_STACK_FRAMES 16 |
331 | #define UPL_DEBUG_COMMIT_RECORDS 4 |
332 | |
333 | struct ucd { |
334 | upl_offset_t c_beg; |
335 | upl_offset_t c_end; |
336 | int c_aborted; |
337 | void * c_retaddr[UPL_DEBUG_STACK_FRAMES]; |
338 | }; |
339 | #endif |
340 | |
341 | struct upl_io_completion { |
342 | |
343 | void *io_context; |
344 | void (*io_done)(void *, int); |
345 | |
346 | int io_error; |
347 | }; |
348 | |
349 | |
350 | struct upl { |
351 | decl_lck_mtx_data(, Lock) /* Synchronization */ |
352 | int ref_count; |
353 | int ext_ref_count; |
354 | int flags; |
355 | vm_object_offset_t offset; |
356 | upl_size_t size; /* size in bytes of the address space */ |
357 | vm_offset_t kaddr; /* secondary mapping in kernel */ |
358 | vm_object_t map_object; |
359 | ppnum_t highest_page; |
360 | void* vector_upl; |
361 | upl_t associated_upl; |
362 | struct upl_io_completion *upl_iodone; |
363 | #if CONFIG_IOSCHED |
364 | int upl_priority; |
365 | uint64_t *upl_reprio_info; |
366 | void *decmp_io_upl; |
367 | #endif |
368 | #if CONFIG_IOSCHED || UPL_DEBUG |
369 | thread_t upl_creator; |
370 | queue_chain_t uplq; /* List of outstanding upls on an obj */ |
371 | #endif |
372 | #if UPL_DEBUG |
373 | uintptr_t ubc_alias1; |
374 | uintptr_t ubc_alias2; |
375 | |
376 | uint32_t upl_state; |
377 | uint32_t upl_commit_index; |
378 | void *upl_create_retaddr[UPL_DEBUG_STACK_FRAMES]; |
379 | |
380 | struct ucd upl_commit_records[UPL_DEBUG_COMMIT_RECORDS]; |
381 | #endif /* UPL_DEBUG */ |
382 | }; |
383 | |
384 | /* upl struct flags */ |
385 | #define UPL_PAGE_LIST_MAPPED 0x1 |
386 | #define UPL_KERNEL_MAPPED 0x2 |
387 | #define UPL_CLEAR_DIRTY 0x4 |
388 | #define UPL_COMPOSITE_LIST 0x8 |
389 | #define UPL_INTERNAL 0x10 |
390 | #define UPL_PAGE_SYNC_DONE 0x20 |
391 | #define UPL_DEVICE_MEMORY 0x40 |
392 | #define UPL_PAGEOUT 0x80 |
393 | #define UPL_LITE 0x100 |
394 | #define UPL_IO_WIRE 0x200 |
395 | #define UPL_ACCESS_BLOCKED 0x400 |
396 | #define UPL_SHADOWED 0x1000 |
397 | #define UPL_KERNEL_OBJECT 0x2000 |
398 | #define UPL_VECTOR 0x4000 |
399 | #define UPL_SET_DIRTY 0x8000 |
400 | #define UPL_HAS_BUSY 0x10000 |
401 | #define UPL_TRACKED_BY_OBJECT 0x20000 |
402 | #define UPL_EXPEDITE_SUPPORTED 0x40000 |
403 | #define UPL_DECMP_REQ 0x80000 |
404 | #define UPL_DECMP_REAL_IO 0x100000 |
405 | |
406 | /* flags for upl_create flags parameter */ |
407 | #define UPL_CREATE_EXTERNAL 0 |
408 | #define UPL_CREATE_INTERNAL 0x1 |
409 | #define UPL_CREATE_LITE 0x2 |
410 | #define UPL_CREATE_IO_TRACKING 0x4 |
411 | #define UPL_CREATE_EXPEDITE_SUP 0x8 |
412 | |
413 | extern upl_t vector_upl_create(vm_offset_t); |
414 | extern void vector_upl_deallocate(upl_t); |
415 | extern boolean_t vector_upl_is_valid(upl_t); |
416 | extern boolean_t vector_upl_set_subupl(upl_t, upl_t, u_int32_t); |
417 | extern void vector_upl_set_pagelist(upl_t); |
418 | extern void vector_upl_set_submap(upl_t, vm_map_t, vm_offset_t); |
419 | extern void vector_upl_get_submap(upl_t, vm_map_t*, vm_offset_t*); |
420 | extern void vector_upl_set_iostate(upl_t, upl_t, upl_offset_t, upl_size_t); |
421 | extern void vector_upl_get_iostate(upl_t, upl_t, upl_offset_t*, upl_size_t*); |
422 | extern void vector_upl_get_iostate_byindex(upl_t, uint32_t, upl_offset_t*, upl_size_t*); |
423 | extern upl_t vector_upl_subupl_byindex(upl_t , uint32_t); |
424 | extern upl_t vector_upl_subupl_byoffset(upl_t , upl_offset_t*, upl_size_t*); |
425 | |
426 | extern void vm_object_set_pmap_cache_attr( |
427 | vm_object_t object, |
428 | upl_page_info_array_t user_page_list, |
429 | unsigned int num_pages, |
430 | boolean_t batch_pmap_op); |
431 | |
432 | extern kern_return_t vm_object_iopl_request( |
433 | vm_object_t object, |
434 | vm_object_offset_t offset, |
435 | upl_size_t size, |
436 | upl_t *upl_ptr, |
437 | upl_page_info_array_t user_page_list, |
438 | unsigned int *page_list_count, |
439 | upl_control_flags_t cntrl_flags, |
440 | vm_tag_t tag); |
441 | |
442 | extern kern_return_t vm_object_super_upl_request( |
443 | vm_object_t object, |
444 | vm_object_offset_t offset, |
445 | upl_size_t size, |
446 | upl_size_t super_cluster, |
447 | upl_t *upl, |
448 | upl_page_info_t *user_page_list, |
449 | unsigned int *page_list_count, |
450 | upl_control_flags_t cntrl_flags, |
451 | vm_tag_t tag); |
452 | |
453 | /* should be just a regular vm_map_enter() */ |
454 | extern kern_return_t vm_map_enter_upl( |
455 | vm_map_t map, |
456 | upl_t upl, |
457 | vm_map_offset_t *dst_addr); |
458 | |
459 | /* should be just a regular vm_map_remove() */ |
460 | extern kern_return_t vm_map_remove_upl( |
461 | vm_map_t map, |
462 | upl_t upl); |
463 | |
464 | /* wired page list structure */ |
465 | typedef uint32_t *wpl_array_t; |
466 | |
467 | extern void vm_page_free_reserve(int pages); |
468 | |
469 | extern void vm_pageout_throttle_down(vm_page_t page); |
470 | extern void vm_pageout_throttle_up(vm_page_t page); |
471 | |
472 | extern kern_return_t vm_paging_map_object( |
473 | vm_page_t page, |
474 | vm_object_t object, |
475 | vm_object_offset_t offset, |
476 | vm_prot_t protection, |
477 | boolean_t can_unlock_object, |
478 | vm_map_size_t *size, /* IN/OUT */ |
479 | vm_map_offset_t *address, /* OUT */ |
480 | boolean_t *need_unmap); /* OUT */ |
481 | extern void vm_paging_unmap_object( |
482 | vm_object_t object, |
483 | vm_map_offset_t start, |
484 | vm_map_offset_t end); |
485 | decl_simple_lock_data(extern, vm_paging_lock) |
486 | |
487 | /* |
488 | * Backing store throttle when BS is exhausted |
489 | */ |
490 | extern unsigned int vm_backing_store_low; |
491 | |
492 | extern void vm_pageout_steal_laundry( |
493 | vm_page_t page, |
494 | boolean_t queues_locked); |
495 | |
496 | #endif /* MACH_KERNEL_PRIVATE */ |
497 | |
498 | #if UPL_DEBUG |
499 | extern kern_return_t upl_ubc_alias_set( |
500 | upl_t upl, |
501 | uintptr_t alias1, |
502 | uintptr_t alias2); |
503 | extern int upl_ubc_alias_get( |
504 | upl_t upl, |
505 | uintptr_t * al, |
506 | uintptr_t * al2); |
507 | #endif /* UPL_DEBUG */ |
508 | |
509 | extern void vm_countdirtypages(void); |
510 | |
511 | extern void vm_backing_store_disable( |
512 | boolean_t suspend); |
513 | |
514 | extern kern_return_t upl_transpose( |
515 | upl_t upl1, |
516 | upl_t upl2); |
517 | |
518 | extern kern_return_t mach_vm_pressure_monitor( |
519 | boolean_t wait_for_pressure, |
520 | unsigned int nsecs_monitored, |
521 | unsigned int *pages_reclaimed_p, |
522 | unsigned int *pages_wanted_p); |
523 | |
524 | extern kern_return_t |
525 | vm_set_buffer_cleanup_callout( |
526 | boolean_t (*func)(int)); |
527 | |
528 | struct vm_page_stats_reusable { |
529 | SInt32 reusable_count; |
530 | uint64_t reusable; |
531 | uint64_t reused; |
532 | uint64_t reused_wire; |
533 | uint64_t reused_remove; |
534 | uint64_t all_reusable_calls; |
535 | uint64_t partial_reusable_calls; |
536 | uint64_t all_reuse_calls; |
537 | uint64_t partial_reuse_calls; |
538 | uint64_t reusable_pages_success; |
539 | uint64_t reusable_pages_failure; |
540 | uint64_t reusable_pages_shared; |
541 | uint64_t reuse_pages_success; |
542 | uint64_t reuse_pages_failure; |
543 | uint64_t can_reuse_success; |
544 | uint64_t can_reuse_failure; |
545 | uint64_t reusable_reclaimed; |
546 | uint64_t reusable_nonwritable; |
547 | uint64_t reusable_shared; |
548 | uint64_t free_shared; |
549 | }; |
550 | extern struct vm_page_stats_reusable vm_page_stats_reusable; |
551 | |
552 | extern int hibernate_flush_memory(void); |
553 | extern void hibernate_reset_stats(void); |
554 | extern void hibernate_create_paddr_map(void); |
555 | |
556 | extern void vm_set_restrictions(void); |
557 | |
558 | extern int vm_compressor_mode; |
559 | extern kern_return_t vm_pageout_compress_page(void **, char *, vm_page_t); |
560 | extern void vm_pageout_anonymous_pages(void); |
561 | extern void vm_pageout_disconnect_all_pages(void); |
562 | |
563 | |
564 | struct vm_config { |
565 | boolean_t compressor_is_present; /* compressor is initialized and can be used by the freezer, the sweep or the pager */ |
566 | boolean_t compressor_is_active; /* pager can actively compress pages... 'compressor_is_present' must be set */ |
567 | boolean_t swap_is_present; /* swap is initialized and can be used by the freezer, the sweep or the pager */ |
568 | boolean_t swap_is_active; /* pager can actively swap out compressed segments... 'swap_is_present' must be set */ |
569 | boolean_t freezer_swap_is_active; /* freezer can swap out frozen tasks... "compressor_is_present + swap_is_present" must be set */ |
570 | }; |
571 | |
572 | extern struct vm_config vm_config; |
573 | |
574 | |
575 | #define 0x0 /* no compresser or swap configured */ |
576 | #define 0x1 /* Use default pager... DEPRECATED */ |
577 | #define 0x2 /* Active in-core compressor only. */ |
578 | #define 0x4 /* Active in-core compressor + swap backend. */ |
579 | #define 0x8 /* Freezer backed by default pager... DEPRECATED */ |
580 | #define 0x10 /* Freezer backed by in-core compressor only i.e. frozen data remain in-core compressed.*/ |
581 | #define 0x20 /* Active in-core compressor + Freezer backed by in-core compressor with swap support too.*/ |
582 | |
583 | #define 6 /* Total number of vm compressor modes supported */ |
584 | |
585 | |
586 | #define VM_CONFIG_COMPRESSOR_IS_PRESENT (vm_config.compressor_is_present == TRUE) |
587 | #define VM_CONFIG_COMPRESSOR_IS_ACTIVE (vm_config.compressor_is_active == TRUE) |
588 | #define VM_CONFIG_SWAP_IS_PRESENT (vm_config.swap_is_present == TRUE) |
589 | #define VM_CONFIG_SWAP_IS_ACTIVE (vm_config.swap_is_active == TRUE) |
590 | #define VM_CONFIG_FREEZER_SWAP_IS_ACTIVE (vm_config.freezer_swap_is_active == TRUE) |
591 | |
592 | #endif /* KERNEL_PRIVATE */ |
593 | |
594 | #ifdef XNU_KERNEL_PRIVATE |
595 | |
596 | struct vm_pageout_state { |
597 | boolean_t vm_pressure_thread_running; |
598 | boolean_t vm_pressure_changed; |
599 | boolean_t vm_restricted_to_single_processor; |
600 | int vm_compressor_thread_count; |
601 | |
602 | unsigned int vm_page_speculative_q_age_ms; |
603 | unsigned int vm_page_speculative_percentage; |
604 | unsigned int vm_page_speculative_target; |
605 | |
606 | unsigned int vm_pageout_swap_wait; |
607 | unsigned int vm_pageout_idle_wait; /* milliseconds */ |
608 | unsigned int vm_pageout_empty_wait; /* milliseconds */ |
609 | unsigned int vm_pageout_burst_wait; /* milliseconds */ |
610 | unsigned int vm_pageout_deadlock_wait; /* milliseconds */ |
611 | unsigned int vm_pageout_deadlock_relief; |
612 | unsigned int vm_pageout_burst_inactive_throttle; |
613 | |
614 | unsigned int vm_pageout_inactive; |
615 | unsigned int vm_pageout_inactive_used; /* debugging */ |
616 | unsigned int vm_pageout_inactive_clean; /* debugging */ |
617 | |
618 | uint32_t vm_page_filecache_min; |
619 | uint32_t vm_page_filecache_min_divisor; |
620 | uint32_t vm_page_xpmapped_min; |
621 | uint32_t vm_page_xpmapped_min_divisor; |
622 | uint64_t vm_pageout_considered_page_last; |
623 | |
624 | int vm_page_free_count_init; |
625 | |
626 | unsigned int vm_memory_pressure; |
627 | |
628 | int memorystatus_purge_on_critical; |
629 | int memorystatus_purge_on_warning; |
630 | int memorystatus_purge_on_urgent; |
631 | |
632 | thread_t vm_pageout_external_iothread; |
633 | thread_t vm_pageout_internal_iothread; |
634 | }; |
635 | |
636 | extern struct vm_pageout_state vm_pageout_state; |
637 | |
638 | /* |
639 | * This structure is used to track the VM_INFO instrumentation |
640 | */ |
641 | struct vm_pageout_vminfo { |
642 | unsigned long vm_pageout_considered_page; |
643 | unsigned long vm_pageout_considered_bq_internal; |
644 | unsigned long vm_pageout_considered_bq_external; |
645 | unsigned long vm_pageout_skipped_external; |
646 | |
647 | unsigned long vm_pageout_pages_evicted; |
648 | unsigned long vm_pageout_pages_purged;; |
649 | unsigned long vm_pageout_freed_cleaned; |
650 | unsigned long vm_pageout_freed_speculative; |
651 | unsigned long vm_pageout_freed_external; |
652 | unsigned long vm_pageout_freed_internal; |
653 | unsigned long vm_pageout_inactive_dirty_internal; |
654 | unsigned long vm_pageout_inactive_dirty_external; |
655 | unsigned long vm_pageout_inactive_referenced; |
656 | unsigned long vm_pageout_reactivation_limit_exceeded; |
657 | unsigned long vm_pageout_inactive_force_reclaim; |
658 | unsigned long vm_pageout_inactive_nolock; |
659 | unsigned long vm_pageout_filecache_min_reactivated; |
660 | unsigned long vm_pageout_scan_inactive_throttled_internal; |
661 | unsigned long vm_pageout_scan_inactive_throttled_external; |
662 | |
663 | uint64_t vm_pageout_compressions; |
664 | uint64_t vm_compressor_pages_grabbed; |
665 | unsigned long vm_compressor_failed; |
666 | |
667 | unsigned long vm_page_pages_freed; |
668 | |
669 | unsigned long vm_phantom_cache_found_ghost; |
670 | unsigned long vm_phantom_cache_added_ghost; |
671 | }; |
672 | |
673 | extern struct vm_pageout_vminfo vm_pageout_vminfo; |
674 | |
675 | |
676 | #if DEVELOPMENT || DEBUG |
677 | |
678 | /* |
679 | * This structure records the pageout daemon's actions: |
680 | * how many pages it looks at and what happens to those pages. |
681 | * No locking needed because only one thread modifies the fields. |
682 | */ |
683 | struct vm_pageout_debug { |
684 | uint32_t vm_pageout_balanced; |
685 | uint32_t vm_pageout_scan_event_counter; |
686 | uint32_t vm_pageout_speculative_dirty; |
687 | |
688 | uint32_t vm_pageout_inactive_busy; |
689 | uint32_t vm_pageout_inactive_absent; |
690 | uint32_t vm_pageout_inactive_notalive; |
691 | uint32_t vm_pageout_inactive_error; |
692 | uint32_t vm_pageout_inactive_deactivated; |
693 | |
694 | uint32_t vm_pageout_enqueued_cleaned; |
695 | |
696 | uint32_t vm_pageout_cleaned_busy; |
697 | uint32_t vm_pageout_cleaned_nolock; |
698 | uint32_t vm_pageout_cleaned_reference_reactivated; |
699 | uint32_t vm_pageout_cleaned_volatile_reactivated; |
700 | uint32_t vm_pageout_cleaned_reactivated; /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */ |
701 | uint32_t vm_pageout_cleaned_fault_reactivated; |
702 | |
703 | uint32_t vm_pageout_dirty_no_pager; |
704 | uint32_t vm_pageout_purged_objects; |
705 | |
706 | uint32_t vm_pageout_scan_throttle; |
707 | uint32_t vm_pageout_scan_reclaimed_throttled; |
708 | uint32_t vm_pageout_scan_burst_throttle; |
709 | uint32_t vm_pageout_scan_empty_throttle; |
710 | uint32_t vm_pageout_scan_swap_throttle; |
711 | uint32_t vm_pageout_scan_deadlock_detected; |
712 | uint32_t vm_pageout_scan_inactive_throttle_success; |
713 | uint32_t vm_pageout_scan_throttle_deferred; |
714 | |
715 | uint32_t vm_pageout_inactive_external_forced_jetsam_count; |
716 | |
717 | uint32_t vm_grab_anon_overrides; |
718 | uint32_t vm_grab_anon_nops; |
719 | |
720 | uint32_t vm_pageout_no_victim; |
721 | unsigned long vm_pageout_throttle_up_count; |
722 | uint32_t vm_page_steal_pageout_page; |
723 | |
724 | uint32_t vm_cs_validated_resets; |
725 | uint32_t vm_object_iopl_request_sleep_for_cleaning; |
726 | uint32_t vm_page_slide_counter; |
727 | uint32_t vm_page_slide_errors; |
728 | uint32_t vm_page_throttle_count; |
729 | /* |
730 | * Statistics about UPL enforcement of copy-on-write obligations. |
731 | */ |
732 | unsigned long upl_cow; |
733 | unsigned long upl_cow_again; |
734 | unsigned long upl_cow_pages; |
735 | unsigned long upl_cow_again_pages; |
736 | unsigned long iopl_cow; |
737 | unsigned long iopl_cow_pages; |
738 | }; |
739 | |
740 | extern struct vm_pageout_debug vm_pageout_debug; |
741 | |
742 | #define VM_PAGEOUT_DEBUG(member, value) \ |
743 | MACRO_BEGIN \ |
744 | vm_pageout_debug.member += value; \ |
745 | MACRO_END |
746 | #else |
747 | #define VM_PAGEOUT_DEBUG(member, value) |
748 | #endif |
749 | |
750 | #define MAX_COMPRESSOR_THREAD_COUNT 8 |
751 | |
752 | #if DEVELOPMENT || DEBUG |
753 | typedef struct vmct_stats_s { |
754 | uint64_t vmct_runtimes[MAX_COMPRESSOR_THREAD_COUNT]; |
755 | uint64_t vmct_pages[MAX_COMPRESSOR_THREAD_COUNT]; |
756 | uint64_t vmct_iterations[MAX_COMPRESSOR_THREAD_COUNT]; |
757 | uint64_t vmct_cthreads_total; |
758 | int32_t vmct_minpages[MAX_COMPRESSOR_THREAD_COUNT]; |
759 | int32_t vmct_maxpages[MAX_COMPRESSOR_THREAD_COUNT]; |
760 | } vmct_stats_t; |
761 | #endif |
762 | #endif |
763 | #endif /* _VM_VM_PAGEOUT_H_ */ |
764 | |