1 | /* |
2 | * Copyright (c) 2000-2016 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | #ifndef _VM_VM_COMPRESSOR_H_ |
29 | #define _VM_VM_COMPRESSOR_H_ |
30 | |
31 | #include <vm/vm_compressor_pager.h> |
32 | #include <vm/vm_kern.h> |
33 | #include <vm/vm_page.h> |
34 | #include <vm/vm_protos.h> |
35 | #include <vm/WKdm_new.h> |
36 | #include <vm/vm_object.h> |
37 | #include <vm/vm_map.h> |
38 | #include <machine/pmap.h> |
39 | #include <kern/locks.h> |
40 | |
41 | #include <sys/kdebug.h> |
42 | |
43 | #if defined(__arm64__) |
44 | #include <arm64/proc_reg.h> |
45 | #endif |
46 | |
47 | #define C_SEG_OFFSET_BITS 16 |
48 | |
49 | #define C_SEG_MAX_POPULATE_SIZE (4 * PAGE_SIZE) |
50 | |
51 | #if defined(__arm64__) && (DEVELOPMENT || DEBUG) |
52 | |
53 | #if defined(XNU_PLATFORM_WatchOS) |
54 | #define VALIDATE_C_SEGMENTS (1) |
55 | #endif |
56 | #endif /* defined(__arm64__) && (DEVELOPMENT || DEBUG) */ |
57 | |
58 | |
59 | #if DEBUG || COMPRESSOR_INTEGRITY_CHECKS |
60 | #define ENABLE_SWAP_CHECKS 1 |
61 | #define ENABLE_COMPRESSOR_CHECKS 1 |
62 | #define POPCOUNT_THE_COMPRESSED_DATA (1) |
63 | #else |
64 | #define ENABLE_SWAP_CHECKS 0 |
65 | #define ENABLE_COMPRESSOR_CHECKS 0 |
66 | #endif |
67 | |
68 | #define CHECKSUM_THE_SWAP ENABLE_SWAP_CHECKS /* Debug swap data */ |
69 | #define CHECKSUM_THE_DATA ENABLE_COMPRESSOR_CHECKS /* Debug compressor/decompressor data */ |
70 | #define CHECKSUM_THE_COMPRESSED_DATA ENABLE_COMPRESSOR_CHECKS /* Debug compressor/decompressor compressed data */ |
71 | |
72 | #ifndef VALIDATE_C_SEGMENTS |
73 | #define VALIDATE_C_SEGMENTS ENABLE_COMPRESSOR_CHECKS /* Debug compaction */ |
74 | #endif |
75 | |
76 | #define RECORD_THE_COMPRESSED_DATA 0 |
77 | |
78 | /* |
79 | * The c_slot structure embeds a packed pointer to a c_slot_mapping |
80 | * (32bits) which we ideally want to span as much VA space as possible |
81 | * to not limit zalloc in how it sets itself up. |
82 | */ |
83 | #if !defined(__LP64__) /* no packing */ |
84 | #define C_SLOT_PACKED_PTR_BITS 32 |
85 | #define C_SLOT_PACKED_PTR_SHIFT 0 |
86 | #define C_SLOT_PACKED_PTR_BASE 0 |
87 | |
88 | #define C_SLOT_C_SIZE_BITS 12 |
89 | #define C_SLOT_C_CODEC_BITS 1 |
90 | #define C_SLOT_C_POPCOUNT_BITS 0 |
91 | #define C_SLOT_C_PADDING_BITS 3 |
92 | |
93 | #elif defined(__arm64__) /* 32G from the heap start */ |
94 | #define C_SLOT_PACKED_PTR_BITS 33 |
95 | #define C_SLOT_PACKED_PTR_SHIFT 2 |
96 | #define C_SLOT_PACKED_PTR_BASE ((uintptr_t)KERNEL_PMAP_HEAP_RANGE_START) |
97 | |
98 | #define C_SLOT_C_SIZE_BITS 14 |
99 | #define C_SLOT_C_CODEC_BITS 1 |
100 | #define C_SLOT_C_POPCOUNT_BITS 0 |
101 | #define C_SLOT_C_PADDING_BITS 0 |
102 | |
103 | #elif defined(__x86_64__) /* 256G from the heap start */ |
104 | #define C_SLOT_PACKED_PTR_BITS 36 |
105 | #define C_SLOT_PACKED_PTR_SHIFT 2 |
106 | #define C_SLOT_PACKED_PTR_BASE ((uintptr_t)KERNEL_PMAP_HEAP_RANGE_START) |
107 | |
108 | #define C_SLOT_C_SIZE_BITS 12 |
109 | #define C_SLOT_C_CODEC_BITS 0 /* not used */ |
110 | #define C_SLOT_C_POPCOUNT_BITS 0 |
111 | #define C_SLOT_C_PADDING_BITS 0 |
112 | |
113 | #else |
114 | #error vm_compressor parameters undefined for this architecture |
115 | #endif |
116 | |
117 | /* |
118 | * Popcounts needs to represent both 0 and full which requires |
119 | * (8 ^ C_SLOT_C_SIZE_BITS) + 1 values and (C_SLOT_C_SIZE_BITS + 4) bits. |
120 | * |
121 | * We us the (2 * (8 ^ C_SLOT_C_SIZE_BITS) - 1) value to mean "unknown". |
122 | */ |
123 | #define C_SLOT_NO_POPCOUNT ((16u << C_SLOT_C_SIZE_BITS) - 1) |
124 | |
125 | static_assert((C_SEG_OFFSET_BITS + C_SLOT_C_SIZE_BITS + |
126 | C_SLOT_C_CODEC_BITS + C_SLOT_C_POPCOUNT_BITS + |
127 | C_SLOT_C_PADDING_BITS + C_SLOT_PACKED_PTR_BITS) % 32 == 0); |
128 | |
129 | struct c_slot { |
130 | uint64_t c_offset:C_SEG_OFFSET_BITS __kernel_ptr_semantics; |
131 | uint64_t c_size:C_SLOT_C_SIZE_BITS; |
132 | #if C_SLOT_C_CODEC_BITS |
133 | uint64_t c_codec:C_SLOT_C_CODEC_BITS; |
134 | #endif |
135 | #if C_SLOT_C_POPCOUNT_BITS |
136 | /* |
137 | * This value may not agree with c_pop_cdata, as it may be the |
138 | * population count of the uncompressed data. |
139 | * |
140 | * This value must be C_SLOT_NO_POPCOUNT when the compression algorithm |
141 | * cannot provide it. |
142 | */ |
143 | uint32_t c_inline_popcount:C_SLOT_C_POPCOUNT_BITS; |
144 | #endif |
145 | #if C_SLOT_C_PADDING_BITS |
146 | uint64_t c_padding:C_SLOT_C_PADDING_BITS; |
147 | #endif |
148 | uint64_t c_packed_ptr:C_SLOT_PACKED_PTR_BITS __kernel_ptr_semantics; |
149 | |
150 | /* debugging fields, typically not present on release kernels */ |
151 | #if CHECKSUM_THE_DATA |
152 | unsigned int c_hash_data; |
153 | #endif |
154 | #if CHECKSUM_THE_COMPRESSED_DATA |
155 | unsigned int c_hash_compressed_data; |
156 | #endif |
157 | #if POPCOUNT_THE_COMPRESSED_DATA |
158 | unsigned int c_pop_cdata; |
159 | #endif |
160 | } __attribute__((packed, aligned(4))); |
161 | |
162 | #define C_IS_EMPTY 0 |
163 | #define C_IS_FREE 1 |
164 | #define C_IS_FILLING 2 |
165 | #define C_ON_AGE_Q 3 |
166 | #define C_ON_SWAPOUT_Q 4 |
167 | #define C_ON_SWAPPEDOUT_Q 5 |
168 | #define C_ON_SWAPPEDOUTSPARSE_Q 6 |
169 | #define C_ON_SWAPPEDIN_Q 7 |
170 | #define C_ON_MAJORCOMPACT_Q 8 |
171 | #define C_ON_BAD_Q 9 |
172 | #define C_ON_SWAPIO_Q 10 |
173 | |
174 | |
175 | struct c_segment { |
176 | lck_mtx_t c_lock; |
177 | queue_chain_t c_age_list; |
178 | queue_chain_t c_list; |
179 | |
180 | #if CONFIG_FREEZE |
181 | queue_chain_t c_task_list_next_cseg; |
182 | task_t c_task_owner; |
183 | #endif /* CONFIG_FREEZE */ |
184 | |
185 | #define C_SEG_MAX_LIMIT (UINT_MAX) /* this needs to track the size of c_mysegno */ |
186 | uint32_t c_mysegno; |
187 | |
188 | uint32_t c_creation_ts; |
189 | uint64_t c_generation_id; |
190 | |
191 | int32_t c_bytes_used; |
192 | int32_t c_bytes_unused; |
193 | uint32_t c_slots_used; |
194 | |
195 | uint16_t c_firstemptyslot; |
196 | uint16_t c_nextslot; |
197 | uint32_t c_nextoffset; |
198 | uint32_t c_populated_offset; |
199 | |
200 | union { |
201 | int32_t *c_buffer; |
202 | uint64_t c_swap_handle; |
203 | } c_store; |
204 | |
205 | #if VALIDATE_C_SEGMENTS |
206 | uint32_t c_was_minor_compacted; |
207 | uint32_t c_was_major_compacted; |
208 | uint32_t c_was_major_donor; |
209 | #endif |
210 | #if CHECKSUM_THE_SWAP |
211 | unsigned int cseg_hash; |
212 | unsigned int cseg_swap_size; |
213 | #endif /* CHECKSUM_THE_SWAP */ |
214 | |
215 | thread_t c_busy_for_thread; |
216 | uint32_t c_agedin_ts; |
217 | uint32_t c_swappedin_ts; |
218 | bool c_swappedin; |
219 | /* |
220 | * Do not pull c_swappedin above into the bitfield below. |
221 | * We update it without always taking the segment |
222 | * lock and rely on the segment being busy instead. |
223 | * The bitfield needs the segment lock. So updating |
224 | * this state, if in the bitfield, without the lock |
225 | * will race with the updates to the other fields and |
226 | * result in a mess. |
227 | */ |
228 | uint32_t c_busy:1, |
229 | c_busy_swapping:1, |
230 | c_wanted:1, |
231 | c_on_minorcompact_q:1, /* can also be on the age_q, the majorcompact_q or the swappedin_q */ |
232 | |
233 | c_state:4, /* what state is the segment in which dictates which q to find it on */ |
234 | c_overage_swap:1, |
235 | c_has_donated_pages:1, |
236 | #if CONFIG_FREEZE |
237 | c_has_freezer_pages:1, |
238 | c_reserved:21; |
239 | #else /* CONFIG_FREEZE */ |
240 | c_reserved:22; |
241 | #endif /* CONFIG_FREEZE */ |
242 | |
243 | int c_slot_var_array_len; |
244 | struct c_slot *c_slot_var_array; |
245 | struct c_slot c_slot_fixed_array[0]; |
246 | }; |
247 | |
248 | |
249 | struct c_slot_mapping { |
250 | #if !CONFIG_TRACK_UNMODIFIED_ANON_PAGES |
251 | uint32_t s_cseg:22, /* segment number + 1 */ |
252 | s_cindx:10; /* index in the segment */ |
253 | #else /* !CONFIG_TRACK_UNMODIFIED_ANON_PAGES */ |
254 | uint32_t s_cseg:21, /* segment number + 1 */ |
255 | s_cindx:10, /* index in the segment */ |
256 | s_uncompressed:1; /* This bit indicates that the page resides uncompressed in a swapfile. |
257 | * This can happen in 2 ways:- |
258 | * 1) Page used to be in the compressor, got decompressed, was not |
259 | * modified, and so was pushed uncompressed to a different swapfile on disk. |
260 | * 2) Page was in its uncompressed form in a swapfile on disk. It got swapped in |
261 | * but was not modified. As we are about to reclaim it, we notice that this bit |
262 | * is set in its current slot. And so we can safely toss this clean anonymous page |
263 | * because its copy exists on disk. |
264 | */ |
265 | #endif /* !CONFIG_TRACK_UNMODIFIED_ANON_PAGES */ |
266 | }; |
267 | #define C_SLOT_MAX_INDEX (1 << 10) |
268 | |
269 | typedef struct c_slot_mapping *c_slot_mapping_t; |
270 | |
271 | |
272 | extern int c_seg_fixed_array_len; |
273 | extern vm_offset_t c_buffers; |
274 | extern int64_t c_segment_compressed_bytes; |
275 | |
276 | #define C_SEG_BUFFER_ADDRESS(c_segno) ((c_buffers + ((uint64_t)c_segno * (uint64_t)c_seg_allocsize))) |
277 | |
278 | #define C_SEG_SLOT_FROM_INDEX(cseg, index) (index < c_seg_fixed_array_len ? &(cseg->c_slot_fixed_array[index]) : &(cseg->c_slot_var_array[index - c_seg_fixed_array_len])) |
279 | |
280 | #define C_SEG_OFFSET_TO_BYTES(off) ((off) * (int) sizeof(int32_t)) |
281 | #define C_SEG_BYTES_TO_OFFSET(bytes) ((bytes) / (int) sizeof(int32_t)) |
282 | |
283 | #define C_SEG_UNUSED_BYTES(cseg) (cseg->c_bytes_unused + (C_SEG_OFFSET_TO_BYTES(cseg->c_populated_offset - cseg->c_nextoffset))) |
284 | |
285 | #ifndef __PLATFORM_WKDM_ALIGNMENT_MASK__ |
286 | #define C_SEG_OFFSET_ALIGNMENT_MASK 0x3ULL |
287 | #define C_SEG_OFFSET_ALIGNMENT_BOUNDARY 0x4 |
288 | #else |
289 | #define C_SEG_OFFSET_ALIGNMENT_MASK __PLATFORM_WKDM_ALIGNMENT_MASK__ |
290 | #define C_SEG_OFFSET_ALIGNMENT_BOUNDARY __PLATFORM_WKDM_ALIGNMENT_BOUNDARY__ |
291 | #endif |
292 | |
293 | #define C_SEG_SHOULD_MINORCOMPACT_NOW(cseg) ((C_SEG_UNUSED_BYTES(cseg) >= (c_seg_bufsize / 4)) ? 1 : 0) |
294 | |
295 | /* |
296 | * the decsion to force a c_seg to be major compacted is based on 2 criteria |
297 | * 1) is the c_seg buffer almost empty (i.e. we have a chance to merge it with another c_seg) |
298 | * 2) are there at least a minimum number of slots unoccupied so that we have a chance |
299 | * of combining this c_seg with another one. |
300 | */ |
301 | #define C_SEG_SHOULD_MAJORCOMPACT_NOW(cseg) \ |
302 | ((((cseg->c_bytes_unused + (c_seg_bufsize - C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))) >= (c_seg_bufsize / 8)) && \ |
303 | ((C_SLOT_MAX_INDEX - cseg->c_slots_used) > (c_seg_bufsize / PAGE_SIZE))) \ |
304 | ? 1 : 0) |
305 | |
306 | #define C_SEG_ONDISK_IS_SPARSE(cseg) ((cseg->c_bytes_used < cseg->c_bytes_unused) ? 1 : 0) |
307 | #define C_SEG_IS_ONDISK(cseg) ((cseg->c_state == C_ON_SWAPPEDOUT_Q || cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q)) |
308 | #define C_SEG_IS_ON_DISK_OR_SOQ(cseg) ((cseg->c_state == C_ON_SWAPPEDOUT_Q || \ |
309 | cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q || \ |
310 | cseg->c_state == C_ON_SWAPOUT_Q || \ |
311 | cseg->c_state == C_ON_SWAPIO_Q)) |
312 | |
313 | |
314 | #define C_SEG_WAKEUP_DONE(cseg) \ |
315 | MACRO_BEGIN \ |
316 | assert((cseg)->c_busy); \ |
317 | (cseg)->c_busy = 0; \ |
318 | assert((cseg)->c_busy_for_thread != NULL); \ |
319 | (cseg)->c_busy_for_thread = NULL; \ |
320 | if ((cseg)->c_wanted) { \ |
321 | (cseg)->c_wanted = 0; \ |
322 | thread_wakeup((event_t) (cseg)); \ |
323 | } \ |
324 | MACRO_END |
325 | |
326 | #define C_SEG_BUSY(cseg) \ |
327 | MACRO_BEGIN \ |
328 | assert((cseg)->c_busy == 0); \ |
329 | (cseg)->c_busy = 1; \ |
330 | assert((cseg)->c_busy_for_thread == NULL); \ |
331 | (cseg)->c_busy_for_thread = current_thread(); \ |
332 | MACRO_END |
333 | |
334 | |
335 | extern vm_map_t compressor_map; |
336 | |
337 | #if DEVELOPMENT || DEBUG |
338 | extern boolean_t write_protect_c_segs; |
339 | extern int vm_compressor_test_seg_wp; |
340 | |
341 | #define C_SEG_MAKE_WRITEABLE(cseg) \ |
342 | MACRO_BEGIN \ |
343 | if (write_protect_c_segs) { \ |
344 | vm_map_protect(compressor_map, \ |
345 | (vm_map_offset_t)cseg->c_store.c_buffer, \ |
346 | (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(c_seg_allocsize)],\ |
347 | VM_PROT_READ | VM_PROT_WRITE, \ |
348 | 0); \ |
349 | } \ |
350 | MACRO_END |
351 | |
352 | #define C_SEG_WRITE_PROTECT(cseg) \ |
353 | MACRO_BEGIN \ |
354 | if (write_protect_c_segs) { \ |
355 | vm_map_protect(compressor_map, \ |
356 | (vm_map_offset_t)cseg->c_store.c_buffer, \ |
357 | (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(c_seg_allocsize)],\ |
358 | VM_PROT_READ, \ |
359 | 0); \ |
360 | } \ |
361 | if (vm_compressor_test_seg_wp) { \ |
362 | volatile uint32_t vmtstmp = *(volatile uint32_t *)cseg->c_store.c_buffer; \ |
363 | *(volatile uint32_t *)cseg->c_store.c_buffer = 0xDEADABCD; \ |
364 | (void) vmtstmp; \ |
365 | } \ |
366 | MACRO_END |
367 | #endif |
368 | |
369 | typedef struct c_segment *c_segment_t; |
370 | typedef struct c_slot *c_slot_t; |
371 | |
372 | uint64_t vm_compressor_total_compressions(void); |
373 | void vm_wake_compactor_swapper(void); |
374 | void vm_run_compactor(void); |
375 | void vm_thrashing_jetsam_done(void); |
376 | void vm_consider_waking_compactor_swapper(void); |
377 | void vm_consider_swapping(void); |
378 | void vm_compressor_flush(void); |
379 | void c_seg_free(c_segment_t); |
380 | bool vm_compressor_is_thrashing(void); |
381 | bool vm_compressor_needs_to_swap(bool wake_memorystatus_thread); |
382 | void c_seg_free_locked(c_segment_t); |
383 | void c_seg_insert_into_age_q(c_segment_t); |
384 | void c_seg_need_delayed_compaction(c_segment_t, boolean_t); |
385 | void c_seg_update_task_owner(c_segment_t, task_t); |
386 | |
387 | void vm_decompressor_lock(void); |
388 | void vm_decompressor_unlock(void); |
389 | |
390 | void vm_compressor_delay_trim(void); |
391 | void vm_compressor_do_warmup(void); |
392 | void vm_compressor_record_warmup_start(void); |
393 | void vm_compressor_record_warmup_end(void); |
394 | |
395 | int vm_wants_task_throttled(task_t); |
396 | |
397 | extern void vm_compaction_swapper_do_init(void); |
398 | extern void vm_compressor_swap_init(void); |
399 | extern lck_rw_t c_master_lock; |
400 | |
401 | #if ENCRYPTED_SWAP |
402 | extern void vm_swap_decrypt(c_segment_t); |
403 | #endif /* ENCRYPTED_SWAP */ |
404 | |
405 | extern int vm_swap_low_on_space(void); |
406 | extern int vm_swap_out_of_space(void); |
407 | extern kern_return_t vm_swap_get(c_segment_t, uint64_t, uint64_t); |
408 | extern void vm_swap_free(uint64_t); |
409 | extern void vm_swap_consider_defragmenting(int); |
410 | |
411 | extern void c_seg_swapin_requeue(c_segment_t, boolean_t, boolean_t, boolean_t); |
412 | extern int c_seg_swapin(c_segment_t, boolean_t, boolean_t); |
413 | extern void c_seg_wait_on_busy(c_segment_t); |
414 | extern void c_seg_trim_tail(c_segment_t); |
415 | extern void c_seg_switch_state(c_segment_t, int, boolean_t); |
416 | |
417 | extern boolean_t fastwake_recording_in_progress; |
418 | extern int compaction_swapper_inited; |
419 | extern int compaction_swapper_running; |
420 | extern uint64_t vm_swap_put_failures; |
421 | |
422 | extern int c_overage_swapped_count; |
423 | extern int c_overage_swapped_limit; |
424 | |
425 | extern queue_head_t c_minor_list_head; |
426 | extern queue_head_t c_age_list_head; |
427 | extern queue_head_t c_major_list_head; |
428 | extern queue_head_t c_early_swapout_list_head; |
429 | extern queue_head_t c_regular_swapout_list_head; |
430 | extern queue_head_t c_late_swapout_list_head; |
431 | extern queue_head_t c_swappedout_list_head; |
432 | extern queue_head_t c_swappedout_sparse_list_head; |
433 | |
434 | extern uint32_t c_age_count; |
435 | extern uint32_t c_early_swapout_count, c_regular_swapout_count, c_late_swapout_count; |
436 | extern uint32_t c_swappedout_count; |
437 | extern uint32_t c_swappedout_sparse_count; |
438 | |
439 | extern int64_t compressor_bytes_used; |
440 | extern uint64_t first_c_segment_to_warm_generation_id; |
441 | extern uint64_t last_c_segment_to_warm_generation_id; |
442 | extern boolean_t hibernate_flushing; |
443 | extern boolean_t hibernate_no_swapspace; |
444 | extern boolean_t hibernate_in_progress_with_pinned_swap; |
445 | extern boolean_t hibernate_flush_timed_out; |
446 | extern uint32_t swapout_target_age; |
447 | |
448 | extern void c_seg_insert_into_q(queue_head_t *, c_segment_t); |
449 | |
450 | extern uint32_t vm_compressor_minorcompact_threshold_divisor; |
451 | extern uint32_t vm_compressor_majorcompact_threshold_divisor; |
452 | extern uint32_t vm_compressor_unthrottle_threshold_divisor; |
453 | extern uint32_t vm_compressor_catchup_threshold_divisor; |
454 | |
455 | extern uint32_t vm_compressor_minorcompact_threshold_divisor_overridden; |
456 | extern uint32_t vm_compressor_majorcompact_threshold_divisor_overridden; |
457 | extern uint32_t vm_compressor_unthrottle_threshold_divisor_overridden; |
458 | extern uint32_t vm_compressor_catchup_threshold_divisor_overridden; |
459 | |
460 | extern uint64_t vm_compressor_compute_elapsed_msecs(clock_sec_t, clock_nsec_t, clock_sec_t, clock_nsec_t); |
461 | |
462 | extern void kdp_compressor_busy_find_owner(event64_t wait_event, thread_waitinfo_t *waitinfo); |
463 | |
464 | #define PAGE_REPLACEMENT_DISALLOWED(enable) (enable == TRUE ? lck_rw_lock_shared(&c_master_lock) : lck_rw_done(&c_master_lock)) |
465 | #define PAGE_REPLACEMENT_ALLOWED(enable) (enable == TRUE ? lck_rw_lock_exclusive(&c_master_lock) : lck_rw_done(&c_master_lock)) |
466 | |
467 | |
468 | #define AVAILABLE_NON_COMPRESSED_MEMORY (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count) |
469 | #define AVAILABLE_MEMORY (AVAILABLE_NON_COMPRESSED_MEMORY + VM_PAGE_COMPRESSOR_COUNT) |
470 | |
471 | /* |
472 | * TODO, there may be a minor optimisation opportunity to replace these divisions |
473 | * with multiplies and shifts |
474 | * |
475 | * By multiplying by 10, the divisors can have more precision w/o resorting to floating point... a divisor specified as 25 is in reality a divide by 2.5 |
476 | * By multiplying by 9, you get a number ~11% smaller which allows us to have another limit point derived from the same base |
477 | * By multiplying by 11, you get a number ~10% bigger which allows us to generate a reset limit derived from the same base which is useful for hysteresis |
478 | */ |
479 | |
480 | #define VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_minorcompact_threshold_divisor ? vm_compressor_minorcompact_threshold_divisor : 10)) |
481 | #define VM_PAGE_COMPRESSOR_SWAP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_majorcompact_threshold_divisor ? vm_compressor_majorcompact_threshold_divisor : 10)) |
482 | |
483 | #define VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 10)) |
484 | #define VM_PAGE_COMPRESSOR_SWAP_RETHROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 11) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 11)) |
485 | |
486 | #define VM_PAGE_COMPRESSOR_SWAP_HAS_CAUGHTUP_THRESHOLD (((AVAILABLE_MEMORY) * 11) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 11)) |
487 | #define VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 10)) |
488 | #define VM_PAGE_COMPRESSOR_HARD_THROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 9) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 9)) |
489 | |
490 | #if !XNU_TARGET_OS_OSX |
491 | #define AVAILABLE_NON_COMPRESSED_MIN 20000 |
492 | #define COMPRESSOR_NEEDS_TO_SWAP() (((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) || \ |
493 | (AVAILABLE_NON_COMPRESSED_MEMORY < AVAILABLE_NON_COMPRESSED_MIN)) ? 1 : 0) |
494 | #else /* !XNU_TARGET_OS_OSX */ |
495 | #define COMPRESSOR_NEEDS_TO_SWAP() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0) |
496 | #endif /* !XNU_TARGET_OS_OSX */ |
497 | |
498 | #define HARD_THROTTLE_LIMIT_REACHED() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_HARD_THROTTLE_THRESHOLD) ? 1 : 0) |
499 | #define SWAPPER_NEEDS_TO_UNTHROTTLE() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) ? 1 : 0) |
500 | #define SWAPPER_NEEDS_TO_RETHROTTLE() ((AVAILABLE_NON_COMPRESSED_MEMORY > VM_PAGE_COMPRESSOR_SWAP_RETHROTTLE_THRESHOLD) ? 1 : 0) |
501 | #define SWAPPER_NEEDS_TO_CATCHUP() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD) ? 1 : 0) |
502 | #define SWAPPER_HAS_CAUGHTUP() ((AVAILABLE_NON_COMPRESSED_MEMORY > VM_PAGE_COMPRESSOR_SWAP_HAS_CAUGHTUP_THRESHOLD) ? 1 : 0) |
503 | #define COMPRESSOR_NEEDS_TO_MINOR_COMPACT() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0) |
504 | |
505 | |
506 | #if !XNU_TARGET_OS_OSX |
507 | #define COMPRESSOR_FREE_RESERVED_LIMIT 28 |
508 | #else /* !XNU_TARGET_OS_OSX */ |
509 | #define COMPRESSOR_FREE_RESERVED_LIMIT 128 |
510 | #endif /* !XNU_TARGET_OS_OSX */ |
511 | |
512 | uint32_t vm_compressor_get_encode_scratch_size(void) __pure2; |
513 | uint32_t vm_compressor_get_decode_scratch_size(void) __pure2; |
514 | |
515 | #define COMPRESSOR_SCRATCH_BUF_SIZE vm_compressor_get_encode_scratch_size() |
516 | |
517 | #if RECORD_THE_COMPRESSED_DATA |
518 | extern void c_compressed_record_init(void); |
519 | extern void c_compressed_record_write(char *, int); |
520 | #endif |
521 | |
522 | extern lck_mtx_t c_list_lock_storage; |
523 | #define c_list_lock (&c_list_lock_storage) |
524 | |
525 | #if DEVELOPMENT || DEBUG |
526 | extern uint32_t vm_ktrace_enabled; |
527 | |
528 | #define VMKDBG(x, ...) \ |
529 | MACRO_BEGIN \ |
530 | if (vm_ktrace_enabled) { \ |
531 | KDBG(x, ## __VA_ARGS__);\ |
532 | } \ |
533 | MACRO_END |
534 | |
535 | #if DEVELOPMENT || DEBUG |
536 | extern bool compressor_running_perf_test; |
537 | extern uint64_t compressor_perf_test_pages_processed; |
538 | #endif /* DEVELOPMENT || DEBUG */ |
539 | #endif |
540 | |
541 | #endif /* _VM_VM_COMPRESSOR_H_ */ |
542 | |