1/*
2 * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#ifndef _VM_VM_COMPRESSOR_H_
29#define _VM_VM_COMPRESSOR_H_
30
31#include <vm/vm_compressor_pager.h>
32#include <vm/vm_kern.h>
33#include <vm/vm_page.h>
34#include <vm/vm_protos.h>
35#include <vm/WKdm_new.h>
36#include <vm/vm_object.h>
37#include <vm/vm_map.h>
38#include <machine/pmap.h>
39#include <kern/locks.h>
40
41#include <sys/kdebug.h>
42
43#if defined(__arm64__)
44#include <arm64/proc_reg.h>
45#endif
46
47#define C_SEG_OFFSET_BITS 16
48
49#define C_SEG_MAX_POPULATE_SIZE (4 * PAGE_SIZE)
50
51#if defined(__arm64__) && (DEVELOPMENT || DEBUG)
52
53#if defined(XNU_PLATFORM_WatchOS)
54#define VALIDATE_C_SEGMENTS (1)
55#endif
56#endif /* defined(__arm64__) && (DEVELOPMENT || DEBUG) */
57
58
59#if DEBUG || COMPRESSOR_INTEGRITY_CHECKS
60#define ENABLE_SWAP_CHECKS 1
61#define ENABLE_COMPRESSOR_CHECKS 1
62#define POPCOUNT_THE_COMPRESSED_DATA (1)
63#else
64#define ENABLE_SWAP_CHECKS 0
65#define ENABLE_COMPRESSOR_CHECKS 0
66#endif
67
68#define CHECKSUM_THE_SWAP ENABLE_SWAP_CHECKS /* Debug swap data */
69#define CHECKSUM_THE_DATA ENABLE_COMPRESSOR_CHECKS /* Debug compressor/decompressor data */
70#define CHECKSUM_THE_COMPRESSED_DATA ENABLE_COMPRESSOR_CHECKS /* Debug compressor/decompressor compressed data */
71
72#ifndef VALIDATE_C_SEGMENTS
73#define VALIDATE_C_SEGMENTS ENABLE_COMPRESSOR_CHECKS /* Debug compaction */
74#endif
75
76#define RECORD_THE_COMPRESSED_DATA 0
77
78/*
79 * The c_slot structure embeds a packed pointer to a c_slot_mapping
80 * (32bits) which we ideally want to span as much VA space as possible
81 * to not limit zalloc in how it sets itself up.
82 */
83#if !defined(__LP64__) /* no packing */
84#define C_SLOT_PACKED_PTR_BITS 32
85#define C_SLOT_PACKED_PTR_SHIFT 0
86#define C_SLOT_PACKED_PTR_BASE 0
87
88#define C_SLOT_C_SIZE_BITS 12
89#define C_SLOT_C_CODEC_BITS 1
90#define C_SLOT_C_POPCOUNT_BITS 0
91#define C_SLOT_C_PADDING_BITS 3
92
93#elif defined(__arm64__) /* 32G from the heap start */
94#define C_SLOT_PACKED_PTR_BITS 33
95#define C_SLOT_PACKED_PTR_SHIFT 2
96#define C_SLOT_PACKED_PTR_BASE ((uintptr_t)KERNEL_PMAP_HEAP_RANGE_START)
97
98#define C_SLOT_C_SIZE_BITS 14
99#define C_SLOT_C_CODEC_BITS 1
100#define C_SLOT_C_POPCOUNT_BITS 0
101#define C_SLOT_C_PADDING_BITS 0
102
103#elif defined(__x86_64__) /* 256G from the heap start */
104#define C_SLOT_PACKED_PTR_BITS 36
105#define C_SLOT_PACKED_PTR_SHIFT 2
106#define C_SLOT_PACKED_PTR_BASE ((uintptr_t)KERNEL_PMAP_HEAP_RANGE_START)
107
108#define C_SLOT_C_SIZE_BITS 12
109#define C_SLOT_C_CODEC_BITS 0 /* not used */
110#define C_SLOT_C_POPCOUNT_BITS 0
111#define C_SLOT_C_PADDING_BITS 0
112
113#else
114#error vm_compressor parameters undefined for this architecture
115#endif
116
117/*
118 * Popcounts needs to represent both 0 and full which requires
119 * (8 ^ C_SLOT_C_SIZE_BITS) + 1 values and (C_SLOT_C_SIZE_BITS + 4) bits.
120 *
121 * We us the (2 * (8 ^ C_SLOT_C_SIZE_BITS) - 1) value to mean "unknown".
122 */
123#define C_SLOT_NO_POPCOUNT ((16u << C_SLOT_C_SIZE_BITS) - 1)
124
125static_assert((C_SEG_OFFSET_BITS + C_SLOT_C_SIZE_BITS +
126 C_SLOT_C_CODEC_BITS + C_SLOT_C_POPCOUNT_BITS +
127 C_SLOT_C_PADDING_BITS + C_SLOT_PACKED_PTR_BITS) % 32 == 0);
128
129struct c_slot {
130 uint64_t c_offset:C_SEG_OFFSET_BITS __kernel_ptr_semantics;
131 uint64_t c_size:C_SLOT_C_SIZE_BITS;
132#if C_SLOT_C_CODEC_BITS
133 uint64_t c_codec:C_SLOT_C_CODEC_BITS;
134#endif
135#if C_SLOT_C_POPCOUNT_BITS
136 /*
137 * This value may not agree with c_pop_cdata, as it may be the
138 * population count of the uncompressed data.
139 *
140 * This value must be C_SLOT_NO_POPCOUNT when the compression algorithm
141 * cannot provide it.
142 */
143 uint32_t c_inline_popcount:C_SLOT_C_POPCOUNT_BITS;
144#endif
145#if C_SLOT_C_PADDING_BITS
146 uint64_t c_padding:C_SLOT_C_PADDING_BITS;
147#endif
148 uint64_t c_packed_ptr:C_SLOT_PACKED_PTR_BITS __kernel_ptr_semantics;
149
150 /* debugging fields, typically not present on release kernels */
151#if CHECKSUM_THE_DATA
152 unsigned int c_hash_data;
153#endif
154#if CHECKSUM_THE_COMPRESSED_DATA
155 unsigned int c_hash_compressed_data;
156#endif
157#if POPCOUNT_THE_COMPRESSED_DATA
158 unsigned int c_pop_cdata;
159#endif
160} __attribute__((packed, aligned(4)));
161
162#define C_IS_EMPTY 0
163#define C_IS_FREE 1
164#define C_IS_FILLING 2
165#define C_ON_AGE_Q 3
166#define C_ON_SWAPOUT_Q 4
167#define C_ON_SWAPPEDOUT_Q 5
168#define C_ON_SWAPPEDOUTSPARSE_Q 6
169#define C_ON_SWAPPEDIN_Q 7
170#define C_ON_MAJORCOMPACT_Q 8
171#define C_ON_BAD_Q 9
172#define C_ON_SWAPIO_Q 10
173
174
175struct c_segment {
176 lck_mtx_t c_lock;
177 queue_chain_t c_age_list;
178 queue_chain_t c_list;
179
180#if CONFIG_FREEZE
181 queue_chain_t c_task_list_next_cseg;
182 task_t c_task_owner;
183#endif /* CONFIG_FREEZE */
184
185#define C_SEG_MAX_LIMIT (UINT_MAX) /* this needs to track the size of c_mysegno */
186 uint32_t c_mysegno;
187
188 uint32_t c_creation_ts;
189 uint64_t c_generation_id;
190
191 int32_t c_bytes_used;
192 int32_t c_bytes_unused;
193 uint32_t c_slots_used;
194
195 uint16_t c_firstemptyslot;
196 uint16_t c_nextslot;
197 uint32_t c_nextoffset;
198 uint32_t c_populated_offset;
199
200 union {
201 int32_t *c_buffer;
202 uint64_t c_swap_handle;
203 } c_store;
204
205#if VALIDATE_C_SEGMENTS
206 uint32_t c_was_minor_compacted;
207 uint32_t c_was_major_compacted;
208 uint32_t c_was_major_donor;
209#endif
210#if CHECKSUM_THE_SWAP
211 unsigned int cseg_hash;
212 unsigned int cseg_swap_size;
213#endif /* CHECKSUM_THE_SWAP */
214
215 thread_t c_busy_for_thread;
216 uint32_t c_agedin_ts;
217 uint32_t c_swappedin_ts;
218 bool c_swappedin;
219 /*
220 * Do not pull c_swappedin above into the bitfield below.
221 * We update it without always taking the segment
222 * lock and rely on the segment being busy instead.
223 * The bitfield needs the segment lock. So updating
224 * this state, if in the bitfield, without the lock
225 * will race with the updates to the other fields and
226 * result in a mess.
227 */
228 uint32_t c_busy:1,
229 c_busy_swapping:1,
230 c_wanted:1,
231 c_on_minorcompact_q:1, /* can also be on the age_q, the majorcompact_q or the swappedin_q */
232
233 c_state:4, /* what state is the segment in which dictates which q to find it on */
234 c_overage_swap:1,
235 c_has_donated_pages:1,
236#if CONFIG_FREEZE
237 c_has_freezer_pages:1,
238 c_reserved:21;
239#else /* CONFIG_FREEZE */
240 c_reserved:22;
241#endif /* CONFIG_FREEZE */
242
243 int c_slot_var_array_len;
244 struct c_slot *c_slot_var_array;
245 struct c_slot c_slot_fixed_array[0];
246};
247
248
249struct c_slot_mapping {
250#if !CONFIG_TRACK_UNMODIFIED_ANON_PAGES
251 uint32_t s_cseg:22, /* segment number + 1 */
252 s_cindx:10; /* index in the segment */
253#else /* !CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
254 uint32_t s_cseg:21, /* segment number + 1 */
255 s_cindx:10, /* index in the segment */
256 s_uncompressed:1; /* This bit indicates that the page resides uncompressed in a swapfile.
257 * This can happen in 2 ways:-
258 * 1) Page used to be in the compressor, got decompressed, was not
259 * modified, and so was pushed uncompressed to a different swapfile on disk.
260 * 2) Page was in its uncompressed form in a swapfile on disk. It got swapped in
261 * but was not modified. As we are about to reclaim it, we notice that this bit
262 * is set in its current slot. And so we can safely toss this clean anonymous page
263 * because its copy exists on disk.
264 */
265#endif /* !CONFIG_TRACK_UNMODIFIED_ANON_PAGES */
266};
267#define C_SLOT_MAX_INDEX (1 << 10)
268
269typedef struct c_slot_mapping *c_slot_mapping_t;
270
271
272extern int c_seg_fixed_array_len;
273extern vm_offset_t c_buffers;
274extern int64_t c_segment_compressed_bytes;
275
276#define C_SEG_BUFFER_ADDRESS(c_segno) ((c_buffers + ((uint64_t)c_segno * (uint64_t)c_seg_allocsize)))
277
278#define C_SEG_SLOT_FROM_INDEX(cseg, index) (index < c_seg_fixed_array_len ? &(cseg->c_slot_fixed_array[index]) : &(cseg->c_slot_var_array[index - c_seg_fixed_array_len]))
279
280#define C_SEG_OFFSET_TO_BYTES(off) ((off) * (int) sizeof(int32_t))
281#define C_SEG_BYTES_TO_OFFSET(bytes) ((bytes) / (int) sizeof(int32_t))
282
283#define C_SEG_UNUSED_BYTES(cseg) (cseg->c_bytes_unused + (C_SEG_OFFSET_TO_BYTES(cseg->c_populated_offset - cseg->c_nextoffset)))
284
285#ifndef __PLATFORM_WKDM_ALIGNMENT_MASK__
286#define C_SEG_OFFSET_ALIGNMENT_MASK 0x3ULL
287#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY 0x4
288#else
289#define C_SEG_OFFSET_ALIGNMENT_MASK __PLATFORM_WKDM_ALIGNMENT_MASK__
290#define C_SEG_OFFSET_ALIGNMENT_BOUNDARY __PLATFORM_WKDM_ALIGNMENT_BOUNDARY__
291#endif
292
293#define C_SEG_SHOULD_MINORCOMPACT_NOW(cseg) ((C_SEG_UNUSED_BYTES(cseg) >= (c_seg_bufsize / 4)) ? 1 : 0)
294
295/*
296 * the decsion to force a c_seg to be major compacted is based on 2 criteria
297 * 1) is the c_seg buffer almost empty (i.e. we have a chance to merge it with another c_seg)
298 * 2) are there at least a minimum number of slots unoccupied so that we have a chance
299 * of combining this c_seg with another one.
300 */
301#define C_SEG_SHOULD_MAJORCOMPACT_NOW(cseg) \
302 ((((cseg->c_bytes_unused + (c_seg_bufsize - C_SEG_OFFSET_TO_BYTES(c_seg->c_nextoffset))) >= (c_seg_bufsize / 8)) && \
303 ((C_SLOT_MAX_INDEX - cseg->c_slots_used) > (c_seg_bufsize / PAGE_SIZE))) \
304 ? 1 : 0)
305
306#define C_SEG_ONDISK_IS_SPARSE(cseg) ((cseg->c_bytes_used < cseg->c_bytes_unused) ? 1 : 0)
307#define C_SEG_IS_ONDISK(cseg) ((cseg->c_state == C_ON_SWAPPEDOUT_Q || cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q))
308#define C_SEG_IS_ON_DISK_OR_SOQ(cseg) ((cseg->c_state == C_ON_SWAPPEDOUT_Q || \
309 cseg->c_state == C_ON_SWAPPEDOUTSPARSE_Q || \
310 cseg->c_state == C_ON_SWAPOUT_Q || \
311 cseg->c_state == C_ON_SWAPIO_Q))
312
313
314#define C_SEG_WAKEUP_DONE(cseg) \
315 MACRO_BEGIN \
316 assert((cseg)->c_busy); \
317 (cseg)->c_busy = 0; \
318 assert((cseg)->c_busy_for_thread != NULL); \
319 (cseg)->c_busy_for_thread = NULL; \
320 if ((cseg)->c_wanted) { \
321 (cseg)->c_wanted = 0; \
322 thread_wakeup((event_t) (cseg)); \
323 } \
324 MACRO_END
325
326#define C_SEG_BUSY(cseg) \
327 MACRO_BEGIN \
328 assert((cseg)->c_busy == 0); \
329 (cseg)->c_busy = 1; \
330 assert((cseg)->c_busy_for_thread == NULL); \
331 (cseg)->c_busy_for_thread = current_thread(); \
332 MACRO_END
333
334
335extern vm_map_t compressor_map;
336
337#if DEVELOPMENT || DEBUG
338extern boolean_t write_protect_c_segs;
339extern int vm_compressor_test_seg_wp;
340
341#define C_SEG_MAKE_WRITEABLE(cseg) \
342 MACRO_BEGIN \
343 if (write_protect_c_segs) { \
344 vm_map_protect(compressor_map, \
345 (vm_map_offset_t)cseg->c_store.c_buffer, \
346 (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(c_seg_allocsize)],\
347 VM_PROT_READ | VM_PROT_WRITE, \
348 0); \
349 } \
350 MACRO_END
351
352#define C_SEG_WRITE_PROTECT(cseg) \
353 MACRO_BEGIN \
354 if (write_protect_c_segs) { \
355 vm_map_protect(compressor_map, \
356 (vm_map_offset_t)cseg->c_store.c_buffer, \
357 (vm_map_offset_t)&cseg->c_store.c_buffer[C_SEG_BYTES_TO_OFFSET(c_seg_allocsize)],\
358 VM_PROT_READ, \
359 0); \
360 } \
361 if (vm_compressor_test_seg_wp) { \
362 volatile uint32_t vmtstmp = *(volatile uint32_t *)cseg->c_store.c_buffer; \
363 *(volatile uint32_t *)cseg->c_store.c_buffer = 0xDEADABCD; \
364 (void) vmtstmp; \
365 } \
366 MACRO_END
367#endif
368
369typedef struct c_segment *c_segment_t;
370typedef struct c_slot *c_slot_t;
371
372uint64_t vm_compressor_total_compressions(void);
373void vm_wake_compactor_swapper(void);
374void vm_run_compactor(void);
375void vm_thrashing_jetsam_done(void);
376void vm_consider_waking_compactor_swapper(void);
377void vm_consider_swapping(void);
378void vm_compressor_flush(void);
379void c_seg_free(c_segment_t);
380bool vm_compressor_is_thrashing(void);
381bool vm_compressor_needs_to_swap(bool wake_memorystatus_thread);
382void c_seg_free_locked(c_segment_t);
383void c_seg_insert_into_age_q(c_segment_t);
384void c_seg_need_delayed_compaction(c_segment_t, boolean_t);
385void c_seg_update_task_owner(c_segment_t, task_t);
386
387void vm_decompressor_lock(void);
388void vm_decompressor_unlock(void);
389
390void vm_compressor_delay_trim(void);
391void vm_compressor_do_warmup(void);
392void vm_compressor_record_warmup_start(void);
393void vm_compressor_record_warmup_end(void);
394
395int vm_wants_task_throttled(task_t);
396
397extern void vm_compaction_swapper_do_init(void);
398extern void vm_compressor_swap_init(void);
399extern lck_rw_t c_master_lock;
400
401#if ENCRYPTED_SWAP
402extern void vm_swap_decrypt(c_segment_t);
403#endif /* ENCRYPTED_SWAP */
404
405extern int vm_swap_low_on_space(void);
406extern int vm_swap_out_of_space(void);
407extern kern_return_t vm_swap_get(c_segment_t, uint64_t, uint64_t);
408extern void vm_swap_free(uint64_t);
409extern void vm_swap_consider_defragmenting(int);
410
411extern void c_seg_swapin_requeue(c_segment_t, boolean_t, boolean_t, boolean_t);
412extern int c_seg_swapin(c_segment_t, boolean_t, boolean_t);
413extern void c_seg_wait_on_busy(c_segment_t);
414extern void c_seg_trim_tail(c_segment_t);
415extern void c_seg_switch_state(c_segment_t, int, boolean_t);
416
417extern boolean_t fastwake_recording_in_progress;
418extern int compaction_swapper_inited;
419extern int compaction_swapper_running;
420extern uint64_t vm_swap_put_failures;
421
422extern int c_overage_swapped_count;
423extern int c_overage_swapped_limit;
424
425extern queue_head_t c_minor_list_head;
426extern queue_head_t c_age_list_head;
427extern queue_head_t c_major_list_head;
428extern queue_head_t c_early_swapout_list_head;
429extern queue_head_t c_regular_swapout_list_head;
430extern queue_head_t c_late_swapout_list_head;
431extern queue_head_t c_swappedout_list_head;
432extern queue_head_t c_swappedout_sparse_list_head;
433
434extern uint32_t c_age_count;
435extern uint32_t c_early_swapout_count, c_regular_swapout_count, c_late_swapout_count;
436extern uint32_t c_swappedout_count;
437extern uint32_t c_swappedout_sparse_count;
438
439extern int64_t compressor_bytes_used;
440extern uint64_t first_c_segment_to_warm_generation_id;
441extern uint64_t last_c_segment_to_warm_generation_id;
442extern boolean_t hibernate_flushing;
443extern boolean_t hibernate_no_swapspace;
444extern boolean_t hibernate_in_progress_with_pinned_swap;
445extern boolean_t hibernate_flush_timed_out;
446extern uint32_t swapout_target_age;
447
448extern void c_seg_insert_into_q(queue_head_t *, c_segment_t);
449
450extern uint32_t vm_compressor_minorcompact_threshold_divisor;
451extern uint32_t vm_compressor_majorcompact_threshold_divisor;
452extern uint32_t vm_compressor_unthrottle_threshold_divisor;
453extern uint32_t vm_compressor_catchup_threshold_divisor;
454
455extern uint32_t vm_compressor_minorcompact_threshold_divisor_overridden;
456extern uint32_t vm_compressor_majorcompact_threshold_divisor_overridden;
457extern uint32_t vm_compressor_unthrottle_threshold_divisor_overridden;
458extern uint32_t vm_compressor_catchup_threshold_divisor_overridden;
459
460extern uint64_t vm_compressor_compute_elapsed_msecs(clock_sec_t, clock_nsec_t, clock_sec_t, clock_nsec_t);
461
462extern void kdp_compressor_busy_find_owner(event64_t wait_event, thread_waitinfo_t *waitinfo);
463
464#define PAGE_REPLACEMENT_DISALLOWED(enable) (enable == TRUE ? lck_rw_lock_shared(&c_master_lock) : lck_rw_done(&c_master_lock))
465#define PAGE_REPLACEMENT_ALLOWED(enable) (enable == TRUE ? lck_rw_lock_exclusive(&c_master_lock) : lck_rw_done(&c_master_lock))
466
467
468#define AVAILABLE_NON_COMPRESSED_MEMORY (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
469#define AVAILABLE_MEMORY (AVAILABLE_NON_COMPRESSED_MEMORY + VM_PAGE_COMPRESSOR_COUNT)
470
471/*
472 * TODO, there may be a minor optimisation opportunity to replace these divisions
473 * with multiplies and shifts
474 *
475 * By multiplying by 10, the divisors can have more precision w/o resorting to floating point... a divisor specified as 25 is in reality a divide by 2.5
476 * By multiplying by 9, you get a number ~11% smaller which allows us to have another limit point derived from the same base
477 * By multiplying by 11, you get a number ~10% bigger which allows us to generate a reset limit derived from the same base which is useful for hysteresis
478 */
479
480#define VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_minorcompact_threshold_divisor ? vm_compressor_minorcompact_threshold_divisor : 10))
481#define VM_PAGE_COMPRESSOR_SWAP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_majorcompact_threshold_divisor ? vm_compressor_majorcompact_threshold_divisor : 10))
482
483#define VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 10))
484#define VM_PAGE_COMPRESSOR_SWAP_RETHROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 11) / (vm_compressor_unthrottle_threshold_divisor ? vm_compressor_unthrottle_threshold_divisor : 11))
485
486#define VM_PAGE_COMPRESSOR_SWAP_HAS_CAUGHTUP_THRESHOLD (((AVAILABLE_MEMORY) * 11) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 11))
487#define VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD (((AVAILABLE_MEMORY) * 10) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 10))
488#define VM_PAGE_COMPRESSOR_HARD_THROTTLE_THRESHOLD (((AVAILABLE_MEMORY) * 9) / (vm_compressor_catchup_threshold_divisor ? vm_compressor_catchup_threshold_divisor : 9))
489
490#if !XNU_TARGET_OS_OSX
491#define AVAILABLE_NON_COMPRESSED_MIN 20000
492#define COMPRESSOR_NEEDS_TO_SWAP() (((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) || \
493 (AVAILABLE_NON_COMPRESSED_MEMORY < AVAILABLE_NON_COMPRESSED_MIN)) ? 1 : 0)
494#else /* !XNU_TARGET_OS_OSX */
495#define COMPRESSOR_NEEDS_TO_SWAP() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_THRESHOLD) ? 1 : 0)
496#endif /* !XNU_TARGET_OS_OSX */
497
498#define HARD_THROTTLE_LIMIT_REACHED() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_HARD_THROTTLE_THRESHOLD) ? 1 : 0)
499#define SWAPPER_NEEDS_TO_UNTHROTTLE() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) ? 1 : 0)
500#define SWAPPER_NEEDS_TO_RETHROTTLE() ((AVAILABLE_NON_COMPRESSED_MEMORY > VM_PAGE_COMPRESSOR_SWAP_RETHROTTLE_THRESHOLD) ? 1 : 0)
501#define SWAPPER_NEEDS_TO_CATCHUP() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_SWAP_CATCHUP_THRESHOLD) ? 1 : 0)
502#define SWAPPER_HAS_CAUGHTUP() ((AVAILABLE_NON_COMPRESSED_MEMORY > VM_PAGE_COMPRESSOR_SWAP_HAS_CAUGHTUP_THRESHOLD) ? 1 : 0)
503#define COMPRESSOR_NEEDS_TO_MINOR_COMPACT() ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0)
504
505
506#if !XNU_TARGET_OS_OSX
507#define COMPRESSOR_FREE_RESERVED_LIMIT 28
508#else /* !XNU_TARGET_OS_OSX */
509#define COMPRESSOR_FREE_RESERVED_LIMIT 128
510#endif /* !XNU_TARGET_OS_OSX */
511
512uint32_t vm_compressor_get_encode_scratch_size(void) __pure2;
513uint32_t vm_compressor_get_decode_scratch_size(void) __pure2;
514
515#define COMPRESSOR_SCRATCH_BUF_SIZE vm_compressor_get_encode_scratch_size()
516
517#if RECORD_THE_COMPRESSED_DATA
518extern void c_compressed_record_init(void);
519extern void c_compressed_record_write(char *, int);
520#endif
521
522extern lck_mtx_t c_list_lock_storage;
523#define c_list_lock (&c_list_lock_storage)
524
525#if DEVELOPMENT || DEBUG
526extern uint32_t vm_ktrace_enabled;
527
528#define VMKDBG(x, ...) \
529MACRO_BEGIN \
530if (vm_ktrace_enabled) { \
531 KDBG(x, ## __VA_ARGS__);\
532} \
533MACRO_END
534
535#if DEVELOPMENT || DEBUG
536extern bool compressor_running_perf_test;
537extern uint64_t compressor_perf_test_pages_processed;
538#endif /* DEVELOPMENT || DEBUG */
539#endif
540
541#endif /* _VM_VM_COMPRESSOR_H_ */
542