1/*
2 * Copyright (c) 2006-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 *
28 */
29
30#ifndef _KERN_MEMORYSTATUS_INTERNAL_H_
31#define _KERN_MEMORYSTATUS_INTERNAL_H_
32
33/*
34 * Contains memorystatus subsystem definitions that are not
35 * exported outside of the memorystatus subsystem.
36 *
37 * For example, all of the mechanisms used by kern_memorystatus_policy.c
38 * should be defined in this header.
39 */
40
41#if BSD_KERNEL_PRIVATE
42
43#include <mach/boolean.h>
44#include <stdbool.h>
45#include <os/base.h>
46#include <os/log.h>
47#include <kern/sched_prim.h>
48
49#if CONFIG_FREEZE
50#include <sys/kern_memorystatus_freeze.h>
51#endif /* CONFIG_FREEZE */
52
53/*
54 * memorystatus subsystem globals
55 */
56#if CONFIG_JETSAM
57extern unsigned int memorystatus_available_pages;
58extern unsigned int memorystatus_available_pages_pressure;
59extern unsigned int memorystatus_available_pages_critical;
60extern uint32_t jetsam_kill_on_low_swap;
61#else /* CONFIG_JETSAM */
62extern uint64_t memorystatus_available_pages;
63extern uint64_t memorystatus_available_pages_pressure;
64extern uint64_t memorystatus_available_pages_critical;
65#endif /* CONFIG_JETSAM */
66extern int block_corpses; /* counter to block new corpses if jetsam purges them */
67extern int system_procs_aging_band;
68extern int applications_aging_band;
69/*
70 * TODO(jason): This should really be calculated dynamically by the zalloc
71 * subsystem before we do a zone map exhaustion kill. But the zone_gc
72 * logic is non-trivial, so for now it just sets this global.
73 */
74extern _Atomic bool memorystatus_zone_map_is_exhausted;
75/*
76 * TODO(jason): We should get rid of this global
77 * and have the memorystatus thread check for compressor space shortages
78 * itself. However, there are 3 async call sites remaining that require more work to get us there:
79 * 2 of them are in vm_swap_defragment. When it's about to swap in a segment, it checks if that
80 * will cause a compressor space shortage & pre-emptively triggers jetsam. vm_compressor_backing_store
81 * needs to keep track of in-flight swapins due to defrag so we can perform those checks
82 * in the memorystatus thread.
83 * The other is in no_paging_space_action. This is only on macOS right now, but will
84 * be needed on iPad when we run out of swap space. This should be a new kill
85 * reason and we need to add a new health check for it.
86 * We need to maintain the macOS behavior though that we kill no more than 1 process
87 * every 5 seconds.
88 */
89extern _Atomic bool memorystatus_compressor_space_shortage;
90/*
91 * TODO(jason): We should also get rid of this global
92 * and check for phantom cache pressure from the memorystatus
93 * thread. But first we need to fix the syncronization in
94 * vm_phantom_cache_check_pressure
95 */
96extern _Atomic bool memorystatus_phantom_cache_pressure;
97
98extern _Atomic bool memorystatus_pageout_starved;
99/*
100 * The actions that the memorystatus thread can perform
101 * when we're low on memory.
102 * See memorystatus_pick_action to see when each action is deployed.
103 */
104OS_CLOSED_ENUM(memorystatus_action, uint32_t,
105 MEMORYSTATUS_KILL_HIWATER, // Kill 1 highwatermark process
106 MEMORYSTATUS_KILL_AGGRESSIVE, // Do aggressive jetsam
107 MEMORYSTATUS_KILL_TOP_PROCESS, // Kill based on jetsam priority
108 MEMORYSTATUS_WAKE_SWAPPER, // Wake up the swap thread
109 MEMORYSTATUS_PROCESS_SWAPIN_QUEUE, // Compact the swapin queue and move segments to the swapout queue
110 MEMORYSTATUS_KILL_SUSPENDED_SWAPPABLE, // Kill a suspended swap-eligible processes based on jetsam priority
111 MEMORYSTATUS_KILL_SWAPPABLE, // Kill a swap-eligible process (even if it's running) based on jetsam priority
112 MEMORYSTATUS_KILL_NONE, // Do nothing
113 );
114
115/*
116 * Structure to hold state for a jetsam thread.
117 * Typically there should be a single jetsam thread
118 * unless parallel jetsam is enabled.
119 */
120typedef struct jetsam_thread_state {
121 uint8_t inited; /* boolean - if the thread is initialized */
122 uint8_t limit_to_low_bands; /* boolean */
123 int index; /* jetsam thread index */
124 thread_t thread; /* jetsam thread pointer */
125 int jld_idle_kills; /* idle jetsam kill counter for this session */
126 uint32_t errors; /* Error accumulator */
127 bool sort_flag; /* Sort the fg band (idle on macOS) before killing? */
128 bool corpse_list_purged; /* Has the corpse list been purged? */
129 bool post_snapshot; /* Do we need to post a jetsam snapshot after this session? */
130 uint64_t memory_reclaimed; /* Amount of memory that was just reclaimed */
131 uint32_t hwm_kills; /* hwm kill counter for this session */
132 sched_cond_atomic_t jt_wakeup_cond; /* condition var used to synchronize wake/sleep operations for this jetsam thread */
133} jetsam_thread_state_t;
134
135/*
136 * The memorystatus thread monitors these conditions
137 * and will continue to act until the system is considered
138 * healthy.
139 */
140typedef struct memorystatus_system_health {
141#if CONFIG_JETSAM
142 bool msh_available_pages_below_pressure;
143 bool msh_available_pages_below_critical;
144 bool msh_compressor_needs_to_swap;
145 bool msh_compressor_is_low_on_space;
146 bool msh_compressor_is_thrashing;
147 bool msh_compressed_pages_nearing_limit;
148 bool msh_filecache_is_thrashing;
149 bool msh_phantom_cache_pressure;
150 bool msh_swappable_compressor_segments_over_limit;
151 bool msh_swapin_queue_over_limit;
152 bool msh_swap_low_on_space;
153 bool msh_swap_out_of_space;
154 bool msh_pageout_starved;
155#endif /* CONFIG_JETSAM */
156 bool msh_zone_map_is_exhausted;
157} memorystatus_system_health_t;
158
159void memorystatus_log_system_health(const memorystatus_system_health_t *health);
160bool memorystatus_is_system_healthy(const memorystatus_system_health_t *status);
161/* Picks a kill cause given an unhealthy system status */
162uint32_t memorystatus_pick_kill_cause(const memorystatus_system_health_t *status);
163
164/*
165 * Agressive jetsam tunables
166 */
167#define kJetsamAgingPolicyNone (0)
168#define kJetsamAgingPolicyLegacy (1)
169#define kJetsamAgingPolicySysProcsReclaimedFirst (2)
170#define kJetsamAgingPolicyAppsReclaimedFirst (3)
171#define kJetsamAgingPolicyMax kJetsamAgingPolicyAppsReclaimedFirst
172extern boolean_t memorystatus_jld_enabled; /* Enable jetsam loop detection */
173extern uint32_t memorystatus_jld_eval_period_msecs; /* Init pass sets this based on device memory size */
174extern int memorystatus_jld_eval_aggressive_count; /* Raise the priority max after 'n' aggressive loops */
175extern int memorystatus_jld_eval_aggressive_priority_band_max; /* Kill aggressively up through this band */
176extern int memorystatus_jld_max_kill_loops; /* How many times should we try and kill up to the target band */
177extern unsigned int memorystatus_sysproc_aging_aggr_pages; /* Aggressive jetsam pages threshold for sysproc aging policy */
178extern int jld_eval_aggressive_count;
179extern int32_t jld_priority_band_max;
180extern uint64_t jld_timestamp_msecs;
181extern int jld_idle_kill_candidates;
182
183
184/*
185 * VM globals read by the memorystatus subsystem
186 */
187extern unsigned int vm_page_free_count;
188extern unsigned int vm_page_active_count;
189extern unsigned int vm_page_inactive_count;
190extern unsigned int vm_page_throttled_count;
191extern unsigned int vm_page_purgeable_count;
192extern unsigned int vm_page_wire_count;
193extern unsigned int vm_page_speculative_count;
194extern uint32_t c_late_swapout_count, c_late_swappedin_count;
195extern uint32_t c_seg_allocsize;
196extern bool vm_swapout_thread_running;
197extern _Atomic bool vm_swapout_wake_pending;
198#define VM_PAGE_DONATE_DISABLED 0
199#define VM_PAGE_DONATE_ENABLED 1
200extern uint32_t vm_page_donate_mode;
201void vm_swapout_thread(void);
202void vm_compressor_process_special_swapped_in_segments(void);
203
204#if CONFIG_JETSAM
205#define MEMORYSTATUS_LOG_AVAILABLE_PAGES memorystatus_available_pages
206#else /* CONFIG_JETSAM */
207#define MEMORYSTATUS_LOG_AVAILABLE_PAGES (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count)
208#endif /* CONFIG_JETSAM */
209
210bool memorystatus_avail_pages_below_pressure(void);
211bool memorystatus_avail_pages_below_critical(void);
212#if CONFIG_JETSAM
213bool memorystatus_swap_over_trigger(uint64_t adjustment_factor);
214bool memorystatus_swapin_over_trigger(void);
215#endif /* CONFIG_JETSAM */
216
217/* Does cause indicate vm or fc thrashing? */
218bool is_reason_thrashing(unsigned cause);
219/* Is the zone map almost full? */
220bool is_reason_zone_map_exhaustion(unsigned cause);
221
222memorystatus_action_t memorystatus_pick_action(struct jetsam_thread_state *jetsam_thread,
223 uint32_t *kill_cause, bool highwater_remaining,
224 bool suspended_swappable_apps_remaining,
225 bool swappable_apps_remaining, int *jld_idle_kills);
226
227#define MEMSTAT_PERCENT_TOTAL_PAGES(p) (p * atop_64(max_mem) / 100)
228
229#pragma mark Logging Utilities
230
231__enum_decl(memorystatus_log_level_t, unsigned int, {
232 MEMORYSTATUS_LOG_LEVEL_DEFAULT = 0,
233 MEMORYSTATUS_LOG_LEVEL_INFO = 1,
234 MEMORYSTATUS_LOG_LEVEL_DEBUG = 2,
235});
236
237extern os_log_t memorystatus_log_handle;
238extern memorystatus_log_level_t memorystatus_log_level;
239
240/*
241 * NB: Critical memorystatus logs (e.g. jetsam kills) are load-bearing for OS
242 * performance testing infrastructure. Be careful when modifying the log-level for
243 * important system events.
244 *
245 * Memorystatus logs are interpreted by a wide audience. To avoid logging information
246 * that could lead to false diagnoses, INFO and DEBUG messages are only logged if the
247 * system has been configured to do so via `kern.memorystatus_log_level` (sysctl) or
248 * `memorystatus_log_level` (boot-arg).
249 *
250 * os_log supports a mechanism for configuring these properties dynamically; however,
251 * this mechanism is currently unsupported in XNU.
252 *
253 * TODO (JC) Deprecate sysctl/boot-arg and move to subsystem preferences pending:
254 * - rdar://27006343 (Custom kernel log handles)
255 * - rdar://80958044 (Kernel Logging Configuration)
256 */
257#define _memorystatus_log_with_type(type, format, ...) os_log_with_type(memorystatus_log_handle, type, format, ##__VA_ARGS__)
258#define memorystatus_log(format, ...) _memorystatus_log_with_type(OS_LOG_TYPE_DEFAULT, format, ##__VA_ARGS__)
259#define memorystatus_log_info(format, ...) if (memorystatus_log_level >= MEMORYSTATUS_LOG_LEVEL_INFO) { _memorystatus_log_with_type(OS_LOG_TYPE_INFO, format, ##__VA_ARGS__); }
260#define memorystatus_log_debug(format, ...) if (memorystatus_log_level >= MEMORYSTATUS_LOG_LEVEL_DEBUG) { _memorystatus_log_with_type(OS_LOG_TYPE_DEBUG, format, ##__VA_ARGS__); }
261#define memorystatus_log_error(format, ...) _memorystatus_log_with_type(OS_LOG_TYPE_ERROR, format, ##__VA_ARGS__)
262#define memorystatus_log_fault(format, ...) _memorystatus_log_with_type(OS_LOG_TYPE_FAULT, format, ##__VA_ARGS__)
263
264#pragma mark Freezer
265#if CONFIG_FREEZE
266/*
267 * Freezer data types
268 */
269
270/* An ordered list of freeze or demotion candidates */
271struct memorystatus_freezer_candidate_list {
272 memorystatus_properties_freeze_entry_v1 *mfcl_list;
273 size_t mfcl_length;
274};
275
276struct memorystatus_freeze_list_iterator {
277 bool refreeze_only;
278 proc_t last_p;
279 size_t global_freeze_list_index;
280};
281
282/*
283 * Freezer globals
284 */
285extern struct memorystatus_freezer_stats_t memorystatus_freezer_stats;
286extern int memorystatus_freezer_use_ordered_list;
287extern struct memorystatus_freezer_candidate_list memorystatus_global_freeze_list;
288extern struct memorystatus_freezer_candidate_list memorystatus_global_demote_list;
289extern uint64_t memorystatus_freezer_thread_next_run_ts;
290bool memorystatus_is_process_eligible_for_freeze(proc_t p);
291bool memorystatus_freeze_proc_is_refreeze_eligible(proc_t p);
292
293proc_t memorystatus_freezer_candidate_list_get_proc(
294 struct memorystatus_freezer_candidate_list *list,
295 size_t index,
296 uint64_t *pid_mismatch_counter);
297/*
298 * Returns the leader of the p's jetsam coalition
299 * and the role of p in that coalition.
300 */
301proc_t memorystatus_get_coalition_leader_and_role(proc_t p, int *role_in_coalition);
302bool memorystatus_freeze_process_is_recommended(const proc_t p);
303
304/*
305 * Ordered iterator over all freeze candidates.
306 * The iterator should initially be zeroed out by the caller and
307 * can be zeroed out whenever the caller wishes to start from the beginning
308 * of the list again.
309 * Returns PROC_NULL when all candidates have been iterated over.
310 */
311proc_t memorystatus_freeze_pick_process(struct memorystatus_freeze_list_iterator *iterator);
312
313/*
314 * Returns the number of processes that the freezer thread should try to freeze
315 * on this wakeup.
316 */
317size_t memorystatus_pick_freeze_count_for_wakeup(void);
318
319/*
320 * Configure the freezer for app-based swap mode.
321 * Should be called at boot.
322 */
323void memorystatus_freeze_configure_for_swap(void);
324/*
325 * Undo memorystatus_freeze_configure_for_swap
326 */
327void memorystatus_freeze_disable_swap(void);
328#endif /* CONFIG_FREEZE */
329
330#endif /* BSD_KERNEL_PRIVATE */
331
332#endif /* _KERN_MEMORYSTATUS_INTERNAL_H_ */
333