1 | /* |
2 | * Copyright (c) 2006-2019 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | * |
28 | */ |
29 | |
30 | #ifndef _KERN_MEMORYSTATUS_INTERNAL_H_ |
31 | #define _KERN_MEMORYSTATUS_INTERNAL_H_ |
32 | |
33 | /* |
34 | * Contains memorystatus subsystem definitions that are not |
35 | * exported outside of the memorystatus subsystem. |
36 | * |
37 | * For example, all of the mechanisms used by kern_memorystatus_policy.c |
38 | * should be defined in this header. |
39 | */ |
40 | |
41 | #if BSD_KERNEL_PRIVATE |
42 | |
43 | #include <mach/boolean.h> |
44 | #include <stdbool.h> |
45 | #include <os/base.h> |
46 | #include <os/log.h> |
47 | #include <kern/sched_prim.h> |
48 | |
49 | #if CONFIG_FREEZE |
50 | #include <sys/kern_memorystatus_freeze.h> |
51 | #endif /* CONFIG_FREEZE */ |
52 | |
53 | /* |
54 | * memorystatus subsystem globals |
55 | */ |
56 | #if CONFIG_JETSAM |
57 | extern unsigned int memorystatus_available_pages; |
58 | extern unsigned int memorystatus_available_pages_pressure; |
59 | extern unsigned int memorystatus_available_pages_critical; |
60 | extern uint32_t jetsam_kill_on_low_swap; |
61 | #else /* CONFIG_JETSAM */ |
62 | extern uint64_t memorystatus_available_pages; |
63 | extern uint64_t memorystatus_available_pages_pressure; |
64 | extern uint64_t memorystatus_available_pages_critical; |
65 | #endif /* CONFIG_JETSAM */ |
66 | extern int block_corpses; /* counter to block new corpses if jetsam purges them */ |
67 | extern int system_procs_aging_band; |
68 | extern int applications_aging_band; |
69 | /* |
70 | * TODO(jason): This should really be calculated dynamically by the zalloc |
71 | * subsystem before we do a zone map exhaustion kill. But the zone_gc |
72 | * logic is non-trivial, so for now it just sets this global. |
73 | */ |
74 | extern _Atomic bool memorystatus_zone_map_is_exhausted; |
75 | /* |
76 | * TODO(jason): We should get rid of this global |
77 | * and have the memorystatus thread check for compressor space shortages |
78 | * itself. However, there are 3 async call sites remaining that require more work to get us there: |
79 | * 2 of them are in vm_swap_defragment. When it's about to swap in a segment, it checks if that |
80 | * will cause a compressor space shortage & pre-emptively triggers jetsam. vm_compressor_backing_store |
81 | * needs to keep track of in-flight swapins due to defrag so we can perform those checks |
82 | * in the memorystatus thread. |
83 | * The other is in no_paging_space_action. This is only on macOS right now, but will |
84 | * be needed on iPad when we run out of swap space. This should be a new kill |
85 | * reason and we need to add a new health check for it. |
86 | * We need to maintain the macOS behavior though that we kill no more than 1 process |
87 | * every 5 seconds. |
88 | */ |
89 | extern _Atomic bool memorystatus_compressor_space_shortage; |
90 | /* |
91 | * TODO(jason): We should also get rid of this global |
92 | * and check for phantom cache pressure from the memorystatus |
93 | * thread. But first we need to fix the syncronization in |
94 | * vm_phantom_cache_check_pressure |
95 | */ |
96 | extern _Atomic bool memorystatus_phantom_cache_pressure; |
97 | |
98 | extern _Atomic bool memorystatus_pageout_starved; |
99 | /* |
100 | * The actions that the memorystatus thread can perform |
101 | * when we're low on memory. |
102 | * See memorystatus_pick_action to see when each action is deployed. |
103 | */ |
104 | OS_CLOSED_ENUM(memorystatus_action, uint32_t, |
105 | MEMORYSTATUS_KILL_HIWATER, // Kill 1 highwatermark process |
106 | MEMORYSTATUS_KILL_AGGRESSIVE, // Do aggressive jetsam |
107 | MEMORYSTATUS_KILL_TOP_PROCESS, // Kill based on jetsam priority |
108 | MEMORYSTATUS_WAKE_SWAPPER, // Wake up the swap thread |
109 | MEMORYSTATUS_PROCESS_SWAPIN_QUEUE, // Compact the swapin queue and move segments to the swapout queue |
110 | MEMORYSTATUS_KILL_SUSPENDED_SWAPPABLE, // Kill a suspended swap-eligible processes based on jetsam priority |
111 | MEMORYSTATUS_KILL_SWAPPABLE, // Kill a swap-eligible process (even if it's running) based on jetsam priority |
112 | MEMORYSTATUS_KILL_NONE, // Do nothing |
113 | ); |
114 | |
115 | /* |
116 | * Structure to hold state for a jetsam thread. |
117 | * Typically there should be a single jetsam thread |
118 | * unless parallel jetsam is enabled. |
119 | */ |
120 | typedef struct jetsam_thread_state { |
121 | uint8_t inited; /* boolean - if the thread is initialized */ |
122 | uint8_t limit_to_low_bands; /* boolean */ |
123 | int index; /* jetsam thread index */ |
124 | thread_t thread; /* jetsam thread pointer */ |
125 | int jld_idle_kills; /* idle jetsam kill counter for this session */ |
126 | uint32_t errors; /* Error accumulator */ |
127 | bool sort_flag; /* Sort the fg band (idle on macOS) before killing? */ |
128 | bool corpse_list_purged; /* Has the corpse list been purged? */ |
129 | bool post_snapshot; /* Do we need to post a jetsam snapshot after this session? */ |
130 | uint64_t memory_reclaimed; /* Amount of memory that was just reclaimed */ |
131 | uint32_t hwm_kills; /* hwm kill counter for this session */ |
132 | sched_cond_atomic_t jt_wakeup_cond; /* condition var used to synchronize wake/sleep operations for this jetsam thread */ |
133 | } jetsam_thread_state_t; |
134 | |
135 | /* |
136 | * The memorystatus thread monitors these conditions |
137 | * and will continue to act until the system is considered |
138 | * healthy. |
139 | */ |
140 | typedef struct memorystatus_system_health { |
141 | #if CONFIG_JETSAM |
142 | bool msh_available_pages_below_pressure; |
143 | bool msh_available_pages_below_critical; |
144 | bool msh_compressor_needs_to_swap; |
145 | bool msh_compressor_is_low_on_space; |
146 | bool msh_compressor_is_thrashing; |
147 | bool msh_compressed_pages_nearing_limit; |
148 | bool msh_filecache_is_thrashing; |
149 | bool msh_phantom_cache_pressure; |
150 | bool msh_swappable_compressor_segments_over_limit; |
151 | bool msh_swapin_queue_over_limit; |
152 | bool msh_swap_low_on_space; |
153 | bool msh_swap_out_of_space; |
154 | bool msh_pageout_starved; |
155 | #endif /* CONFIG_JETSAM */ |
156 | bool msh_zone_map_is_exhausted; |
157 | } memorystatus_system_health_t; |
158 | |
159 | void memorystatus_log_system_health(const memorystatus_system_health_t *health); |
160 | bool memorystatus_is_system_healthy(const memorystatus_system_health_t *status); |
161 | /* Picks a kill cause given an unhealthy system status */ |
162 | uint32_t memorystatus_pick_kill_cause(const memorystatus_system_health_t *status); |
163 | |
164 | /* |
165 | * Agressive jetsam tunables |
166 | */ |
167 | #define kJetsamAgingPolicyNone (0) |
168 | #define kJetsamAgingPolicyLegacy (1) |
169 | #define kJetsamAgingPolicySysProcsReclaimedFirst (2) |
170 | #define kJetsamAgingPolicyAppsReclaimedFirst (3) |
171 | #define kJetsamAgingPolicyMax kJetsamAgingPolicyAppsReclaimedFirst |
172 | extern boolean_t memorystatus_jld_enabled; /* Enable jetsam loop detection */ |
173 | extern uint32_t memorystatus_jld_eval_period_msecs; /* Init pass sets this based on device memory size */ |
174 | extern int memorystatus_jld_eval_aggressive_count; /* Raise the priority max after 'n' aggressive loops */ |
175 | extern int memorystatus_jld_eval_aggressive_priority_band_max; /* Kill aggressively up through this band */ |
176 | extern int memorystatus_jld_max_kill_loops; /* How many times should we try and kill up to the target band */ |
177 | extern unsigned int memorystatus_sysproc_aging_aggr_pages; /* Aggressive jetsam pages threshold for sysproc aging policy */ |
178 | extern int jld_eval_aggressive_count; |
179 | extern int32_t jld_priority_band_max; |
180 | extern uint64_t jld_timestamp_msecs; |
181 | extern int jld_idle_kill_candidates; |
182 | |
183 | |
184 | /* |
185 | * VM globals read by the memorystatus subsystem |
186 | */ |
187 | extern unsigned int vm_page_free_count; |
188 | extern unsigned int vm_page_active_count; |
189 | extern unsigned int vm_page_inactive_count; |
190 | extern unsigned int vm_page_throttled_count; |
191 | extern unsigned int vm_page_purgeable_count; |
192 | extern unsigned int vm_page_wire_count; |
193 | extern unsigned int vm_page_speculative_count; |
194 | extern uint32_t c_late_swapout_count, c_late_swappedin_count; |
195 | extern uint32_t c_seg_allocsize; |
196 | extern bool vm_swapout_thread_running; |
197 | extern _Atomic bool vm_swapout_wake_pending; |
198 | #define VM_PAGE_DONATE_DISABLED 0 |
199 | #define VM_PAGE_DONATE_ENABLED 1 |
200 | extern uint32_t vm_page_donate_mode; |
201 | void vm_swapout_thread(void); |
202 | void vm_compressor_process_special_swapped_in_segments(void); |
203 | |
204 | #if CONFIG_JETSAM |
205 | #define MEMORYSTATUS_LOG_AVAILABLE_PAGES memorystatus_available_pages |
206 | #else /* CONFIG_JETSAM */ |
207 | #define MEMORYSTATUS_LOG_AVAILABLE_PAGES (vm_page_active_count + vm_page_inactive_count + vm_page_free_count + vm_page_speculative_count) |
208 | #endif /* CONFIG_JETSAM */ |
209 | |
210 | bool memorystatus_avail_pages_below_pressure(void); |
211 | bool memorystatus_avail_pages_below_critical(void); |
212 | #if CONFIG_JETSAM |
213 | bool memorystatus_swap_over_trigger(uint64_t adjustment_factor); |
214 | bool memorystatus_swapin_over_trigger(void); |
215 | #endif /* CONFIG_JETSAM */ |
216 | |
217 | /* Does cause indicate vm or fc thrashing? */ |
218 | bool is_reason_thrashing(unsigned cause); |
219 | /* Is the zone map almost full? */ |
220 | bool is_reason_zone_map_exhaustion(unsigned cause); |
221 | |
222 | memorystatus_action_t memorystatus_pick_action(struct jetsam_thread_state *jetsam_thread, |
223 | uint32_t *kill_cause, bool highwater_remaining, |
224 | bool suspended_swappable_apps_remaining, |
225 | bool swappable_apps_remaining, int *jld_idle_kills); |
226 | |
227 | #define MEMSTAT_PERCENT_TOTAL_PAGES(p) (p * atop_64(max_mem) / 100) |
228 | |
229 | #pragma mark Logging Utilities |
230 | |
231 | __enum_decl(memorystatus_log_level_t, unsigned int, { |
232 | MEMORYSTATUS_LOG_LEVEL_DEFAULT = 0, |
233 | MEMORYSTATUS_LOG_LEVEL_INFO = 1, |
234 | MEMORYSTATUS_LOG_LEVEL_DEBUG = 2, |
235 | }); |
236 | |
237 | extern os_log_t memorystatus_log_handle; |
238 | extern memorystatus_log_level_t memorystatus_log_level; |
239 | |
240 | /* |
241 | * NB: Critical memorystatus logs (e.g. jetsam kills) are load-bearing for OS |
242 | * performance testing infrastructure. Be careful when modifying the log-level for |
243 | * important system events. |
244 | * |
245 | * Memorystatus logs are interpreted by a wide audience. To avoid logging information |
246 | * that could lead to false diagnoses, INFO and DEBUG messages are only logged if the |
247 | * system has been configured to do so via `kern.memorystatus_log_level` (sysctl) or |
248 | * `memorystatus_log_level` (boot-arg). |
249 | * |
250 | * os_log supports a mechanism for configuring these properties dynamically; however, |
251 | * this mechanism is currently unsupported in XNU. |
252 | * |
253 | * TODO (JC) Deprecate sysctl/boot-arg and move to subsystem preferences pending: |
254 | * - rdar://27006343 (Custom kernel log handles) |
255 | * - rdar://80958044 (Kernel Logging Configuration) |
256 | */ |
257 | #define _memorystatus_log_with_type(type, format, ...) os_log_with_type(memorystatus_log_handle, type, format, ##__VA_ARGS__) |
258 | #define memorystatus_log(format, ...) _memorystatus_log_with_type(OS_LOG_TYPE_DEFAULT, format, ##__VA_ARGS__) |
259 | #define memorystatus_log_info(format, ...) if (memorystatus_log_level >= MEMORYSTATUS_LOG_LEVEL_INFO) { _memorystatus_log_with_type(OS_LOG_TYPE_INFO, format, ##__VA_ARGS__); } |
260 | #define memorystatus_log_debug(format, ...) if (memorystatus_log_level >= MEMORYSTATUS_LOG_LEVEL_DEBUG) { _memorystatus_log_with_type(OS_LOG_TYPE_DEBUG, format, ##__VA_ARGS__); } |
261 | #define memorystatus_log_error(format, ...) _memorystatus_log_with_type(OS_LOG_TYPE_ERROR, format, ##__VA_ARGS__) |
262 | #define memorystatus_log_fault(format, ...) _memorystatus_log_with_type(OS_LOG_TYPE_FAULT, format, ##__VA_ARGS__) |
263 | |
264 | #pragma mark Freezer |
265 | #if CONFIG_FREEZE |
266 | /* |
267 | * Freezer data types |
268 | */ |
269 | |
270 | /* An ordered list of freeze or demotion candidates */ |
271 | struct memorystatus_freezer_candidate_list { |
272 | memorystatus_properties_freeze_entry_v1 *mfcl_list; |
273 | size_t mfcl_length; |
274 | }; |
275 | |
276 | struct memorystatus_freeze_list_iterator { |
277 | bool refreeze_only; |
278 | proc_t last_p; |
279 | size_t global_freeze_list_index; |
280 | }; |
281 | |
282 | /* |
283 | * Freezer globals |
284 | */ |
285 | extern struct memorystatus_freezer_stats_t memorystatus_freezer_stats; |
286 | extern int memorystatus_freezer_use_ordered_list; |
287 | extern struct memorystatus_freezer_candidate_list memorystatus_global_freeze_list; |
288 | extern struct memorystatus_freezer_candidate_list memorystatus_global_demote_list; |
289 | extern uint64_t memorystatus_freezer_thread_next_run_ts; |
290 | bool memorystatus_is_process_eligible_for_freeze(proc_t p); |
291 | bool memorystatus_freeze_proc_is_refreeze_eligible(proc_t p); |
292 | |
293 | proc_t memorystatus_freezer_candidate_list_get_proc( |
294 | struct memorystatus_freezer_candidate_list *list, |
295 | size_t index, |
296 | uint64_t *pid_mismatch_counter); |
297 | /* |
298 | * Returns the leader of the p's jetsam coalition |
299 | * and the role of p in that coalition. |
300 | */ |
301 | proc_t memorystatus_get_coalition_leader_and_role(proc_t p, int *role_in_coalition); |
302 | bool memorystatus_freeze_process_is_recommended(const proc_t p); |
303 | |
304 | /* |
305 | * Ordered iterator over all freeze candidates. |
306 | * The iterator should initially be zeroed out by the caller and |
307 | * can be zeroed out whenever the caller wishes to start from the beginning |
308 | * of the list again. |
309 | * Returns PROC_NULL when all candidates have been iterated over. |
310 | */ |
311 | proc_t memorystatus_freeze_pick_process(struct memorystatus_freeze_list_iterator *iterator); |
312 | |
313 | /* |
314 | * Returns the number of processes that the freezer thread should try to freeze |
315 | * on this wakeup. |
316 | */ |
317 | size_t memorystatus_pick_freeze_count_for_wakeup(void); |
318 | |
319 | /* |
320 | * Configure the freezer for app-based swap mode. |
321 | * Should be called at boot. |
322 | */ |
323 | void memorystatus_freeze_configure_for_swap(void); |
324 | /* |
325 | * Undo memorystatus_freeze_configure_for_swap |
326 | */ |
327 | void memorystatus_freeze_disable_swap(void); |
328 | #endif /* CONFIG_FREEZE */ |
329 | |
330 | #endif /* BSD_KERNEL_PRIVATE */ |
331 | |
332 | #endif /* _KERN_MEMORYSTATUS_INTERNAL_H_ */ |
333 | |