1 | /* |
2 | * Copyright (c) 2019-2020 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <kern/kern_types.h> |
30 | #include <mach/mach_types.h> |
31 | #include <mach/boolean.h> |
32 | |
33 | #include <kern/coalition.h> |
34 | #include <kern/exc_resource.h> |
35 | #include <kern/host.h> |
36 | #include <kern/ledger.h> |
37 | #include <kern/mach_param.h> /* for TASK_CHUNK */ |
38 | #include <kern/monotonic.h> |
39 | #include <kern/policy_internal.h> |
40 | #include <kern/task.h> |
41 | #include <kern/smr_hash.h> |
42 | #include <kern/thread_group.h> |
43 | #include <kern/zalloc.h> |
44 | #include <vm/vm_pageout.h> |
45 | |
46 | #include <libkern/OSAtomic.h> |
47 | |
48 | #include <mach/coalition_notification_server.h> |
49 | #include <mach/host_priv.h> |
50 | #include <mach/host_special_ports.h> |
51 | |
52 | #include <os/log.h> |
53 | |
54 | #include <sys/errno.h> |
55 | |
56 | /* |
57 | * BSD interface functions |
58 | */ |
59 | size_t coalitions_get_list(int type, struct procinfo_coalinfo *coal_list, size_t list_sz); |
60 | coalition_t task_get_coalition(task_t task, int type); |
61 | boolean_t coalition_is_leader(task_t task, coalition_t coal); |
62 | task_t coalition_get_leader(coalition_t coal); |
63 | int coalition_get_task_count(coalition_t coal); |
64 | uint64_t coalition_get_page_count(coalition_t coal, int *ntasks); |
65 | int coalition_get_pid_list(coalition_t coal, uint32_t rolemask, int sort_order, |
66 | int *pid_list, int list_sz); |
67 | |
68 | /* defined in task.c */ |
69 | extern ledger_template_t task_ledger_template; |
70 | |
71 | /* |
72 | * Templates; task template is copied due to potential allocation limits on |
73 | * task ledgers. |
74 | */ |
75 | ledger_template_t coalition_task_ledger_template = NULL; |
76 | ledger_template_t coalition_ledger_template = NULL; |
77 | |
78 | extern int proc_selfpid(void); |
79 | /* |
80 | * Coalition zone needs limits. We expect there will be as many coalitions as |
81 | * tasks (same order of magnitude), so use the task zone's limits. |
82 | * */ |
83 | #define CONFIG_COALITION_MAX CONFIG_TASK_MAX |
84 | #define COALITION_CHUNK TASK_CHUNK |
85 | |
86 | #if DEBUG || DEVELOPMENT |
87 | TUNABLE_WRITEABLE(int, unrestrict_coalition_syscalls, "unrestrict_coalition_syscalls" , 0); |
88 | #else |
89 | #define unrestrict_coalition_syscalls false |
90 | #endif |
91 | |
92 | static LCK_GRP_DECLARE(coalitions_lck_grp, "coalition" ); |
93 | |
94 | /* coalitions_list_lock protects coalition_hash, next_coalition_id. */ |
95 | static LCK_MTX_DECLARE(coalitions_list_lock, &coalitions_lck_grp); |
96 | static uint64_t coalition_next_id = 1; |
97 | static struct smr_hash coalition_hash; |
98 | |
99 | coalition_t init_coalition[COALITION_NUM_TYPES]; |
100 | coalition_t corpse_coalition[COALITION_NUM_TYPES]; |
101 | |
102 | static const char * |
103 | coal_type_str(int type) |
104 | { |
105 | switch (type) { |
106 | case COALITION_TYPE_RESOURCE: |
107 | return "RESOURCE" ; |
108 | case COALITION_TYPE_JETSAM: |
109 | return "JETSAM" ; |
110 | default: |
111 | return "<unknown>" ; |
112 | } |
113 | } |
114 | |
115 | struct coalition_type { |
116 | int type; |
117 | int has_default; |
118 | /* |
119 | * init |
120 | * pre-condition: coalition just allocated (unlocked), unreferenced, |
121 | * type field set |
122 | */ |
123 | kern_return_t (*init)(coalition_t coal, boolean_t privileged, boolean_t efficient); |
124 | |
125 | /* |
126 | * dealloc |
127 | * pre-condition: coalition unlocked |
128 | * pre-condition: coalition refcount=0, active_count=0, |
129 | * termrequested=1, terminated=1, reaped=1 |
130 | */ |
131 | void (*dealloc)(coalition_t coal); |
132 | |
133 | /* |
134 | * adopt_task |
135 | * pre-condition: coalition locked |
136 | * pre-condition: coalition !repead and !terminated |
137 | */ |
138 | kern_return_t (*adopt_task)(coalition_t coal, task_t task); |
139 | |
140 | /* |
141 | * remove_task |
142 | * pre-condition: coalition locked |
143 | * pre-condition: task has been removed from coalition's task list |
144 | */ |
145 | kern_return_t (*remove_task)(coalition_t coal, task_t task); |
146 | |
147 | /* |
148 | * set_taskrole |
149 | * pre-condition: coalition locked |
150 | * pre-condition: task added to coalition's task list, |
151 | * active_count >= 1 (at least the given task is active) |
152 | */ |
153 | kern_return_t (*set_taskrole)(coalition_t coal, task_t task, int role); |
154 | |
155 | /* |
156 | * get_taskrole |
157 | * pre-condition: coalition locked |
158 | * pre-condition: task added to coalition's task list, |
159 | * active_count >= 1 (at least the given task is active) |
160 | */ |
161 | int (*get_taskrole)(coalition_t coal, task_t task); |
162 | |
163 | /* |
164 | * iterate_tasks |
165 | * pre-condition: coalition locked |
166 | */ |
167 | void (*iterate_tasks)(coalition_t coal, void *ctx, void (*callback)(coalition_t, void *, task_t)); |
168 | }; |
169 | |
170 | /* |
171 | * COALITION_TYPE_RESOURCE |
172 | */ |
173 | |
174 | static kern_return_t i_coal_resource_init(coalition_t coal, boolean_t privileged, boolean_t efficient); |
175 | static void i_coal_resource_dealloc(coalition_t coal); |
176 | static kern_return_t i_coal_resource_adopt_task(coalition_t coal, task_t task); |
177 | static kern_return_t i_coal_resource_remove_task(coalition_t coal, task_t task); |
178 | static kern_return_t i_coal_resource_set_taskrole(coalition_t coal, |
179 | task_t task, int role); |
180 | static int i_coal_resource_get_taskrole(coalition_t coal, task_t task); |
181 | static void i_coal_resource_iterate_tasks(coalition_t coal, void *ctx, |
182 | void (*callback)(coalition_t, void *, task_t)); |
183 | |
184 | /* |
185 | * Ensure COALITION_NUM_THREAD_QOS_TYPES defined in mach/coalition.h still |
186 | * matches THREAD_QOS_LAST defined in mach/thread_policy.h |
187 | */ |
188 | static_assert(COALITION_NUM_THREAD_QOS_TYPES == THREAD_QOS_LAST); |
189 | |
190 | struct i_resource_coalition { |
191 | /* |
192 | * This keeps track of resource utilization of tasks that are no longer active |
193 | * in the coalition and is updated when a task is removed from the coalition. |
194 | */ |
195 | ledger_t ledger; |
196 | uint64_t bytesread; |
197 | uint64_t byteswritten; |
198 | uint64_t energy; |
199 | uint64_t gpu_time; |
200 | uint64_t logical_immediate_writes; |
201 | uint64_t logical_deferred_writes; |
202 | uint64_t logical_invalidated_writes; |
203 | uint64_t logical_metadata_writes; |
204 | uint64_t logical_immediate_writes_to_external; |
205 | uint64_t logical_deferred_writes_to_external; |
206 | uint64_t logical_invalidated_writes_to_external; |
207 | uint64_t logical_metadata_writes_to_external; |
208 | uint64_t cpu_time_eqos[COALITION_NUM_THREAD_QOS_TYPES]; /* cpu time per effective QoS class */ |
209 | uint64_t cpu_time_rqos[COALITION_NUM_THREAD_QOS_TYPES]; /* cpu time per requested QoS class */ |
210 | uint64_t cpu_instructions; |
211 | uint64_t cpu_cycles; |
212 | struct recount_coalition co_recount; |
213 | |
214 | uint64_t task_count; /* tasks that have started in this coalition */ |
215 | uint64_t dead_task_count; /* tasks that have exited in this coalition; |
216 | * subtract from task_count to get count |
217 | * of "active" tasks */ |
218 | /* |
219 | * Count the length of time this coalition had at least one active task. |
220 | * This can be a 'denominator' to turn e.g. cpu_time to %cpu. |
221 | * */ |
222 | uint64_t last_became_nonempty_time; |
223 | uint64_t time_nonempty; |
224 | |
225 | queue_head_t tasks; /* List of active tasks in the coalition */ |
226 | /* |
227 | * This ledger is used for triggering resource exception. For the tracked resources, this is updated |
228 | * when the member tasks' resource usage changes. |
229 | */ |
230 | ledger_t resource_monitor_ledger; |
231 | #if CONFIG_PHYS_WRITE_ACCT |
232 | uint64_t fs_metadata_writes; |
233 | #endif /* CONFIG_PHYS_WRITE_ACCT */ |
234 | }; |
235 | |
236 | /* |
237 | * COALITION_TYPE_JETSAM |
238 | */ |
239 | |
240 | static kern_return_t i_coal_jetsam_init(coalition_t coal, boolean_t privileged, boolean_t efficient); |
241 | static void i_coal_jetsam_dealloc(coalition_t coal); |
242 | static kern_return_t i_coal_jetsam_adopt_task(coalition_t coal, task_t task); |
243 | static kern_return_t i_coal_jetsam_remove_task(coalition_t coal, task_t task); |
244 | static kern_return_t i_coal_jetsam_set_taskrole(coalition_t coal, |
245 | task_t task, int role); |
246 | int i_coal_jetsam_get_taskrole(coalition_t coal, task_t task); |
247 | static void i_coal_jetsam_iterate_tasks(coalition_t coal, void *ctx, |
248 | void (*callback)(coalition_t, void *, task_t)); |
249 | |
250 | struct i_jetsam_coalition { |
251 | task_t leader; |
252 | queue_head_t extensions; |
253 | queue_head_t services; |
254 | queue_head_t other; |
255 | struct thread_group *thread_group; |
256 | bool swap_enabled; |
257 | }; |
258 | |
259 | |
260 | /* |
261 | * main coalition structure |
262 | */ |
263 | struct coalition { |
264 | uint64_t id; /* monotonically increasing */ |
265 | uint32_t type; |
266 | uint32_t role; /* default task role (background, adaptive, interactive, etc) */ |
267 | os_ref_atomic_t ref_count; /* Number of references to the memory containing this struct */ |
268 | uint32_t active_count; /* Number of members of (tasks in) the |
269 | * coalition, plus vouchers referring |
270 | * to the coalition */ |
271 | uint32_t focal_task_count; /* Number of TASK_FOREGROUND_APPLICATION tasks in the coalition */ |
272 | uint32_t nonfocal_task_count; /* Number of TASK_BACKGROUND_APPLICATION tasks in the coalition */ |
273 | uint32_t game_task_count; /* Number of GAME_MODE tasks in the coalition */ |
274 | |
275 | /* coalition flags */ |
276 | uint32_t privileged : 1; /* Members of this coalition may create |
277 | * and manage coalitions and may posix_spawn |
278 | * processes into selected coalitions */ |
279 | /* ast? */ |
280 | /* voucher */ |
281 | uint32_t termrequested : 1; /* launchd has requested termination when coalition becomes empty */ |
282 | uint32_t terminated : 1; /* coalition became empty and spawns are now forbidden */ |
283 | uint32_t reaped : 1; /* reaped, invisible to userspace, but waiting for ref_count to go to zero */ |
284 | uint32_t notified : 1; /* no-more-processes notification was sent via special port */ |
285 | uint32_t efficient : 1; /* launchd has marked the coalition as efficient */ |
286 | #if DEVELOPMENT || DEBUG |
287 | uint32_t should_notify : 1; /* should this coalition send notifications (default: yes) */ |
288 | #endif |
289 | |
290 | struct smrq_slink link; /* global list of coalitions */ |
291 | |
292 | union { |
293 | lck_mtx_t lock; /* Coalition lock. */ |
294 | struct smr_node smr_node; |
295 | }; |
296 | |
297 | /* put coalition type-specific structures here */ |
298 | union { |
299 | struct i_resource_coalition r; |
300 | struct i_jetsam_coalition j; |
301 | }; |
302 | }; |
303 | |
304 | os_refgrp_decl(static, coal_ref_grp, "coalitions" , NULL); |
305 | #define coal_ref_init(coal, c) os_ref_init_count_raw(&(coal)->ref_count, &coal_ref_grp, c) |
306 | #define coal_ref_count(coal) os_ref_get_count_raw(&(coal)->ref_count) |
307 | #define coal_ref_try_retain(coal) os_ref_retain_try_raw(&(coal)->ref_count, &coal_ref_grp) |
308 | #define coal_ref_retain(coal) os_ref_retain_raw(&(coal)->ref_count, &coal_ref_grp) |
309 | #define coal_ref_release(coal) os_ref_release_raw(&(coal)->ref_count, &coal_ref_grp) |
310 | #define coal_ref_release_live(coal) os_ref_release_live_raw(&(coal)->ref_count, &coal_ref_grp) |
311 | |
312 | #define COALITION_HASH_SIZE_MIN 16 |
313 | |
314 | static bool |
315 | coal_hash_obj_try_get(void *coal) |
316 | { |
317 | return coal_ref_try_retain((struct coalition *)coal); |
318 | } |
319 | |
320 | SMRH_TRAITS_DEFINE_SCALAR(coal_hash_traits, struct coalition, id, link, |
321 | .domain = &smr_proc_task, |
322 | .obj_try_get = coal_hash_obj_try_get); |
323 | |
324 | /* |
325 | * register different coalition types: |
326 | * these must be kept in the order specified in coalition.h |
327 | */ |
328 | static const struct coalition_type s_coalition_types[COALITION_NUM_TYPES] = { |
329 | { |
330 | COALITION_TYPE_RESOURCE, |
331 | .has_default: 1, |
332 | .init: i_coal_resource_init, |
333 | .dealloc: i_coal_resource_dealloc, |
334 | .adopt_task: i_coal_resource_adopt_task, |
335 | .remove_task: i_coal_resource_remove_task, |
336 | .set_taskrole: i_coal_resource_set_taskrole, |
337 | .get_taskrole: i_coal_resource_get_taskrole, |
338 | .iterate_tasks: i_coal_resource_iterate_tasks, |
339 | }, { |
340 | COALITION_TYPE_JETSAM, |
341 | .has_default: 1, |
342 | .init: i_coal_jetsam_init, |
343 | .dealloc: i_coal_jetsam_dealloc, |
344 | .adopt_task: i_coal_jetsam_adopt_task, |
345 | .remove_task: i_coal_jetsam_remove_task, |
346 | .set_taskrole: i_coal_jetsam_set_taskrole, |
347 | .get_taskrole: i_coal_jetsam_get_taskrole, |
348 | .iterate_tasks: i_coal_jetsam_iterate_tasks, |
349 | }, |
350 | }; |
351 | |
352 | static KALLOC_TYPE_DEFINE(coalition_zone, struct coalition, KT_PRIV_ACCT); |
353 | |
354 | #define coal_call(coal, func, ...) \ |
355 | (s_coalition_types[(coal)->type].func)(coal, ## __VA_ARGS__) |
356 | |
357 | |
358 | #define coalition_lock(c) lck_mtx_lock(&(c)->lock) |
359 | #define coalition_unlock(c) lck_mtx_unlock(&(c)->lock) |
360 | |
361 | /* |
362 | * Define the coalition type to track focal tasks. |
363 | * On embedded, track them using jetsam coalitions since they have associated thread |
364 | * groups which reflect this property as a flag (and pass it down to CLPC). |
365 | * On non-embedded platforms, since not all coalitions have jetsam coalitions |
366 | * track focal counts on the resource coalition. |
367 | */ |
368 | #if !XNU_TARGET_OS_OSX |
369 | #define COALITION_FOCAL_TASKS_ACCOUNTING COALITION_TYPE_JETSAM |
370 | #else /* !XNU_TARGET_OS_OSX */ |
371 | #define COALITION_FOCAL_TASKS_ACCOUNTING COALITION_TYPE_RESOURCE |
372 | #endif /* !XNU_TARGET_OS_OSX */ |
373 | |
374 | |
375 | /* |
376 | * |
377 | * Coalition ledger implementation |
378 | * |
379 | */ |
380 | |
381 | struct coalition_ledger_indices coalition_ledgers = |
382 | {.logical_writes = -1, }; |
383 | void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(int flavor); |
384 | |
385 | ledger_t |
386 | coalition_ledger_get_from_task(task_t task) |
387 | { |
388 | ledger_t ledger = LEDGER_NULL; |
389 | coalition_t coal = task->coalition[COALITION_TYPE_RESOURCE]; |
390 | |
391 | if (coal != NULL && (!queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE]))) { |
392 | ledger = coal->r.resource_monitor_ledger; |
393 | ledger_reference(ledger); |
394 | } |
395 | return ledger; |
396 | } |
397 | |
398 | |
399 | enum { |
400 | COALITION_IO_LEDGER_ENABLE, |
401 | COALITION_IO_LEDGER_DISABLE |
402 | }; |
403 | |
404 | void |
405 | coalition_io_monitor_ctl(struct coalition *coalition, uint32_t flags, int64_t limit) |
406 | { |
407 | ledger_t ledger = coalition->r.resource_monitor_ledger; |
408 | |
409 | if (flags == COALITION_IO_LEDGER_ENABLE) { |
410 | /* Configure the logical I/O ledger */ |
411 | ledger_set_limit(ledger, entry: coalition_ledgers.logical_writes, limit: (limit * 1024 * 1024), warn_level_percentage: 0); |
412 | ledger_set_period(ledger, entry: coalition_ledgers.logical_writes, period: (COALITION_LEDGER_MONITOR_INTERVAL_SECS * NSEC_PER_SEC)); |
413 | } else if (flags == COALITION_IO_LEDGER_DISABLE) { |
414 | ledger_disable_refill(l: ledger, entry: coalition_ledgers.logical_writes); |
415 | ledger_disable_callback(ledger, entry: coalition_ledgers.logical_writes); |
416 | } |
417 | } |
418 | |
419 | int |
420 | coalition_ledger_set_logical_writes_limit(struct coalition *coalition, int64_t limit) |
421 | { |
422 | int error = 0; |
423 | |
424 | /* limit = -1 will be used to disable the limit and the callback */ |
425 | if (limit > COALITION_MAX_LOGICAL_WRITES_LIMIT || limit == 0 || limit < -1) { |
426 | error = EINVAL; |
427 | goto out; |
428 | } |
429 | |
430 | coalition_lock(coalition); |
431 | if (limit == -1) { |
432 | coalition_io_monitor_ctl(coalition, flags: COALITION_IO_LEDGER_DISABLE, limit); |
433 | } else { |
434 | coalition_io_monitor_ctl(coalition, flags: COALITION_IO_LEDGER_ENABLE, limit); |
435 | } |
436 | coalition_unlock(coalition); |
437 | out: |
438 | return error; |
439 | } |
440 | |
441 | void __attribute__((noinline)) |
442 | SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(int flavor) |
443 | { |
444 | int pid = proc_selfpid(); |
445 | ledger_amount_t new_limit; |
446 | task_t task = current_task(); |
447 | struct ledger_entry_info lei; |
448 | kern_return_t kr; |
449 | ledger_t ledger; |
450 | struct coalition *coalition = task->coalition[COALITION_TYPE_RESOURCE]; |
451 | |
452 | assert(coalition != NULL); |
453 | ledger = coalition->r.resource_monitor_ledger; |
454 | |
455 | switch (flavor) { |
456 | case FLAVOR_IO_LOGICAL_WRITES: |
457 | ledger_get_entry_info(ledger, entry: coalition_ledgers.logical_writes, lei: &lei); |
458 | trace_resource_violation(RMON_LOGWRITES_VIOLATED, ledger_info: &lei); |
459 | break; |
460 | default: |
461 | goto Exit; |
462 | } |
463 | |
464 | os_log(OS_LOG_DEFAULT, "Coalition [%lld] caught causing excessive I/O (flavor: %d). Task I/O: %lld MB. [Limit : %lld MB per %lld secs]. Triggered by process [%d]\n" , |
465 | coalition->id, flavor, (lei.lei_balance / (1024 * 1024)), (lei.lei_limit / (1024 * 1024)), |
466 | (lei.lei_refill_period / NSEC_PER_SEC), pid); |
467 | |
468 | kr = send_resource_violation(send_disk_writes_violation, violator: task, ledger_info: &lei, kRNFlagsNone); |
469 | if (kr) { |
470 | os_log(OS_LOG_DEFAULT, "ERROR %#x returned from send_resource_violation(disk_writes, ...)\n" , kr); |
471 | } |
472 | |
473 | /* |
474 | * Continue to monitor the coalition after it hits the initital limit, but increase |
475 | * the limit exponentially so that we don't spam the listener. |
476 | */ |
477 | new_limit = (lei.lei_limit / 1024 / 1024) * 4; |
478 | coalition_lock(coalition); |
479 | if (new_limit > COALITION_MAX_LOGICAL_WRITES_LIMIT) { |
480 | coalition_io_monitor_ctl(coalition, flags: COALITION_IO_LEDGER_DISABLE, limit: -1); |
481 | } else { |
482 | coalition_io_monitor_ctl(coalition, flags: COALITION_IO_LEDGER_ENABLE, limit: new_limit); |
483 | } |
484 | coalition_unlock(coalition); |
485 | |
486 | Exit: |
487 | return; |
488 | } |
489 | |
490 | void |
491 | coalition_io_rate_exceeded(int warning, const void *param0, __unused const void *param1) |
492 | { |
493 | if (warning == 0) { |
494 | SENDING_NOTIFICATION__THIS_COALITION_IS_CAUSING_TOO_MUCH_IO(flavor: (int)param0); |
495 | } |
496 | } |
497 | |
498 | void |
499 | init_coalition_ledgers(void) |
500 | { |
501 | ledger_template_t t; |
502 | assert(coalition_ledger_template == NULL); |
503 | |
504 | if ((t = ledger_template_create(name: "Per-coalition ledgers" )) == NULL) { |
505 | panic("couldn't create coalition ledger template" ); |
506 | } |
507 | |
508 | coalition_ledgers.logical_writes = ledger_entry_add(template: t, key: "logical_writes" , group: "res" , units: "bytes" ); |
509 | |
510 | if (coalition_ledgers.logical_writes < 0) { |
511 | panic("couldn't create entries for coaliton ledger template" ); |
512 | } |
513 | |
514 | ledger_set_callback(template: t, entry: coalition_ledgers.logical_writes, callback: coalition_io_rate_exceeded, param0: (void *)FLAVOR_IO_LOGICAL_WRITES, NULL); |
515 | ledger_template_complete(template: t); |
516 | |
517 | coalition_task_ledger_template = ledger_template_copy(template: task_ledger_template, name: "Coalition task ledgers" ); |
518 | |
519 | if (coalition_task_ledger_template == NULL) { |
520 | panic("couldn't create coalition task ledger template" ); |
521 | } |
522 | |
523 | ledger_template_complete(template: coalition_task_ledger_template); |
524 | |
525 | coalition_ledger_template = t; |
526 | } |
527 | |
528 | void |
529 | coalition_io_ledger_update(task_t task, int32_t flavor, boolean_t is_credit, uint32_t io_size) |
530 | { |
531 | ledger_t ledger; |
532 | coalition_t coal = task->coalition[COALITION_TYPE_RESOURCE]; |
533 | |
534 | assert(coal != NULL); |
535 | ledger = coal->r.resource_monitor_ledger; |
536 | if (LEDGER_VALID(ledger)) { |
537 | if (flavor == FLAVOR_IO_LOGICAL_WRITES) { |
538 | if (is_credit) { |
539 | ledger_credit(ledger, entry: coalition_ledgers.logical_writes, amount: io_size); |
540 | } else { |
541 | ledger_debit(ledger, entry: coalition_ledgers.logical_writes, amount: io_size); |
542 | } |
543 | } |
544 | } |
545 | } |
546 | |
547 | static void |
548 | coalition_notify_user(uint64_t id, uint32_t flags) |
549 | { |
550 | mach_port_t user_port; |
551 | kern_return_t kr; |
552 | |
553 | kr = host_get_coalition_port(host_priv_self(), &user_port); |
554 | if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) { |
555 | return; |
556 | } |
557 | |
558 | coalition_notification(coalition_port: user_port, id, flags); |
559 | ipc_port_release_send(port: user_port); |
560 | } |
561 | |
562 | /* |
563 | * |
564 | * COALITION_TYPE_RESOURCE |
565 | * |
566 | */ |
567 | static kern_return_t |
568 | i_coal_resource_init(coalition_t coal, boolean_t privileged, boolean_t efficient) |
569 | { |
570 | #pragma unused(privileged, efficient) |
571 | |
572 | assert(coal && coal->type == COALITION_TYPE_RESOURCE); |
573 | |
574 | recount_coalition_init(co: &coal->r.co_recount); |
575 | coal->r.ledger = ledger_instantiate(template: coalition_task_ledger_template, |
576 | LEDGER_CREATE_ACTIVE_ENTRIES); |
577 | if (coal->r.ledger == NULL) { |
578 | return KERN_RESOURCE_SHORTAGE; |
579 | } |
580 | |
581 | coal->r.resource_monitor_ledger = ledger_instantiate(template: coalition_ledger_template, |
582 | LEDGER_CREATE_ACTIVE_ENTRIES); |
583 | if (coal->r.resource_monitor_ledger == NULL) { |
584 | return KERN_RESOURCE_SHORTAGE; |
585 | } |
586 | |
587 | queue_init(&coal->r.tasks); |
588 | |
589 | return KERN_SUCCESS; |
590 | } |
591 | |
592 | static void |
593 | i_coal_resource_dealloc(coalition_t coal) |
594 | { |
595 | assert(coal && coal->type == COALITION_TYPE_RESOURCE); |
596 | |
597 | recount_coalition_deinit(co: &coal->r.co_recount); |
598 | ledger_dereference(ledger: coal->r.ledger); |
599 | ledger_dereference(ledger: coal->r.resource_monitor_ledger); |
600 | } |
601 | |
602 | static kern_return_t |
603 | i_coal_resource_adopt_task(coalition_t coal, task_t task) |
604 | { |
605 | struct i_resource_coalition *cr; |
606 | |
607 | assert(coal && coal->type == COALITION_TYPE_RESOURCE); |
608 | assert(queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE])); |
609 | |
610 | cr = &coal->r; |
611 | cr->task_count++; |
612 | |
613 | if (cr->task_count < cr->dead_task_count) { |
614 | panic("%s: coalition %p id:%llu type:%s task_count(%llu) < dead_task_count(%llu)" , |
615 | __func__, coal, coal->id, coal_type_str(coal->type), |
616 | cr->task_count, cr->dead_task_count); |
617 | } |
618 | |
619 | /* If moving from 0->1 active tasks */ |
620 | if (cr->task_count - cr->dead_task_count == 1) { |
621 | cr->last_became_nonempty_time = mach_absolute_time(); |
622 | } |
623 | |
624 | /* put the task on the coalition's list of tasks */ |
625 | enqueue_tail(que: &cr->tasks, elt: &task->task_coalition[COALITION_TYPE_RESOURCE]); |
626 | |
627 | coal_dbg("Added PID:%d to id:%llu, task_count:%llu, dead_count:%llu, nonempty_time:%llu" , |
628 | task_pid(task), coal->id, cr->task_count, cr->dead_task_count, |
629 | cr->last_became_nonempty_time); |
630 | |
631 | return KERN_SUCCESS; |
632 | } |
633 | |
634 | static kern_return_t |
635 | i_coal_resource_remove_task(coalition_t coal, task_t task) |
636 | { |
637 | struct i_resource_coalition *cr; |
638 | |
639 | assert(coal && coal->type == COALITION_TYPE_RESOURCE); |
640 | assert(task->coalition[COALITION_TYPE_RESOURCE] == coal); |
641 | assert(!queue_empty(&task->task_coalition[COALITION_TYPE_RESOURCE])); |
642 | |
643 | /* |
644 | * handle resource coalition accounting rollup for dead tasks |
645 | */ |
646 | cr = &coal->r; |
647 | |
648 | cr->dead_task_count++; |
649 | |
650 | if (cr->task_count < cr->dead_task_count) { |
651 | panic("%s: coalition %p id:%llu type:%s task_count(%llu) < dead_task_count(%llu)" , |
652 | __func__, coal, coal->id, coal_type_str(coal->type), cr->task_count, cr->dead_task_count); |
653 | } |
654 | |
655 | /* If moving from 1->0 active tasks */ |
656 | if (cr->task_count - cr->dead_task_count == 0) { |
657 | uint64_t last_time_nonempty = mach_absolute_time() - cr->last_became_nonempty_time; |
658 | cr->last_became_nonempty_time = 0; |
659 | cr->time_nonempty += last_time_nonempty; |
660 | } |
661 | |
662 | /* Do not roll up for exec'd task or exec copy task */ |
663 | if (!task_is_exec_copy(task) && !task_did_exec(task)) { |
664 | ledger_rollup(to_ledger: cr->ledger, from_ledger: task->ledger); |
665 | cr->bytesread += task->task_io_stats->disk_reads.size; |
666 | cr->byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size; |
667 | #if defined(__x86_64__) |
668 | cr->gpu_time += task_gpu_utilisation(task); |
669 | #endif /* defined(__x86_64__) */ |
670 | |
671 | cr->logical_immediate_writes += task->task_writes_counters_internal.task_immediate_writes; |
672 | cr->logical_deferred_writes += task->task_writes_counters_internal.task_deferred_writes; |
673 | cr->logical_invalidated_writes += task->task_writes_counters_internal.task_invalidated_writes; |
674 | cr->logical_metadata_writes += task->task_writes_counters_internal.task_metadata_writes; |
675 | cr->logical_immediate_writes_to_external += task->task_writes_counters_external.task_immediate_writes; |
676 | cr->logical_deferred_writes_to_external += task->task_writes_counters_external.task_deferred_writes; |
677 | cr->logical_invalidated_writes_to_external += task->task_writes_counters_external.task_invalidated_writes; |
678 | cr->logical_metadata_writes_to_external += task->task_writes_counters_external.task_metadata_writes; |
679 | #if CONFIG_PHYS_WRITE_ACCT |
680 | cr->fs_metadata_writes += task->task_fs_metadata_writes; |
681 | #endif /* CONFIG_PHYS_WRITE_ACCT */ |
682 | task_update_cpu_time_qos_stats(task, eqos_stats: cr->cpu_time_eqos, rqos_stats: cr->cpu_time_rqos); |
683 | recount_coalition_rollup_task(co: &cr->co_recount, tk: &task->tk_recount); |
684 | } |
685 | |
686 | /* remove the task from the coalition's list */ |
687 | remqueue(elt: &task->task_coalition[COALITION_TYPE_RESOURCE]); |
688 | queue_chain_init(task->task_coalition[COALITION_TYPE_RESOURCE]); |
689 | |
690 | coal_dbg("removed PID:%d from id:%llu, task_count:%llu, dead_count:%llu" , |
691 | task_pid(task), coal->id, cr->task_count, cr->dead_task_count); |
692 | |
693 | return KERN_SUCCESS; |
694 | } |
695 | |
696 | static kern_return_t |
697 | i_coal_resource_set_taskrole(__unused coalition_t coal, |
698 | __unused task_t task, __unused int role) |
699 | { |
700 | return KERN_SUCCESS; |
701 | } |
702 | |
703 | static int |
704 | i_coal_resource_get_taskrole(__unused coalition_t coal, __unused task_t task) |
705 | { |
706 | task_t t; |
707 | |
708 | assert(coal && coal->type == COALITION_TYPE_RESOURCE); |
709 | |
710 | qe_foreach_element(t, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) { |
711 | if (t == task) { |
712 | return COALITION_TASKROLE_UNDEF; |
713 | } |
714 | } |
715 | |
716 | return -1; |
717 | } |
718 | |
719 | static void |
720 | i_coal_resource_iterate_tasks(coalition_t coal, void *ctx, void (*callback)(coalition_t, void *, task_t)) |
721 | { |
722 | task_t t; |
723 | assert(coal && coal->type == COALITION_TYPE_RESOURCE); |
724 | |
725 | qe_foreach_element(t, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) |
726 | callback(coal, ctx, t); |
727 | } |
728 | |
729 | #if CONFIG_PHYS_WRITE_ACCT |
730 | extern uint64_t kernel_pm_writes; |
731 | #endif /* CONFIG_PHYS_WRITE_ACCT */ |
732 | |
733 | kern_return_t |
734 | coalition_resource_usage_internal(coalition_t coal, struct coalition_resource_usage *cru_out) |
735 | { |
736 | kern_return_t kr; |
737 | ledger_amount_t credit, debit; |
738 | int i; |
739 | |
740 | if (coal->type != COALITION_TYPE_RESOURCE) { |
741 | return KERN_INVALID_ARGUMENT; |
742 | } |
743 | |
744 | /* Return KERN_INVALID_ARGUMENT for Corpse coalition */ |
745 | for (i = 0; i < COALITION_NUM_TYPES; i++) { |
746 | if (coal == corpse_coalition[i]) { |
747 | return KERN_INVALID_ARGUMENT; |
748 | } |
749 | } |
750 | |
751 | ledger_t sum_ledger = ledger_instantiate(template: coalition_task_ledger_template, LEDGER_CREATE_ACTIVE_ENTRIES); |
752 | if (sum_ledger == LEDGER_NULL) { |
753 | return KERN_RESOURCE_SHORTAGE; |
754 | } |
755 | |
756 | coalition_lock(coal); |
757 | |
758 | /* |
759 | * Start with the coalition's ledger, which holds the totals from all |
760 | * the dead tasks. |
761 | */ |
762 | ledger_rollup(to_ledger: sum_ledger, from_ledger: coal->r.ledger); |
763 | uint64_t bytesread = coal->r.bytesread; |
764 | uint64_t byteswritten = coal->r.byteswritten; |
765 | uint64_t gpu_time = coal->r.gpu_time; |
766 | uint64_t logical_immediate_writes = coal->r.logical_immediate_writes; |
767 | uint64_t logical_deferred_writes = coal->r.logical_deferred_writes; |
768 | uint64_t logical_invalidated_writes = coal->r.logical_invalidated_writes; |
769 | uint64_t logical_metadata_writes = coal->r.logical_metadata_writes; |
770 | uint64_t logical_immediate_writes_to_external = coal->r.logical_immediate_writes_to_external; |
771 | uint64_t logical_deferred_writes_to_external = coal->r.logical_deferred_writes_to_external; |
772 | uint64_t logical_invalidated_writes_to_external = coal->r.logical_invalidated_writes_to_external; |
773 | uint64_t logical_metadata_writes_to_external = coal->r.logical_metadata_writes_to_external; |
774 | #if CONFIG_PHYS_WRITE_ACCT |
775 | uint64_t fs_metadata_writes = coal->r.fs_metadata_writes; |
776 | #endif /* CONFIG_PHYS_WRITE_ACCT */ |
777 | int64_t conclave_mem = 0; |
778 | int64_t cpu_time_billed_to_me = 0; |
779 | int64_t cpu_time_billed_to_others = 0; |
780 | int64_t energy_billed_to_me = 0; |
781 | int64_t energy_billed_to_others = 0; |
782 | struct recount_usage stats_sum = { 0 }; |
783 | struct recount_usage stats_perf_only = { 0 }; |
784 | recount_coalition_usage_perf_only(coal: &coal->r.co_recount, sum: &stats_sum, |
785 | sum_perf_only: &stats_perf_only); |
786 | uint64_t cpu_time_eqos[COALITION_NUM_THREAD_QOS_TYPES] = { 0 }; |
787 | uint64_t cpu_time_rqos[COALITION_NUM_THREAD_QOS_TYPES] = { 0 }; |
788 | /* |
789 | * Add to that all the active tasks' ledgers. Tasks cannot deallocate |
790 | * out from under us, since we hold the coalition lock. |
791 | */ |
792 | task_t task; |
793 | qe_foreach_element(task, &coal->r.tasks, task_coalition[COALITION_TYPE_RESOURCE]) { |
794 | /* |
795 | * Rolling up stats for exec copy task or exec'd task will lead to double accounting. |
796 | * Cannot take task lock after taking coaliton lock |
797 | */ |
798 | if (task_is_exec_copy(task) || task_did_exec(task)) { |
799 | continue; |
800 | } |
801 | |
802 | ledger_rollup(to_ledger: sum_ledger, from_ledger: task->ledger); |
803 | bytesread += task->task_io_stats->disk_reads.size; |
804 | byteswritten += task->task_io_stats->total_io.size - task->task_io_stats->disk_reads.size; |
805 | #if defined(__x86_64__) |
806 | gpu_time += task_gpu_utilisation(task); |
807 | #endif /* defined(__x86_64__) */ |
808 | |
809 | logical_immediate_writes += task->task_writes_counters_internal.task_immediate_writes; |
810 | logical_deferred_writes += task->task_writes_counters_internal.task_deferred_writes; |
811 | logical_invalidated_writes += task->task_writes_counters_internal.task_invalidated_writes; |
812 | logical_metadata_writes += task->task_writes_counters_internal.task_metadata_writes; |
813 | logical_immediate_writes_to_external += task->task_writes_counters_external.task_immediate_writes; |
814 | logical_deferred_writes_to_external += task->task_writes_counters_external.task_deferred_writes; |
815 | logical_invalidated_writes_to_external += task->task_writes_counters_external.task_invalidated_writes; |
816 | logical_metadata_writes_to_external += task->task_writes_counters_external.task_metadata_writes; |
817 | #if CONFIG_PHYS_WRITE_ACCT |
818 | fs_metadata_writes += task->task_fs_metadata_writes; |
819 | #endif /* CONFIG_PHYS_WRITE_ACCT */ |
820 | |
821 | task_update_cpu_time_qos_stats(task, eqos_stats: cpu_time_eqos, rqos_stats: cpu_time_rqos); |
822 | recount_task_usage_perf_only(task, sum: &stats_sum, sum_perf_only: &stats_perf_only); |
823 | } |
824 | |
825 | kr = ledger_get_balance(ledger: sum_ledger, entry: task_ledgers.conclave_mem, balance: (int64_t *)&conclave_mem); |
826 | if (kr != KERN_SUCCESS || conclave_mem < 0) { |
827 | conclave_mem = 0; |
828 | } |
829 | |
830 | kr = ledger_get_balance(ledger: sum_ledger, entry: task_ledgers.cpu_time_billed_to_me, balance: (int64_t *)&cpu_time_billed_to_me); |
831 | if (kr != KERN_SUCCESS || cpu_time_billed_to_me < 0) { |
832 | cpu_time_billed_to_me = 0; |
833 | } |
834 | |
835 | kr = ledger_get_balance(ledger: sum_ledger, entry: task_ledgers.cpu_time_billed_to_others, balance: (int64_t *)&cpu_time_billed_to_others); |
836 | if (kr != KERN_SUCCESS || cpu_time_billed_to_others < 0) { |
837 | cpu_time_billed_to_others = 0; |
838 | } |
839 | |
840 | kr = ledger_get_balance(ledger: sum_ledger, entry: task_ledgers.energy_billed_to_me, balance: (int64_t *)&energy_billed_to_me); |
841 | if (kr != KERN_SUCCESS || energy_billed_to_me < 0) { |
842 | energy_billed_to_me = 0; |
843 | } |
844 | |
845 | kr = ledger_get_balance(ledger: sum_ledger, entry: task_ledgers.energy_billed_to_others, balance: (int64_t *)&energy_billed_to_others); |
846 | if (kr != KERN_SUCCESS || energy_billed_to_others < 0) { |
847 | energy_billed_to_others = 0; |
848 | } |
849 | |
850 | /* collect information from the coalition itself */ |
851 | cru_out->tasks_started = coal->r.task_count; |
852 | cru_out->tasks_exited = coal->r.dead_task_count; |
853 | |
854 | uint64_t time_nonempty = coal->r.time_nonempty; |
855 | uint64_t last_became_nonempty_time = coal->r.last_became_nonempty_time; |
856 | |
857 | coalition_unlock(coal); |
858 | |
859 | /* Copy the totals out of sum_ledger */ |
860 | kr = ledger_get_entries(ledger: sum_ledger, entry: task_ledgers.cpu_time, |
861 | credit: &credit, debit: &debit); |
862 | if (kr != KERN_SUCCESS) { |
863 | credit = 0; |
864 | } |
865 | cru_out->conclave_mem = conclave_mem; |
866 | cru_out->cpu_time = credit; |
867 | cru_out->cpu_time_billed_to_me = (uint64_t)cpu_time_billed_to_me; |
868 | cru_out->cpu_time_billed_to_others = (uint64_t)cpu_time_billed_to_others; |
869 | cru_out->energy_billed_to_me = (uint64_t)energy_billed_to_me; |
870 | cru_out->energy_billed_to_others = (uint64_t)energy_billed_to_others; |
871 | |
872 | kr = ledger_get_entries(ledger: sum_ledger, entry: task_ledgers.interrupt_wakeups, |
873 | credit: &credit, debit: &debit); |
874 | if (kr != KERN_SUCCESS) { |
875 | credit = 0; |
876 | } |
877 | cru_out->interrupt_wakeups = credit; |
878 | |
879 | kr = ledger_get_entries(ledger: sum_ledger, entry: task_ledgers.platform_idle_wakeups, |
880 | credit: &credit, debit: &debit); |
881 | if (kr != KERN_SUCCESS) { |
882 | credit = 0; |
883 | } |
884 | cru_out->platform_idle_wakeups = credit; |
885 | |
886 | cru_out->bytesread = bytesread; |
887 | cru_out->byteswritten = byteswritten; |
888 | cru_out->gpu_time = gpu_time; |
889 | cru_out->logical_immediate_writes = logical_immediate_writes; |
890 | cru_out->logical_deferred_writes = logical_deferred_writes; |
891 | cru_out->logical_invalidated_writes = logical_invalidated_writes; |
892 | cru_out->logical_metadata_writes = logical_metadata_writes; |
893 | cru_out->logical_immediate_writes_to_external = logical_immediate_writes_to_external; |
894 | cru_out->logical_deferred_writes_to_external = logical_deferred_writes_to_external; |
895 | cru_out->logical_invalidated_writes_to_external = logical_invalidated_writes_to_external; |
896 | cru_out->logical_metadata_writes_to_external = logical_metadata_writes_to_external; |
897 | #if CONFIG_PHYS_WRITE_ACCT |
898 | cru_out->fs_metadata_writes = fs_metadata_writes; |
899 | #else |
900 | cru_out->fs_metadata_writes = 0; |
901 | #endif /* CONFIG_PHYS_WRITE_ACCT */ |
902 | cru_out->cpu_time_eqos_len = COALITION_NUM_THREAD_QOS_TYPES; |
903 | memcpy(dst: cru_out->cpu_time_eqos, src: cpu_time_eqos, n: sizeof(cru_out->cpu_time_eqos)); |
904 | |
905 | cru_out->cpu_ptime = recount_usage_time_mach(usage: &stats_perf_only); |
906 | #if CONFIG_PERVASIVE_CPI |
907 | cru_out->cpu_instructions = recount_usage_instructions(&stats_sum); |
908 | cru_out->cpu_cycles = recount_usage_cycles(&stats_sum); |
909 | cru_out->cpu_pinstructions = recount_usage_instructions(&stats_perf_only); |
910 | cru_out->cpu_pcycles = recount_usage_cycles(&stats_perf_only); |
911 | #endif // CONFIG_PERVASIVE_CPI |
912 | |
913 | ledger_dereference(ledger: sum_ledger); |
914 | sum_ledger = LEDGER_NULL; |
915 | |
916 | #if CONFIG_PERVASIVE_ENERGY |
917 | cru_out->energy = stats_sum.ru_energy_nj; |
918 | #endif /* CONFIG_PERVASIVE_ENERGY */ |
919 | |
920 | #if CONFIG_PHYS_WRITE_ACCT |
921 | // kernel_pm_writes are only recorded under kernel_task coalition |
922 | if (coalition_id(coal) == COALITION_ID_KERNEL) { |
923 | cru_out->pm_writes = kernel_pm_writes; |
924 | } else { |
925 | cru_out->pm_writes = 0; |
926 | } |
927 | #else |
928 | cru_out->pm_writes = 0; |
929 | #endif /* CONFIG_PHYS_WRITE_ACCT */ |
930 | |
931 | if (last_became_nonempty_time) { |
932 | time_nonempty += mach_absolute_time() - last_became_nonempty_time; |
933 | } |
934 | absolutetime_to_nanoseconds(abstime: time_nonempty, result: &cru_out->time_nonempty); |
935 | |
936 | return KERN_SUCCESS; |
937 | } |
938 | |
939 | kern_return_t |
940 | coalition_debug_info_internal(coalition_t coal, |
941 | struct coalinfo_debuginfo *c_debuginfo) |
942 | { |
943 | /* Return KERN_INVALID_ARGUMENT for Corpse coalition */ |
944 | for (int i = 0; i < COALITION_NUM_TYPES; i++) { |
945 | if (coal == corpse_coalition[i]) { |
946 | return KERN_INVALID_ARGUMENT; |
947 | } |
948 | } |
949 | |
950 | if (coal->type == COALITION_FOCAL_TASKS_ACCOUNTING) { |
951 | c_debuginfo->focal_task_count = coal->focal_task_count; |
952 | c_debuginfo->nonfocal_task_count = coal->nonfocal_task_count; |
953 | c_debuginfo->game_task_count = coal->game_task_count; |
954 | } |
955 | |
956 | #if CONFIG_THREAD_GROUPS |
957 | struct thread_group * group = coalition_get_thread_group(coal); |
958 | |
959 | if (group != NULL) { |
960 | c_debuginfo->thread_group_id = thread_group_id(tg: group); |
961 | c_debuginfo->thread_group_flags = thread_group_get_flags(group); |
962 | c_debuginfo->thread_group_recommendation = thread_group_recommendation(tg: group); |
963 | } |
964 | #endif /* CONFIG_THREAD_GROUPS */ |
965 | |
966 | return KERN_SUCCESS; |
967 | } |
968 | |
969 | /* |
970 | * |
971 | * COALITION_TYPE_JETSAM |
972 | * |
973 | */ |
974 | static kern_return_t |
975 | i_coal_jetsam_init(coalition_t coal, boolean_t privileged, boolean_t efficient) |
976 | { |
977 | assert(coal && coal->type == COALITION_TYPE_JETSAM); |
978 | (void)privileged; |
979 | (void)efficient; |
980 | |
981 | coal->j.leader = TASK_NULL; |
982 | queue_head_init(coal->j.extensions); |
983 | queue_head_init(coal->j.services); |
984 | queue_head_init(coal->j.other); |
985 | |
986 | #if CONFIG_THREAD_GROUPS |
987 | switch (coal->role) { |
988 | case COALITION_ROLE_SYSTEM: |
989 | coal->j.thread_group = thread_group_find_by_id_and_retain(THREAD_GROUP_SYSTEM); |
990 | break; |
991 | case COALITION_ROLE_BACKGROUND: |
992 | coal->j.thread_group = thread_group_find_by_id_and_retain(THREAD_GROUP_BACKGROUND); |
993 | break; |
994 | default: |
995 | coal->j.thread_group = thread_group_create_and_retain(flags: efficient ? THREAD_GROUP_FLAGS_EFFICIENT : THREAD_GROUP_FLAGS_DEFAULT); |
996 | } |
997 | assert(coal->j.thread_group != NULL); |
998 | #endif |
999 | return KERN_SUCCESS; |
1000 | } |
1001 | |
1002 | static void |
1003 | i_coal_jetsam_dealloc(__unused coalition_t coal) |
1004 | { |
1005 | assert(coal && coal->type == COALITION_TYPE_JETSAM); |
1006 | |
1007 | /* the coalition should be completely clear at this point */ |
1008 | assert(queue_empty(&coal->j.extensions)); |
1009 | assert(queue_empty(&coal->j.services)); |
1010 | assert(queue_empty(&coal->j.other)); |
1011 | assert(coal->j.leader == TASK_NULL); |
1012 | |
1013 | #if CONFIG_THREAD_GROUPS |
1014 | /* disassociate from the thread group */ |
1015 | assert(coal->j.thread_group != NULL); |
1016 | thread_group_release(tg: coal->j.thread_group); |
1017 | coal->j.thread_group = NULL; |
1018 | #endif |
1019 | } |
1020 | |
1021 | static kern_return_t |
1022 | i_coal_jetsam_adopt_task(coalition_t coal, task_t task) |
1023 | { |
1024 | struct i_jetsam_coalition *cj; |
1025 | assert(coal && coal->type == COALITION_TYPE_JETSAM); |
1026 | |
1027 | cj = &coal->j; |
1028 | |
1029 | assert(queue_empty(&task->task_coalition[COALITION_TYPE_JETSAM])); |
1030 | |
1031 | /* put each task initially in the "other" list */ |
1032 | enqueue_tail(que: &cj->other, elt: &task->task_coalition[COALITION_TYPE_JETSAM]); |
1033 | coal_dbg("coalition %lld adopted PID:%d as UNDEF" , |
1034 | coal->id, task_pid(task)); |
1035 | |
1036 | return KERN_SUCCESS; |
1037 | } |
1038 | |
1039 | static kern_return_t |
1040 | i_coal_jetsam_remove_task(coalition_t coal, task_t task) |
1041 | { |
1042 | assert(coal && coal->type == COALITION_TYPE_JETSAM); |
1043 | assert(task->coalition[COALITION_TYPE_JETSAM] == coal); |
1044 | |
1045 | coal_dbg("removing PID:%d from coalition id:%lld" , |
1046 | task_pid(task), coal->id); |
1047 | |
1048 | if (task == coal->j.leader) { |
1049 | coal->j.leader = NULL; |
1050 | coal_dbg(" PID:%d was the leader!" , task_pid(task)); |
1051 | } else { |
1052 | assert(!queue_empty(&task->task_coalition[COALITION_TYPE_JETSAM])); |
1053 | } |
1054 | |
1055 | /* remove the task from the specific coalition role queue */ |
1056 | remqueue(elt: &task->task_coalition[COALITION_TYPE_JETSAM]); |
1057 | queue_chain_init(task->task_coalition[COALITION_TYPE_RESOURCE]); |
1058 | |
1059 | return KERN_SUCCESS; |
1060 | } |
1061 | |
1062 | static kern_return_t |
1063 | i_coal_jetsam_set_taskrole(coalition_t coal, task_t task, int role) |
1064 | { |
1065 | struct i_jetsam_coalition *cj; |
1066 | queue_t q = NULL; |
1067 | assert(coal && coal->type == COALITION_TYPE_JETSAM); |
1068 | assert(task->coalition[COALITION_TYPE_JETSAM] == coal); |
1069 | |
1070 | cj = &coal->j; |
1071 | |
1072 | switch (role) { |
1073 | case COALITION_TASKROLE_LEADER: |
1074 | coal_dbg("setting PID:%d as LEADER of %lld" , |
1075 | task_pid(task), coal->id); |
1076 | if (cj->leader != TASK_NULL) { |
1077 | /* re-queue the exiting leader onto the "other" list */ |
1078 | coal_dbg(" re-queue existing leader (%d) as OTHER" , |
1079 | task_pid(cj->leader)); |
1080 | re_queue_tail(que: &cj->other, elt: &cj->leader->task_coalition[COALITION_TYPE_JETSAM]); |
1081 | } |
1082 | /* |
1083 | * remove the task from the "other" list |
1084 | * (where it was put by default) |
1085 | */ |
1086 | remqueue(elt: &task->task_coalition[COALITION_TYPE_JETSAM]); |
1087 | queue_chain_init(task->task_coalition[COALITION_TYPE_JETSAM]); |
1088 | |
1089 | /* set the coalition leader */ |
1090 | cj->leader = task; |
1091 | break; |
1092 | case COALITION_TASKROLE_XPC: |
1093 | coal_dbg("setting PID:%d as XPC in %lld" , |
1094 | task_pid(task), coal->id); |
1095 | q = (queue_t)&cj->services; |
1096 | break; |
1097 | case COALITION_TASKROLE_EXT: |
1098 | coal_dbg("setting PID:%d as EXT in %lld" , |
1099 | task_pid(task), coal->id); |
1100 | q = (queue_t)&cj->extensions; |
1101 | break; |
1102 | case COALITION_TASKROLE_NONE: |
1103 | /* |
1104 | * Tasks with a role of "none" should fall through to an |
1105 | * undefined role so long as the task is currently a member |
1106 | * of the coalition. This scenario can happen if a task is |
1107 | * killed (usually via jetsam) during exec. |
1108 | */ |
1109 | if (task->coalition[COALITION_TYPE_JETSAM] != coal) { |
1110 | panic("%s: task %p attempting to set role %d " |
1111 | "in coalition %p to which it does not belong!" , __func__, task, role, coal); |
1112 | } |
1113 | OS_FALLTHROUGH; |
1114 | case COALITION_TASKROLE_UNDEF: |
1115 | coal_dbg("setting PID:%d as UNDEF in %lld" , |
1116 | task_pid(task), coal->id); |
1117 | q = (queue_t)&cj->other; |
1118 | break; |
1119 | default: |
1120 | panic("%s: invalid role(%d) for task" , __func__, role); |
1121 | return KERN_INVALID_ARGUMENT; |
1122 | } |
1123 | |
1124 | if (q != NULL) { |
1125 | re_queue_tail(que: q, elt: &task->task_coalition[COALITION_TYPE_JETSAM]); |
1126 | } |
1127 | |
1128 | return KERN_SUCCESS; |
1129 | } |
1130 | |
1131 | int |
1132 | i_coal_jetsam_get_taskrole(coalition_t coal, task_t task) |
1133 | { |
1134 | struct i_jetsam_coalition *cj; |
1135 | task_t t; |
1136 | |
1137 | assert(coal && coal->type == COALITION_TYPE_JETSAM); |
1138 | assert(task->coalition[COALITION_TYPE_JETSAM] == coal); |
1139 | |
1140 | cj = &coal->j; |
1141 | |
1142 | if (task == cj->leader) { |
1143 | return COALITION_TASKROLE_LEADER; |
1144 | } |
1145 | |
1146 | qe_foreach_element(t, &cj->services, task_coalition[COALITION_TYPE_JETSAM]) { |
1147 | if (t == task) { |
1148 | return COALITION_TASKROLE_XPC; |
1149 | } |
1150 | } |
1151 | |
1152 | qe_foreach_element(t, &cj->extensions, task_coalition[COALITION_TYPE_JETSAM]) { |
1153 | if (t == task) { |
1154 | return COALITION_TASKROLE_EXT; |
1155 | } |
1156 | } |
1157 | |
1158 | qe_foreach_element(t, &cj->other, task_coalition[COALITION_TYPE_JETSAM]) { |
1159 | if (t == task) { |
1160 | return COALITION_TASKROLE_UNDEF; |
1161 | } |
1162 | } |
1163 | |
1164 | /* task not in the coalition?! */ |
1165 | return COALITION_TASKROLE_NONE; |
1166 | } |
1167 | |
1168 | static void |
1169 | i_coal_jetsam_iterate_tasks(coalition_t coal, void *ctx, void (*callback)(coalition_t, void *, task_t)) |
1170 | { |
1171 | struct i_jetsam_coalition *cj; |
1172 | task_t t; |
1173 | |
1174 | assert(coal && coal->type == COALITION_TYPE_JETSAM); |
1175 | |
1176 | cj = &coal->j; |
1177 | |
1178 | if (cj->leader) { |
1179 | callback(coal, ctx, cj->leader); |
1180 | } |
1181 | |
1182 | qe_foreach_element(t, &cj->services, task_coalition[COALITION_TYPE_JETSAM]) |
1183 | callback(coal, ctx, t); |
1184 | |
1185 | qe_foreach_element(t, &cj->extensions, task_coalition[COALITION_TYPE_JETSAM]) |
1186 | callback(coal, ctx, t); |
1187 | |
1188 | qe_foreach_element(t, &cj->other, task_coalition[COALITION_TYPE_JETSAM]) |
1189 | callback(coal, ctx, t); |
1190 | } |
1191 | |
1192 | |
1193 | /* |
1194 | * |
1195 | * Main Coalition implementation |
1196 | * |
1197 | */ |
1198 | |
1199 | /* |
1200 | * coalition_create_internal |
1201 | * Returns: New coalition object, referenced for the caller and unlocked. |
1202 | * Condition: coalitions_list_lock must be UNLOCKED. |
1203 | */ |
1204 | kern_return_t |
1205 | coalition_create_internal(int type, int role, boolean_t privileged, boolean_t efficient, coalition_t *out, uint64_t *coalition_id) |
1206 | { |
1207 | kern_return_t kr; |
1208 | struct coalition *new_coal; |
1209 | uint64_t cid; |
1210 | |
1211 | if (type < 0 || type > COALITION_TYPE_MAX) { |
1212 | return KERN_INVALID_ARGUMENT; |
1213 | } |
1214 | |
1215 | new_coal = zalloc_flags(coalition_zone, Z_WAITOK | Z_ZERO | Z_NOFAIL); |
1216 | |
1217 | new_coal->type = type; |
1218 | new_coal->role = role; |
1219 | |
1220 | /* initialize type-specific resources */ |
1221 | kr = coal_call(new_coal, init, privileged, efficient); |
1222 | if (kr != KERN_SUCCESS) { |
1223 | zfree(coalition_zone, new_coal); |
1224 | return kr; |
1225 | } |
1226 | |
1227 | /* One for caller, one for coalitions list */ |
1228 | coal_ref_init(new_coal, 2); |
1229 | |
1230 | new_coal->privileged = privileged ? TRUE : FALSE; |
1231 | new_coal->efficient = efficient ? TRUE : FALSE; |
1232 | #if DEVELOPMENT || DEBUG |
1233 | new_coal->should_notify = 1; |
1234 | #endif |
1235 | |
1236 | lck_mtx_init(lck: &new_coal->lock, grp: &coalitions_lck_grp, LCK_ATTR_NULL); |
1237 | |
1238 | lck_mtx_lock(lck: &coalitions_list_lock); |
1239 | new_coal->id = cid = coalition_next_id++; |
1240 | |
1241 | smr_hash_serialized_insert(&coalition_hash, &new_coal->link, |
1242 | &coal_hash_traits); |
1243 | |
1244 | #if CONFIG_THREAD_GROUPS |
1245 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_NEW), |
1246 | new_coal->id, new_coal->type, |
1247 | (new_coal->type == COALITION_TYPE_JETSAM && new_coal->j.thread_group) ? |
1248 | thread_group_get_id(new_coal->j.thread_group) : 0); |
1249 | |
1250 | #else |
1251 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_NEW), |
1252 | new_coal->id, new_coal->type); |
1253 | #endif |
1254 | |
1255 | if (smr_hash_serialized_should_grow(smrh: &coalition_hash, size_factor: 1, count_factor: 4)) { |
1256 | /* grow if more more than 4 elements per 1 bucket */ |
1257 | smr_hash_grow_and_unlock(&coalition_hash, |
1258 | &coalitions_list_lock, &coal_hash_traits); |
1259 | } else { |
1260 | lck_mtx_unlock(lck: &coalitions_list_lock); |
1261 | } |
1262 | |
1263 | coal_dbg("id:%llu, type:%s" , cid, coal_type_str(type)); |
1264 | |
1265 | if (coalition_id != NULL) { |
1266 | *coalition_id = cid; |
1267 | } |
1268 | |
1269 | *out = new_coal; |
1270 | return KERN_SUCCESS; |
1271 | } |
1272 | |
1273 | static void |
1274 | coalition_free(struct smr_node *node) |
1275 | { |
1276 | struct coalition *coal; |
1277 | |
1278 | coal = __container_of(node, struct coalition, smr_node); |
1279 | zfree(coalition_zone, coal); |
1280 | } |
1281 | |
1282 | static __attribute__((noinline)) void |
1283 | coalition_retire(coalition_t coal) |
1284 | { |
1285 | coalition_lock(coal); |
1286 | |
1287 | coal_dbg("id:%llu type:%s active_count:%u%s" , |
1288 | coal->id, coal_type_str(coal->type), coal->active_count, |
1289 | rc <= 0 ? ", will deallocate now" : "" ); |
1290 | |
1291 | assert(coal->termrequested); |
1292 | assert(coal->terminated); |
1293 | assert(coal->active_count == 0); |
1294 | assert(coal->reaped); |
1295 | assert(coal->focal_task_count == 0); |
1296 | assert(coal->nonfocal_task_count == 0); |
1297 | assert(coal->game_task_count == 0); |
1298 | #if CONFIG_THREAD_GROUPS |
1299 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_FREE), |
1300 | coal->id, coal->type, |
1301 | coal->type == COALITION_TYPE_JETSAM ? |
1302 | coal->j.thread_group : 0); |
1303 | #else |
1304 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_FREE), |
1305 | coal->id, coal->type); |
1306 | #endif |
1307 | |
1308 | coal_call(coal, dealloc); |
1309 | |
1310 | coalition_unlock(coal); |
1311 | |
1312 | lck_mtx_destroy(lck: &coal->lock, grp: &coalitions_lck_grp); |
1313 | |
1314 | smr_proc_task_call(&coal->smr_node, sizeof(*coal), coalition_free); |
1315 | } |
1316 | |
1317 | /* |
1318 | * coalition_release |
1319 | * Condition: coalition must be UNLOCKED. |
1320 | * */ |
1321 | void |
1322 | coalition_release(coalition_t coal) |
1323 | { |
1324 | if (coal_ref_release(coal) > 0) { |
1325 | return; |
1326 | } |
1327 | |
1328 | coalition_retire(coal); |
1329 | } |
1330 | |
1331 | /* |
1332 | * coalition_find_by_id |
1333 | * Returns: Coalition object with specified id, referenced. |
1334 | */ |
1335 | coalition_t |
1336 | coalition_find_by_id(uint64_t cid) |
1337 | { |
1338 | smrh_key_t key = SMRH_SCALAR_KEY(cid); |
1339 | |
1340 | if (cid == 0) { |
1341 | return COALITION_NULL; |
1342 | } |
1343 | |
1344 | return smr_hash_get(&coalition_hash, key, &coal_hash_traits); |
1345 | } |
1346 | |
1347 | /* |
1348 | * coalition_find_and_activate_by_id |
1349 | * Returns: Coalition object with specified id, referenced, and activated. |
1350 | * This is the function to use when putting a 'new' thing into a coalition, |
1351 | * like posix_spawn of an XPC service by launchd. |
1352 | * See also coalition_extend_active. |
1353 | */ |
1354 | coalition_t |
1355 | coalition_find_and_activate_by_id(uint64_t cid) |
1356 | { |
1357 | coalition_t coal = coalition_find_by_id(cid); |
1358 | |
1359 | if (coal == COALITION_NULL) { |
1360 | return COALITION_NULL; |
1361 | } |
1362 | |
1363 | coalition_lock(coal); |
1364 | |
1365 | if (coal->reaped || coal->terminated) { |
1366 | /* Too late to put something new into this coalition, it's |
1367 | * already on its way out the door */ |
1368 | coalition_unlock(coal); |
1369 | coalition_release(coal); |
1370 | return COALITION_NULL; |
1371 | } |
1372 | |
1373 | coal->active_count++; |
1374 | |
1375 | #if COALITION_DEBUG |
1376 | uint32_t rc = coal_ref_count(coal); |
1377 | uint32_t ac = coal->active_count; |
1378 | #endif |
1379 | coalition_unlock(coal); |
1380 | |
1381 | coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u" , |
1382 | coal->id, coal_type_str(coal->type), rc, ac); |
1383 | |
1384 | return coal; |
1385 | } |
1386 | |
1387 | uint64_t |
1388 | coalition_id(coalition_t coal) |
1389 | { |
1390 | assert(coal != COALITION_NULL); |
1391 | return coal->id; |
1392 | } |
1393 | |
1394 | void |
1395 | task_coalition_ids(task_t task, uint64_t ids[COALITION_NUM_TYPES]) |
1396 | { |
1397 | int i; |
1398 | for (i = 0; i < COALITION_NUM_TYPES; i++) { |
1399 | if (task->coalition[i]) { |
1400 | ids[i] = task->coalition[i]->id; |
1401 | } else { |
1402 | ids[i] = 0; |
1403 | } |
1404 | } |
1405 | } |
1406 | |
1407 | void |
1408 | task_coalition_roles(task_t task, int roles[COALITION_NUM_TYPES]) |
1409 | { |
1410 | int i; |
1411 | memset(s: roles, c: 0, COALITION_NUM_TYPES * sizeof(roles[0])); |
1412 | |
1413 | for (i = 0; i < COALITION_NUM_TYPES; i++) { |
1414 | if (task->coalition[i]) { |
1415 | coalition_lock(task->coalition[i]); |
1416 | roles[i] = coal_call(task->coalition[i], |
1417 | get_taskrole, task); |
1418 | coalition_unlock(task->coalition[i]); |
1419 | } else { |
1420 | roles[i] = COALITION_TASKROLE_NONE; |
1421 | } |
1422 | } |
1423 | } |
1424 | |
1425 | int |
1426 | task_coalition_role_for_type(task_t task, int coalition_type) |
1427 | { |
1428 | coalition_t coal; |
1429 | int role; |
1430 | if (coalition_type >= COALITION_NUM_TYPES) { |
1431 | panic("Attempt to call task_coalition_role_for_type with invalid coalition_type: %d\n" , coalition_type); |
1432 | } |
1433 | coal = task->coalition[coalition_type]; |
1434 | if (coal == NULL) { |
1435 | return COALITION_TASKROLE_NONE; |
1436 | } |
1437 | coalition_lock(coal); |
1438 | role = coal_call(coal, get_taskrole, task); |
1439 | coalition_unlock(coal); |
1440 | return role; |
1441 | } |
1442 | |
1443 | int |
1444 | coalition_type(coalition_t coal) |
1445 | { |
1446 | return coal->type; |
1447 | } |
1448 | |
1449 | boolean_t |
1450 | coalition_term_requested(coalition_t coal) |
1451 | { |
1452 | return coal->termrequested; |
1453 | } |
1454 | |
1455 | boolean_t |
1456 | coalition_is_terminated(coalition_t coal) |
1457 | { |
1458 | return coal->terminated; |
1459 | } |
1460 | |
1461 | boolean_t |
1462 | coalition_is_reaped(coalition_t coal) |
1463 | { |
1464 | return coal->reaped; |
1465 | } |
1466 | |
1467 | boolean_t |
1468 | coalition_is_privileged(coalition_t coal) |
1469 | { |
1470 | return coal->privileged || unrestrict_coalition_syscalls; |
1471 | } |
1472 | |
1473 | boolean_t |
1474 | task_is_in_privileged_coalition(task_t task, int type) |
1475 | { |
1476 | if (type < 0 || type > COALITION_TYPE_MAX) { |
1477 | return FALSE; |
1478 | } |
1479 | if (unrestrict_coalition_syscalls) { |
1480 | return TRUE; |
1481 | } |
1482 | if (!task->coalition[type]) { |
1483 | return FALSE; |
1484 | } |
1485 | return task->coalition[type]->privileged; |
1486 | } |
1487 | |
1488 | void |
1489 | task_coalition_update_gpu_stats(task_t task, uint64_t gpu_ns_delta) |
1490 | { |
1491 | coalition_t coal; |
1492 | |
1493 | assert(task != TASK_NULL); |
1494 | if (gpu_ns_delta == 0) { |
1495 | return; |
1496 | } |
1497 | |
1498 | coal = task->coalition[COALITION_TYPE_RESOURCE]; |
1499 | assert(coal != COALITION_NULL); |
1500 | |
1501 | coalition_lock(coal); |
1502 | coal->r.gpu_time += gpu_ns_delta; |
1503 | coalition_unlock(coal); |
1504 | } |
1505 | |
1506 | boolean_t |
1507 | task_coalition_adjust_focal_count(task_t task, int count, uint32_t *new_count) |
1508 | { |
1509 | coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; |
1510 | if (coal == COALITION_NULL) { |
1511 | return FALSE; |
1512 | } |
1513 | |
1514 | *new_count = os_atomic_add(&coal->focal_task_count, count, relaxed); |
1515 | assert(*new_count != UINT32_MAX); |
1516 | return TRUE; |
1517 | } |
1518 | |
1519 | uint32_t |
1520 | task_coalition_focal_count(task_t task) |
1521 | { |
1522 | coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; |
1523 | if (coal == COALITION_NULL) { |
1524 | return 0; |
1525 | } |
1526 | |
1527 | return coal->focal_task_count; |
1528 | } |
1529 | |
1530 | uint32_t |
1531 | task_coalition_game_mode_count(task_t task) |
1532 | { |
1533 | coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; |
1534 | if (coal == COALITION_NULL) { |
1535 | return 0; |
1536 | } |
1537 | |
1538 | return coal->game_task_count; |
1539 | } |
1540 | |
1541 | |
1542 | boolean_t |
1543 | task_coalition_adjust_nonfocal_count(task_t task, int count, uint32_t *new_count) |
1544 | { |
1545 | coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; |
1546 | if (coal == COALITION_NULL) { |
1547 | return FALSE; |
1548 | } |
1549 | |
1550 | *new_count = os_atomic_add(&coal->nonfocal_task_count, count, relaxed); |
1551 | assert(*new_count != UINT32_MAX); |
1552 | return TRUE; |
1553 | } |
1554 | |
1555 | uint32_t |
1556 | task_coalition_nonfocal_count(task_t task) |
1557 | { |
1558 | coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; |
1559 | if (coal == COALITION_NULL) { |
1560 | return 0; |
1561 | } |
1562 | |
1563 | return coal->nonfocal_task_count; |
1564 | } |
1565 | |
1566 | bool |
1567 | task_coalition_adjust_game_mode_count(task_t task, int count, uint32_t *new_count) |
1568 | { |
1569 | coalition_t coal = task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING]; |
1570 | if (coal == COALITION_NULL) { |
1571 | return false; |
1572 | } |
1573 | |
1574 | *new_count = os_atomic_add(&coal->game_task_count, count, relaxed); |
1575 | assert(*new_count != UINT32_MAX); |
1576 | return true; |
1577 | } |
1578 | |
1579 | #if CONFIG_THREAD_GROUPS |
1580 | |
1581 | /* Thread group lives as long as the task is holding the coalition reference */ |
1582 | struct thread_group * |
1583 | task_coalition_get_thread_group(task_t task) |
1584 | { |
1585 | coalition_t coal = task->coalition[COALITION_TYPE_JETSAM]; |
1586 | /* return system thread group for non-jetsam coalitions */ |
1587 | if (coal == COALITION_NULL) { |
1588 | return init_coalition[COALITION_TYPE_JETSAM]->j.thread_group; |
1589 | } |
1590 | return coal->j.thread_group; |
1591 | } |
1592 | |
1593 | |
1594 | struct thread_group * |
1595 | kdp_coalition_get_thread_group(coalition_t coal) |
1596 | { |
1597 | if (coal->type != COALITION_TYPE_JETSAM) { |
1598 | return NULL; |
1599 | } |
1600 | assert(coal->j.thread_group != NULL); |
1601 | return coal->j.thread_group; |
1602 | } |
1603 | |
1604 | /* Thread group lives as long as the coalition reference is held */ |
1605 | struct thread_group * |
1606 | coalition_get_thread_group(coalition_t coal) |
1607 | { |
1608 | if (coal->type != COALITION_TYPE_JETSAM) { |
1609 | return NULL; |
1610 | } |
1611 | assert(coal->j.thread_group != NULL); |
1612 | return coal->j.thread_group; |
1613 | } |
1614 | |
1615 | /* Donates the thread group reference to the coalition */ |
1616 | void |
1617 | coalition_set_thread_group(coalition_t coal, struct thread_group *tg) |
1618 | { |
1619 | assert(coal != COALITION_NULL); |
1620 | assert(tg != NULL); |
1621 | |
1622 | if (coal->type != COALITION_TYPE_JETSAM) { |
1623 | return; |
1624 | } |
1625 | struct thread_group *old_tg = coal->j.thread_group; |
1626 | assert(old_tg != NULL); |
1627 | coal->j.thread_group = tg; |
1628 | |
1629 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_THREAD_GROUP_SET), |
1630 | coal->id, coal->type, thread_group_get_id(tg)); |
1631 | |
1632 | thread_group_release(tg: old_tg); |
1633 | } |
1634 | |
1635 | void |
1636 | task_coalition_thread_group_focal_update(task_t task) |
1637 | { |
1638 | assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL); |
1639 | thread_group_flags_update_lock(); |
1640 | uint32_t focal_count = task_coalition_focal_count(task); |
1641 | if (focal_count) { |
1642 | thread_group_set_flags_locked(tg: task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_UI_APP); |
1643 | } else { |
1644 | thread_group_clear_flags_locked(tg: task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_UI_APP); |
1645 | } |
1646 | thread_group_flags_update_unlock(); |
1647 | } |
1648 | |
1649 | void |
1650 | task_coalition_thread_group_game_mode_update(task_t task) |
1651 | { |
1652 | assert(task->coalition[COALITION_FOCAL_TASKS_ACCOUNTING] != COALITION_NULL); |
1653 | thread_group_flags_update_lock(); |
1654 | if (task_coalition_game_mode_count(task)) { |
1655 | thread_group_set_flags_locked(tg: task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_GAME_MODE); |
1656 | } else { |
1657 | thread_group_clear_flags_locked(tg: task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_GAME_MODE); |
1658 | } |
1659 | thread_group_flags_update_unlock(); |
1660 | } |
1661 | |
1662 | void |
1663 | task_coalition_thread_group_application_set(task_t task) |
1664 | { |
1665 | /* |
1666 | * Setting the "Application" flag on the thread group is a one way transition. |
1667 | * Once a coalition has a single task with an application apptype, the |
1668 | * thread group associated with the coalition is tagged as Application. |
1669 | */ |
1670 | thread_group_flags_update_lock(); |
1671 | thread_group_set_flags_locked(tg: task_coalition_get_thread_group(task), THREAD_GROUP_FLAGS_APPLICATION); |
1672 | thread_group_flags_update_unlock(); |
1673 | } |
1674 | |
1675 | #endif /* CONFIG_THREAD_GROUPS */ |
1676 | |
1677 | void |
1678 | coalition_for_each_task(coalition_t coal, void *ctx, |
1679 | void (*callback)(coalition_t, void *, task_t)) |
1680 | { |
1681 | assert(coal != COALITION_NULL); |
1682 | |
1683 | coal_dbg("iterating tasks in coalition %p id:%llu type:%s, active_count:%u" , |
1684 | coal, coal->id, coal_type_str(coal->type), coal->active_count); |
1685 | |
1686 | coalition_lock(coal); |
1687 | |
1688 | coal_call(coal, iterate_tasks, ctx, callback); |
1689 | |
1690 | coalition_unlock(coal); |
1691 | } |
1692 | |
1693 | |
1694 | void |
1695 | coalition_remove_active(coalition_t coal) |
1696 | { |
1697 | coalition_lock(coal); |
1698 | |
1699 | assert(!coalition_is_reaped(coal)); |
1700 | assert(coal->active_count > 0); |
1701 | |
1702 | coal->active_count--; |
1703 | |
1704 | boolean_t do_notify = FALSE; |
1705 | uint64_t notify_id = 0; |
1706 | uint32_t notify_flags = 0; |
1707 | if (coal->termrequested && coal->active_count == 0) { |
1708 | /* We only notify once, when active_count reaches zero. |
1709 | * We just decremented, so if it reached zero, we mustn't have |
1710 | * notified already. |
1711 | */ |
1712 | assert(!coal->terminated); |
1713 | coal->terminated = TRUE; |
1714 | |
1715 | assert(!coal->notified); |
1716 | |
1717 | coal->notified = TRUE; |
1718 | #if DEVELOPMENT || DEBUG |
1719 | do_notify = coal->should_notify; |
1720 | #else |
1721 | do_notify = TRUE; |
1722 | #endif |
1723 | notify_id = coal->id; |
1724 | notify_flags = 0; |
1725 | } |
1726 | |
1727 | #if COALITION_DEBUG |
1728 | uint64_t cid = coal->id; |
1729 | uint32_t rc = coal_ref_count(coal); |
1730 | int ac = coal->active_count; |
1731 | int ct = coal->type; |
1732 | #endif |
1733 | coalition_unlock(coal); |
1734 | |
1735 | coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u,%s" , |
1736 | cid, coal_type_str(ct), rc, ac, do_notify ? " NOTIFY" : " " ); |
1737 | |
1738 | if (do_notify) { |
1739 | coalition_notify_user(id: notify_id, flags: notify_flags); |
1740 | } |
1741 | } |
1742 | |
1743 | /* Used for kernel_task, launchd, launchd's early boot tasks... */ |
1744 | kern_return_t |
1745 | coalitions_adopt_init_task(task_t task) |
1746 | { |
1747 | kern_return_t kr; |
1748 | kr = coalitions_adopt_task(coaltions: init_coalition, task); |
1749 | if (kr != KERN_SUCCESS) { |
1750 | panic("failed to adopt task %p into default coalition: %d" , task, kr); |
1751 | } |
1752 | return kr; |
1753 | } |
1754 | |
1755 | /* Used for forked corpses. */ |
1756 | kern_return_t |
1757 | coalitions_adopt_corpse_task(task_t task) |
1758 | { |
1759 | kern_return_t kr; |
1760 | kr = coalitions_adopt_task(coaltions: corpse_coalition, task); |
1761 | if (kr != KERN_SUCCESS) { |
1762 | panic("failed to adopt task %p into corpse coalition: %d" , task, kr); |
1763 | } |
1764 | return kr; |
1765 | } |
1766 | |
1767 | /* |
1768 | * coalition_adopt_task_internal |
1769 | * Condition: Coalition must be referenced and unlocked. Will fail if coalition |
1770 | * is already terminated. |
1771 | */ |
1772 | static kern_return_t |
1773 | coalition_adopt_task_internal(coalition_t coal, task_t task) |
1774 | { |
1775 | kern_return_t kr; |
1776 | |
1777 | if (task->coalition[coal->type]) { |
1778 | return KERN_ALREADY_IN_SET; |
1779 | } |
1780 | |
1781 | coalition_lock(coal); |
1782 | |
1783 | if (coal->reaped || coal->terminated) { |
1784 | coalition_unlock(coal); |
1785 | return KERN_TERMINATED; |
1786 | } |
1787 | |
1788 | kr = coal_call(coal, adopt_task, task); |
1789 | if (kr != KERN_SUCCESS) { |
1790 | goto out_unlock; |
1791 | } |
1792 | |
1793 | coal->active_count++; |
1794 | |
1795 | coal_ref_retain(coal); |
1796 | |
1797 | task->coalition[coal->type] = coal; |
1798 | |
1799 | out_unlock: |
1800 | #if COALITION_DEBUG |
1801 | (void)coal; /* need expression after label */ |
1802 | uint64_t cid = coal->id; |
1803 | uint32_t rc = coal_ref_count(coal); |
1804 | uint32_t ct = coal->type; |
1805 | #endif |
1806 | if (get_task_uniqueid(task) != UINT64_MAX) { |
1807 | /* On 32-bit targets, uniqueid will get truncated to 32 bits */ |
1808 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_ADOPT), |
1809 | coal->id, get_task_uniqueid(task)); |
1810 | } |
1811 | |
1812 | coalition_unlock(coal); |
1813 | |
1814 | coal_dbg("task:%d, id:%llu type:%s ref_count:%u, kr=%d" , |
1815 | task_pid(task), cid, coal_type_str(ct), rc, kr); |
1816 | return kr; |
1817 | } |
1818 | |
1819 | static kern_return_t |
1820 | coalition_remove_task_internal(task_t task, int type) |
1821 | { |
1822 | kern_return_t kr; |
1823 | |
1824 | coalition_t coal = task->coalition[type]; |
1825 | |
1826 | if (!coal) { |
1827 | return KERN_SUCCESS; |
1828 | } |
1829 | |
1830 | assert(coal->type == (uint32_t)type); |
1831 | |
1832 | coalition_lock(coal); |
1833 | |
1834 | kr = coal_call(coal, remove_task, task); |
1835 | |
1836 | #if COALITION_DEBUG |
1837 | uint64_t cid = coal->id; |
1838 | uint32_t rc = coal_ref_count(coal); |
1839 | int ac = coal->active_count; |
1840 | int ct = coal->type; |
1841 | #endif |
1842 | KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_COALITION, MACH_COALITION_REMOVE), |
1843 | coal->id, get_task_uniqueid(task)); |
1844 | coalition_unlock(coal); |
1845 | |
1846 | coal_dbg("id:%llu type:%s ref_count:%u, active_count:%u, kr=%d" , |
1847 | cid, coal_type_str(ct), rc, ac, kr); |
1848 | |
1849 | coalition_remove_active(coal); |
1850 | |
1851 | return kr; |
1852 | } |
1853 | |
1854 | /* |
1855 | * coalitions_adopt_task |
1856 | * Condition: All coalitions must be referenced and unlocked. |
1857 | * Will fail if any coalition is already terminated. |
1858 | */ |
1859 | kern_return_t |
1860 | coalitions_adopt_task(coalition_t *coals, task_t task) |
1861 | { |
1862 | int i; |
1863 | kern_return_t kr; |
1864 | |
1865 | if (!coals || coals[COALITION_TYPE_RESOURCE] == COALITION_NULL) { |
1866 | return KERN_INVALID_ARGUMENT; |
1867 | } |
1868 | |
1869 | /* verify that the incoming coalitions are what they say they are */ |
1870 | for (i = 0; i < COALITION_NUM_TYPES; i++) { |
1871 | if (coals[i] && coals[i]->type != (uint32_t)i) { |
1872 | return KERN_INVALID_ARGUMENT; |
1873 | } |
1874 | } |
1875 | |
1876 | for (i = 0; i < COALITION_NUM_TYPES; i++) { |
1877 | kr = KERN_SUCCESS; |
1878 | if (coals[i]) { |
1879 | kr = coalition_adopt_task_internal(coal: coals[i], task); |
1880 | } |
1881 | if (kr != KERN_SUCCESS) { |
1882 | /* dis-associate any coalitions that just adopted this task */ |
1883 | while (--i >= 0) { |
1884 | if (task->coalition[i]) { |
1885 | coalition_remove_task_internal(task, type: i); |
1886 | } |
1887 | } |
1888 | break; |
1889 | } |
1890 | } |
1891 | return kr; |
1892 | } |
1893 | |
1894 | /* |
1895 | * coalitions_remove_task |
1896 | * Condition: task must be referenced and UNLOCKED; all task's coalitions must be UNLOCKED |
1897 | */ |
1898 | kern_return_t |
1899 | coalitions_remove_task(task_t task) |
1900 | { |
1901 | kern_return_t kr; |
1902 | int i; |
1903 | |
1904 | task_lock(task); |
1905 | if (!task_is_coalition_member(task)) { |
1906 | task_unlock(task); |
1907 | return KERN_SUCCESS; |
1908 | } |
1909 | |
1910 | task_clear_coalition_member(task); |
1911 | task_unlock(task); |
1912 | |
1913 | for (i = 0; i < COALITION_NUM_TYPES; i++) { |
1914 | kr = coalition_remove_task_internal(task, type: i); |
1915 | assert(kr == KERN_SUCCESS); |
1916 | } |
1917 | |
1918 | return kr; |
1919 | } |
1920 | |
1921 | /* |
1922 | * task_release_coalitions |
1923 | * helper function to release references to all coalitions in which |
1924 | * 'task' is a member. |
1925 | */ |
1926 | void |
1927 | task_release_coalitions(task_t task) |
1928 | { |
1929 | int i; |
1930 | for (i = 0; i < COALITION_NUM_TYPES; i++) { |
1931 | if (task->coalition[i]) { |
1932 | coalition_release(coal: task->coalition[i]); |
1933 | } else if (i == COALITION_TYPE_RESOURCE) { |
1934 | panic("deallocating task %p was not a member of a resource coalition" , task); |
1935 | } |
1936 | } |
1937 | } |
1938 | |
1939 | /* |
1940 | * coalitions_set_roles |
1941 | * for each type of coalition, if the task is a member of a coalition of |
1942 | * that type (given in the coalitions parameter) then set the role of |
1943 | * the task within that that coalition. |
1944 | */ |
1945 | kern_return_t |
1946 | coalitions_set_roles(coalition_t coalitions[COALITION_NUM_TYPES], |
1947 | task_t task, int roles[COALITION_NUM_TYPES]) |
1948 | { |
1949 | kern_return_t kr = KERN_SUCCESS; |
1950 | int i; |
1951 | |
1952 | for (i = 0; i < COALITION_NUM_TYPES; i++) { |
1953 | if (!coalitions[i]) { |
1954 | continue; |
1955 | } |
1956 | coalition_lock(coalitions[i]); |
1957 | kr = coal_call(coalitions[i], set_taskrole, task, roles[i]); |
1958 | coalition_unlock(coalitions[i]); |
1959 | assert(kr == KERN_SUCCESS); |
1960 | } |
1961 | |
1962 | return kr; |
1963 | } |
1964 | |
1965 | /* |
1966 | * coalition_terminate_internal |
1967 | * Condition: Coalition must be referenced and UNLOCKED. |
1968 | */ |
1969 | kern_return_t |
1970 | coalition_request_terminate_internal(coalition_t coal) |
1971 | { |
1972 | assert(coal->type >= 0 && coal->type <= COALITION_TYPE_MAX); |
1973 | |
1974 | if (coal == init_coalition[coal->type]) { |
1975 | return KERN_DEFAULT_SET; |
1976 | } |
1977 | |
1978 | coalition_lock(coal); |
1979 | |
1980 | if (coal->reaped) { |
1981 | coalition_unlock(coal); |
1982 | return KERN_INVALID_NAME; |
1983 | } |
1984 | |
1985 | if (coal->terminated || coal->termrequested) { |
1986 | coalition_unlock(coal); |
1987 | return KERN_TERMINATED; |
1988 | } |
1989 | |
1990 | coal->termrequested = TRUE; |
1991 | |
1992 | boolean_t do_notify = FALSE; |
1993 | uint64_t note_id = 0; |
1994 | uint32_t note_flags = 0; |
1995 | |
1996 | if (coal->active_count == 0) { |
1997 | /* |
1998 | * We only notify once, when active_count reaches zero. |
1999 | * We just set termrequested to zero. If the active count |
2000 | * was already at zero (tasks died before we could request |
2001 | * a termination notification), we should notify. |
2002 | */ |
2003 | assert(!coal->terminated); |
2004 | coal->terminated = TRUE; |
2005 | |
2006 | assert(!coal->notified); |
2007 | |
2008 | coal->notified = TRUE; |
2009 | #if DEVELOPMENT || DEBUG |
2010 | do_notify = coal->should_notify; |
2011 | #else |
2012 | do_notify = TRUE; |
2013 | #endif |
2014 | note_id = coal->id; |
2015 | note_flags = 0; |
2016 | } |
2017 | |
2018 | coalition_unlock(coal); |
2019 | |
2020 | if (do_notify) { |
2021 | coalition_notify_user(id: note_id, flags: note_flags); |
2022 | } |
2023 | |
2024 | return KERN_SUCCESS; |
2025 | } |
2026 | |
2027 | /* |
2028 | * coalition_reap_internal |
2029 | * Condition: Coalition must be referenced and UNLOCKED. |
2030 | */ |
2031 | kern_return_t |
2032 | coalition_reap_internal(coalition_t coal) |
2033 | { |
2034 | assert(coal->type <= COALITION_TYPE_MAX); |
2035 | |
2036 | if (coal == init_coalition[coal->type]) { |
2037 | return KERN_DEFAULT_SET; |
2038 | } |
2039 | |
2040 | coalition_lock(coal); |
2041 | if (coal->reaped) { |
2042 | coalition_unlock(coal); |
2043 | return KERN_TERMINATED; |
2044 | } |
2045 | if (!coal->terminated) { |
2046 | coalition_unlock(coal); |
2047 | return KERN_FAILURE; |
2048 | } |
2049 | assert(coal->termrequested); |
2050 | if (coal->active_count > 0) { |
2051 | coalition_unlock(coal); |
2052 | return KERN_FAILURE; |
2053 | } |
2054 | |
2055 | coal->reaped = TRUE; |
2056 | |
2057 | coalition_unlock(coal); |
2058 | |
2059 | lck_mtx_lock(lck: &coalitions_list_lock); |
2060 | smr_hash_serialized_remove(&coalition_hash, &coal->link, |
2061 | &coal_hash_traits); |
2062 | if (smr_hash_serialized_should_shrink(smrh: &coalition_hash, |
2063 | COALITION_HASH_SIZE_MIN, size_factor: 2, count_factor: 1)) { |
2064 | /* shrink if more than 2 buckets per 1 element */ |
2065 | smr_hash_shrink_and_unlock(&coalition_hash, |
2066 | &coalitions_list_lock, &coal_hash_traits); |
2067 | } else { |
2068 | lck_mtx_unlock(lck: &coalitions_list_lock); |
2069 | } |
2070 | |
2071 | /* Release the list's reference and launchd's reference. */ |
2072 | coal_ref_release_live(coal); |
2073 | coalition_release(coal); |
2074 | |
2075 | return KERN_SUCCESS; |
2076 | } |
2077 | |
2078 | #if DEVELOPMENT || DEBUG |
2079 | int |
2080 | coalition_should_notify(coalition_t coal) |
2081 | { |
2082 | int should; |
2083 | if (!coal) { |
2084 | return -1; |
2085 | } |
2086 | coalition_lock(coal); |
2087 | should = coal->should_notify; |
2088 | coalition_unlock(coal); |
2089 | |
2090 | return should; |
2091 | } |
2092 | |
2093 | void |
2094 | coalition_set_notify(coalition_t coal, int notify) |
2095 | { |
2096 | if (!coal) { |
2097 | return; |
2098 | } |
2099 | coalition_lock(coal); |
2100 | coal->should_notify = !!notify; |
2101 | coalition_unlock(coal); |
2102 | } |
2103 | #endif |
2104 | |
2105 | void |
2106 | coalitions_init(void) |
2107 | { |
2108 | kern_return_t kr; |
2109 | int i; |
2110 | const struct coalition_type *ctype; |
2111 | |
2112 | smr_hash_init(smrh: &coalition_hash, COALITION_HASH_SIZE_MIN); |
2113 | |
2114 | init_task_ledgers(); |
2115 | |
2116 | init_coalition_ledgers(); |
2117 | |
2118 | for (i = 0, ctype = &s_coalition_types[0]; i < COALITION_NUM_TYPES; ctype++, i++) { |
2119 | /* verify the entry in the global coalition types array */ |
2120 | if (ctype->type != i || |
2121 | !ctype->init || |
2122 | !ctype->dealloc || |
2123 | !ctype->adopt_task || |
2124 | !ctype->remove_task) { |
2125 | panic("%s: Malformed coalition type %s(%d) in slot for type:%s(%d)" , |
2126 | __func__, coal_type_str(ctype->type), ctype->type, coal_type_str(i), i); |
2127 | } |
2128 | if (!ctype->has_default) { |
2129 | continue; |
2130 | } |
2131 | kr = coalition_create_internal(type: ctype->type, COALITION_ROLE_SYSTEM, TRUE, FALSE, out: &init_coalition[ctype->type], NULL); |
2132 | if (kr != KERN_SUCCESS) { |
2133 | panic("%s: could not create init %s coalition: kr:%d" , |
2134 | __func__, coal_type_str(i), kr); |
2135 | } |
2136 | if (i == COALITION_TYPE_RESOURCE) { |
2137 | assert(COALITION_ID_KERNEL == init_coalition[ctype->type]->id); |
2138 | } |
2139 | kr = coalition_create_internal(type: ctype->type, COALITION_ROLE_SYSTEM, FALSE, FALSE, out: &corpse_coalition[ctype->type], NULL); |
2140 | if (kr != KERN_SUCCESS) { |
2141 | panic("%s: could not create corpse %s coalition: kr:%d" , |
2142 | __func__, coal_type_str(i), kr); |
2143 | } |
2144 | } |
2145 | |
2146 | /* "Leak" our reference to the global object */ |
2147 | } |
2148 | |
2149 | /* |
2150 | * BSD Kernel interface functions |
2151 | * |
2152 | */ |
2153 | static void |
2154 | coalition_fill_procinfo(struct coalition *coal, |
2155 | struct procinfo_coalinfo *coalinfo) |
2156 | { |
2157 | coalinfo->coalition_id = coal->id; |
2158 | coalinfo->coalition_type = coal->type; |
2159 | coalinfo->coalition_tasks = coalition_get_task_count(coal); |
2160 | } |
2161 | |
2162 | |
2163 | size_t |
2164 | coalitions_get_list(int type, struct procinfo_coalinfo *coal_list, size_t list_sz) |
2165 | { |
2166 | size_t ncoals = 0; |
2167 | struct coalition *coal; |
2168 | |
2169 | lck_mtx_lock(lck: &coalitions_list_lock); |
2170 | smr_hash_foreach(coal, &coalition_hash, &coal_hash_traits) { |
2171 | if (!coal->reaped && (type < 0 || type == (int)coal->type)) { |
2172 | if (coal_list && ncoals < list_sz) { |
2173 | coalition_fill_procinfo(coal, coalinfo: &coal_list[ncoals]); |
2174 | } |
2175 | ++ncoals; |
2176 | } |
2177 | } |
2178 | lck_mtx_unlock(lck: &coalitions_list_lock); |
2179 | |
2180 | return ncoals; |
2181 | } |
2182 | |
2183 | /* |
2184 | * Return the coaltion of the given type to which the task belongs. |
2185 | */ |
2186 | coalition_t |
2187 | task_get_coalition(task_t task, int coal_type) |
2188 | { |
2189 | coalition_t c; |
2190 | |
2191 | if (task == NULL || coal_type > COALITION_TYPE_MAX) { |
2192 | return COALITION_NULL; |
2193 | } |
2194 | |
2195 | c = task->coalition[coal_type]; |
2196 | assert(c == COALITION_NULL || (int)c->type == coal_type); |
2197 | return c; |
2198 | } |
2199 | |
2200 | /* |
2201 | * Report if the given task is the leader of the given jetsam coalition. |
2202 | */ |
2203 | boolean_t |
2204 | coalition_is_leader(task_t task, coalition_t coal) |
2205 | { |
2206 | boolean_t ret = FALSE; |
2207 | |
2208 | if (coal != COALITION_NULL) { |
2209 | coalition_lock(coal); |
2210 | |
2211 | ret = (coal->type == COALITION_TYPE_JETSAM && coal->j.leader == task); |
2212 | |
2213 | coalition_unlock(coal); |
2214 | } |
2215 | |
2216 | return ret; |
2217 | } |
2218 | |
2219 | kern_return_t |
2220 | coalition_iterate_stackshot(coalition_iterate_fn_t callout, void *arg, uint32_t coalition_type) |
2221 | { |
2222 | coalition_t coal; |
2223 | int i = 0; |
2224 | |
2225 | smr_hash_foreach(coal, &coalition_hash, &coal_hash_traits) { |
2226 | if (coal == NULL || !ml_validate_nofault(virtsrc: (vm_offset_t)coal, size: sizeof(struct coalition))) { |
2227 | return KERN_FAILURE; |
2228 | } |
2229 | |
2230 | if (coalition_type == coal->type) { |
2231 | callout(arg, i++, coal); |
2232 | } |
2233 | } |
2234 | |
2235 | return KERN_SUCCESS; |
2236 | } |
2237 | |
2238 | task_t |
2239 | kdp_coalition_get_leader(coalition_t coal) |
2240 | { |
2241 | if (!coal) { |
2242 | return TASK_NULL; |
2243 | } |
2244 | |
2245 | if (coal->type == COALITION_TYPE_JETSAM) { |
2246 | return coal->j.leader; |
2247 | } |
2248 | return TASK_NULL; |
2249 | } |
2250 | |
2251 | task_t |
2252 | coalition_get_leader(coalition_t coal) |
2253 | { |
2254 | task_t leader = TASK_NULL; |
2255 | |
2256 | if (!coal) { |
2257 | return TASK_NULL; |
2258 | } |
2259 | |
2260 | coalition_lock(coal); |
2261 | if (coal->type != COALITION_TYPE_JETSAM) { |
2262 | goto out_unlock; |
2263 | } |
2264 | |
2265 | leader = coal->j.leader; |
2266 | if (leader != TASK_NULL) { |
2267 | task_reference(leader); |
2268 | } |
2269 | |
2270 | out_unlock: |
2271 | coalition_unlock(coal); |
2272 | return leader; |
2273 | } |
2274 | |
2275 | |
2276 | int |
2277 | coalition_get_task_count(coalition_t coal) |
2278 | { |
2279 | int ntasks = 0; |
2280 | struct queue_entry *qe; |
2281 | if (!coal) { |
2282 | return 0; |
2283 | } |
2284 | |
2285 | coalition_lock(coal); |
2286 | switch (coal->type) { |
2287 | case COALITION_TYPE_RESOURCE: |
2288 | qe_foreach(qe, &coal->r.tasks) |
2289 | ntasks++; |
2290 | break; |
2291 | case COALITION_TYPE_JETSAM: |
2292 | if (coal->j.leader) { |
2293 | ntasks++; |
2294 | } |
2295 | qe_foreach(qe, &coal->j.other) |
2296 | ntasks++; |
2297 | qe_foreach(qe, &coal->j.extensions) |
2298 | ntasks++; |
2299 | qe_foreach(qe, &coal->j.services) |
2300 | ntasks++; |
2301 | break; |
2302 | default: |
2303 | break; |
2304 | } |
2305 | coalition_unlock(coal); |
2306 | |
2307 | return ntasks; |
2308 | } |
2309 | |
2310 | |
2311 | static uint64_t |
2312 | (queue_t list, int type, int *ntasks) |
2313 | { |
2314 | task_t task; |
2315 | uint64_t bytes = 0; |
2316 | |
2317 | qe_foreach_element(task, list, task_coalition[type]) { |
2318 | bytes += get_task_phys_footprint(task); |
2319 | coal_dbg(" [%d] task_pid:%d, type:%d, footprint:%lld" , |
2320 | *ntasks, task_pid(task), type, bytes); |
2321 | *ntasks += 1; |
2322 | } |
2323 | |
2324 | return bytes; |
2325 | } |
2326 | |
2327 | uint64_t |
2328 | coalition_get_page_count(coalition_t coal, int *ntasks) |
2329 | { |
2330 | uint64_t bytes = 0; |
2331 | int num_tasks = 0; |
2332 | |
2333 | if (ntasks) { |
2334 | *ntasks = 0; |
2335 | } |
2336 | if (!coal) { |
2337 | return bytes; |
2338 | } |
2339 | |
2340 | coalition_lock(coal); |
2341 | |
2342 | switch (coal->type) { |
2343 | case COALITION_TYPE_RESOURCE: |
2344 | bytes += i_get_list_footprint(list: &coal->r.tasks, COALITION_TYPE_RESOURCE, ntasks: &num_tasks); |
2345 | break; |
2346 | case COALITION_TYPE_JETSAM: |
2347 | if (coal->j.leader) { |
2348 | bytes += get_task_phys_footprint(coal->j.leader); |
2349 | num_tasks = 1; |
2350 | } |
2351 | bytes += i_get_list_footprint(list: &coal->j.extensions, COALITION_TYPE_JETSAM, ntasks: &num_tasks); |
2352 | bytes += i_get_list_footprint(list: &coal->j.services, COALITION_TYPE_JETSAM, ntasks: &num_tasks); |
2353 | bytes += i_get_list_footprint(list: &coal->j.other, COALITION_TYPE_JETSAM, ntasks: &num_tasks); |
2354 | break; |
2355 | default: |
2356 | break; |
2357 | } |
2358 | |
2359 | coalition_unlock(coal); |
2360 | |
2361 | if (ntasks) { |
2362 | *ntasks = num_tasks; |
2363 | } |
2364 | |
2365 | return bytes / PAGE_SIZE_64; |
2366 | } |
2367 | |
2368 | struct coal_sort_s { |
2369 | int pid; |
2370 | int usr_order; |
2371 | uint64_t bytes; |
2372 | }; |
2373 | |
2374 | /* |
2375 | * return < 0 for a < b |
2376 | * 0 for a == b |
2377 | * > 0 for a > b |
2378 | */ |
2379 | typedef int (*cmpfunc_t)(const void *a, const void *b); |
2380 | |
2381 | extern void |
2382 | qsort(void *a, size_t n, size_t es, cmpfunc_t cmp); |
2383 | |
2384 | static int |
2385 | dflt_cmp(const void *a, const void *b) |
2386 | { |
2387 | const struct coal_sort_s *csA = (const struct coal_sort_s *)a; |
2388 | const struct coal_sort_s *csB = (const struct coal_sort_s *)b; |
2389 | |
2390 | /* |
2391 | * if both A and B are equal, use a memory descending sort |
2392 | */ |
2393 | if (csA->usr_order == csB->usr_order) { |
2394 | return (int)((int64_t)csB->bytes - (int64_t)csA->bytes); |
2395 | } |
2396 | |
2397 | /* otherwise, return the relationship between user specified orders */ |
2398 | return csA->usr_order - csB->usr_order; |
2399 | } |
2400 | |
2401 | static int |
2402 | mem_asc_cmp(const void *a, const void *b) |
2403 | { |
2404 | const struct coal_sort_s *csA = (const struct coal_sort_s *)a; |
2405 | const struct coal_sort_s *csB = (const struct coal_sort_s *)b; |
2406 | |
2407 | return (int)((int64_t)csA->bytes - (int64_t)csB->bytes); |
2408 | } |
2409 | |
2410 | static int |
2411 | mem_dec_cmp(const void *a, const void *b) |
2412 | { |
2413 | const struct coal_sort_s *csA = (const struct coal_sort_s *)a; |
2414 | const struct coal_sort_s *csB = (const struct coal_sort_s *)b; |
2415 | |
2416 | return (int)((int64_t)csB->bytes - (int64_t)csA->bytes); |
2417 | } |
2418 | |
2419 | static int |
2420 | usr_asc_cmp(const void *a, const void *b) |
2421 | { |
2422 | const struct coal_sort_s *csA = (const struct coal_sort_s *)a; |
2423 | const struct coal_sort_s *csB = (const struct coal_sort_s *)b; |
2424 | |
2425 | return csA->usr_order - csB->usr_order; |
2426 | } |
2427 | |
2428 | static int |
2429 | usr_dec_cmp(const void *a, const void *b) |
2430 | { |
2431 | const struct coal_sort_s *csA = (const struct coal_sort_s *)a; |
2432 | const struct coal_sort_s *csB = (const struct coal_sort_s *)b; |
2433 | |
2434 | return csB->usr_order - csA->usr_order; |
2435 | } |
2436 | |
2437 | /* avoid dynamic allocation in this path */ |
2438 | #define MAX_SORTED_PIDS 80 |
2439 | |
2440 | static int |
2441 | coalition_get_sort_list(coalition_t coal, int sort_order, queue_t list, |
2442 | struct coal_sort_s *sort_array, int array_sz) |
2443 | { |
2444 | int ntasks = 0; |
2445 | task_t task; |
2446 | |
2447 | assert(sort_array != NULL); |
2448 | |
2449 | if (array_sz <= 0) { |
2450 | return 0; |
2451 | } |
2452 | |
2453 | if (!list) { |
2454 | /* |
2455 | * this function will only be called with a NULL |
2456 | * list for JETSAM-type coalitions, and is intended |
2457 | * to investigate the leader process |
2458 | */ |
2459 | if (coal->type != COALITION_TYPE_JETSAM || |
2460 | coal->j.leader == TASK_NULL) { |
2461 | return 0; |
2462 | } |
2463 | sort_array[0].pid = task_pid(task: coal->j.leader); |
2464 | switch (sort_order) { |
2465 | case COALITION_SORT_DEFAULT: |
2466 | sort_array[0].usr_order = 0; |
2467 | OS_FALLTHROUGH; |
2468 | case COALITION_SORT_MEM_ASC: |
2469 | case COALITION_SORT_MEM_DEC: |
2470 | sort_array[0].bytes = get_task_phys_footprint(coal->j.leader); |
2471 | break; |
2472 | case COALITION_SORT_USER_ASC: |
2473 | case COALITION_SORT_USER_DEC: |
2474 | sort_array[0].usr_order = 0; |
2475 | break; |
2476 | default: |
2477 | break; |
2478 | } |
2479 | return 1; |
2480 | } |
2481 | |
2482 | qe_foreach_element(task, list, task_coalition[coal->type]) { |
2483 | if (ntasks >= array_sz) { |
2484 | printf(format: "WARNING: more than %d pids in coalition %llu\n" , |
2485 | MAX_SORTED_PIDS, coal->id); |
2486 | break; |
2487 | } |
2488 | |
2489 | sort_array[ntasks].pid = task_pid(task); |
2490 | |
2491 | switch (sort_order) { |
2492 | case COALITION_SORT_DEFAULT: |
2493 | sort_array[ntasks].usr_order = 0; |
2494 | OS_FALLTHROUGH; |
2495 | case COALITION_SORT_MEM_ASC: |
2496 | case COALITION_SORT_MEM_DEC: |
2497 | sort_array[ntasks].bytes = get_task_phys_footprint(task); |
2498 | break; |
2499 | case COALITION_SORT_USER_ASC: |
2500 | case COALITION_SORT_USER_DEC: |
2501 | sort_array[ntasks].usr_order = 0; |
2502 | break; |
2503 | default: |
2504 | break; |
2505 | } |
2506 | |
2507 | ntasks++; |
2508 | } |
2509 | |
2510 | return ntasks; |
2511 | } |
2512 | |
2513 | int |
2514 | coalition_get_pid_list(coalition_t coal, uint32_t rolemask, int sort_order, |
2515 | int *pid_list, int list_sz) |
2516 | { |
2517 | struct i_jetsam_coalition *cj; |
2518 | int ntasks = 0; |
2519 | cmpfunc_t cmp_func = NULL; |
2520 | struct coal_sort_s sort_array[MAX_SORTED_PIDS] = { {0, 0, 0} }; /* keep to < 2k */ |
2521 | |
2522 | if (!coal || |
2523 | !(rolemask & COALITION_ROLEMASK_ALLROLES) || |
2524 | !pid_list || list_sz < 1) { |
2525 | coal_dbg("Invalid parameters: coal:%p, type:%d, rolemask:0x%x, " |
2526 | "pid_list:%p, list_sz:%d" , coal, coal ? coal->type : -1, |
2527 | rolemask, pid_list, list_sz); |
2528 | return -EINVAL; |
2529 | } |
2530 | |
2531 | switch (sort_order) { |
2532 | case COALITION_SORT_NOSORT: |
2533 | cmp_func = NULL; |
2534 | break; |
2535 | case COALITION_SORT_DEFAULT: |
2536 | cmp_func = dflt_cmp; |
2537 | break; |
2538 | case COALITION_SORT_MEM_ASC: |
2539 | cmp_func = mem_asc_cmp; |
2540 | break; |
2541 | case COALITION_SORT_MEM_DEC: |
2542 | cmp_func = mem_dec_cmp; |
2543 | break; |
2544 | case COALITION_SORT_USER_ASC: |
2545 | cmp_func = usr_asc_cmp; |
2546 | break; |
2547 | case COALITION_SORT_USER_DEC: |
2548 | cmp_func = usr_dec_cmp; |
2549 | break; |
2550 | default: |
2551 | return -ENOTSUP; |
2552 | } |
2553 | |
2554 | coalition_lock(coal); |
2555 | |
2556 | if (coal->type == COALITION_TYPE_RESOURCE) { |
2557 | ntasks += coalition_get_sort_list(coal, sort_order, list: &coal->r.tasks, |
2558 | sort_array, MAX_SORTED_PIDS); |
2559 | goto unlock_coal; |
2560 | } |
2561 | |
2562 | cj = &coal->j; |
2563 | |
2564 | if (rolemask & COALITION_ROLEMASK_UNDEF) { |
2565 | ntasks += coalition_get_sort_list(coal, sort_order, list: &cj->other, |
2566 | sort_array: sort_array + ntasks, |
2567 | MAX_SORTED_PIDS - ntasks); |
2568 | } |
2569 | |
2570 | if (rolemask & COALITION_ROLEMASK_XPC) { |
2571 | ntasks += coalition_get_sort_list(coal, sort_order, list: &cj->services, |
2572 | sort_array: sort_array + ntasks, |
2573 | MAX_SORTED_PIDS - ntasks); |
2574 | } |
2575 | |
2576 | if (rolemask & COALITION_ROLEMASK_EXT) { |
2577 | ntasks += coalition_get_sort_list(coal, sort_order, list: &cj->extensions, |
2578 | sort_array: sort_array + ntasks, |
2579 | MAX_SORTED_PIDS - ntasks); |
2580 | } |
2581 | |
2582 | if (rolemask & COALITION_ROLEMASK_LEADER) { |
2583 | ntasks += coalition_get_sort_list(coal, sort_order, NULL, |
2584 | sort_array: sort_array + ntasks, |
2585 | MAX_SORTED_PIDS - ntasks); |
2586 | } |
2587 | |
2588 | unlock_coal: |
2589 | coalition_unlock(coal); |
2590 | |
2591 | /* sort based on the chosen criterion (no sense sorting 1 item) */ |
2592 | if (cmp_func && ntasks > 1) { |
2593 | qsort(a: sort_array, n: ntasks, es: sizeof(struct coal_sort_s), cmp: cmp_func); |
2594 | } |
2595 | |
2596 | for (int i = 0; i < ntasks; i++) { |
2597 | if (i >= list_sz) { |
2598 | break; |
2599 | } |
2600 | coal_dbg(" [%d] PID:%d, footprint:%lld, usr_order:%d" , |
2601 | i, sort_array[i].pid, sort_array[i].bytes, |
2602 | sort_array[i].usr_order); |
2603 | pid_list[i] = sort_array[i].pid; |
2604 | } |
2605 | |
2606 | return ntasks; |
2607 | } |
2608 | |
2609 | static void |
2610 | mark_coalition_member_as_swappable(__unused coalition_t coal, __unused void *ctx, task_t task) |
2611 | { |
2612 | vm_task_set_selfdonate_pages(task, true); |
2613 | } |
2614 | |
2615 | void |
2616 | coalition_mark_swappable(coalition_t coal) |
2617 | { |
2618 | struct i_jetsam_coalition *cj = NULL; |
2619 | |
2620 | coalition_lock(coal); |
2621 | assert(coal && coal->type == COALITION_TYPE_JETSAM); |
2622 | |
2623 | cj = &coal->j; |
2624 | cj->swap_enabled = true; |
2625 | |
2626 | i_coal_jetsam_iterate_tasks(coal, NULL, callback: mark_coalition_member_as_swappable); |
2627 | |
2628 | coalition_unlock(coal); |
2629 | } |
2630 | |
2631 | bool |
2632 | coalition_is_swappable(coalition_t coal) |
2633 | { |
2634 | struct i_jetsam_coalition *cj = NULL; |
2635 | |
2636 | coalition_lock(coal); |
2637 | assert(coal && coal->type == COALITION_TYPE_JETSAM); |
2638 | |
2639 | cj = &coal->j; |
2640 | bool enabled = cj->swap_enabled; |
2641 | |
2642 | coalition_unlock(coal); |
2643 | |
2644 | return enabled; |
2645 | } |
2646 | |