1/*
2 * Copyright (c) 2010-2020 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31
32#include <kern/kern_types.h>
33#include <kern/ledger.h>
34#include <kern/kalloc.h>
35#include <kern/task.h>
36#include <kern/thread.h>
37#include <kern/coalition.h>
38
39#include <kern/processor.h>
40#include <kern/machine.h>
41#include <kern/queue.h>
42#include <kern/policy_internal.h>
43
44#include <sys/errno.h>
45
46#include <libkern/OSAtomic.h>
47#include <mach/mach_types.h>
48#include <os/overflow.h>
49
50#include <vm/pmap.h>
51
52/*
53 * Ledger entry flags. Bits in second nibble (masked by 0xF0) are used for
54 * ledger actions (LEDGER_ACTION_BLOCK, etc).
55 */
56#define LF_ENTRY_ACTIVE 0x0001 /* entry is active if set */
57#define LF_WAKE_NEEDED 0x0100 /* one or more threads are asleep */
58#define LF_WAKE_INPROGRESS 0x0200 /* the wait queue is being processed */
59#define LF_REFILL_SCHEDULED 0x0400 /* a refill timer has been set */
60#define LF_REFILL_INPROGRESS 0x0800 /* the ledger is being refilled */
61#define LF_CALLED_BACK 0x1000 /* callback was called for balance in deficit */
62#define LF_WARNED 0x2000 /* callback was called for balance warning */
63#define LF_TRACKING_MAX 0x4000 /* track max balance. Exclusive w.r.t refill */
64#define LF_PANIC_ON_NEGATIVE 0x8000 /* panic if it goes negative */
65#define LF_TRACK_CREDIT_ONLY 0x10000 /* only update "credit" */
66#define LF_DIAG_WARNED 0x20000 /* callback was called for balance diag */
67#define LF_DIAG_DISABLED 0x40000 /* diagnostics threshold are disabled at the moment */
68
69
70/*
71 * Ledger entry IDs are actually a tuple of (size, offset).
72 * For backwards compatibility, they're stored in an int.
73 * Size is stored in the upper 16 bits, and offset is stored in the lower 16 bits.
74 *
75 * Use the ENTRY_ID_SIZE and ENTRY_ID_OFFSET macros to extract size and offset.
76 */
77#define ENTRY_ID_SIZE_SHIFT 16
78#define ENTRY_ID_OFFSET_MASK ((1 << ENTRY_ID_SIZE_SHIFT) - 1)
79#define ENTRY_ID_OFFSET(x) ((x) & (ENTRY_ID_OFFSET_MASK))
80#define ENTRY_ID_SIZE_MASK (ENTRY_ID_OFFSET_MASK << ENTRY_ID_SIZE_SHIFT)
81#define ENTRY_ID_SIZE(x) ((((uint32_t) (x)) & (ENTRY_ID_SIZE_MASK)) >> ENTRY_ID_SIZE_SHIFT)
82_Static_assert(((sizeof(struct ledger_entry_small) << ENTRY_ID_SIZE_SHIFT) | (UINT16_MAX / sizeof(struct ledger_entry_small))) > 0, "Valid ledger index < 0");
83_Static_assert(((sizeof(struct ledger_entry) << ENTRY_ID_SIZE_SHIFT) | (UINT16_MAX / sizeof(struct ledger_entry_small))) > 0, "Valid ledger index < 0");
84_Static_assert(sizeof(int) * 8 >= ENTRY_ID_SIZE_SHIFT * 2, "Ledger indices don't fit in an int.");
85#define MAX_LEDGER_ENTRIES (UINT16_MAX / sizeof(struct ledger_entry_small))
86
87#define LEDGER_DIAG_MEM_THRESHOLD_SHIFT 20
88#define LEDGER_DIAG_MEM_AMOUNT_TO_THRESHOLD(X) ((X) >> (LEDGER_DIAG_MEM_THRESHOLD_SHIFT))
89#define LEDGER_DIAG_MEM_AMOUNT_FROM_THRESHOLD(X) (((ledger_amount_t)(X)) << (LEDGER_DIAG_MEM_THRESHOLD_SHIFT))
90
91/* These features can fit in a small ledger entry. All others require a full size ledger entry */
92#define LEDGER_ENTRY_SMALL_FLAGS (LEDGER_ENTRY_ALLOW_PANIC_ON_NEGATIVE | LEDGER_ENTRY_ALLOW_INACTIVE)
93
94/* Turn on to debug invalid ledger accesses */
95#if MACH_ASSERT
96#define PANIC_ON_INVALID_LEDGER_ACCESS 1
97#endif /* MACH_ASSERT */
98
99static inline volatile uint32_t *
100get_entry_flags(ledger_t l, int index)
101{
102 assert(l != NULL);
103
104 uint16_t size, offset;
105 size = ENTRY_ID_SIZE(index);
106 offset = ENTRY_ID_OFFSET(index);
107 struct ledger_entry_small *les = &l->l_entries[offset];
108 if (size == sizeof(struct ledger_entry)) {
109 return &((struct ledger_entry *)les)->le_flags;
110 } else if (size == sizeof(struct ledger_entry_small)) {
111 return &les->les_flags;
112 } else {
113 panic("Unknown ledger entry size! ledger=%p, index=0x%x, entry_size=%d\n", l, index, size);
114 }
115}
116
117#if PANIC_ON_INVALID_LEDGER_ACCESS
118#define INVALID_LEDGER_ACCESS(l, e) if ((e) != -1) panic("Invalid ledger access: ledger=%p, entry=0x%x, entry_size=0x%x, entry_offset=0x%x\n", \
119 (l), (e), (ENTRY_ID_SIZE((e))), ENTRY_ID_OFFSET((e)));
120#else
121#define INVALID_LEDGER_ACCESS(l, e)
122#endif /* PANIC_ON_INVALID_LEDGER_ACCESS */
123
124/* Determine whether a ledger entry exists */
125static inline bool
126is_entry_valid(ledger_t l, int entry)
127{
128 uint32_t size, offset, end_offset;
129 size = ENTRY_ID_SIZE(entry);
130 offset = ENTRY_ID_OFFSET(entry);
131 if (l == NULL) {
132 return false;
133 }
134 if (os_mul_overflow(offset, sizeof(struct ledger_entry_small), &offset) || offset >= l->l_size) {
135 INVALID_LEDGER_ACCESS(l, entry);
136 return false;
137 }
138 if (os_add_overflow(size, offset, &end_offset) || end_offset > l->l_size) {
139 INVALID_LEDGER_ACCESS(l, entry);
140 return false;
141 }
142 return true;
143}
144
145static inline bool
146is_entry_active(ledger_t l, int entry)
147{
148 uint32_t flags = *get_entry_flags(l, index: entry);
149 if ((flags & LF_ENTRY_ACTIVE) != LF_ENTRY_ACTIVE) {
150 return false;
151 }
152
153 return true;
154}
155
156static inline bool
157is_entry_valid_and_active(ledger_t l, int entry)
158{
159 return is_entry_valid(l, entry) && is_entry_active(l, entry);
160}
161
162#define ASSERT(a) assert(a)
163
164#ifdef LEDGER_DEBUG
165int ledger_debug = 0;
166
167#define lprintf(a) if (ledger_debug) { \
168 printf("%lld ", abstime_to_nsecs(mach_absolute_time() / 1000000)); \
169 printf a ; \
170}
171#else
172#define lprintf(a)
173#endif
174
175struct ledger_callback {
176 ledger_callback_t lc_func;
177 const void *lc_param0;
178 const void *lc_param1;
179};
180
181struct entry_template {
182 char et_key[LEDGER_NAME_MAX];
183 char et_group[LEDGER_NAME_MAX];
184 char et_units[LEDGER_NAME_MAX];
185 uint32_t et_flags;
186 uint16_t et_size;
187 uint16_t et_offset;
188 struct ledger_callback *et_callback;
189};
190
191LCK_GRP_DECLARE(ledger_lck_grp, "ledger");
192os_refgrp_decl(static, ledger_refgrp, "ledger", NULL);
193
194/*
195 * Modifying the reference count, table size, table contents, lt_next_offset, or lt_entries_lut,
196 * requires holding the lt_lock. Modfying the table address requires both
197 * lt_lock and setting the inuse bit. This means that the lt_entries field can
198 * be safely dereferenced if you hold either the lock or the inuse bit. The
199 * inuse bit exists solely to allow us to swap in a new, larger entries
200 * table without requiring a full lock to be acquired on each lookup.
201 * Accordingly, the inuse bit should never be held for longer than it takes
202 * to extract a value from the table - i.e., 2 or 3 memory references.
203 */
204struct ledger_template {
205 const char *lt_name;
206 int lt_refs;
207 volatile uint32_t lt_inuse;
208 lck_mtx_t lt_lock;
209 zone_t lt_zone;
210 bool lt_initialized;
211 uint16_t lt_next_offset;
212 uint16_t lt_cnt;
213 uint16_t lt_table_size;
214 struct entry_template *lt_entries;
215 /* Lookup table to go from entry_offset to index in the lt_entries table. */
216 uint16_t *lt_entries_lut;
217};
218
219static inline uint16_t
220ledger_template_entries_lut_size(uint16_t lt_table_size)
221{
222 /*
223 * The lookup table needs to be big enough to store lt_table_size entries of the largest
224 * entry size (struct ledger_entry) given a stride of the smallest entry size (struct ledger_entry_small)
225 */
226 if (os_mul_overflow(lt_table_size, (sizeof(struct ledger_entry) / sizeof(struct ledger_entry_small)), &lt_table_size)) {
227 /*
228 * This means MAX_LEDGER_ENTRIES is misconfigured or
229 * someone has accidently passed in an lt_table_size that is > MAX_LEDGER_ENTRIES
230 */
231 panic("Attempt to create a lookup table for a ledger template with too many entries. lt_table_size=%u, MAX_LEDGER_ENTRIES=%lu\n", lt_table_size, MAX_LEDGER_ENTRIES);
232 }
233 return lt_table_size;
234}
235
236#define template_lock(template) lck_mtx_lock(&(template)->lt_lock)
237#define template_unlock(template) lck_mtx_unlock(&(template)->lt_lock)
238
239#define TEMPLATE_INUSE(s, t) { \
240 s = splsched(); \
241 while (OSCompareAndSwap(0, 1, &((t)->lt_inuse))) \
242 ; \
243}
244
245#define TEMPLATE_IDLE(s, t) { \
246 (t)->lt_inuse = 0; \
247 splx(s); \
248}
249
250static int ledger_cnt = 0;
251/* ledger ast helper functions */
252static uint32_t ledger_check_needblock(ledger_t l, uint64_t now);
253static kern_return_t ledger_perform_blocking(ledger_t l);
254static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit);
255static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit);
256
257static void ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
258 int entry);
259#if DEBUG || DEVELOPMENT
260static inline bool ledger_is_diag_threshold_enabled_internal(struct ledger_entry *le);
261#endif
262#if 0
263static void
264debug_callback(const void *p0, __unused const void *p1)
265{
266 printf("ledger: resource exhausted [%s] for task %p\n",
267 (const char *)p0, p1);
268}
269#endif
270
271/************************************/
272
273static uint64_t
274abstime_to_nsecs(uint64_t abstime)
275{
276 uint64_t nsecs;
277
278 absolutetime_to_nanoseconds(abstime, result: &nsecs);
279 return nsecs;
280}
281
282static uint64_t
283nsecs_to_abstime(uint64_t nsecs)
284{
285 uint64_t abstime;
286
287 nanoseconds_to_absolutetime(nanoseconds: nsecs, result: &abstime);
288 return abstime;
289}
290
291static const uint16_t *
292ledger_entry_to_template_idx(ledger_template_t template, int index)
293{
294 uint16_t offset = ENTRY_ID_OFFSET(index);
295 if (offset / sizeof(struct ledger_entry_small) >= template->lt_cnt) {
296 return NULL;
297 }
298
299 return &template->lt_entries_lut[offset];
300}
301
302/*
303 * Convert the id to a ledger entry.
304 * It's the callers responsibility to ensure the id is valid and a full size
305 * ledger entry.
306 */
307static struct ledger_entry *
308ledger_entry_identifier_to_entry(ledger_t ledger, int id)
309{
310 assert(is_entry_valid(ledger, id));
311 assert(ENTRY_ID_SIZE(id) == sizeof(struct ledger_entry));
312 return (struct ledger_entry *) &ledger->l_entries[ENTRY_ID_OFFSET(id)];
313}
314
315
316ledger_template_t
317ledger_template_create(const char *name)
318{
319 ledger_template_t template;
320
321 template = kalloc_type(struct ledger_template, Z_WAITOK | Z_ZERO | Z_NOFAIL);
322 template->lt_name = name;
323 template->lt_refs = 1;
324 template->lt_table_size = 1;
325 lck_mtx_init(lck: &template->lt_lock, grp: &ledger_lck_grp, LCK_ATTR_NULL);
326
327 template->lt_entries = kalloc_type(struct entry_template,
328 template->lt_table_size, Z_WAITOK | Z_ZERO);
329 if (template->lt_entries == NULL) {
330 kfree_type(struct ledger_template, template);
331 template = NULL;
332 }
333 template->lt_entries_lut = kalloc_type(uint16_t, ledger_template_entries_lut_size(template->lt_table_size),
334 Z_WAITOK | Z_ZERO);
335 if (template->lt_entries_lut == NULL) {
336 kfree_type(struct entry_template, template->lt_entries);
337 kfree_type(struct ledger_template, template);
338 template = NULL;
339 }
340
341 return template;
342}
343
344ledger_template_t
345ledger_template_copy(ledger_template_t template, const char *name)
346{
347 struct entry_template * new_entries = NULL;
348 uint16_t *new_entries_lut = NULL;
349 size_t new_entries_lut_size = 0;
350 ledger_template_t new_template = ledger_template_create(name);
351
352 if (new_template == NULL) {
353 return new_template;
354 }
355
356 template_lock(template);
357 assert(template->lt_initialized);
358
359 new_entries = kalloc_type(struct entry_template, template->lt_table_size,
360 Z_WAITOK | Z_ZERO);
361
362 if (new_entries == NULL) {
363 /* Tear down the new template; we've failed. :( */
364 ledger_template_dereference(template: new_template);
365 new_template = NULL;
366 goto out;
367 }
368 new_entries_lut_size = ledger_template_entries_lut_size(lt_table_size: template->lt_table_size);
369
370 new_entries_lut = kalloc_type(uint16_t, new_entries_lut_size,
371 Z_WAITOK | Z_ZERO);
372 if (new_entries_lut == NULL) {
373 /* Tear down the new template; we've failed. :( */
374 ledger_template_dereference(template: new_template);
375 new_template = NULL;
376 goto out;
377 }
378
379 /* Copy the template entries. */
380 bcopy(src: template->lt_entries, dst: new_entries, n: sizeof(struct entry_template) * template->lt_table_size);
381 kfree_type(struct entry_template, new_template->lt_table_size, new_template->lt_entries);
382 /* Copy the look up table. */
383 bcopy(src: template->lt_entries_lut, dst: new_entries_lut, n: sizeof(uint16_t) * new_entries_lut_size);
384 kfree_type(uint16_t, ledger_template_entries_lut_size(new_template->lt_table_size), new_template->lt_entries_lut);
385
386 new_template->lt_entries = new_entries;
387 new_template->lt_table_size = template->lt_table_size;
388 new_template->lt_cnt = template->lt_cnt;
389 new_template->lt_next_offset = template->lt_next_offset;
390 new_template->lt_entries_lut = new_entries_lut;
391
392out:
393 template_unlock(template);
394
395 return new_template;
396}
397
398void
399ledger_template_dereference(ledger_template_t template)
400{
401 template_lock(template);
402 template->lt_refs--;
403 template_unlock(template);
404
405 if (template->lt_refs == 0) {
406 kfree_type(struct entry_template, template->lt_table_size, template->lt_entries);
407 kfree_type(uint16_t, ledger_template_entries_lut_size(template->lt_table_size), template->lt_entries_lut);
408 lck_mtx_destroy(lck: &template->lt_lock, grp: &ledger_lck_grp);
409 if (template->lt_zone) {
410 zdestroy(zone: template->lt_zone);
411 }
412 kfree_type(struct ledger_template, template);
413 }
414}
415
416static inline int
417ledger_entry_id(uint16_t size, uint16_t offset)
418{
419 int idx = offset;
420 idx |= (size << ENTRY_ID_SIZE_SHIFT);
421 assert(idx >= 0);
422 return idx;
423}
424
425static inline int
426ledger_entry_id_from_template_entry(const struct entry_template *et)
427{
428 return ledger_entry_id(size: et->et_size, offset: et->et_offset);
429}
430
431int
432ledger_entry_add_with_flags(ledger_template_t template, const char *key,
433 const char *group, const char *units, uint64_t flags)
434{
435 uint16_t template_idx;
436 struct entry_template *et;
437 uint16_t size = 0, next_offset = 0, entry_idx = 0;
438
439 if ((key == NULL) || (strlen(s: key) >= LEDGER_NAME_MAX) || (template->lt_zone != NULL)) {
440 return -1;
441 }
442
443 template_lock(template);
444
445 /* Make sure we have space for this entry */
446 if (template->lt_cnt == MAX_LEDGER_ENTRIES) {
447 template_unlock(template);
448 return -1;
449 }
450
451 /* If the table is full, attempt to double its size */
452 if (template->lt_cnt == template->lt_table_size) {
453 struct entry_template *new_entries, *old_entries;
454 uint16_t *new_entries_lut = NULL, *old_entries_lut = NULL;
455 uint16_t old_cnt, new_cnt;
456 spl_t s;
457
458 old_cnt = template->lt_table_size;
459 /* double old_sz allocation, but check for overflow */
460 if (os_mul_overflow(old_cnt, 2, &new_cnt)) {
461 template_unlock(template);
462 return -1;
463 }
464
465 if (new_cnt > MAX_LEDGER_ENTRIES) {
466 template_unlock(template);
467 panic("Attempt to create a ledger template with more than MAX_LEDGER_ENTRIES. MAX_LEDGER_ENTRIES=%lu, old_cnt=%u, new_cnt=%u\n", MAX_LEDGER_ENTRIES, old_cnt, new_cnt);
468 }
469
470 new_entries = kalloc_type(struct entry_template, new_cnt,
471 Z_WAITOK | Z_ZERO);
472 if (new_entries == NULL) {
473 template_unlock(template);
474 return -1;
475 }
476 new_entries_lut = kalloc_type(uint16_t, ledger_template_entries_lut_size(new_cnt),
477 Z_WAITOK | Z_ZERO);
478 if (new_entries_lut == NULL) {
479 template_unlock(template);
480 kfree_type(struct entry_template, new_cnt, new_entries);
481 return -1;
482 }
483
484 memcpy(dst: new_entries, src: template->lt_entries,
485 n: old_cnt * sizeof(struct entry_template));
486 template->lt_table_size = new_cnt;
487
488 memcpy(dst: new_entries_lut, src: template->lt_entries_lut,
489 n: ledger_template_entries_lut_size(lt_table_size: old_cnt) * sizeof(uint16_t));
490
491 old_entries = template->lt_entries;
492 old_entries_lut = template->lt_entries_lut;
493
494 TEMPLATE_INUSE(s, template);
495 template->lt_entries = new_entries;
496 template->lt_entries_lut = new_entries_lut;
497 TEMPLATE_IDLE(s, template);
498
499 kfree_type(struct entry_template, old_cnt, old_entries);
500 kfree_type(uint16_t, ledger_template_entries_lut_size(old_cnt), old_entries_lut);
501 }
502
503 et = &template->lt_entries[template->lt_cnt];
504 strlcpy(dst: et->et_key, src: key, LEDGER_NAME_MAX);
505 strlcpy(dst: et->et_group, src: group, LEDGER_NAME_MAX);
506 strlcpy(dst: et->et_units, src: units, LEDGER_NAME_MAX);
507 et->et_flags = LF_ENTRY_ACTIVE;
508 /*
509 * Currently we only have two types of variable sized entries
510 * CREDIT_ONLY and full-fledged leger_entry.
511 * In the future, we can add more gradations based on the flags.
512 */
513 if ((flags & ~(LEDGER_ENTRY_SMALL_FLAGS)) == 0) {
514 size = sizeof(struct ledger_entry_small);
515 et->et_flags |= LF_TRACK_CREDIT_ONLY;
516 } else {
517 size = sizeof(struct ledger_entry);
518 }
519 et->et_size = size;
520 et->et_offset = (template->lt_next_offset / sizeof(struct ledger_entry_small));
521 et->et_callback = NULL;
522
523 template_idx = template->lt_cnt++;
524 next_offset = template->lt_next_offset;
525 entry_idx = next_offset / sizeof(struct ledger_entry_small);
526 template->lt_next_offset += size;
527 assert(template->lt_next_offset > next_offset);
528 template->lt_entries_lut[entry_idx] = template_idx;
529 template_unlock(template);
530
531 return ledger_entry_id(size, offset: entry_idx);
532}
533
534/*
535 * Add a new entry to the list of entries in a ledger template. There is
536 * currently no mechanism to remove an entry. Implementing such a mechanism
537 * would require us to maintain per-entry reference counts, which we would
538 * prefer to avoid if possible.
539 */
540int
541ledger_entry_add(ledger_template_t template, const char *key,
542 const char *group, const char *units)
543{
544 /*
545 * When using the legacy interface we have to be pessimistic
546 * and allocate memory for all of the features.
547 */
548 return ledger_entry_add_with_flags(template, key, group, units,
549 flags: LEDGER_ENTRY_ALLOW_CALLBACK | LEDGER_ENTRY_ALLOW_MAXIMUM |
550 LEDGER_ENTRY_ALLOW_DEBIT | LEDGER_ENTRY_ALLOW_LIMIT |
551 LEDGER_ENTRY_ALLOW_ACTION | LEDGER_ENTRY_ALLOW_INACTIVE);
552}
553
554
555kern_return_t
556ledger_entry_setactive(ledger_t ledger, int entry)
557{
558 volatile uint32_t *flags = NULL;
559
560 if (!is_entry_valid(l: ledger, entry)) {
561 return KERN_INVALID_ARGUMENT;
562 }
563
564 flags = get_entry_flags(l: ledger, index: entry);
565
566 if ((*flags & LF_ENTRY_ACTIVE) == 0) {
567 flag_set(flags, LF_ENTRY_ACTIVE);
568 }
569 return KERN_SUCCESS;
570}
571
572
573int
574ledger_key_lookup(ledger_template_t template, const char *key)
575{
576 int id = -1;
577 struct entry_template *et = NULL;
578
579 template_lock(template);
580 if (template->lt_entries != NULL) {
581 for (uint16_t idx = 0; idx < template->lt_cnt; idx++) {
582 et = &template->lt_entries[idx];
583 if (strcmp(s1: key, s2: et->et_key) == 0) {
584 id = ledger_entry_id(size: et->et_size, offset: et->et_offset);
585 break;
586 }
587 }
588 }
589
590 template_unlock(template);
591
592 return id;
593}
594
595/*
596 * Complete the initialization of ledger template
597 * by initializing ledger zone. After initializing
598 * the ledger zone, adding an entry in the ledger
599 * template will fail.
600 */
601void
602ledger_template_complete(ledger_template_t template)
603{
604 size_t ledger_size;
605 ledger_size = sizeof(struct ledger) + template->lt_next_offset;
606 assert(ledger_size > sizeof(struct ledger));
607 template->lt_zone = zone_create(name: template->lt_name, size: ledger_size,
608 flags: ZC_PGZ_USE_GUARDS | ZC_DESTRUCTIBLE);
609 template->lt_initialized = true;
610}
611
612/*
613 * Like ledger_template_complete, except we'll ask
614 * the pmap layer to manage allocations for us.
615 * Meant for ledgers that should be owned by the
616 * pmap layer.
617 */
618void
619ledger_template_complete_secure_alloc(ledger_template_t template)
620{
621 size_t ledger_size;
622 ledger_size = sizeof(struct ledger) + template->lt_next_offset;
623
624 /**
625 * Ensure that the amount of space being allocated by the PPL for each
626 * ledger is large enough.
627 */
628 pmap_ledger_verify_size(ledger_size);
629 template->lt_initialized = true;
630}
631
632/*
633 * Create a new ledger based on the specified template. As part of the
634 * ledger creation we need to allocate space for a table of ledger entries.
635 * The size of the table is based on the size of the template at the time
636 * the ledger is created. If additional entries are added to the template
637 * after the ledger is created, they will not be tracked in this ledger.
638 */
639ledger_t
640ledger_instantiate(ledger_template_t template, int entry_type)
641{
642 ledger_t ledger;
643 uint16_t entries_size;
644 uint16_t num_entries;
645 uint16_t i;
646
647 template_lock(template);
648 template->lt_refs++;
649 entries_size = template->lt_next_offset;
650 num_entries = template->lt_cnt;
651 template_unlock(template);
652
653 if (template->lt_zone) {
654 ledger = (ledger_t)zalloc(zone: template->lt_zone);
655 } else {
656 /**
657 * If the template doesn't contain a zone to allocate ledger objects
658 * from, then assume that these ledger objects should be allocated by
659 * the pmap. This is done on PPL-enabled systems to give the PPL a
660 * method of validating ledger objects when updating them from within
661 * the PPL.
662 */
663 ledger = pmap_ledger_alloc();
664 }
665
666 if (ledger == NULL) {
667 ledger_template_dereference(template);
668 return LEDGER_NULL;
669 }
670
671 ledger->l_template = template;
672 ledger->l_id = ledger_cnt++;
673 os_ref_init(&ledger->l_refs, &ledger_refgrp);
674 assert(entries_size > 0);
675 ledger->l_size = (uint16_t) entries_size;
676
677 template_lock(template);
678 assert(ledger->l_size <= template->lt_next_offset);
679 for (i = 0; i < num_entries; i++) {
680 uint16_t size, offset;
681 struct entry_template *et = &template->lt_entries[i];
682 size = et->et_size;
683 offset = et->et_offset;
684 assert(offset < ledger->l_size);
685
686 struct ledger_entry_small *les = &ledger->l_entries[offset];
687 if (size == sizeof(struct ledger_entry)) {
688 struct ledger_entry *le = (struct ledger_entry *) les;
689
690 le->le_flags = et->et_flags;
691 /* make entry inactive by removing active bit */
692 if (entry_type == LEDGER_CREATE_INACTIVE_ENTRIES) {
693 flag_clear(flags: &le->le_flags, LF_ENTRY_ACTIVE);
694 }
695 /*
696 * If template has a callback, this entry is opted-in,
697 * by default.
698 */
699 if (et->et_callback != NULL) {
700 flag_set(flags: &le->le_flags, LEDGER_ACTION_CALLBACK);
701 }
702 le->le_credit = 0;
703 le->le_debit = 0;
704 le->le_limit = LEDGER_LIMIT_INFINITY;
705 le->le_warn_percent = LEDGER_PERCENT_NONE;
706 le->le_diag_threshold_scaled = LEDGER_DIAG_MEM_THRESHOLD_INFINITY;
707 le->_le.le_refill.le_refill_period = 0;
708 le->_le.le_refill.le_last_refill = 0;
709 } else {
710 les->les_flags = et->et_flags;
711 les->les_credit = 0;
712 }
713 }
714 template_unlock(template);
715
716 return ledger;
717}
718
719static uint32_t
720flag_set(volatile uint32_t *flags, uint32_t bit)
721{
722 return OSBitOrAtomic(bit, flags);
723}
724
725static uint32_t
726flag_clear(volatile uint32_t *flags, uint32_t bit)
727{
728 return OSBitAndAtomic(~bit, flags);
729}
730
731/*
732 * Take a reference on a ledger
733 */
734void
735ledger_reference(ledger_t ledger)
736{
737 if (!LEDGER_VALID(ledger)) {
738 return;
739 }
740
741 os_ref_retain(rc: &ledger->l_refs);
742}
743
744/*
745 * Remove a reference on a ledger. If this is the last reference,
746 * deallocate the unused ledger.
747 */
748void
749ledger_dereference(ledger_t ledger)
750{
751 if (!LEDGER_VALID(ledger)) {
752 return;
753 }
754
755 if (os_ref_release(rc: &ledger->l_refs) == 0) {
756 ledger_template_t template = ledger->l_template;
757 if (template->lt_zone) {
758 zfree(template->lt_zone, ledger);
759 } else {
760 /**
761 * If the template doesn't contain a zone to allocate ledger objects
762 * from, then assume that these ledger objects were allocated by the
763 * pmap. This is done on PPL-enabled systems to give the PPL a
764 * method of validating ledger objects when updating them from
765 * within the PPL.
766 */
767 pmap_ledger_free(ledger);
768 }
769 ledger_template_dereference(template);
770 }
771}
772
773/*
774 * Determine whether an entry has exceeded its warning level.
775 */
776static inline bool
777warn_level_exceeded(struct ledger_entry *le)
778{
779 ledger_amount_t balance;
780
781 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
782 assert(le->le_debit == 0);
783 } else {
784 assert((le->le_credit >= 0) && (le->le_debit >= 0));
785 }
786
787 /*
788 * XXX - Currently, we only support warnings for ledgers which
789 * use positive limits.
790 */
791 balance = le->le_credit - le->le_debit;
792 if (le->le_warn_percent != LEDGER_PERCENT_NONE &&
793 ((balance > (le->le_limit * le->le_warn_percent) >> 16))) {
794 return true;
795 }
796 return false;
797}
798#if DEBUG || DEVELOPMENT
799
800/*
801 * Determine whether an entry has exceeded its diag mem threshold level.
802 */
803static inline bool
804diag_mem_threshold_exceeded(struct ledger_entry *le)
805{
806 ledger_amount_t balance;
807 ledger_amount_t diag_mem_threshold;
808
809 if ((le->le_diag_threshold_scaled != LEDGER_DIAG_MEM_THRESHOLD_INFINITY) && (ledger_is_diag_threshold_enabled_internal(le) == true)) {
810 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
811 assert(le->le_debit == 0);
812 } else {
813 assert((le->le_credit >= 0) && (le->le_debit >= 0));
814 }
815
816 diag_mem_threshold = LEDGER_DIAG_MEM_AMOUNT_FROM_THRESHOLD(le->le_diag_threshold_scaled);
817 balance = le->le_credit - le->le_debit;
818 if ((diag_mem_threshold <= 0) && (balance < diag_mem_threshold)) {
819 return 1;
820 }
821 if ((diag_mem_threshold > 0) && (balance > diag_mem_threshold)) {
822 return 1;
823 }
824 }
825 return 0;
826}
827#endif
828/*
829 * Determine whether an entry has exceeded its limit.
830 */
831static inline bool
832limit_exceeded(struct ledger_entry *le)
833{
834 ledger_amount_t balance;
835
836 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
837 assert(le->le_debit == 0);
838 } else {
839 assert((le->le_credit >= 0) && (le->le_debit >= 0));
840 }
841
842 balance = le->le_credit - le->le_debit;
843 if ((le->le_limit <= 0) && (balance < le->le_limit)) {
844 return true;
845 }
846
847 if ((le->le_limit > 0) && (balance > le->le_limit)) {
848 return true;
849 }
850 return false;
851}
852
853static inline struct ledger_callback *
854entry_get_callback(ledger_t ledger, int entry)
855{
856 struct ledger_callback *callback = NULL;
857 spl_t s;
858 const uint16_t *ledger_template_idx_p = NULL;
859
860 TEMPLATE_INUSE(s, ledger->l_template);
861 ledger_template_idx_p = ledger_entry_to_template_idx(template: ledger->l_template, index: entry);
862 if (ledger_template_idx_p != NULL) {
863 callback = ledger->l_template->lt_entries[*ledger_template_idx_p].et_callback;
864 }
865 TEMPLATE_IDLE(s, ledger->l_template);
866
867 return callback;
868}
869
870/*
871 * If the ledger value is positive, wake up anybody waiting on it.
872 */
873static inline void
874ledger_limit_entry_wakeup(struct ledger_entry *le)
875{
876 uint32_t flags;
877
878 if (!limit_exceeded(le)) {
879 flags = flag_clear(flags: &le->le_flags, LF_CALLED_BACK);
880
881 while (le->le_flags & LF_WAKE_NEEDED) {
882 flag_clear(flags: &le->le_flags, LF_WAKE_NEEDED);
883 thread_wakeup((event_t)le);
884 }
885 }
886}
887
888/*
889 * Refill the coffers.
890 */
891static void
892ledger_refill(uint64_t now, ledger_t ledger, int entry)
893{
894 uint64_t elapsed, period, periods;
895 struct ledger_entry *le;
896 ledger_amount_t balance, due;
897
898 if (!is_entry_valid(l: ledger, entry)) {
899 return;
900 }
901
902 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
903 /* Small entries can't do refills */
904 return;
905 }
906
907 le = ledger_entry_identifier_to_entry(ledger, id: entry);
908
909 assert(le->le_limit != LEDGER_LIMIT_INFINITY);
910
911 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
912 assert(le->le_debit == 0);
913 return;
914 }
915
916 /*
917 * If another thread is handling the refill already, we're not
918 * needed.
919 */
920 if (flag_set(flags: &le->le_flags, LF_REFILL_INPROGRESS) & LF_REFILL_INPROGRESS) {
921 return;
922 }
923
924 /*
925 * If the timestamp we're about to use to refill is older than the
926 * last refill, then someone else has already refilled this ledger
927 * and there's nothing for us to do here.
928 */
929 if (now <= le->_le.le_refill.le_last_refill) {
930 flag_clear(flags: &le->le_flags, LF_REFILL_INPROGRESS);
931 return;
932 }
933
934 /*
935 * See how many refill periods have passed since we last
936 * did a refill.
937 */
938 period = le->_le.le_refill.le_refill_period;
939 elapsed = now - le->_le.le_refill.le_last_refill;
940 if ((period == 0) || (elapsed < period)) {
941 flag_clear(flags: &le->le_flags, LF_REFILL_INPROGRESS);
942 return;
943 }
944
945 /*
946 * Optimize for the most common case of only one or two
947 * periods elapsing.
948 */
949 periods = 0;
950 while ((periods < 2) && (elapsed > 0)) {
951 periods++;
952 elapsed -= period;
953 }
954
955 /*
956 * OK, it's been a long time. Do a divide to figure out
957 * how long.
958 */
959 if (elapsed > 0) {
960 periods = (now - le->_le.le_refill.le_last_refill) / period;
961 }
962
963 balance = le->le_credit - le->le_debit;
964 due = periods * le->le_limit;
965
966 if (balance - due < 0) {
967 due = balance;
968 }
969
970 if (due < 0 && (le->le_flags & LF_PANIC_ON_NEGATIVE)) {
971 assertf(due >= 0, "now=%llu, ledger=%p, entry=%d, balance=%lld, due=%lld", now, ledger, entry, balance, due);
972 } else {
973 OSAddAtomic64(due, &le->le_debit);
974 assert(le->le_debit >= 0);
975 }
976 /*
977 * If we've completely refilled the pool, set the refill time to now.
978 * Otherwise set it to the time at which it last should have been
979 * fully refilled.
980 */
981 if (balance == due) {
982 le->_le.le_refill.le_last_refill = now;
983 } else {
984 le->_le.le_refill.le_last_refill += (le->_le.le_refill.le_refill_period * periods);
985 }
986
987 flag_clear(flags: &le->le_flags, LF_REFILL_INPROGRESS);
988
989 lprintf(("Refill %lld %lld->%lld\n", periods, balance, balance - due));
990 if (!limit_exceeded(le)) {
991 ledger_limit_entry_wakeup(le);
992 }
993}
994
995void
996ledger_entry_check_new_balance(thread_t thread, ledger_t ledger,
997 int entry)
998{
999 uint16_t size, offset;
1000 struct ledger_entry *le = NULL;
1001 struct ledger_entry_small *les = NULL;
1002 if (!is_entry_valid(l: ledger, entry)) {
1003 return;
1004 }
1005 size = ENTRY_ID_SIZE(entry);
1006 offset = ENTRY_ID_OFFSET(entry);
1007 les = &ledger->l_entries[offset];
1008 if (size == sizeof(struct ledger_entry_small)) {
1009 if ((les->les_flags & LF_PANIC_ON_NEGATIVE) && les->les_credit < 0) {
1010 panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld debit:0 balance:%lld",
1011 ledger, entry, les,
1012 les->les_credit,
1013 les->les_credit);
1014 }
1015 } else if (size == sizeof(struct ledger_entry)) {
1016 le = (struct ledger_entry *)les;
1017 if (le->le_flags & LF_TRACKING_MAX) {
1018 ledger_amount_t balance = le->le_credit - le->le_debit;
1019
1020 if (balance > le->_le._le_max.le_lifetime_max) {
1021 le->_le._le_max.le_lifetime_max = balance;
1022 }
1023
1024#if CONFIG_LEDGER_INTERVAL_MAX
1025 if (balance > le->_le._le_max.le_interval_max) {
1026 le->_le._le_max.le_interval_max = balance;
1027 }
1028#endif /* LEDGER_CONFIG_INTERVAL_MAX */
1029 }
1030
1031 /* Check to see whether we're due a refill */
1032 if (le->le_flags & LF_REFILL_SCHEDULED) {
1033 assert(!(le->le_flags & LF_TRACKING_MAX));
1034
1035 uint64_t now = mach_absolute_time();
1036 if ((now - le->_le.le_refill.le_last_refill) > le->_le.le_refill.le_refill_period) {
1037 ledger_refill(now, ledger, entry);
1038 }
1039 }
1040
1041 if (limit_exceeded(le)) {
1042 /*
1043 * We've exceeded the limit for this entry. There
1044 * are several possible ways to handle it. We can block,
1045 * we can execute a callback, or we can ignore it. In
1046 * either of the first two cases, we want to set the AST
1047 * flag so we can take the appropriate action just before
1048 * leaving the kernel. The one caveat is that if we have
1049 * already called the callback, we don't want to do it
1050 * again until it gets rearmed.
1051 */
1052 if ((le->le_flags & LEDGER_ACTION_BLOCK) ||
1053 (!(le->le_flags & LF_CALLED_BACK) &&
1054 entry_get_callback(ledger, entry))) {
1055 act_set_astledger_async(thread);
1056 }
1057 } else {
1058 /*
1059 * The balance on the account is below the limit.
1060 *
1061 * If there are any threads blocked on this entry, now would
1062 * be a good time to wake them up.
1063 */
1064 if (le->le_flags & LF_WAKE_NEEDED) {
1065 ledger_limit_entry_wakeup(le);
1066 }
1067
1068 if (le->le_flags & LEDGER_ACTION_CALLBACK) {
1069 if (warn_level_exceeded(le)) {
1070 /*
1071 * This ledger's balance is above the warning level.
1072 */
1073 if ((le->le_flags & LF_WARNED) == 0) {
1074 /*
1075 * If we are above the warning level and
1076 * have not yet invoked the callback,
1077 * set the AST so it can be done before returning
1078 * to userland.
1079 */
1080 act_set_astledger_async(thread);
1081 }
1082 } else {
1083 /*
1084 * This ledger's balance is below the warning level.
1085 */
1086 if (le->le_flags & LF_WARNED) {
1087 /*
1088 * If we are below the warning level and
1089 * the LF_WARNED flag is still set, we need
1090 * to invoke the callback to let the client
1091 * know the ledger balance is now back below
1092 * the warning level.
1093 */
1094 act_set_astledger_async(thread);
1095 }
1096 }
1097 }
1098 }
1099#if DEBUG || DEVELOPMENT
1100 if (diag_mem_threshold_exceeded(le)) {
1101 /*
1102 * Even if the limit is below the threshold, we may be interested
1103 * in diagnostics limits. Lets process them if the ast is not
1104 * invoked
1105 */
1106 if ((le->le_flags & LF_DIAG_WARNED) == 0) {
1107 act_set_astledger_async(thread);
1108 }
1109 }
1110#endif
1111 if ((le->le_flags & LF_PANIC_ON_NEGATIVE) &&
1112 (le->le_credit < le->le_debit)) {
1113 panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld debit:%lld balance:%lld",
1114 ledger, entry, le,
1115 le->le_credit,
1116 le->le_debit,
1117 le->le_credit - le->le_debit);
1118 }
1119 } else {
1120 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, size);
1121 }
1122}
1123
1124void
1125ledger_check_new_balance(thread_t thread, ledger_t ledger, int entry)
1126{
1127 ledger_entry_check_new_balance(thread, ledger, entry);
1128}
1129
1130/*
1131 * Add value to an entry in a ledger for a specific thread.
1132 */
1133kern_return_t
1134ledger_credit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)
1135{
1136 ledger_amount_t old, new;
1137 struct ledger_entry *le;
1138 uint16_t entry_size = ENTRY_ID_SIZE(entry);
1139
1140 if (!is_entry_valid_and_active(l: ledger, entry) || (amount < 0)) {
1141 return KERN_INVALID_VALUE;
1142 }
1143
1144 if (amount == 0) {
1145 return KERN_SUCCESS;
1146 }
1147
1148 if (entry_size == sizeof(struct ledger_entry_small)) {
1149 struct ledger_entry_small *les = &ledger->l_entries[ENTRY_ID_OFFSET(entry)];
1150 old = OSAddAtomic64(amount, &les->les_credit);
1151 new = old + amount;
1152 } else if (entry_size == sizeof(struct ledger_entry)) {
1153 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1154
1155 old = OSAddAtomic64(amount, &le->le_credit);
1156 new = old + amount;
1157 } else {
1158 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, entry_size);
1159 }
1160
1161 lprintf(("%p Credit %lld->%lld\n", thread, old, new));
1162 if (thread) {
1163 ledger_entry_check_new_balance(thread, ledger, entry);
1164 }
1165
1166 return KERN_SUCCESS;
1167}
1168
1169/*
1170 * Add value to an entry in a ledger.
1171 */
1172kern_return_t
1173ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount)
1174{
1175 return ledger_credit_thread(thread: current_thread(), ledger, entry, amount);
1176}
1177
1178/*
1179 * Add value to an entry in a ledger; do not check balance after update.
1180 */
1181kern_return_t
1182ledger_credit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount)
1183{
1184 return ledger_credit_thread(NULL, ledger, entry, amount);
1185}
1186
1187/* Add all of one ledger's values into another.
1188 * They must have been created from the same template.
1189 * This is not done atomically. Another thread (if not otherwise synchronized)
1190 * may see bogus values when comparing one entry to another.
1191 * As each entry's credit & debit are modified one at a time, the warning/limit
1192 * may spuriously trip, or spuriously fail to trip, or another thread (if not
1193 * otherwise synchronized) may see a bogus balance.
1194 */
1195kern_return_t
1196ledger_rollup(ledger_t to_ledger, ledger_t from_ledger)
1197{
1198 int id;
1199 ledger_template_t template = NULL;
1200 struct entry_template *et = NULL;
1201
1202 assert(to_ledger->l_template->lt_cnt == from_ledger->l_template->lt_cnt);
1203 template = from_ledger->l_template;
1204 assert(template->lt_initialized);
1205
1206 for (uint16_t i = 0; i < template->lt_cnt; i++) {
1207 et = &template->lt_entries[i];
1208 uint16_t size = et->et_size;
1209 id = ledger_entry_id(size, offset: et->et_offset);
1210 ledger_rollup_entry(to_ledger, from_ledger, entry: id);
1211 }
1212
1213 return KERN_SUCCESS;
1214}
1215
1216/* Add one ledger entry value to another.
1217 * They must have been created from the same template.
1218 * Since the credit and debit values are added one
1219 * at a time, other thread might read the a bogus value.
1220 */
1221kern_return_t
1222ledger_rollup_entry(ledger_t to_ledger, ledger_t from_ledger, int entry)
1223{
1224 struct ledger_entry_small *from_les, *to_les;
1225 uint16_t entry_size, entry_offset;
1226 entry_size = ENTRY_ID_SIZE(entry);
1227 entry_offset = ENTRY_ID_OFFSET(entry);
1228
1229 assert(to_ledger->l_template->lt_cnt == from_ledger->l_template->lt_cnt);
1230 if (is_entry_valid(l: from_ledger, entry) && is_entry_valid(l: to_ledger, entry)) {
1231 from_les = &from_ledger->l_entries[entry_offset];
1232 to_les = &to_ledger->l_entries[entry_offset];
1233 if (entry_size == sizeof(struct ledger_entry)) {
1234 struct ledger_entry *from = (struct ledger_entry *)from_les;
1235 struct ledger_entry *to = (struct ledger_entry *)to_les;
1236 OSAddAtomic64(from->le_credit, &to->le_credit);
1237 OSAddAtomic64(from->le_debit, &to->le_debit);
1238 } else if (entry_size == sizeof(struct ledger_entry_small)) {
1239 OSAddAtomic64(from_les->les_credit, &to_les->les_credit);
1240 } else {
1241 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", from_ledger, entry, entry_size);
1242 }
1243 }
1244
1245 return KERN_SUCCESS;
1246}
1247
1248/*
1249 * Zero the balance of a ledger by adding to its credit or debit, whichever is smaller.
1250 * Note that some clients of ledgers (notably, task wakeup statistics) require that
1251 * le_credit only ever increase as a function of ledger_credit().
1252 */
1253kern_return_t
1254ledger_zero_balance(ledger_t ledger, int entry)
1255{
1256 struct ledger_entry *le;
1257 struct ledger_entry_small *les;
1258 ledger_amount_t debit, credit;
1259 uint16_t entry_size, entry_offset;
1260 entry_size = ENTRY_ID_SIZE(entry);
1261 entry_offset = ENTRY_ID_OFFSET(entry);
1262
1263 if (!is_entry_valid_and_active(l: ledger, entry)) {
1264 return KERN_INVALID_VALUE;
1265 }
1266
1267 les = &ledger->l_entries[entry_offset];
1268 if (entry_size == sizeof(struct ledger_entry_small)) {
1269 while (true) {
1270 credit = les->les_credit;
1271 if (OSCompareAndSwap64(credit, 0, &les->les_credit)) {
1272 break;
1273 }
1274 }
1275 } else if (entry_size == sizeof(struct ledger_entry)) {
1276 le = (struct ledger_entry *)les;
1277top:
1278 debit = le->le_debit;
1279 credit = le->le_credit;
1280
1281 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
1282 assert(le->le_debit == 0);
1283 if (!OSCompareAndSwap64(credit, 0, &le->le_credit)) {
1284 goto top;
1285 }
1286 lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, 0));
1287 } else if (credit > debit) {
1288 if (!OSCompareAndSwap64(debit, credit, &le->le_debit)) {
1289 goto top;
1290 }
1291 lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_debit, le->le_credit));
1292 } else if (credit < debit) {
1293 if (!OSCompareAndSwap64(credit, debit, &le->le_credit)) {
1294 goto top;
1295 }
1296 lprintf(("%p zeroed %lld->%lld\n", current_thread(), le->le_credit, le->le_debit));
1297 }
1298 } else {
1299 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, entry_size);
1300 }
1301
1302 return KERN_SUCCESS;
1303}
1304
1305kern_return_t
1306ledger_get_limit(ledger_t ledger, int entry, ledger_amount_t *limit)
1307{
1308 struct ledger_entry *le;
1309
1310 if (!is_entry_valid_and_active(l: ledger, entry)) {
1311 return KERN_INVALID_VALUE;
1312 }
1313
1314 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1315 /* Small entries can't have limits */
1316 *limit = LEDGER_LIMIT_INFINITY;
1317 } else {
1318 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1319 *limit = le->le_limit;
1320 }
1321
1322 lprintf(("ledger_get_limit: %lld\n", *limit));
1323
1324 return KERN_SUCCESS;
1325}
1326
1327/*
1328 * Adjust the limit of a limited resource. This does not affect the
1329 * current balance, so the change doesn't affect the thread until the
1330 * next refill.
1331 *
1332 * warn_level: If non-zero, causes the callback to be invoked when
1333 * the balance exceeds this level. Specified as a percentage [of the limit].
1334 */
1335kern_return_t
1336ledger_set_limit(ledger_t ledger, int entry, ledger_amount_t limit,
1337 uint8_t warn_level_percentage)
1338{
1339 struct ledger_entry *le;
1340
1341 if (!is_entry_valid_and_active(l: ledger, entry)) {
1342 return KERN_INVALID_VALUE;
1343 }
1344
1345 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1346 /* Small entries can't have limits */
1347 return KERN_INVALID_ARGUMENT;
1348 }
1349
1350 lprintf(("ledger_set_limit: %lld\n", limit));
1351 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1352
1353 if (limit == LEDGER_LIMIT_INFINITY) {
1354 /*
1355 * Caller wishes to disable the limit. This will implicitly
1356 * disable automatic refill, as refills implicitly depend
1357 * on the limit.
1358 */
1359 ledger_disable_refill(l: ledger, entry);
1360 }
1361
1362 le->le_limit = limit;
1363 if (le->le_flags & LF_REFILL_SCHEDULED) {
1364 assert(!(le->le_flags & LF_TRACKING_MAX));
1365 le->_le.le_refill.le_last_refill = 0;
1366 }
1367 flag_clear(flags: &le->le_flags, LF_CALLED_BACK);
1368 flag_clear(flags: &le->le_flags, LF_WARNED);
1369 ledger_limit_entry_wakeup(le);
1370
1371 if (warn_level_percentage != 0) {
1372 assert(warn_level_percentage <= 100);
1373 assert(limit > 0); /* no negative limit support for warnings */
1374 assert(limit != LEDGER_LIMIT_INFINITY); /* warn % without limit makes no sense */
1375 le->le_warn_percent = warn_level_percentage * (1u << 16) / 100;
1376 } else {
1377 le->le_warn_percent = LEDGER_PERCENT_NONE;
1378 }
1379
1380 return KERN_SUCCESS;
1381}
1382
1383#if CONFIG_LEDGER_INTERVAL_MAX
1384kern_return_t
1385ledger_get_interval_max(ledger_t ledger, int entry,
1386 ledger_amount_t *max_interval_balance, int reset)
1387{
1388 kern_return_t kr = KERN_SUCCESS;
1389 struct ledger_entry *le;
1390
1391 if (!is_entry_valid_and_active(l: ledger, entry)) {
1392 return KERN_INVALID_VALUE;
1393 }
1394
1395 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1396 /* Small entries can't track max */
1397 return KERN_INVALID_ARGUMENT;
1398 }
1399
1400 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1401
1402 if (!(le->le_flags & LF_TRACKING_MAX)) {
1403 return KERN_INVALID_VALUE;
1404 }
1405
1406 *max_interval_balance = le->_le._le_max.le_interval_max;
1407 lprintf(("ledger_get_interval_max: %lld%s\n", *max_interval_balance,
1408 (reset) ? " --> 0" : ""));
1409
1410 if (reset) {
1411 kr = ledger_get_balance(ledger, entry, balance: &le->_le._le_max.le_interval_max);
1412 }
1413
1414 return kr;
1415}
1416#endif /* CONFIG_LEDGER_INTERVAL_MAX */
1417
1418kern_return_t
1419ledger_get_lifetime_max(ledger_t ledger, int entry,
1420 ledger_amount_t *max_lifetime_balance)
1421{
1422 struct ledger_entry *le;
1423
1424 if (!is_entry_valid_and_active(l: ledger, entry)) {
1425 return KERN_INVALID_VALUE;
1426 }
1427
1428 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1429 /* Small entries can't track max */
1430 return KERN_INVALID_ARGUMENT;
1431 }
1432
1433 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1434
1435 if (!(le->le_flags & LF_TRACKING_MAX)) {
1436 return KERN_INVALID_VALUE;
1437 }
1438
1439 *max_lifetime_balance = le->_le._le_max.le_lifetime_max;
1440 lprintf(("ledger_get_lifetime_max: %lld\n", *max_lifetime_balance));
1441
1442 return KERN_SUCCESS;
1443}
1444
1445/*
1446 * Enable tracking of periodic maximums for this ledger entry.
1447 */
1448kern_return_t
1449ledger_track_maximum(ledger_template_t template, int entry,
1450 __unused int period_in_secs)
1451{
1452 uint16_t idx;
1453 const uint16_t *idx_p;
1454 struct entry_template *et = NULL;
1455 kern_return_t kr = KERN_INVALID_VALUE;
1456
1457 template_lock(template);
1458
1459 idx_p = ledger_entry_to_template_idx(template, index: entry);
1460 if (idx_p == NULL) {
1461 kr = KERN_INVALID_VALUE;
1462 goto out;
1463 }
1464 idx = *idx_p;
1465 if (idx >= template->lt_cnt) {
1466 kr = KERN_INVALID_VALUE;
1467 goto out;
1468 }
1469 et = &template->lt_entries[idx];
1470 /* Ensure the caller asked for enough space up front */
1471 if (et->et_size != sizeof(struct ledger_entry)) {
1472 kr = KERN_INVALID_VALUE;
1473 goto out;
1474 }
1475
1476 /* Refill is incompatible with max tracking. */
1477 if (et->et_flags & LF_REFILL_SCHEDULED) {
1478 kr = KERN_INVALID_VALUE;
1479 goto out;
1480 }
1481
1482 et->et_flags |= LF_TRACKING_MAX;
1483 kr = KERN_SUCCESS;
1484out:
1485 template_unlock(template);
1486
1487 return kr;
1488}
1489
1490kern_return_t
1491ledger_panic_on_negative(ledger_template_t template, int entry)
1492{
1493 const uint16_t *idx_p;
1494 uint16_t idx;
1495 template_lock(template);
1496
1497 idx_p = ledger_entry_to_template_idx(template, index: entry);
1498 if (idx_p == NULL) {
1499 template_unlock(template);
1500 return KERN_INVALID_VALUE;
1501 }
1502 idx = *idx_p;
1503 if (idx >= template->lt_cnt) {
1504 template_unlock(template);
1505 return KERN_INVALID_VALUE;
1506 }
1507
1508 template->lt_entries[idx].et_flags |= LF_PANIC_ON_NEGATIVE;
1509
1510 template_unlock(template);
1511
1512 return KERN_SUCCESS;
1513}
1514
1515kern_return_t
1516ledger_track_credit_only(ledger_template_t template, int entry)
1517{
1518 const uint16_t *idx_p;
1519 uint16_t idx;
1520 struct entry_template *et = NULL;
1521 kern_return_t kr = KERN_INVALID_VALUE;
1522 template_lock(template);
1523
1524 idx_p = ledger_entry_to_template_idx(template, index: entry);
1525 if (idx_p == NULL) {
1526 kr = KERN_INVALID_VALUE;
1527 goto out;
1528 }
1529 idx = *idx_p;
1530 if (idx >= template->lt_cnt) {
1531 kr = KERN_INVALID_VALUE;
1532 goto out;
1533 }
1534 et = &template->lt_entries[idx];
1535 /* Ensure the caller asked for enough space up front */
1536 if (et->et_size != sizeof(struct ledger_entry)) {
1537 kr = KERN_INVALID_VALUE;
1538 goto out;
1539 }
1540
1541 et->et_flags |= LF_TRACK_CREDIT_ONLY;
1542 kr = KERN_SUCCESS;
1543
1544out:
1545 template_unlock(template);
1546
1547 return kr;
1548}
1549
1550/*
1551 * Add a callback to be executed when the resource goes into deficit.
1552 */
1553kern_return_t
1554ledger_set_callback(ledger_template_t template, int entry,
1555 ledger_callback_t func, const void *param0, const void *param1)
1556{
1557 struct entry_template *et;
1558 struct ledger_callback *old_cb, *new_cb;
1559 const uint16_t *idx_p;
1560 uint16_t idx;
1561
1562 idx_p = ledger_entry_to_template_idx(template, index: entry);
1563 if (idx_p == NULL) {
1564 return KERN_INVALID_VALUE;
1565 }
1566 idx = *idx_p;
1567
1568 if (idx >= template->lt_cnt) {
1569 return KERN_INVALID_VALUE;
1570 }
1571
1572 if (func) {
1573 new_cb = kalloc_type(struct ledger_callback, Z_WAITOK);
1574 new_cb->lc_func = func;
1575 new_cb->lc_param0 = param0;
1576 new_cb->lc_param1 = param1;
1577 } else {
1578 new_cb = NULL;
1579 }
1580
1581 template_lock(template);
1582 et = &template->lt_entries[idx];
1583 /* Ensure the caller asked for enough space up front */
1584 if (et->et_size != sizeof(struct ledger_entry)) {
1585 kfree_type(struct ledger_callback, new_cb);
1586 template_unlock(template);
1587 return KERN_INVALID_VALUE;
1588 }
1589 old_cb = et->et_callback;
1590 et->et_callback = new_cb;
1591 template_unlock(template);
1592 if (old_cb) {
1593 kfree_type(struct ledger_callback, old_cb);
1594 }
1595
1596 return KERN_SUCCESS;
1597}
1598
1599/*
1600 * Disable callback notification for a specific ledger entry.
1601 *
1602 * Otherwise, if using a ledger template which specified a
1603 * callback function (ledger_set_callback()), it will be invoked when
1604 * the resource goes into deficit.
1605 */
1606kern_return_t
1607ledger_disable_callback(ledger_t ledger, int entry)
1608{
1609 struct ledger_entry *le = NULL;
1610
1611 if (!is_entry_valid_and_active(l: ledger, entry)) {
1612 return KERN_INVALID_VALUE;
1613 }
1614
1615 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1616 /* Small entries can't have callbacks */
1617 return KERN_INVALID_ARGUMENT;
1618 }
1619
1620 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1621
1622 /*
1623 * le_warn_percent is used to indicate *if* this ledger has a warning configured,
1624 * in addition to what that warning level is set to.
1625 * This means a side-effect of ledger_disable_callback() is that the
1626 * warning level is forgotten.
1627 */
1628 le->le_warn_percent = LEDGER_PERCENT_NONE;
1629 flag_clear(flags: &le->le_flags, LEDGER_ACTION_CALLBACK);
1630 return KERN_SUCCESS;
1631}
1632
1633/*
1634 * Enable callback notification for a specific ledger entry.
1635 *
1636 * This is only needed if ledger_disable_callback() has previously
1637 * been invoked against an entry; there must already be a callback
1638 * configured.
1639 */
1640kern_return_t
1641ledger_enable_callback(ledger_t ledger, int entry)
1642{
1643 struct ledger_entry *le = NULL;
1644
1645 if (!is_entry_valid_and_active(l: ledger, entry)) {
1646 return KERN_INVALID_VALUE;
1647 }
1648
1649 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1650 /* Small entries can't have callbacks */
1651 return KERN_INVALID_ARGUMENT;
1652 }
1653
1654 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1655
1656 assert(entry_get_callback(ledger, entry) != NULL);
1657
1658 flag_set(flags: &le->le_flags, LEDGER_ACTION_CALLBACK);
1659 return KERN_SUCCESS;
1660}
1661
1662/*
1663 * Query the automatic refill period for this ledger entry.
1664 *
1665 * A period of 0 means this entry has none configured.
1666 */
1667kern_return_t
1668ledger_get_period(ledger_t ledger, int entry, uint64_t *period)
1669{
1670 struct ledger_entry *le;
1671
1672 if (!is_entry_valid_and_active(l: ledger, entry)) {
1673 return KERN_INVALID_VALUE;
1674 }
1675
1676 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1677 /* Small entries can't do refills */
1678 return KERN_INVALID_ARGUMENT;
1679 }
1680
1681 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1682
1683 *period = abstime_to_nsecs(abstime: le->_le.le_refill.le_refill_period);
1684 lprintf(("ledger_get_period: %llx\n", *period));
1685 return KERN_SUCCESS;
1686}
1687
1688/*
1689 * Adjust the automatic refill period.
1690 */
1691kern_return_t
1692ledger_set_period(ledger_t ledger, int entry, uint64_t period)
1693{
1694 struct ledger_entry *le = NULL;
1695
1696 if (!is_entry_valid_and_active(l: ledger, entry)) {
1697 return KERN_INVALID_VALUE;
1698 }
1699
1700 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1701 /* Small entries can't do refills */
1702 return KERN_INVALID_ARGUMENT;
1703 }
1704
1705 lprintf(("ledger_set_period: %llx\n", period));
1706
1707 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1708
1709 /*
1710 * A refill period refills the ledger in multiples of the limit,
1711 * so if you haven't set one yet, you need a lesson on ledgers.
1712 */
1713 assert(le->le_limit != LEDGER_LIMIT_INFINITY);
1714
1715 if (le->le_flags & LF_TRACKING_MAX) {
1716 /*
1717 * Refill is incompatible with rolling max tracking.
1718 */
1719 return KERN_INVALID_VALUE;
1720 }
1721
1722 le->_le.le_refill.le_refill_period = nsecs_to_abstime(nsecs: period);
1723
1724 /*
1725 * Set the 'starting time' for the next refill to now. Since
1726 * we're resetting the balance to zero here, we consider this
1727 * moment the starting time for accumulating a balance that
1728 * counts towards the limit.
1729 */
1730 le->_le.le_refill.le_last_refill = mach_absolute_time();
1731 ledger_zero_balance(ledger, entry);
1732
1733 flag_set(flags: &le->le_flags, LF_REFILL_SCHEDULED);
1734
1735 return KERN_SUCCESS;
1736}
1737
1738/*
1739 * Disable automatic refill.
1740 */
1741kern_return_t
1742ledger_disable_refill(ledger_t ledger, int entry)
1743{
1744 struct ledger_entry *le = NULL;
1745
1746 if (!is_entry_valid_and_active(l: ledger, entry)) {
1747 return KERN_INVALID_VALUE;
1748 }
1749
1750 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1751 /* Small entries can't do refills */
1752 return KERN_INVALID_ARGUMENT;
1753 }
1754
1755 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1756
1757 flag_clear(flags: &le->le_flags, LF_REFILL_SCHEDULED);
1758
1759 return KERN_SUCCESS;
1760}
1761
1762kern_return_t
1763ledger_get_actions(ledger_t ledger, int entry, int *actions)
1764{
1765 struct ledger_entry *le = NULL;
1766 *actions = 0;
1767
1768 if (!is_entry_valid_and_active(l: ledger, entry)) {
1769 return KERN_INVALID_VALUE;
1770 }
1771
1772 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1773 /* Small entries can't have actions */
1774 return KERN_INVALID_ARGUMENT;
1775 }
1776
1777 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1778
1779 *actions = le->le_flags & LEDGER_ACTION_MASK;
1780 lprintf(("ledger_get_actions: %#x\n", *actions));
1781 return KERN_SUCCESS;
1782}
1783
1784kern_return_t
1785ledger_set_action(ledger_t ledger, int entry, int action)
1786{
1787 lprintf(("ledger_set_action: %#x\n", action));
1788 struct ledger_entry *le = NULL;
1789
1790 if (!is_entry_valid_and_active(l: ledger, entry)) {
1791 return KERN_INVALID_VALUE;
1792 }
1793
1794 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
1795 /* Small entries can't have actions */
1796 return KERN_INVALID_ARGUMENT;
1797 }
1798
1799 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1800
1801 flag_set(flags: &le->le_flags, bit: action);
1802 return KERN_SUCCESS;
1803}
1804
1805kern_return_t
1806ledger_debit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount)
1807{
1808 struct ledger_entry *le;
1809 ledger_amount_t old, new;
1810 uint16_t entry_size = ENTRY_ID_SIZE(entry);
1811
1812 if (!is_entry_valid_and_active(l: ledger, entry) || (amount < 0)) {
1813 return KERN_INVALID_ARGUMENT;
1814 }
1815
1816 if (amount == 0) {
1817 return KERN_SUCCESS;
1818 }
1819
1820 if (entry_size == sizeof(struct ledger_entry_small)) {
1821 struct ledger_entry_small *les = &ledger->l_entries[ENTRY_ID_OFFSET(entry)];
1822 old = OSAddAtomic64(-amount, &les->les_credit);
1823 new = old - amount;
1824 } else if (entry_size == sizeof(struct ledger_entry)) {
1825 le = ledger_entry_identifier_to_entry(ledger, id: entry);
1826
1827 if (le->le_flags & LF_TRACK_CREDIT_ONLY) {
1828 assert(le->le_debit == 0);
1829 old = OSAddAtomic64(-amount, &le->le_credit);
1830 new = old - amount;
1831 } else {
1832 old = OSAddAtomic64(amount, &le->le_debit);
1833 new = old + amount;
1834 }
1835 } else {
1836 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, entry_size);
1837 }
1838 lprintf(("%p Debit %lld->%lld\n", thread, old, new));
1839
1840 if (thread) {
1841 ledger_entry_check_new_balance(thread, ledger, entry);
1842 }
1843
1844 return KERN_SUCCESS;
1845}
1846
1847kern_return_t
1848ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount)
1849{
1850 return ledger_debit_thread(thread: current_thread(), ledger, entry, amount);
1851}
1852
1853kern_return_t
1854ledger_debit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount)
1855{
1856 return ledger_debit_thread(NULL, ledger, entry, amount);
1857}
1858
1859void
1860ledger_ast(thread_t thread)
1861{
1862 struct ledger *l = thread->t_ledger;
1863 struct ledger *thl;
1864 struct ledger *coalition_ledger;
1865 uint32_t block;
1866 uint64_t now;
1867 uint8_t task_flags;
1868 uint8_t task_percentage;
1869 uint64_t task_interval;
1870
1871 kern_return_t ret;
1872 task_t task = get_threadtask(thread);
1873
1874 lprintf(("Ledger AST for %p\n", thread));
1875
1876 ASSERT(task != NULL);
1877 ASSERT(thread == current_thread());
1878
1879#if CONFIG_SCHED_RT_ALLOW
1880 /*
1881 * The RT policy may have forced a CPU limit on the thread. Check if
1882 * that's the case and apply the limit as requested.
1883 */
1884 spl_t s = splsched();
1885 thread_lock(thread);
1886
1887 int req_action = thread->t_ledger_req_action;
1888 uint8_t req_percentage = thread->t_ledger_req_percentage;
1889 uint64_t req_interval_ns = thread->t_ledger_req_interval_ms * NSEC_PER_MSEC;
1890
1891 thread->t_ledger_req_action = 0;
1892
1893 thread_unlock(thread);
1894 splx(s);
1895
1896 if (req_action != 0) {
1897 assert(req_action == THREAD_CPULIMIT_DISABLE ||
1898 req_action == THREAD_CPULIMIT_BLOCK);
1899
1900 if (req_action == THREAD_CPULIMIT_DISABLE &&
1901 (thread->options & TH_OPT_FORCED_LEDGER) != 0) {
1902 thread->options &= ~TH_OPT_FORCED_LEDGER;
1903 ret = thread_set_cpulimit(THREAD_CPULIMIT_DISABLE, 0, 0);
1904 assert3u(ret, ==, KERN_SUCCESS);
1905 }
1906
1907 if (req_action == THREAD_CPULIMIT_BLOCK) {
1908 thread->options &= ~TH_OPT_FORCED_LEDGER;
1909 ret = thread_set_cpulimit(THREAD_CPULIMIT_BLOCK,
1910 req_percentage, req_interval_ns);
1911 assert3u(ret, ==, KERN_SUCCESS);
1912 thread->options |= TH_OPT_FORCED_LEDGER;
1913 }
1914 }
1915#endif /* CONFIG_SCHED_RT_ALLOW */
1916
1917top:
1918 /*
1919 * Take a self-consistent snapshot of the CPU usage monitor parameters. The task
1920 * can change them at any point (with the task locked).
1921 */
1922 task_lock(task);
1923 task_flags = task->rusage_cpu_flags;
1924 task_percentage = task->rusage_cpu_perthr_percentage;
1925 task_interval = task->rusage_cpu_perthr_interval;
1926 task_unlock(task);
1927
1928 /*
1929 * Make sure this thread is up to date with regards to any task-wide per-thread
1930 * CPU limit, but only if it doesn't have a thread-private blocking CPU limit.
1931 */
1932 if (((task_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) != 0) &&
1933 ((thread->options & TH_OPT_PRVT_CPULIMIT) == 0)) {
1934 uint8_t percentage;
1935 uint64_t interval;
1936 int action;
1937
1938 thread_get_cpulimit(action: &action, percentage: &percentage, interval_ns: &interval);
1939
1940 /*
1941 * If the thread's CPU limits no longer match the task's, or the
1942 * task has a limit but the thread doesn't, update the limit.
1943 */
1944 if (((thread->options & TH_OPT_PROC_CPULIMIT) == 0) ||
1945 (interval != task_interval) || (percentage != task_percentage)) {
1946 thread_set_cpulimit(THREAD_CPULIMIT_EXCEPTION, percentage: task_percentage, interval_ns: task_interval);
1947 assert((thread->options & TH_OPT_PROC_CPULIMIT) != 0);
1948 }
1949 } else if (((task_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) == 0) &&
1950 (thread->options & TH_OPT_PROC_CPULIMIT)) {
1951 assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0);
1952
1953 /*
1954 * Task no longer has a per-thread CPU limit; remove this thread's
1955 * corresponding CPU limit.
1956 */
1957 thread_set_cpulimit(THREAD_CPULIMIT_DISABLE, percentage: 0, interval_ns: 0);
1958 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0);
1959 }
1960
1961 /*
1962 * If the task or thread is being terminated, let's just get on with it
1963 */
1964 if ((l == NULL) || !task->active || task->halting || !thread->active) {
1965 return;
1966 }
1967
1968 /*
1969 * Examine all entries in deficit to see which might be eligble for
1970 * an automatic refill, which require callbacks to be issued, and
1971 * which require blocking.
1972 */
1973 block = 0;
1974 now = mach_absolute_time();
1975
1976 /*
1977 * Note that thread->t_threadledger may have been changed by the
1978 * thread_set_cpulimit() call above - so don't examine it until afterwards.
1979 */
1980 thl = thread->t_threadledger;
1981 if (LEDGER_VALID(thl)) {
1982 block |= ledger_check_needblock(l: thl, now);
1983 }
1984 block |= ledger_check_needblock(l, now);
1985
1986 coalition_ledger = coalition_ledger_get_from_task(task);
1987 if (LEDGER_VALID(coalition_ledger)) {
1988 block |= ledger_check_needblock(l: coalition_ledger, now);
1989 }
1990 ledger_dereference(ledger: coalition_ledger);
1991 /*
1992 * If we are supposed to block on the availability of one or more
1993 * resources, find the first entry in deficit for which we should wait.
1994 * Schedule a refill if necessary and then sleep until the resource
1995 * becomes available.
1996 */
1997 if (block) {
1998 if (LEDGER_VALID(thl)) {
1999 ret = ledger_perform_blocking(l: thl);
2000 if (ret != KERN_SUCCESS) {
2001 goto top;
2002 }
2003 }
2004 ret = ledger_perform_blocking(l);
2005 if (ret != KERN_SUCCESS) {
2006 goto top;
2007 }
2008 } /* block */
2009}
2010
2011static uint32_t
2012ledger_check_needblock(ledger_t l, uint64_t now)
2013{
2014 int i;
2015 uint32_t flags, block = 0;
2016 struct ledger_entry *le;
2017 struct ledger_callback *lc;
2018 struct entry_template *et = NULL;
2019 ledger_template_t template = NULL;
2020
2021 template = l->l_template;
2022 assert(template != NULL);
2023 assert(template->lt_initialized);
2024 /*
2025 * The template has been initialized so the entries table can't change.
2026 * Thus we don't need to acquire the template lock or the inuse bit.
2027 */
2028
2029
2030 for (i = 0; i < template->lt_cnt; i++) {
2031 spl_t s;
2032 et = &template->lt_entries[i];
2033 if (et->et_size == sizeof(struct ledger_entry_small)) {
2034 /* Small entries don't track limits or have callbacks */
2035 continue;
2036 }
2037 assert(et->et_size == sizeof(struct ledger_entry));
2038 le = (struct ledger_entry *) &l->l_entries[et->et_offset];
2039
2040 TEMPLATE_INUSE(s, template);
2041 lc = template->lt_entries[i].et_callback;
2042 TEMPLATE_IDLE(s, template);
2043
2044 if (limit_exceeded(le) == FALSE) {
2045 if (le->le_flags & LEDGER_ACTION_CALLBACK) {
2046 /*
2047 * If needed, invoke the callback as a warning.
2048 * This needs to happen both when the balance rises above
2049 * the warning level, and also when it dips back below it.
2050 */
2051 assert(lc != NULL);
2052 /*
2053 * See comments for matching logic in ledger_check_new_balance().
2054 */
2055 if (warn_level_exceeded(le)) {
2056 flags = flag_set(flags: &le->le_flags, LF_WARNED);
2057 if ((flags & LF_WARNED) == 0) {
2058 lc->lc_func(LEDGER_WARNING_ROSE_ABOVE, lc->lc_param0, lc->lc_param1);
2059 }
2060 } else {
2061 flags = flag_clear(flags: &le->le_flags, LF_WARNED);
2062 if (flags & LF_WARNED) {
2063 lc->lc_func(LEDGER_WARNING_DIPPED_BELOW, lc->lc_param0, lc->lc_param1);
2064 }
2065 }
2066 }
2067#if DEBUG || DEVELOPMENT
2068 if (diag_mem_threshold_exceeded(le)) {
2069 if (le->le_flags & LEDGER_ACTION_CALLBACK) {
2070 assert(lc != NULL);
2071 flags = flag_set(&le->le_flags, LF_DIAG_WARNED);
2072 if ((flags & LF_DIAG_WARNED) == 0) {
2073 lc->lc_func(LEDGER_WARNING_DIAG_MEM_THRESHOLD, lc->lc_param0, lc->lc_param1);
2074 }
2075 }
2076 }
2077#endif
2078 continue;
2079 }
2080#if DEBUG || DEVELOPMENT
2081 if (diag_mem_threshold_exceeded(le)) {
2082 if (le->le_flags & LEDGER_ACTION_CALLBACK) {
2083 assert(lc != NULL);
2084 flags = flag_set(&le->le_flags, LF_DIAG_WARNED);
2085 if ((flags & LF_DIAG_WARNED) == 0) {
2086 lc->lc_func(LEDGER_WARNING_DIAG_MEM_THRESHOLD, lc->lc_param0, lc->lc_param1);
2087 }
2088 }
2089 }
2090#endif
2091
2092 /* We're over the limit, so refill if we are eligible and past due. */
2093 if (le->le_flags & LF_REFILL_SCHEDULED) {
2094 assert(!(le->le_flags & LF_TRACKING_MAX));
2095
2096 if ((le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period) <= now) {
2097 ledger_refill(now, ledger: l, entry: i);
2098 if (limit_exceeded(le) == FALSE) {
2099 continue;
2100 }
2101 }
2102 }
2103
2104 if (le->le_flags & LEDGER_ACTION_BLOCK) {
2105 block = 1;
2106 }
2107 if ((le->le_flags & LEDGER_ACTION_CALLBACK) == 0) {
2108 continue;
2109 }
2110
2111 /*
2112 * If the LEDGER_ACTION_CALLBACK flag is on, we expect there to
2113 * be a registered callback.
2114 */
2115 assert(lc != NULL);
2116 flags = flag_set(flags: &le->le_flags, LF_CALLED_BACK);
2117 /* Callback has already been called */
2118 if (flags & LF_CALLED_BACK) {
2119 continue;
2120 }
2121 lc->lc_func(FALSE, lc->lc_param0, lc->lc_param1);
2122 }
2123 return block;
2124}
2125
2126
2127/* return KERN_SUCCESS to continue, KERN_FAILURE to restart */
2128static kern_return_t
2129ledger_perform_blocking(ledger_t l)
2130{
2131 int i;
2132 kern_return_t ret;
2133 struct ledger_entry *le;
2134 ledger_template_t template = NULL;
2135 struct entry_template *et = NULL;
2136
2137 template = l->l_template;
2138 assert(template->lt_initialized);
2139
2140 for (i = 0; i < template->lt_cnt; i++) {
2141 et = &template->lt_entries[i];
2142 if (et->et_size != sizeof(struct ledger_entry)) {
2143 /* Small entries do not block for anything. */
2144 continue;
2145 }
2146 le = (struct ledger_entry *) &l->l_entries[et->et_offset];
2147 if ((!limit_exceeded(le)) ||
2148 ((le->le_flags & LEDGER_ACTION_BLOCK) == 0)) {
2149 continue;
2150 }
2151
2152 assert(!(le->le_flags & LF_TRACKING_MAX));
2153
2154 /* Prepare to sleep until the resource is refilled */
2155 ret = assert_wait_deadline(event: le, THREAD_INTERRUPTIBLE,
2156 deadline: le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period);
2157 if (ret != THREAD_WAITING) {
2158 return KERN_SUCCESS;
2159 }
2160
2161 /* Mark that somebody is waiting on this entry */
2162 flag_set(flags: &le->le_flags, LF_WAKE_NEEDED);
2163
2164 ret = thread_block_reason(THREAD_CONTINUE_NULL, NULL,
2165 AST_LEDGER);
2166 if (ret != THREAD_AWAKENED) {
2167 return KERN_SUCCESS;
2168 }
2169
2170 /*
2171 * The world may have changed while we were asleep.
2172 * Some other resource we need may have gone into
2173 * deficit. Or maybe we're supposed to die now.
2174 * Go back to the top and reevaluate.
2175 */
2176 return KERN_FAILURE;
2177 }
2178 return KERN_SUCCESS;
2179}
2180
2181
2182kern_return_t
2183ledger_get_entries(ledger_t ledger, int entry, ledger_amount_t *credit,
2184 ledger_amount_t *debit)
2185{
2186 struct ledger_entry *le = NULL;
2187 struct ledger_entry_small *les = NULL;
2188 uint16_t entry_size, entry_offset;
2189
2190 if (!is_entry_valid_and_active(l: ledger, entry)) {
2191 return KERN_INVALID_ARGUMENT;
2192 }
2193
2194 entry_size = ENTRY_ID_SIZE(entry);
2195 entry_offset = ENTRY_ID_OFFSET(entry);
2196 les = &ledger->l_entries[entry_offset];
2197 if (entry_size == sizeof(struct ledger_entry)) {
2198 le = (struct ledger_entry *)les;
2199 *credit = le->le_credit;
2200 *debit = le->le_debit;
2201 } else if (entry_size == sizeof(struct ledger_entry_small)) {
2202 *credit = les->les_credit;
2203 *debit = 0;
2204 } else {
2205 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, entry_size);
2206 }
2207
2208 return KERN_SUCCESS;
2209}
2210
2211kern_return_t
2212ledger_reset_callback_state(ledger_t ledger, int entry)
2213{
2214 struct ledger_entry *le = NULL;
2215
2216 if (!is_entry_valid_and_active(l: ledger, entry)) {
2217 return KERN_INVALID_ARGUMENT;
2218 }
2219
2220 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2221 /* small entries can't have callbacks */
2222 return KERN_INVALID_ARGUMENT;
2223 }
2224
2225 le = ledger_entry_identifier_to_entry(ledger, id: entry);
2226
2227 flag_clear(flags: &le->le_flags, LF_CALLED_BACK);
2228
2229 return KERN_SUCCESS;
2230}
2231
2232kern_return_t
2233ledger_disable_panic_on_negative(ledger_t ledger, int entry)
2234{
2235 volatile uint32_t *flags;
2236
2237 if (!is_entry_valid_and_active(l: ledger, entry)) {
2238 return KERN_INVALID_ARGUMENT;
2239 }
2240 flags = get_entry_flags(l: ledger, index: entry);
2241
2242 flag_clear(flags, LF_PANIC_ON_NEGATIVE);
2243
2244 return KERN_SUCCESS;
2245}
2246
2247kern_return_t
2248ledger_get_panic_on_negative(ledger_t ledger, int entry, int *panic_on_negative)
2249{
2250 volatile uint32_t flags;
2251
2252 if (!is_entry_valid_and_active(l: ledger, entry)) {
2253 return KERN_INVALID_ARGUMENT;
2254 }
2255 flags = *get_entry_flags(l: ledger, index: entry);
2256
2257 if (flags & LF_PANIC_ON_NEGATIVE) {
2258 *panic_on_negative = TRUE;
2259 } else {
2260 *panic_on_negative = FALSE;
2261 }
2262
2263 return KERN_SUCCESS;
2264}
2265
2266kern_return_t
2267ledger_get_balance(ledger_t ledger, int entry, ledger_amount_t *balance)
2268{
2269 kern_return_t kr;
2270 ledger_amount_t credit, debit;
2271
2272 kr = ledger_get_entries(ledger, entry, credit: &credit, debit: &debit);
2273 if (kr != KERN_SUCCESS) {
2274 return kr;
2275 }
2276 *balance = credit - debit;
2277
2278 return KERN_SUCCESS;
2279}
2280
2281int
2282ledger_template_info(void **buf, int *len)
2283{
2284 struct ledger_template_info *lti;
2285 struct entry_template *et;
2286 ledger_template_t template;
2287 int i;
2288 ledger_t l;
2289
2290 /*
2291 * Since all tasks share a ledger template, we'll just use the
2292 * caller's as the source.
2293 */
2294 l = current_task()->ledger;
2295 if ((*len < 0) || (l == NULL)) {
2296 return EINVAL;
2297 }
2298 template = l->l_template;
2299 assert(template);
2300 assert(template->lt_initialized);
2301
2302 if (*len > template->lt_cnt) {
2303 *len = template->lt_cnt;
2304 }
2305 lti = kalloc_data((*len) * sizeof(struct ledger_template_info),
2306 Z_WAITOK);
2307 if (lti == NULL) {
2308 return ENOMEM;
2309 }
2310 *buf = lti;
2311
2312 template_lock(template);
2313 et = template->lt_entries;
2314
2315 for (i = 0; i < *len; i++) {
2316 memset(s: lti, c: 0, n: sizeof(*lti));
2317 strlcpy(dst: lti->lti_name, src: et->et_key, LEDGER_NAME_MAX);
2318 strlcpy(dst: lti->lti_group, src: et->et_group, LEDGER_NAME_MAX);
2319 strlcpy(dst: lti->lti_units, src: et->et_units, LEDGER_NAME_MAX);
2320 et++;
2321 lti++;
2322 }
2323 template_unlock(template);
2324
2325 return 0;
2326}
2327
2328static kern_return_t
2329ledger_fill_entry_info(ledger_t ledger,
2330 int entry,
2331 struct ledger_entry_info *lei,
2332 uint64_t now)
2333{
2334 assert(ledger != NULL);
2335 assert(lei != NULL);
2336 if (!is_entry_valid(l: ledger, entry)) {
2337 return KERN_INVALID_ARGUMENT;
2338 }
2339 uint16_t entry_size, entry_offset;
2340 struct ledger_entry_small *les = NULL;
2341 struct ledger_entry *le = NULL;
2342 entry_size = ENTRY_ID_SIZE(entry);
2343 entry_offset = ENTRY_ID_OFFSET(entry);
2344
2345 les = &ledger->l_entries[entry_offset];
2346 memset(s: lei, c: 0, n: sizeof(*lei));
2347 if (entry_size == sizeof(struct ledger_entry_small)) {
2348 lei->lei_limit = LEDGER_LIMIT_INFINITY;
2349 lei->lei_credit = les->les_credit;
2350 lei->lei_debit = 0;
2351 lei->lei_refill_period = 0;
2352 lei->lei_last_refill = abstime_to_nsecs(abstime: now);
2353 } else if (entry_size == sizeof(struct ledger_entry)) {
2354 le = (struct ledger_entry *) les;
2355 lei->lei_limit = le->le_limit;
2356 lei->lei_credit = le->le_credit;
2357 lei->lei_debit = le->le_debit;
2358 lei->lei_refill_period = (le->le_flags & LF_REFILL_SCHEDULED) ?
2359 abstime_to_nsecs(abstime: le->_le.le_refill.le_refill_period) : 0;
2360 lei->lei_last_refill = abstime_to_nsecs(abstime: now - le->_le.le_refill.le_last_refill);
2361 } else {
2362 panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n", ledger, entry, entry_size);
2363 }
2364
2365 lei->lei_balance = lei->lei_credit - lei->lei_debit;
2366
2367 return KERN_SUCCESS;
2368}
2369
2370int
2371ledger_get_task_entry_info_multiple(task_t task, void **buf, int *len)
2372{
2373 struct ledger_entry_info *lei_buf = NULL, *lei_curr = NULL;
2374 uint64_t now = mach_absolute_time();
2375 vm_size_t size = 0;
2376 int i;
2377 ledger_t l;
2378 ledger_template_t template;
2379 struct entry_template *et = NULL;
2380
2381 if ((*len < 0) || ((l = task->ledger) == NULL)) {
2382 return EINVAL;
2383 }
2384 template = l->l_template;
2385 assert(template && template->lt_initialized);
2386
2387 if (*len > template->lt_cnt) {
2388 *len = template->lt_cnt;
2389 }
2390 size = (*len) * sizeof(struct ledger_entry_info);
2391 lei_buf = kalloc_data(size, Z_WAITOK);
2392 if (lei_buf == NULL) {
2393 return ENOMEM;
2394 }
2395 lei_curr = lei_buf;
2396
2397 for (i = 0; i < *len; i++) {
2398 et = &template->lt_entries[i];
2399 int index = ledger_entry_id_from_template_entry(et);
2400 if (ledger_fill_entry_info(ledger: l, entry: index, lei: lei_curr, now) != KERN_SUCCESS) {
2401 kfree_data(lei_buf, size);
2402 lei_buf = NULL;
2403 return EINVAL;
2404 }
2405 lei_curr++;
2406 }
2407
2408 *buf = lei_buf;
2409 return 0;
2410}
2411
2412void
2413ledger_get_entry_info(ledger_t ledger,
2414 int entry,
2415 struct ledger_entry_info *lei)
2416{
2417 uint64_t now = mach_absolute_time();
2418
2419 assert(ledger != NULL);
2420 assert(lei != NULL);
2421
2422 ledger_fill_entry_info(ledger, entry, lei, now);
2423}
2424
2425int
2426ledger_info(task_t task, struct ledger_info *info)
2427{
2428 ledger_t l;
2429
2430 if ((l = task->ledger) == NULL) {
2431 return ENOENT;
2432 }
2433
2434 memset(s: info, c: 0, n: sizeof(*info));
2435
2436 strlcpy(dst: info->li_name, src: l->l_template->lt_name, LEDGER_NAME_MAX);
2437 info->li_id = l->l_id;
2438 info->li_entries = l->l_template->lt_cnt;
2439 return 0;
2440}
2441
2442/*
2443 * Returns the amount that would be required to hit the limit.
2444 * Must be a valid, active, full-sized ledger.
2445 */
2446ledger_amount_t
2447ledger_get_remaining(ledger_t ledger, int entry)
2448{
2449 const struct ledger_entry *le =
2450 ledger_entry_identifier_to_entry(ledger, id: entry);
2451 const ledger_amount_t limit = le->le_limit;
2452 const ledger_amount_t balance = le->le_credit - le->le_debit;
2453
2454 /* +1 here as the limit isn't hit until the limit is exceeded. */
2455 return limit > balance ? limit - balance + 1 : 0;
2456}
2457
2458/*
2459 * Balances the ledger by modifying the debit only and sets the last refill time
2460 * to `now`.
2461 * WARNING: It is up to the caller to enforce consistency.
2462 * Must be a valid, active, full-sized ledger.
2463 */
2464void
2465ledger_restart(ledger_t ledger, int entry, uint64_t now)
2466{
2467 struct ledger_entry *le = ledger_entry_identifier_to_entry(ledger, id: entry);
2468
2469 le->le_debit = le->le_credit;
2470 le->_le.le_refill.le_last_refill = now;
2471}
2472
2473/*
2474 * Returns the amount of time that would have to pass to expire the current
2475 * interval.
2476 * Must be a valid, active, full-sized ledger.
2477 */
2478uint64_t
2479ledger_get_interval_remaining(ledger_t ledger, int entry, uint64_t now)
2480{
2481 const struct ledger_entry *le =
2482 ledger_entry_identifier_to_entry(ledger, id: entry);
2483
2484 if ((now - le->_le.le_refill.le_last_refill) >
2485 le->_le.le_refill.le_refill_period) {
2486 return 0;
2487 } else {
2488 return le->_le.le_refill.le_refill_period -
2489 (now - le->_le.le_refill.le_last_refill) + 1;
2490 }
2491}
2492
2493#ifdef LEDGER_DEBUG
2494int
2495ledger_limit(task_t task, struct ledger_limit_args *args)
2496{
2497 ledger_t l;
2498 int64_t limit;
2499 int idx;
2500
2501 if ((l = task->ledger) == NULL) {
2502 return EINVAL;
2503 }
2504
2505 idx = ledger_key_lookup(l->l_template, args->lla_name);
2506 if (idx < 0) {
2507 return EINVAL;
2508 }
2509 if (ENTRY_ID_SIZE(idx) == sizeof(ledger_entry_small)) {
2510 /* Small entries can't have limits */
2511 return EINVAL;
2512 }
2513
2514 /*
2515 * XXX - this doesn't really seem like the right place to have
2516 * a context-sensitive conversion of userspace units into kernel
2517 * units. For now I'll handwave and say that the ledger() system
2518 * call isn't meant for civilians to use - they should be using
2519 * the process policy interfaces.
2520 */
2521 if (idx == task_ledgers.cpu_time) {
2522 int64_t nsecs;
2523
2524 if (args->lla_refill_period) {
2525 /*
2526 * If a refill is scheduled, then the limit is
2527 * specified as a percentage of one CPU. The
2528 * syscall specifies the refill period in terms of
2529 * milliseconds, so we need to convert to nsecs.
2530 */
2531 args->lla_refill_period *= 1000000;
2532 nsecs = args->lla_limit *
2533 (args->lla_refill_period / 100);
2534 lprintf(("CPU limited to %lld nsecs per second\n",
2535 nsecs));
2536 } else {
2537 /*
2538 * If no refill is scheduled, then this is a
2539 * fixed amount of CPU time (in nsecs) that can
2540 * be consumed.
2541 */
2542 nsecs = args->lla_limit;
2543 lprintf(("CPU limited to %lld nsecs\n", nsecs));
2544 }
2545 limit = nsecs_to_abstime(nsecs);
2546 } else {
2547 limit = args->lla_limit;
2548 lprintf(("%s limited to %lld\n", args->lla_name, limit));
2549 }
2550
2551 if (args->lla_refill_period > 0) {
2552 ledger_set_period(l, idx, args->lla_refill_period);
2553 }
2554
2555 ledger_set_limit(l, idx, limit);
2556
2557 flag_set(ledger_entry_identifier_to_entry(l, idx)->le_flags, LEDGER_ACTION_BLOCK);
2558 return 0;
2559}
2560#endif
2561
2562/*
2563 * Adjust the diag mem threshold limit of a resource. The diag mem threshold limit only
2564 * works prescaled by 20 bits (mb)
2565 */
2566#if DEBUG || DEVELOPMENT
2567kern_return_t
2568ledger_set_diag_mem_threshold(ledger_t ledger, int entry, ledger_amount_t limit)
2569{
2570 struct ledger_entry *le;
2571
2572 if (!is_entry_valid_and_active(ledger, entry)) {
2573 return KERN_INVALID_VALUE;
2574 }
2575
2576 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2577 /* Small entries can't have limits */
2578 return KERN_INVALID_ARGUMENT;
2579 }
2580
2581 lprintf(("ledger_set_diag mem threshold_limit: %lld\n", limit));
2582 le = ledger_entry_identifier_to_entry(ledger, entry);
2583 le->le_diag_threshold_scaled = (int16_t)LEDGER_DIAG_MEM_AMOUNT_TO_THRESHOLD(limit);
2584 lprintf(("ledger_set_diag mem threshold_limit new : %lld\n", limit));
2585 flag_clear(&le->le_flags, LF_DIAG_WARNED);
2586
2587 return KERN_SUCCESS;
2588}
2589
2590kern_return_t
2591ledger_get_diag_mem_threshold(ledger_t ledger, int entry, ledger_amount_t *limit)
2592{
2593 struct ledger_entry *le;
2594
2595 if (!is_entry_valid_and_active(ledger, entry)) {
2596 return KERN_INVALID_VALUE;
2597 }
2598
2599 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2600 /* Small entries can't have limits */
2601 *limit = LEDGER_LIMIT_INFINITY;
2602 } else {
2603 le = ledger_entry_identifier_to_entry(ledger, entry);
2604 if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) {
2605 *limit = LEDGER_LIMIT_INFINITY;
2606 } else {
2607 *limit = LEDGER_DIAG_MEM_AMOUNT_FROM_THRESHOLD(le->le_diag_threshold_scaled);
2608 }
2609 }
2610
2611 lprintf(("ledger_get_diag mem threshold_limit: %lld\n", *limit));
2612
2613 return KERN_SUCCESS;
2614}
2615
2616static inline void
2617ledger_set_diag_mem_threshold_flag_disabled_internal(struct ledger_entry *le, bool value)
2618{
2619 if (value == true) {
2620 flag_set(&le->le_flags, LF_DIAG_DISABLED);
2621 } else {
2622 flag_clear(&le->le_flags, LF_DIAG_DISABLED);
2623 }
2624}
2625
2626static inline bool
2627ledger_is_diag_threshold_enabled_internal( struct ledger_entry *le)
2628{
2629 return ((le->le_flags & LF_DIAG_DISABLED) == 0)? true : false;
2630}
2631
2632/**
2633 * Disable the diagnostics threshold due to overlap with footprint limit
2634 */
2635kern_return_t
2636ledger_set_diag_mem_threshold_disabled(ledger_t ledger, int entry)
2637{
2638 struct ledger_entry *le;
2639
2640 if (!is_entry_valid_and_active(ledger, entry)) {
2641 return KERN_INVALID_VALUE;
2642 }
2643
2644 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2645 /* Small entries can't have limits */
2646 return KERN_INVALID_ARGUMENT;
2647 }
2648
2649 lprintf(("ledger_set_diag_mem_threshold_disabled"));
2650 le = ledger_entry_identifier_to_entry(ledger, entry);
2651 if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) {
2652 lprintf(("ledger_set_diag_mem_threshold_disabled, cannot disable a ledger entry that have no value, returning error"));
2653 return KERN_INVALID_ARGUMENT;
2654 }
2655 ledger_set_diag_mem_threshold_flag_disabled_internal(le, true);
2656 return KERN_SUCCESS;
2657}
2658/**
2659 * Enable the diagnostics threshold for a specific entry
2660 */
2661kern_return_t
2662ledger_set_diag_mem_threshold_enabled(ledger_t ledger, int entry)
2663{
2664 struct ledger_entry *le;
2665
2666 if (!is_entry_valid_and_active(ledger, entry)) {
2667 return KERN_INVALID_VALUE;
2668 }
2669
2670 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2671 /* Small entries can't have limits */
2672 return KERN_INVALID_ARGUMENT;
2673 }
2674
2675 lprintf(("ledger_set_diag_mem_threshold_enabled"));
2676 le = ledger_entry_identifier_to_entry(ledger, entry);
2677 /*
2678 * if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) {
2679 * lprintf(("ledger_set_diag_mem_threshold_enabled, cannot disable a ledger entry that have no value, returning error"));
2680 * return KERN_INVALID_ARGUMENT;
2681 * }
2682 */
2683 ledger_set_diag_mem_threshold_flag_disabled_internal(le, false);
2684
2685 return KERN_SUCCESS;
2686}
2687/**
2688 * Obtain the diagnostics threshold enabled flag. If the diagnostics threshold is enabled, returns true
2689 * else returns false.
2690 */
2691kern_return_t
2692ledger_is_diag_threshold_enabled(ledger_t ledger, int entry, bool *status)
2693{
2694 struct ledger_entry *le;
2695
2696 if (!is_entry_valid_and_active(ledger, entry)) {
2697 return KERN_INVALID_VALUE;
2698 }
2699
2700 if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) {
2701 /* Small entries can't have limits */
2702 return KERN_INVALID_ARGUMENT;
2703 }
2704
2705 lprintf(("ledger_is_diag_threshold_enabled"));
2706 le = ledger_entry_identifier_to_entry(ledger, entry);
2707 /*
2708 * if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) {
2709 * lprintf(("ledger_is_diag_threshold_enabled, get enabled flag for a ledger entry that have no value, returning error"));
2710 * return KERN_INVALID_ARGUMENT;
2711 * }
2712 */
2713 *status = ledger_is_diag_threshold_enabled_internal(le);
2714 return KERN_SUCCESS;
2715}
2716#endif // DEBUG || DEVELOPMENT
2717