1 | /* |
2 | * Copyright (c) 2010-2020 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | |
32 | #include <kern/kern_types.h> |
33 | #include <kern/ledger.h> |
34 | #include <kern/kalloc.h> |
35 | #include <kern/task.h> |
36 | #include <kern/thread.h> |
37 | #include <kern/coalition.h> |
38 | |
39 | #include <kern/processor.h> |
40 | #include <kern/machine.h> |
41 | #include <kern/queue.h> |
42 | #include <kern/policy_internal.h> |
43 | |
44 | #include <sys/errno.h> |
45 | |
46 | #include <libkern/OSAtomic.h> |
47 | #include <mach/mach_types.h> |
48 | #include <os/overflow.h> |
49 | |
50 | #include <vm/pmap.h> |
51 | |
52 | /* |
53 | * Ledger entry flags. Bits in second nibble (masked by 0xF0) are used for |
54 | * ledger actions (LEDGER_ACTION_BLOCK, etc). |
55 | */ |
56 | #define LF_ENTRY_ACTIVE 0x0001 /* entry is active if set */ |
57 | #define LF_WAKE_NEEDED 0x0100 /* one or more threads are asleep */ |
58 | #define LF_WAKE_INPROGRESS 0x0200 /* the wait queue is being processed */ |
59 | #define LF_REFILL_SCHEDULED 0x0400 /* a refill timer has been set */ |
60 | #define LF_REFILL_INPROGRESS 0x0800 /* the ledger is being refilled */ |
61 | #define LF_CALLED_BACK 0x1000 /* callback was called for balance in deficit */ |
62 | #define LF_WARNED 0x2000 /* callback was called for balance warning */ |
63 | #define LF_TRACKING_MAX 0x4000 /* track max balance. Exclusive w.r.t refill */ |
64 | #define LF_PANIC_ON_NEGATIVE 0x8000 /* panic if it goes negative */ |
65 | #define LF_TRACK_CREDIT_ONLY 0x10000 /* only update "credit" */ |
66 | #define LF_DIAG_WARNED 0x20000 /* callback was called for balance diag */ |
67 | #define LF_DIAG_DISABLED 0x40000 /* diagnostics threshold are disabled at the moment */ |
68 | |
69 | |
70 | /* |
71 | * Ledger entry IDs are actually a tuple of (size, offset). |
72 | * For backwards compatibility, they're stored in an int. |
73 | * Size is stored in the upper 16 bits, and offset is stored in the lower 16 bits. |
74 | * |
75 | * Use the ENTRY_ID_SIZE and ENTRY_ID_OFFSET macros to extract size and offset. |
76 | */ |
77 | #define ENTRY_ID_SIZE_SHIFT 16 |
78 | #define ENTRY_ID_OFFSET_MASK ((1 << ENTRY_ID_SIZE_SHIFT) - 1) |
79 | #define ENTRY_ID_OFFSET(x) ((x) & (ENTRY_ID_OFFSET_MASK)) |
80 | #define ENTRY_ID_SIZE_MASK (ENTRY_ID_OFFSET_MASK << ENTRY_ID_SIZE_SHIFT) |
81 | #define ENTRY_ID_SIZE(x) ((((uint32_t) (x)) & (ENTRY_ID_SIZE_MASK)) >> ENTRY_ID_SIZE_SHIFT) |
82 | _Static_assert(((sizeof(struct ledger_entry_small) << ENTRY_ID_SIZE_SHIFT) | (UINT16_MAX / sizeof(struct ledger_entry_small))) > 0, "Valid ledger index < 0" ); |
83 | _Static_assert(((sizeof(struct ledger_entry) << ENTRY_ID_SIZE_SHIFT) | (UINT16_MAX / sizeof(struct ledger_entry_small))) > 0, "Valid ledger index < 0" ); |
84 | _Static_assert(sizeof(int) * 8 >= ENTRY_ID_SIZE_SHIFT * 2, "Ledger indices don't fit in an int." ); |
85 | #define MAX_LEDGER_ENTRIES (UINT16_MAX / sizeof(struct ledger_entry_small)) |
86 | |
87 | #define LEDGER_DIAG_MEM_THRESHOLD_SHIFT 20 |
88 | #define LEDGER_DIAG_MEM_AMOUNT_TO_THRESHOLD(X) ((X) >> (LEDGER_DIAG_MEM_THRESHOLD_SHIFT)) |
89 | #define LEDGER_DIAG_MEM_AMOUNT_FROM_THRESHOLD(X) (((ledger_amount_t)(X)) << (LEDGER_DIAG_MEM_THRESHOLD_SHIFT)) |
90 | |
91 | /* These features can fit in a small ledger entry. All others require a full size ledger entry */ |
92 | #define LEDGER_ENTRY_SMALL_FLAGS (LEDGER_ENTRY_ALLOW_PANIC_ON_NEGATIVE | LEDGER_ENTRY_ALLOW_INACTIVE) |
93 | |
94 | /* Turn on to debug invalid ledger accesses */ |
95 | #if MACH_ASSERT |
96 | #define PANIC_ON_INVALID_LEDGER_ACCESS 1 |
97 | #endif /* MACH_ASSERT */ |
98 | |
99 | static inline volatile uint32_t * |
100 | get_entry_flags(ledger_t l, int index) |
101 | { |
102 | assert(l != NULL); |
103 | |
104 | uint16_t size, offset; |
105 | size = ENTRY_ID_SIZE(index); |
106 | offset = ENTRY_ID_OFFSET(index); |
107 | struct ledger_entry_small *les = &l->l_entries[offset]; |
108 | if (size == sizeof(struct ledger_entry)) { |
109 | return &((struct ledger_entry *)les)->le_flags; |
110 | } else if (size == sizeof(struct ledger_entry_small)) { |
111 | return &les->les_flags; |
112 | } else { |
113 | panic("Unknown ledger entry size! ledger=%p, index=0x%x, entry_size=%d\n" , l, index, size); |
114 | } |
115 | } |
116 | |
117 | #if PANIC_ON_INVALID_LEDGER_ACCESS |
118 | #define INVALID_LEDGER_ACCESS(l, e) if ((e) != -1) panic("Invalid ledger access: ledger=%p, entry=0x%x, entry_size=0x%x, entry_offset=0x%x\n", \ |
119 | (l), (e), (ENTRY_ID_SIZE((e))), ENTRY_ID_OFFSET((e))); |
120 | #else |
121 | #define INVALID_LEDGER_ACCESS(l, e) |
122 | #endif /* PANIC_ON_INVALID_LEDGER_ACCESS */ |
123 | |
124 | /* Determine whether a ledger entry exists */ |
125 | static inline bool |
126 | is_entry_valid(ledger_t l, int entry) |
127 | { |
128 | uint32_t size, offset, end_offset; |
129 | size = ENTRY_ID_SIZE(entry); |
130 | offset = ENTRY_ID_OFFSET(entry); |
131 | if (l == NULL) { |
132 | return false; |
133 | } |
134 | if (os_mul_overflow(offset, sizeof(struct ledger_entry_small), &offset) || offset >= l->l_size) { |
135 | INVALID_LEDGER_ACCESS(l, entry); |
136 | return false; |
137 | } |
138 | if (os_add_overflow(size, offset, &end_offset) || end_offset > l->l_size) { |
139 | INVALID_LEDGER_ACCESS(l, entry); |
140 | return false; |
141 | } |
142 | return true; |
143 | } |
144 | |
145 | static inline bool |
146 | is_entry_active(ledger_t l, int entry) |
147 | { |
148 | uint32_t flags = *get_entry_flags(l, index: entry); |
149 | if ((flags & LF_ENTRY_ACTIVE) != LF_ENTRY_ACTIVE) { |
150 | return false; |
151 | } |
152 | |
153 | return true; |
154 | } |
155 | |
156 | static inline bool |
157 | is_entry_valid_and_active(ledger_t l, int entry) |
158 | { |
159 | return is_entry_valid(l, entry) && is_entry_active(l, entry); |
160 | } |
161 | |
162 | #define ASSERT(a) assert(a) |
163 | |
164 | #ifdef LEDGER_DEBUG |
165 | int ledger_debug = 0; |
166 | |
167 | #define lprintf(a) if (ledger_debug) { \ |
168 | printf("%lld ", abstime_to_nsecs(mach_absolute_time() / 1000000)); \ |
169 | printf a ; \ |
170 | } |
171 | #else |
172 | #define lprintf(a) |
173 | #endif |
174 | |
175 | struct ledger_callback { |
176 | ledger_callback_t lc_func; |
177 | const void *lc_param0; |
178 | const void *lc_param1; |
179 | }; |
180 | |
181 | struct entry_template { |
182 | char et_key[LEDGER_NAME_MAX]; |
183 | char et_group[LEDGER_NAME_MAX]; |
184 | char et_units[LEDGER_NAME_MAX]; |
185 | uint32_t et_flags; |
186 | uint16_t et_size; |
187 | uint16_t et_offset; |
188 | struct ledger_callback *et_callback; |
189 | }; |
190 | |
191 | LCK_GRP_DECLARE(ledger_lck_grp, "ledger" ); |
192 | os_refgrp_decl(static, ledger_refgrp, "ledger" , NULL); |
193 | |
194 | /* |
195 | * Modifying the reference count, table size, table contents, lt_next_offset, or lt_entries_lut, |
196 | * requires holding the lt_lock. Modfying the table address requires both |
197 | * lt_lock and setting the inuse bit. This means that the lt_entries field can |
198 | * be safely dereferenced if you hold either the lock or the inuse bit. The |
199 | * inuse bit exists solely to allow us to swap in a new, larger entries |
200 | * table without requiring a full lock to be acquired on each lookup. |
201 | * Accordingly, the inuse bit should never be held for longer than it takes |
202 | * to extract a value from the table - i.e., 2 or 3 memory references. |
203 | */ |
204 | struct ledger_template { |
205 | const char *lt_name; |
206 | int lt_refs; |
207 | volatile uint32_t lt_inuse; |
208 | lck_mtx_t lt_lock; |
209 | zone_t lt_zone; |
210 | bool lt_initialized; |
211 | uint16_t lt_next_offset; |
212 | uint16_t lt_cnt; |
213 | uint16_t lt_table_size; |
214 | struct entry_template *lt_entries; |
215 | /* Lookup table to go from entry_offset to index in the lt_entries table. */ |
216 | uint16_t *lt_entries_lut; |
217 | }; |
218 | |
219 | static inline uint16_t |
220 | ledger_template_entries_lut_size(uint16_t lt_table_size) |
221 | { |
222 | /* |
223 | * The lookup table needs to be big enough to store lt_table_size entries of the largest |
224 | * entry size (struct ledger_entry) given a stride of the smallest entry size (struct ledger_entry_small) |
225 | */ |
226 | if (os_mul_overflow(lt_table_size, (sizeof(struct ledger_entry) / sizeof(struct ledger_entry_small)), <_table_size)) { |
227 | /* |
228 | * This means MAX_LEDGER_ENTRIES is misconfigured or |
229 | * someone has accidently passed in an lt_table_size that is > MAX_LEDGER_ENTRIES |
230 | */ |
231 | panic("Attempt to create a lookup table for a ledger template with too many entries. lt_table_size=%u, MAX_LEDGER_ENTRIES=%lu\n" , lt_table_size, MAX_LEDGER_ENTRIES); |
232 | } |
233 | return lt_table_size; |
234 | } |
235 | |
236 | #define template_lock(template) lck_mtx_lock(&(template)->lt_lock) |
237 | #define template_unlock(template) lck_mtx_unlock(&(template)->lt_lock) |
238 | |
239 | #define TEMPLATE_INUSE(s, t) { \ |
240 | s = splsched(); \ |
241 | while (OSCompareAndSwap(0, 1, &((t)->lt_inuse))) \ |
242 | ; \ |
243 | } |
244 | |
245 | #define TEMPLATE_IDLE(s, t) { \ |
246 | (t)->lt_inuse = 0; \ |
247 | splx(s); \ |
248 | } |
249 | |
250 | static int ledger_cnt = 0; |
251 | /* ledger ast helper functions */ |
252 | static uint32_t ledger_check_needblock(ledger_t l, uint64_t now); |
253 | static kern_return_t ledger_perform_blocking(ledger_t l); |
254 | static uint32_t flag_set(volatile uint32_t *flags, uint32_t bit); |
255 | static uint32_t flag_clear(volatile uint32_t *flags, uint32_t bit); |
256 | |
257 | static void ledger_entry_check_new_balance(thread_t thread, ledger_t ledger, |
258 | int entry); |
259 | #if DEBUG || DEVELOPMENT |
260 | static inline bool ledger_is_diag_threshold_enabled_internal(struct ledger_entry *le); |
261 | #endif |
262 | #if 0 |
263 | static void |
264 | debug_callback(const void *p0, __unused const void *p1) |
265 | { |
266 | printf("ledger: resource exhausted [%s] for task %p\n" , |
267 | (const char *)p0, p1); |
268 | } |
269 | #endif |
270 | |
271 | /************************************/ |
272 | |
273 | static uint64_t |
274 | abstime_to_nsecs(uint64_t abstime) |
275 | { |
276 | uint64_t nsecs; |
277 | |
278 | absolutetime_to_nanoseconds(abstime, result: &nsecs); |
279 | return nsecs; |
280 | } |
281 | |
282 | static uint64_t |
283 | nsecs_to_abstime(uint64_t nsecs) |
284 | { |
285 | uint64_t abstime; |
286 | |
287 | nanoseconds_to_absolutetime(nanoseconds: nsecs, result: &abstime); |
288 | return abstime; |
289 | } |
290 | |
291 | static const uint16_t * |
292 | ledger_entry_to_template_idx(ledger_template_t template, int index) |
293 | { |
294 | uint16_t offset = ENTRY_ID_OFFSET(index); |
295 | if (offset / sizeof(struct ledger_entry_small) >= template->lt_cnt) { |
296 | return NULL; |
297 | } |
298 | |
299 | return &template->lt_entries_lut[offset]; |
300 | } |
301 | |
302 | /* |
303 | * Convert the id to a ledger entry. |
304 | * It's the callers responsibility to ensure the id is valid and a full size |
305 | * ledger entry. |
306 | */ |
307 | static struct ledger_entry * |
308 | ledger_entry_identifier_to_entry(ledger_t ledger, int id) |
309 | { |
310 | assert(is_entry_valid(ledger, id)); |
311 | assert(ENTRY_ID_SIZE(id) == sizeof(struct ledger_entry)); |
312 | return (struct ledger_entry *) &ledger->l_entries[ENTRY_ID_OFFSET(id)]; |
313 | } |
314 | |
315 | |
316 | ledger_template_t |
317 | ledger_template_create(const char *name) |
318 | { |
319 | ledger_template_t template; |
320 | |
321 | template = kalloc_type(struct ledger_template, Z_WAITOK | Z_ZERO | Z_NOFAIL); |
322 | template->lt_name = name; |
323 | template->lt_refs = 1; |
324 | template->lt_table_size = 1; |
325 | lck_mtx_init(lck: &template->lt_lock, grp: &ledger_lck_grp, LCK_ATTR_NULL); |
326 | |
327 | template->lt_entries = kalloc_type(struct entry_template, |
328 | template->lt_table_size, Z_WAITOK | Z_ZERO); |
329 | if (template->lt_entries == NULL) { |
330 | kfree_type(struct ledger_template, template); |
331 | template = NULL; |
332 | } |
333 | template->lt_entries_lut = kalloc_type(uint16_t, ledger_template_entries_lut_size(template->lt_table_size), |
334 | Z_WAITOK | Z_ZERO); |
335 | if (template->lt_entries_lut == NULL) { |
336 | kfree_type(struct entry_template, template->lt_entries); |
337 | kfree_type(struct ledger_template, template); |
338 | template = NULL; |
339 | } |
340 | |
341 | return template; |
342 | } |
343 | |
344 | ledger_template_t |
345 | ledger_template_copy(ledger_template_t template, const char *name) |
346 | { |
347 | struct entry_template * new_entries = NULL; |
348 | uint16_t *new_entries_lut = NULL; |
349 | size_t new_entries_lut_size = 0; |
350 | ledger_template_t new_template = ledger_template_create(name); |
351 | |
352 | if (new_template == NULL) { |
353 | return new_template; |
354 | } |
355 | |
356 | template_lock(template); |
357 | assert(template->lt_initialized); |
358 | |
359 | new_entries = kalloc_type(struct entry_template, template->lt_table_size, |
360 | Z_WAITOK | Z_ZERO); |
361 | |
362 | if (new_entries == NULL) { |
363 | /* Tear down the new template; we've failed. :( */ |
364 | ledger_template_dereference(template: new_template); |
365 | new_template = NULL; |
366 | goto out; |
367 | } |
368 | new_entries_lut_size = ledger_template_entries_lut_size(lt_table_size: template->lt_table_size); |
369 | |
370 | new_entries_lut = kalloc_type(uint16_t, new_entries_lut_size, |
371 | Z_WAITOK | Z_ZERO); |
372 | if (new_entries_lut == NULL) { |
373 | /* Tear down the new template; we've failed. :( */ |
374 | ledger_template_dereference(template: new_template); |
375 | new_template = NULL; |
376 | goto out; |
377 | } |
378 | |
379 | /* Copy the template entries. */ |
380 | bcopy(src: template->lt_entries, dst: new_entries, n: sizeof(struct entry_template) * template->lt_table_size); |
381 | kfree_type(struct entry_template, new_template->lt_table_size, new_template->lt_entries); |
382 | /* Copy the look up table. */ |
383 | bcopy(src: template->lt_entries_lut, dst: new_entries_lut, n: sizeof(uint16_t) * new_entries_lut_size); |
384 | kfree_type(uint16_t, ledger_template_entries_lut_size(new_template->lt_table_size), new_template->lt_entries_lut); |
385 | |
386 | new_template->lt_entries = new_entries; |
387 | new_template->lt_table_size = template->lt_table_size; |
388 | new_template->lt_cnt = template->lt_cnt; |
389 | new_template->lt_next_offset = template->lt_next_offset; |
390 | new_template->lt_entries_lut = new_entries_lut; |
391 | |
392 | out: |
393 | template_unlock(template); |
394 | |
395 | return new_template; |
396 | } |
397 | |
398 | void |
399 | ledger_template_dereference(ledger_template_t template) |
400 | { |
401 | template_lock(template); |
402 | template->lt_refs--; |
403 | template_unlock(template); |
404 | |
405 | if (template->lt_refs == 0) { |
406 | kfree_type(struct entry_template, template->lt_table_size, template->lt_entries); |
407 | kfree_type(uint16_t, ledger_template_entries_lut_size(template->lt_table_size), template->lt_entries_lut); |
408 | lck_mtx_destroy(lck: &template->lt_lock, grp: &ledger_lck_grp); |
409 | if (template->lt_zone) { |
410 | zdestroy(zone: template->lt_zone); |
411 | } |
412 | kfree_type(struct ledger_template, template); |
413 | } |
414 | } |
415 | |
416 | static inline int |
417 | ledger_entry_id(uint16_t size, uint16_t offset) |
418 | { |
419 | int idx = offset; |
420 | idx |= (size << ENTRY_ID_SIZE_SHIFT); |
421 | assert(idx >= 0); |
422 | return idx; |
423 | } |
424 | |
425 | static inline int |
426 | ledger_entry_id_from_template_entry(const struct entry_template *et) |
427 | { |
428 | return ledger_entry_id(size: et->et_size, offset: et->et_offset); |
429 | } |
430 | |
431 | int |
432 | ledger_entry_add_with_flags(ledger_template_t template, const char *key, |
433 | const char *group, const char *units, uint64_t flags) |
434 | { |
435 | uint16_t template_idx; |
436 | struct entry_template *et; |
437 | uint16_t size = 0, next_offset = 0, entry_idx = 0; |
438 | |
439 | if ((key == NULL) || (strlen(s: key) >= LEDGER_NAME_MAX) || (template->lt_zone != NULL)) { |
440 | return -1; |
441 | } |
442 | |
443 | template_lock(template); |
444 | |
445 | /* Make sure we have space for this entry */ |
446 | if (template->lt_cnt == MAX_LEDGER_ENTRIES) { |
447 | template_unlock(template); |
448 | return -1; |
449 | } |
450 | |
451 | /* If the table is full, attempt to double its size */ |
452 | if (template->lt_cnt == template->lt_table_size) { |
453 | struct entry_template *new_entries, *old_entries; |
454 | uint16_t *new_entries_lut = NULL, *old_entries_lut = NULL; |
455 | uint16_t old_cnt, new_cnt; |
456 | spl_t s; |
457 | |
458 | old_cnt = template->lt_table_size; |
459 | /* double old_sz allocation, but check for overflow */ |
460 | if (os_mul_overflow(old_cnt, 2, &new_cnt)) { |
461 | template_unlock(template); |
462 | return -1; |
463 | } |
464 | |
465 | if (new_cnt > MAX_LEDGER_ENTRIES) { |
466 | template_unlock(template); |
467 | panic("Attempt to create a ledger template with more than MAX_LEDGER_ENTRIES. MAX_LEDGER_ENTRIES=%lu, old_cnt=%u, new_cnt=%u\n" , MAX_LEDGER_ENTRIES, old_cnt, new_cnt); |
468 | } |
469 | |
470 | new_entries = kalloc_type(struct entry_template, new_cnt, |
471 | Z_WAITOK | Z_ZERO); |
472 | if (new_entries == NULL) { |
473 | template_unlock(template); |
474 | return -1; |
475 | } |
476 | new_entries_lut = kalloc_type(uint16_t, ledger_template_entries_lut_size(new_cnt), |
477 | Z_WAITOK | Z_ZERO); |
478 | if (new_entries_lut == NULL) { |
479 | template_unlock(template); |
480 | kfree_type(struct entry_template, new_cnt, new_entries); |
481 | return -1; |
482 | } |
483 | |
484 | memcpy(dst: new_entries, src: template->lt_entries, |
485 | n: old_cnt * sizeof(struct entry_template)); |
486 | template->lt_table_size = new_cnt; |
487 | |
488 | memcpy(dst: new_entries_lut, src: template->lt_entries_lut, |
489 | n: ledger_template_entries_lut_size(lt_table_size: old_cnt) * sizeof(uint16_t)); |
490 | |
491 | old_entries = template->lt_entries; |
492 | old_entries_lut = template->lt_entries_lut; |
493 | |
494 | TEMPLATE_INUSE(s, template); |
495 | template->lt_entries = new_entries; |
496 | template->lt_entries_lut = new_entries_lut; |
497 | TEMPLATE_IDLE(s, template); |
498 | |
499 | kfree_type(struct entry_template, old_cnt, old_entries); |
500 | kfree_type(uint16_t, ledger_template_entries_lut_size(old_cnt), old_entries_lut); |
501 | } |
502 | |
503 | et = &template->lt_entries[template->lt_cnt]; |
504 | strlcpy(dst: et->et_key, src: key, LEDGER_NAME_MAX); |
505 | strlcpy(dst: et->et_group, src: group, LEDGER_NAME_MAX); |
506 | strlcpy(dst: et->et_units, src: units, LEDGER_NAME_MAX); |
507 | et->et_flags = LF_ENTRY_ACTIVE; |
508 | /* |
509 | * Currently we only have two types of variable sized entries |
510 | * CREDIT_ONLY and full-fledged leger_entry. |
511 | * In the future, we can add more gradations based on the flags. |
512 | */ |
513 | if ((flags & ~(LEDGER_ENTRY_SMALL_FLAGS)) == 0) { |
514 | size = sizeof(struct ledger_entry_small); |
515 | et->et_flags |= LF_TRACK_CREDIT_ONLY; |
516 | } else { |
517 | size = sizeof(struct ledger_entry); |
518 | } |
519 | et->et_size = size; |
520 | et->et_offset = (template->lt_next_offset / sizeof(struct ledger_entry_small)); |
521 | et->et_callback = NULL; |
522 | |
523 | template_idx = template->lt_cnt++; |
524 | next_offset = template->lt_next_offset; |
525 | entry_idx = next_offset / sizeof(struct ledger_entry_small); |
526 | template->lt_next_offset += size; |
527 | assert(template->lt_next_offset > next_offset); |
528 | template->lt_entries_lut[entry_idx] = template_idx; |
529 | template_unlock(template); |
530 | |
531 | return ledger_entry_id(size, offset: entry_idx); |
532 | } |
533 | |
534 | /* |
535 | * Add a new entry to the list of entries in a ledger template. There is |
536 | * currently no mechanism to remove an entry. Implementing such a mechanism |
537 | * would require us to maintain per-entry reference counts, which we would |
538 | * prefer to avoid if possible. |
539 | */ |
540 | int |
541 | ledger_entry_add(ledger_template_t template, const char *key, |
542 | const char *group, const char *units) |
543 | { |
544 | /* |
545 | * When using the legacy interface we have to be pessimistic |
546 | * and allocate memory for all of the features. |
547 | */ |
548 | return ledger_entry_add_with_flags(template, key, group, units, |
549 | flags: LEDGER_ENTRY_ALLOW_CALLBACK | LEDGER_ENTRY_ALLOW_MAXIMUM | |
550 | LEDGER_ENTRY_ALLOW_DEBIT | LEDGER_ENTRY_ALLOW_LIMIT | |
551 | LEDGER_ENTRY_ALLOW_ACTION | LEDGER_ENTRY_ALLOW_INACTIVE); |
552 | } |
553 | |
554 | |
555 | kern_return_t |
556 | ledger_entry_setactive(ledger_t ledger, int entry) |
557 | { |
558 | volatile uint32_t *flags = NULL; |
559 | |
560 | if (!is_entry_valid(l: ledger, entry)) { |
561 | return KERN_INVALID_ARGUMENT; |
562 | } |
563 | |
564 | flags = get_entry_flags(l: ledger, index: entry); |
565 | |
566 | if ((*flags & LF_ENTRY_ACTIVE) == 0) { |
567 | flag_set(flags, LF_ENTRY_ACTIVE); |
568 | } |
569 | return KERN_SUCCESS; |
570 | } |
571 | |
572 | |
573 | int |
574 | ledger_key_lookup(ledger_template_t template, const char *key) |
575 | { |
576 | int id = -1; |
577 | struct entry_template *et = NULL; |
578 | |
579 | template_lock(template); |
580 | if (template->lt_entries != NULL) { |
581 | for (uint16_t idx = 0; idx < template->lt_cnt; idx++) { |
582 | et = &template->lt_entries[idx]; |
583 | if (strcmp(s1: key, s2: et->et_key) == 0) { |
584 | id = ledger_entry_id(size: et->et_size, offset: et->et_offset); |
585 | break; |
586 | } |
587 | } |
588 | } |
589 | |
590 | template_unlock(template); |
591 | |
592 | return id; |
593 | } |
594 | |
595 | /* |
596 | * Complete the initialization of ledger template |
597 | * by initializing ledger zone. After initializing |
598 | * the ledger zone, adding an entry in the ledger |
599 | * template will fail. |
600 | */ |
601 | void |
602 | ledger_template_complete(ledger_template_t template) |
603 | { |
604 | size_t ledger_size; |
605 | ledger_size = sizeof(struct ledger) + template->lt_next_offset; |
606 | assert(ledger_size > sizeof(struct ledger)); |
607 | template->lt_zone = zone_create(name: template->lt_name, size: ledger_size, |
608 | flags: ZC_PGZ_USE_GUARDS | ZC_DESTRUCTIBLE); |
609 | template->lt_initialized = true; |
610 | } |
611 | |
612 | /* |
613 | * Like ledger_template_complete, except we'll ask |
614 | * the pmap layer to manage allocations for us. |
615 | * Meant for ledgers that should be owned by the |
616 | * pmap layer. |
617 | */ |
618 | void |
619 | ledger_template_complete_secure_alloc(ledger_template_t template) |
620 | { |
621 | size_t ledger_size; |
622 | ledger_size = sizeof(struct ledger) + template->lt_next_offset; |
623 | |
624 | /** |
625 | * Ensure that the amount of space being allocated by the PPL for each |
626 | * ledger is large enough. |
627 | */ |
628 | pmap_ledger_verify_size(ledger_size); |
629 | template->lt_initialized = true; |
630 | } |
631 | |
632 | /* |
633 | * Create a new ledger based on the specified template. As part of the |
634 | * ledger creation we need to allocate space for a table of ledger entries. |
635 | * The size of the table is based on the size of the template at the time |
636 | * the ledger is created. If additional entries are added to the template |
637 | * after the ledger is created, they will not be tracked in this ledger. |
638 | */ |
639 | ledger_t |
640 | ledger_instantiate(ledger_template_t template, int entry_type) |
641 | { |
642 | ledger_t ledger; |
643 | uint16_t entries_size; |
644 | uint16_t num_entries; |
645 | uint16_t i; |
646 | |
647 | template_lock(template); |
648 | template->lt_refs++; |
649 | entries_size = template->lt_next_offset; |
650 | num_entries = template->lt_cnt; |
651 | template_unlock(template); |
652 | |
653 | if (template->lt_zone) { |
654 | ledger = (ledger_t)zalloc(zone: template->lt_zone); |
655 | } else { |
656 | /** |
657 | * If the template doesn't contain a zone to allocate ledger objects |
658 | * from, then assume that these ledger objects should be allocated by |
659 | * the pmap. This is done on PPL-enabled systems to give the PPL a |
660 | * method of validating ledger objects when updating them from within |
661 | * the PPL. |
662 | */ |
663 | ledger = pmap_ledger_alloc(); |
664 | } |
665 | |
666 | if (ledger == NULL) { |
667 | ledger_template_dereference(template); |
668 | return LEDGER_NULL; |
669 | } |
670 | |
671 | ledger->l_template = template; |
672 | ledger->l_id = ledger_cnt++; |
673 | os_ref_init(&ledger->l_refs, &ledger_refgrp); |
674 | assert(entries_size > 0); |
675 | ledger->l_size = (uint16_t) entries_size; |
676 | |
677 | template_lock(template); |
678 | assert(ledger->l_size <= template->lt_next_offset); |
679 | for (i = 0; i < num_entries; i++) { |
680 | uint16_t size, offset; |
681 | struct entry_template *et = &template->lt_entries[i]; |
682 | size = et->et_size; |
683 | offset = et->et_offset; |
684 | assert(offset < ledger->l_size); |
685 | |
686 | struct ledger_entry_small *les = &ledger->l_entries[offset]; |
687 | if (size == sizeof(struct ledger_entry)) { |
688 | struct ledger_entry *le = (struct ledger_entry *) les; |
689 | |
690 | le->le_flags = et->et_flags; |
691 | /* make entry inactive by removing active bit */ |
692 | if (entry_type == LEDGER_CREATE_INACTIVE_ENTRIES) { |
693 | flag_clear(flags: &le->le_flags, LF_ENTRY_ACTIVE); |
694 | } |
695 | /* |
696 | * If template has a callback, this entry is opted-in, |
697 | * by default. |
698 | */ |
699 | if (et->et_callback != NULL) { |
700 | flag_set(flags: &le->le_flags, LEDGER_ACTION_CALLBACK); |
701 | } |
702 | le->le_credit = 0; |
703 | le->le_debit = 0; |
704 | le->le_limit = LEDGER_LIMIT_INFINITY; |
705 | le->le_warn_percent = LEDGER_PERCENT_NONE; |
706 | le->le_diag_threshold_scaled = LEDGER_DIAG_MEM_THRESHOLD_INFINITY; |
707 | le->_le.le_refill.le_refill_period = 0; |
708 | le->_le.le_refill.le_last_refill = 0; |
709 | } else { |
710 | les->les_flags = et->et_flags; |
711 | les->les_credit = 0; |
712 | } |
713 | } |
714 | template_unlock(template); |
715 | |
716 | return ledger; |
717 | } |
718 | |
719 | static uint32_t |
720 | flag_set(volatile uint32_t *flags, uint32_t bit) |
721 | { |
722 | return OSBitOrAtomic(bit, flags); |
723 | } |
724 | |
725 | static uint32_t |
726 | flag_clear(volatile uint32_t *flags, uint32_t bit) |
727 | { |
728 | return OSBitAndAtomic(~bit, flags); |
729 | } |
730 | |
731 | /* |
732 | * Take a reference on a ledger |
733 | */ |
734 | void |
735 | ledger_reference(ledger_t ledger) |
736 | { |
737 | if (!LEDGER_VALID(ledger)) { |
738 | return; |
739 | } |
740 | |
741 | os_ref_retain(rc: &ledger->l_refs); |
742 | } |
743 | |
744 | /* |
745 | * Remove a reference on a ledger. If this is the last reference, |
746 | * deallocate the unused ledger. |
747 | */ |
748 | void |
749 | ledger_dereference(ledger_t ledger) |
750 | { |
751 | if (!LEDGER_VALID(ledger)) { |
752 | return; |
753 | } |
754 | |
755 | if (os_ref_release(rc: &ledger->l_refs) == 0) { |
756 | ledger_template_t template = ledger->l_template; |
757 | if (template->lt_zone) { |
758 | zfree(template->lt_zone, ledger); |
759 | } else { |
760 | /** |
761 | * If the template doesn't contain a zone to allocate ledger objects |
762 | * from, then assume that these ledger objects were allocated by the |
763 | * pmap. This is done on PPL-enabled systems to give the PPL a |
764 | * method of validating ledger objects when updating them from |
765 | * within the PPL. |
766 | */ |
767 | pmap_ledger_free(ledger); |
768 | } |
769 | ledger_template_dereference(template); |
770 | } |
771 | } |
772 | |
773 | /* |
774 | * Determine whether an entry has exceeded its warning level. |
775 | */ |
776 | static inline bool |
777 | warn_level_exceeded(struct ledger_entry *le) |
778 | { |
779 | ledger_amount_t balance; |
780 | |
781 | if (le->le_flags & LF_TRACK_CREDIT_ONLY) { |
782 | assert(le->le_debit == 0); |
783 | } else { |
784 | assert((le->le_credit >= 0) && (le->le_debit >= 0)); |
785 | } |
786 | |
787 | /* |
788 | * XXX - Currently, we only support warnings for ledgers which |
789 | * use positive limits. |
790 | */ |
791 | balance = le->le_credit - le->le_debit; |
792 | if (le->le_warn_percent != LEDGER_PERCENT_NONE && |
793 | ((balance > (le->le_limit * le->le_warn_percent) >> 16))) { |
794 | return true; |
795 | } |
796 | return false; |
797 | } |
798 | #if DEBUG || DEVELOPMENT |
799 | |
800 | /* |
801 | * Determine whether an entry has exceeded its diag mem threshold level. |
802 | */ |
803 | static inline bool |
804 | diag_mem_threshold_exceeded(struct ledger_entry *le) |
805 | { |
806 | ledger_amount_t balance; |
807 | ledger_amount_t diag_mem_threshold; |
808 | |
809 | if ((le->le_diag_threshold_scaled != LEDGER_DIAG_MEM_THRESHOLD_INFINITY) && (ledger_is_diag_threshold_enabled_internal(le) == true)) { |
810 | if (le->le_flags & LF_TRACK_CREDIT_ONLY) { |
811 | assert(le->le_debit == 0); |
812 | } else { |
813 | assert((le->le_credit >= 0) && (le->le_debit >= 0)); |
814 | } |
815 | |
816 | diag_mem_threshold = LEDGER_DIAG_MEM_AMOUNT_FROM_THRESHOLD(le->le_diag_threshold_scaled); |
817 | balance = le->le_credit - le->le_debit; |
818 | if ((diag_mem_threshold <= 0) && (balance < diag_mem_threshold)) { |
819 | return 1; |
820 | } |
821 | if ((diag_mem_threshold > 0) && (balance > diag_mem_threshold)) { |
822 | return 1; |
823 | } |
824 | } |
825 | return 0; |
826 | } |
827 | #endif |
828 | /* |
829 | * Determine whether an entry has exceeded its limit. |
830 | */ |
831 | static inline bool |
832 | limit_exceeded(struct ledger_entry *le) |
833 | { |
834 | ledger_amount_t balance; |
835 | |
836 | if (le->le_flags & LF_TRACK_CREDIT_ONLY) { |
837 | assert(le->le_debit == 0); |
838 | } else { |
839 | assert((le->le_credit >= 0) && (le->le_debit >= 0)); |
840 | } |
841 | |
842 | balance = le->le_credit - le->le_debit; |
843 | if ((le->le_limit <= 0) && (balance < le->le_limit)) { |
844 | return true; |
845 | } |
846 | |
847 | if ((le->le_limit > 0) && (balance > le->le_limit)) { |
848 | return true; |
849 | } |
850 | return false; |
851 | } |
852 | |
853 | static inline struct ledger_callback * |
854 | entry_get_callback(ledger_t ledger, int entry) |
855 | { |
856 | struct ledger_callback *callback = NULL; |
857 | spl_t s; |
858 | const uint16_t *ledger_template_idx_p = NULL; |
859 | |
860 | TEMPLATE_INUSE(s, ledger->l_template); |
861 | ledger_template_idx_p = ledger_entry_to_template_idx(template: ledger->l_template, index: entry); |
862 | if (ledger_template_idx_p != NULL) { |
863 | callback = ledger->l_template->lt_entries[*ledger_template_idx_p].et_callback; |
864 | } |
865 | TEMPLATE_IDLE(s, ledger->l_template); |
866 | |
867 | return callback; |
868 | } |
869 | |
870 | /* |
871 | * If the ledger value is positive, wake up anybody waiting on it. |
872 | */ |
873 | static inline void |
874 | ledger_limit_entry_wakeup(struct ledger_entry *le) |
875 | { |
876 | uint32_t flags; |
877 | |
878 | if (!limit_exceeded(le)) { |
879 | flags = flag_clear(flags: &le->le_flags, LF_CALLED_BACK); |
880 | |
881 | while (le->le_flags & LF_WAKE_NEEDED) { |
882 | flag_clear(flags: &le->le_flags, LF_WAKE_NEEDED); |
883 | thread_wakeup((event_t)le); |
884 | } |
885 | } |
886 | } |
887 | |
888 | /* |
889 | * Refill the coffers. |
890 | */ |
891 | static void |
892 | ledger_refill(uint64_t now, ledger_t ledger, int entry) |
893 | { |
894 | uint64_t elapsed, period, periods; |
895 | struct ledger_entry *le; |
896 | ledger_amount_t balance, due; |
897 | |
898 | if (!is_entry_valid(l: ledger, entry)) { |
899 | return; |
900 | } |
901 | |
902 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
903 | /* Small entries can't do refills */ |
904 | return; |
905 | } |
906 | |
907 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
908 | |
909 | assert(le->le_limit != LEDGER_LIMIT_INFINITY); |
910 | |
911 | if (le->le_flags & LF_TRACK_CREDIT_ONLY) { |
912 | assert(le->le_debit == 0); |
913 | return; |
914 | } |
915 | |
916 | /* |
917 | * If another thread is handling the refill already, we're not |
918 | * needed. |
919 | */ |
920 | if (flag_set(flags: &le->le_flags, LF_REFILL_INPROGRESS) & LF_REFILL_INPROGRESS) { |
921 | return; |
922 | } |
923 | |
924 | /* |
925 | * If the timestamp we're about to use to refill is older than the |
926 | * last refill, then someone else has already refilled this ledger |
927 | * and there's nothing for us to do here. |
928 | */ |
929 | if (now <= le->_le.le_refill.le_last_refill) { |
930 | flag_clear(flags: &le->le_flags, LF_REFILL_INPROGRESS); |
931 | return; |
932 | } |
933 | |
934 | /* |
935 | * See how many refill periods have passed since we last |
936 | * did a refill. |
937 | */ |
938 | period = le->_le.le_refill.le_refill_period; |
939 | elapsed = now - le->_le.le_refill.le_last_refill; |
940 | if ((period == 0) || (elapsed < period)) { |
941 | flag_clear(flags: &le->le_flags, LF_REFILL_INPROGRESS); |
942 | return; |
943 | } |
944 | |
945 | /* |
946 | * Optimize for the most common case of only one or two |
947 | * periods elapsing. |
948 | */ |
949 | periods = 0; |
950 | while ((periods < 2) && (elapsed > 0)) { |
951 | periods++; |
952 | elapsed -= period; |
953 | } |
954 | |
955 | /* |
956 | * OK, it's been a long time. Do a divide to figure out |
957 | * how long. |
958 | */ |
959 | if (elapsed > 0) { |
960 | periods = (now - le->_le.le_refill.le_last_refill) / period; |
961 | } |
962 | |
963 | balance = le->le_credit - le->le_debit; |
964 | due = periods * le->le_limit; |
965 | |
966 | if (balance - due < 0) { |
967 | due = balance; |
968 | } |
969 | |
970 | if (due < 0 && (le->le_flags & LF_PANIC_ON_NEGATIVE)) { |
971 | assertf(due >= 0, "now=%llu, ledger=%p, entry=%d, balance=%lld, due=%lld" , now, ledger, entry, balance, due); |
972 | } else { |
973 | OSAddAtomic64(due, &le->le_debit); |
974 | assert(le->le_debit >= 0); |
975 | } |
976 | /* |
977 | * If we've completely refilled the pool, set the refill time to now. |
978 | * Otherwise set it to the time at which it last should have been |
979 | * fully refilled. |
980 | */ |
981 | if (balance == due) { |
982 | le->_le.le_refill.le_last_refill = now; |
983 | } else { |
984 | le->_le.le_refill.le_last_refill += (le->_le.le_refill.le_refill_period * periods); |
985 | } |
986 | |
987 | flag_clear(flags: &le->le_flags, LF_REFILL_INPROGRESS); |
988 | |
989 | lprintf(("Refill %lld %lld->%lld\n" , periods, balance, balance - due)); |
990 | if (!limit_exceeded(le)) { |
991 | ledger_limit_entry_wakeup(le); |
992 | } |
993 | } |
994 | |
995 | void |
996 | ledger_entry_check_new_balance(thread_t thread, ledger_t ledger, |
997 | int entry) |
998 | { |
999 | uint16_t size, offset; |
1000 | struct ledger_entry *le = NULL; |
1001 | struct ledger_entry_small *les = NULL; |
1002 | if (!is_entry_valid(l: ledger, entry)) { |
1003 | return; |
1004 | } |
1005 | size = ENTRY_ID_SIZE(entry); |
1006 | offset = ENTRY_ID_OFFSET(entry); |
1007 | les = &ledger->l_entries[offset]; |
1008 | if (size == sizeof(struct ledger_entry_small)) { |
1009 | if ((les->les_flags & LF_PANIC_ON_NEGATIVE) && les->les_credit < 0) { |
1010 | panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld debit:0 balance:%lld" , |
1011 | ledger, entry, les, |
1012 | les->les_credit, |
1013 | les->les_credit); |
1014 | } |
1015 | } else if (size == sizeof(struct ledger_entry)) { |
1016 | le = (struct ledger_entry *)les; |
1017 | if (le->le_flags & LF_TRACKING_MAX) { |
1018 | ledger_amount_t balance = le->le_credit - le->le_debit; |
1019 | |
1020 | if (balance > le->_le._le_max.le_lifetime_max) { |
1021 | le->_le._le_max.le_lifetime_max = balance; |
1022 | } |
1023 | |
1024 | #if CONFIG_LEDGER_INTERVAL_MAX |
1025 | if (balance > le->_le._le_max.le_interval_max) { |
1026 | le->_le._le_max.le_interval_max = balance; |
1027 | } |
1028 | #endif /* LEDGER_CONFIG_INTERVAL_MAX */ |
1029 | } |
1030 | |
1031 | /* Check to see whether we're due a refill */ |
1032 | if (le->le_flags & LF_REFILL_SCHEDULED) { |
1033 | assert(!(le->le_flags & LF_TRACKING_MAX)); |
1034 | |
1035 | uint64_t now = mach_absolute_time(); |
1036 | if ((now - le->_le.le_refill.le_last_refill) > le->_le.le_refill.le_refill_period) { |
1037 | ledger_refill(now, ledger, entry); |
1038 | } |
1039 | } |
1040 | |
1041 | if (limit_exceeded(le)) { |
1042 | /* |
1043 | * We've exceeded the limit for this entry. There |
1044 | * are several possible ways to handle it. We can block, |
1045 | * we can execute a callback, or we can ignore it. In |
1046 | * either of the first two cases, we want to set the AST |
1047 | * flag so we can take the appropriate action just before |
1048 | * leaving the kernel. The one caveat is that if we have |
1049 | * already called the callback, we don't want to do it |
1050 | * again until it gets rearmed. |
1051 | */ |
1052 | if ((le->le_flags & LEDGER_ACTION_BLOCK) || |
1053 | (!(le->le_flags & LF_CALLED_BACK) && |
1054 | entry_get_callback(ledger, entry))) { |
1055 | act_set_astledger_async(thread); |
1056 | } |
1057 | } else { |
1058 | /* |
1059 | * The balance on the account is below the limit. |
1060 | * |
1061 | * If there are any threads blocked on this entry, now would |
1062 | * be a good time to wake them up. |
1063 | */ |
1064 | if (le->le_flags & LF_WAKE_NEEDED) { |
1065 | ledger_limit_entry_wakeup(le); |
1066 | } |
1067 | |
1068 | if (le->le_flags & LEDGER_ACTION_CALLBACK) { |
1069 | if (warn_level_exceeded(le)) { |
1070 | /* |
1071 | * This ledger's balance is above the warning level. |
1072 | */ |
1073 | if ((le->le_flags & LF_WARNED) == 0) { |
1074 | /* |
1075 | * If we are above the warning level and |
1076 | * have not yet invoked the callback, |
1077 | * set the AST so it can be done before returning |
1078 | * to userland. |
1079 | */ |
1080 | act_set_astledger_async(thread); |
1081 | } |
1082 | } else { |
1083 | /* |
1084 | * This ledger's balance is below the warning level. |
1085 | */ |
1086 | if (le->le_flags & LF_WARNED) { |
1087 | /* |
1088 | * If we are below the warning level and |
1089 | * the LF_WARNED flag is still set, we need |
1090 | * to invoke the callback to let the client |
1091 | * know the ledger balance is now back below |
1092 | * the warning level. |
1093 | */ |
1094 | act_set_astledger_async(thread); |
1095 | } |
1096 | } |
1097 | } |
1098 | } |
1099 | #if DEBUG || DEVELOPMENT |
1100 | if (diag_mem_threshold_exceeded(le)) { |
1101 | /* |
1102 | * Even if the limit is below the threshold, we may be interested |
1103 | * in diagnostics limits. Lets process them if the ast is not |
1104 | * invoked |
1105 | */ |
1106 | if ((le->le_flags & LF_DIAG_WARNED) == 0) { |
1107 | act_set_astledger_async(thread); |
1108 | } |
1109 | } |
1110 | #endif |
1111 | if ((le->le_flags & LF_PANIC_ON_NEGATIVE) && |
1112 | (le->le_credit < le->le_debit)) { |
1113 | panic("ledger_entry_check_new_balance(%p,%d): negative ledger %p credit:%lld debit:%lld balance:%lld" , |
1114 | ledger, entry, le, |
1115 | le->le_credit, |
1116 | le->le_debit, |
1117 | le->le_credit - le->le_debit); |
1118 | } |
1119 | } else { |
1120 | panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n" , ledger, entry, size); |
1121 | } |
1122 | } |
1123 | |
1124 | void |
1125 | ledger_check_new_balance(thread_t thread, ledger_t ledger, int entry) |
1126 | { |
1127 | ledger_entry_check_new_balance(thread, ledger, entry); |
1128 | } |
1129 | |
1130 | /* |
1131 | * Add value to an entry in a ledger for a specific thread. |
1132 | */ |
1133 | kern_return_t |
1134 | ledger_credit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount) |
1135 | { |
1136 | ledger_amount_t old, new; |
1137 | struct ledger_entry *le; |
1138 | uint16_t entry_size = ENTRY_ID_SIZE(entry); |
1139 | |
1140 | if (!is_entry_valid_and_active(l: ledger, entry) || (amount < 0)) { |
1141 | return KERN_INVALID_VALUE; |
1142 | } |
1143 | |
1144 | if (amount == 0) { |
1145 | return KERN_SUCCESS; |
1146 | } |
1147 | |
1148 | if (entry_size == sizeof(struct ledger_entry_small)) { |
1149 | struct ledger_entry_small *les = &ledger->l_entries[ENTRY_ID_OFFSET(entry)]; |
1150 | old = OSAddAtomic64(amount, &les->les_credit); |
1151 | new = old + amount; |
1152 | } else if (entry_size == sizeof(struct ledger_entry)) { |
1153 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1154 | |
1155 | old = OSAddAtomic64(amount, &le->le_credit); |
1156 | new = old + amount; |
1157 | } else { |
1158 | panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n" , ledger, entry, entry_size); |
1159 | } |
1160 | |
1161 | lprintf(("%p Credit %lld->%lld\n" , thread, old, new)); |
1162 | if (thread) { |
1163 | ledger_entry_check_new_balance(thread, ledger, entry); |
1164 | } |
1165 | |
1166 | return KERN_SUCCESS; |
1167 | } |
1168 | |
1169 | /* |
1170 | * Add value to an entry in a ledger. |
1171 | */ |
1172 | kern_return_t |
1173 | ledger_credit(ledger_t ledger, int entry, ledger_amount_t amount) |
1174 | { |
1175 | return ledger_credit_thread(thread: current_thread(), ledger, entry, amount); |
1176 | } |
1177 | |
1178 | /* |
1179 | * Add value to an entry in a ledger; do not check balance after update. |
1180 | */ |
1181 | kern_return_t |
1182 | ledger_credit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount) |
1183 | { |
1184 | return ledger_credit_thread(NULL, ledger, entry, amount); |
1185 | } |
1186 | |
1187 | /* Add all of one ledger's values into another. |
1188 | * They must have been created from the same template. |
1189 | * This is not done atomically. Another thread (if not otherwise synchronized) |
1190 | * may see bogus values when comparing one entry to another. |
1191 | * As each entry's credit & debit are modified one at a time, the warning/limit |
1192 | * may spuriously trip, or spuriously fail to trip, or another thread (if not |
1193 | * otherwise synchronized) may see a bogus balance. |
1194 | */ |
1195 | kern_return_t |
1196 | ledger_rollup(ledger_t to_ledger, ledger_t from_ledger) |
1197 | { |
1198 | int id; |
1199 | ledger_template_t template = NULL; |
1200 | struct entry_template *et = NULL; |
1201 | |
1202 | assert(to_ledger->l_template->lt_cnt == from_ledger->l_template->lt_cnt); |
1203 | template = from_ledger->l_template; |
1204 | assert(template->lt_initialized); |
1205 | |
1206 | for (uint16_t i = 0; i < template->lt_cnt; i++) { |
1207 | et = &template->lt_entries[i]; |
1208 | uint16_t size = et->et_size; |
1209 | id = ledger_entry_id(size, offset: et->et_offset); |
1210 | ledger_rollup_entry(to_ledger, from_ledger, entry: id); |
1211 | } |
1212 | |
1213 | return KERN_SUCCESS; |
1214 | } |
1215 | |
1216 | /* Add one ledger entry value to another. |
1217 | * They must have been created from the same template. |
1218 | * Since the credit and debit values are added one |
1219 | * at a time, other thread might read the a bogus value. |
1220 | */ |
1221 | kern_return_t |
1222 | ledger_rollup_entry(ledger_t to_ledger, ledger_t from_ledger, int entry) |
1223 | { |
1224 | struct ledger_entry_small *from_les, *to_les; |
1225 | uint16_t entry_size, entry_offset; |
1226 | entry_size = ENTRY_ID_SIZE(entry); |
1227 | entry_offset = ENTRY_ID_OFFSET(entry); |
1228 | |
1229 | assert(to_ledger->l_template->lt_cnt == from_ledger->l_template->lt_cnt); |
1230 | if (is_entry_valid(l: from_ledger, entry) && is_entry_valid(l: to_ledger, entry)) { |
1231 | from_les = &from_ledger->l_entries[entry_offset]; |
1232 | to_les = &to_ledger->l_entries[entry_offset]; |
1233 | if (entry_size == sizeof(struct ledger_entry)) { |
1234 | struct ledger_entry *from = (struct ledger_entry *)from_les; |
1235 | struct ledger_entry *to = (struct ledger_entry *)to_les; |
1236 | OSAddAtomic64(from->le_credit, &to->le_credit); |
1237 | OSAddAtomic64(from->le_debit, &to->le_debit); |
1238 | } else if (entry_size == sizeof(struct ledger_entry_small)) { |
1239 | OSAddAtomic64(from_les->les_credit, &to_les->les_credit); |
1240 | } else { |
1241 | panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n" , from_ledger, entry, entry_size); |
1242 | } |
1243 | } |
1244 | |
1245 | return KERN_SUCCESS; |
1246 | } |
1247 | |
1248 | /* |
1249 | * Zero the balance of a ledger by adding to its credit or debit, whichever is smaller. |
1250 | * Note that some clients of ledgers (notably, task wakeup statistics) require that |
1251 | * le_credit only ever increase as a function of ledger_credit(). |
1252 | */ |
1253 | kern_return_t |
1254 | ledger_zero_balance(ledger_t ledger, int entry) |
1255 | { |
1256 | struct ledger_entry *le; |
1257 | struct ledger_entry_small *les; |
1258 | ledger_amount_t debit, credit; |
1259 | uint16_t entry_size, entry_offset; |
1260 | entry_size = ENTRY_ID_SIZE(entry); |
1261 | entry_offset = ENTRY_ID_OFFSET(entry); |
1262 | |
1263 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1264 | return KERN_INVALID_VALUE; |
1265 | } |
1266 | |
1267 | les = &ledger->l_entries[entry_offset]; |
1268 | if (entry_size == sizeof(struct ledger_entry_small)) { |
1269 | while (true) { |
1270 | credit = les->les_credit; |
1271 | if (OSCompareAndSwap64(credit, 0, &les->les_credit)) { |
1272 | break; |
1273 | } |
1274 | } |
1275 | } else if (entry_size == sizeof(struct ledger_entry)) { |
1276 | le = (struct ledger_entry *)les; |
1277 | top: |
1278 | debit = le->le_debit; |
1279 | credit = le->le_credit; |
1280 | |
1281 | if (le->le_flags & LF_TRACK_CREDIT_ONLY) { |
1282 | assert(le->le_debit == 0); |
1283 | if (!OSCompareAndSwap64(credit, 0, &le->le_credit)) { |
1284 | goto top; |
1285 | } |
1286 | lprintf(("%p zeroed %lld->%lld\n" , current_thread(), le->le_credit, 0)); |
1287 | } else if (credit > debit) { |
1288 | if (!OSCompareAndSwap64(debit, credit, &le->le_debit)) { |
1289 | goto top; |
1290 | } |
1291 | lprintf(("%p zeroed %lld->%lld\n" , current_thread(), le->le_debit, le->le_credit)); |
1292 | } else if (credit < debit) { |
1293 | if (!OSCompareAndSwap64(credit, debit, &le->le_credit)) { |
1294 | goto top; |
1295 | } |
1296 | lprintf(("%p zeroed %lld->%lld\n" , current_thread(), le->le_credit, le->le_debit)); |
1297 | } |
1298 | } else { |
1299 | panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n" , ledger, entry, entry_size); |
1300 | } |
1301 | |
1302 | return KERN_SUCCESS; |
1303 | } |
1304 | |
1305 | kern_return_t |
1306 | ledger_get_limit(ledger_t ledger, int entry, ledger_amount_t *limit) |
1307 | { |
1308 | struct ledger_entry *le; |
1309 | |
1310 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1311 | return KERN_INVALID_VALUE; |
1312 | } |
1313 | |
1314 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1315 | /* Small entries can't have limits */ |
1316 | *limit = LEDGER_LIMIT_INFINITY; |
1317 | } else { |
1318 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1319 | *limit = le->le_limit; |
1320 | } |
1321 | |
1322 | lprintf(("ledger_get_limit: %lld\n" , *limit)); |
1323 | |
1324 | return KERN_SUCCESS; |
1325 | } |
1326 | |
1327 | /* |
1328 | * Adjust the limit of a limited resource. This does not affect the |
1329 | * current balance, so the change doesn't affect the thread until the |
1330 | * next refill. |
1331 | * |
1332 | * warn_level: If non-zero, causes the callback to be invoked when |
1333 | * the balance exceeds this level. Specified as a percentage [of the limit]. |
1334 | */ |
1335 | kern_return_t |
1336 | ledger_set_limit(ledger_t ledger, int entry, ledger_amount_t limit, |
1337 | uint8_t warn_level_percentage) |
1338 | { |
1339 | struct ledger_entry *le; |
1340 | |
1341 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1342 | return KERN_INVALID_VALUE; |
1343 | } |
1344 | |
1345 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1346 | /* Small entries can't have limits */ |
1347 | return KERN_INVALID_ARGUMENT; |
1348 | } |
1349 | |
1350 | lprintf(("ledger_set_limit: %lld\n" , limit)); |
1351 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1352 | |
1353 | if (limit == LEDGER_LIMIT_INFINITY) { |
1354 | /* |
1355 | * Caller wishes to disable the limit. This will implicitly |
1356 | * disable automatic refill, as refills implicitly depend |
1357 | * on the limit. |
1358 | */ |
1359 | ledger_disable_refill(l: ledger, entry); |
1360 | } |
1361 | |
1362 | le->le_limit = limit; |
1363 | if (le->le_flags & LF_REFILL_SCHEDULED) { |
1364 | assert(!(le->le_flags & LF_TRACKING_MAX)); |
1365 | le->_le.le_refill.le_last_refill = 0; |
1366 | } |
1367 | flag_clear(flags: &le->le_flags, LF_CALLED_BACK); |
1368 | flag_clear(flags: &le->le_flags, LF_WARNED); |
1369 | ledger_limit_entry_wakeup(le); |
1370 | |
1371 | if (warn_level_percentage != 0) { |
1372 | assert(warn_level_percentage <= 100); |
1373 | assert(limit > 0); /* no negative limit support for warnings */ |
1374 | assert(limit != LEDGER_LIMIT_INFINITY); /* warn % without limit makes no sense */ |
1375 | le->le_warn_percent = warn_level_percentage * (1u << 16) / 100; |
1376 | } else { |
1377 | le->le_warn_percent = LEDGER_PERCENT_NONE; |
1378 | } |
1379 | |
1380 | return KERN_SUCCESS; |
1381 | } |
1382 | |
1383 | #if CONFIG_LEDGER_INTERVAL_MAX |
1384 | kern_return_t |
1385 | ledger_get_interval_max(ledger_t ledger, int entry, |
1386 | ledger_amount_t *max_interval_balance, int reset) |
1387 | { |
1388 | kern_return_t kr = KERN_SUCCESS; |
1389 | struct ledger_entry *le; |
1390 | |
1391 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1392 | return KERN_INVALID_VALUE; |
1393 | } |
1394 | |
1395 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1396 | /* Small entries can't track max */ |
1397 | return KERN_INVALID_ARGUMENT; |
1398 | } |
1399 | |
1400 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1401 | |
1402 | if (!(le->le_flags & LF_TRACKING_MAX)) { |
1403 | return KERN_INVALID_VALUE; |
1404 | } |
1405 | |
1406 | *max_interval_balance = le->_le._le_max.le_interval_max; |
1407 | lprintf(("ledger_get_interval_max: %lld%s\n" , *max_interval_balance, |
1408 | (reset) ? " --> 0" : "" )); |
1409 | |
1410 | if (reset) { |
1411 | kr = ledger_get_balance(ledger, entry, balance: &le->_le._le_max.le_interval_max); |
1412 | } |
1413 | |
1414 | return kr; |
1415 | } |
1416 | #endif /* CONFIG_LEDGER_INTERVAL_MAX */ |
1417 | |
1418 | kern_return_t |
1419 | ledger_get_lifetime_max(ledger_t ledger, int entry, |
1420 | ledger_amount_t *max_lifetime_balance) |
1421 | { |
1422 | struct ledger_entry *le; |
1423 | |
1424 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1425 | return KERN_INVALID_VALUE; |
1426 | } |
1427 | |
1428 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1429 | /* Small entries can't track max */ |
1430 | return KERN_INVALID_ARGUMENT; |
1431 | } |
1432 | |
1433 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1434 | |
1435 | if (!(le->le_flags & LF_TRACKING_MAX)) { |
1436 | return KERN_INVALID_VALUE; |
1437 | } |
1438 | |
1439 | *max_lifetime_balance = le->_le._le_max.le_lifetime_max; |
1440 | lprintf(("ledger_get_lifetime_max: %lld\n" , *max_lifetime_balance)); |
1441 | |
1442 | return KERN_SUCCESS; |
1443 | } |
1444 | |
1445 | /* |
1446 | * Enable tracking of periodic maximums for this ledger entry. |
1447 | */ |
1448 | kern_return_t |
1449 | ledger_track_maximum(ledger_template_t template, int entry, |
1450 | __unused int period_in_secs) |
1451 | { |
1452 | uint16_t idx; |
1453 | const uint16_t *idx_p; |
1454 | struct entry_template *et = NULL; |
1455 | kern_return_t kr = KERN_INVALID_VALUE; |
1456 | |
1457 | template_lock(template); |
1458 | |
1459 | idx_p = ledger_entry_to_template_idx(template, index: entry); |
1460 | if (idx_p == NULL) { |
1461 | kr = KERN_INVALID_VALUE; |
1462 | goto out; |
1463 | } |
1464 | idx = *idx_p; |
1465 | if (idx >= template->lt_cnt) { |
1466 | kr = KERN_INVALID_VALUE; |
1467 | goto out; |
1468 | } |
1469 | et = &template->lt_entries[idx]; |
1470 | /* Ensure the caller asked for enough space up front */ |
1471 | if (et->et_size != sizeof(struct ledger_entry)) { |
1472 | kr = KERN_INVALID_VALUE; |
1473 | goto out; |
1474 | } |
1475 | |
1476 | /* Refill is incompatible with max tracking. */ |
1477 | if (et->et_flags & LF_REFILL_SCHEDULED) { |
1478 | kr = KERN_INVALID_VALUE; |
1479 | goto out; |
1480 | } |
1481 | |
1482 | et->et_flags |= LF_TRACKING_MAX; |
1483 | kr = KERN_SUCCESS; |
1484 | out: |
1485 | template_unlock(template); |
1486 | |
1487 | return kr; |
1488 | } |
1489 | |
1490 | kern_return_t |
1491 | ledger_panic_on_negative(ledger_template_t template, int entry) |
1492 | { |
1493 | const uint16_t *idx_p; |
1494 | uint16_t idx; |
1495 | template_lock(template); |
1496 | |
1497 | idx_p = ledger_entry_to_template_idx(template, index: entry); |
1498 | if (idx_p == NULL) { |
1499 | template_unlock(template); |
1500 | return KERN_INVALID_VALUE; |
1501 | } |
1502 | idx = *idx_p; |
1503 | if (idx >= template->lt_cnt) { |
1504 | template_unlock(template); |
1505 | return KERN_INVALID_VALUE; |
1506 | } |
1507 | |
1508 | template->lt_entries[idx].et_flags |= LF_PANIC_ON_NEGATIVE; |
1509 | |
1510 | template_unlock(template); |
1511 | |
1512 | return KERN_SUCCESS; |
1513 | } |
1514 | |
1515 | kern_return_t |
1516 | ledger_track_credit_only(ledger_template_t template, int entry) |
1517 | { |
1518 | const uint16_t *idx_p; |
1519 | uint16_t idx; |
1520 | struct entry_template *et = NULL; |
1521 | kern_return_t kr = KERN_INVALID_VALUE; |
1522 | template_lock(template); |
1523 | |
1524 | idx_p = ledger_entry_to_template_idx(template, index: entry); |
1525 | if (idx_p == NULL) { |
1526 | kr = KERN_INVALID_VALUE; |
1527 | goto out; |
1528 | } |
1529 | idx = *idx_p; |
1530 | if (idx >= template->lt_cnt) { |
1531 | kr = KERN_INVALID_VALUE; |
1532 | goto out; |
1533 | } |
1534 | et = &template->lt_entries[idx]; |
1535 | /* Ensure the caller asked for enough space up front */ |
1536 | if (et->et_size != sizeof(struct ledger_entry)) { |
1537 | kr = KERN_INVALID_VALUE; |
1538 | goto out; |
1539 | } |
1540 | |
1541 | et->et_flags |= LF_TRACK_CREDIT_ONLY; |
1542 | kr = KERN_SUCCESS; |
1543 | |
1544 | out: |
1545 | template_unlock(template); |
1546 | |
1547 | return kr; |
1548 | } |
1549 | |
1550 | /* |
1551 | * Add a callback to be executed when the resource goes into deficit. |
1552 | */ |
1553 | kern_return_t |
1554 | ledger_set_callback(ledger_template_t template, int entry, |
1555 | ledger_callback_t func, const void *param0, const void *param1) |
1556 | { |
1557 | struct entry_template *et; |
1558 | struct ledger_callback *old_cb, *new_cb; |
1559 | const uint16_t *idx_p; |
1560 | uint16_t idx; |
1561 | |
1562 | idx_p = ledger_entry_to_template_idx(template, index: entry); |
1563 | if (idx_p == NULL) { |
1564 | return KERN_INVALID_VALUE; |
1565 | } |
1566 | idx = *idx_p; |
1567 | |
1568 | if (idx >= template->lt_cnt) { |
1569 | return KERN_INVALID_VALUE; |
1570 | } |
1571 | |
1572 | if (func) { |
1573 | new_cb = kalloc_type(struct ledger_callback, Z_WAITOK); |
1574 | new_cb->lc_func = func; |
1575 | new_cb->lc_param0 = param0; |
1576 | new_cb->lc_param1 = param1; |
1577 | } else { |
1578 | new_cb = NULL; |
1579 | } |
1580 | |
1581 | template_lock(template); |
1582 | et = &template->lt_entries[idx]; |
1583 | /* Ensure the caller asked for enough space up front */ |
1584 | if (et->et_size != sizeof(struct ledger_entry)) { |
1585 | kfree_type(struct ledger_callback, new_cb); |
1586 | template_unlock(template); |
1587 | return KERN_INVALID_VALUE; |
1588 | } |
1589 | old_cb = et->et_callback; |
1590 | et->et_callback = new_cb; |
1591 | template_unlock(template); |
1592 | if (old_cb) { |
1593 | kfree_type(struct ledger_callback, old_cb); |
1594 | } |
1595 | |
1596 | return KERN_SUCCESS; |
1597 | } |
1598 | |
1599 | /* |
1600 | * Disable callback notification for a specific ledger entry. |
1601 | * |
1602 | * Otherwise, if using a ledger template which specified a |
1603 | * callback function (ledger_set_callback()), it will be invoked when |
1604 | * the resource goes into deficit. |
1605 | */ |
1606 | kern_return_t |
1607 | ledger_disable_callback(ledger_t ledger, int entry) |
1608 | { |
1609 | struct ledger_entry *le = NULL; |
1610 | |
1611 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1612 | return KERN_INVALID_VALUE; |
1613 | } |
1614 | |
1615 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1616 | /* Small entries can't have callbacks */ |
1617 | return KERN_INVALID_ARGUMENT; |
1618 | } |
1619 | |
1620 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1621 | |
1622 | /* |
1623 | * le_warn_percent is used to indicate *if* this ledger has a warning configured, |
1624 | * in addition to what that warning level is set to. |
1625 | * This means a side-effect of ledger_disable_callback() is that the |
1626 | * warning level is forgotten. |
1627 | */ |
1628 | le->le_warn_percent = LEDGER_PERCENT_NONE; |
1629 | flag_clear(flags: &le->le_flags, LEDGER_ACTION_CALLBACK); |
1630 | return KERN_SUCCESS; |
1631 | } |
1632 | |
1633 | /* |
1634 | * Enable callback notification for a specific ledger entry. |
1635 | * |
1636 | * This is only needed if ledger_disable_callback() has previously |
1637 | * been invoked against an entry; there must already be a callback |
1638 | * configured. |
1639 | */ |
1640 | kern_return_t |
1641 | ledger_enable_callback(ledger_t ledger, int entry) |
1642 | { |
1643 | struct ledger_entry *le = NULL; |
1644 | |
1645 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1646 | return KERN_INVALID_VALUE; |
1647 | } |
1648 | |
1649 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1650 | /* Small entries can't have callbacks */ |
1651 | return KERN_INVALID_ARGUMENT; |
1652 | } |
1653 | |
1654 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1655 | |
1656 | assert(entry_get_callback(ledger, entry) != NULL); |
1657 | |
1658 | flag_set(flags: &le->le_flags, LEDGER_ACTION_CALLBACK); |
1659 | return KERN_SUCCESS; |
1660 | } |
1661 | |
1662 | /* |
1663 | * Query the automatic refill period for this ledger entry. |
1664 | * |
1665 | * A period of 0 means this entry has none configured. |
1666 | */ |
1667 | kern_return_t |
1668 | ledger_get_period(ledger_t ledger, int entry, uint64_t *period) |
1669 | { |
1670 | struct ledger_entry *le; |
1671 | |
1672 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1673 | return KERN_INVALID_VALUE; |
1674 | } |
1675 | |
1676 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1677 | /* Small entries can't do refills */ |
1678 | return KERN_INVALID_ARGUMENT; |
1679 | } |
1680 | |
1681 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1682 | |
1683 | *period = abstime_to_nsecs(abstime: le->_le.le_refill.le_refill_period); |
1684 | lprintf(("ledger_get_period: %llx\n" , *period)); |
1685 | return KERN_SUCCESS; |
1686 | } |
1687 | |
1688 | /* |
1689 | * Adjust the automatic refill period. |
1690 | */ |
1691 | kern_return_t |
1692 | ledger_set_period(ledger_t ledger, int entry, uint64_t period) |
1693 | { |
1694 | struct ledger_entry *le = NULL; |
1695 | |
1696 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1697 | return KERN_INVALID_VALUE; |
1698 | } |
1699 | |
1700 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1701 | /* Small entries can't do refills */ |
1702 | return KERN_INVALID_ARGUMENT; |
1703 | } |
1704 | |
1705 | lprintf(("ledger_set_period: %llx\n" , period)); |
1706 | |
1707 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1708 | |
1709 | /* |
1710 | * A refill period refills the ledger in multiples of the limit, |
1711 | * so if you haven't set one yet, you need a lesson on ledgers. |
1712 | */ |
1713 | assert(le->le_limit != LEDGER_LIMIT_INFINITY); |
1714 | |
1715 | if (le->le_flags & LF_TRACKING_MAX) { |
1716 | /* |
1717 | * Refill is incompatible with rolling max tracking. |
1718 | */ |
1719 | return KERN_INVALID_VALUE; |
1720 | } |
1721 | |
1722 | le->_le.le_refill.le_refill_period = nsecs_to_abstime(nsecs: period); |
1723 | |
1724 | /* |
1725 | * Set the 'starting time' for the next refill to now. Since |
1726 | * we're resetting the balance to zero here, we consider this |
1727 | * moment the starting time for accumulating a balance that |
1728 | * counts towards the limit. |
1729 | */ |
1730 | le->_le.le_refill.le_last_refill = mach_absolute_time(); |
1731 | ledger_zero_balance(ledger, entry); |
1732 | |
1733 | flag_set(flags: &le->le_flags, LF_REFILL_SCHEDULED); |
1734 | |
1735 | return KERN_SUCCESS; |
1736 | } |
1737 | |
1738 | /* |
1739 | * Disable automatic refill. |
1740 | */ |
1741 | kern_return_t |
1742 | ledger_disable_refill(ledger_t ledger, int entry) |
1743 | { |
1744 | struct ledger_entry *le = NULL; |
1745 | |
1746 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1747 | return KERN_INVALID_VALUE; |
1748 | } |
1749 | |
1750 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1751 | /* Small entries can't do refills */ |
1752 | return KERN_INVALID_ARGUMENT; |
1753 | } |
1754 | |
1755 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1756 | |
1757 | flag_clear(flags: &le->le_flags, LF_REFILL_SCHEDULED); |
1758 | |
1759 | return KERN_SUCCESS; |
1760 | } |
1761 | |
1762 | kern_return_t |
1763 | ledger_get_actions(ledger_t ledger, int entry, int *actions) |
1764 | { |
1765 | struct ledger_entry *le = NULL; |
1766 | *actions = 0; |
1767 | |
1768 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1769 | return KERN_INVALID_VALUE; |
1770 | } |
1771 | |
1772 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1773 | /* Small entries can't have actions */ |
1774 | return KERN_INVALID_ARGUMENT; |
1775 | } |
1776 | |
1777 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1778 | |
1779 | *actions = le->le_flags & LEDGER_ACTION_MASK; |
1780 | lprintf(("ledger_get_actions: %#x\n" , *actions)); |
1781 | return KERN_SUCCESS; |
1782 | } |
1783 | |
1784 | kern_return_t |
1785 | ledger_set_action(ledger_t ledger, int entry, int action) |
1786 | { |
1787 | lprintf(("ledger_set_action: %#x\n" , action)); |
1788 | struct ledger_entry *le = NULL; |
1789 | |
1790 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
1791 | return KERN_INVALID_VALUE; |
1792 | } |
1793 | |
1794 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
1795 | /* Small entries can't have actions */ |
1796 | return KERN_INVALID_ARGUMENT; |
1797 | } |
1798 | |
1799 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1800 | |
1801 | flag_set(flags: &le->le_flags, bit: action); |
1802 | return KERN_SUCCESS; |
1803 | } |
1804 | |
1805 | kern_return_t |
1806 | ledger_debit_thread(thread_t thread, ledger_t ledger, int entry, ledger_amount_t amount) |
1807 | { |
1808 | struct ledger_entry *le; |
1809 | ledger_amount_t old, new; |
1810 | uint16_t entry_size = ENTRY_ID_SIZE(entry); |
1811 | |
1812 | if (!is_entry_valid_and_active(l: ledger, entry) || (amount < 0)) { |
1813 | return KERN_INVALID_ARGUMENT; |
1814 | } |
1815 | |
1816 | if (amount == 0) { |
1817 | return KERN_SUCCESS; |
1818 | } |
1819 | |
1820 | if (entry_size == sizeof(struct ledger_entry_small)) { |
1821 | struct ledger_entry_small *les = &ledger->l_entries[ENTRY_ID_OFFSET(entry)]; |
1822 | old = OSAddAtomic64(-amount, &les->les_credit); |
1823 | new = old - amount; |
1824 | } else if (entry_size == sizeof(struct ledger_entry)) { |
1825 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
1826 | |
1827 | if (le->le_flags & LF_TRACK_CREDIT_ONLY) { |
1828 | assert(le->le_debit == 0); |
1829 | old = OSAddAtomic64(-amount, &le->le_credit); |
1830 | new = old - amount; |
1831 | } else { |
1832 | old = OSAddAtomic64(amount, &le->le_debit); |
1833 | new = old + amount; |
1834 | } |
1835 | } else { |
1836 | panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n" , ledger, entry, entry_size); |
1837 | } |
1838 | lprintf(("%p Debit %lld->%lld\n" , thread, old, new)); |
1839 | |
1840 | if (thread) { |
1841 | ledger_entry_check_new_balance(thread, ledger, entry); |
1842 | } |
1843 | |
1844 | return KERN_SUCCESS; |
1845 | } |
1846 | |
1847 | kern_return_t |
1848 | ledger_debit(ledger_t ledger, int entry, ledger_amount_t amount) |
1849 | { |
1850 | return ledger_debit_thread(thread: current_thread(), ledger, entry, amount); |
1851 | } |
1852 | |
1853 | kern_return_t |
1854 | ledger_debit_nocheck(ledger_t ledger, int entry, ledger_amount_t amount) |
1855 | { |
1856 | return ledger_debit_thread(NULL, ledger, entry, amount); |
1857 | } |
1858 | |
1859 | void |
1860 | ledger_ast(thread_t thread) |
1861 | { |
1862 | struct ledger *l = thread->t_ledger; |
1863 | struct ledger *thl; |
1864 | struct ledger *coalition_ledger; |
1865 | uint32_t block; |
1866 | uint64_t now; |
1867 | uint8_t task_flags; |
1868 | uint8_t task_percentage; |
1869 | uint64_t task_interval; |
1870 | |
1871 | kern_return_t ret; |
1872 | task_t task = get_threadtask(thread); |
1873 | |
1874 | lprintf(("Ledger AST for %p\n" , thread)); |
1875 | |
1876 | ASSERT(task != NULL); |
1877 | ASSERT(thread == current_thread()); |
1878 | |
1879 | #if CONFIG_SCHED_RT_ALLOW |
1880 | /* |
1881 | * The RT policy may have forced a CPU limit on the thread. Check if |
1882 | * that's the case and apply the limit as requested. |
1883 | */ |
1884 | spl_t s = splsched(); |
1885 | thread_lock(thread); |
1886 | |
1887 | int req_action = thread->t_ledger_req_action; |
1888 | uint8_t req_percentage = thread->t_ledger_req_percentage; |
1889 | uint64_t req_interval_ns = thread->t_ledger_req_interval_ms * NSEC_PER_MSEC; |
1890 | |
1891 | thread->t_ledger_req_action = 0; |
1892 | |
1893 | thread_unlock(thread); |
1894 | splx(s); |
1895 | |
1896 | if (req_action != 0) { |
1897 | assert(req_action == THREAD_CPULIMIT_DISABLE || |
1898 | req_action == THREAD_CPULIMIT_BLOCK); |
1899 | |
1900 | if (req_action == THREAD_CPULIMIT_DISABLE && |
1901 | (thread->options & TH_OPT_FORCED_LEDGER) != 0) { |
1902 | thread->options &= ~TH_OPT_FORCED_LEDGER; |
1903 | ret = thread_set_cpulimit(THREAD_CPULIMIT_DISABLE, 0, 0); |
1904 | assert3u(ret, ==, KERN_SUCCESS); |
1905 | } |
1906 | |
1907 | if (req_action == THREAD_CPULIMIT_BLOCK) { |
1908 | thread->options &= ~TH_OPT_FORCED_LEDGER; |
1909 | ret = thread_set_cpulimit(THREAD_CPULIMIT_BLOCK, |
1910 | req_percentage, req_interval_ns); |
1911 | assert3u(ret, ==, KERN_SUCCESS); |
1912 | thread->options |= TH_OPT_FORCED_LEDGER; |
1913 | } |
1914 | } |
1915 | #endif /* CONFIG_SCHED_RT_ALLOW */ |
1916 | |
1917 | top: |
1918 | /* |
1919 | * Take a self-consistent snapshot of the CPU usage monitor parameters. The task |
1920 | * can change them at any point (with the task locked). |
1921 | */ |
1922 | task_lock(task); |
1923 | task_flags = task->rusage_cpu_flags; |
1924 | task_percentage = task->rusage_cpu_perthr_percentage; |
1925 | task_interval = task->rusage_cpu_perthr_interval; |
1926 | task_unlock(task); |
1927 | |
1928 | /* |
1929 | * Make sure this thread is up to date with regards to any task-wide per-thread |
1930 | * CPU limit, but only if it doesn't have a thread-private blocking CPU limit. |
1931 | */ |
1932 | if (((task_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) != 0) && |
1933 | ((thread->options & TH_OPT_PRVT_CPULIMIT) == 0)) { |
1934 | uint8_t percentage; |
1935 | uint64_t interval; |
1936 | int action; |
1937 | |
1938 | thread_get_cpulimit(action: &action, percentage: &percentage, interval_ns: &interval); |
1939 | |
1940 | /* |
1941 | * If the thread's CPU limits no longer match the task's, or the |
1942 | * task has a limit but the thread doesn't, update the limit. |
1943 | */ |
1944 | if (((thread->options & TH_OPT_PROC_CPULIMIT) == 0) || |
1945 | (interval != task_interval) || (percentage != task_percentage)) { |
1946 | thread_set_cpulimit(THREAD_CPULIMIT_EXCEPTION, percentage: task_percentage, interval_ns: task_interval); |
1947 | assert((thread->options & TH_OPT_PROC_CPULIMIT) != 0); |
1948 | } |
1949 | } else if (((task_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) == 0) && |
1950 | (thread->options & TH_OPT_PROC_CPULIMIT)) { |
1951 | assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0); |
1952 | |
1953 | /* |
1954 | * Task no longer has a per-thread CPU limit; remove this thread's |
1955 | * corresponding CPU limit. |
1956 | */ |
1957 | thread_set_cpulimit(THREAD_CPULIMIT_DISABLE, percentage: 0, interval_ns: 0); |
1958 | assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0); |
1959 | } |
1960 | |
1961 | /* |
1962 | * If the task or thread is being terminated, let's just get on with it |
1963 | */ |
1964 | if ((l == NULL) || !task->active || task->halting || !thread->active) { |
1965 | return; |
1966 | } |
1967 | |
1968 | /* |
1969 | * Examine all entries in deficit to see which might be eligble for |
1970 | * an automatic refill, which require callbacks to be issued, and |
1971 | * which require blocking. |
1972 | */ |
1973 | block = 0; |
1974 | now = mach_absolute_time(); |
1975 | |
1976 | /* |
1977 | * Note that thread->t_threadledger may have been changed by the |
1978 | * thread_set_cpulimit() call above - so don't examine it until afterwards. |
1979 | */ |
1980 | thl = thread->t_threadledger; |
1981 | if (LEDGER_VALID(thl)) { |
1982 | block |= ledger_check_needblock(l: thl, now); |
1983 | } |
1984 | block |= ledger_check_needblock(l, now); |
1985 | |
1986 | coalition_ledger = coalition_ledger_get_from_task(task); |
1987 | if (LEDGER_VALID(coalition_ledger)) { |
1988 | block |= ledger_check_needblock(l: coalition_ledger, now); |
1989 | } |
1990 | ledger_dereference(ledger: coalition_ledger); |
1991 | /* |
1992 | * If we are supposed to block on the availability of one or more |
1993 | * resources, find the first entry in deficit for which we should wait. |
1994 | * Schedule a refill if necessary and then sleep until the resource |
1995 | * becomes available. |
1996 | */ |
1997 | if (block) { |
1998 | if (LEDGER_VALID(thl)) { |
1999 | ret = ledger_perform_blocking(l: thl); |
2000 | if (ret != KERN_SUCCESS) { |
2001 | goto top; |
2002 | } |
2003 | } |
2004 | ret = ledger_perform_blocking(l); |
2005 | if (ret != KERN_SUCCESS) { |
2006 | goto top; |
2007 | } |
2008 | } /* block */ |
2009 | } |
2010 | |
2011 | static uint32_t |
2012 | ledger_check_needblock(ledger_t l, uint64_t now) |
2013 | { |
2014 | int i; |
2015 | uint32_t flags, block = 0; |
2016 | struct ledger_entry *le; |
2017 | struct ledger_callback *lc; |
2018 | struct entry_template *et = NULL; |
2019 | ledger_template_t template = NULL; |
2020 | |
2021 | template = l->l_template; |
2022 | assert(template != NULL); |
2023 | assert(template->lt_initialized); |
2024 | /* |
2025 | * The template has been initialized so the entries table can't change. |
2026 | * Thus we don't need to acquire the template lock or the inuse bit. |
2027 | */ |
2028 | |
2029 | |
2030 | for (i = 0; i < template->lt_cnt; i++) { |
2031 | spl_t s; |
2032 | et = &template->lt_entries[i]; |
2033 | if (et->et_size == sizeof(struct ledger_entry_small)) { |
2034 | /* Small entries don't track limits or have callbacks */ |
2035 | continue; |
2036 | } |
2037 | assert(et->et_size == sizeof(struct ledger_entry)); |
2038 | le = (struct ledger_entry *) &l->l_entries[et->et_offset]; |
2039 | |
2040 | TEMPLATE_INUSE(s, template); |
2041 | lc = template->lt_entries[i].et_callback; |
2042 | TEMPLATE_IDLE(s, template); |
2043 | |
2044 | if (limit_exceeded(le) == FALSE) { |
2045 | if (le->le_flags & LEDGER_ACTION_CALLBACK) { |
2046 | /* |
2047 | * If needed, invoke the callback as a warning. |
2048 | * This needs to happen both when the balance rises above |
2049 | * the warning level, and also when it dips back below it. |
2050 | */ |
2051 | assert(lc != NULL); |
2052 | /* |
2053 | * See comments for matching logic in ledger_check_new_balance(). |
2054 | */ |
2055 | if (warn_level_exceeded(le)) { |
2056 | flags = flag_set(flags: &le->le_flags, LF_WARNED); |
2057 | if ((flags & LF_WARNED) == 0) { |
2058 | lc->lc_func(LEDGER_WARNING_ROSE_ABOVE, lc->lc_param0, lc->lc_param1); |
2059 | } |
2060 | } else { |
2061 | flags = flag_clear(flags: &le->le_flags, LF_WARNED); |
2062 | if (flags & LF_WARNED) { |
2063 | lc->lc_func(LEDGER_WARNING_DIPPED_BELOW, lc->lc_param0, lc->lc_param1); |
2064 | } |
2065 | } |
2066 | } |
2067 | #if DEBUG || DEVELOPMENT |
2068 | if (diag_mem_threshold_exceeded(le)) { |
2069 | if (le->le_flags & LEDGER_ACTION_CALLBACK) { |
2070 | assert(lc != NULL); |
2071 | flags = flag_set(&le->le_flags, LF_DIAG_WARNED); |
2072 | if ((flags & LF_DIAG_WARNED) == 0) { |
2073 | lc->lc_func(LEDGER_WARNING_DIAG_MEM_THRESHOLD, lc->lc_param0, lc->lc_param1); |
2074 | } |
2075 | } |
2076 | } |
2077 | #endif |
2078 | continue; |
2079 | } |
2080 | #if DEBUG || DEVELOPMENT |
2081 | if (diag_mem_threshold_exceeded(le)) { |
2082 | if (le->le_flags & LEDGER_ACTION_CALLBACK) { |
2083 | assert(lc != NULL); |
2084 | flags = flag_set(&le->le_flags, LF_DIAG_WARNED); |
2085 | if ((flags & LF_DIAG_WARNED) == 0) { |
2086 | lc->lc_func(LEDGER_WARNING_DIAG_MEM_THRESHOLD, lc->lc_param0, lc->lc_param1); |
2087 | } |
2088 | } |
2089 | } |
2090 | #endif |
2091 | |
2092 | /* We're over the limit, so refill if we are eligible and past due. */ |
2093 | if (le->le_flags & LF_REFILL_SCHEDULED) { |
2094 | assert(!(le->le_flags & LF_TRACKING_MAX)); |
2095 | |
2096 | if ((le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period) <= now) { |
2097 | ledger_refill(now, ledger: l, entry: i); |
2098 | if (limit_exceeded(le) == FALSE) { |
2099 | continue; |
2100 | } |
2101 | } |
2102 | } |
2103 | |
2104 | if (le->le_flags & LEDGER_ACTION_BLOCK) { |
2105 | block = 1; |
2106 | } |
2107 | if ((le->le_flags & LEDGER_ACTION_CALLBACK) == 0) { |
2108 | continue; |
2109 | } |
2110 | |
2111 | /* |
2112 | * If the LEDGER_ACTION_CALLBACK flag is on, we expect there to |
2113 | * be a registered callback. |
2114 | */ |
2115 | assert(lc != NULL); |
2116 | flags = flag_set(flags: &le->le_flags, LF_CALLED_BACK); |
2117 | /* Callback has already been called */ |
2118 | if (flags & LF_CALLED_BACK) { |
2119 | continue; |
2120 | } |
2121 | lc->lc_func(FALSE, lc->lc_param0, lc->lc_param1); |
2122 | } |
2123 | return block; |
2124 | } |
2125 | |
2126 | |
2127 | /* return KERN_SUCCESS to continue, KERN_FAILURE to restart */ |
2128 | static kern_return_t |
2129 | ledger_perform_blocking(ledger_t l) |
2130 | { |
2131 | int i; |
2132 | kern_return_t ret; |
2133 | struct ledger_entry *le; |
2134 | ledger_template_t template = NULL; |
2135 | struct entry_template *et = NULL; |
2136 | |
2137 | template = l->l_template; |
2138 | assert(template->lt_initialized); |
2139 | |
2140 | for (i = 0; i < template->lt_cnt; i++) { |
2141 | et = &template->lt_entries[i]; |
2142 | if (et->et_size != sizeof(struct ledger_entry)) { |
2143 | /* Small entries do not block for anything. */ |
2144 | continue; |
2145 | } |
2146 | le = (struct ledger_entry *) &l->l_entries[et->et_offset]; |
2147 | if ((!limit_exceeded(le)) || |
2148 | ((le->le_flags & LEDGER_ACTION_BLOCK) == 0)) { |
2149 | continue; |
2150 | } |
2151 | |
2152 | assert(!(le->le_flags & LF_TRACKING_MAX)); |
2153 | |
2154 | /* Prepare to sleep until the resource is refilled */ |
2155 | ret = assert_wait_deadline(event: le, THREAD_INTERRUPTIBLE, |
2156 | deadline: le->_le.le_refill.le_last_refill + le->_le.le_refill.le_refill_period); |
2157 | if (ret != THREAD_WAITING) { |
2158 | return KERN_SUCCESS; |
2159 | } |
2160 | |
2161 | /* Mark that somebody is waiting on this entry */ |
2162 | flag_set(flags: &le->le_flags, LF_WAKE_NEEDED); |
2163 | |
2164 | ret = thread_block_reason(THREAD_CONTINUE_NULL, NULL, |
2165 | AST_LEDGER); |
2166 | if (ret != THREAD_AWAKENED) { |
2167 | return KERN_SUCCESS; |
2168 | } |
2169 | |
2170 | /* |
2171 | * The world may have changed while we were asleep. |
2172 | * Some other resource we need may have gone into |
2173 | * deficit. Or maybe we're supposed to die now. |
2174 | * Go back to the top and reevaluate. |
2175 | */ |
2176 | return KERN_FAILURE; |
2177 | } |
2178 | return KERN_SUCCESS; |
2179 | } |
2180 | |
2181 | |
2182 | kern_return_t |
2183 | ledger_get_entries(ledger_t ledger, int entry, ledger_amount_t *credit, |
2184 | ledger_amount_t *debit) |
2185 | { |
2186 | struct ledger_entry *le = NULL; |
2187 | struct ledger_entry_small *les = NULL; |
2188 | uint16_t entry_size, entry_offset; |
2189 | |
2190 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
2191 | return KERN_INVALID_ARGUMENT; |
2192 | } |
2193 | |
2194 | entry_size = ENTRY_ID_SIZE(entry); |
2195 | entry_offset = ENTRY_ID_OFFSET(entry); |
2196 | les = &ledger->l_entries[entry_offset]; |
2197 | if (entry_size == sizeof(struct ledger_entry)) { |
2198 | le = (struct ledger_entry *)les; |
2199 | *credit = le->le_credit; |
2200 | *debit = le->le_debit; |
2201 | } else if (entry_size == sizeof(struct ledger_entry_small)) { |
2202 | *credit = les->les_credit; |
2203 | *debit = 0; |
2204 | } else { |
2205 | panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n" , ledger, entry, entry_size); |
2206 | } |
2207 | |
2208 | return KERN_SUCCESS; |
2209 | } |
2210 | |
2211 | kern_return_t |
2212 | ledger_reset_callback_state(ledger_t ledger, int entry) |
2213 | { |
2214 | struct ledger_entry *le = NULL; |
2215 | |
2216 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
2217 | return KERN_INVALID_ARGUMENT; |
2218 | } |
2219 | |
2220 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
2221 | /* small entries can't have callbacks */ |
2222 | return KERN_INVALID_ARGUMENT; |
2223 | } |
2224 | |
2225 | le = ledger_entry_identifier_to_entry(ledger, id: entry); |
2226 | |
2227 | flag_clear(flags: &le->le_flags, LF_CALLED_BACK); |
2228 | |
2229 | return KERN_SUCCESS; |
2230 | } |
2231 | |
2232 | kern_return_t |
2233 | ledger_disable_panic_on_negative(ledger_t ledger, int entry) |
2234 | { |
2235 | volatile uint32_t *flags; |
2236 | |
2237 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
2238 | return KERN_INVALID_ARGUMENT; |
2239 | } |
2240 | flags = get_entry_flags(l: ledger, index: entry); |
2241 | |
2242 | flag_clear(flags, LF_PANIC_ON_NEGATIVE); |
2243 | |
2244 | return KERN_SUCCESS; |
2245 | } |
2246 | |
2247 | kern_return_t |
2248 | ledger_get_panic_on_negative(ledger_t ledger, int entry, int *panic_on_negative) |
2249 | { |
2250 | volatile uint32_t flags; |
2251 | |
2252 | if (!is_entry_valid_and_active(l: ledger, entry)) { |
2253 | return KERN_INVALID_ARGUMENT; |
2254 | } |
2255 | flags = *get_entry_flags(l: ledger, index: entry); |
2256 | |
2257 | if (flags & LF_PANIC_ON_NEGATIVE) { |
2258 | *panic_on_negative = TRUE; |
2259 | } else { |
2260 | *panic_on_negative = FALSE; |
2261 | } |
2262 | |
2263 | return KERN_SUCCESS; |
2264 | } |
2265 | |
2266 | kern_return_t |
2267 | ledger_get_balance(ledger_t ledger, int entry, ledger_amount_t *balance) |
2268 | { |
2269 | kern_return_t kr; |
2270 | ledger_amount_t credit, debit; |
2271 | |
2272 | kr = ledger_get_entries(ledger, entry, credit: &credit, debit: &debit); |
2273 | if (kr != KERN_SUCCESS) { |
2274 | return kr; |
2275 | } |
2276 | *balance = credit - debit; |
2277 | |
2278 | return KERN_SUCCESS; |
2279 | } |
2280 | |
2281 | int |
2282 | ledger_template_info(void **buf, int *len) |
2283 | { |
2284 | struct ledger_template_info *lti; |
2285 | struct entry_template *et; |
2286 | ledger_template_t template; |
2287 | int i; |
2288 | ledger_t l; |
2289 | |
2290 | /* |
2291 | * Since all tasks share a ledger template, we'll just use the |
2292 | * caller's as the source. |
2293 | */ |
2294 | l = current_task()->ledger; |
2295 | if ((*len < 0) || (l == NULL)) { |
2296 | return EINVAL; |
2297 | } |
2298 | template = l->l_template; |
2299 | assert(template); |
2300 | assert(template->lt_initialized); |
2301 | |
2302 | if (*len > template->lt_cnt) { |
2303 | *len = template->lt_cnt; |
2304 | } |
2305 | lti = kalloc_data((*len) * sizeof(struct ledger_template_info), |
2306 | Z_WAITOK); |
2307 | if (lti == NULL) { |
2308 | return ENOMEM; |
2309 | } |
2310 | *buf = lti; |
2311 | |
2312 | template_lock(template); |
2313 | et = template->lt_entries; |
2314 | |
2315 | for (i = 0; i < *len; i++) { |
2316 | memset(s: lti, c: 0, n: sizeof(*lti)); |
2317 | strlcpy(dst: lti->lti_name, src: et->et_key, LEDGER_NAME_MAX); |
2318 | strlcpy(dst: lti->lti_group, src: et->et_group, LEDGER_NAME_MAX); |
2319 | strlcpy(dst: lti->lti_units, src: et->et_units, LEDGER_NAME_MAX); |
2320 | et++; |
2321 | lti++; |
2322 | } |
2323 | template_unlock(template); |
2324 | |
2325 | return 0; |
2326 | } |
2327 | |
2328 | static kern_return_t |
2329 | ledger_fill_entry_info(ledger_t ledger, |
2330 | int entry, |
2331 | struct ledger_entry_info *lei, |
2332 | uint64_t now) |
2333 | { |
2334 | assert(ledger != NULL); |
2335 | assert(lei != NULL); |
2336 | if (!is_entry_valid(l: ledger, entry)) { |
2337 | return KERN_INVALID_ARGUMENT; |
2338 | } |
2339 | uint16_t entry_size, entry_offset; |
2340 | struct ledger_entry_small *les = NULL; |
2341 | struct ledger_entry *le = NULL; |
2342 | entry_size = ENTRY_ID_SIZE(entry); |
2343 | entry_offset = ENTRY_ID_OFFSET(entry); |
2344 | |
2345 | les = &ledger->l_entries[entry_offset]; |
2346 | memset(s: lei, c: 0, n: sizeof(*lei)); |
2347 | if (entry_size == sizeof(struct ledger_entry_small)) { |
2348 | lei->lei_limit = LEDGER_LIMIT_INFINITY; |
2349 | lei->lei_credit = les->les_credit; |
2350 | lei->lei_debit = 0; |
2351 | lei->lei_refill_period = 0; |
2352 | lei->lei_last_refill = abstime_to_nsecs(abstime: now); |
2353 | } else if (entry_size == sizeof(struct ledger_entry)) { |
2354 | le = (struct ledger_entry *) les; |
2355 | lei->lei_limit = le->le_limit; |
2356 | lei->lei_credit = le->le_credit; |
2357 | lei->lei_debit = le->le_debit; |
2358 | lei->lei_refill_period = (le->le_flags & LF_REFILL_SCHEDULED) ? |
2359 | abstime_to_nsecs(abstime: le->_le.le_refill.le_refill_period) : 0; |
2360 | lei->lei_last_refill = abstime_to_nsecs(abstime: now - le->_le.le_refill.le_last_refill); |
2361 | } else { |
2362 | panic("Unknown ledger entry size! ledger=%p, entry=0x%x, entry_size=%d\n" , ledger, entry, entry_size); |
2363 | } |
2364 | |
2365 | lei->lei_balance = lei->lei_credit - lei->lei_debit; |
2366 | |
2367 | return KERN_SUCCESS; |
2368 | } |
2369 | |
2370 | int |
2371 | ledger_get_task_entry_info_multiple(task_t task, void **buf, int *len) |
2372 | { |
2373 | struct ledger_entry_info *lei_buf = NULL, *lei_curr = NULL; |
2374 | uint64_t now = mach_absolute_time(); |
2375 | vm_size_t size = 0; |
2376 | int i; |
2377 | ledger_t l; |
2378 | ledger_template_t template; |
2379 | struct entry_template *et = NULL; |
2380 | |
2381 | if ((*len < 0) || ((l = task->ledger) == NULL)) { |
2382 | return EINVAL; |
2383 | } |
2384 | template = l->l_template; |
2385 | assert(template && template->lt_initialized); |
2386 | |
2387 | if (*len > template->lt_cnt) { |
2388 | *len = template->lt_cnt; |
2389 | } |
2390 | size = (*len) * sizeof(struct ledger_entry_info); |
2391 | lei_buf = kalloc_data(size, Z_WAITOK); |
2392 | if (lei_buf == NULL) { |
2393 | return ENOMEM; |
2394 | } |
2395 | lei_curr = lei_buf; |
2396 | |
2397 | for (i = 0; i < *len; i++) { |
2398 | et = &template->lt_entries[i]; |
2399 | int index = ledger_entry_id_from_template_entry(et); |
2400 | if (ledger_fill_entry_info(ledger: l, entry: index, lei: lei_curr, now) != KERN_SUCCESS) { |
2401 | kfree_data(lei_buf, size); |
2402 | lei_buf = NULL; |
2403 | return EINVAL; |
2404 | } |
2405 | lei_curr++; |
2406 | } |
2407 | |
2408 | *buf = lei_buf; |
2409 | return 0; |
2410 | } |
2411 | |
2412 | void |
2413 | ledger_get_entry_info(ledger_t ledger, |
2414 | int entry, |
2415 | struct ledger_entry_info *lei) |
2416 | { |
2417 | uint64_t now = mach_absolute_time(); |
2418 | |
2419 | assert(ledger != NULL); |
2420 | assert(lei != NULL); |
2421 | |
2422 | ledger_fill_entry_info(ledger, entry, lei, now); |
2423 | } |
2424 | |
2425 | int |
2426 | ledger_info(task_t task, struct ledger_info *info) |
2427 | { |
2428 | ledger_t l; |
2429 | |
2430 | if ((l = task->ledger) == NULL) { |
2431 | return ENOENT; |
2432 | } |
2433 | |
2434 | memset(s: info, c: 0, n: sizeof(*info)); |
2435 | |
2436 | strlcpy(dst: info->li_name, src: l->l_template->lt_name, LEDGER_NAME_MAX); |
2437 | info->li_id = l->l_id; |
2438 | info->li_entries = l->l_template->lt_cnt; |
2439 | return 0; |
2440 | } |
2441 | |
2442 | /* |
2443 | * Returns the amount that would be required to hit the limit. |
2444 | * Must be a valid, active, full-sized ledger. |
2445 | */ |
2446 | ledger_amount_t |
2447 | ledger_get_remaining(ledger_t ledger, int entry) |
2448 | { |
2449 | const struct ledger_entry *le = |
2450 | ledger_entry_identifier_to_entry(ledger, id: entry); |
2451 | const ledger_amount_t limit = le->le_limit; |
2452 | const ledger_amount_t balance = le->le_credit - le->le_debit; |
2453 | |
2454 | /* +1 here as the limit isn't hit until the limit is exceeded. */ |
2455 | return limit > balance ? limit - balance + 1 : 0; |
2456 | } |
2457 | |
2458 | /* |
2459 | * Balances the ledger by modifying the debit only and sets the last refill time |
2460 | * to `now`. |
2461 | * WARNING: It is up to the caller to enforce consistency. |
2462 | * Must be a valid, active, full-sized ledger. |
2463 | */ |
2464 | void |
2465 | ledger_restart(ledger_t ledger, int entry, uint64_t now) |
2466 | { |
2467 | struct ledger_entry *le = ledger_entry_identifier_to_entry(ledger, id: entry); |
2468 | |
2469 | le->le_debit = le->le_credit; |
2470 | le->_le.le_refill.le_last_refill = now; |
2471 | } |
2472 | |
2473 | /* |
2474 | * Returns the amount of time that would have to pass to expire the current |
2475 | * interval. |
2476 | * Must be a valid, active, full-sized ledger. |
2477 | */ |
2478 | uint64_t |
2479 | ledger_get_interval_remaining(ledger_t ledger, int entry, uint64_t now) |
2480 | { |
2481 | const struct ledger_entry *le = |
2482 | ledger_entry_identifier_to_entry(ledger, id: entry); |
2483 | |
2484 | if ((now - le->_le.le_refill.le_last_refill) > |
2485 | le->_le.le_refill.le_refill_period) { |
2486 | return 0; |
2487 | } else { |
2488 | return le->_le.le_refill.le_refill_period - |
2489 | (now - le->_le.le_refill.le_last_refill) + 1; |
2490 | } |
2491 | } |
2492 | |
2493 | #ifdef LEDGER_DEBUG |
2494 | int |
2495 | ledger_limit(task_t task, struct ledger_limit_args *args) |
2496 | { |
2497 | ledger_t l; |
2498 | int64_t limit; |
2499 | int idx; |
2500 | |
2501 | if ((l = task->ledger) == NULL) { |
2502 | return EINVAL; |
2503 | } |
2504 | |
2505 | idx = ledger_key_lookup(l->l_template, args->lla_name); |
2506 | if (idx < 0) { |
2507 | return EINVAL; |
2508 | } |
2509 | if (ENTRY_ID_SIZE(idx) == sizeof(ledger_entry_small)) { |
2510 | /* Small entries can't have limits */ |
2511 | return EINVAL; |
2512 | } |
2513 | |
2514 | /* |
2515 | * XXX - this doesn't really seem like the right place to have |
2516 | * a context-sensitive conversion of userspace units into kernel |
2517 | * units. For now I'll handwave and say that the ledger() system |
2518 | * call isn't meant for civilians to use - they should be using |
2519 | * the process policy interfaces. |
2520 | */ |
2521 | if (idx == task_ledgers.cpu_time) { |
2522 | int64_t nsecs; |
2523 | |
2524 | if (args->lla_refill_period) { |
2525 | /* |
2526 | * If a refill is scheduled, then the limit is |
2527 | * specified as a percentage of one CPU. The |
2528 | * syscall specifies the refill period in terms of |
2529 | * milliseconds, so we need to convert to nsecs. |
2530 | */ |
2531 | args->lla_refill_period *= 1000000; |
2532 | nsecs = args->lla_limit * |
2533 | (args->lla_refill_period / 100); |
2534 | lprintf(("CPU limited to %lld nsecs per second\n" , |
2535 | nsecs)); |
2536 | } else { |
2537 | /* |
2538 | * If no refill is scheduled, then this is a |
2539 | * fixed amount of CPU time (in nsecs) that can |
2540 | * be consumed. |
2541 | */ |
2542 | nsecs = args->lla_limit; |
2543 | lprintf(("CPU limited to %lld nsecs\n" , nsecs)); |
2544 | } |
2545 | limit = nsecs_to_abstime(nsecs); |
2546 | } else { |
2547 | limit = args->lla_limit; |
2548 | lprintf(("%s limited to %lld\n" , args->lla_name, limit)); |
2549 | } |
2550 | |
2551 | if (args->lla_refill_period > 0) { |
2552 | ledger_set_period(l, idx, args->lla_refill_period); |
2553 | } |
2554 | |
2555 | ledger_set_limit(l, idx, limit); |
2556 | |
2557 | flag_set(ledger_entry_identifier_to_entry(l, idx)->le_flags, LEDGER_ACTION_BLOCK); |
2558 | return 0; |
2559 | } |
2560 | #endif |
2561 | |
2562 | /* |
2563 | * Adjust the diag mem threshold limit of a resource. The diag mem threshold limit only |
2564 | * works prescaled by 20 bits (mb) |
2565 | */ |
2566 | #if DEBUG || DEVELOPMENT |
2567 | kern_return_t |
2568 | ledger_set_diag_mem_threshold(ledger_t ledger, int entry, ledger_amount_t limit) |
2569 | { |
2570 | struct ledger_entry *le; |
2571 | |
2572 | if (!is_entry_valid_and_active(ledger, entry)) { |
2573 | return KERN_INVALID_VALUE; |
2574 | } |
2575 | |
2576 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
2577 | /* Small entries can't have limits */ |
2578 | return KERN_INVALID_ARGUMENT; |
2579 | } |
2580 | |
2581 | lprintf(("ledger_set_diag mem threshold_limit: %lld\n" , limit)); |
2582 | le = ledger_entry_identifier_to_entry(ledger, entry); |
2583 | le->le_diag_threshold_scaled = (int16_t)LEDGER_DIAG_MEM_AMOUNT_TO_THRESHOLD(limit); |
2584 | lprintf(("ledger_set_diag mem threshold_limit new : %lld\n" , limit)); |
2585 | flag_clear(&le->le_flags, LF_DIAG_WARNED); |
2586 | |
2587 | return KERN_SUCCESS; |
2588 | } |
2589 | |
2590 | kern_return_t |
2591 | ledger_get_diag_mem_threshold(ledger_t ledger, int entry, ledger_amount_t *limit) |
2592 | { |
2593 | struct ledger_entry *le; |
2594 | |
2595 | if (!is_entry_valid_and_active(ledger, entry)) { |
2596 | return KERN_INVALID_VALUE; |
2597 | } |
2598 | |
2599 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
2600 | /* Small entries can't have limits */ |
2601 | *limit = LEDGER_LIMIT_INFINITY; |
2602 | } else { |
2603 | le = ledger_entry_identifier_to_entry(ledger, entry); |
2604 | if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) { |
2605 | *limit = LEDGER_LIMIT_INFINITY; |
2606 | } else { |
2607 | *limit = LEDGER_DIAG_MEM_AMOUNT_FROM_THRESHOLD(le->le_diag_threshold_scaled); |
2608 | } |
2609 | } |
2610 | |
2611 | lprintf(("ledger_get_diag mem threshold_limit: %lld\n" , *limit)); |
2612 | |
2613 | return KERN_SUCCESS; |
2614 | } |
2615 | |
2616 | static inline void |
2617 | ledger_set_diag_mem_threshold_flag_disabled_internal(struct ledger_entry *le, bool value) |
2618 | { |
2619 | if (value == true) { |
2620 | flag_set(&le->le_flags, LF_DIAG_DISABLED); |
2621 | } else { |
2622 | flag_clear(&le->le_flags, LF_DIAG_DISABLED); |
2623 | } |
2624 | } |
2625 | |
2626 | static inline bool |
2627 | ledger_is_diag_threshold_enabled_internal( struct ledger_entry *le) |
2628 | { |
2629 | return ((le->le_flags & LF_DIAG_DISABLED) == 0)? true : false; |
2630 | } |
2631 | |
2632 | /** |
2633 | * Disable the diagnostics threshold due to overlap with footprint limit |
2634 | */ |
2635 | kern_return_t |
2636 | ledger_set_diag_mem_threshold_disabled(ledger_t ledger, int entry) |
2637 | { |
2638 | struct ledger_entry *le; |
2639 | |
2640 | if (!is_entry_valid_and_active(ledger, entry)) { |
2641 | return KERN_INVALID_VALUE; |
2642 | } |
2643 | |
2644 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
2645 | /* Small entries can't have limits */ |
2646 | return KERN_INVALID_ARGUMENT; |
2647 | } |
2648 | |
2649 | lprintf(("ledger_set_diag_mem_threshold_disabled" )); |
2650 | le = ledger_entry_identifier_to_entry(ledger, entry); |
2651 | if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) { |
2652 | lprintf(("ledger_set_diag_mem_threshold_disabled, cannot disable a ledger entry that have no value, returning error" )); |
2653 | return KERN_INVALID_ARGUMENT; |
2654 | } |
2655 | ledger_set_diag_mem_threshold_flag_disabled_internal(le, true); |
2656 | return KERN_SUCCESS; |
2657 | } |
2658 | /** |
2659 | * Enable the diagnostics threshold for a specific entry |
2660 | */ |
2661 | kern_return_t |
2662 | ledger_set_diag_mem_threshold_enabled(ledger_t ledger, int entry) |
2663 | { |
2664 | struct ledger_entry *le; |
2665 | |
2666 | if (!is_entry_valid_and_active(ledger, entry)) { |
2667 | return KERN_INVALID_VALUE; |
2668 | } |
2669 | |
2670 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
2671 | /* Small entries can't have limits */ |
2672 | return KERN_INVALID_ARGUMENT; |
2673 | } |
2674 | |
2675 | lprintf(("ledger_set_diag_mem_threshold_enabled" )); |
2676 | le = ledger_entry_identifier_to_entry(ledger, entry); |
2677 | /* |
2678 | * if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) { |
2679 | * lprintf(("ledger_set_diag_mem_threshold_enabled, cannot disable a ledger entry that have no value, returning error")); |
2680 | * return KERN_INVALID_ARGUMENT; |
2681 | * } |
2682 | */ |
2683 | ledger_set_diag_mem_threshold_flag_disabled_internal(le, false); |
2684 | |
2685 | return KERN_SUCCESS; |
2686 | } |
2687 | /** |
2688 | * Obtain the diagnostics threshold enabled flag. If the diagnostics threshold is enabled, returns true |
2689 | * else returns false. |
2690 | */ |
2691 | kern_return_t |
2692 | ledger_is_diag_threshold_enabled(ledger_t ledger, int entry, bool *status) |
2693 | { |
2694 | struct ledger_entry *le; |
2695 | |
2696 | if (!is_entry_valid_and_active(ledger, entry)) { |
2697 | return KERN_INVALID_VALUE; |
2698 | } |
2699 | |
2700 | if (ENTRY_ID_SIZE(entry) != sizeof(struct ledger_entry)) { |
2701 | /* Small entries can't have limits */ |
2702 | return KERN_INVALID_ARGUMENT; |
2703 | } |
2704 | |
2705 | lprintf(("ledger_is_diag_threshold_enabled" )); |
2706 | le = ledger_entry_identifier_to_entry(ledger, entry); |
2707 | /* |
2708 | * if (le->le_diag_threshold_scaled == LEDGER_DIAG_MEM_THRESHOLD_INFINITY) { |
2709 | * lprintf(("ledger_is_diag_threshold_enabled, get enabled flag for a ledger entry that have no value, returning error")); |
2710 | * return KERN_INVALID_ARGUMENT; |
2711 | * } |
2712 | */ |
2713 | *status = ledger_is_diag_threshold_enabled_internal(le); |
2714 | return KERN_SUCCESS; |
2715 | } |
2716 | #endif // DEBUG || DEVELOPMENT |
2717 | |