1 | /* |
2 | * Copyright (c) 2020-2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. Please obtain a copy of the License at |
10 | * http://www.opensource.apple.com/apsl/ and read it before using this |
11 | * file. |
12 | * |
13 | * The Original Code and all software distributed under the License are |
14 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
15 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
16 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
18 | * Please see the License for the specific language governing rights and |
19 | * limitations under the License. |
20 | * log_queue_failed_intr); |
21 | * |
22 | * @APPLE_LICENSE_HEADER_END@ |
23 | */ |
24 | |
25 | #include <kern/assert.h> |
26 | #include <kern/counter.h> |
27 | #include <kern/cpu_data.h> |
28 | #include <kern/percpu.h> |
29 | #include <kern/kalloc.h> |
30 | #include <kern/thread_call.h> |
31 | #include <libkern/libkern.h> |
32 | #include <sys/queue.h> |
33 | #include <vm/vm_kern.h> |
34 | |
35 | #include "log_queue.h" |
36 | #include "log_mem.h" |
37 | |
38 | #define LQ_DEFAULT_SZ_ORDER 15 // 32K per slot |
39 | #define LQ_DEFAULT_FREE_AFTER_CNT 15000 // Deallocate log queue after N logs |
40 | #define LQ_MAX_SZ_ORDER 20 // 1MB per CPU should really be enough and a hard cap |
41 | #define LQ_MIN_LOG_SZ_ORDER 5 |
42 | #define LQ_MAX_LOG_SZ_ORDER 11 |
43 | #define LQ_BATCH_SIZE 24 |
44 | #define LQ_MAX_LM_SLOTS 8 |
45 | #define LQ_LOW_MEM_SCALE 3 |
46 | |
47 | #define LQ_MEM_ENABLE(q, i) ((q)->lq_mem_set |= (1 << (i))) |
48 | #define LQ_MEM_ENABLED(q, i) ((q)->lq_mem_set & (1 << (i))) |
49 | #define LQ_MEM_DISABLE(q, i) ((q)->lq_mem_set &= ~(1 << (i))) |
50 | |
51 | OS_ENUM(log_queue_entry_state, uint8_t, |
52 | LOG_QUEUE_ENTRY_STATE_INVALID = 0, |
53 | LOG_QUEUE_ENTRY_STATE_STORED, |
54 | LOG_QUEUE_ENTRY_STATE_DISPATCHED, |
55 | LOG_QUEUE_ENTRY_STATE_SENT, |
56 | LOG_QUEUE_ENTRY_STATE_FAILED |
57 | ); |
58 | |
59 | OS_ENUM(lq_mem_state, uint8_t, |
60 | LQ_MEM_STATE_READY = 0, |
61 | LQ_MEM_STATE_ALLOCATING, |
62 | LQ_MEM_STATE_RELEASING |
63 | ); |
64 | |
65 | OS_ENUM(lq_req_state, uint8_t, |
66 | LQ_REQ_STATE_INVALID = 0, |
67 | LQ_REQ_STATE_ALLOCATING, |
68 | LQ_REQ_STATE_RELEASING, |
69 | LQ_REQ_STATE_READY |
70 | ); |
71 | |
72 | typedef struct log_queue_entry { |
73 | STAILQ_ENTRY(log_queue_entry) lqe_link; |
74 | uint16_t lqe_size; |
75 | uint16_t lqe_lm_id; |
76 | _Atomic log_queue_entry_state_t lqe_state; |
77 | log_payload_s lqe_payload; |
78 | } log_queue_entry_s, *log_queue_entry_t; |
79 | |
80 | typedef STAILQ_HEAD(, log_queue_entry) log_queue_list_s, *log_queue_list_t; |
81 | |
82 | typedef struct { |
83 | log_queue_list_s lq_log_list; |
84 | log_queue_list_s lq_dispatch_list; |
85 | logmem_t lq_mem[LQ_MAX_LM_SLOTS]; |
86 | size_t lq_mem_set; |
87 | size_t lq_mem_size; |
88 | size_t lq_mem_size_order; |
89 | lq_mem_state_t lq_mem_state; |
90 | thread_call_t lq_mem_handler; |
91 | size_t lq_cnt_mem_active; |
92 | size_t lq_cnt_mem_avail; |
93 | _Atomic lq_req_state_t lq_req_state; |
94 | void *lq_req_mem; |
95 | uint32_t lq_ready : 1; |
96 | uint32_t lq_suspend : 1; |
97 | } log_queue_s, *log_queue_t; |
98 | |
99 | extern bool os_log_disabled(void); |
100 | |
101 | /* |
102 | * Log Queue |
103 | * |
104 | * Log queues are allocated and set up per cpu. When a firehose memory is full |
105 | * logs are stored in a log queue and sent into the firehose once it has a free |
106 | * space again. Each log queue (memory) can grow and shrink based on demand by |
107 | * adding/removing additional memory to/from its memory slots. There are |
108 | * LQ_MAX_LM_SLOTS memory slots available for every log queue to use. Memory |
109 | * slots are released when not needed, with one slot always allocated per queue |
110 | * as a minimum. |
111 | * |
112 | * Boot args: |
113 | * |
114 | * lq_size_order: Per slot memory size defined as a power of 2 exponent |
115 | * (i.e. 2^lq_bootarg_size_order). Zero disables queues. |
116 | * |
117 | * lq_nslots: Number of allocated slots to boot with per each log queue. |
118 | * Once initial log traffic decreases, log queues release |
119 | * slots as needed. |
120 | * |
121 | * If extensive number of logs is expected, setting aforementioned boot-args as |
122 | * needed allows to capture the vast majority of logs and avoid drops. |
123 | */ |
124 | TUNABLE(size_t, lq_bootarg_size_order, "lq_size_order" , LQ_DEFAULT_SZ_ORDER); |
125 | TUNABLE(size_t, lq_bootarg_nslots, "lq_nslots" , LQ_MAX_LM_SLOTS); |
126 | |
127 | SCALABLE_COUNTER_DEFINE(log_queue_cnt_received); |
128 | SCALABLE_COUNTER_DEFINE(log_queue_cnt_rejected_fh); |
129 | SCALABLE_COUNTER_DEFINE(log_queue_cnt_queued); |
130 | SCALABLE_COUNTER_DEFINE(log_queue_cnt_sent); |
131 | SCALABLE_COUNTER_DEFINE(log_queue_cnt_dropped_nomem); |
132 | SCALABLE_COUNTER_DEFINE(log_queue_cnt_dropped_off); |
133 | SCALABLE_COUNTER_DEFINE(log_queue_cnt_mem_allocated); |
134 | SCALABLE_COUNTER_DEFINE(log_queue_cnt_mem_released); |
135 | SCALABLE_COUNTER_DEFINE(log_queue_cnt_mem_failed); |
136 | |
137 | static log_queue_s PERCPU_DATA(oslog_queue); |
138 | static size_t lq_low_mem_limit; |
139 | |
140 | static void * |
141 | log_queue_buffer_alloc(size_t amount) |
142 | { |
143 | return kalloc_data_tag(amount, Z_WAITOK_ZERO, VM_KERN_MEMORY_LOG); |
144 | } |
145 | |
146 | static void |
147 | log_queue_buffer_free(void *addr, size_t amount) |
148 | { |
149 | kfree_data(addr, amount); |
150 | } |
151 | |
152 | #define log_queue_entry_size(p) (sizeof(log_queue_entry_s) + (p)->lp_data_size) |
153 | |
154 | #define publish(a, v) os_atomic_store((a), (v), release) |
155 | #define read_dependency(v) os_atomic_load((v), dependency) |
156 | #define read_dependent(v, t) os_atomic_load_with_dependency_on((v), (uintptr_t)(t)) |
157 | #define read_dependent_w(v, t) ({ \ |
158 | __auto_type _v = os_atomic_inject_dependency((v), (uintptr_t)(t)); \ |
159 | os_atomic_load_wide(_v, dependency); \ |
160 | }) |
161 | |
162 | static log_queue_entry_state_t |
163 | log_queue_entry_state(const log_queue_entry_t lqe) |
164 | { |
165 | log_queue_entry_state_t state = read_dependency(&lqe->lqe_state); |
166 | assert(state != LOG_QUEUE_ENTRY_STATE_INVALID); |
167 | return state; |
168 | } |
169 | |
170 | static log_queue_entry_t |
171 | log_queue_entry_alloc(log_queue_t lq, size_t lqe_size) |
172 | { |
173 | for (short i = 0; i < LQ_MAX_LM_SLOTS; i++) { |
174 | if (!LQ_MEM_ENABLED(lq, i)) { |
175 | continue; |
176 | } |
177 | log_queue_entry_t lqe = logmem_alloc(&lq->lq_mem[i], &lqe_size); |
178 | if (lqe) { |
179 | assert(lqe_size <= lq->lq_cnt_mem_avail); |
180 | lq->lq_cnt_mem_avail -= lqe_size; |
181 | assert(lqe_size <= UINT16_MAX); |
182 | lqe->lqe_size = (uint16_t)lqe_size; |
183 | lqe->lqe_lm_id = i; |
184 | return lqe; |
185 | } |
186 | } |
187 | |
188 | return NULL; |
189 | } |
190 | |
191 | static void |
192 | log_queue_entry_free(log_queue_t lq, log_queue_entry_t lqe) |
193 | { |
194 | const size_t lqe_size = lqe->lqe_size; |
195 | const uint16_t lqe_lm_id = lqe->lqe_lm_id; |
196 | |
197 | bzero(s: lqe, n: lqe_size); |
198 | logmem_free(&lq->lq_mem[lqe_lm_id], lqe, lqe_size); |
199 | lq->lq_cnt_mem_avail += lqe_size; |
200 | } |
201 | |
202 | static bool |
203 | log_queue_add_entry(log_queue_t lq, log_payload_t lp, const uint8_t *lp_data) |
204 | { |
205 | log_queue_entry_t lqe = log_queue_entry_alloc(lq, log_queue_entry_size(lp)); |
206 | if (!lqe) { |
207 | counter_inc_preemption_disabled(&log_queue_cnt_dropped_nomem); |
208 | return false; |
209 | } |
210 | assert(lqe->lqe_size >= lp->lp_data_size); |
211 | |
212 | lqe->lqe_payload = *lp; |
213 | (void) memcpy(dst: (uint8_t *)lqe + sizeof(*lqe), src: lp_data, n: lqe->lqe_payload.lp_data_size); |
214 | STAILQ_INSERT_TAIL(&lq->lq_log_list, lqe, lqe_link); |
215 | publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_STORED); |
216 | |
217 | counter_inc_preemption_disabled(&log_queue_cnt_queued); |
218 | |
219 | return true; |
220 | } |
221 | |
222 | /* |
223 | * Remove successfully sent logs from a dispatch list and free them. |
224 | */ |
225 | static size_t |
226 | dispatch_list_cleanup(log_queue_t lq) |
227 | { |
228 | log_queue_entry_t lqe, lqe_tmp; |
229 | size_t freed = 0; |
230 | |
231 | STAILQ_FOREACH_SAFE(lqe, &lq->lq_dispatch_list, lqe_link, lqe_tmp) { |
232 | log_queue_entry_state_t lqe_state = log_queue_entry_state(lqe); |
233 | assert(lqe_state != LOG_QUEUE_ENTRY_STATE_STORED); |
234 | |
235 | if (lqe_state == LOG_QUEUE_ENTRY_STATE_SENT) { |
236 | STAILQ_REMOVE(&lq->lq_dispatch_list, lqe, log_queue_entry, lqe_link); |
237 | publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_INVALID); |
238 | log_queue_entry_free(lq, lqe); |
239 | counter_dec_preemption_disabled(&log_queue_cnt_queued); |
240 | freed++; |
241 | } |
242 | } |
243 | |
244 | return freed; |
245 | } |
246 | |
247 | /* |
248 | * Walk and collect logs stored in the log queue suitable for dispatching. |
249 | * First, collect previously failed logs, then (if still enough space) grab new |
250 | * logs. |
251 | */ |
252 | static size_t |
253 | log_dispatch_prepare(log_queue_t lq, size_t requested, log_queue_entry_t *buf) |
254 | { |
255 | log_queue_entry_t lqe, lqe_tmp; |
256 | size_t collected = 0; |
257 | |
258 | STAILQ_FOREACH(lqe, &lq->lq_dispatch_list, lqe_link) { |
259 | log_queue_entry_state_t lqe_state = log_queue_entry_state(lqe); |
260 | assert(lqe_state != LOG_QUEUE_ENTRY_STATE_STORED); |
261 | |
262 | if (lqe_state == LOG_QUEUE_ENTRY_STATE_FAILED) { |
263 | publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_DISPATCHED); |
264 | buf[collected++] = lqe; |
265 | } |
266 | |
267 | if (collected == requested) { |
268 | return collected; |
269 | } |
270 | } |
271 | assert(collected < requested); |
272 | |
273 | STAILQ_FOREACH_SAFE(lqe, &lq->lq_log_list, lqe_link, lqe_tmp) { |
274 | assert(log_queue_entry_state(lqe) == LOG_QUEUE_ENTRY_STATE_STORED); |
275 | |
276 | STAILQ_REMOVE(&lq->lq_log_list, lqe, log_queue_entry, lqe_link); |
277 | STAILQ_INSERT_TAIL(&lq->lq_dispatch_list, lqe, lqe_link); |
278 | publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_DISPATCHED); |
279 | |
280 | buf[collected++] = lqe; |
281 | if (collected == requested) { |
282 | break; |
283 | } |
284 | } |
285 | |
286 | return collected; |
287 | } |
288 | |
289 | /* |
290 | * Send dispatched logs to the firehose. Skip streaming when replaying. |
291 | * Streaming does not process timestamps and would therefore show logs out of |
292 | * order. |
293 | */ |
294 | static void |
295 | log_queue_dispatch_logs(size_t logs_count, log_queue_entry_t *logs) |
296 | { |
297 | for (size_t i = 0; i < logs_count; i++) { |
298 | const log_queue_entry_t lqe = logs[i]; |
299 | log_queue_entry_state_t lqe_state = log_queue_entry_state(lqe); |
300 | |
301 | if (lqe_state == LOG_QUEUE_ENTRY_STATE_DISPATCHED) { |
302 | const log_payload_t lqe_lp = &lqe->lqe_payload; |
303 | |
304 | log_payload_s lp = { |
305 | .lp_ftid = read_dependent_w(&lqe_lp->lp_ftid, lqe_state), |
306 | .lp_timestamp = read_dependent_w(&lqe_lp->lp_timestamp, lqe_state), |
307 | .lp_stream = read_dependent(&lqe_lp->lp_stream, lqe_state), |
308 | .lp_pub_data_size = read_dependent(&lqe_lp->lp_pub_data_size, lqe_state), |
309 | .lp_data_size = read_dependent(&lqe_lp->lp_data_size, lqe_state) |
310 | }; |
311 | const void *lp_data = (uint8_t *)lqe + sizeof(*lqe); |
312 | |
313 | /* |
314 | * The log queue mechanism expects only the state to be |
315 | * modified here since we are likely running on a |
316 | * different cpu. Queue cleanup will be done safely |
317 | * later in dispatch_list_cleanup(). |
318 | */ |
319 | if (log_payload_send(&lp, lp_data, false)) { |
320 | publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_SENT); |
321 | counter_inc(&log_queue_cnt_sent); |
322 | } else { |
323 | publish(&lqe->lqe_state, LOG_QUEUE_ENTRY_STATE_FAILED); |
324 | } |
325 | } |
326 | } |
327 | } |
328 | |
329 | static bool |
330 | log_queue_empty(const log_queue_t lq) |
331 | { |
332 | return STAILQ_EMPTY(&lq->lq_log_list) && STAILQ_EMPTY(&lq->lq_dispatch_list); |
333 | } |
334 | |
335 | static boolean_t |
336 | log_queue_low_mem(const log_queue_t lq) |
337 | { |
338 | return lq->lq_cnt_mem_avail < (lq->lq_cnt_mem_active * lq_low_mem_limit); |
339 | } |
340 | |
341 | static lq_req_state_t |
342 | log_queue_request_state(log_queue_t lq) |
343 | { |
344 | lq_req_state_t req_state = read_dependency(&lq->lq_req_state); |
345 | return req_state; |
346 | } |
347 | |
348 | static void |
349 | log_queue_mem_init(log_queue_t lq, size_t idx, void *buf, size_t buflen) |
350 | { |
351 | assert(buf); |
352 | assert(buflen > 0); |
353 | assert(idx < LQ_MAX_LM_SLOTS); |
354 | assert(!LQ_MEM_ENABLED(lq, idx)); |
355 | |
356 | logmem_init(&lq->lq_mem[idx], buf, buflen, lq->lq_mem_size_order, |
357 | LQ_MIN_LOG_SZ_ORDER, LQ_MAX_LOG_SZ_ORDER); |
358 | } |
359 | |
360 | static int |
361 | log_queue_mem_free_slot(log_queue_t lq) |
362 | { |
363 | assert(LQ_MEM_ENABLED(lq, 0)); |
364 | |
365 | for (int i = 1; i < LQ_MAX_LM_SLOTS; i++) { |
366 | if (!LQ_MEM_ENABLED(lq, i)) { |
367 | return i; |
368 | } |
369 | } |
370 | return -1; |
371 | } |
372 | |
373 | static void |
374 | log_queue_memory_handler(thread_call_param_t a0, __unused thread_call_param_t a1) |
375 | { |
376 | log_queue_t lq = (log_queue_t)a0; |
377 | lq_req_state_t req_state = log_queue_request_state(lq); |
378 | |
379 | assert(req_state != LQ_REQ_STATE_INVALID); |
380 | |
381 | if (req_state == LQ_REQ_STATE_ALLOCATING) { |
382 | lq->lq_req_mem = log_queue_buffer_alloc(amount: lq->lq_mem_size); |
383 | publish(&lq->lq_req_state, LQ_REQ_STATE_READY); |
384 | |
385 | if (lq->lq_req_mem) { |
386 | counter_inc(&log_queue_cnt_mem_allocated); |
387 | } else { |
388 | counter_inc(&log_queue_cnt_mem_failed); |
389 | } |
390 | } else if (req_state == LQ_REQ_STATE_RELEASING) { |
391 | void *buf = read_dependent(&lq->lq_req_mem, req_state); |
392 | |
393 | log_queue_buffer_free(addr: buf, amount: lq->lq_mem_size); |
394 | lq->lq_req_mem = NULL; |
395 | publish(&lq->lq_req_state, LQ_REQ_STATE_READY); |
396 | |
397 | counter_inc(&log_queue_cnt_mem_released); |
398 | } |
399 | } |
400 | |
401 | static void |
402 | log_queue_order_memory(log_queue_t lq) |
403 | { |
404 | boolean_t __assert_only running; |
405 | |
406 | lq->lq_req_mem = NULL; |
407 | publish(&lq->lq_req_state, LQ_REQ_STATE_ALLOCATING); |
408 | |
409 | running = thread_call_enter(call: lq->lq_mem_handler); |
410 | assert(!running); |
411 | } |
412 | |
413 | static void |
414 | log_queue_release_memory(log_queue_t lq, void *buf) |
415 | { |
416 | boolean_t __assert_only running; |
417 | |
418 | assert(buf); |
419 | lq->lq_req_mem = buf; |
420 | publish(&lq->lq_req_state, LQ_REQ_STATE_RELEASING); |
421 | |
422 | running = thread_call_enter(call: lq->lq_mem_handler); |
423 | assert(!running); |
424 | } |
425 | |
426 | static void |
427 | log_queue_mem_enable(log_queue_t lq, size_t i) |
428 | { |
429 | logmem_t *lm = &lq->lq_mem[i]; |
430 | assert(!LQ_MEM_ENABLED(lq, i)); |
431 | |
432 | LQ_MEM_ENABLE(lq, i); |
433 | lq->lq_cnt_mem_active++; |
434 | lq->lq_cnt_mem_avail += lm->lm_cnt_free; |
435 | } |
436 | |
437 | static void |
438 | log_queue_mem_disable(log_queue_t lq, size_t i) |
439 | { |
440 | logmem_t *lm = &lq->lq_mem[i]; |
441 | assert(LQ_MEM_ENABLED(lq, i)); |
442 | |
443 | LQ_MEM_DISABLE(lq, i); |
444 | lq->lq_cnt_mem_active--; |
445 | lq->lq_cnt_mem_avail -= lm->lm_cnt_free; |
446 | } |
447 | |
448 | static void * |
449 | log_queue_mem_reclaim(log_queue_t lq) |
450 | { |
451 | for (int i = 1; i < LQ_MAX_LM_SLOTS; i++) { |
452 | logmem_t *lm = &lq->lq_mem[i]; |
453 | if (LQ_MEM_ENABLED(lq, i) && logmem_empty(lm)) { |
454 | assert(lm->lm_mem_size == lq->lq_mem_size); |
455 | void *reclaimed = lm->lm_mem; |
456 | log_queue_mem_disable(lq, i); |
457 | /* Do not use bzero here, see rdar://116922009 */ |
458 | *lm = (logmem_t){ }; |
459 | return reclaimed; |
460 | } |
461 | } |
462 | return NULL; |
463 | } |
464 | |
465 | static void |
466 | log_queue_mem_reconfigure(log_queue_t lq) |
467 | { |
468 | assert(lq->lq_mem_state == LQ_MEM_STATE_ALLOCATING || |
469 | lq->lq_mem_state == LQ_MEM_STATE_RELEASING); |
470 | |
471 | lq_req_state_t req_state = log_queue_request_state(lq); |
472 | |
473 | if (req_state == LQ_REQ_STATE_READY) { |
474 | if (lq->lq_mem_state == LQ_MEM_STATE_ALLOCATING) { |
475 | void *buf = read_dependent(&lq->lq_req_mem, req_state); |
476 | if (buf) { |
477 | const int i = log_queue_mem_free_slot(lq); |
478 | assert(i > 0); |
479 | log_queue_mem_init(lq, idx: i, buf, buflen: lq->lq_mem_size); |
480 | log_queue_mem_enable(lq, i); |
481 | } |
482 | } |
483 | lq->lq_mem_state = LQ_MEM_STATE_READY; |
484 | publish(&lq->lq_req_state, LQ_REQ_STATE_INVALID); |
485 | } |
486 | } |
487 | |
488 | static boolean_t |
489 | log_queue_needs_memory(log_queue_t lq, boolean_t new_suspend) |
490 | { |
491 | if (new_suspend || log_queue_low_mem(lq)) { |
492 | return lq->lq_cnt_mem_active < LQ_MAX_LM_SLOTS; |
493 | } |
494 | return false; |
495 | } |
496 | |
497 | static boolean_t |
498 | log_queue_can_release_memory(log_queue_t lq) |
499 | { |
500 | assert(lq->lq_mem_state == LQ_MEM_STATE_READY); |
501 | |
502 | if (lq->lq_cnt_mem_active > 1 && log_queue_empty(lq) && !lq->lq_suspend) { |
503 | const uint64_t total_log_cnt = counter_load(&log_queue_cnt_received); |
504 | return total_log_cnt > LQ_DEFAULT_FREE_AFTER_CNT; |
505 | } |
506 | return false; |
507 | } |
508 | |
509 | extern boolean_t tasks_suspend_state; |
510 | |
511 | static boolean_t |
512 | detect_new_suspend(log_queue_t lq) |
513 | { |
514 | if (!tasks_suspend_state) { |
515 | lq->lq_suspend = false; |
516 | return false; |
517 | } |
518 | |
519 | if (!lq->lq_suspend) { |
520 | lq->lq_suspend = true; |
521 | return true; |
522 | } |
523 | |
524 | return false; |
525 | } |
526 | |
527 | static void |
528 | log_queue_dispatch(void) |
529 | { |
530 | lq_mem_state_t new_mem_state = LQ_MEM_STATE_READY; |
531 | void *reclaimed_memory = NULL; |
532 | |
533 | disable_preemption(); |
534 | |
535 | log_queue_t lq = PERCPU_GET(oslog_queue); |
536 | if (__improbable(!lq->lq_ready)) { |
537 | enable_preemption(); |
538 | return; |
539 | } |
540 | |
541 | dispatch_list_cleanup(lq); |
542 | |
543 | log_queue_entry_t logs[LQ_BATCH_SIZE]; |
544 | size_t logs_count = log_dispatch_prepare(lq, LQ_BATCH_SIZE, buf: (log_queue_entry_t *)&logs); |
545 | |
546 | boolean_t new_suspend = detect_new_suspend(lq); |
547 | |
548 | if (__improbable(lq->lq_mem_state != LQ_MEM_STATE_READY)) { |
549 | log_queue_mem_reconfigure(lq); |
550 | } else if (logs_count == 0 && log_queue_can_release_memory(lq)) { |
551 | reclaimed_memory = log_queue_mem_reclaim(lq); |
552 | if (reclaimed_memory) { |
553 | lq->lq_mem_state = LQ_MEM_STATE_RELEASING; |
554 | new_mem_state = lq->lq_mem_state; |
555 | } |
556 | } else if (log_queue_needs_memory(lq, new_suspend)) { |
557 | lq->lq_mem_state = LQ_MEM_STATE_ALLOCATING; |
558 | new_mem_state = lq->lq_mem_state; |
559 | } |
560 | |
561 | enable_preemption(); |
562 | |
563 | switch (new_mem_state) { |
564 | case LQ_MEM_STATE_RELEASING: |
565 | assert(logs_count == 0); |
566 | log_queue_release_memory(lq, buf: reclaimed_memory); |
567 | break; |
568 | case LQ_MEM_STATE_ALLOCATING: |
569 | log_queue_order_memory(lq); |
570 | /* FALLTHROUGH */ |
571 | case LQ_MEM_STATE_READY: |
572 | log_queue_dispatch_logs(logs_count, logs); |
573 | break; |
574 | default: |
575 | panic("Invalid log memory state %u" , new_mem_state); |
576 | break; |
577 | } |
578 | } |
579 | |
580 | static bool |
581 | log_queue_add(log_payload_t lp, const uint8_t *lp_data) |
582 | { |
583 | boolean_t order_memory = false; |
584 | |
585 | disable_preemption(); |
586 | |
587 | log_queue_t lq = PERCPU_GET(oslog_queue); |
588 | if (__improbable(!lq->lq_ready)) { |
589 | enable_preemption(); |
590 | counter_inc(&log_queue_cnt_dropped_off); |
591 | return false; |
592 | } |
593 | |
594 | boolean_t new_suspend = detect_new_suspend(lq); |
595 | |
596 | if (__improbable(lq->lq_mem_state != LQ_MEM_STATE_READY)) { |
597 | log_queue_mem_reconfigure(lq); |
598 | } else if (log_queue_needs_memory(lq, new_suspend)) { |
599 | lq->lq_mem_state = LQ_MEM_STATE_ALLOCATING; |
600 | order_memory = true; |
601 | } |
602 | |
603 | bool added = log_queue_add_entry(lq, lp, lp_data); |
604 | enable_preemption(); |
605 | |
606 | if (order_memory) { |
607 | log_queue_order_memory(lq); |
608 | } |
609 | |
610 | return added; |
611 | } |
612 | |
613 | __startup_func |
614 | static size_t |
615 | log_queue_init_memory(log_queue_t lq, size_t lm_count) |
616 | { |
617 | assert(lm_count <= LQ_MAX_LM_SLOTS); |
618 | |
619 | for (size_t i = 0; i < lm_count; i++) { |
620 | void *buf = log_queue_buffer_alloc(amount: lq->lq_mem_size); |
621 | if (!buf) { |
622 | return i; |
623 | } |
624 | counter_inc(&log_queue_cnt_mem_allocated); |
625 | log_queue_mem_init(lq, idx: i, buf, buflen: lq->lq_mem_size); |
626 | log_queue_mem_enable(lq, i); |
627 | } |
628 | |
629 | return lm_count; |
630 | } |
631 | |
632 | __startup_func |
633 | static void |
634 | oslog_init_log_queues(void) |
635 | { |
636 | if (os_log_disabled()) { |
637 | printf("Log queues disabled: Logging disabled by ATM\n" ); |
638 | return; |
639 | } |
640 | |
641 | if (lq_bootarg_size_order == 0) { |
642 | printf("Log queues disabled: Zero lq_size_order boot argument\n" ); |
643 | return; |
644 | } |
645 | |
646 | lq_bootarg_size_order = MAX(lq_bootarg_size_order, PAGE_SHIFT); |
647 | lq_bootarg_size_order = MIN(lq_bootarg_size_order, LQ_MAX_SZ_ORDER); |
648 | |
649 | lq_bootarg_nslots = MAX(lq_bootarg_nslots, 1); |
650 | lq_bootarg_nslots = MIN(lq_bootarg_nslots, LQ_MAX_LM_SLOTS); |
651 | |
652 | lq_low_mem_limit = MAX(1 << (lq_bootarg_size_order - LQ_LOW_MEM_SCALE), 1024); |
653 | |
654 | unsigned int slot_count = 0; |
655 | |
656 | percpu_foreach(lq, oslog_queue) { |
657 | lq->lq_mem_size_order = lq_bootarg_size_order; |
658 | lq->lq_mem_size = round_page(x: logmem_required_size(lq->lq_mem_size_order, LQ_MIN_LOG_SZ_ORDER)); |
659 | lq->lq_mem_handler = thread_call_allocate(func: log_queue_memory_handler, param0: (thread_call_param_t)lq); |
660 | slot_count += log_queue_init_memory(lq, lm_count: lq_bootarg_nslots); |
661 | STAILQ_INIT(&lq->lq_log_list); |
662 | STAILQ_INIT(&lq->lq_dispatch_list); |
663 | lq->lq_ready = true; |
664 | } |
665 | |
666 | printf("Log queues configured: slot count: %u, per-slot size: %u, total size: %u\n" , |
667 | slot_count, (1 << lq_bootarg_size_order), |
668 | slot_count * (1 << lq_bootarg_size_order)); |
669 | } |
670 | STARTUP(OSLOG, STARTUP_RANK_SECOND, oslog_init_log_queues); |
671 | |
672 | bool |
673 | log_queue_log(log_payload_t lp, const void *lp_data, bool stream) |
674 | { |
675 | assert(lp); |
676 | assert(oslog_is_safe() || startup_phase < STARTUP_SUB_EARLY_BOOT); |
677 | |
678 | counter_inc(&log_queue_cnt_received); |
679 | |
680 | if (log_payload_send(lp, lp_data, stream)) { |
681 | counter_inc(&log_queue_cnt_sent); |
682 | log_queue_dispatch(); |
683 | return true; |
684 | } |
685 | counter_inc(&log_queue_cnt_rejected_fh); |
686 | |
687 | if (!log_queue_add(lp, lp_data)) { |
688 | return false; |
689 | } |
690 | |
691 | return true; |
692 | } |
693 | |