1 | /* |
2 | * Copyright (c) 2000-2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @Apple_LICENSE_HEADER_START@ |
5 | * |
6 | * The contents of this file constitute Original Code as defined in and |
7 | * are subject to the Apple Public Source License Version 1.1 (the |
8 | * "License"). You may not use this file except in compliance with the |
9 | * License. Please obtain a copy of the License at |
10 | * http://www.apple.com/publicsource and read it before using this file. |
11 | * |
12 | * This Original Code and all software distributed under the License are |
13 | * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
14 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
15 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the |
17 | * License for the specific language governing rights and limitations |
18 | * under the License. |
19 | * |
20 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
21 | */ |
22 | |
23 | #include <sys/errno.h> |
24 | #include <sys/kdebug_private.h> |
25 | #include <sys/proc_internal.h> |
26 | #include <sys/vm.h> |
27 | #include <sys/sysctl.h> |
28 | #include <sys/kdebug_common.h> |
29 | #include <sys/kdebug.h> |
30 | #include <sys/kdebug_triage.h> |
31 | #include <sys/kauth.h> |
32 | #include <sys/ktrace.h> |
33 | #include <sys/sysproto.h> |
34 | #include <sys/bsdtask_info.h> |
35 | #include <sys/random.h> |
36 | |
37 | #include <mach/mach_vm.h> |
38 | #include <machine/atomic.h> |
39 | |
40 | #include <mach/machine.h> |
41 | #include <mach/vm_map.h> |
42 | #include <kern/clock.h> |
43 | |
44 | #include <kern/task.h> |
45 | #include <kern/debug.h> |
46 | #include <kern/kalloc.h> |
47 | #include <kern/telemetry.h> |
48 | #include <kern/sched_prim.h> |
49 | #include <sys/lock.h> |
50 | #include <pexpert/device_tree.h> |
51 | |
52 | #include <sys/malloc.h> |
53 | |
54 | #include <sys/vnode.h> |
55 | #include <sys/vnode_internal.h> |
56 | #include <sys/fcntl.h> |
57 | #include <sys/file_internal.h> |
58 | #include <sys/ubc.h> |
59 | #include <sys/param.h> /* for isset() */ |
60 | |
61 | #include <libkern/OSAtomic.h> |
62 | |
63 | #include <machine/pal_routines.h> |
64 | #include <machine/atomic.h> |
65 | |
66 | |
67 | extern unsigned int wake_nkdbufs; |
68 | extern unsigned int trace_wrap; |
69 | |
70 | // Coprocessors (or "IOP"s) |
71 | // |
72 | // Coprocessors are auxiliary cores that want to participate in kdebug event |
73 | // logging. They are registered dynamically, as devices match hardware, and are |
74 | // each assigned an ID at registration. |
75 | // |
76 | // Once registered, a coprocessor is permanent; it cannot be unregistered. |
77 | // The current implementation depends on this for thread safety. |
78 | // |
79 | // The `kd_coprocs` list may be safely walked at any time, without holding |
80 | // locks. |
81 | // |
82 | // When starting a trace session, the current `kd_coprocs` head is captured. Any |
83 | // operations that depend on the buffer state (such as flushing IOP traces on |
84 | // reads, etc.) should use the captured list head. This will allow registrations |
85 | // to take place while trace is in use, though their events will be rejected |
86 | // until the next time a trace session is started. |
87 | |
88 | struct kd_coproc { |
89 | char full_name[32]; |
90 | kdebug_coproc_flags_t flags; |
91 | kd_callback_t callback; |
92 | uint32_t cpu_id; |
93 | struct kd_coproc *next; |
94 | struct mpsc_queue_chain chain; |
95 | }; |
96 | |
97 | static struct kd_coproc *kd_coprocs = NULL; |
98 | |
99 | // Use an MPSC queue to notify coprocessors of the current trace state during |
100 | // registration, if space is available for them in the current trace session. |
101 | static struct mpsc_daemon_queue _coproc_notify_queue; |
102 | |
103 | // Typefilter(s) |
104 | // |
105 | // A typefilter is a 8KB bitmap that is used to selectively filter events |
106 | // being recorded. It is able to individually address every class & subclass. |
107 | // |
108 | // There is a shared typefilter in the kernel which is lazily allocated. Once |
109 | // allocated, the shared typefilter is never deallocated. The shared typefilter |
110 | // is also mapped on demand into userspace processes that invoke kdebug_trace |
111 | // API from Libsyscall. When mapped into a userspace process, the memory is |
112 | // read only, and does not have a fixed address. |
113 | // |
114 | // It is a requirement that the kernel's shared typefilter always pass DBG_TRACE |
115 | // events. This is enforced automatically, by having the needed bits set any |
116 | // time the shared typefilter is mutated. |
117 | |
118 | typedef uint8_t *typefilter_t; |
119 | |
120 | static typefilter_t kdbg_typefilter; |
121 | static mach_port_t kdbg_typefilter_memory_entry; |
122 | |
123 | /* |
124 | * There are 3 combinations of page sizes: |
125 | * |
126 | * 4KB / 4KB |
127 | * 4KB / 16KB |
128 | * 16KB / 16KB |
129 | * |
130 | * The typefilter is exactly 8KB. In the first two scenarios, we would like |
131 | * to use 2 pages exactly; in the third scenario we must make certain that |
132 | * a full page is allocated so we do not inadvertantly share 8KB of random |
133 | * data to userspace. The round_page_32 macro rounds to kernel page size. |
134 | */ |
135 | #define TYPEFILTER_ALLOC_SIZE MAX(round_page_32(KDBG_TYPEFILTER_BITMAP_SIZE), KDBG_TYPEFILTER_BITMAP_SIZE) |
136 | |
137 | static typefilter_t |
138 | typefilter_create(void) |
139 | { |
140 | typefilter_t tf; |
141 | if (KERN_SUCCESS == kmem_alloc(map: kernel_map, addrp: (vm_offset_t*)&tf, |
142 | TYPEFILTER_ALLOC_SIZE, flags: KMA_DATA | KMA_ZERO, VM_KERN_MEMORY_DIAG)) { |
143 | return tf; |
144 | } |
145 | return NULL; |
146 | } |
147 | |
148 | static void |
149 | typefilter_deallocate(typefilter_t tf) |
150 | { |
151 | assert(tf != NULL); |
152 | assert(tf != kdbg_typefilter); |
153 | kmem_free(map: kernel_map, addr: (vm_offset_t)tf, TYPEFILTER_ALLOC_SIZE); |
154 | } |
155 | |
156 | static void |
157 | typefilter_copy(typefilter_t dst, typefilter_t src) |
158 | { |
159 | assert(src != NULL); |
160 | assert(dst != NULL); |
161 | memcpy(dst, src, KDBG_TYPEFILTER_BITMAP_SIZE); |
162 | } |
163 | |
164 | static void |
165 | typefilter_reject_all(typefilter_t tf) |
166 | { |
167 | assert(tf != NULL); |
168 | memset(s: tf, c: 0, KDBG_TYPEFILTER_BITMAP_SIZE); |
169 | } |
170 | |
171 | static void |
172 | typefilter_allow_all(typefilter_t tf) |
173 | { |
174 | assert(tf != NULL); |
175 | memset(s: tf, c: ~0, KDBG_TYPEFILTER_BITMAP_SIZE); |
176 | } |
177 | |
178 | static void |
179 | typefilter_allow_class(typefilter_t tf, uint8_t class) |
180 | { |
181 | assert(tf != NULL); |
182 | const uint32_t BYTES_PER_CLASS = 256 / 8; // 256 subclasses, 1 bit each |
183 | memset(s: &tf[class * BYTES_PER_CLASS], c: 0xFF, n: BYTES_PER_CLASS); |
184 | } |
185 | |
186 | static void |
187 | typefilter_allow_csc(typefilter_t tf, uint16_t csc) |
188 | { |
189 | assert(tf != NULL); |
190 | setbit(tf, csc); |
191 | } |
192 | |
193 | static bool |
194 | typefilter_is_debugid_allowed(typefilter_t tf, uint32_t id) |
195 | { |
196 | assert(tf != NULL); |
197 | return isset(tf, KDBG_EXTRACT_CSC(id)); |
198 | } |
199 | |
200 | static mach_port_t |
201 | typefilter_create_memory_entry(typefilter_t tf) |
202 | { |
203 | assert(tf != NULL); |
204 | |
205 | mach_port_t memory_entry = MACH_PORT_NULL; |
206 | memory_object_size_t size = TYPEFILTER_ALLOC_SIZE; |
207 | |
208 | kern_return_t kr = mach_make_memory_entry_64(target_task: kernel_map, |
209 | size: &size, |
210 | offset: (memory_object_offset_t)tf, |
211 | VM_PROT_READ, |
212 | object_handle: &memory_entry, |
213 | MACH_PORT_NULL); |
214 | if (kr != KERN_SUCCESS) { |
215 | return MACH_PORT_NULL; |
216 | } |
217 | |
218 | return memory_entry; |
219 | } |
220 | |
221 | static int kdbg_copyin_typefilter(user_addr_t addr, size_t size); |
222 | static void kdbg_enable_typefilter(void); |
223 | static void kdbg_disable_typefilter(void); |
224 | |
225 | // External prototypes |
226 | |
227 | void commpage_update_kdebug_state(void); |
228 | |
229 | static int kdbg_readcurthrmap(user_addr_t, size_t *); |
230 | static int kdbg_setpidex(kd_regtype *); |
231 | static int kdbg_setpid(kd_regtype *); |
232 | static int kdbg_reinit(unsigned int ); |
233 | #if DEVELOPMENT || DEBUG |
234 | static int kdbg_test(size_t flavor); |
235 | #endif /* DEVELOPMENT || DEBUG */ |
236 | |
237 | static int _write_legacy_header(bool write_thread_map, vnode_t vp, |
238 | vfs_context_t ctx); |
239 | static int kdbg_write_thread_map(vnode_t vp, vfs_context_t ctx); |
240 | static int kdbg_copyout_thread_map(user_addr_t buffer, size_t *buffer_size); |
241 | static void _clear_thread_map(void); |
242 | |
243 | static bool kdbg_wait(uint64_t timeout_ms); |
244 | static void kdbg_wakeup(void); |
245 | |
246 | static int _copy_cpu_map(int version, void **dst, size_t *size); |
247 | |
248 | static kd_threadmap *_thread_map_create_live(size_t max_count, |
249 | vm_size_t *map_size, vm_size_t *map_count); |
250 | |
251 | static bool kdebug_current_proc_enabled(uint32_t debugid); |
252 | static errno_t kdebug_check_trace_string(uint32_t debugid, uint64_t str_id); |
253 | |
254 | int kernel_debug_trace_write_to_file(user_addr_t *buffer, size_t *number, |
255 | size_t *count, size_t tempbuf_number, vnode_t vp, vfs_context_t ctx, |
256 | bool chunk); |
257 | |
258 | extern void IOSleep(int); |
259 | |
260 | unsigned int kdebug_enable = 0; |
261 | |
262 | // A static buffer to record events prior to the start of regular logging. |
263 | |
264 | #define KD_EARLY_BUFFER_SIZE (16 * 1024) |
265 | #define KD_EARLY_EVENT_COUNT (KD_EARLY_BUFFER_SIZE / sizeof(kd_buf)) |
266 | #if defined(__x86_64__) |
267 | __attribute__((aligned(KD_EARLY_BUFFER_SIZE))) |
268 | static kd_buf kd_early_buffer[KD_EARLY_EVENT_COUNT]; |
269 | #else /* defined(__x86_64__) */ |
270 | // On ARM, the space for this is carved out by osfmk/arm/data.s -- clang |
271 | // has problems aligning to greater than 4K. |
272 | extern kd_buf kd_early_buffer[KD_EARLY_EVENT_COUNT]; |
273 | #endif /* !defined(__x86_64__) */ |
274 | |
275 | static __security_const_late unsigned int kd_early_index = 0; |
276 | static __security_const_late bool kd_early_overflow = false; |
277 | static __security_const_late bool kd_early_done = false; |
278 | |
279 | static bool kd_waiter = false; |
280 | static LCK_SPIN_DECLARE(kd_wait_lock, &kdebug_lck_grp); |
281 | // Synchronize access to coprocessor list for kdebug trace. |
282 | static LCK_SPIN_DECLARE(kd_coproc_spinlock, &kdebug_lck_grp); |
283 | |
284 | #define TRACE_KDCOPYBUF_COUNT 8192 |
285 | #define TRACE_KDCOPYBUF_SIZE (TRACE_KDCOPYBUF_COUNT * sizeof(kd_buf)) |
286 | |
287 | struct kd_control kd_control_trace = { |
288 | .kds_free_list = {.raw = KDS_PTR_NULL}, |
289 | .enabled = 0, |
290 | .mode = KDEBUG_MODE_TRACE, |
291 | .kdebug_events_per_storage_unit = TRACE_EVENTS_PER_STORAGE_UNIT, |
292 | .kdebug_min_storage_units_per_cpu = TRACE_MIN_STORAGE_UNITS_PER_CPU, |
293 | .kdebug_kdcopybuf_count = TRACE_KDCOPYBUF_COUNT, |
294 | .kdebug_kdcopybuf_size = TRACE_KDCOPYBUF_SIZE, |
295 | .kdc_flags = 0, |
296 | .kdc_emit = KDEMIT_DISABLE, |
297 | .kdc_oldest_time = 0 |
298 | }; |
299 | |
300 | struct kd_buffer kd_buffer_trace = { |
301 | .kdb_event_count = 0, |
302 | .kdb_storage_count = 0, |
303 | .kdb_storage_threshold = 0, |
304 | .kdb_region_count = 0, |
305 | .kdb_info = NULL, |
306 | .kd_bufs = NULL, |
307 | .kdcopybuf = NULL |
308 | }; |
309 | |
310 | unsigned int kdlog_beg = 0; |
311 | unsigned int kdlog_end = 0; |
312 | unsigned int kdlog_value1 = 0; |
313 | unsigned int kdlog_value2 = 0; |
314 | unsigned int kdlog_value3 = 0; |
315 | unsigned int kdlog_value4 = 0; |
316 | |
317 | kd_threadmap *kd_mapptr = 0; |
318 | vm_size_t kd_mapsize = 0; |
319 | vm_size_t kd_mapcount = 0; |
320 | |
321 | off_t RAW_file_offset = 0; |
322 | int RAW_file_written = 0; |
323 | |
324 | /* |
325 | * A globally increasing counter for identifying strings in trace. Starts at |
326 | * 1 because 0 is a reserved return value. |
327 | */ |
328 | __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE))) |
329 | static uint64_t g_curr_str_id = 1; |
330 | |
331 | #define STR_ID_SIG_OFFSET (48) |
332 | #define STR_ID_MASK ((1ULL << STR_ID_SIG_OFFSET) - 1) |
333 | #define STR_ID_SIG_MASK (~STR_ID_MASK) |
334 | |
335 | /* |
336 | * A bit pattern for identifying string IDs generated by |
337 | * kdebug_trace_string(2). |
338 | */ |
339 | static uint64_t g_str_id_signature = (0x70acULL << STR_ID_SIG_OFFSET); |
340 | |
341 | #define RAW_VERSION3 0x00001000 |
342 | |
343 | #define V3_RAW_EVENTS 0x00001e00 |
344 | |
345 | static void |
346 | _coproc_lock(void) |
347 | { |
348 | lck_spin_lock_grp(lck: &kd_coproc_spinlock, grp: &kdebug_lck_grp); |
349 | } |
350 | |
351 | static void |
352 | _coproc_unlock(void) |
353 | { |
354 | lck_spin_unlock(lck: &kd_coproc_spinlock); |
355 | } |
356 | |
357 | static void |
358 | _coproc_list_check(void) |
359 | { |
360 | #if MACH_ASSERT |
361 | _coproc_lock(); |
362 | struct kd_coproc *coproc = kd_control_trace.kdc_coprocs; |
363 | if (coproc) { |
364 | /* Is list sorted by cpu_id? */ |
365 | struct kd_coproc* temp = coproc; |
366 | do { |
367 | assert(!temp->next || temp->next->cpu_id == temp->cpu_id - 1); |
368 | assert(temp->next || (temp->cpu_id == kdbg_cpu_count())); |
369 | } while ((temp = temp->next)); |
370 | |
371 | /* Does each entry have a function and a name? */ |
372 | temp = coproc; |
373 | do { |
374 | assert(temp->callback.func); |
375 | assert(strlen(temp->callback.iop_name) < sizeof(temp->callback.iop_name)); |
376 | } while ((temp = temp->next)); |
377 | } |
378 | _coproc_unlock(); |
379 | #endif // MACH_ASSERT |
380 | } |
381 | |
382 | static void |
383 | _coproc_list_callback(kd_callback_type type, void *arg) |
384 | { |
385 | if (kd_control_trace.kdc_flags & KDBG_DISABLE_COPROCS) { |
386 | return; |
387 | } |
388 | |
389 | _coproc_lock(); |
390 | // Coprocessor list is only ever prepended to. |
391 | struct kd_coproc *head = kd_control_trace.kdc_coprocs; |
392 | _coproc_unlock(); |
393 | while (head) { |
394 | head->callback.func(head->callback.context, type, arg); |
395 | head = head->next; |
396 | } |
397 | } |
398 | |
399 | // Leave some extra space for coprocessors to register while tracing is active. |
400 | #define (16) |
401 | // There are more coprocessors registering during boot tracing. |
402 | #define (32) |
403 | |
404 | static kdebug_emit_filter_t |
405 | _trace_emit_filter(void) |
406 | { |
407 | if (!kdebug_enable) { |
408 | return KDEMIT_DISABLE; |
409 | } else if (kd_control_trace.kdc_flags & KDBG_TYPEFILTER_CHECK) { |
410 | return KDEMIT_TYPEFILTER; |
411 | } else if (kd_control_trace.kdc_flags & KDBG_RANGECHECK) { |
412 | return KDEMIT_RANGE; |
413 | } else if (kd_control_trace.kdc_flags & KDBG_VALCHECK) { |
414 | return KDEMIT_EXACT; |
415 | } else { |
416 | return KDEMIT_ALL; |
417 | } |
418 | } |
419 | |
420 | static void |
421 | kdbg_set_tracing_enabled(bool enabled, uint32_t trace_type) |
422 | { |
423 | // Drain any events from coprocessors before making the state change. On |
424 | // enabling, this removes any stale events from before tracing. On |
425 | // disabling, this saves any events up to the point tracing is disabled. |
426 | _coproc_list_callback(type: KD_CALLBACK_SYNC_FLUSH, NULL); |
427 | |
428 | if (!enabled) { |
429 | // Give coprocessors a chance to log any events before tracing is |
430 | // disabled, outside the lock. |
431 | _coproc_list_callback(type: KD_CALLBACK_KDEBUG_DISABLED, NULL); |
432 | } |
433 | |
434 | int intrs_en = kdebug_storage_lock(ctl: &kd_control_trace); |
435 | if (enabled) { |
436 | // The oldest valid time is now; reject past events from coprocessors. |
437 | kd_control_trace.kdc_oldest_time = kdebug_timestamp(); |
438 | kdebug_enable |= trace_type; |
439 | kd_control_trace.kdc_emit = _trace_emit_filter(); |
440 | kd_control_trace.enabled = 1; |
441 | commpage_update_kdebug_state(); |
442 | } else { |
443 | kdebug_enable = 0; |
444 | kd_control_trace.kdc_emit = KDEMIT_DISABLE; |
445 | kd_control_trace.enabled = 0; |
446 | commpage_update_kdebug_state(); |
447 | } |
448 | kdebug_storage_unlock(ctl: &kd_control_trace, intrs_en); |
449 | |
450 | if (enabled) { |
451 | _coproc_list_callback(type: KD_CALLBACK_KDEBUG_ENABLED, NULL); |
452 | } |
453 | } |
454 | |
455 | static int |
456 | create_buffers_trace(unsigned int ) |
457 | { |
458 | int events_per_storage_unit = kd_control_trace.kdebug_events_per_storage_unit; |
459 | int min_storage_units_per_cpu = kd_control_trace.kdebug_min_storage_units_per_cpu; |
460 | |
461 | // For the duration of this allocation, trace code will only reference |
462 | // kdc_coprocs. |
463 | kd_control_trace.kdc_coprocs = kd_coprocs; |
464 | _coproc_list_check(); |
465 | |
466 | // If the list is valid, it is sorted from newest to oldest. Each entry is |
467 | // prepended, so the CPU IDs are sorted in descending order. |
468 | kd_control_trace.kdebug_cpus = kd_control_trace.kdc_coprocs ? |
469 | kd_control_trace.kdc_coprocs->cpu_id + 1 : kdbg_cpu_count(); |
470 | kd_control_trace.alloc_cpus = kd_control_trace.kdebug_cpus + extra_cpus; |
471 | |
472 | size_t min_event_count = kd_control_trace.alloc_cpus * |
473 | events_per_storage_unit * min_storage_units_per_cpu; |
474 | if (kd_buffer_trace.kdb_event_count < min_event_count) { |
475 | kd_buffer_trace.kdb_storage_count = kd_control_trace.alloc_cpus * min_storage_units_per_cpu; |
476 | } else { |
477 | kd_buffer_trace.kdb_storage_count = kd_buffer_trace.kdb_event_count / events_per_storage_unit; |
478 | } |
479 | |
480 | kd_buffer_trace.kdb_event_count = kd_buffer_trace.kdb_storage_count * events_per_storage_unit; |
481 | |
482 | kd_buffer_trace.kd_bufs = NULL; |
483 | |
484 | int error = create_buffers(ctl: &kd_control_trace, buf: &kd_buffer_trace, |
485 | VM_KERN_MEMORY_DIAG); |
486 | if (!error) { |
487 | struct kd_bufinfo *info = kd_buffer_trace.kdb_info; |
488 | struct kd_coproc *cur_iop = kd_control_trace.kdc_coprocs; |
489 | while (cur_iop != NULL) { |
490 | info[cur_iop->cpu_id].continuous_timestamps = ISSET(cur_iop->flags, |
491 | KDCP_CONTINUOUS_TIME); |
492 | cur_iop = cur_iop->next; |
493 | } |
494 | kd_buffer_trace.kdb_storage_threshold = kd_buffer_trace.kdb_storage_count / 2; |
495 | } |
496 | |
497 | return error; |
498 | } |
499 | |
500 | static void |
501 | delete_buffers_trace(void) |
502 | { |
503 | delete_buffers(ctl: &kd_control_trace, buf: &kd_buffer_trace); |
504 | } |
505 | |
506 | static int |
507 | _register_coproc_internal(const char *name, kdebug_coproc_flags_t flags, |
508 | kd_callback_fn callback, void *context) |
509 | { |
510 | struct kd_coproc *coproc = NULL; |
511 | |
512 | coproc = zalloc_permanent_type(struct kd_coproc); |
513 | coproc->callback.func = callback; |
514 | coproc->callback.context = context; |
515 | coproc->flags = flags; |
516 | strlcpy(dst: coproc->full_name, src: name, n: sizeof(coproc->full_name)); |
517 | |
518 | _coproc_lock(); |
519 | coproc->next = kd_coprocs; |
520 | coproc->cpu_id = kd_coprocs == NULL ? kdbg_cpu_count() : kd_coprocs->cpu_id + 1; |
521 | kd_coprocs = coproc; |
522 | if (coproc->cpu_id < kd_control_trace.alloc_cpus) { |
523 | kd_control_trace.kdc_coprocs = kd_coprocs; |
524 | kd_control_trace.kdebug_cpus += 1; |
525 | if (kdebug_enable) { |
526 | mpsc_daemon_enqueue(dq: &_coproc_notify_queue, elm: &coproc->chain, |
527 | options: MPSC_QUEUE_NONE); |
528 | } |
529 | } |
530 | _coproc_unlock(); |
531 | |
532 | return coproc->cpu_id; |
533 | } |
534 | |
535 | int |
536 | kernel_debug_register_callback(kd_callback_t callback) |
537 | { |
538 | // Be paranoid about using the provided name, but it's too late to reject |
539 | // it. |
540 | bool is_valid_name = false; |
541 | for (uint32_t length = 0; length < sizeof(callback.iop_name); ++length) { |
542 | if (callback.iop_name[length] > 0x20 && callback.iop_name[length] < 0x7F) { |
543 | continue; |
544 | } |
545 | if (callback.iop_name[length] == 0) { |
546 | if (length) { |
547 | is_valid_name = true; |
548 | } |
549 | break; |
550 | } |
551 | } |
552 | kd_callback_t sane_cb = callback; |
553 | if (!is_valid_name) { |
554 | strlcpy(dst: sane_cb.iop_name, src: "IOP-???" , n: sizeof(sane_cb.iop_name)); |
555 | } |
556 | |
557 | return _register_coproc_internal(name: sane_cb.iop_name, flags: 0, callback: sane_cb.func, |
558 | context: sane_cb.context); |
559 | } |
560 | |
561 | int |
562 | kdebug_register_coproc(const char *name, kdebug_coproc_flags_t flags, |
563 | kd_callback_fn callback, void *context) |
564 | { |
565 | size_t name_len = strlen(s: name); |
566 | if (!name || name_len == 0) { |
567 | panic("kdebug: invalid name for coprocessor: %p" , name); |
568 | } |
569 | for (size_t i = 0; i < name_len; i++) { |
570 | if (name[i] <= 0x20 || name[i] >= 0x7F) { |
571 | panic("kdebug: invalid name for coprocessor: %s" , name); |
572 | } |
573 | } |
574 | if (!callback) { |
575 | panic("kdebug: no callback for coprocessor `%s'" , name); |
576 | } |
577 | return _register_coproc_internal(name, flags, callback, context); |
578 | } |
579 | |
580 | static inline bool |
581 | _should_emit_debugid(kdebug_emit_filter_t emit, uint32_t debugid) |
582 | { |
583 | switch (emit) { |
584 | case KDEMIT_DISABLE: |
585 | return false; |
586 | case KDEMIT_TYPEFILTER: |
587 | return typefilter_is_debugid_allowed(tf: kdbg_typefilter, id: debugid); |
588 | case KDEMIT_RANGE: |
589 | return debugid >= kdlog_beg && debugid <= kdlog_end; |
590 | case KDEMIT_EXACT:; |
591 | uint32_t eventid = debugid & KDBG_EVENTID_MASK; |
592 | return eventid == kdlog_value1 || eventid == kdlog_value2 || |
593 | eventid == kdlog_value3 || eventid == kdlog_value4; |
594 | case KDEMIT_ALL: |
595 | return true; |
596 | } |
597 | } |
598 | |
599 | static void |
600 | _try_wakeup_above_threshold(uint32_t debugid) |
601 | { |
602 | bool over_threshold = kd_control_trace.kdc_storage_used >= |
603 | kd_buffer_trace.kdb_storage_threshold; |
604 | if (kd_waiter && over_threshold) { |
605 | // Wakeup any waiters if called from a safe context. |
606 | |
607 | const uint32_t INTERRUPT_EVENT = 0x01050000; |
608 | const uint32_t VMFAULT_EVENT = 0x01300008; |
609 | const uint32_t BSD_SYSCALL_CSC = 0x040c0000; |
610 | const uint32_t MACH_SYSCALL_CSC = 0x010c0000; |
611 | |
612 | uint32_t eventid = debugid & KDBG_EVENTID_MASK; |
613 | uint32_t csc = debugid & KDBG_CSC_MASK; |
614 | |
615 | if (eventid == INTERRUPT_EVENT || eventid == VMFAULT_EVENT || |
616 | csc == BSD_SYSCALL_CSC || csc == MACH_SYSCALL_CSC) { |
617 | kdbg_wakeup(); |
618 | } |
619 | } |
620 | } |
621 | |
622 | // Emit events from coprocessors. |
623 | void |
624 | kernel_debug_enter( |
625 | uint32_t coreid, |
626 | uint32_t debugid, |
627 | uint64_t timestamp, |
628 | uintptr_t arg1, |
629 | uintptr_t arg2, |
630 | uintptr_t arg3, |
631 | uintptr_t arg4, |
632 | uintptr_t threadid |
633 | ) |
634 | { |
635 | if (kd_control_trace.kdc_flags & KDBG_DISABLE_COPROCS) { |
636 | return; |
637 | } |
638 | kdebug_emit_filter_t emit = kd_control_trace.kdc_emit; |
639 | if (!emit || !kdebug_enable) { |
640 | return; |
641 | } |
642 | if (!_should_emit_debugid(emit, debugid)) { |
643 | return; |
644 | } |
645 | |
646 | struct kd_record kd_rec = { |
647 | .cpu = (int32_t)coreid, |
648 | .timestamp = (int64_t)timestamp, |
649 | .debugid = debugid, |
650 | .arg1 = arg1, |
651 | .arg2 = arg2, |
652 | .arg3 = arg3, |
653 | .arg4 = arg4, |
654 | .arg5 = threadid, |
655 | }; |
656 | kernel_debug_write(ctl: &kd_control_trace, buf: &kd_buffer_trace, kd_rec); |
657 | } |
658 | |
659 | __pure2 |
660 | static inline proc_t |
661 | kdebug_current_proc_unsafe(void) |
662 | { |
663 | return get_thread_ro_unchecked(current_thread())->tro_proc; |
664 | } |
665 | |
666 | // Return true iff the debug ID should be traced by the current process. |
667 | static inline bool |
668 | kdebug_debugid_procfilt_allowed(uint32_t debugid) |
669 | { |
670 | uint32_t procfilt_flags = kd_control_trace.kdc_flags & |
671 | (KDBG_PIDCHECK | KDBG_PIDEXCLUDE); |
672 | if (!procfilt_flags) { |
673 | return true; |
674 | } |
675 | |
676 | // DBG_TRACE and MACH_SCHED tracepoints ignore the process filter. |
677 | if ((debugid & KDBG_CSC_MASK) == MACHDBG_CODE(DBG_MACH_SCHED, 0) || |
678 | (KDBG_EXTRACT_CLASS(debugid) == DBG_TRACE)) { |
679 | return true; |
680 | } |
681 | |
682 | struct proc *curproc = kdebug_current_proc_unsafe(); |
683 | // If the process is missing (early in boot), allow it. |
684 | if (!curproc) { |
685 | return true; |
686 | } |
687 | |
688 | switch (procfilt_flags) { |
689 | case KDBG_PIDCHECK: |
690 | return curproc->p_kdebug; |
691 | case KDBG_PIDEXCLUDE: |
692 | return !curproc->p_kdebug; |
693 | default: |
694 | panic("kdebug: invalid procfilt flags %x" , kd_control_trace.kdc_flags); |
695 | } |
696 | } |
697 | |
698 | static void |
699 | kdebug_emit_internal(kdebug_emit_filter_t emit, |
700 | uint32_t debugid, |
701 | uintptr_t arg1, |
702 | uintptr_t arg2, |
703 | uintptr_t arg3, |
704 | uintptr_t arg4, |
705 | uintptr_t arg5, |
706 | uint64_t flags) |
707 | { |
708 | bool only_filter = flags & KDBG_FLAG_FILTERED; |
709 | bool observe_procfilt = !(flags & KDBG_FLAG_NOPROCFILT); |
710 | |
711 | if (!_should_emit_debugid(emit, debugid)) { |
712 | return; |
713 | } |
714 | if (emit == KDEMIT_ALL && only_filter) { |
715 | return; |
716 | } |
717 | if (!ml_at_interrupt_context() && observe_procfilt && |
718 | !kdebug_debugid_procfilt_allowed(debugid)) { |
719 | return; |
720 | } |
721 | |
722 | struct kd_record kd_rec = { |
723 | .cpu = -1, |
724 | .timestamp = -1, |
725 | .debugid = debugid, |
726 | .arg1 = arg1, |
727 | .arg2 = arg2, |
728 | .arg3 = arg3, |
729 | .arg4 = arg4, |
730 | .arg5 = arg5, |
731 | }; |
732 | kernel_debug_write(ctl: &kd_control_trace, buf: &kd_buffer_trace, kd_rec); |
733 | |
734 | #if KPERF |
735 | kperf_kdebug_callback(debugid: kd_rec.debugid, starting_fp: __builtin_frame_address(0)); |
736 | #endif // KPERF |
737 | } |
738 | |
739 | static void |
740 | kernel_debug_internal( |
741 | uint32_t debugid, |
742 | uintptr_t arg1, |
743 | uintptr_t arg2, |
744 | uintptr_t arg3, |
745 | uintptr_t arg4, |
746 | uintptr_t arg5, |
747 | uint64_t flags) |
748 | { |
749 | kdebug_emit_filter_t emit = kd_control_trace.kdc_emit; |
750 | if (!emit || !kdebug_enable) { |
751 | return; |
752 | } |
753 | kdebug_emit_internal(emit, debugid, arg1, arg2, arg3, arg4, arg5, flags); |
754 | _try_wakeup_above_threshold(debugid); |
755 | } |
756 | |
757 | __attribute__((noinline)) |
758 | void |
759 | kernel_debug(uint32_t debugid, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, |
760 | uintptr_t arg4, __unused uintptr_t arg5) |
761 | { |
762 | kernel_debug_internal(debugid, arg1, arg2, arg3, arg4, |
763 | arg5: (uintptr_t)thread_tid(thread: current_thread()), flags: 0); |
764 | } |
765 | |
766 | __attribute__((noinline)) |
767 | void |
768 | kernel_debug1(uint32_t debugid, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, |
769 | uintptr_t arg4, uintptr_t arg5) |
770 | { |
771 | kernel_debug_internal(debugid, arg1, arg2, arg3, arg4, arg5, flags: 0); |
772 | } |
773 | |
774 | __attribute__((noinline)) |
775 | void |
776 | kernel_debug_flags( |
777 | uint32_t debugid, |
778 | uintptr_t arg1, |
779 | uintptr_t arg2, |
780 | uintptr_t arg3, |
781 | uintptr_t arg4, |
782 | uint64_t flags) |
783 | { |
784 | kernel_debug_internal(debugid, arg1, arg2, arg3, arg4, |
785 | arg5: (uintptr_t)thread_tid(thread: current_thread()), flags); |
786 | } |
787 | |
788 | __attribute__((noinline)) |
789 | void |
790 | kernel_debug_filtered( |
791 | uint32_t debugid, |
792 | uintptr_t arg1, |
793 | uintptr_t arg2, |
794 | uintptr_t arg3, |
795 | uintptr_t arg4) |
796 | { |
797 | kernel_debug_flags(debugid, arg1, arg2, arg3, arg4, KDBG_FLAG_FILTERED); |
798 | } |
799 | |
800 | void |
801 | kernel_debug_string_early(const char *message) |
802 | { |
803 | uintptr_t a[4] = { 0 }; |
804 | strncpy((char *)a, message, sizeof(a)); |
805 | KERNEL_DEBUG_EARLY(TRACE_INFO_STRING, a[0], a[1], a[2], a[3]); |
806 | } |
807 | |
808 | #define SIMPLE_STR_LEN (64) |
809 | static_assert(SIMPLE_STR_LEN % sizeof(uintptr_t) == 0); |
810 | |
811 | void |
812 | kernel_debug_string_simple(uint32_t eventid, const char *str) |
813 | { |
814 | if (!kdebug_enable) { |
815 | return; |
816 | } |
817 | |
818 | /* array of uintptr_ts simplifies emitting the string as arguments */ |
819 | uintptr_t str_buf[(SIMPLE_STR_LEN / sizeof(uintptr_t)) + 1] = { 0 }; |
820 | size_t len = strlcpy(dst: (char *)str_buf, src: str, SIMPLE_STR_LEN + 1); |
821 | len = MIN(len, SIMPLE_STR_LEN); |
822 | |
823 | uintptr_t thread_id = (uintptr_t)thread_tid(thread: current_thread()); |
824 | uint32_t debugid = eventid | DBG_FUNC_START; |
825 | |
826 | /* string can fit in a single tracepoint */ |
827 | if (len <= (4 * sizeof(uintptr_t))) { |
828 | debugid |= DBG_FUNC_END; |
829 | } |
830 | |
831 | kernel_debug_internal(debugid, arg1: str_buf[0], arg2: str_buf[1], arg3: str_buf[2], |
832 | arg4: str_buf[3], arg5: thread_id, flags: 0); |
833 | |
834 | debugid &= KDBG_EVENTID_MASK; |
835 | int i = 4; |
836 | size_t written = 4 * sizeof(uintptr_t); |
837 | |
838 | for (; written < len; i += 4, written += 4 * sizeof(uintptr_t)) { |
839 | /* if this is the last tracepoint to be emitted */ |
840 | if ((written + (4 * sizeof(uintptr_t))) >= len) { |
841 | debugid |= DBG_FUNC_END; |
842 | } |
843 | kernel_debug_internal(debugid, arg1: str_buf[i], |
844 | arg2: str_buf[i + 1], |
845 | arg3: str_buf[i + 2], |
846 | arg4: str_buf[i + 3], arg5: thread_id, flags: 0); |
847 | } |
848 | } |
849 | |
850 | extern int master_cpu; /* MACH_KERNEL_PRIVATE */ |
851 | /* |
852 | * Used prior to start_kern_tracing() being called. |
853 | * Log temporarily into a static buffer. |
854 | */ |
855 | void |
856 | kernel_debug_early( |
857 | uint32_t debugid, |
858 | uintptr_t arg1, |
859 | uintptr_t arg2, |
860 | uintptr_t arg3, |
861 | uintptr_t arg4) |
862 | { |
863 | #if defined(__x86_64__) |
864 | extern int early_boot; |
865 | /* |
866 | * Note that "early" isn't early enough in some cases where |
867 | * we're invoked before gsbase is set on x86, hence the |
868 | * check of "early_boot". |
869 | */ |
870 | if (early_boot) { |
871 | return; |
872 | } |
873 | #endif |
874 | |
875 | /* If early tracing is over, use the normal path. */ |
876 | if (kd_early_done) { |
877 | KDBG_RELEASE(debugid, arg1, arg2, arg3, arg4); |
878 | return; |
879 | } |
880 | |
881 | /* Do nothing if the buffer is full or we're not on the boot cpu. */ |
882 | kd_early_overflow = kd_early_index >= KD_EARLY_EVENT_COUNT; |
883 | if (kd_early_overflow || cpu_number() != master_cpu) { |
884 | return; |
885 | } |
886 | |
887 | kd_early_buffer[kd_early_index].debugid = debugid; |
888 | kd_early_buffer[kd_early_index].timestamp = mach_absolute_time(); |
889 | kd_early_buffer[kd_early_index].arg1 = arg1; |
890 | kd_early_buffer[kd_early_index].arg2 = arg2; |
891 | kd_early_buffer[kd_early_index].arg3 = arg3; |
892 | kd_early_buffer[kd_early_index].arg4 = arg4; |
893 | kd_early_buffer[kd_early_index].arg5 = 0; |
894 | kd_early_index++; |
895 | } |
896 | |
897 | /* |
898 | * Transfer the contents of the temporary buffer into the trace buffers. |
899 | * Precede that by logging the rebase time (offset) - the TSC-based time (in ns) |
900 | * when mach_absolute_time is set to 0. |
901 | */ |
902 | static void |
903 | kernel_debug_early_end(void) |
904 | { |
905 | if (cpu_number() != master_cpu) { |
906 | panic("kernel_debug_early_end() not call on boot processor" ); |
907 | } |
908 | |
909 | /* reset the current oldest time to allow early events */ |
910 | kd_control_trace.kdc_oldest_time = 0; |
911 | |
912 | #if defined(__x86_64__) |
913 | /* Fake sentinel marking the start of kernel time relative to TSC */ |
914 | kernel_debug_enter(0, TRACE_TIMESTAMPS, 0, |
915 | (uint32_t)(tsc_rebase_abs_time >> 32), (uint32_t)tsc_rebase_abs_time, |
916 | tsc_at_boot, 0, 0); |
917 | #endif /* defined(__x86_64__) */ |
918 | for (unsigned int i = 0; i < kd_early_index; i++) { |
919 | kernel_debug_enter(coreid: 0, |
920 | debugid: kd_early_buffer[i].debugid, |
921 | timestamp: kd_early_buffer[i].timestamp, |
922 | arg1: kd_early_buffer[i].arg1, |
923 | arg2: kd_early_buffer[i].arg2, |
924 | arg3: kd_early_buffer[i].arg3, |
925 | arg4: kd_early_buffer[i].arg4, |
926 | threadid: 0); |
927 | } |
928 | |
929 | /* Cut events-lost event on overflow */ |
930 | if (kd_early_overflow) { |
931 | KDBG_RELEASE(TRACE_LOST_EVENTS, 1); |
932 | } |
933 | |
934 | kd_early_done = true; |
935 | |
936 | /* This trace marks the start of kernel tracing */ |
937 | kernel_debug_string_early(message: "early trace done" ); |
938 | } |
939 | |
940 | void |
941 | kernel_debug_disable(void) |
942 | { |
943 | if (kdebug_enable) { |
944 | kdbg_set_tracing_enabled(false, trace_type: 0); |
945 | kdbg_wakeup(); |
946 | } |
947 | } |
948 | |
949 | // Returns true if debugid should only be traced from the kernel. |
950 | static int |
951 | _kernel_only_event(uint32_t debugid) |
952 | { |
953 | return KDBG_EXTRACT_CLASS(debugid) == DBG_TRACE; |
954 | } |
955 | |
956 | /* |
957 | * Support syscall SYS_kdebug_typefilter. |
958 | */ |
959 | int |
960 | kdebug_typefilter(__unused struct proc* p, struct kdebug_typefilter_args* uap, |
961 | __unused int *retval) |
962 | { |
963 | if (uap->addr == USER_ADDR_NULL || uap->size == USER_ADDR_NULL) { |
964 | return EINVAL; |
965 | } |
966 | |
967 | mach_vm_offset_t user_addr = 0; |
968 | vm_map_t user_map = current_map(); |
969 | const bool copy = false; |
970 | kern_return_t kr = mach_vm_map_kernel(target_map: user_map, address: &user_addr, |
971 | TYPEFILTER_ALLOC_SIZE, mask: 0, VM_MAP_KERNEL_FLAGS_ANYWHERE(), |
972 | port: kdbg_typefilter_memory_entry, offset: 0, copy, |
973 | VM_PROT_READ, VM_PROT_READ, VM_INHERIT_SHARE); |
974 | if (kr != KERN_SUCCESS) { |
975 | return mach_to_bsd_errno(mach_err: kr); |
976 | } |
977 | |
978 | vm_size_t user_ptr_size = vm_map_is_64bit(map: user_map) ? 8 : 4; |
979 | int error = copyout((void *)&user_addr, uap->addr, user_ptr_size); |
980 | if (error != 0) { |
981 | mach_vm_deallocate(target: user_map, address: user_addr, TYPEFILTER_ALLOC_SIZE); |
982 | } |
983 | return error; |
984 | } |
985 | |
986 | // Support SYS_kdebug_trace. |
987 | int |
988 | kdebug_trace(struct proc *p, struct kdebug_trace_args *uap, int32_t *retval) |
989 | { |
990 | struct kdebug_trace64_args uap64 = { |
991 | .code = uap->code, |
992 | .arg1 = uap->arg1, |
993 | .arg2 = uap->arg2, |
994 | .arg3 = uap->arg3, |
995 | .arg4 = uap->arg4, |
996 | }; |
997 | return kdebug_trace64(p, &uap64, retval); |
998 | } |
999 | |
1000 | // Support kdebug_trace(2). 64-bit arguments on K32 will get truncated |
1001 | // to fit in the 32-bit record format. |
1002 | // |
1003 | // It is intentional that error conditions are not checked until kdebug is |
1004 | // enabled. This is to match the userspace wrapper behavior, which is optimizing |
1005 | // for non-error case performance. |
1006 | int |
1007 | kdebug_trace64(__unused struct proc *p, struct kdebug_trace64_args *uap, |
1008 | __unused int32_t *retval) |
1009 | { |
1010 | if (__probable(kdebug_enable == 0)) { |
1011 | return 0; |
1012 | } |
1013 | if (_kernel_only_event(debugid: uap->code)) { |
1014 | return EPERM; |
1015 | } |
1016 | kernel_debug_internal(debugid: uap->code, arg1: (uintptr_t)uap->arg1, |
1017 | arg2: (uintptr_t)uap->arg2, arg3: (uintptr_t)uap->arg3, arg4: (uintptr_t)uap->arg4, |
1018 | arg5: (uintptr_t)thread_tid(thread: current_thread()), flags: 0); |
1019 | return 0; |
1020 | } |
1021 | |
1022 | /* |
1023 | * Adding enough padding to contain a full tracepoint for the last |
1024 | * portion of the string greatly simplifies the logic of splitting the |
1025 | * string between tracepoints. Full tracepoints can be generated using |
1026 | * the buffer itself, without having to manually add zeros to pad the |
1027 | * arguments. |
1028 | */ |
1029 | |
1030 | /* 2 string args in first tracepoint and 9 string data tracepoints */ |
1031 | #define STR_BUF_ARGS (2 + (32 * 4)) |
1032 | /* times the size of each arg on K64 */ |
1033 | #define MAX_STR_LEN (STR_BUF_ARGS * sizeof(uint64_t)) |
1034 | /* on K32, ending straddles a tracepoint, so reserve blanks */ |
1035 | #define STR_BUF_SIZE (MAX_STR_LEN + (2 * sizeof(uint32_t))) |
1036 | |
1037 | /* |
1038 | * This function does no error checking and assumes that it is called with |
1039 | * the correct arguments, including that the buffer pointed to by str is at |
1040 | * least STR_BUF_SIZE bytes. However, str must be aligned to word-size and |
1041 | * be NUL-terminated. In cases where a string can fit evenly into a final |
1042 | * tracepoint without its NUL-terminator, this function will not end those |
1043 | * strings with a NUL in trace. It's up to clients to look at the function |
1044 | * qualifier for DBG_FUNC_END in this case, to end the string. |
1045 | */ |
1046 | static uint64_t |
1047 | kernel_debug_string_internal(uint32_t debugid, uint64_t str_id, void *vstr, |
1048 | size_t str_len) |
1049 | { |
1050 | /* str must be word-aligned */ |
1051 | uintptr_t *str = vstr; |
1052 | size_t written = 0; |
1053 | uintptr_t thread_id; |
1054 | int i; |
1055 | uint32_t trace_debugid = TRACEDBG_CODE(DBG_TRACE_STRING, |
1056 | TRACE_STRING_GLOBAL); |
1057 | |
1058 | thread_id = (uintptr_t)thread_tid(thread: current_thread()); |
1059 | |
1060 | /* if the ID is being invalidated, just emit that */ |
1061 | if (str_id != 0 && str_len == 0) { |
1062 | kernel_debug_internal(debugid: trace_debugid | DBG_FUNC_START | DBG_FUNC_END, |
1063 | arg1: (uintptr_t)debugid, arg2: (uintptr_t)str_id, arg3: 0, arg4: 0, arg5: thread_id, flags: 0); |
1064 | return str_id; |
1065 | } |
1066 | |
1067 | /* generate an ID, if necessary */ |
1068 | if (str_id == 0) { |
1069 | str_id = OSIncrementAtomic64(address: (SInt64 *)&g_curr_str_id); |
1070 | str_id = (str_id & STR_ID_MASK) | g_str_id_signature; |
1071 | } |
1072 | |
1073 | trace_debugid |= DBG_FUNC_START; |
1074 | /* string can fit in a single tracepoint */ |
1075 | if (str_len <= (2 * sizeof(uintptr_t))) { |
1076 | trace_debugid |= DBG_FUNC_END; |
1077 | } |
1078 | |
1079 | kernel_debug_internal(debugid: trace_debugid, arg1: (uintptr_t)debugid, arg2: (uintptr_t)str_id, |
1080 | arg3: str[0], arg4: str[1], arg5: thread_id, flags: 0); |
1081 | |
1082 | trace_debugid &= KDBG_EVENTID_MASK; |
1083 | i = 2; |
1084 | written += 2 * sizeof(uintptr_t); |
1085 | |
1086 | for (; written < str_len; i += 4, written += 4 * sizeof(uintptr_t)) { |
1087 | if ((written + (4 * sizeof(uintptr_t))) >= str_len) { |
1088 | trace_debugid |= DBG_FUNC_END; |
1089 | } |
1090 | kernel_debug_internal(debugid: trace_debugid, arg1: str[i], |
1091 | arg2: str[i + 1], |
1092 | arg3: str[i + 2], |
1093 | arg4: str[i + 3], arg5: thread_id, flags: 0); |
1094 | } |
1095 | |
1096 | return str_id; |
1097 | } |
1098 | |
1099 | /* |
1100 | * Returns true if the current process can emit events, and false otherwise. |
1101 | * Trace system and scheduling events circumvent this check, as do events |
1102 | * emitted in interrupt context. |
1103 | */ |
1104 | static bool |
1105 | kdebug_current_proc_enabled(uint32_t debugid) |
1106 | { |
1107 | /* can't determine current process in interrupt context */ |
1108 | if (ml_at_interrupt_context()) { |
1109 | return true; |
1110 | } |
1111 | |
1112 | /* always emit trace system and scheduling events */ |
1113 | if ((KDBG_EXTRACT_CLASS(debugid) == DBG_TRACE || |
1114 | (debugid & KDBG_CSC_MASK) == MACHDBG_CODE(DBG_MACH_SCHED, 0))) { |
1115 | return true; |
1116 | } |
1117 | |
1118 | if (kd_control_trace.kdc_flags & KDBG_PIDCHECK) { |
1119 | proc_t cur_proc = kdebug_current_proc_unsafe(); |
1120 | |
1121 | /* only the process with the kdebug bit set is allowed */ |
1122 | if (cur_proc && !(cur_proc->p_kdebug)) { |
1123 | return false; |
1124 | } |
1125 | } else if (kd_control_trace.kdc_flags & KDBG_PIDEXCLUDE) { |
1126 | proc_t cur_proc = kdebug_current_proc_unsafe(); |
1127 | |
1128 | /* every process except the one with the kdebug bit set is allowed */ |
1129 | if (cur_proc && cur_proc->p_kdebug) { |
1130 | return false; |
1131 | } |
1132 | } |
1133 | |
1134 | return true; |
1135 | } |
1136 | |
1137 | bool |
1138 | kdebug_debugid_enabled(uint32_t debugid) |
1139 | { |
1140 | return _should_emit_debugid(emit: kd_control_trace.kdc_emit, debugid); |
1141 | } |
1142 | |
1143 | bool |
1144 | kdebug_debugid_explicitly_enabled(uint32_t debugid) |
1145 | { |
1146 | if (kd_control_trace.kdc_flags & KDBG_TYPEFILTER_CHECK) { |
1147 | return typefilter_is_debugid_allowed(tf: kdbg_typefilter, id: debugid); |
1148 | } else if (KDBG_EXTRACT_CLASS(debugid) == DBG_TRACE) { |
1149 | return true; |
1150 | } else if (kd_control_trace.kdc_flags & KDBG_RANGECHECK) { |
1151 | if (debugid < kdlog_beg || debugid > kdlog_end) { |
1152 | return false; |
1153 | } |
1154 | } else if (kd_control_trace.kdc_flags & KDBG_VALCHECK) { |
1155 | if ((debugid & KDBG_EVENTID_MASK) != kdlog_value1 && |
1156 | (debugid & KDBG_EVENTID_MASK) != kdlog_value2 && |
1157 | (debugid & KDBG_EVENTID_MASK) != kdlog_value3 && |
1158 | (debugid & KDBG_EVENTID_MASK) != kdlog_value4) { |
1159 | return false; |
1160 | } |
1161 | } |
1162 | |
1163 | return true; |
1164 | } |
1165 | |
1166 | /* |
1167 | * Returns 0 if a string can be traced with these arguments. Returns errno |
1168 | * value if error occurred. |
1169 | */ |
1170 | static errno_t |
1171 | kdebug_check_trace_string(uint32_t debugid, uint64_t str_id) |
1172 | { |
1173 | if (debugid & (DBG_FUNC_START | DBG_FUNC_END)) { |
1174 | return EINVAL; |
1175 | } |
1176 | if (_kernel_only_event(debugid)) { |
1177 | return EPERM; |
1178 | } |
1179 | if (str_id != 0 && (str_id & STR_ID_SIG_MASK) != g_str_id_signature) { |
1180 | return EINVAL; |
1181 | } |
1182 | return 0; |
1183 | } |
1184 | |
1185 | /* |
1186 | * Implementation of KPI kernel_debug_string. |
1187 | */ |
1188 | int |
1189 | kernel_debug_string(uint32_t debugid, uint64_t *str_id, const char *str) |
1190 | { |
1191 | /* arguments to tracepoints must be word-aligned */ |
1192 | __attribute__((aligned(sizeof(uintptr_t)))) char str_buf[STR_BUF_SIZE]; |
1193 | static_assert(sizeof(str_buf) > MAX_STR_LEN); |
1194 | vm_size_t len_copied; |
1195 | int err; |
1196 | |
1197 | assert(str_id); |
1198 | |
1199 | if (__probable(kdebug_enable == 0)) { |
1200 | return 0; |
1201 | } |
1202 | |
1203 | if (!kdebug_current_proc_enabled(debugid)) { |
1204 | return 0; |
1205 | } |
1206 | |
1207 | if (!kdebug_debugid_enabled(debugid)) { |
1208 | return 0; |
1209 | } |
1210 | |
1211 | if ((err = kdebug_check_trace_string(debugid, str_id: *str_id)) != 0) { |
1212 | return err; |
1213 | } |
1214 | |
1215 | if (str == NULL) { |
1216 | if (str_id == 0) { |
1217 | return EINVAL; |
1218 | } |
1219 | |
1220 | *str_id = kernel_debug_string_internal(debugid, str_id: *str_id, NULL, str_len: 0); |
1221 | return 0; |
1222 | } |
1223 | |
1224 | memset(s: str_buf, c: 0, n: sizeof(str_buf)); |
1225 | len_copied = strlcpy(dst: str_buf, src: str, MAX_STR_LEN + 1); |
1226 | *str_id = kernel_debug_string_internal(debugid, str_id: *str_id, vstr: str_buf, |
1227 | str_len: len_copied); |
1228 | return 0; |
1229 | } |
1230 | |
1231 | // Support kdebug_trace_string(2). |
1232 | int |
1233 | kdebug_trace_string(__unused struct proc *p, |
1234 | struct kdebug_trace_string_args *uap, |
1235 | uint64_t *retval) |
1236 | { |
1237 | __attribute__((aligned(sizeof(uintptr_t)))) char str_buf[STR_BUF_SIZE]; |
1238 | static_assert(sizeof(str_buf) > MAX_STR_LEN); |
1239 | size_t len_copied; |
1240 | int err; |
1241 | |
1242 | if (__probable(kdebug_enable == 0)) { |
1243 | return 0; |
1244 | } |
1245 | |
1246 | if (!kdebug_current_proc_enabled(debugid: uap->debugid)) { |
1247 | return 0; |
1248 | } |
1249 | |
1250 | if (!kdebug_debugid_enabled(debugid: uap->debugid)) { |
1251 | return 0; |
1252 | } |
1253 | |
1254 | if ((err = kdebug_check_trace_string(debugid: uap->debugid, str_id: uap->str_id)) != 0) { |
1255 | return err; |
1256 | } |
1257 | |
1258 | if (uap->str == USER_ADDR_NULL) { |
1259 | if (uap->str_id == 0) { |
1260 | return EINVAL; |
1261 | } |
1262 | |
1263 | *retval = kernel_debug_string_internal(debugid: uap->debugid, str_id: uap->str_id, |
1264 | NULL, str_len: 0); |
1265 | return 0; |
1266 | } |
1267 | |
1268 | memset(s: str_buf, c: 0, n: sizeof(str_buf)); |
1269 | err = copyinstr(uaddr: uap->str, kaddr: str_buf, MAX_STR_LEN + 1, done: &len_copied); |
1270 | |
1271 | /* it's alright to truncate the string, so allow ENAMETOOLONG */ |
1272 | if (err == ENAMETOOLONG) { |
1273 | str_buf[MAX_STR_LEN] = '\0'; |
1274 | } else if (err) { |
1275 | return err; |
1276 | } |
1277 | |
1278 | if (len_copied <= 1) { |
1279 | return EINVAL; |
1280 | } |
1281 | |
1282 | /* convert back to a length */ |
1283 | len_copied--; |
1284 | |
1285 | *retval = kernel_debug_string_internal(debugid: uap->debugid, str_id: uap->str_id, vstr: str_buf, |
1286 | str_len: len_copied); |
1287 | return 0; |
1288 | } |
1289 | |
1290 | int |
1291 | kdbg_reinit(unsigned int ) |
1292 | { |
1293 | kernel_debug_disable(); |
1294 | // Wait for any event writers to see the disable status. |
1295 | IOSleep(100); |
1296 | delete_buffers_trace(); |
1297 | |
1298 | _clear_thread_map(); |
1299 | kd_control_trace.kdc_live_flags &= ~KDBG_WRAPPED; |
1300 | |
1301 | RAW_file_offset = 0; |
1302 | RAW_file_written = 0; |
1303 | |
1304 | return create_buffers_trace(extra_cpus); |
1305 | } |
1306 | |
1307 | void |
1308 | kdbg_trace_data(struct proc *proc, long *arg_pid, long *arg_uniqueid) |
1309 | { |
1310 | if (proc) { |
1311 | *arg_pid = proc_getpid(proc); |
1312 | *arg_uniqueid = (long)proc_uniqueid(proc); |
1313 | if ((uint64_t)*arg_uniqueid != proc_uniqueid(proc)) { |
1314 | *arg_uniqueid = 0; |
1315 | } |
1316 | } else { |
1317 | *arg_pid = 0; |
1318 | *arg_uniqueid = 0; |
1319 | } |
1320 | } |
1321 | |
1322 | void kdebug_proc_name_args(struct proc *proc, long args[static 4]); |
1323 | void |
1324 | kdebug_proc_name_args(struct proc *proc, long args[static 4]) |
1325 | { |
1326 | if (proc) { |
1327 | strncpy((char *)args, proc_best_name(proc), 4 * sizeof(args[0])); |
1328 | } |
1329 | } |
1330 | |
1331 | static void |
1332 | _copy_ap_name(unsigned int cpuid, void *dst, size_t size) |
1333 | { |
1334 | const char *name = "AP" ; |
1335 | #if defined(__arm64__) |
1336 | const ml_topology_info_t *topology = ml_get_topology_info(); |
1337 | switch (topology->cpus[cpuid].cluster_type) { |
1338 | case CLUSTER_TYPE_E: |
1339 | name = "AP-E" ; |
1340 | break; |
1341 | case CLUSTER_TYPE_P: |
1342 | name = "AP-P" ; |
1343 | break; |
1344 | default: |
1345 | break; |
1346 | } |
1347 | #else /* defined(__arm64__) */ |
1348 | #pragma unused(cpuid) |
1349 | #endif /* !defined(__arm64__) */ |
1350 | strlcpy(dst, src: name, n: size); |
1351 | } |
1352 | |
1353 | // Write the specified `map_version` of CPU map to the `dst` buffer, using at |
1354 | // most `size` bytes. Returns 0 on success and sets `size` to the number of |
1355 | // bytes written, and either ENOMEM or EINVAL on failure. |
1356 | // |
1357 | // If the value pointed to by `dst` is NULL, memory is allocated, and `size` is |
1358 | // adjusted to the allocated buffer's size. |
1359 | // |
1360 | // NB: `coprocs` is used to determine whether the stashed CPU map captured at |
1361 | // the start of tracing should be used. |
1362 | static errno_t |
1363 | _copy_cpu_map(int map_version, void **dst, size_t *size) |
1364 | { |
1365 | _coproc_lock(); |
1366 | struct kd_coproc *coprocs = kd_control_trace.kdc_coprocs; |
1367 | unsigned int cpu_count = kd_control_trace.kdebug_cpus; |
1368 | _coproc_unlock(); |
1369 | |
1370 | assert(cpu_count != 0); |
1371 | assert(coprocs == NULL || coprocs[0].cpu_id + 1 == cpu_count); |
1372 | |
1373 | bool ext = map_version != RAW_VERSION1; |
1374 | size_t stride = ext ? sizeof(kd_cpumap_ext) : sizeof(kd_cpumap); |
1375 | |
1376 | size_t size_needed = sizeof(kd_cpumap_header) + cpu_count * stride; |
1377 | size_t size_avail = *size; |
1378 | *size = size_needed; |
1379 | |
1380 | if (*dst == NULL) { |
1381 | kern_return_t alloc_ret = kmem_alloc(map: kernel_map, addrp: (vm_offset_t *)dst, |
1382 | size: (vm_size_t)size_needed, flags: KMA_DATA | KMA_ZERO, VM_KERN_MEMORY_DIAG); |
1383 | if (alloc_ret != KERN_SUCCESS) { |
1384 | return ENOMEM; |
1385 | } |
1386 | } else if (size_avail < size_needed) { |
1387 | return EINVAL; |
1388 | } |
1389 | |
1390 | kd_cpumap_header * = *dst; |
1391 | header->version_no = map_version; |
1392 | header->cpu_count = cpu_count; |
1393 | |
1394 | void *cpus = &header[1]; |
1395 | size_t name_size = ext ? sizeof(((kd_cpumap_ext *)NULL)->name) : |
1396 | sizeof(((kd_cpumap *)NULL)->name); |
1397 | |
1398 | int i = cpu_count - 1; |
1399 | for (struct kd_coproc *cur_coproc = coprocs; cur_coproc != NULL; |
1400 | cur_coproc = cur_coproc->next, i--) { |
1401 | kd_cpumap_ext *cpu = (kd_cpumap_ext *)((uintptr_t)cpus + stride * i); |
1402 | cpu->cpu_id = cur_coproc->cpu_id; |
1403 | cpu->flags = KDBG_CPUMAP_IS_IOP; |
1404 | strlcpy(dst: (void *)&cpu->name, src: cur_coproc->full_name, n: name_size); |
1405 | } |
1406 | for (; i >= 0; i--) { |
1407 | kd_cpumap *cpu = (kd_cpumap *)((uintptr_t)cpus + stride * i); |
1408 | cpu->cpu_id = i; |
1409 | cpu->flags = 0; |
1410 | _copy_ap_name(cpuid: i, dst: &cpu->name, size: name_size); |
1411 | } |
1412 | |
1413 | return 0; |
1414 | } |
1415 | |
1416 | static void |
1417 | _threadmap_init(void) |
1418 | { |
1419 | ktrace_assert_lock_held(); |
1420 | |
1421 | if (kd_control_trace.kdc_flags & KDBG_MAPINIT) { |
1422 | return; |
1423 | } |
1424 | |
1425 | kd_mapptr = _thread_map_create_live(max_count: 0, map_size: &kd_mapsize, map_count: &kd_mapcount); |
1426 | |
1427 | if (kd_mapptr) { |
1428 | kd_control_trace.kdc_flags |= KDBG_MAPINIT; |
1429 | } |
1430 | } |
1431 | |
1432 | struct kd_resolver { |
1433 | kd_threadmap *krs_map; |
1434 | vm_size_t krs_count; |
1435 | vm_size_t krs_maxcount; |
1436 | }; |
1437 | |
1438 | static int |
1439 | _resolve_iterator(proc_t proc, void *opaque) |
1440 | { |
1441 | if (proc == kernproc) { |
1442 | /* Handled specially as it lacks uthreads. */ |
1443 | return PROC_RETURNED; |
1444 | } |
1445 | struct kd_resolver *resolver = opaque; |
1446 | struct uthread *uth = NULL; |
1447 | const char *proc_name = proc_best_name(p: proc); |
1448 | pid_t pid = proc_getpid(proc); |
1449 | |
1450 | proc_lock(proc); |
1451 | TAILQ_FOREACH(uth, &proc->p_uthlist, uu_list) { |
1452 | if (resolver->krs_count >= resolver->krs_maxcount) { |
1453 | break; |
1454 | } |
1455 | kd_threadmap *map = &resolver->krs_map[resolver->krs_count]; |
1456 | map->thread = (uintptr_t)uthread_tid(uth); |
1457 | (void)strlcpy(dst: map->command, src: proc_name, n: sizeof(map->command)); |
1458 | map->valid = pid; |
1459 | resolver->krs_count++; |
1460 | } |
1461 | proc_unlock(proc); |
1462 | |
1463 | bool done = resolver->krs_count >= resolver->krs_maxcount; |
1464 | return done ? PROC_RETURNED_DONE : PROC_RETURNED; |
1465 | } |
1466 | |
1467 | static void |
1468 | _resolve_kernel_task(thread_t thread, void *opaque) |
1469 | { |
1470 | struct kd_resolver *resolver = opaque; |
1471 | if (resolver->krs_count >= resolver->krs_maxcount) { |
1472 | return; |
1473 | } |
1474 | kd_threadmap *map = &resolver->krs_map[resolver->krs_count]; |
1475 | map->thread = (uintptr_t)thread_tid(thread); |
1476 | (void)strlcpy(dst: map->command, src: "kernel_task" , n: sizeof(map->command)); |
1477 | map->valid = 1; |
1478 | resolver->krs_count++; |
1479 | } |
1480 | |
1481 | static vm_size_t |
1482 | _resolve_threads(kd_threadmap *map, vm_size_t nthreads) |
1483 | { |
1484 | struct kd_resolver resolver = { |
1485 | .krs_map = map, .krs_count = 0, .krs_maxcount = nthreads, |
1486 | }; |
1487 | |
1488 | // Handle kernel_task specially, as it lacks uthreads. |
1489 | extern void task_act_iterate_wth_args(task_t, void (*)(thread_t, void *), |
1490 | void *); |
1491 | task_act_iterate_wth_args(kernel_task, _resolve_kernel_task, &resolver); |
1492 | proc_iterate(PROC_ALLPROCLIST | PROC_NOWAITTRANS, callout: _resolve_iterator, |
1493 | arg: &resolver, NULL, NULL); |
1494 | return resolver.krs_count; |
1495 | } |
1496 | |
1497 | static kd_threadmap * |
1498 | _thread_map_create_live(size_t maxthreads, vm_size_t *mapsize, |
1499 | vm_size_t *mapcount) |
1500 | { |
1501 | kd_threadmap *thread_map = NULL; |
1502 | |
1503 | assert(mapsize != NULL); |
1504 | assert(mapcount != NULL); |
1505 | |
1506 | extern int threads_count; |
1507 | vm_size_t nthreads = threads_count; |
1508 | |
1509 | // Allow 25% more threads to be started while iterating processes. |
1510 | if (os_add_overflow(nthreads, nthreads / 4, &nthreads)) { |
1511 | return NULL; |
1512 | } |
1513 | |
1514 | *mapcount = nthreads; |
1515 | if (os_mul_overflow(nthreads, sizeof(kd_threadmap), mapsize)) { |
1516 | return NULL; |
1517 | } |
1518 | |
1519 | // Wait until the out-parameters have been filled with the needed size to |
1520 | // do the bounds checking on the provided maximum. |
1521 | if (maxthreads != 0 && maxthreads < nthreads) { |
1522 | return NULL; |
1523 | } |
1524 | |
1525 | // This allocation can be too large for `Z_NOFAIL`. |
1526 | thread_map = kalloc_data_tag(*mapsize, Z_WAITOK | Z_ZERO, |
1527 | VM_KERN_MEMORY_DIAG); |
1528 | if (thread_map != NULL) { |
1529 | *mapcount = _resolve_threads(map: thread_map, nthreads); |
1530 | } |
1531 | return thread_map; |
1532 | } |
1533 | |
1534 | static void |
1535 | kdbg_clear(void) |
1536 | { |
1537 | kernel_debug_disable(); |
1538 | kdbg_disable_typefilter(); |
1539 | |
1540 | // Wait for any event writers to see the disable status. |
1541 | IOSleep(100); |
1542 | |
1543 | // Reset kdebug status for each process. |
1544 | if (kd_control_trace.kdc_flags & (KDBG_PIDCHECK | KDBG_PIDEXCLUDE)) { |
1545 | proc_list_lock(); |
1546 | proc_t p; |
1547 | ALLPROC_FOREACH(p) { |
1548 | p->p_kdebug = 0; |
1549 | } |
1550 | proc_list_unlock(); |
1551 | } |
1552 | |
1553 | kd_control_trace.kdc_flags &= (unsigned int)~KDBG_CKTYPES; |
1554 | kd_control_trace.kdc_flags &= ~(KDBG_RANGECHECK | KDBG_VALCHECK); |
1555 | kd_control_trace.kdc_flags &= ~(KDBG_PIDCHECK | KDBG_PIDEXCLUDE); |
1556 | kd_control_trace.kdc_flags &= ~KDBG_CONTINUOUS_TIME; |
1557 | kd_control_trace.kdc_flags &= ~KDBG_DISABLE_COPROCS; |
1558 | kd_control_trace.kdc_flags &= ~KDBG_MATCH_DISABLE; |
1559 | kd_control_trace.kdc_live_flags &= ~(KDBG_NOWRAP | KDBG_WRAPPED); |
1560 | |
1561 | kd_control_trace.kdc_oldest_time = 0; |
1562 | |
1563 | delete_buffers_trace(); |
1564 | kd_buffer_trace.kdb_event_count = 0; |
1565 | |
1566 | _clear_thread_map(); |
1567 | |
1568 | RAW_file_offset = 0; |
1569 | RAW_file_written = 0; |
1570 | } |
1571 | |
1572 | void |
1573 | kdebug_reset(void) |
1574 | { |
1575 | ktrace_assert_lock_held(); |
1576 | |
1577 | kdbg_clear(); |
1578 | typefilter_reject_all(tf: kdbg_typefilter); |
1579 | typefilter_allow_class(tf: kdbg_typefilter, DBG_TRACE); |
1580 | } |
1581 | |
1582 | void |
1583 | kdebug_free_early_buf(void) |
1584 | { |
1585 | #if defined(__x86_64__) |
1586 | ml_static_mfree((vm_offset_t)&kd_early_buffer, sizeof(kd_early_buffer)); |
1587 | #endif /* defined(__x86_64__) */ |
1588 | // ARM handles this as part of the BOOTDATA segment. |
1589 | } |
1590 | |
1591 | int |
1592 | kdbg_setpid(kd_regtype *kdr) |
1593 | { |
1594 | pid_t pid; |
1595 | int flag, ret = 0; |
1596 | struct proc *p; |
1597 | |
1598 | pid = (pid_t)kdr->value1; |
1599 | flag = (int)kdr->value2; |
1600 | |
1601 | if (pid >= 0) { |
1602 | if ((p = proc_find(pid)) == NULL) { |
1603 | ret = ESRCH; |
1604 | } else { |
1605 | if (flag == 1) { |
1606 | /* |
1607 | * turn on pid check for this and all pids |
1608 | */ |
1609 | kd_control_trace.kdc_flags |= KDBG_PIDCHECK; |
1610 | kd_control_trace.kdc_flags &= ~KDBG_PIDEXCLUDE; |
1611 | |
1612 | p->p_kdebug = 1; |
1613 | } else { |
1614 | /* |
1615 | * turn off pid check for this pid value |
1616 | * Don't turn off all pid checking though |
1617 | * |
1618 | * kd_control_trace.kdc_flags &= ~KDBG_PIDCHECK; |
1619 | */ |
1620 | p->p_kdebug = 0; |
1621 | } |
1622 | proc_rele(p); |
1623 | } |
1624 | } else { |
1625 | ret = EINVAL; |
1626 | } |
1627 | |
1628 | return ret; |
1629 | } |
1630 | |
1631 | /* This is for pid exclusion in the trace buffer */ |
1632 | int |
1633 | kdbg_setpidex(kd_regtype *kdr) |
1634 | { |
1635 | pid_t pid; |
1636 | int flag, ret = 0; |
1637 | struct proc *p; |
1638 | |
1639 | pid = (pid_t)kdr->value1; |
1640 | flag = (int)kdr->value2; |
1641 | |
1642 | if (pid >= 0) { |
1643 | if ((p = proc_find(pid)) == NULL) { |
1644 | ret = ESRCH; |
1645 | } else { |
1646 | if (flag == 1) { |
1647 | /* |
1648 | * turn on pid exclusion |
1649 | */ |
1650 | kd_control_trace.kdc_flags |= KDBG_PIDEXCLUDE; |
1651 | kd_control_trace.kdc_flags &= ~KDBG_PIDCHECK; |
1652 | |
1653 | p->p_kdebug = 1; |
1654 | } else { |
1655 | /* |
1656 | * turn off pid exclusion for this pid value |
1657 | * Don't turn off all pid exclusion though |
1658 | * |
1659 | * kd_control_trace.kdc_flags &= ~KDBG_PIDEXCLUDE; |
1660 | */ |
1661 | p->p_kdebug = 0; |
1662 | } |
1663 | proc_rele(p); |
1664 | } |
1665 | } else { |
1666 | ret = EINVAL; |
1667 | } |
1668 | |
1669 | return ret; |
1670 | } |
1671 | |
1672 | /* |
1673 | * The following functions all operate on the typefilter singleton. |
1674 | */ |
1675 | |
1676 | static int |
1677 | kdbg_copyin_typefilter(user_addr_t addr, size_t size) |
1678 | { |
1679 | int ret = ENOMEM; |
1680 | typefilter_t tf; |
1681 | |
1682 | ktrace_assert_lock_held(); |
1683 | |
1684 | if (size != KDBG_TYPEFILTER_BITMAP_SIZE) { |
1685 | return EINVAL; |
1686 | } |
1687 | |
1688 | if ((tf = typefilter_create())) { |
1689 | if ((ret = copyin(addr, tf, KDBG_TYPEFILTER_BITMAP_SIZE)) == 0) { |
1690 | /* The kernel typefilter must always allow DBG_TRACE */ |
1691 | typefilter_allow_class(tf, DBG_TRACE); |
1692 | |
1693 | typefilter_copy(dst: kdbg_typefilter, src: tf); |
1694 | |
1695 | kdbg_enable_typefilter(); |
1696 | _coproc_list_callback(type: KD_CALLBACK_TYPEFILTER_CHANGED, arg: kdbg_typefilter); |
1697 | } |
1698 | |
1699 | if (tf) { |
1700 | typefilter_deallocate(tf); |
1701 | } |
1702 | } |
1703 | |
1704 | return ret; |
1705 | } |
1706 | |
1707 | /* |
1708 | * Enable the flags in the control page for the typefilter. Assumes that |
1709 | * kdbg_typefilter has already been allocated, so events being written |
1710 | * don't see a bad typefilter. |
1711 | */ |
1712 | static void |
1713 | kdbg_enable_typefilter(void) |
1714 | { |
1715 | kd_control_trace.kdc_flags &= ~(KDBG_RANGECHECK | KDBG_VALCHECK); |
1716 | kd_control_trace.kdc_flags |= KDBG_TYPEFILTER_CHECK; |
1717 | if (kdebug_enable) { |
1718 | kd_control_trace.kdc_emit = _trace_emit_filter(); |
1719 | } |
1720 | commpage_update_kdebug_state(); |
1721 | } |
1722 | |
1723 | // Disable the flags in the control page for the typefilter. The typefilter |
1724 | // may be safely deallocated shortly after this function returns. |
1725 | static void |
1726 | kdbg_disable_typefilter(void) |
1727 | { |
1728 | bool notify_coprocs = kd_control_trace.kdc_flags & KDBG_TYPEFILTER_CHECK; |
1729 | kd_control_trace.kdc_flags &= ~KDBG_TYPEFILTER_CHECK; |
1730 | |
1731 | commpage_update_kdebug_state(); |
1732 | |
1733 | if (notify_coprocs) { |
1734 | // Notify coprocessors that the typefilter will now allow everything. |
1735 | // Otherwise, they won't know a typefilter is no longer in effect. |
1736 | typefilter_allow_all(tf: kdbg_typefilter); |
1737 | _coproc_list_callback(type: KD_CALLBACK_TYPEFILTER_CHANGED, arg: kdbg_typefilter); |
1738 | } |
1739 | } |
1740 | |
1741 | uint32_t |
1742 | kdebug_commpage_state(void) |
1743 | { |
1744 | uint32_t state = 0; |
1745 | if (kdebug_enable) { |
1746 | state |= KDEBUG_COMMPAGE_ENABLE_TRACE; |
1747 | if (kd_control_trace.kdc_flags & KDBG_TYPEFILTER_CHECK) { |
1748 | state |= KDEBUG_COMMPAGE_ENABLE_TYPEFILTER; |
1749 | } |
1750 | if (kd_control_trace.kdc_flags & KDBG_CONTINUOUS_TIME) { |
1751 | state |= KDEBUG_COMMPAGE_CONTINUOUS; |
1752 | } |
1753 | } |
1754 | return state; |
1755 | } |
1756 | |
1757 | static int |
1758 | kdbg_setreg(kd_regtype * kdr) |
1759 | { |
1760 | switch (kdr->type) { |
1761 | case KDBG_CLASSTYPE: |
1762 | kdlog_beg = KDBG_EVENTID(kdr->value1 & 0xff, 0, 0); |
1763 | kdlog_end = KDBG_EVENTID(kdr->value2 & 0xff, 0, 0); |
1764 | kd_control_trace.kdc_flags &= ~KDBG_VALCHECK; |
1765 | kd_control_trace.kdc_flags |= KDBG_RANGECHECK; |
1766 | break; |
1767 | case KDBG_SUBCLSTYPE:; |
1768 | unsigned int cls = kdr->value1 & 0xff; |
1769 | unsigned int subcls = kdr->value2 & 0xff; |
1770 | unsigned int subcls_end = subcls + 1; |
1771 | kdlog_beg = KDBG_EVENTID(cls, subcls, 0); |
1772 | kdlog_end = KDBG_EVENTID(cls, subcls_end, 0); |
1773 | kd_control_trace.kdc_flags &= ~KDBG_VALCHECK; |
1774 | kd_control_trace.kdc_flags |= KDBG_RANGECHECK; |
1775 | break; |
1776 | case KDBG_RANGETYPE: |
1777 | kdlog_beg = kdr->value1; |
1778 | kdlog_end = kdr->value2; |
1779 | kd_control_trace.kdc_flags &= ~KDBG_VALCHECK; |
1780 | kd_control_trace.kdc_flags |= KDBG_RANGECHECK; |
1781 | break; |
1782 | case KDBG_VALCHECK: |
1783 | kdlog_value1 = kdr->value1; |
1784 | kdlog_value2 = kdr->value2; |
1785 | kdlog_value3 = kdr->value3; |
1786 | kdlog_value4 = kdr->value4; |
1787 | kd_control_trace.kdc_flags &= ~KDBG_RANGECHECK; |
1788 | kd_control_trace.kdc_flags |= KDBG_VALCHECK; |
1789 | break; |
1790 | case KDBG_TYPENONE: |
1791 | kd_control_trace.kdc_flags &= ~(KDBG_RANGECHECK | KDBG_VALCHECK); |
1792 | kdlog_beg = 0; |
1793 | kdlog_end = 0; |
1794 | break; |
1795 | default: |
1796 | return EINVAL; |
1797 | } |
1798 | if (kdebug_enable) { |
1799 | kd_control_trace.kdc_emit = _trace_emit_filter(); |
1800 | } |
1801 | return 0; |
1802 | } |
1803 | |
1804 | static int |
1805 | _copyin_event_disable_mask(user_addr_t uaddr, size_t usize) |
1806 | { |
1807 | if (usize < 2 * sizeof(kd_event_matcher)) { |
1808 | return ERANGE; |
1809 | } |
1810 | int ret = copyin(uaddr, &kd_control_trace.disable_event_match, |
1811 | sizeof(kd_event_matcher)); |
1812 | if (ret != 0) { |
1813 | return ret; |
1814 | } |
1815 | ret = copyin(uaddr + sizeof(kd_event_matcher), |
1816 | &kd_control_trace.disable_event_mask, sizeof(kd_event_matcher)); |
1817 | if (ret != 0) { |
1818 | memset(s: &kd_control_trace.disable_event_match, c: 0, |
1819 | n: sizeof(kd_event_matcher)); |
1820 | return ret; |
1821 | } |
1822 | return 0; |
1823 | } |
1824 | |
1825 | static int |
1826 | _copyout_event_disable_mask(user_addr_t uaddr, size_t usize) |
1827 | { |
1828 | if (usize < 2 * sizeof(kd_event_matcher)) { |
1829 | return ERANGE; |
1830 | } |
1831 | int ret = copyout(&kd_control_trace.disable_event_match, uaddr, |
1832 | sizeof(kd_event_matcher)); |
1833 | if (ret != 0) { |
1834 | return ret; |
1835 | } |
1836 | ret = copyout(&kd_control_trace.disable_event_mask, |
1837 | uaddr + sizeof(kd_event_matcher), sizeof(kd_event_matcher)); |
1838 | if (ret != 0) { |
1839 | return ret; |
1840 | } |
1841 | return 0; |
1842 | } |
1843 | |
1844 | static int |
1845 | kdbg_write_to_vnode(caddr_t buffer, size_t size, vnode_t vp, vfs_context_t ctx, off_t file_offset) |
1846 | { |
1847 | assert(size < INT_MAX); |
1848 | return vn_rdwr(rw: UIO_WRITE, vp, base: buffer, len: (int)size, offset: file_offset, segflg: UIO_SYSSPACE, |
1849 | IO_NODELOCKED | IO_UNIT, cred: vfs_context_ucred(ctx), aresid: (int *) 0, |
1850 | p: vfs_context_proc(ctx)); |
1851 | } |
1852 | |
1853 | static errno_t |
1854 | _copyout_cpu_map(int map_version, user_addr_t udst, size_t *usize) |
1855 | { |
1856 | if ((kd_control_trace.kdc_flags & KDBG_BUFINIT) == 0) { |
1857 | return EINVAL; |
1858 | } |
1859 | |
1860 | void *cpu_map = NULL; |
1861 | size_t size = 0; |
1862 | int error = _copy_cpu_map(map_version, dst: &cpu_map, size: &size); |
1863 | if (0 == error) { |
1864 | if (udst) { |
1865 | size_t copy_size = MIN(*usize, size); |
1866 | error = copyout(cpu_map, udst, copy_size); |
1867 | } |
1868 | *usize = size; |
1869 | kmem_free(map: kernel_map, addr: (vm_offset_t)cpu_map, size); |
1870 | } |
1871 | if (EINVAL == error && 0 == udst) { |
1872 | *usize = size; |
1873 | // User space only needs the size if it passes NULL; |
1874 | error = 0; |
1875 | } |
1876 | return error; |
1877 | } |
1878 | |
1879 | int |
1880 | kdbg_readcurthrmap(user_addr_t buffer, size_t *bufsize) |
1881 | { |
1882 | kd_threadmap *mapptr; |
1883 | vm_size_t mapsize; |
1884 | vm_size_t mapcount; |
1885 | int ret = 0; |
1886 | size_t count = *bufsize / sizeof(kd_threadmap); |
1887 | |
1888 | *bufsize = 0; |
1889 | |
1890 | if ((mapptr = _thread_map_create_live(maxthreads: count, mapsize: &mapsize, mapcount: &mapcount))) { |
1891 | if (copyout(mapptr, buffer, mapcount * sizeof(kd_threadmap))) { |
1892 | ret = EFAULT; |
1893 | } else { |
1894 | *bufsize = (mapcount * sizeof(kd_threadmap)); |
1895 | } |
1896 | |
1897 | kfree_data(mapptr, mapsize); |
1898 | } else { |
1899 | ret = EINVAL; |
1900 | } |
1901 | |
1902 | return ret; |
1903 | } |
1904 | |
1905 | static int |
1906 | (bool write_thread_map, vnode_t vp, vfs_context_t ctx) |
1907 | { |
1908 | int ret = 0; |
1909 | RAW_header ; |
1910 | clock_sec_t secs; |
1911 | clock_usec_t usecs; |
1912 | void *pad_buf; |
1913 | uint32_t pad_size; |
1914 | uint32_t = 0; |
1915 | uint32_t cpumap_size; |
1916 | size_t map_size = 0; |
1917 | uint32_t map_count = 0; |
1918 | |
1919 | if (write_thread_map) { |
1920 | assert(kd_control_trace.kdc_flags & KDBG_MAPINIT); |
1921 | if (kd_mapcount > UINT32_MAX) { |
1922 | return ERANGE; |
1923 | } |
1924 | map_count = (uint32_t)kd_mapcount; |
1925 | if (os_mul_overflow(map_count, sizeof(kd_threadmap), &map_size)) { |
1926 | return ERANGE; |
1927 | } |
1928 | if (map_size >= INT_MAX) { |
1929 | return ERANGE; |
1930 | } |
1931 | } |
1932 | |
1933 | /* |
1934 | * Without the buffers initialized, we cannot construct a CPU map or a |
1935 | * thread map, and cannot write a header. |
1936 | */ |
1937 | if (!(kd_control_trace.kdc_flags & KDBG_BUFINIT)) { |
1938 | return EINVAL; |
1939 | } |
1940 | |
1941 | /* |
1942 | * To write a RAW_VERSION1+ file, we must embed a cpumap in the |
1943 | * "padding" used to page align the events following the threadmap. If |
1944 | * the threadmap happens to not require enough padding, we artificially |
1945 | * increase its footprint until it needs enough padding. |
1946 | */ |
1947 | |
1948 | assert(vp); |
1949 | assert(ctx); |
1950 | |
1951 | pad_size = 16384 - ((sizeof(RAW_header) + map_size) & PAGE_MASK); |
1952 | cpumap_size = sizeof(kd_cpumap_header) + kd_control_trace.kdebug_cpus * sizeof(kd_cpumap); |
1953 | |
1954 | if (cpumap_size > pad_size) { |
1955 | /* If the cpu map doesn't fit in the current available pad_size, |
1956 | * we increase the pad_size by 16K. We do this so that the event |
1957 | * data is always available on a page aligned boundary for both |
1958 | * 4k and 16k systems. We enforce this alignment for the event |
1959 | * data so that we can take advantage of optimized file/disk writes. |
1960 | */ |
1961 | pad_size += 16384; |
1962 | } |
1963 | |
1964 | /* The way we are silently embedding a cpumap in the "padding" is by artificially |
1965 | * increasing the number of thread entries. However, we'll also need to ensure that |
1966 | * the cpumap is embedded in the last 4K page before when the event data is expected. |
1967 | * This way the tools can read the data starting the next page boundary on both |
1968 | * 4K and 16K systems preserving compatibility with older versions of the tools |
1969 | */ |
1970 | if (pad_size > 4096) { |
1971 | pad_size -= 4096; |
1972 | extra_thread_count = (pad_size / sizeof(kd_threadmap)) + 1; |
1973 | } |
1974 | |
1975 | memset(s: &header, c: 0, n: sizeof(header)); |
1976 | header.version_no = RAW_VERSION1; |
1977 | header.thread_count = map_count + extra_thread_count; |
1978 | |
1979 | clock_get_calendar_microtime(secs: &secs, microsecs: &usecs); |
1980 | header.TOD_secs = secs; |
1981 | header.TOD_usecs = usecs; |
1982 | |
1983 | ret = vn_rdwr(rw: UIO_WRITE, vp, base: (caddr_t)&header, len: (int)sizeof(RAW_header), offset: RAW_file_offset, |
1984 | segflg: UIO_SYSSPACE, IO_NODELOCKED | IO_UNIT, cred: vfs_context_ucred(ctx), aresid: (int *) 0, p: vfs_context_proc(ctx)); |
1985 | if (ret) { |
1986 | goto write_error; |
1987 | } |
1988 | RAW_file_offset += sizeof(RAW_header); |
1989 | RAW_file_written += sizeof(RAW_header); |
1990 | |
1991 | if (write_thread_map) { |
1992 | assert(map_size < INT_MAX); |
1993 | ret = vn_rdwr(rw: UIO_WRITE, vp, base: (caddr_t)kd_mapptr, len: (int)map_size, offset: RAW_file_offset, |
1994 | segflg: UIO_SYSSPACE, IO_NODELOCKED | IO_UNIT, cred: vfs_context_ucred(ctx), aresid: (int *) 0, p: vfs_context_proc(ctx)); |
1995 | if (ret) { |
1996 | goto write_error; |
1997 | } |
1998 | |
1999 | RAW_file_offset += map_size; |
2000 | RAW_file_written += map_size; |
2001 | } |
2002 | |
2003 | if (extra_thread_count) { |
2004 | pad_size = extra_thread_count * sizeof(kd_threadmap); |
2005 | pad_buf = (char *)kalloc_data(pad_size, Z_WAITOK | Z_ZERO); |
2006 | if (!pad_buf) { |
2007 | ret = ENOMEM; |
2008 | goto write_error; |
2009 | } |
2010 | |
2011 | assert(pad_size < INT_MAX); |
2012 | ret = vn_rdwr(rw: UIO_WRITE, vp, base: (caddr_t)pad_buf, len: (int)pad_size, offset: RAW_file_offset, |
2013 | segflg: UIO_SYSSPACE, IO_NODELOCKED | IO_UNIT, cred: vfs_context_ucred(ctx), aresid: (int *) 0, p: vfs_context_proc(ctx)); |
2014 | kfree_data(pad_buf, pad_size); |
2015 | if (ret) { |
2016 | goto write_error; |
2017 | } |
2018 | |
2019 | RAW_file_offset += pad_size; |
2020 | RAW_file_written += pad_size; |
2021 | } |
2022 | |
2023 | pad_size = PAGE_SIZE - (RAW_file_offset & PAGE_MASK); |
2024 | if (pad_size) { |
2025 | pad_buf = (char *)kalloc_data(pad_size, Z_WAITOK | Z_ZERO); |
2026 | if (!pad_buf) { |
2027 | ret = ENOMEM; |
2028 | goto write_error; |
2029 | } |
2030 | |
2031 | /* |
2032 | * Embed the CPU map in the padding bytes -- old code will skip it, |
2033 | * while newer code knows it's there. |
2034 | */ |
2035 | size_t temp = pad_size; |
2036 | errno_t error = _copy_cpu_map(RAW_VERSION1, dst: &pad_buf, size: &temp); |
2037 | if (0 != error) { |
2038 | memset(s: pad_buf, c: 0, n: pad_size); |
2039 | } |
2040 | |
2041 | assert(pad_size < INT_MAX); |
2042 | ret = vn_rdwr(rw: UIO_WRITE, vp, base: (caddr_t)pad_buf, len: (int)pad_size, offset: RAW_file_offset, |
2043 | segflg: UIO_SYSSPACE, IO_NODELOCKED | IO_UNIT, cred: vfs_context_ucred(ctx), aresid: (int *) 0, p: vfs_context_proc(ctx)); |
2044 | kfree_data(pad_buf, pad_size); |
2045 | if (ret) { |
2046 | goto write_error; |
2047 | } |
2048 | |
2049 | RAW_file_offset += pad_size; |
2050 | RAW_file_written += pad_size; |
2051 | } |
2052 | |
2053 | write_error: |
2054 | return ret; |
2055 | } |
2056 | |
2057 | static void |
2058 | _clear_thread_map(void) |
2059 | { |
2060 | ktrace_assert_lock_held(); |
2061 | |
2062 | if (kd_control_trace.kdc_flags & KDBG_MAPINIT) { |
2063 | assert(kd_mapptr != NULL); |
2064 | kfree_data(kd_mapptr, kd_mapsize); |
2065 | kd_mapptr = NULL; |
2066 | kd_mapsize = 0; |
2067 | kd_mapcount = 0; |
2068 | kd_control_trace.kdc_flags &= ~KDBG_MAPINIT; |
2069 | } |
2070 | } |
2071 | |
2072 | /* |
2073 | * Write out a version 1 header and the thread map, if it is initialized, to a |
2074 | * vnode. Used by KDWRITEMAP and kdbg_dump_trace_to_file. |
2075 | * |
2076 | * Returns write errors from vn_rdwr if a write fails. Returns ENODATA if the |
2077 | * thread map has not been initialized, but the header will still be written. |
2078 | * Returns ENOMEM if padding could not be allocated. Returns 0 otherwise. |
2079 | */ |
2080 | static int |
2081 | kdbg_write_thread_map(vnode_t vp, vfs_context_t ctx) |
2082 | { |
2083 | int ret = 0; |
2084 | bool map_initialized; |
2085 | |
2086 | ktrace_assert_lock_held(); |
2087 | assert(ctx != NULL); |
2088 | |
2089 | map_initialized = (kd_control_trace.kdc_flags & KDBG_MAPINIT); |
2090 | |
2091 | ret = _write_legacy_header(write_thread_map: map_initialized, vp, ctx); |
2092 | if (ret == 0) { |
2093 | if (map_initialized) { |
2094 | _clear_thread_map(); |
2095 | } else { |
2096 | ret = ENODATA; |
2097 | } |
2098 | } |
2099 | |
2100 | return ret; |
2101 | } |
2102 | |
2103 | /* |
2104 | * Copy out the thread map to a user space buffer. Used by KDTHRMAP. |
2105 | * |
2106 | * Returns copyout errors if the copyout fails. Returns ENODATA if the thread |
2107 | * map has not been initialized. Returns EINVAL if the buffer provided is not |
2108 | * large enough for the entire thread map. Returns 0 otherwise. |
2109 | */ |
2110 | static int |
2111 | kdbg_copyout_thread_map(user_addr_t buffer, size_t *buffer_size) |
2112 | { |
2113 | bool map_initialized; |
2114 | size_t map_size; |
2115 | int ret = 0; |
2116 | |
2117 | ktrace_assert_lock_held(); |
2118 | assert(buffer_size != NULL); |
2119 | |
2120 | map_initialized = (kd_control_trace.kdc_flags & KDBG_MAPINIT); |
2121 | if (!map_initialized) { |
2122 | return ENODATA; |
2123 | } |
2124 | |
2125 | map_size = kd_mapcount * sizeof(kd_threadmap); |
2126 | if (*buffer_size < map_size) { |
2127 | return EINVAL; |
2128 | } |
2129 | |
2130 | ret = copyout(kd_mapptr, buffer, map_size); |
2131 | if (ret == 0) { |
2132 | _clear_thread_map(); |
2133 | } |
2134 | |
2135 | return ret; |
2136 | } |
2137 | |
2138 | static void |
2139 | kdbg_set_nkdbufs_trace(unsigned int req_nkdbufs_trace) |
2140 | { |
2141 | /* |
2142 | * Only allow allocations of up to half the kernel's data range or "sane |
2143 | * size", whichever is smaller. |
2144 | */ |
2145 | const uint64_t max_nkdbufs_trace_64 = |
2146 | MIN(kmem_range_id_size(KMEM_RANGE_ID_DATA), sane_size) / 2 / |
2147 | sizeof(kd_buf); |
2148 | /* |
2149 | * Can't allocate more than 2^38 (2^32 * 64) bytes of events without |
2150 | * switching to a 64-bit event count; should be fine. |
2151 | */ |
2152 | const unsigned int max_nkdbufs_trace = |
2153 | (unsigned int)MIN(max_nkdbufs_trace_64, UINT_MAX); |
2154 | |
2155 | kd_buffer_trace.kdb_event_count = MIN(req_nkdbufs_trace, max_nkdbufs_trace); |
2156 | } |
2157 | |
2158 | /* |
2159 | * Block until there are `kd_buffer_trace.kdb_storage_threshold` storage units filled with |
2160 | * events or `timeout_ms` milliseconds have passed. If `locked_wait` is true, |
2161 | * `ktrace_lock` is held while waiting. This is necessary while waiting to |
2162 | * write events out of the buffers. |
2163 | * |
2164 | * Returns true if the threshold was reached and false otherwise. |
2165 | * |
2166 | * Called with `ktrace_lock` locked and interrupts enabled. |
2167 | */ |
2168 | static bool |
2169 | kdbg_wait(uint64_t timeout_ms) |
2170 | { |
2171 | int wait_result = THREAD_AWAKENED; |
2172 | uint64_t deadline_mach = 0; |
2173 | |
2174 | ktrace_assert_lock_held(); |
2175 | |
2176 | if (timeout_ms != 0) { |
2177 | uint64_t ns = timeout_ms * NSEC_PER_MSEC; |
2178 | nanoseconds_to_absolutetime(nanoseconds: ns, result: &deadline_mach); |
2179 | clock_absolutetime_interval_to_deadline(abstime: deadline_mach, result: &deadline_mach); |
2180 | } |
2181 | |
2182 | bool s = ml_set_interrupts_enabled(false); |
2183 | if (!s) { |
2184 | panic("kdbg_wait() called with interrupts disabled" ); |
2185 | } |
2186 | lck_spin_lock_grp(lck: &kd_wait_lock, grp: &kdebug_lck_grp); |
2187 | |
2188 | /* drop the mutex to allow others to access trace */ |
2189 | ktrace_unlock(); |
2190 | |
2191 | while (wait_result == THREAD_AWAKENED && |
2192 | kd_control_trace.kdc_storage_used < kd_buffer_trace.kdb_storage_threshold) { |
2193 | kd_waiter = true; |
2194 | |
2195 | if (deadline_mach) { |
2196 | wait_result = lck_spin_sleep_deadline(lck: &kd_wait_lock, lck_sleep_action: 0, event: &kd_waiter, |
2197 | THREAD_ABORTSAFE, deadline: deadline_mach); |
2198 | } else { |
2199 | wait_result = lck_spin_sleep(lck: &kd_wait_lock, lck_sleep_action: 0, event: &kd_waiter, |
2200 | THREAD_ABORTSAFE); |
2201 | } |
2202 | } |
2203 | |
2204 | bool threshold_exceeded = (kd_control_trace.kdc_storage_used >= kd_buffer_trace.kdb_storage_threshold); |
2205 | |
2206 | lck_spin_unlock(lck: &kd_wait_lock); |
2207 | ml_set_interrupts_enabled(enable: s); |
2208 | |
2209 | ktrace_lock(); |
2210 | |
2211 | return threshold_exceeded; |
2212 | } |
2213 | |
2214 | /* |
2215 | * Wakeup a thread waiting using `kdbg_wait` if there are at least |
2216 | * `kd_buffer_trace.kdb_storage_threshold` storage units in use. |
2217 | */ |
2218 | static void |
2219 | kdbg_wakeup(void) |
2220 | { |
2221 | bool need_kds_wakeup = false; |
2222 | |
2223 | /* |
2224 | * Try to take the lock here to synchronize with the waiter entering |
2225 | * the blocked state. Use the try mode to prevent deadlocks caused by |
2226 | * re-entering this routine due to various trace points triggered in the |
2227 | * lck_spin_sleep_xxxx routines used to actually enter one of our 2 wait |
2228 | * conditions. No problem if we fail, there will be lots of additional |
2229 | * events coming in that will eventually succeed in grabbing this lock. |
2230 | */ |
2231 | bool s = ml_set_interrupts_enabled(false); |
2232 | |
2233 | if (lck_spin_try_lock(lck: &kd_wait_lock)) { |
2234 | if (kd_waiter && |
2235 | (kd_control_trace.kdc_storage_used >= kd_buffer_trace.kdb_storage_threshold)) { |
2236 | kd_waiter = 0; |
2237 | need_kds_wakeup = true; |
2238 | } |
2239 | lck_spin_unlock(lck: &kd_wait_lock); |
2240 | } |
2241 | |
2242 | ml_set_interrupts_enabled(enable: s); |
2243 | |
2244 | if (need_kds_wakeup == true) { |
2245 | wakeup(chan: &kd_waiter); |
2246 | } |
2247 | } |
2248 | |
2249 | static int |
2250 | _read_merged_trace_events(user_addr_t buffer, size_t *number, vnode_t vp, |
2251 | vfs_context_t ctx, bool chunk) |
2252 | { |
2253 | ktrace_assert_lock_held(); |
2254 | size_t count = *number / sizeof(kd_buf); |
2255 | if (count == 0 || !(kd_control_trace.kdc_flags & KDBG_BUFINIT) || |
2256 | kd_buffer_trace.kdcopybuf == 0) { |
2257 | *number = 0; |
2258 | return EINVAL; |
2259 | } |
2260 | |
2261 | // Before merging, make sure coprocessors have provided up-to-date events. |
2262 | _coproc_list_callback(type: KD_CALLBACK_SYNC_FLUSH, NULL); |
2263 | return kernel_debug_read(ctl: &kd_control_trace, buf: &kd_buffer_trace, buffer, |
2264 | number, vp, ctx, file_version: chunk); |
2265 | } |
2266 | |
2267 | struct { |
2268 | uint32_t ; |
2269 | uint32_t ; |
2270 | uint64_t ; |
2271 | uint64_t ; |
2272 | }; |
2273 | |
2274 | static int |
2275 | (user_addr_t udst, vnode_t vp, vfs_context_t ctx, |
2276 | uint64_t length) |
2277 | { |
2278 | struct event_chunk_header = { |
2279 | .tag = V3_RAW_EVENTS, |
2280 | .sub_tag = 1, |
2281 | .length = length, |
2282 | }; |
2283 | |
2284 | if (vp) { |
2285 | assert(udst == USER_ADDR_NULL); |
2286 | assert(ctx != NULL); |
2287 | int error = kdbg_write_to_vnode(buffer: (caddr_t)&header, size: sizeof(header), vp, |
2288 | ctx, file_offset: RAW_file_offset); |
2289 | if (0 == error) { |
2290 | RAW_file_offset += sizeof(header); |
2291 | } |
2292 | return error; |
2293 | } else { |
2294 | assert(udst != USER_ADDR_NULL); |
2295 | return copyout(&header, udst, sizeof(header)); |
2296 | } |
2297 | } |
2298 | |
2299 | int |
2300 | kernel_debug_trace_write_to_file(user_addr_t *buffer, size_t *number, |
2301 | size_t *count, size_t tempbuf_number, vnode_t vp, vfs_context_t ctx, |
2302 | bool chunk) |
2303 | { |
2304 | int error = 0; |
2305 | |
2306 | if (chunk) { |
2307 | error = _write_event_chunk_header(udst: *buffer, vp, ctx, |
2308 | length: tempbuf_number * sizeof(kd_buf)); |
2309 | if (error) { |
2310 | return error; |
2311 | } |
2312 | if (buffer) { |
2313 | *buffer += sizeof(struct event_chunk_header); |
2314 | } |
2315 | |
2316 | assert(*count >= sizeof(struct event_chunk_header)); |
2317 | *count -= sizeof(struct event_chunk_header); |
2318 | *number += sizeof(struct event_chunk_header); |
2319 | } |
2320 | if (vp) { |
2321 | size_t write_size = tempbuf_number * sizeof(kd_buf); |
2322 | error = kdbg_write_to_vnode(buffer: (caddr_t)kd_buffer_trace.kdcopybuf, |
2323 | size: write_size, vp, ctx, file_offset: RAW_file_offset); |
2324 | if (!error) { |
2325 | RAW_file_offset += write_size; |
2326 | } |
2327 | |
2328 | if (RAW_file_written >= RAW_FLUSH_SIZE) { |
2329 | error = VNOP_FSYNC(vp, MNT_NOWAIT, ctx); |
2330 | |
2331 | RAW_file_written = 0; |
2332 | } |
2333 | } else { |
2334 | error = copyout(kd_buffer_trace.kdcopybuf, *buffer, tempbuf_number * sizeof(kd_buf)); |
2335 | *buffer += (tempbuf_number * sizeof(kd_buf)); |
2336 | } |
2337 | |
2338 | return error; |
2339 | } |
2340 | |
2341 | #pragma mark - User space interface |
2342 | |
2343 | static int |
2344 | _kd_sysctl_internal(int op, int value, user_addr_t where, size_t *sizep) |
2345 | { |
2346 | size_t size = *sizep; |
2347 | kd_regtype kd_Reg; |
2348 | proc_t p; |
2349 | |
2350 | bool read_only = (op == KERN_KDGETBUF || op == KERN_KDREADCURTHRMAP); |
2351 | int perm_error = read_only ? ktrace_read_check() : |
2352 | ktrace_configure(KTRACE_KDEBUG); |
2353 | if (perm_error != 0) { |
2354 | return perm_error; |
2355 | } |
2356 | |
2357 | switch (op) { |
2358 | case KERN_KDGETBUF:; |
2359 | pid_t owning_pid = ktrace_get_owning_pid(); |
2360 | kbufinfo_t info = { |
2361 | .nkdbufs = kd_buffer_trace.kdb_event_count, |
2362 | .nkdthreads = (int)MIN(kd_mapcount, INT_MAX), |
2363 | .nolog = kd_control_trace.kdc_emit == KDEMIT_DISABLE, |
2364 | .flags = kd_control_trace.kdc_flags | kd_control_trace.kdc_live_flags, |
2365 | .bufid = owning_pid ?: -1, |
2366 | }; |
2367 | #if defined(__LP64__) |
2368 | info.flags |= KDBG_LP64; |
2369 | #endif // defined(__LP64__) |
2370 | |
2371 | size = MIN(size, sizeof(info)); |
2372 | return copyout(&info, where, size); |
2373 | case KERN_KDREADCURTHRMAP: |
2374 | return kdbg_readcurthrmap(buffer: where, bufsize: sizep); |
2375 | case KERN_KDEFLAGS: |
2376 | value &= KDBG_USERFLAGS; |
2377 | kd_control_trace.kdc_flags |= value; |
2378 | return 0; |
2379 | case KERN_KDDFLAGS: |
2380 | value &= KDBG_USERFLAGS; |
2381 | kd_control_trace.kdc_flags &= ~value; |
2382 | return 0; |
2383 | case KERN_KDENABLE: |
2384 | if (value) { |
2385 | if (!(kd_control_trace.kdc_flags & KDBG_BUFINIT) || |
2386 | !(value == KDEBUG_ENABLE_TRACE || value == KDEBUG_ENABLE_PPT)) { |
2387 | return EINVAL; |
2388 | } |
2389 | _threadmap_init(); |
2390 | |
2391 | kdbg_set_tracing_enabled(true, trace_type: value); |
2392 | } else { |
2393 | if (!kdebug_enable) { |
2394 | return 0; |
2395 | } |
2396 | |
2397 | kernel_debug_disable(); |
2398 | } |
2399 | return 0; |
2400 | case KERN_KDSETBUF: |
2401 | kdbg_set_nkdbufs_trace(req_nkdbufs_trace: value); |
2402 | return 0; |
2403 | case KERN_KDSETUP: |
2404 | return kdbg_reinit(EXTRA_COPROC_COUNT); |
2405 | case KERN_KDREMOVE: |
2406 | ktrace_reset(KTRACE_KDEBUG); |
2407 | return 0; |
2408 | case KERN_KDSETREG: |
2409 | if (size < sizeof(kd_regtype)) { |
2410 | return EINVAL; |
2411 | } |
2412 | if (copyin(where, &kd_Reg, sizeof(kd_regtype))) { |
2413 | return EINVAL; |
2414 | } |
2415 | return kdbg_setreg(kdr: &kd_Reg); |
2416 | case KERN_KDGETREG: |
2417 | return EINVAL; |
2418 | case KERN_KDREADTR: |
2419 | return _read_merged_trace_events(buffer: where, number: sizep, NULL, NULL, false); |
2420 | case KERN_KDWRITETR: |
2421 | case KERN_KDWRITETR_V3: |
2422 | case KERN_KDWRITEMAP: { |
2423 | struct vfs_context context; |
2424 | struct fileproc *fp; |
2425 | size_t number; |
2426 | vnode_t vp; |
2427 | int fd; |
2428 | int ret = 0; |
2429 | |
2430 | if (op == KERN_KDWRITETR || op == KERN_KDWRITETR_V3) { |
2431 | (void)kdbg_wait(timeout_ms: size); |
2432 | // Re-check whether this process can configure ktrace, since waiting |
2433 | // will drop the ktrace lock. |
2434 | int no_longer_owner_error = ktrace_configure(KTRACE_KDEBUG); |
2435 | if (no_longer_owner_error != 0) { |
2436 | return no_longer_owner_error; |
2437 | } |
2438 | } |
2439 | |
2440 | p = current_proc(); |
2441 | fd = value; |
2442 | |
2443 | if (fp_get_ftype(p, fd, ftype: DTYPE_VNODE, EBADF, fpp: &fp)) { |
2444 | return EBADF; |
2445 | } |
2446 | |
2447 | vp = fp_get_data(fp); |
2448 | context.vc_thread = current_thread(); |
2449 | context.vc_ucred = fp->fp_glob->fg_cred; |
2450 | |
2451 | if ((ret = vnode_getwithref(vp)) == 0) { |
2452 | RAW_file_offset = fp->fp_glob->fg_offset; |
2453 | if (op == KERN_KDWRITETR || op == KERN_KDWRITETR_V3) { |
2454 | number = kd_buffer_trace.kdb_event_count * sizeof(kd_buf); |
2455 | |
2456 | KDBG_RELEASE(TRACE_WRITING_EVENTS | DBG_FUNC_START); |
2457 | ret = _read_merged_trace_events(buffer: 0, number: &number, vp, ctx: &context, |
2458 | chunk: op == KERN_KDWRITETR_V3); |
2459 | KDBG_RELEASE(TRACE_WRITING_EVENTS | DBG_FUNC_END, number); |
2460 | |
2461 | *sizep = number; |
2462 | } else { |
2463 | number = kd_mapcount * sizeof(kd_threadmap); |
2464 | ret = kdbg_write_thread_map(vp, ctx: &context); |
2465 | } |
2466 | fp->fp_glob->fg_offset = RAW_file_offset; |
2467 | vnode_put(vp); |
2468 | } |
2469 | fp_drop(p, fd, fp, locked: 0); |
2470 | |
2471 | return ret; |
2472 | } |
2473 | case KERN_KDBUFWAIT: |
2474 | *sizep = kdbg_wait(timeout_ms: size); |
2475 | return 0; |
2476 | case KERN_KDPIDTR: |
2477 | if (size < sizeof(kd_regtype)) { |
2478 | return EINVAL; |
2479 | } |
2480 | if (copyin(where, &kd_Reg, sizeof(kd_regtype))) { |
2481 | return EINVAL; |
2482 | } |
2483 | return kdbg_setpid(kdr: &kd_Reg); |
2484 | case KERN_KDPIDEX: |
2485 | if (size < sizeof(kd_regtype)) { |
2486 | return EINVAL; |
2487 | } |
2488 | if (copyin(where, &kd_Reg, sizeof(kd_regtype))) { |
2489 | return EINVAL; |
2490 | } |
2491 | return kdbg_setpidex(kdr: &kd_Reg); |
2492 | case KERN_KDCPUMAP: |
2493 | return _copyout_cpu_map(RAW_VERSION1, udst: where, usize: sizep); |
2494 | case KERN_KDCPUMAP_EXT: |
2495 | return _copyout_cpu_map(map_version: 1, udst: where, usize: sizep); |
2496 | case KERN_KDTHRMAP: |
2497 | return kdbg_copyout_thread_map(buffer: where, buffer_size: sizep); |
2498 | case KERN_KDSET_TYPEFILTER: |
2499 | return kdbg_copyin_typefilter(addr: where, size); |
2500 | case KERN_KDSET_EDM: |
2501 | return _copyin_event_disable_mask(uaddr: where, usize: size); |
2502 | case KERN_KDGET_EDM: |
2503 | return _copyout_event_disable_mask(uaddr: where, usize: size); |
2504 | #if DEVELOPMENT || DEBUG |
2505 | case KERN_KDTEST: |
2506 | return kdbg_test(size); |
2507 | #endif // DEVELOPMENT || DEBUG |
2508 | |
2509 | default: |
2510 | return ENOTSUP; |
2511 | } |
2512 | } |
2513 | |
2514 | static int |
2515 | kdebug_sysctl SYSCTL_HANDLER_ARGS |
2516 | { |
2517 | int *names = arg1; |
2518 | int name_count = arg2; |
2519 | user_addr_t udst = req->oldptr; |
2520 | size_t *usize = &req->oldlen; |
2521 | int value = 0; |
2522 | |
2523 | if (name_count == 0) { |
2524 | return ENOTSUP; |
2525 | } |
2526 | |
2527 | int op = names[0]; |
2528 | |
2529 | // Some operations have an argument stuffed into the next OID argument. |
2530 | switch (op) { |
2531 | case KERN_KDWRITETR: |
2532 | case KERN_KDWRITETR_V3: |
2533 | case KERN_KDWRITEMAP: |
2534 | case KERN_KDEFLAGS: |
2535 | case KERN_KDDFLAGS: |
2536 | case KERN_KDENABLE: |
2537 | case KERN_KDSETBUF: |
2538 | if (name_count < 2) { |
2539 | return EINVAL; |
2540 | } |
2541 | value = names[1]; |
2542 | break; |
2543 | default: |
2544 | break; |
2545 | } |
2546 | |
2547 | ktrace_lock(); |
2548 | int ret = _kd_sysctl_internal(op, value, where: udst, sizep: usize); |
2549 | ktrace_unlock(); |
2550 | if (0 == ret) { |
2551 | req->oldidx += req->oldlen; |
2552 | } |
2553 | return ret; |
2554 | } |
2555 | SYSCTL_PROC(_kern, KERN_KDEBUG, kdebug, |
2556 | CTLTYPE_NODE | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, kdebug_sysctl, NULL, "" ); |
2557 | |
2558 | #pragma mark - Tests |
2559 | |
2560 | #if DEVELOPMENT || DEBUG |
2561 | |
2562 | static int test_coproc = 0; |
2563 | static int sync_flush_coproc = 0; |
2564 | |
2565 | #define KDEBUG_TEST_CODE(code) BSDDBG_CODE(DBG_BSD_KDEBUG_TEST, (code)) |
2566 | |
2567 | /* |
2568 | * A test IOP for the SYNC_FLUSH callback. |
2569 | */ |
2570 | |
2571 | static void |
2572 | sync_flush_callback(void * __unused context, kd_callback_type reason, |
2573 | void * __unused arg) |
2574 | { |
2575 | assert(sync_flush_coproc > 0); |
2576 | |
2577 | if (reason == KD_CALLBACK_SYNC_FLUSH) { |
2578 | kernel_debug_enter(sync_flush_coproc, KDEBUG_TEST_CODE(0xff), |
2579 | kdebug_timestamp(), 0, 0, 0, 0, 0); |
2580 | } |
2581 | } |
2582 | |
2583 | static struct kd_callback sync_flush_kdcb = { |
2584 | .func = sync_flush_callback, |
2585 | .iop_name = "test_sf" , |
2586 | }; |
2587 | |
2588 | #define TEST_COPROC_CTX 0xabadcafe |
2589 | |
2590 | static void |
2591 | test_coproc_cb(void *context, kd_callback_type __unused reason, |
2592 | void * __unused arg) |
2593 | { |
2594 | assert((uintptr_t)context == TEST_COPROC_CTX); |
2595 | } |
2596 | |
2597 | static int |
2598 | kdbg_test(size_t flavor) |
2599 | { |
2600 | int code = 0; |
2601 | int dummy_iop = 0; |
2602 | |
2603 | switch (flavor) { |
2604 | case KDTEST_KERNEL_MACROS: |
2605 | /* try each macro */ |
2606 | KDBG(KDEBUG_TEST_CODE(code)); code++; |
2607 | KDBG(KDEBUG_TEST_CODE(code), 1); code++; |
2608 | KDBG(KDEBUG_TEST_CODE(code), 1, 2); code++; |
2609 | KDBG(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; |
2610 | KDBG(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; |
2611 | |
2612 | KDBG_RELEASE(KDEBUG_TEST_CODE(code)); code++; |
2613 | KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1); code++; |
2614 | KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2); code++; |
2615 | KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; |
2616 | KDBG_RELEASE(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; |
2617 | |
2618 | KDBG_FILTERED(KDEBUG_TEST_CODE(code)); code++; |
2619 | KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1); code++; |
2620 | KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2); code++; |
2621 | KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; |
2622 | KDBG_FILTERED(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; |
2623 | |
2624 | KDBG_RELEASE_NOPROCFILT(KDEBUG_TEST_CODE(code)); code++; |
2625 | KDBG_RELEASE_NOPROCFILT(KDEBUG_TEST_CODE(code), 1); code++; |
2626 | KDBG_RELEASE_NOPROCFILT(KDEBUG_TEST_CODE(code), 1, 2); code++; |
2627 | KDBG_RELEASE_NOPROCFILT(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; |
2628 | KDBG_RELEASE_NOPROCFILT(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; |
2629 | |
2630 | KDBG_DEBUG(KDEBUG_TEST_CODE(code)); code++; |
2631 | KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1); code++; |
2632 | KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2); code++; |
2633 | KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2, 3); code++; |
2634 | KDBG_DEBUG(KDEBUG_TEST_CODE(code), 1, 2, 3, 4); code++; |
2635 | break; |
2636 | |
2637 | case KDTEST_OLD_TIMESTAMP: |
2638 | if (kd_control_trace.kdc_coprocs) { |
2639 | /* avoid the assertion in kernel_debug_enter for a valid IOP */ |
2640 | dummy_iop = kd_control_trace.kdc_coprocs[0].cpu_id; |
2641 | } |
2642 | |
2643 | /* ensure old timestamps are not emitted from kernel_debug_enter */ |
2644 | kernel_debug_enter(dummy_iop, KDEBUG_TEST_CODE(code), |
2645 | 100 /* very old timestamp */, 0, 0, 0, 0, 0); |
2646 | code++; |
2647 | kernel_debug_enter(dummy_iop, KDEBUG_TEST_CODE(code), |
2648 | kdebug_timestamp(), 0, 0, 0, 0, 0); |
2649 | code++; |
2650 | break; |
2651 | |
2652 | case KDTEST_FUTURE_TIMESTAMP: |
2653 | if (kd_control_trace.kdc_coprocs) { |
2654 | dummy_iop = kd_control_trace.kdc_coprocs[0].cpu_id; |
2655 | } |
2656 | kernel_debug_enter(dummy_iop, KDEBUG_TEST_CODE(code), |
2657 | kdebug_timestamp() * 2 /* !!! */, 0, 0, 0, 0, 0); |
2658 | break; |
2659 | |
2660 | case KDTEST_SETUP_IOP: |
2661 | if (!sync_flush_coproc) { |
2662 | ktrace_unlock(); |
2663 | int new_sync_flush_coproc = kernel_debug_register_callback( |
2664 | sync_flush_kdcb); |
2665 | assert(new_sync_flush_coproc > 0); |
2666 | ktrace_lock(); |
2667 | if (!sync_flush_coproc) { |
2668 | sync_flush_coproc = new_sync_flush_coproc; |
2669 | } |
2670 | } |
2671 | break; |
2672 | |
2673 | case KDTEST_SETUP_COPROCESSOR: |
2674 | if (!test_coproc) { |
2675 | ktrace_unlock(); |
2676 | int new_test_coproc = kdebug_register_coproc("test_coproc" , |
2677 | KDCP_CONTINUOUS_TIME, test_coproc_cb, (void *)TEST_COPROC_CTX); |
2678 | assert(new_test_coproc > 0); |
2679 | ktrace_lock(); |
2680 | if (!test_coproc) { |
2681 | test_coproc = new_test_coproc; |
2682 | } |
2683 | } |
2684 | break; |
2685 | |
2686 | case KDTEST_ABSOLUTE_TIMESTAMP:; |
2687 | uint64_t atime = mach_absolute_time(); |
2688 | kernel_debug_enter(sync_flush_coproc, KDEBUG_TEST_CODE(0), |
2689 | atime, (uintptr_t)atime, (uintptr_t)(atime >> 32), 0, 0, 0); |
2690 | break; |
2691 | |
2692 | case KDTEST_CONTINUOUS_TIMESTAMP:; |
2693 | uint64_t ctime = mach_continuous_time(); |
2694 | kernel_debug_enter(test_coproc, KDEBUG_TEST_CODE(1), |
2695 | ctime, (uintptr_t)ctime, (uintptr_t)(ctime >> 32), 0, 0, 0); |
2696 | break; |
2697 | |
2698 | case KDTEST_PAST_EVENT:; |
2699 | uint64_t old_time = 1; |
2700 | kernel_debug_enter(test_coproc, KDEBUG_TEST_CODE(1), old_time, 0, 0, 0, |
2701 | 0, 0); |
2702 | kernel_debug_enter(test_coproc, KDEBUG_TEST_CODE(1), kdebug_timestamp(), |
2703 | 0, 0, 0, 0, 0); |
2704 | break; |
2705 | |
2706 | default: |
2707 | return ENOTSUP; |
2708 | } |
2709 | |
2710 | return 0; |
2711 | } |
2712 | |
2713 | #undef KDEBUG_TEST_CODE |
2714 | |
2715 | #endif /* DEVELOPMENT || DEBUG */ |
2716 | |
2717 | static void |
2718 | _deferred_coproc_notify(mpsc_queue_chain_t e, mpsc_daemon_queue_t queue __unused) |
2719 | { |
2720 | struct kd_coproc *coproc = mpsc_queue_element(e, struct kd_coproc, chain); |
2721 | if (kd_control_trace.kdc_emit == KDEMIT_TYPEFILTER) { |
2722 | coproc->callback.func(coproc->callback.context, |
2723 | KD_CALLBACK_TYPEFILTER_CHANGED, kdbg_typefilter); |
2724 | } |
2725 | if (kdebug_enable) { |
2726 | coproc->callback.func(coproc->callback.context, |
2727 | KD_CALLBACK_KDEBUG_ENABLED, kdbg_typefilter); |
2728 | } |
2729 | } |
2730 | |
2731 | void |
2732 | kdebug_init(unsigned int n_events, char *filter_desc, enum kdebug_opts opts) |
2733 | { |
2734 | assert(filter_desc != NULL); |
2735 | |
2736 | kdbg_typefilter = typefilter_create(); |
2737 | assert(kdbg_typefilter != NULL); |
2738 | kdbg_typefilter_memory_entry = typefilter_create_memory_entry(tf: kdbg_typefilter); |
2739 | assert(kdbg_typefilter_memory_entry != MACH_PORT_NULL); |
2740 | |
2741 | (void)mpsc_daemon_queue_init_with_thread_call(dq: &_coproc_notify_queue, |
2742 | invoke: _deferred_coproc_notify, pri: THREAD_CALL_PRIORITY_KERNEL, |
2743 | flags: MPSC_DAEMON_INIT_NONE); |
2744 | |
2745 | kdebug_trace_start(n_events, filterdesc: filter_desc, opts); |
2746 | } |
2747 | |
2748 | static void |
2749 | kdbg_set_typefilter_string(const char *filter_desc) |
2750 | { |
2751 | char *end = NULL; |
2752 | |
2753 | ktrace_assert_lock_held(); |
2754 | |
2755 | assert(filter_desc != NULL); |
2756 | |
2757 | typefilter_reject_all(tf: kdbg_typefilter); |
2758 | typefilter_allow_class(tf: kdbg_typefilter, DBG_TRACE); |
2759 | |
2760 | /* if the filter description starts with a number, assume it's a csc */ |
2761 | if (filter_desc[0] >= '0' && filter_desc[0] <= '9') { |
2762 | unsigned long csc = strtoul(filter_desc, NULL, 0); |
2763 | if (filter_desc != end && csc <= KDBG_CSC_MAX) { |
2764 | typefilter_allow_csc(tf: kdbg_typefilter, csc: (uint16_t)csc); |
2765 | } |
2766 | return; |
2767 | } |
2768 | |
2769 | while (filter_desc[0] != '\0') { |
2770 | unsigned long allow_value; |
2771 | |
2772 | char filter_type = filter_desc[0]; |
2773 | if (filter_type != 'C' && filter_type != 'S') { |
2774 | printf("kdebug: unexpected filter type `%c'\n" , filter_type); |
2775 | return; |
2776 | } |
2777 | filter_desc++; |
2778 | |
2779 | allow_value = strtoul(filter_desc, &end, 0); |
2780 | if (filter_desc == end) { |
2781 | printf("kdebug: cannot parse `%s' as integer\n" , filter_desc); |
2782 | return; |
2783 | } |
2784 | |
2785 | switch (filter_type) { |
2786 | case 'C': |
2787 | if (allow_value > KDBG_CLASS_MAX) { |
2788 | printf("kdebug: class 0x%lx is invalid\n" , allow_value); |
2789 | return; |
2790 | } |
2791 | printf("kdebug: C 0x%lx\n" , allow_value); |
2792 | typefilter_allow_class(tf: kdbg_typefilter, class: (uint8_t)allow_value); |
2793 | break; |
2794 | case 'S': |
2795 | if (allow_value > KDBG_CSC_MAX) { |
2796 | printf("kdebug: class-subclass 0x%lx is invalid\n" , allow_value); |
2797 | return; |
2798 | } |
2799 | printf("kdebug: S 0x%lx\n" , allow_value); |
2800 | typefilter_allow_csc(tf: kdbg_typefilter, csc: (uint16_t)allow_value); |
2801 | break; |
2802 | default: |
2803 | __builtin_unreachable(); |
2804 | } |
2805 | |
2806 | /* advance to next filter entry */ |
2807 | filter_desc = end; |
2808 | if (filter_desc[0] == ',') { |
2809 | filter_desc++; |
2810 | } |
2811 | } |
2812 | } |
2813 | |
2814 | uint64_t |
2815 | kdebug_wake(void) |
2816 | { |
2817 | if (!wake_nkdbufs) { |
2818 | return 0; |
2819 | } |
2820 | uint64_t start = mach_absolute_time(); |
2821 | kdebug_trace_start(n_events: wake_nkdbufs, NULL, opts: trace_wrap ? KDOPT_WRAPPING : 0); |
2822 | return mach_absolute_time() - start; |
2823 | } |
2824 | |
2825 | /* |
2826 | * This function is meant to be called from the bootstrap thread or kdebug_wake. |
2827 | */ |
2828 | void |
2829 | kdebug_trace_start(unsigned int n_events, const char *filter_desc, |
2830 | enum kdebug_opts opts) |
2831 | { |
2832 | if (!n_events) { |
2833 | kd_early_done = true; |
2834 | return; |
2835 | } |
2836 | |
2837 | ktrace_start_single_threaded(); |
2838 | |
2839 | ktrace_kernel_configure(KTRACE_KDEBUG); |
2840 | |
2841 | kdbg_set_nkdbufs_trace(req_nkdbufs_trace: n_events); |
2842 | |
2843 | kernel_debug_string_early(message: "start_kern_tracing" ); |
2844 | |
2845 | int error = kdbg_reinit(EXTRA_COPROC_COUNT_BOOT); |
2846 | if (error != 0) { |
2847 | printf("kdebug: allocation failed, kernel tracing not started: %d\n" , |
2848 | error); |
2849 | kd_early_done = true; |
2850 | goto out; |
2851 | } |
2852 | |
2853 | /* |
2854 | * Wrapping is disabled because boot and wake tracing is interested in |
2855 | * the earliest events, at the expense of later ones. |
2856 | */ |
2857 | if ((opts & KDOPT_WRAPPING) == 0) { |
2858 | kd_control_trace.kdc_live_flags |= KDBG_NOWRAP; |
2859 | } |
2860 | |
2861 | if (filter_desc && filter_desc[0] != '\0') { |
2862 | kdbg_set_typefilter_string(filter_desc); |
2863 | kdbg_enable_typefilter(); |
2864 | } |
2865 | |
2866 | /* |
2867 | * Hold off interrupts between getting a thread map and enabling trace |
2868 | * and until the early traces are recorded. |
2869 | */ |
2870 | bool s = ml_set_interrupts_enabled(false); |
2871 | |
2872 | if (!(opts & KDOPT_ATBOOT)) { |
2873 | _threadmap_init(); |
2874 | } |
2875 | |
2876 | kdbg_set_tracing_enabled(true, KDEBUG_ENABLE_TRACE); |
2877 | |
2878 | if ((opts & KDOPT_ATBOOT)) { |
2879 | /* |
2880 | * Transfer all very early events from the static buffer into the real |
2881 | * buffers. |
2882 | */ |
2883 | kernel_debug_early_end(); |
2884 | } |
2885 | |
2886 | ml_set_interrupts_enabled(enable: s); |
2887 | |
2888 | printf("kernel tracing started with %u events, filter = %s\n" , n_events, |
2889 | filter_desc ?: "none" ); |
2890 | |
2891 | out: |
2892 | ktrace_end_single_threaded(); |
2893 | } |
2894 | |
2895 | void |
2896 | kdbg_dump_trace_to_file(const char *filename, bool reenable) |
2897 | { |
2898 | vfs_context_t ctx; |
2899 | vnode_t vp; |
2900 | size_t write_size; |
2901 | int ret; |
2902 | int reenable_trace = 0; |
2903 | |
2904 | ktrace_lock(); |
2905 | |
2906 | if (!(kdebug_enable & KDEBUG_ENABLE_TRACE)) { |
2907 | goto out; |
2908 | } |
2909 | |
2910 | if (ktrace_get_owning_pid() != 0) { |
2911 | /* |
2912 | * Another process owns ktrace and is still active, disable tracing to |
2913 | * prevent wrapping. |
2914 | */ |
2915 | kdebug_enable = 0; |
2916 | kd_control_trace.enabled = 0; |
2917 | commpage_update_kdebug_state(); |
2918 | goto out; |
2919 | } |
2920 | |
2921 | KDBG_RELEASE(TRACE_WRITING_EVENTS | DBG_FUNC_START); |
2922 | |
2923 | reenable_trace = reenable ? kdebug_enable : 0; |
2924 | kdebug_enable = 0; |
2925 | kd_control_trace.enabled = 0; |
2926 | commpage_update_kdebug_state(); |
2927 | |
2928 | ctx = vfs_context_kernel(); |
2929 | |
2930 | if (vnode_open(path: filename, fmode: (O_CREAT | FWRITE | O_NOFOLLOW), cmode: 0600, flags: 0, vpp: &vp, ctx)) { |
2931 | goto out; |
2932 | } |
2933 | |
2934 | kdbg_write_thread_map(vp, ctx); |
2935 | |
2936 | write_size = kd_buffer_trace.kdb_event_count * sizeof(kd_buf); |
2937 | ret = _read_merged_trace_events(buffer: 0, number: &write_size, vp, ctx, false); |
2938 | if (ret) { |
2939 | goto out_close; |
2940 | } |
2941 | |
2942 | /* |
2943 | * Wait to synchronize the file to capture the I/O in the |
2944 | * TRACE_WRITING_EVENTS interval. |
2945 | */ |
2946 | ret = VNOP_FSYNC(vp, MNT_WAIT, ctx); |
2947 | if (ret == KERN_SUCCESS) { |
2948 | ret = VNOP_IOCTL(vp, F_FULLFSYNC, data: (caddr_t)NULL, fflag: 0, ctx); |
2949 | } |
2950 | |
2951 | /* |
2952 | * Balance the starting TRACE_WRITING_EVENTS tracepoint manually. |
2953 | */ |
2954 | kd_buf end_event = { |
2955 | .debugid = TRACE_WRITING_EVENTS | DBG_FUNC_END, |
2956 | .arg1 = write_size, |
2957 | .arg2 = ret, |
2958 | .arg5 = (kd_buf_argtype)thread_tid(thread: current_thread()), |
2959 | }; |
2960 | kdbg_set_timestamp_and_cpu(kp: &end_event, thetime: kdebug_timestamp(), |
2961 | cpu: cpu_number()); |
2962 | |
2963 | /* this is best effort -- ignore any errors */ |
2964 | (void)kdbg_write_to_vnode(buffer: (caddr_t)&end_event, size: sizeof(kd_buf), vp, ctx, |
2965 | file_offset: RAW_file_offset); |
2966 | |
2967 | out_close: |
2968 | vnode_close(vp, FWRITE, ctx); |
2969 | sync(current_proc(), (void *)NULL, (int *)NULL); |
2970 | |
2971 | out: |
2972 | if (reenable_trace != 0) { |
2973 | kdebug_enable = reenable_trace; |
2974 | kd_control_trace.enabled = 1; |
2975 | commpage_update_kdebug_state(); |
2976 | } |
2977 | |
2978 | ktrace_unlock(); |
2979 | } |
2980 | |
2981 | SYSCTL_NODE(_kern, OID_AUTO, kdbg, CTLFLAG_RD | CTLFLAG_LOCKED, 0, |
2982 | "kdbg" ); |
2983 | |
2984 | SYSCTL_INT(_kern_kdbg, OID_AUTO, debug, |
2985 | CTLFLAG_RW | CTLFLAG_LOCKED, |
2986 | &kdbg_debug, 0, "Set kdebug debug mode" ); |
2987 | |
2988 | SYSCTL_QUAD(_kern_kdbg, OID_AUTO, oldest_time, |
2989 | CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_LOCKED, |
2990 | &kd_control_trace.kdc_oldest_time, |
2991 | "Find the oldest timestamp still in trace" ); |
2992 | |