| 1 | /* |
| 2 | * Copyright (c) 2000-2020 Apple, Inc. All rights reserved. |
| 3 | * |
| 4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
| 5 | * |
| 6 | * This file contains Original Code and/or Modifications of Original Code |
| 7 | * as defined in and that are subject to the Apple Public Source License |
| 8 | * Version 2.0 (the 'License'). You may not use this file except in |
| 9 | * compliance with the License. The rights granted to you under the License |
| 10 | * may not be used to create, or enable the creation or redistribution of, |
| 11 | * unlawful or unlicensed copies of an Apple operating system, or to |
| 12 | * circumvent, violate, or enable the circumvention or violation of, any |
| 13 | * terms of an Apple operating system software license agreement. |
| 14 | * |
| 15 | * Please obtain a copy of the License at |
| 16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
| 17 | * |
| 18 | * The Original Code and all software distributed under the License are |
| 19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
| 20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
| 21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
| 22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
| 23 | * Please see the License for the specific language governing rights and |
| 24 | * limitations under the License. |
| 25 | * |
| 26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
| 27 | */ |
| 28 | |
| 29 | #ifdef __x86_64__ |
| 30 | #include <i386/mp.h> |
| 31 | #include <i386/cpu_data.h> |
| 32 | #include <i386/bit_routines.h> |
| 33 | #include <i386/machine_routines.h> |
| 34 | #include <i386/misc_protos.h> |
| 35 | #include <i386/serial_io.h> |
| 36 | #endif /* __x86_64__ */ |
| 37 | |
| 38 | #include <machine/machine_cpu.h> |
| 39 | #include <libkern/OSAtomic.h> |
| 40 | #include <vm/vm_kern.h> |
| 41 | #include <vm/vm_map.h> |
| 42 | #include <console/video_console.h> |
| 43 | #include <console/serial_protos.h> |
| 44 | #include <kern/startup.h> |
| 45 | #include <kern/thread.h> |
| 46 | #include <kern/cpu_data.h> |
| 47 | #include <kern/sched_prim.h> |
| 48 | #include <libkern/section_keywords.h> |
| 49 | |
| 50 | #if __arm64__ |
| 51 | #include <machine/machine_routines.h> |
| 52 | #include <arm/cpu_data_internal.h> |
| 53 | #endif |
| 54 | |
| 55 | #ifdef CONFIG_XNUPOST |
| 56 | #include <tests/xnupost.h> |
| 57 | kern_return_t console_serial_test(void); |
| 58 | kern_return_t console_serial_parallel_log_tests(void); |
| 59 | #endif |
| 60 | |
| 61 | /* Structure representing the console ring buffer. */ |
| 62 | static struct { |
| 63 | /* The ring buffer backing store. */ |
| 64 | char *buffer; |
| 65 | |
| 66 | /* The total length of the ring buffer. */ |
| 67 | int len; |
| 68 | |
| 69 | /** |
| 70 | * The number of characters that have been written into the buffer that need |
| 71 | * to be drained. |
| 72 | */ |
| 73 | int used; |
| 74 | |
| 75 | /** |
| 76 | * Number of reserved regions in the buffer. These are regions that are |
| 77 | * currently being written into by various CPUs. We use this as a way of |
| 78 | * determining when it's safe to drain the buffer. |
| 79 | */ |
| 80 | int nreserved; |
| 81 | |
| 82 | /* The location in the buffer thats written to next. */ |
| 83 | char *write_ptr; |
| 84 | |
| 85 | /* The location in the buffer that will be drained next. */ |
| 86 | char *read_ptr; |
| 87 | |
| 88 | /* Synchronizes the flushing of the ring buffer to hardware */ |
| 89 | lck_mtx_t flush_lock; |
| 90 | |
| 91 | /** |
| 92 | * Synchronizes reserving space in the ring buffer and ensures that only |
| 93 | * completed writes are flushed. |
| 94 | */ |
| 95 | lck_ticket_t write_lock; |
| 96 | } console_ring; |
| 97 | |
| 98 | /** |
| 99 | * We don't dedicate any buffer space to specific CPUs, but this value is used |
| 100 | * to scale the size of the console buffer by the number of CPUs. |
| 101 | * |
| 102 | * How many bytes-per-cpu to allocate in the console ring buffer. Also affects |
| 103 | * the maximum number of bytes a single console thread can drain. |
| 104 | */ |
| 105 | #define CPU_CONS_BUF_SIZE 256 |
| 106 | |
| 107 | /* Scale the size of the console ring buffer by the number of CPUs. */ |
| 108 | #define KERN_CONSOLE_RING_SIZE vm_map_round_page(CPU_CONS_BUF_SIZE * (MAX_CPUS + 1), PAGE_SIZE - 1) |
| 109 | |
| 110 | #define MAX_FLUSH_SIZE_LOCK_HELD 16 |
| 111 | #define MAX_TOTAL_FLUSH_SIZE (MAX(2, MAX_CPUS) * CPU_CONS_BUF_SIZE) |
| 112 | |
| 113 | extern int serial_getc(void); |
| 114 | extern void serial_putc_options(char, bool); |
| 115 | |
| 116 | #if DEBUG || DEVELOPMENT |
| 117 | TUNABLE(bool, allow_printf_from_interrupts_disabled_context, "nointr_consio" , false); |
| 118 | #else |
| 119 | #define allow_printf_from_interrupts_disabled_context false |
| 120 | #endif |
| 121 | |
| 122 | SECURITY_READ_ONLY_EARLY(struct console_ops) cons_ops[] = { |
| 123 | { |
| 124 | .putc = serial_putc_options, .getc = _serial_getc, |
| 125 | }, |
| 126 | { |
| 127 | .putc = vcputc_options, .getc = _vcgetc, |
| 128 | }, |
| 129 | }; |
| 130 | |
| 131 | SECURITY_READ_ONLY_EARLY(uint32_t) nconsops = (sizeof cons_ops / sizeof cons_ops[0]); |
| 132 | |
| 133 | #if __x86_64__ |
| 134 | uint32_t cons_ops_index = VC_CONS_OPS; |
| 135 | #else |
| 136 | SECURITY_READ_ONLY_LATE(uint32_t) cons_ops_index = VC_CONS_OPS; |
| 137 | #endif |
| 138 | |
| 139 | LCK_GRP_DECLARE(console_lck_grp, "console" ); |
| 140 | |
| 141 | /* If the NMI string is entered into the console, the system will enter the debugger. */ |
| 142 | #define NMI_STRING_SIZE 32 |
| 143 | char nmi_string[NMI_STRING_SIZE] = "afDIGHr84A84jh19Kphgp428DNPdnapq" ; |
| 144 | static int nmi_counter = 0; |
| 145 | |
| 146 | /** |
| 147 | * This is used to prevent console output from going through the console ring |
| 148 | * buffer synchronization in cases where that could cause issues (e.g., during |
| 149 | * panics/stackshots and going down for sleep). |
| 150 | */ |
| 151 | static bool console_suspended = false; |
| 152 | |
| 153 | /** |
| 154 | * Controls console output for underlying serial or video console. |
| 155 | * To be used only by core console and init accessors. |
| 156 | */ |
| 157 | int disableConsoleOutput; |
| 158 | |
| 159 | /** |
| 160 | * Enforce policies around when console I/O is allowed. Most importantly about |
| 161 | * not performing console I/O while interrupts are disabled (which can cause |
| 162 | * serious latency issues). |
| 163 | * |
| 164 | * @return True if console I/O should be allowed, false otherwise. |
| 165 | */ |
| 166 | static inline bool |
| 167 | console_io_allowed(void) |
| 168 | { |
| 169 | if (!allow_printf_from_interrupts_disabled_context && |
| 170 | !console_suspended && |
| 171 | startup_phase >= STARTUP_SUB_EARLY_BOOT && |
| 172 | !ml_get_interrupts_enabled()) { |
| 173 | #if defined(__arm64__) || DEBUG || DEVELOPMENT |
| 174 | panic("Console I/O from interrupt-disabled context" ); |
| 175 | #else |
| 176 | return false; |
| 177 | #endif |
| 178 | } |
| 179 | |
| 180 | return true; |
| 181 | } |
| 182 | |
| 183 | /** |
| 184 | * Initialize the console ring buffer and console lock. It's still possible to |
| 185 | * call console_write() before initializing the ring buffer. In that case the |
| 186 | * data will get outputted directly to the underlying serial/video console |
| 187 | * without synchronization. |
| 188 | * |
| 189 | * This function is also safe to call multiple times. Any call after the first |
| 190 | * will return early without doing anything. |
| 191 | */ |
| 192 | void |
| 193 | console_init(void) |
| 194 | { |
| 195 | if (console_ring.len != 0) { |
| 196 | return; |
| 197 | } |
| 198 | |
| 199 | kmem_alloc(map: kernel_map, addrp: (vm_offset_t *)&console_ring.buffer, |
| 200 | KERN_CONSOLE_RING_SIZE + ptoa(2), flags: KMA_NOFAIL | KMA_PERMANENT | |
| 201 | KMA_KOBJECT | KMA_PERMANENT | KMA_GUARD_FIRST | KMA_GUARD_LAST | |
| 202 | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_OSFMK); |
| 203 | |
| 204 | console_ring.buffer += PAGE_SIZE; /* Skip past the first guard page. */ |
| 205 | console_ring.len = KERN_CONSOLE_RING_SIZE; |
| 206 | console_ring.used = 0; |
| 207 | console_ring.nreserved = 0; |
| 208 | console_ring.read_ptr = console_ring.buffer; |
| 209 | console_ring.write_ptr = console_ring.buffer; |
| 210 | |
| 211 | lck_mtx_init(lck: &console_ring.flush_lock, grp: &console_lck_grp, LCK_ATTR_NULL); |
| 212 | lck_ticket_init(tlock: &console_ring.write_lock, grp: &console_lck_grp); |
| 213 | } |
| 214 | |
| 215 | /** |
| 216 | * Returns true when the console has already been initialized. |
| 217 | */ |
| 218 | static inline bool |
| 219 | is_console_initialized(void) |
| 220 | { |
| 221 | return console_ring.len == KERN_CONSOLE_RING_SIZE; |
| 222 | } |
| 223 | |
| 224 | /** |
| 225 | * Return the index to the currently selected console (serial/video). This is |
| 226 | * an index into the "cons_ops[]" array of function pointer structs. |
| 227 | */ |
| 228 | static inline uint32_t |
| 229 | get_cons_ops_index(void) |
| 230 | { |
| 231 | uint32_t idx = cons_ops_index; |
| 232 | |
| 233 | if (idx >= nconsops) { |
| 234 | panic("Bad cons_ops_index: %d" , idx); |
| 235 | } |
| 236 | |
| 237 | return idx; |
| 238 | } |
| 239 | |
| 240 | /** |
| 241 | * Helper function for outputting a character to the underlying console |
| 242 | * (either video or serial) with the possibility of sleeping waiting for |
| 243 | * an interrupt indicating the console is ready. |
| 244 | * |
| 245 | * @note assumes console_ring.read lock is held if poll == false |
| 246 | * |
| 247 | * @param c The character to print. |
| 248 | * @param poll Whether or not this call should poll instead of going to sleep |
| 249 | * waiting for an interrupt when the hardware device isn't ready |
| 250 | */ |
| 251 | static inline void |
| 252 | _cnputc(char c, bool poll) |
| 253 | { |
| 254 | bool in_debugger = (kernel_debugger_entry_count > 0); |
| 255 | const uint32_t idx = get_cons_ops_index(); |
| 256 | |
| 257 | poll = poll || in_debugger; |
| 258 | |
| 259 | if (c == '\n') { |
| 260 | _cnputc(c: '\r', poll); |
| 261 | } |
| 262 | |
| 263 | cons_ops[idx].putc(c, poll); |
| 264 | } |
| 265 | |
| 266 | /** |
| 267 | * Helper function for outputting characters directly to the underlying console |
| 268 | * (either video or serial). |
| 269 | * |
| 270 | * @note disableConsoleOutput is to be used only by core console and init accessors |
| 271 | * such as this function. Returns early if the serial output is disabled. |
| 272 | * |
| 273 | * @param c The array of characters to print. |
| 274 | * @param poll Whether or not this call should poll instead of going to sleep |
| 275 | * waiting for an interrupt when the hardware device isn't ready |
| 276 | * @param size The number of characters to print to the console. |
| 277 | */ |
| 278 | static inline void |
| 279 | _cnputs(char *c, int size, bool poll) |
| 280 | { |
| 281 | if (disableConsoleOutput) { |
| 282 | return; |
| 283 | } |
| 284 | |
| 285 | assert(c != NULL); |
| 286 | |
| 287 | while (size-- > 0) { |
| 288 | _cnputc(c: *c, poll); |
| 289 | c++; |
| 290 | } |
| 291 | } |
| 292 | |
| 293 | /** |
| 294 | * Attempt to reserve space for a number of characters in the console ring |
| 295 | * buffer. Space in the ring buffer must be reserved before new characters can |
| 296 | * be entered. |
| 297 | * |
| 298 | * Every call to this function should be paired with a corresponding call to |
| 299 | * console_ring_unreserve_space(). |
| 300 | * |
| 301 | * @note If space is successfully reserved, this will disable preemption because |
| 302 | * otherwise, console_ring_try_empty() could take arbitrarily long. |
| 303 | * |
| 304 | * @param nchars The number of characters to reserve. |
| 305 | * |
| 306 | * @return If the wanted number of characters could not be reserved, then return |
| 307 | * NULL. Otherwise, return a pointer to the beginning of the reserved |
| 308 | * space. |
| 309 | */ |
| 310 | static inline char* |
| 311 | console_ring_reserve_space(int nchars) |
| 312 | { |
| 313 | char *write_ptr = NULL; |
| 314 | lck_ticket_lock(tlock: &console_ring.write_lock, grp: &console_lck_grp); |
| 315 | if ((console_ring.len - console_ring.used) >= nchars) { |
| 316 | console_ring.used += nchars; |
| 317 | mp_disable_preemption(); |
| 318 | os_atomic_inc(&console_ring.nreserved, relaxed); |
| 319 | |
| 320 | /* Return out the pointer to the beginning of the just reserved space. */ |
| 321 | write_ptr = console_ring.write_ptr; |
| 322 | |
| 323 | /* Move the console ring's write pointer to the beginning of the next free space. */ |
| 324 | const ptrdiff_t write_index = console_ring.write_ptr - console_ring.buffer; |
| 325 | console_ring.write_ptr = console_ring.buffer + ((write_index + nchars) % console_ring.len); |
| 326 | } |
| 327 | lck_ticket_unlock(tlock: &console_ring.write_lock); |
| 328 | return write_ptr; |
| 329 | } |
| 330 | |
| 331 | /** |
| 332 | * Decrement the number of reserved spaces in the console ring (now that the data |
| 333 | * has been written) and re-enable preemption. |
| 334 | * |
| 335 | * Every call to this function should be paired with a corresponding call to |
| 336 | * console_ring_reserve_space(). |
| 337 | */ |
| 338 | static inline void |
| 339 | console_ring_unreserve_space(void) |
| 340 | { |
| 341 | assert(console_ring.nreserved > 0); |
| 342 | |
| 343 | os_atomic_dec(&console_ring.nreserved, relaxed); |
| 344 | mp_enable_preemption(); |
| 345 | } |
| 346 | |
| 347 | /** |
| 348 | * Write a single character into the console ring buffer and handle moving the |
| 349 | * write pointer circularly around the buffer. |
| 350 | * |
| 351 | * @note Space to write this character must have already been reserved using |
| 352 | * console_ring_reserve_space(). |
| 353 | * |
| 354 | * @param write_ptr Pointer into the reserved space in the buffer to write the |
| 355 | * character. This pointer will get moved to the next valid |
| 356 | * location to write a character so the same pointer can be |
| 357 | * passed into subsequent calls to write multiple characters. |
| 358 | * @param ch The character to insert into the ring buffer. |
| 359 | */ |
| 360 | static inline void |
| 361 | console_ring_put(char **write_ptr, char ch) |
| 362 | { |
| 363 | assert(console_ring.nreserved > 0); |
| 364 | **write_ptr = ch; |
| 365 | ++(*write_ptr); |
| 366 | if ((*write_ptr - console_ring.buffer) == console_ring.len) { |
| 367 | *write_ptr = console_ring.buffer; |
| 368 | } |
| 369 | } |
| 370 | |
| 371 | /** |
| 372 | * Attempt to drain the console ring buffer if no other CPUs are already doing |
| 373 | * so. |
| 374 | * |
| 375 | * @param fail_fast If true, this function returns immediately instead of |
| 376 | * sleeping if the thread fails to acquire the console flush |
| 377 | * mutex. |
| 378 | * |
| 379 | * @note This function should not be called with preemption disabled. |
| 380 | * |
| 381 | * @note To prevent one CPU from holding the console lock for too long, only |
| 382 | * MAX_FLUSH_SIZE_LOCK_HELD number of characters can be drained at a time |
| 383 | * with the lock held. The lock will be dropped between each drain of size |
| 384 | * MAX_FLUSH_SIZE_LOCK_HELD to allow another CPU to grab the lock. If |
| 385 | * another CPU grabs the lock, then the original thread can stop draining |
| 386 | * and return instead of sleeping for the lock. |
| 387 | * |
| 388 | * @note To prevent one thread from being the drain thread for too long (presumably |
| 389 | * that thread has other things it wants to do besides draining serial), the |
| 390 | * total number of characters a single call to this function can drain is |
| 391 | * restricted to MAX_TOTAL_FLUSH_SIZE. |
| 392 | */ |
| 393 | static void |
| 394 | console_ring_try_empty(bool fail_fast) |
| 395 | { |
| 396 | char flush_buf[MAX_FLUSH_SIZE_LOCK_HELD]; |
| 397 | |
| 398 | int nchars_out = 0; |
| 399 | int total_chars_out = 0; |
| 400 | int size_before_wrap = 0; |
| 401 | bool in_debugger = (kernel_debugger_entry_count > 0); |
| 402 | |
| 403 | if (__improbable(!console_io_allowed()) || get_preemption_level() != 0) { |
| 404 | return; |
| 405 | } |
| 406 | |
| 407 | do { |
| 408 | if (__probable(!in_debugger) && fail_fast && !lck_mtx_try_lock(lck: &console_ring.flush_lock)) { |
| 409 | return; |
| 410 | } else if (__probable(!in_debugger) && !fail_fast) { |
| 411 | lck_mtx_lock(lck: &console_ring.flush_lock); |
| 412 | } |
| 413 | |
| 414 | if (__probable(!in_debugger)) { |
| 415 | lck_ticket_lock(tlock: &console_ring.write_lock, grp: &console_lck_grp); |
| 416 | |
| 417 | /** |
| 418 | * If we've managed to grab the write lock, but there's still space |
| 419 | * reserved in the buffer, then other CPUs are actively writing into |
| 420 | * the ring, wait for them to finish. |
| 421 | */ |
| 422 | while (os_atomic_load(&console_ring.nreserved, relaxed) > 0) { |
| 423 | cpu_pause(); |
| 424 | } |
| 425 | } |
| 426 | |
| 427 | /* Try small chunk at a time, so we allow writes from other cpus into the buffer. */ |
| 428 | nchars_out = MIN(console_ring.used, (int)sizeof(flush_buf)); |
| 429 | |
| 430 | /* Account for data to be read before wrap around. */ |
| 431 | size_before_wrap = (int)((console_ring.buffer + console_ring.len) - console_ring.read_ptr); |
| 432 | if (nchars_out > size_before_wrap) { |
| 433 | nchars_out = size_before_wrap; |
| 434 | } |
| 435 | |
| 436 | /** |
| 437 | * Copy the characters to be drained into a separate flush buffer, and |
| 438 | * move the console read pointer to the next chunk of data that needs to |
| 439 | * be drained. |
| 440 | */ |
| 441 | if (nchars_out > 0) { |
| 442 | memcpy(dst: flush_buf, src: console_ring.read_ptr, n: nchars_out); |
| 443 | const ptrdiff_t read_index = console_ring.read_ptr - console_ring.buffer; |
| 444 | console_ring.read_ptr = console_ring.buffer + ((read_index + nchars_out) % console_ring.len); |
| 445 | console_ring.used -= nchars_out; |
| 446 | } |
| 447 | |
| 448 | if (__probable(!in_debugger)) { |
| 449 | lck_ticket_unlock(tlock: &console_ring.write_lock); |
| 450 | } |
| 451 | |
| 452 | /** |
| 453 | * Output characters to the underlying console (serial/video). We should |
| 454 | * only poll if the console is suspended. |
| 455 | */ |
| 456 | if (nchars_out > 0) { |
| 457 | total_chars_out += nchars_out; |
| 458 | _cnputs(c: flush_buf, size: nchars_out, poll: console_suspended); |
| 459 | } |
| 460 | |
| 461 | if (__probable(!in_debugger)) { |
| 462 | lck_mtx_unlock(lck: &console_ring.flush_lock); |
| 463 | } |
| 464 | |
| 465 | /** |
| 466 | * Prevent this thread from sleeping on the lock again if another thread |
| 467 | * grabs it after we drop it. |
| 468 | */ |
| 469 | fail_fast = true; |
| 470 | |
| 471 | /* |
| 472 | * In case we end up being the console drain thread for far too long, |
| 473 | * break out. Except in panic/suspend cases where we should clear out |
| 474 | * the full buffer. |
| 475 | */ |
| 476 | if (!console_suspended && (total_chars_out >= MAX_TOTAL_FLUSH_SIZE)) { |
| 477 | break; |
| 478 | } |
| 479 | } while (nchars_out > 0); |
| 480 | } |
| 481 | |
| 482 | /** |
| 483 | * Notify the console subystem that all following console writes should skip |
| 484 | * synchronization and get outputted directly to the underlying console. This is |
| 485 | * important for cases like panic/stackshots and going down for sleep where |
| 486 | * assumptions about the state of the system could cause hangs or nested panics. |
| 487 | */ |
| 488 | void |
| 489 | console_suspend() |
| 490 | { |
| 491 | console_suspended = true; |
| 492 | console_ring_try_empty(false); |
| 493 | } |
| 494 | |
| 495 | /** |
| 496 | * Notify the console subsystem that it is now safe to use the console ring |
| 497 | * buffer synchronization when writing console data. |
| 498 | */ |
| 499 | void |
| 500 | console_resume() |
| 501 | { |
| 502 | console_suspended = false; |
| 503 | } |
| 504 | |
| 505 | /** |
| 506 | * Write a string of characters to the underlying video or serial console in a |
| 507 | * synchronized manner. By synchronizing access to a global console buffer, this |
| 508 | * prevents the serial output from appearing interleaved to the end user when |
| 509 | * multiple CPUs are outputting to the console at the same time. |
| 510 | * |
| 511 | * @note It's safe to call this function even before the console buffer has been |
| 512 | * initialized. In that case, the data will be sent directly to the |
| 513 | * underlying console with no buffering. This is the same for when the |
| 514 | * console is suspended. |
| 515 | * |
| 516 | * @note disableConsoleOutput is to be used only by core console and init accessors |
| 517 | * such as this function. Returns early if the serial output is disabled and |
| 518 | * skips lock acquisition. |
| 519 | * |
| 520 | * @param str The string of characters to print. |
| 521 | * @param size The number of characters in `str` to print. |
| 522 | */ |
| 523 | void |
| 524 | console_write(char *str, int size) |
| 525 | { |
| 526 | if (disableConsoleOutput) { |
| 527 | return; |
| 528 | } |
| 529 | |
| 530 | assert(str != NULL); |
| 531 | |
| 532 | char *write_ptr = NULL; |
| 533 | int chunk_size = CPU_CONS_BUF_SIZE; |
| 534 | int i = 0; |
| 535 | |
| 536 | if (__improbable(console_suspended || !is_console_initialized() || pmap_in_ppl())) { |
| 537 | /* |
| 538 | * Output directly to console in the following cases: |
| 539 | * 1. If this is early in boot before the console has been initialized. |
| 540 | * 2. If we're heading into suspend. |
| 541 | * 3. If we're in the kernel debugger for a panic/stackshot. If any of |
| 542 | * the other cores happened to halt while holding any of the console |
| 543 | * locks, attempting to use the normal path will result in sadness. |
| 544 | * 4. If we're in the PPL. As we synchronize the ring buffer with a |
| 545 | * mutex and preemption is disabled in the PPL, any writes must go |
| 546 | * directly to the hardware device. |
| 547 | */ |
| 548 | _cnputs(c: str, size, true); |
| 549 | return; |
| 550 | } else if (__improbable(!console_io_allowed())) { |
| 551 | return; |
| 552 | } |
| 553 | |
| 554 | while (size > 0) { |
| 555 | /** |
| 556 | * Restrict the maximum number of characters that can be reserved at |
| 557 | * once. This helps prevent one CPU from reserving too much and starving |
| 558 | * out the other CPUs. |
| 559 | */ |
| 560 | if (size < chunk_size) { |
| 561 | chunk_size = size; |
| 562 | } |
| 563 | |
| 564 | /** |
| 565 | * Attempt to reserve space in the ring buffer and if that fails, then |
| 566 | * keep attempting to drain the ring buffer until there's enough space. |
| 567 | * We can't flush the serial console with preemption disabled so return |
| 568 | * early to drop the message in that case. |
| 569 | */ |
| 570 | while ((write_ptr = console_ring_reserve_space(nchars: chunk_size)) == NULL) { |
| 571 | if (get_preemption_level() != 0) { |
| 572 | return; |
| 573 | } |
| 574 | |
| 575 | console_ring_try_empty(false); |
| 576 | } |
| 577 | |
| 578 | for (i = 0; i < chunk_size; i++) { |
| 579 | console_ring_put(write_ptr: &write_ptr, ch: str[i]); |
| 580 | } |
| 581 | |
| 582 | console_ring_unreserve_space(); |
| 583 | str = &str[i]; |
| 584 | size -= chunk_size; |
| 585 | } |
| 586 | |
| 587 | /* Do good faith flush if preemption is not disabled */ |
| 588 | if (get_preemption_level() == 0) { |
| 589 | console_ring_try_empty(true); |
| 590 | } |
| 591 | } |
| 592 | |
| 593 | /** |
| 594 | * Output a character directly to the underlying console (either video or serial). |
| 595 | * This directly bypasses the console serial buffer (as provided by console_write()) |
| 596 | * and all of the synchronization that provides. |
| 597 | * |
| 598 | * @note This function can cause serial data to get printed interleaved if being |
| 599 | * called on multiple CPUs at the same time. Only use this function if |
| 600 | * there's a specific reason why this serial data can't get synchronized |
| 601 | * through the console buffer. |
| 602 | * |
| 603 | * @note disableConsoleOutput is to be used only by core console and init accessors |
| 604 | * such as this function. Returns early if the serial output is disabled. |
| 605 | * |
| 606 | * @param c The character to print. |
| 607 | */ |
| 608 | void |
| 609 | console_write_unbuffered(char c) |
| 610 | { |
| 611 | if (disableConsoleOutput) { |
| 612 | return; |
| 613 | } |
| 614 | |
| 615 | _cnputc(c, true); |
| 616 | } |
| 617 | |
| 618 | /** |
| 619 | * Write a single character to the selected console (video or serial). |
| 620 | * |
| 621 | * @param c The character to print. |
| 622 | */ |
| 623 | void |
| 624 | console_write_char(char c) |
| 625 | { |
| 626 | console_write(str: &c, size: 1); |
| 627 | } |
| 628 | |
| 629 | /** |
| 630 | * Wrapper around the platform-dependent serial input method which handles |
| 631 | * waiting for a new character and checking for the NMI string. |
| 632 | * |
| 633 | * @param wait True if this function should block until a character appears. |
| 634 | * |
| 635 | * @return The character if one was read, -1 otherwise. |
| 636 | */ |
| 637 | int |
| 638 | _serial_getc(bool wait) |
| 639 | { |
| 640 | int c = -1; |
| 641 | |
| 642 | do { |
| 643 | c = serial_getc(); |
| 644 | } while (wait && c < 0); |
| 645 | |
| 646 | /* Check for the NMI string. */ |
| 647 | if (c == nmi_string[nmi_counter]) { |
| 648 | nmi_counter++; |
| 649 | if (nmi_counter == NMI_STRING_SIZE) { |
| 650 | /* We've got the NMI string, now do an NMI. */ |
| 651 | Debugger(message: "Automatic NMI" ); |
| 652 | nmi_counter = 0; |
| 653 | return '\n'; |
| 654 | } |
| 655 | } else if (c != -1) { |
| 656 | nmi_counter = 0; |
| 657 | } |
| 658 | |
| 659 | return c; |
| 660 | } |
| 661 | |
| 662 | /** |
| 663 | * Typically the video console doesn't support input, but we call into the |
| 664 | * pexpert to give each platform an opportunity to provide console input through |
| 665 | * alternative methods if it so desires. |
| 666 | * |
| 667 | * Usually a platform will either not provide any input, or will grab input from |
| 668 | * the serial driver. |
| 669 | * |
| 670 | * @return The character if one was read, or -1 otherwise. |
| 671 | */ |
| 672 | int |
| 673 | _vcgetc(__unused bool wait) |
| 674 | { |
| 675 | char c; |
| 676 | |
| 677 | if (0 == PE_stub_poll_input(options: 0, c: &c)) { |
| 678 | return c; |
| 679 | } else { |
| 680 | return -1; |
| 681 | } |
| 682 | } |
| 683 | |
| 684 | /** |
| 685 | * Block until a character is available from the console and return it. |
| 686 | * |
| 687 | * @return The character retrieved from the console. |
| 688 | */ |
| 689 | int |
| 690 | console_read_char(void) |
| 691 | { |
| 692 | const uint32_t idx = get_cons_ops_index(); |
| 693 | return cons_ops[idx].getc(true); |
| 694 | } |
| 695 | |
| 696 | /** |
| 697 | * Attempt to read a character from the console, and if one isn't available, |
| 698 | * then return immediately. |
| 699 | * |
| 700 | * @return The character if one is available, -1 otherwise. |
| 701 | */ |
| 702 | int |
| 703 | console_try_read_char(void) |
| 704 | { |
| 705 | const uint32_t idx = get_cons_ops_index(); |
| 706 | return cons_ops[idx].getc(false); |
| 707 | } |
| 708 | |
| 709 | #ifdef CONFIG_XNUPOST |
| 710 | static uint32_t cons_test_ops_count = 0; |
| 711 | |
| 712 | /* |
| 713 | * Log to console by multiple methods - printf, unbuffered write, console_write() |
| 714 | */ |
| 715 | static void |
| 716 | log_to_console_func(void * arg __unused, wait_result_t wres __unused) |
| 717 | { |
| 718 | uint64_t thread_id = current_thread()->thread_id; |
| 719 | char somedata[10] = "123456789" ; |
| 720 | for (int i = 0; i < 26; i++) { |
| 721 | os_atomic_inc(&cons_test_ops_count, relaxed); |
| 722 | printf(" thid: %llu printf iteration %d\n" , thread_id, i); |
| 723 | console_write_unbuffered((char)('A' + i)); |
| 724 | console_write_unbuffered('\n'); |
| 725 | console_write((char *)somedata, sizeof(somedata)); |
| 726 | delay(10); |
| 727 | } |
| 728 | printf("finished the log_to_console_func operations\n\n" ); |
| 729 | } |
| 730 | |
| 731 | /* Test that outputting to the console can occur on multiple threads at the same time. */ |
| 732 | kern_return_t |
| 733 | console_serial_parallel_log_tests(void) |
| 734 | { |
| 735 | thread_t thread; |
| 736 | kern_return_t kr; |
| 737 | cons_test_ops_count = 0; |
| 738 | |
| 739 | kr = kernel_thread_start(log_to_console_func, NULL, &thread); |
| 740 | T_ASSERT_EQ_INT(kr, KERN_SUCCESS, "kernel_thread_start returned successfully" ); |
| 741 | |
| 742 | delay(100); |
| 743 | |
| 744 | log_to_console_func(NULL, 0); |
| 745 | |
| 746 | /* wait until other thread has also finished */ |
| 747 | while (cons_test_ops_count < 52) { |
| 748 | delay(1000); |
| 749 | } |
| 750 | |
| 751 | thread_deallocate(thread); |
| 752 | T_LOG("parallel_logging tests is now complete. From this point forward we expect full lines\n" ); |
| 753 | return KERN_SUCCESS; |
| 754 | } |
| 755 | |
| 756 | /* Basic serial test that prints serial output through various methods (printf/T_LOG). */ |
| 757 | kern_return_t |
| 758 | console_serial_test(void) |
| 759 | { |
| 760 | unsigned long i; |
| 761 | char buffer[CPU_CONS_BUF_SIZE]; |
| 762 | |
| 763 | T_LOG("Checking console_ring status." ); |
| 764 | T_ASSERT_EQ_INT(console_ring.len, KERN_CONSOLE_RING_SIZE, "Console ring size is not correct." ); |
| 765 | |
| 766 | /* setup buffer to be chars */ |
| 767 | for (i = 0; i < CPU_CONS_BUF_SIZE; i++) { |
| 768 | buffer[i] = (char)('0' + (i % 10)); |
| 769 | } |
| 770 | buffer[CPU_CONS_BUF_SIZE - 1] = '\0'; |
| 771 | |
| 772 | T_LOG("Printing %d char string to serial one char at a time." , CPU_CONS_BUF_SIZE); |
| 773 | for (i = 0; i < CPU_CONS_BUF_SIZE; i++) { |
| 774 | printf("%c" , buffer[i]); |
| 775 | } |
| 776 | printf("End\n" ); |
| 777 | T_LOG("Printing %d char string to serial as a whole" , CPU_CONS_BUF_SIZE); |
| 778 | printf("%s\n" , buffer); |
| 779 | |
| 780 | T_LOG("Using console_write call repeatedly for 100 iterations" ); |
| 781 | for (i = 0; i < 100; i++) { |
| 782 | console_write(&buffer[0], 14); |
| 783 | if ((i % 6) == 0) { |
| 784 | printf("\n" ); |
| 785 | } |
| 786 | } |
| 787 | printf("\n" ); |
| 788 | |
| 789 | T_LOG("Using T_LOG to print buffer %s" , buffer); |
| 790 | return KERN_SUCCESS; |
| 791 | } |
| 792 | #endif |
| 793 | |