1/*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * This file contains the low-level serial drivers used on ARM/ARM64 devices.
5 * The generic serial console code in osfmk/console/serial_console.c will call
6 * into this code to transmit and receive serial data.
7 *
8 * Logging can be performed on multiple serial interfaces at once through a
9 * method called serial multiplexing. This is implemented by enumerating which
10 * serial interfaces are available on boot and registering them into a linked
11 * list of interfaces pointed to by gPESF. When outputting or receiving
12 * characters, each interface is queried in turn.
13 *
14 * Please view doc/arm_serial.md for an in-depth description of these drivers.
15 */
16#include <kern/clock.h>
17#include <kern/debug.h>
18#include <libkern/OSBase.h>
19#include <libkern/section_keywords.h>
20#include <mach/mach_time.h>
21#include <machine/atomic.h>
22#include <machine/machine_routines.h>
23#include <pexpert/pexpert.h>
24#include <pexpert/protos.h>
25#include <pexpert/device_tree.h>
26#include <pexpert/arm/consistent_debug.h>
27#include <pexpert/arm64/board_config.h>
28#include <arm64/proc_reg.h>
29#include <pexpert/arm/protos.h>
30#include <kern/sched_prim.h>
31#if HIBERNATION
32#include <machine/pal_hibernate.h>
33#endif /* HIBERNATION */
34
35/**
36 * Fun and helpful icon to use for announcements regarding serial deprecation,
37 * disablement. This helps catch people's eye to get them to realize the message
38 * is something other than normal serial spew. Please don't remove. :)
39 */
40const char hexley[] =
41 " \n"
42 " %( \n"
43 " #% /((( %(& #((( \n"
44 " *(((( #(((( &(((& %((((((((((&((((((((% \n"
45 " %(. (% *(( %((((((((((%((((((((((((# \n"
46 " *(% (( #(( (((@(%((((((((%((#((((((((. \n"
47 " &(& ((# ((& (( ((((((((((@(%((((((((((% \n"
48 " &((, ((% &((# & (((((/ %(%(((((((((((((((( \n"
49 " &((((&#,%((#&%((((/ @((& &((( (&(((((((#((((((( \n"
50 " %#((((((((%# *## (((((((((@ %@@((&((((((((&((& \n"
51 " ((/ &##&######%( @&((((((((((((((% \n"
52 " ((& ###########&#((((&(((((((((& \n"
53 " #(% /##########%%%(((((((&/ \n"
54 " &(( &%///&@,##&(((((((( \n"
55 " (((((( .#&/##@((( ((((% \n"
56 " #(((%(((, ((((*.....*(((((& (((((% \n"
57 " %(&&(((((((((#.......((((((((( &#%% \n"
58 " ((# (((((((@......../(%(((((((( \n"
59 " %(( (((((@..........((((@(((((( \n"
60 " &(( ............(((((((((, \n"
61 " #((# /............(((((# \n"
62 " ((& .............(((((( \n"
63 " ((# ((...........#((((((% \n"
64 " ((( @(((((.......#(((((((( .@&&@/ \n"
65 " %((. @(((((((#.@((((((((#(@((((((((((#(@ \n"
66 " &((&@##@@((((((((&((((((((@#(((((((((&. \n"
67 " *((& ##&( /##(&#((#(& \n"
68 " %##@&### \n\n";
69
70struct pe_serial_functions {
71 /* Initialize the underlying serial hardware. */
72 void (*init) (void);
73
74 /* Return a non-zero value if the serial interface is ready to send more data. */
75 unsigned int (*transmit_ready) (void);
76
77 /* Write a single byte of data to serial. */
78 void (*transmit_data) (uint8_t c);
79
80 /* Return a non-zero value if there's a byte of data available. */
81 unsigned int (*receive_ready) (void);
82
83 /* Read a single byte from serial. */
84 uint8_t (*receive_data) (void);
85
86 /* Enables IRQs from this device. */
87 void (*enable_irq) (void);
88
89 /* Disables IRQs from this device and reports whether IRQs were enabled. */
90 bool (*disable_irq) (void);
91
92 /* Clears this device's IRQs targeting this agent, returning true if at least one IRQ was cleared. */
93 bool (*acknowledge_irq) (void);
94
95 /**
96 * Whether this serial driver can handle irqs. This value should be set by
97 * querying the device tree to see if the serial device has interrupts
98 * associated with it.
99 *
100 * For a device to support IRQs:
101 * - enable_irq, disable_irq, and acknowledge_irq must be non-null
102 * - The AppleSerialShim kext must be able to match to the serial device
103 * in the IORegistry and call serial_enable_irq with the proper
104 * serial_device_t
105 * - The device tree entry for the serial device should have an interrupt
106 * associated with it.
107 */
108 bool has_irq;
109
110 /* enum identifying which serial device these functions belong to. */
111 serial_device_t device;
112
113 /* Pointer to the next serial interface in the linked-list. */
114 struct pe_serial_functions *next;
115};
116
117MARK_AS_HIBERNATE_DATA_CONST_LATE static struct pe_serial_functions* gPESF = NULL;
118
119/**
120 * Whether uart has been initialized already. This value is kept across a
121 * sleep/wake cycle so we know we need to reinitialize when serial_init is
122 * called again after wake.
123 */
124MARK_AS_HIBERNATE_DATA static bool uart_initted = false;
125
126/* Whether uart should run in simple mode that works during hibernation resume. */
127MARK_AS_HIBERNATE_DATA static bool uart_hibernation = false;
128
129/** Set <=> transmission is authorized.
130 * Always set, unless SERIALMODE_ON_DEMAND is provided at boot,
131 * and no data has yet been received.
132 * Originaly meant to be a per-pe_serial_functions variable,
133 * but the data protection on the structs prevents it. */
134static bool serial_do_transmit = 1;
135
136/**
137 * Used to track if all IRQs have been initialized. Each bit of this variable
138 * represents whether or not a serial device that reports supporting IRQs has
139 * been initialized yet (1 -> not initialized, 0 -> initialized)
140 */
141static uint32_t serial_irq_status = 0;
142
143/**
144 * Set by the 'disable-uart-irq' boot-arg to force serial IRQs into polling mode
145 * by preventing the serial driver shim kext from registering itself with
146 * serial_enable_irq.
147 */
148static bool disable_uart_irq = 0;
149
150/**
151 * Indicates whether or not a given device's irqs have been set up by calling
152 * serial_enable_irq for that particular device.
153 *
154 * @param device_fns Serial functions for the device that is being checked
155 * @return Whether or not the irqs have been initialized for that device
156 */
157static bool
158irq_initialized(struct pe_serial_functions *device_fns)
159{
160 return (serial_irq_status & device_fns->device) == 0;
161}
162
163/**
164 * Indicates whether or not a given device supports irqs and if they are ready
165 * to be used.
166 *
167 * @param device_fns Serial functions for the device that is being checked
168 * @return Whether or not the device can and will send IRQs.
169 */
170static bool
171irq_available_and_ready(struct pe_serial_functions *device_fns)
172{
173 return device_fns->has_irq && irq_initialized(device_fns);
174}
175
176/**
177 * Searches through the global serial functions list and returns the serial function for a particular device
178 *
179 * @param device The device identifier to search for
180 * @return Serial functions for the specified device
181 */
182static struct pe_serial_functions *
183get_serial_functions(serial_device_t device)
184{
185 struct pe_serial_functions *fns = gPESF;
186 while (fns != NULL) {
187 if (fns->device == device) {
188 return fns;
189 }
190 fns = fns->next;
191 }
192 return NULL;
193}
194
195/**
196 * The action to take when polling and waiting for a serial device to be ready
197 * for output. On ARM64, takes a WFE because the WFE timeout will wake us up in
198 * the worst case. On ARMv7 devices, we need to hot poll.
199 */
200static inline void
201serial_poll(void)
202{
203#if __arm64__
204 if (!uart_hibernation) {
205 __builtin_arm_wfe();
206 }
207#endif
208}
209
210/**
211 * This ensures that if we have a future product that supports hibernation, but
212 * doesn't support either UART serial or dock-channels, then hibernation will
213 * gracefully fall back to the serial method that is supported.
214 */
215#if HIBERNATION || defined(APPLE_UART)
216MARK_AS_HIBERNATE_DATA static volatile apple_uart_registers_t *apple_uart_registers = 0;
217#endif /* HIBERNATION || defined(APPLE_UART) */
218
219#if HIBERNATION || defined(DOCKCHANNEL_UART)
220MARK_AS_HIBERNATE_DATA static vm_offset_t dockchannel_uart_base = 0;
221#endif /* HIBERNATION || defined(DOCKCHANNEL_UART) */
222
223/*****************************************************************************/
224
225#ifdef APPLE_UART
226static int32_t dt_sampling = -1;
227static int32_t dt_ubrdiv = -1;
228
229static void apple_uart_set_baud_rate(uint32_t baud_rate);
230
231/**
232 * The Apple UART is configured to use 115200-8-N-1 communication.
233 */
234static void
235apple_uart_init(void)
236{
237 ucon_t ucon = { .raw = 0 };
238 // Use NCLK (which is constant) instead of PCLK (which is variable).
239 ucon.clock_selection = UCON_CLOCK_SELECTION_NCLK;
240 ucon.transmit_mode = UCON_TRANSMIT_MODE_INTERRUPT_OR_POLLING;
241 ucon.receive_mode = UCON_RECEIVE_MODE_INTERRUPT_OR_POLLING;
242 apple_uart_registers->ucon = ucon;
243
244 // Configure 8-N-1 communication.
245 ulcon_t ulcon = { .raw = 0 };
246 ulcon.word_length = ULCON_WORD_LENGTH_8_BITS;
247 ulcon.parity_mode = ULCON_PARITY_MODE_NONE;
248 ulcon.number_of_stop_bits = ULCON_STOP_BITS_1;
249 apple_uart_registers->ulcon = ulcon;
250
251 apple_uart_set_baud_rate(baud_rate: 115200);
252
253 // Enable and reset FIFOs.
254 ufcon_t ufcon = { .raw = 0 };
255 ufcon.fifo_enable = 1;
256 ufcon.tx_fifo_reset = 1;
257 ufcon.rx_fifo_reset = 1;
258 apple_uart_registers->ufcon = ufcon;
259}
260
261static void
262apple_uart_enable_irq(void)
263{
264 // Set the Tx FIFO interrupt trigger level to 0 bytes so interrupts occur when
265 // the Tx FIFO is completely empty; this leads to higher Tx throughput.
266 apple_uart_registers->ufcon.tx_fifo_interrupt_trigger_level_dma_watermark = UFCON_TX_FIFO_ITL_0_BYTES;
267
268 // Enable Tx interrupts.
269 apple_uart_registers->ucon.transmit_interrupt = 1;
270}
271
272static bool
273apple_uart_disable_irq(void)
274{
275 /* Disables Tx interrupts */
276 ucon_t ucon = apple_uart_registers->ucon;
277 const bool irqs_were_enabled = ucon.transmit_interrupt;
278
279 if (irqs_were_enabled) {
280 ucon.transmit_interrupt = 0;
281 apple_uart_registers->ucon = ucon;
282 }
283
284 return irqs_were_enabled;
285}
286
287static bool
288apple_uart_ack_irq(void)
289{
290 apple_uart_registers->utrstat.transmit_interrupt_status = 1;
291 return true;
292}
293
294static inline bool
295apple_uart_fifo_is_empty(void)
296{
297 const ufstat_t ufstat = apple_uart_registers->ufstat;
298 return !(ufstat.tx_fifo_full || ufstat.tx_fifo_count);
299}
300
301static void
302apple_uart_drain_fifo(void)
303{
304 while (!apple_uart_fifo_is_empty()) {
305 serial_poll();
306 }
307}
308
309static void
310apple_uart_set_baud_rate(uint32_t baud_rate)
311{
312 uint32_t div = 0;
313 const uint32_t uart_clock = (uint32_t)gPEClockFrequencyInfo.fix_frequency_hz;
314 uint32_t sample_rate = 16;
315
316 if (baud_rate < 300) {
317 baud_rate = 9600;
318 }
319
320 if (dt_sampling != -1) {
321 // Use the sampling rate specified in the Device Tree
322 sample_rate = dt_sampling & 0xf;
323 }
324
325 if (dt_ubrdiv != -1) {
326 // Use the ubrdiv specified in the Device Tree
327 div = dt_ubrdiv & 0xffff;
328 } else {
329 // Calculate ubrdiv. UBRDIV = (SourceClock / (BPS * Sample Rate)) - 1
330 div = uart_clock / (baud_rate * sample_rate);
331
332 uint32_t actual_baud = uart_clock / ((div + 0) * sample_rate);
333 uint32_t baud_low = uart_clock / ((div + 1) * sample_rate);
334
335 // Adjust div to get the closest target baudrate
336 if ((baud_rate - baud_low) > (actual_baud - baud_rate)) {
337 div--;
338 }
339 }
340
341 ubrdiv_t ubrdiv = apple_uart_registers->ubrdiv;
342 ubrdiv.sample_rate = 16 - sample_rate;
343 ubrdiv.ubr_div = div;
344 apple_uart_registers->ubrdiv = ubrdiv;
345}
346
347MARK_AS_HIBERNATE_TEXT static unsigned int
348apple_uart_transmit_ready(void)
349{
350 return !apple_uart_registers->ufstat.tx_fifo_full;
351}
352
353MARK_AS_HIBERNATE_TEXT static void
354apple_uart_transmit_data(uint8_t c)
355{
356 apple_uart_registers->utxh.txdata = c;
357}
358
359static unsigned int
360apple_uart_receive_ready(void)
361{
362 const ufstat_t ufstat = apple_uart_registers->ufstat;
363 return ufstat.rx_fifo_full || ufstat.rx_fifo_count;
364}
365
366static uint8_t
367apple_uart_receive_data(void)
368{
369 return apple_uart_registers->urxh.rxdata;
370}
371
372MARK_AS_HIBERNATE_DATA_CONST_LATE
373static struct pe_serial_functions apple_serial_functions =
374{
375 .init = apple_uart_init,
376 .transmit_ready = apple_uart_transmit_ready,
377 .transmit_data = apple_uart_transmit_data,
378 .receive_ready = apple_uart_receive_ready,
379 .receive_data = apple_uart_receive_data,
380 .enable_irq = apple_uart_enable_irq,
381 .disable_irq = apple_uart_disable_irq,
382 .acknowledge_irq = apple_uart_ack_irq,
383 .device = SERIAL_APPLE_UART
384};
385
386#endif /* APPLE_UART */
387
388/*****************************************************************************/
389
390#ifdef DOCKCHANNEL_UART
391#define DOCKCHANNEL_WR_MAX_STALL_US (30*1000)
392
393static vm_offset_t dock_agent_base;
394static uint32_t max_dockchannel_drain_period;
395static uint64_t dockchannel_drain_deadline; // Deadline for external agent to drain before a software drain occurs
396static bool use_sw_drain;
397static uint32_t dock_wstat_mask;
398static uint64_t prev_dockchannel_spaces; // Previous w_stat level of the DockChannel.
399static uint64_t dockchannel_stall_grace;
400MARK_AS_HIBERNATE_DATA static bool use_sw_drain;
401MARK_AS_HIBERNATE_DATA static uint32_t dock_wstat_mask;
402
403// forward reference
404static struct pe_serial_functions dockchannel_serial_functions;
405
406//=======================
407// Local funtions
408//=======================
409
410static int
411dockchannel_drain_on_stall()
412{
413 // Called when DockChannel runs out of spaces.
414 // Check if the DockChannel reader has stalled. If so, empty the DockChannel ourselves.
415 // Return number of bytes drained.
416
417 if (mach_absolute_time() >= dockchannel_drain_deadline) {
418 // It's been more than DOCKCHANEL_WR_MAX_STALL_US and nobody read from the FIFO
419 // Drop a character.
420 (void)rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
421 os_atomic_inc(&prev_dockchannel_spaces, relaxed);
422 return 1;
423 }
424 return 0;
425}
426
427static void
428dockchannel_clear_intr(void)
429{
430 rDOCKCHANNELS_AGENT_AP_INTR_CTRL &= ~(0x3);
431 rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x3;
432 rDOCKCHANNELS_AGENT_AP_ERR_INTR_CTRL &= ~(0x3);
433 rDOCKCHANNELS_AGENT_AP_ERR_INTR_STATUS |= 0x3;
434}
435
436static bool
437dockchannel_disable_irq(void)
438{
439 const uint32_t ap_intr_ctrl = rDOCKCHANNELS_AGENT_AP_INTR_CTRL;
440 const bool irqs_were_enabled = ap_intr_ctrl & 0x1;
441 if (irqs_were_enabled) {
442 rDOCKCHANNELS_AGENT_AP_INTR_CTRL = ap_intr_ctrl & ~(0x1);
443 }
444 return irqs_were_enabled;
445}
446
447static void
448dockchannel_enable_irq(void)
449{
450 // set interrupt to be when fifo has 255 empty
451 rDOCKCHANNELS_DEV_WR_WATERMARK(DOCKCHANNEL_UART_CHANNEL) = 0xFF;
452 rDOCKCHANNELS_AGENT_AP_INTR_CTRL |= 0x1;
453}
454
455static bool
456dockchannel_ack_irq(void)
457{
458 /* First check if the IRQ is for the kernel */
459 if (rDOCKCHANNELS_AGENT_AP_INTR_STATUS & 0x1) {
460 rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x1;
461 return true;
462 }
463 return false;
464}
465
466MARK_AS_HIBERNATE_TEXT static void
467dockchannel_transmit_data(uint8_t c)
468{
469 rDOCKCHANNELS_DEV_WDATA1(DOCKCHANNEL_UART_CHANNEL) = (unsigned)c;
470
471 if (use_sw_drain && !uart_hibernation) {
472 os_atomic_dec(&prev_dockchannel_spaces, relaxed); // After writing a byte we have one fewer space than previously expected.
473 }
474}
475
476static unsigned int
477dockchannel_receive_ready(void)
478{
479 return rDOCKCHANNELS_DEV_RDATA0(DOCKCHANNEL_UART_CHANNEL) & 0x7f;
480}
481
482static uint8_t
483dockchannel_receive_data(void)
484{
485 return (uint8_t)((rDOCKCHANNELS_DEV_RDATA1(DOCKCHANNEL_UART_CHANNEL) >> 8) & 0xff);
486}
487
488MARK_AS_HIBERNATE_TEXT static unsigned int
489dockchannel_transmit_ready(void)
490{
491 uint32_t spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;
492
493 if (!uart_hibernation) {
494 if (use_sw_drain) {
495 if (spaces > prev_dockchannel_spaces) {
496 // More spaces showed up. That can only mean someone read the FIFO.
497 // Note that if the DockFIFO is empty we cannot tell if someone is listening,
498 // we can only give them the benefit of the doubt.
499 dockchannel_drain_deadline = mach_absolute_time() + dockchannel_stall_grace;
500 }
501 prev_dockchannel_spaces = spaces;
502 return spaces || dockchannel_drain_on_stall();
503 }
504 }
505
506 return spaces;
507}
508
509static void
510dockchannel_init(void)
511{
512 if (use_sw_drain) {
513 nanoseconds_to_absolutetime(DOCKCHANNEL_WR_MAX_STALL_US * NSEC_PER_USEC, &dockchannel_stall_grace);
514 }
515
516 // Clear all interrupt enable and status bits
517 dockchannel_clear_intr();
518
519 // Setup DRAIN timer
520 rDOCKCHANNELS_DEV_DRAIN_CFG(DOCKCHANNEL_UART_CHANNEL) = max_dockchannel_drain_period;
521
522 // Drain timer doesn't get loaded with value from drain period register if fifo
523 // is already full. Drop a character from the fifo.
524 rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
525}
526
527MARK_AS_HIBERNATE_DATA_CONST_LATE
528static struct pe_serial_functions dockchannel_serial_functions =
529{
530 .init = dockchannel_init,
531 .transmit_ready = dockchannel_transmit_ready,
532 .transmit_data = dockchannel_transmit_data,
533 .receive_ready = dockchannel_receive_ready,
534 .receive_data = dockchannel_receive_data,
535 .enable_irq = dockchannel_enable_irq,
536 .disable_irq = dockchannel_disable_irq,
537 .acknowledge_irq = dockchannel_ack_irq,
538 .device = SERIAL_DOCKCHANNEL
539};
540
541#endif /* DOCKCHANNEL_UART */
542
543/****************************************************************************/
544#ifdef PI3_UART
545vm_offset_t pi3_gpio_base_vaddr = 0;
546vm_offset_t pi3_aux_base_vaddr = 0;
547static unsigned int
548pi3_uart_tr0(void)
549{
550 return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x20;
551}
552
553static void
554pi3_uart_td0(uint8_t c)
555{
556 BCM2837_PUT32(BCM2837_AUX_MU_IO_REG_V, (uint32_t) c);
557}
558
559static unsigned int
560pi3_uart_rr0(void)
561{
562 return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x01;
563}
564
565static uint8_t
566pi3_uart_rd0(void)
567{
568 return (uint8_t) BCM2837_GET32(BCM2837_AUX_MU_IO_REG_V);
569}
570
571static void
572pi3_uart_init(void)
573{
574 // Scratch variable
575 uint32_t i;
576
577 // Reset mini uart registers
578 BCM2837_PUT32(BCM2837_AUX_ENABLES_V, 1);
579 BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 0);
580 BCM2837_PUT32(BCM2837_AUX_MU_LCR_REG_V, 3);
581 BCM2837_PUT32(BCM2837_AUX_MU_MCR_REG_V, 0);
582 BCM2837_PUT32(BCM2837_AUX_MU_IER_REG_V, 0);
583 BCM2837_PUT32(BCM2837_AUX_MU_IIR_REG_V, 0xC6);
584 BCM2837_PUT32(BCM2837_AUX_MU_BAUD_REG_V, 270);
585
586 i = (uint32_t)BCM2837_FSEL_REG(14);
587 // Configure GPIOs 14 & 15 for alternate function 5
588 i &= ~(BCM2837_FSEL_MASK(14));
589 i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(14));
590 i &= ~(BCM2837_FSEL_MASK(15));
591 i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(15));
592
593 BCM2837_PUT32(BCM2837_FSEL_REG(14), i);
594
595 BCM2837_PUT32(BCM2837_GPPUD_V, 0);
596
597 // Barrier before AP spinning for 150 cycles
598 __builtin_arm_isb(ISB_SY);
599
600 for (i = 0; i < 150; i++) {
601 asm volatile ("add x0, x0, xzr");
602 }
603
604 __builtin_arm_isb(ISB_SY);
605
606 BCM2837_PUT32(BCM2837_GPPUDCLK0_V, (1 << 14) | (1 << 15));
607
608 __builtin_arm_isb(ISB_SY);
609
610 for (i = 0; i < 150; i++) {
611 asm volatile ("add x0, x0, xzr");
612 }
613
614 __builtin_arm_isb(ISB_SY);
615
616 BCM2837_PUT32(BCM2837_GPPUDCLK0_V, 0);
617
618 BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 3);
619}
620
621SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) pi3_uart_serial_functions =
622{
623 .init = pi3_uart_init,
624 .transmit_ready = pi3_uart_tr0,
625 .transmit_data = pi3_uart_td0,
626 .receive_ready = pi3_uart_rr0,
627 .receive_data = pi3_uart_rd0,
628 .device = SERIAL_PI3_UART
629};
630
631#endif /* PI3_UART */
632
633/*****************************************************************************/
634
635#ifdef VMAPPLE_UART
636
637static vm_offset_t vmapple_uart0_base_vaddr = 0;
638
639#define PL011_LCR_WORD_LENGTH_8 0x60u
640#define PL011_LCR_FIFO_DISABLE 0x00u
641
642#define PL011_LCR_FIFO_ENABLE 0x10u
643
644#define PL011_LCR_ONE_STOP_BIT 0x00u
645#define PL011_LCR_PARITY_DISABLE 0x00u
646#define PL011_LCR_BREAK_DISABLE 0x00u
647#define PL011_IBRD_DIV_38400 0x27u
648#define PL011_FBRD_DIV_38400 0x09u
649#define PL011_ICR_CLR_ALL_IRQS 0x07ffu
650#define PL011_CR_UART_ENABLE 0x01u
651#define PL011_CR_TX_ENABLE 0x100u
652#define PL011_CR_RX_ENABLE 0x200u
653
654#define VMAPPLE_UART0_DR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x00))
655#define VMAPPLE_UART0_ECR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x04))
656#define VMAPPLE_UART0_FR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x18))
657#define VMAPPLE_UART0_IBRD *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x24))
658#define VMAPPLE_UART0_FBRD *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x28))
659#define VMAPPLE_UART0_LCR_H *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x2c))
660#define VMAPPLE_UART0_CR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x30))
661#define VMAPPLE_UART0_TIMSC *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x38))
662#define VMAPPLE_UART0_ICR *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x44))
663
664static unsigned int
665vmapple_uart_transmit_ready(void)
666{
667 return (unsigned int) !(VMAPPLE_UART0_FR & 0x20);
668}
669
670static void
671vmapple_uart_transmit_data(uint8_t c)
672{
673 VMAPPLE_UART0_DR = (uint32_t) c;
674}
675
676static unsigned int
677vmapple_uart_receive_ready(void)
678{
679 return (unsigned int) !(VMAPPLE_UART0_FR & 0x10);
680}
681
682static uint8_t
683vmapple_uart_receive_data(void)
684{
685 return (uint8_t) (VMAPPLE_UART0_DR & 0xff);
686}
687
688static void
689vmapple_uart_init(void)
690{
691 VMAPPLE_UART0_CR = 0x0;
692 VMAPPLE_UART0_ECR = 0x0;
693 VMAPPLE_UART0_LCR_H = (
694 PL011_LCR_WORD_LENGTH_8 |
695 PL011_LCR_FIFO_ENABLE |
696 PL011_LCR_ONE_STOP_BIT |
697 PL011_LCR_PARITY_DISABLE |
698 PL011_LCR_BREAK_DISABLE
699 );
700 VMAPPLE_UART0_IBRD = PL011_IBRD_DIV_38400;
701 VMAPPLE_UART0_FBRD = PL011_FBRD_DIV_38400;
702 VMAPPLE_UART0_TIMSC = 0x0;
703 VMAPPLE_UART0_ICR = PL011_ICR_CLR_ALL_IRQS;
704 VMAPPLE_UART0_CR = (
705 PL011_CR_UART_ENABLE |
706 PL011_CR_TX_ENABLE |
707 PL011_CR_RX_ENABLE
708 );
709}
710
711SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) vmapple_uart_serial_functions =
712{
713 .init = vmapple_uart_init,
714 .transmit_ready = vmapple_uart_transmit_ready,
715 .transmit_data = vmapple_uart_transmit_data,
716 .receive_ready = vmapple_uart_receive_ready,
717 .receive_data = vmapple_uart_receive_data,
718 .device = SERIAL_VMAPPLE_UART
719};
720
721#endif /* VMAPPLE_UART */
722
723/*****************************************************************************/
724
725/**
726 * Output @str onto every registered serial interface by polling.
727 *
728 * @param str The string to output.
729 */
730static void uart_puts_force_poll(
731 const char *str);
732
733/**
734 * Output @str onto a specific serial interface by polling.
735 *
736 * @param str The string to output.
737 * @param fns The functions to use to output the message.
738 */
739static void uart_puts_force_poll_device(
740 const char *str,
741 struct pe_serial_functions *fns);
742
743static void
744register_serial_functions(struct pe_serial_functions *fns)
745{
746 fns->next = gPESF;
747 gPESF = fns;
748}
749
750#if HIBERNATION
751/**
752 * Transitions the serial driver into a mode that can be run in the hibernation
753 * resume context. In this mode, the serial driver runs at a barebones level
754 * without making sure the serial devices are properly initialized or utilizing
755 * features such as the software drain timer for dockchannels.
756 *
757 * Upon the next call to serial_init (once the hibernation image has been
758 * loaded), this mode is exited and we return to the normal operation of the
759 * driver.
760 */
761MARK_AS_HIBERNATE_TEXT void
762serial_hibernation_init(void)
763{
764 uart_hibernation = true;
765#if defined(APPLE_UART)
766 apple_uart_registers = (apple_uart_registers_t *)gHibernateGlobals.hibUartRegPhysBase;
767#endif /* defined(APPLE_UART) */
768#if defined(DOCKCHANNEL_UART)
769 dockchannel_uart_base = gHibernateGlobals.dockChannelRegPhysBase;
770#endif /* defined(DOCKCHANNEL_UART) */
771}
772
773/**
774 * Transitions the serial driver back to non-hibernation mode so it can resume
775 * normal operations. Should only be called from serial_init on a hibernation
776 * resume.
777 */
778MARK_AS_HIBERNATE_TEXT static void
779serial_hibernation_cleanup(void)
780{
781 uart_hibernation = false;
782#if defined(APPLE_UART)
783 apple_uart_registers = (apple_uart_registers_t *)gHibernateGlobals.hibUartRegVirtBase;
784#endif /* defined(APPLE_UART) */
785#if defined(DOCKCHANNEL_UART)
786 dockchannel_uart_base = gHibernateGlobals.dockChannelRegVirtBase;
787#endif /* defined(DOCKCHANNEL_UART) */
788}
789#endif /* HIBERNATION */
790
791int
792serial_init(void)
793{
794 DTEntry entryP = NULL;
795 uint32_t prop_size;
796 vm_offset_t soc_base;
797 uintptr_t const *reg_prop;
798 uint32_t const *prop_value __unused = NULL;
799
800 struct pe_serial_functions *fns = gPESF;
801
802 /**
803 * Even if the serial devices have already been initialized on cold boot,
804 * when coming out of a sleep/wake, they'll need to be re-initialized. Since
805 * the uart_initted value is kept across a sleep/wake, always re-initialize
806 * to be safe.
807 */
808 if (uart_initted) {
809#if HIBERNATION
810 if (uart_hibernation) {
811 serial_hibernation_cleanup();
812 }
813#endif /* HIBERNATION */
814 while (fns != NULL) {
815 fns->init();
816 fns = fns->next;
817 }
818
819 return gPESF != NULL;
820 }
821
822 soc_base = pe_arm_get_soc_base_phys();
823
824 if (soc_base == 0) {
825 uart_initted = true;
826 return 0;
827 }
828
829 PE_parse_boot_argn(arg_string: "disable-uart-irq", arg_ptr: &disable_uart_irq, max_arg: sizeof(disable_uart_irq));
830
831#ifdef PI3_UART
832 if (SecureDTFindEntry("name", "gpio", &entryP) == kSuccess) {
833 SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
834 pi3_gpio_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
835 }
836 if (SecureDTFindEntry("name", "aux", &entryP) == kSuccess) {
837 SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
838 pi3_aux_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
839 }
840 if ((pi3_gpio_base_vaddr != 0) && (pi3_aux_base_vaddr != 0)) {
841 register_serial_functions(&pi3_uart_serial_functions);
842 }
843#endif /* PI3_UART */
844
845#ifdef VMAPPLE_UART
846 if (SecureDTFindEntry(propName: "name", propValue: "uart0", entryH: &entryP) == kSuccess) {
847 SecureDTGetProperty(entry: entryP, propertyName: "reg", propertyValue: (void const **)&reg_prop, propertySize: &prop_size);
848 vmapple_uart0_base_vaddr = ml_io_map(phys_addr: soc_base + *reg_prop, size: *(reg_prop + 1));
849 }
850
851 if (vmapple_uart0_base_vaddr != 0) {
852 register_serial_functions(fns: &vmapple_uart_serial_functions);
853 }
854#endif /* VMAPPLE_UART */
855
856#ifdef DOCKCHANNEL_UART
857 uint32_t no_dockchannel_uart = 0;
858 if (SecureDTFindEntry("name", "dockchannel-uart", &entryP) == kSuccess) {
859 SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
860 // Should be two reg entries
861 if (prop_size / sizeof(uintptr_t) != 4) {
862 panic("Malformed dockchannel-uart property");
863 }
864 dockchannel_uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
865 dock_agent_base = ml_io_map(soc_base + *(reg_prop + 2), *(reg_prop + 3));
866 PE_parse_boot_argn("no-dockfifo-uart", &no_dockchannel_uart, sizeof(no_dockchannel_uart));
867 // Keep the old name for boot-arg
868 if (no_dockchannel_uart == 0) {
869 register_serial_functions(&dockchannel_serial_functions);
870 SecureDTGetProperty(entryP, "max-aop-clk", (void const **)&prop_value, &prop_size);
871 max_dockchannel_drain_period = (uint32_t)((prop_value)? (*prop_value * 0.03) : DOCKCHANNEL_DRAIN_PERIOD);
872 prop_value = NULL;
873 SecureDTGetProperty(entryP, "enable-sw-drain", (void const **)&prop_value, &prop_size);
874 use_sw_drain = (prop_value)? *prop_value : 0;
875 prop_value = NULL;
876 SecureDTGetProperty(entryP, "dock-wstat-mask", (void const **)&prop_value, &prop_size);
877 dock_wstat_mask = (prop_value)? *prop_value : 0x1ff;
878 prop_value = NULL;
879 SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
880 if (prop_value) {
881 dockchannel_serial_functions.has_irq = true;
882 }
883
884 /* Set sane defaults for dockchannel globals. */
885 prev_dockchannel_spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;
886 dockchannel_drain_deadline = mach_absolute_time() + dockchannel_stall_grace;
887 } else {
888 dockchannel_clear_intr();
889 }
890 // If no dockchannel-uart is found in the device tree, fall back
891 // to looking for the traditional UART serial console.
892 }
893
894#endif /* DOCKCHANNEL_UART */
895
896#ifdef APPLE_UART
897 char const *serial_compat = 0;
898 uint32_t use_legacy_uart = 1;
899
900 /*
901 * The boot serial port should have a property named "boot-console".
902 * If we don't find it there, look for "uart0" and "uart1".
903 */
904 if (SecureDTFindEntry(propName: "boot-console", NULL, entryH: &entryP) == kSuccess) {
905 SecureDTGetProperty(entry: entryP, propertyName: "reg", propertyValue: (void const **)&reg_prop, propertySize: &prop_size);
906 apple_uart_registers = (apple_uart_registers_t *)ml_io_map(phys_addr: soc_base + *reg_prop, size: *(reg_prop + 1));
907 SecureDTGetProperty(entry: entryP, propertyName: "compatible", propertyValue: (void const **)&serial_compat, propertySize: &prop_size);
908 } else if (SecureDTFindEntry(propName: "name", propValue: "uart0", entryH: &entryP) == kSuccess) {
909 SecureDTGetProperty(entry: entryP, propertyName: "reg", propertyValue: (void const **)&reg_prop, propertySize: &prop_size);
910 apple_uart_registers = (apple_uart_registers_t *)ml_io_map(phys_addr: soc_base + *reg_prop, size: *(reg_prop + 1));
911 SecureDTGetProperty(entry: entryP, propertyName: "compatible", propertyValue: (void const **)&serial_compat, propertySize: &prop_size);
912 } else if (SecureDTFindEntry(propName: "name", propValue: "uart1", entryH: &entryP) == kSuccess) {
913 SecureDTGetProperty(entry: entryP, propertyName: "reg", propertyValue: (void const **)&reg_prop, propertySize: &prop_size);
914 apple_uart_registers = (apple_uart_registers_t *)ml_io_map(phys_addr: soc_base + *reg_prop, size: *(reg_prop + 1));
915 SecureDTGetProperty(entry: entryP, propertyName: "compatible", propertyValue: (void const **)&serial_compat, propertySize: &prop_size);
916 }
917
918 if (NULL != entryP) {
919 prop_value = NULL;
920 SecureDTGetProperty(entry: entryP, propertyName: "sampling", propertyValue: (void const **)&prop_value, propertySize: &prop_size);
921 if (prop_value) {
922 dt_sampling = *prop_value;
923 }
924
925 prop_value = NULL;
926 SecureDTGetProperty(entry: entryP, propertyName: "ubrdiv", propertyValue: (void const **)&prop_value, propertySize: &prop_size);
927 if (prop_value) {
928 dt_ubrdiv = *prop_value;
929 }
930
931 prop_value = NULL;
932 SecureDTGetProperty(entry: entryP, propertyName: "interrupts", propertyValue: (void const **)&prop_value, propertySize: &prop_size);
933 if (prop_value) {
934 apple_serial_functions.has_irq = true;
935 }
936
937 prop_value = NULL;
938 SecureDTGetProperty(entry: entryP, propertyName: "disable-legacy-uart", propertyValue: (void const **)&prop_value, propertySize: &prop_size);
939 if (prop_value) {
940 use_legacy_uart = 0;
941 }
942 }
943
944 /* Check if we should enable this deprecated serial device. */
945 PE_parse_boot_argn(arg_string: "use-legacy-uart", arg_ptr: &use_legacy_uart, max_arg: sizeof(use_legacy_uart));
946
947 if (serial_compat && !strcmp(s1: serial_compat, s2: "uart-1,samsung")) {
948 if (use_legacy_uart) {
949 register_serial_functions(fns: &apple_serial_functions);
950 } else {
951 char legacy_serial_msg[] =
952 "Expecting more output? Wondering why Clippy turned into a platypus?\n\n"
953 "UART was manually disabled either via the 'use-legacy-uart=0' boot-arg or via\n"
954 "the disable-legacy-uart property in the boot-console uart device tree node.\n"
955 "Serial output over dockchannels is still enabled on devices with support.\n";
956 apple_serial_functions.init();
957 uart_puts_force_poll_device(str: hexley, fns: &apple_serial_functions);
958 uart_puts_force_poll_device(str: legacy_serial_msg, fns: &apple_serial_functions);
959 }
960 }
961#endif /* APPLE_UART */
962
963 fns = gPESF;
964 while (fns != NULL) {
965 serial_do_transmit = 1;
966 fns->init();
967 if (fns->has_irq) {
968 serial_irq_status |= fns->device; // serial_device_t is one-hot
969 }
970 fns = fns->next;
971 }
972
973#if HIBERNATION
974 /* hibernation needs to know the UART register addresses since it can't directly use this serial driver */
975 if (dockchannel_uart_base) {
976 gHibernateGlobals.dockChannelRegPhysBase = ml_vtophys(dockchannel_uart_base);
977 gHibernateGlobals.dockChannelRegVirtBase = dockchannel_uart_base;
978 gHibernateGlobals.dockChannelWstatMask = dock_wstat_mask;
979 }
980 if (apple_uart_registers) {
981 gHibernateGlobals.hibUartRegPhysBase = ml_vtophys((vm_offset_t)apple_uart_registers);
982 gHibernateGlobals.hibUartRegVirtBase = (vm_offset_t)apple_uart_registers;
983 }
984#endif /* HIBERNATION */
985
986 /* Complete. */
987 uart_initted = true;
988 return gPESF != NULL;
989}
990
991/**
992 * Forbid or allow transmission over each serial until they receive data.
993 */
994void
995serial_set_on_demand(bool on_demand)
996{
997 /* Enable or disable transmission. */
998 serial_do_transmit = !on_demand;
999
1000 /* If on-demand is enabled, report it. */
1001 if (on_demand) {
1002 uart_puts_force_poll(
1003 str: "On-demand serial mode selected.\n"
1004 "Waiting for user input to send logs.\n"
1005 );
1006 }
1007}
1008
1009/**
1010 * Returns a deadline for the longest time the serial driver should wait for an
1011 * interrupt for. This serves as a timeout for the IRQ to allow for the software
1012 * drain timer that dockchannels supports.
1013 *
1014 * @param fns serial functions representing the device to find the deadline for
1015 *
1016 * @returns absolutetime deadline for this device's IRQ.
1017 */
1018static uint64_t
1019serial_interrupt_deadline(__unused struct pe_serial_functions *fns)
1020{
1021#if defined(DOCKCHANNEL_UART)
1022 if (fns->device == SERIAL_DOCKCHANNEL && use_sw_drain) {
1023 return dockchannel_drain_deadline;
1024 }
1025#endif
1026
1027 /**
1028 * Default to 1.5ms for all other devices. 1.5ms was chosen as the baudrate
1029 * of the AppleSerialDevice is 115200, meaning that it should only take
1030 * ~1.5ms to drain the 16 character buffer completely.
1031 */
1032 uint64_t timeout_interval;
1033 nanoseconds_to_absolutetime(nanoseconds: 1500 * NSEC_PER_USEC, result: &timeout_interval);
1034 return mach_absolute_time() + timeout_interval;
1035}
1036
1037/**
1038 * Goes to sleep waiting for an interrupt from a specificed serial device.
1039 *
1040 * @param fns serial functions representing the device to wait for
1041 */
1042static void
1043serial_wait_for_interrupt(struct pe_serial_functions *fns)
1044{
1045 /**
1046 * This block of code is set up to avoid a race condition in which the IRQ
1047 * is transmitted and processed by IOKit in between the time we check if the
1048 * device is ready to transmit and when we call thread_block. If the IRQ
1049 * fires in that time, thread_wakeup may have already been called in which
1050 * case we would be blocking and have nothing to wake us up.
1051 *
1052 * To avoid this issue, we first call assert_wait_deadline, which prepares
1053 * the thread to be blocked, but does not actually block the thread. After
1054 * this point, any call to thread_wakeup from IRQ handler will prevent
1055 * thread_block from actually blocking. As a performance optimization, we
1056 * then double check if the device is ready to transmit and if it is, then
1057 * we cancel the wait and just continue normally.
1058 */
1059 assert_wait_deadline(event: fns, THREAD_UNINT, deadline: serial_interrupt_deadline(fns));
1060 if (!fns->transmit_ready()) {
1061 fns->enable_irq();
1062 thread_block(THREAD_CONTINUE_NULL);
1063 } else {
1064 clear_wait(thread: current_thread(), THREAD_AWAKENED);
1065 }
1066}
1067
1068/**
1069 * Transmit a character over the specified serial output device.
1070 *
1071 * @param c Character to send
1072 * @param poll Whether we should poll or wait for an interrupt.
1073 * @param force Whether we should force this over the device if output has not been enabled yet.
1074 * @param fns Functions for the device to output over.
1075 */
1076static inline void
1077uart_putc_device(char c, bool poll, bool force, struct pe_serial_functions *fns)
1078{
1079 if (!(serial_do_transmit || force)) {
1080 return;
1081 }
1082
1083 while (!fns->transmit_ready()) {
1084 if (irq_available_and_ready(device_fns: fns) && !poll) {
1085 serial_wait_for_interrupt(fns);
1086 } else {
1087 serial_poll();
1088 }
1089 }
1090 fns->transmit_data((uint8_t)c);
1091}
1092
1093/**
1094 * Output a character onto every registered serial interface whose
1095 * transmission is enabled..
1096 *
1097 * @param c The character to output.
1098 * @param poll Whether the driver should poll to send the character or if it can
1099 * wait for an interrupt
1100 */
1101MARK_AS_HIBERNATE_TEXT void
1102uart_putc_options(char c, bool poll)
1103{
1104 struct pe_serial_functions *fns = gPESF;
1105
1106 while (fns != NULL) {
1107 uart_putc_device(c, poll, false, fns);
1108 fns = fns->next;
1109 }
1110}
1111
1112/**
1113 * Output a character onto every registered serial interface whose
1114 * transmission is enabled by polling.
1115 *
1116 * @param c The character to output.
1117 */
1118void
1119uart_putc(char c)
1120{
1121 uart_putc_options(c, true);
1122}
1123
1124/**
1125 * Output @str onto every registered serial interface by polling.
1126 *
1127 * @param str The string to output.
1128 */
1129static void
1130uart_puts_force_poll(
1131 const char *str)
1132{
1133 struct pe_serial_functions *fns = gPESF;
1134 while (fns != NULL) {
1135 uart_puts_force_poll_device(str, fns);
1136 fns = fns->next;
1137 }
1138}
1139
1140/**
1141 * Output @str onto a specific serial interface by polling.
1142 *
1143 * @param str The string to output.
1144 * @param fns The functions to use to output the message.
1145 */
1146static void
1147uart_puts_force_poll_device(
1148 const char *str,
1149 struct pe_serial_functions *fns)
1150{
1151 char c;
1152 while ((c = *(str++))) {
1153 uart_putc_device(c, true, true, fns);
1154 }
1155}
1156
1157/**
1158 * Read a character from the first registered serial interface that has data
1159 * available.
1160 *
1161 * @return The character if any interfaces have data available, otherwise -1.
1162 */
1163int
1164uart_getc(void)
1165{
1166 struct pe_serial_functions *fns = gPESF;
1167 while (fns != NULL) {
1168 if (fns->receive_ready()) {
1169 serial_do_transmit = 1;
1170 return (int)fns->receive_data();
1171 }
1172 fns = fns->next;
1173 }
1174 return -1;
1175}
1176
1177/**
1178 * Enables IRQs for a specific serial device and returns whether or not IRQs for
1179 * that device where enabled successfully. For a serial driver to have irqs
1180 * enabled, it must have the enable_irq, disable_irq, and acknowledge_irq
1181 * functions defined and the has_irq flag set.
1182 *
1183 * @param device Serial device to enable irqs on
1184 * @note This function should only be called from the AppleSerialShim kext
1185 */
1186kern_return_t
1187serial_irq_enable(serial_device_t device)
1188{
1189 struct pe_serial_functions *fns = get_serial_functions(device);
1190
1191 if (!fns || !fns->has_irq || disable_uart_irq) {
1192 return KERN_FAILURE;
1193 }
1194
1195 serial_irq_status &= ~device;
1196
1197 return KERN_SUCCESS;
1198}
1199
1200/**
1201 * Performs any actions needed to handle this IRQ. Wakes up the thread waiting
1202 * on the interrupt if one exists.
1203 *
1204 * @param device Serial device that generated the IRQ.
1205 * @note Interrupts will have already been cleared and disabled by serial_irq_filter.
1206 * @note This function should only be called from the AppleSerialShim kext.
1207 */
1208kern_return_t
1209serial_irq_action(serial_device_t device)
1210{
1211 struct pe_serial_functions *fns = get_serial_functions(device);
1212
1213 if (!fns || !fns->has_irq) {
1214 return KERN_FAILURE;
1215 }
1216
1217 /**
1218 * Because IRQs are enabled only when we know a thread is about to sleep, we
1219 * can call wake up and reasonably expect there to be a thread waiting.
1220 */
1221 thread_wakeup(fns);
1222
1223 return KERN_SUCCESS;
1224}
1225
1226/**
1227 * Returns true if the pending IRQ for device is one that can be handled by the
1228 * platform serial driver.
1229 *
1230 * @param device Serial device that generated the IRQ.
1231 * @note This function is called from a primary interrupt context and should be
1232 * kept lightweight.
1233 * @note This function should only be called from the AppleSerialShim kext
1234 */
1235bool
1236serial_irq_filter(serial_device_t device)
1237{
1238 struct pe_serial_functions *fns = get_serial_functions(device);
1239
1240 if (!fns || !fns->has_irq) {
1241 return false;
1242 }
1243
1244 /**
1245 * Disable IRQs until next time a thread waits for an interrupt to prevent an interrupt storm.
1246 */
1247 const bool had_irqs_enabled = fns->disable_irq();
1248 const bool was_our_interrupt = fns->acknowledge_irq();
1249
1250 /* Re-enable IRQs if the interrupt wasn't for us. */
1251 if (had_irqs_enabled && !was_our_interrupt) {
1252 fns->enable_irq();
1253 }
1254
1255 return was_our_interrupt;
1256}
1257
1258/**
1259 * Prepares all serial devices to go to sleep by draining the hardware FIFOs
1260 * and disabling interrupts.
1261 */
1262void
1263serial_go_to_sleep(void)
1264{
1265 struct pe_serial_functions *fns = gPESF;
1266 while (fns != NULL) {
1267 if (irq_available_and_ready(device_fns: fns)) {
1268 fns->disable_irq();
1269 }
1270 fns = fns->next;
1271 }
1272
1273#ifdef APPLE_UART
1274 /* APPLE_UART needs to drain FIFO before sleeping */
1275 if (get_serial_functions(device: SERIAL_APPLE_UART)) {
1276 apple_uart_drain_fifo();
1277 }
1278#endif /* APPLE_UART */
1279}
1280