1/*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 */
4
5#include <stdint.h>
6#include <arm64/proc_reg.h>
7#include <kern/clock.h>
8#include <mach/mach_time.h>
9#include <machine/atomic.h>
10#include <machine/machine_routines.h>
11#include <pexpert/device_tree.h>
12#include <pexpert/arm64/board_config.h>
13
14
15#if HAS_GIC_V3
16#define GICR_WAKE_TIMEOUT_NS (1000000000ULL) // timeout for redistributor wakeup (default 1s)
17MACHINE_TIMEOUT(gicr_wake_timeout_ns, "gicr-wake-timeout", GICR_WAKE_TIMEOUT_NS, MACHINE_TIMEOUT_UNIT_NSEC, NULL);
18
19static vm_offset_t gicd_base;
20static vm_offset_t gicr_base;
21static vm_offset_t gicr_size;
22
23static uint32_t
24_gic_read32(vm_offset_t addr)
25{
26 return *((volatile uint32_t *) addr);
27}
28
29static uint64_t
30_gic_read64(vm_offset_t addr)
31{
32 return *((volatile uint64_t *) addr);
33}
34
35static void
36_gic_write32(vm_offset_t addr, uint32_t value)
37{
38 *((volatile uint32_t *) addr) = value;
39}
40
41#define gicd_read32(offset) (_gic_read32(gicd_base + (offset)))
42#define gicd_write32(offset, data) (_gic_write32(gicd_base + (offset), (data)))
43#define gicr_read32(offset) (_gic_read32(gicr_pe_base + (offset)))
44#define gicr_write32(offset, data) (_gic_write32(gicr_pe_base + (offset), (data)))
45#define gicr_read64(offset) (_gic_read64(gicr_pe_base + (offset)))
46
47static vm_offset_t
48find_gicr_pe_base()
49{
50 // We only care about aff1 and aff0
51 uint32_t phys_id = __builtin_arm_rsr64("MPIDR_EL1") & (MPIDR_AFF1_MASK | MPIDR_AFF0_MASK);
52
53 for (vm_offset_t offset = 0; offset < gicr_size; offset += GICR_PE_SIZE) {
54 vm_offset_t gicr_pe_base = gicr_base + offset;
55 uint64_t gicr_typer = gicr_read64(GICR_TYPER);
56 uint32_t aff_value = (uint32_t) (gicr_typer >> GICR_TYPER_AFFINITY_VALUE_SHIFT) & (MPIDR_AFF1_MASK | MPIDR_AFF0_MASK);
57
58 if (phys_id == aff_value) {
59 return gicr_pe_base;
60 }
61
62 if (gicr_typer & GICR_TYPER_LAST) {
63 break;
64 }
65 }
66
67 panic("%s: cannot find GICR base for core %u", __func__, ml_get_cpu_number(phys_id));
68}
69
70void
71pe_init_fiq()
72{
73 int error;
74 DTEntry entry;
75
76 // gicd_base is 0x0 only before it's initialized by the boot processor.
77 // Avoid calling SecureDT* routines on secondary processors to avoid
78 // race conditions because they are not thread-safe.
79 if (!gicd_base) {
80 // Find GIC DT node
81 error = SecureDTLookupEntry(NULL, pathName: "/arm-io/gic", foundEntry: &entry);
82 if (error != kSuccess) {
83 panic("%s: cannot find GIC node in DT", __func__);
84 }
85
86 // Find "reg" property
87 void const *prop;
88 unsigned int prop_size;
89 error = SecureDTGetProperty(entry, propertyName: "reg", propertyValue: &prop, propertySize: &prop_size);
90 if (error != kSuccess) {
91 panic("%s: cannot find GIC MMIO regions in DT", __func__);
92 }
93
94 // Need at least GICD base, GICD size, GICR base and GICR size
95 if (prop_size < 4 * sizeof(uint64_t)) {
96 panic("%s: incorrect reg property size in GIC DT node; expecting 32 bytes but got %u bytes", __func__, prop_size);
97 }
98
99 vm_offset_t soc_base_phys = pe_arm_get_soc_base_phys();
100
101 uint64_t const gicd_base_prop = ((uint64_t const *) prop)[0];
102 uint64_t const gicd_size_prop = ((uint64_t const *) prop)[1];
103 uint64_t const gicr_base_prop = ((uint64_t const *) prop)[2];
104 uint64_t const gicr_size_prop = ((uint64_t const *) prop)[3];
105
106 // Find GICD base address
107 gicd_base = ml_io_map(phys_addr: soc_base_phys + gicd_base_prop, size: gicd_size_prop);
108
109 if (!gicd_base) {
110 panic("%s: cannot map GICD region", __func__);
111 }
112
113 // Find GICR base address
114 gicr_base = ml_io_map(phys_addr: soc_base_phys + gicr_base_prop, size: gicr_size_prop);
115
116 if (!gicr_base) {
117 panic("%s: cannot map GICR region", __func__);
118 }
119
120 gicr_size = gicr_size_prop;
121 }
122
123 // Find the redistributor for this processor
124 vm_offset_t gicr_pe_base = find_gicr_pe_base();
125
126 // Mark this PE to be awake
127 uint32_t gicr_waker = gicr_read32(GICR_WAKER);
128 if (gicr_waker & GICR_WAKER_CHILDRENASLEEP) {
129 gicr_waker &= ~GICR_WAKER_PROCESSORSLEEP;
130
131 gicr_write32(GICR_WAKER, gicr_waker);
132
133 uint64_t gicr_wake_deadline;
134 nanoseconds_to_deadline(os_atomic_load(&gicr_wake_timeout_ns, relaxed), result: &gicr_wake_deadline);
135 while (gicr_read32(GICR_WAKER) & GICR_WAKER_CHILDRENASLEEP) {
136 // Spin
137 if (gicr_wake_timeout_ns > 0 && mach_absolute_time() > gicr_wake_deadline) {
138 panic("%s: core %u timed out waiting for redistributor to wake up",
139 __func__, ml_get_cpu_number_local());
140 }
141 }
142 }
143
144 // Configure timers and legacy FIQ to be group 0
145 gicr_write32(GICR_IGROUPR0, 0x81FFFFFF);
146
147 // Enable PPI 27
148 gicr_write32(GICR_ISENABLER0, (1 << 27));
149
150 // Enable system register access
151 uint64_t icc_sre = __builtin_arm_rsr64("ICC_SRE_EL1");
152 icc_sre |= ICC_SRE_SRE;
153 __builtin_arm_wsr64("ICC_SRE_EL1", icc_sre);
154 __builtin_arm_isb(ISB_SY);
155
156 // Set priority masks and binary point for group 0
157 __builtin_arm_wsr64("ICC_BPR0_EL1", 0);
158 __builtin_arm_wsr64("ICC_PMR_EL1", 0xFF);
159
160 // Set EOI mode of this processor
161 uint64_t icc_ctlr = __builtin_arm_rsr64("ICC_CTLR_EL1");
162 icc_ctlr &= ~ICC_CTLR_EOIMODE;
163 __builtin_arm_wsr64("ICC_CTLR_EL1", icc_ctlr);
164
165 // Enable the forwarding of the vtimer interrupt
166 uint32_t gicd_ctlr = gicd_read32(GICD_CTLR);
167 gicd_ctlr |= GICD_CTLR_ENABLEGRP0;
168 gicd_write32(GICD_CTLR, gicd_ctlr);
169 __builtin_arm_wsr64("ICC_IGRPEN0_EL1", 1);
170 __builtin_arm_isb(ISB_SY);
171}
172#else
173void
174pe_init_fiq()
175{
176}
177#endif
178