1 | /* |
2 | * Copyright (c) 2017 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <arm/cpu_data_internal.h> |
30 | #include <arm/dbgwrap.h> |
31 | #include <arm64/proc_reg.h> |
32 | #include <machine/atomic.h> |
33 | #include <pexpert/arm64/board_config.h> |
34 | |
35 | #define DBGWRAP_REG_OFFSET 0 |
36 | #define DBGWRAP_DBGHALT (1ULL << 31) |
37 | #define DBGWRAP_DBGACK (1ULL << 28) |
38 | |
39 | #define EDDTRRX_REG_OFFSET 0x80 |
40 | #define EDITR_REG_OFFSET 0x84 |
41 | #define EDSCR_REG_OFFSET 0x88 |
42 | #define EDSCR_TXFULL (1ULL << 29) |
43 | #define EDSCR_ITE (1ULL << 24) |
44 | #define EDSCR_MA (1ULL << 20) |
45 | #define EDSCR_ERR (1ULL << 6) |
46 | #define EDDTRTX_REG_OFFSET 0x8C |
47 | #define EDRCR_REG_OFFSET 0x90 |
48 | #define EDRCR_CSE (1ULL << 2) |
49 | #define EDPRSR_REG_OFFSET 0x314 |
50 | #define EDPRSR_OSLK (1ULL << 5) |
51 | |
52 | #define MAX_EDITR_RETRIES 16 |
53 | |
54 | /* Older SoCs require 32-bit accesses for DBGWRAP; |
55 | * newer ones require 64-bit accesses. */ |
56 | #ifdef HAS_32BIT_DBGWRAP |
57 | typedef uint32_t dbgwrap_reg_t; |
58 | #else |
59 | typedef uint64_t dbgwrap_reg_t; |
60 | #endif |
61 | |
62 | #if DEVELOPMENT || DEBUG |
63 | #define MAX_STUFFED_INSTRS 64 |
64 | uint32_t stuffed_instrs[MAX_STUFFED_INSTRS]; |
65 | volatile uint32_t stuffed_instr_count = 0; |
66 | #endif |
67 | |
68 | static volatile uint32_t halt_from_cpu = (uint32_t)-1; |
69 | |
70 | boolean_t |
71 | ml_dbgwrap_cpu_is_halted(int cpu_index) |
72 | { |
73 | cpu_data_t *cdp = cpu_datap(cpu: cpu_index); |
74 | if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) { |
75 | return FALSE; |
76 | } |
77 | |
78 | return (*(volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET) & DBGWRAP_DBGACK) != 0; |
79 | } |
80 | |
81 | dbgwrap_status_t |
82 | ml_dbgwrap_wait_cpu_halted(int cpu_index, uint64_t timeout_ns) |
83 | { |
84 | cpu_data_t *cdp = cpu_datap(cpu: cpu_index); |
85 | if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) { |
86 | return DBGWRAP_ERR_UNSUPPORTED; |
87 | } |
88 | |
89 | volatile dbgwrap_reg_t *dbgWrapReg = (volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET); |
90 | |
91 | uint64_t interval; |
92 | nanoseconds_to_absolutetime(nanoseconds: timeout_ns, result: &interval); |
93 | uint64_t deadline = mach_absolute_time() + interval; |
94 | while (!(*dbgWrapReg & DBGWRAP_DBGACK)) { |
95 | if (mach_absolute_time() > deadline) { |
96 | return DBGWRAP_ERR_HALT_TIMEOUT; |
97 | } |
98 | } |
99 | |
100 | return DBGWRAP_SUCCESS; |
101 | } |
102 | |
103 | dbgwrap_status_t |
104 | ml_dbgwrap_halt_cpu(int cpu_index, uint64_t timeout_ns) |
105 | { |
106 | cpu_data_t *cdp = cpu_datap(cpu: cpu_index); |
107 | if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) { |
108 | return DBGWRAP_ERR_UNSUPPORTED; |
109 | } |
110 | |
111 | /* Only one cpu is allowed to initiate the halt sequence, to prevent cpus from cross-halting |
112 | * each other. The first cpu to request a halt may then halt any and all other cpus besides itself. */ |
113 | int curcpu = cpu_number(); |
114 | if (cpu_index == curcpu) { |
115 | return DBGWRAP_ERR_SELF_HALT; |
116 | } |
117 | |
118 | if (!os_atomic_cmpxchg(&halt_from_cpu, (uint32_t)-1, (unsigned int)curcpu, acq_rel) && |
119 | (halt_from_cpu != (uint32_t)curcpu)) { |
120 | return DBGWRAP_ERR_INPROGRESS; |
121 | } |
122 | |
123 | volatile dbgwrap_reg_t *dbgWrapReg = (volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET); |
124 | |
125 | if (ml_dbgwrap_cpu_is_halted(cpu_index)) { |
126 | return DBGWRAP_WARN_ALREADY_HALTED; |
127 | } |
128 | |
129 | |
130 | /* Clear all other writable bits besides dbgHalt; none of the power-down or reset bits must be set. */ |
131 | *dbgWrapReg = DBGWRAP_DBGHALT; |
132 | |
133 | |
134 | if (timeout_ns != 0) { |
135 | dbgwrap_status_t stat = ml_dbgwrap_wait_cpu_halted(cpu_index, timeout_ns); |
136 | return stat; |
137 | } else { |
138 | return DBGWRAP_SUCCESS; |
139 | } |
140 | } |
141 | |
142 | static void |
143 | ml_dbgwrap_stuff_instr(cpu_data_t *cdp, uint32_t instr, uint64_t timeout_ns, dbgwrap_status_t *status) |
144 | { |
145 | if (*status < 0) { |
146 | return; |
147 | } |
148 | |
149 | volatile uint32_t *editr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDITR_REG_OFFSET); |
150 | volatile uint32_t *edscr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDSCR_REG_OFFSET); |
151 | volatile uint32_t *edrcr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDRCR_REG_OFFSET); |
152 | |
153 | int retries = 0; |
154 | |
155 | uint64_t interval; |
156 | nanoseconds_to_absolutetime(nanoseconds: timeout_ns, result: &interval); |
157 | uint64_t deadline = mach_absolute_time() + interval; |
158 | |
159 | #if DEVELOPMENT || DEBUG |
160 | uint32_t stuffed_instr_index = os_atomic_inc(&stuffed_instr_count, relaxed); |
161 | stuffed_instrs[(stuffed_instr_index - 1) % MAX_STUFFED_INSTRS] = instr; |
162 | #endif |
163 | |
164 | do { |
165 | *editr = instr; |
166 | volatile uint32_t edscr_val; |
167 | while (!((edscr_val = *edscr) & EDSCR_ITE)) { |
168 | if (mach_absolute_time() > deadline) { |
169 | *status = DBGWRAP_ERR_INSTR_TIMEOUT; |
170 | return; |
171 | } |
172 | if (edscr_val & EDSCR_ERR) { |
173 | break; |
174 | } |
175 | } |
176 | if (edscr_val & EDSCR_ERR) { |
177 | /* If memory access mode was enable by a debugger, clear it. |
178 | * This will cause ERR to be set on any attempt to use EDITR. */ |
179 | if (edscr_val & EDSCR_MA) { |
180 | *edscr = edscr_val & ~EDSCR_MA; |
181 | } |
182 | *edrcr = EDRCR_CSE; |
183 | ++retries; |
184 | } else { |
185 | break; |
186 | } |
187 | } while (retries < MAX_EDITR_RETRIES); |
188 | |
189 | if (retries >= MAX_EDITR_RETRIES) { |
190 | *status = DBGWRAP_ERR_INSTR_ERROR; |
191 | return; |
192 | } |
193 | } |
194 | |
195 | static uint64_t |
196 | ml_dbgwrap_read_dtr(cpu_data_t *cdp, uint64_t timeout_ns, dbgwrap_status_t *status) |
197 | { |
198 | if (*status < 0) { |
199 | return 0; |
200 | } |
201 | |
202 | uint64_t interval; |
203 | nanoseconds_to_absolutetime(nanoseconds: timeout_ns, result: &interval); |
204 | uint64_t deadline = mach_absolute_time() + interval; |
205 | |
206 | /* Per armv8 debug spec, writes to DBGDTR_EL0 on target cpu will set EDSCR.TXFull, |
207 | * with bits 63:32 available in EDDTRRX and bits 31:0 availabe in EDDTRTX. */ |
208 | volatile uint32_t *edscr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDSCR_REG_OFFSET); |
209 | |
210 | while (!(*edscr & EDSCR_TXFULL)) { |
211 | if (*edscr & EDSCR_ERR) { |
212 | *status = DBGWRAP_ERR_INSTR_ERROR; |
213 | return 0; |
214 | } |
215 | if (mach_absolute_time() > deadline) { |
216 | *status = DBGWRAP_ERR_INSTR_TIMEOUT; |
217 | return 0; |
218 | } |
219 | } |
220 | |
221 | uint32_t dtrrx = *((volatile uint32_t*)(cdp->coresight_base[CORESIGHT_ED] + EDDTRRX_REG_OFFSET)); |
222 | uint32_t dtrtx = *((volatile uint32_t*)(cdp->coresight_base[CORESIGHT_ED] + EDDTRTX_REG_OFFSET)); |
223 | |
224 | return ((uint64_t)dtrrx << 32) | dtrtx; |
225 | } |
226 | |
227 | dbgwrap_status_t |
228 | ml_dbgwrap_halt_cpu_with_state(int cpu_index, uint64_t timeout_ns, dbgwrap_thread_state_t *state) |
229 | { |
230 | cpu_data_t *cdp = cpu_datap(cpu: cpu_index); |
231 | if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_ED] == 0)) { |
232 | return DBGWRAP_ERR_UNSUPPORTED; |
233 | } |
234 | |
235 | /* Ensure memory-mapped coresight registers can be written */ |
236 | *((volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR)) = ARM_DBG_LOCK_ACCESS_KEY; |
237 | |
238 | dbgwrap_status_t status = ml_dbgwrap_halt_cpu(cpu_index, timeout_ns); |
239 | |
240 | /* A core that is not fully powered (e.g. idling in wfi) can still be halted; the dbgwrap |
241 | * register and certain coresight registers such EDPRSR are in the always-on domain. |
242 | * However, EDSCR/EDITR are not in the always-on domain and will generate a parity abort |
243 | * on read. EDPRSR can be safely read in all cases, and the OS lock defaults to being set |
244 | * but we clear it first thing, so use that to detect the offline state. */ |
245 | if (*((volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDPRSR_REG_OFFSET)) & EDPRSR_OSLK) { |
246 | bzero(s: state, n: sizeof(*state)); |
247 | return DBGWRAP_WARN_CPU_OFFLINE; |
248 | } |
249 | |
250 | uint32_t instr; |
251 | |
252 | for (unsigned int i = 0; i < (sizeof(state->x) / sizeof(state->x[0])); ++i) { |
253 | instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | i; // msr DBGDTR0, x<i> |
254 | ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status); |
255 | state->x[i] = ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status); |
256 | } |
257 | |
258 | instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 29; // msr DBGDTR0, fp |
259 | ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status); |
260 | state->fp = ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status); |
261 | |
262 | instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 30; // msr DBGDTR0, lr |
263 | ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status); |
264 | state->lr = ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status); |
265 | |
266 | /* Stack pointer (x31) can't be used as a register operand for msr; register 31 is treated as xzr |
267 | * rather than sp when used as the transfer operand there. Instead, load sp into a GPR |
268 | * we've already saved off and then store that register in the DTR. I've chosen x18 |
269 | * as the temporary GPR since it's reserved by the arm64 ABI and unused by xnu, so overwriting |
270 | * it poses the least risk of causing trouble for external debuggers. */ |
271 | |
272 | instr = (0x91U << 24) | (31 << 5) | 18; // mov x18, sp |
273 | ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status); |
274 | instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18 |
275 | ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status); |
276 | state->sp = ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status); |
277 | |
278 | /* reading PC (e.g. through adr) is undefined in debug state. Instead use DLR_EL0, |
279 | * which contains PC at time of entry into debug state.*/ |
280 | |
281 | instr = (0xD53U << 20) | (1 << 19) | (3 << 16) | (4 << 12) | (5 << 8) | (1 << 5) | 18; // mrs x18, DLR_EL0 |
282 | ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status); |
283 | instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18 |
284 | ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status); |
285 | state->pc = ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status); |
286 | |
287 | /* reading CPSR is undefined in debug state. Instead use DSPSR_EL0, |
288 | * which contains CPSR at time of entry into debug state.*/ |
289 | instr = (0xD53U << 20) | (1 << 19) | (3 << 16) | (4 << 12) | (5 << 8) | 18; // mrs x18, DSPSR_EL0 |
290 | ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status); |
291 | instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18 |
292 | ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status); |
293 | state->cpsr = (uint32_t)ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status); |
294 | |
295 | return status; |
296 | } |
297 | |