1/*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <arm/cpu_data_internal.h>
30#include <arm/dbgwrap.h>
31#include <arm64/proc_reg.h>
32#include <machine/atomic.h>
33#include <pexpert/arm64/board_config.h>
34
35#define DBGWRAP_REG_OFFSET 0
36#define DBGWRAP_DBGHALT (1ULL << 31)
37#define DBGWRAP_DBGACK (1ULL << 28)
38
39#define EDDTRRX_REG_OFFSET 0x80
40#define EDITR_REG_OFFSET 0x84
41#define EDSCR_REG_OFFSET 0x88
42#define EDSCR_TXFULL (1ULL << 29)
43#define EDSCR_ITE (1ULL << 24)
44#define EDSCR_MA (1ULL << 20)
45#define EDSCR_ERR (1ULL << 6)
46#define EDDTRTX_REG_OFFSET 0x8C
47#define EDRCR_REG_OFFSET 0x90
48#define EDRCR_CSE (1ULL << 2)
49#define EDPRSR_REG_OFFSET 0x314
50#define EDPRSR_OSLK (1ULL << 5)
51
52#define MAX_EDITR_RETRIES 16
53
54/* Older SoCs require 32-bit accesses for DBGWRAP;
55 * newer ones require 64-bit accesses. */
56#ifdef HAS_32BIT_DBGWRAP
57typedef uint32_t dbgwrap_reg_t;
58#else
59typedef uint64_t dbgwrap_reg_t;
60#endif
61
62#if DEVELOPMENT || DEBUG
63#define MAX_STUFFED_INSTRS 64
64uint32_t stuffed_instrs[MAX_STUFFED_INSTRS];
65volatile uint32_t stuffed_instr_count = 0;
66#endif
67
68static volatile uint32_t halt_from_cpu = (uint32_t)-1;
69
70boolean_t
71ml_dbgwrap_cpu_is_halted(int cpu_index)
72{
73 cpu_data_t *cdp = cpu_datap(cpu: cpu_index);
74 if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) {
75 return FALSE;
76 }
77
78 return (*(volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET) & DBGWRAP_DBGACK) != 0;
79}
80
81dbgwrap_status_t
82ml_dbgwrap_wait_cpu_halted(int cpu_index, uint64_t timeout_ns)
83{
84 cpu_data_t *cdp = cpu_datap(cpu: cpu_index);
85 if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) {
86 return DBGWRAP_ERR_UNSUPPORTED;
87 }
88
89 volatile dbgwrap_reg_t *dbgWrapReg = (volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET);
90
91 uint64_t interval;
92 nanoseconds_to_absolutetime(nanoseconds: timeout_ns, result: &interval);
93 uint64_t deadline = mach_absolute_time() + interval;
94 while (!(*dbgWrapReg & DBGWRAP_DBGACK)) {
95 if (mach_absolute_time() > deadline) {
96 return DBGWRAP_ERR_HALT_TIMEOUT;
97 }
98 }
99
100 return DBGWRAP_SUCCESS;
101}
102
103dbgwrap_status_t
104ml_dbgwrap_halt_cpu(int cpu_index, uint64_t timeout_ns)
105{
106 cpu_data_t *cdp = cpu_datap(cpu: cpu_index);
107 if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) {
108 return DBGWRAP_ERR_UNSUPPORTED;
109 }
110
111 /* Only one cpu is allowed to initiate the halt sequence, to prevent cpus from cross-halting
112 * each other. The first cpu to request a halt may then halt any and all other cpus besides itself. */
113 int curcpu = cpu_number();
114 if (cpu_index == curcpu) {
115 return DBGWRAP_ERR_SELF_HALT;
116 }
117
118 if (!os_atomic_cmpxchg(&halt_from_cpu, (uint32_t)-1, (unsigned int)curcpu, acq_rel) &&
119 (halt_from_cpu != (uint32_t)curcpu)) {
120 return DBGWRAP_ERR_INPROGRESS;
121 }
122
123 volatile dbgwrap_reg_t *dbgWrapReg = (volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET);
124
125 if (ml_dbgwrap_cpu_is_halted(cpu_index)) {
126 return DBGWRAP_WARN_ALREADY_HALTED;
127 }
128
129
130 /* Clear all other writable bits besides dbgHalt; none of the power-down or reset bits must be set. */
131 *dbgWrapReg = DBGWRAP_DBGHALT;
132
133
134 if (timeout_ns != 0) {
135 dbgwrap_status_t stat = ml_dbgwrap_wait_cpu_halted(cpu_index, timeout_ns);
136 return stat;
137 } else {
138 return DBGWRAP_SUCCESS;
139 }
140}
141
142static void
143ml_dbgwrap_stuff_instr(cpu_data_t *cdp, uint32_t instr, uint64_t timeout_ns, dbgwrap_status_t *status)
144{
145 if (*status < 0) {
146 return;
147 }
148
149 volatile uint32_t *editr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDITR_REG_OFFSET);
150 volatile uint32_t *edscr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDSCR_REG_OFFSET);
151 volatile uint32_t *edrcr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDRCR_REG_OFFSET);
152
153 int retries = 0;
154
155 uint64_t interval;
156 nanoseconds_to_absolutetime(nanoseconds: timeout_ns, result: &interval);
157 uint64_t deadline = mach_absolute_time() + interval;
158
159#if DEVELOPMENT || DEBUG
160 uint32_t stuffed_instr_index = os_atomic_inc(&stuffed_instr_count, relaxed);
161 stuffed_instrs[(stuffed_instr_index - 1) % MAX_STUFFED_INSTRS] = instr;
162#endif
163
164 do {
165 *editr = instr;
166 volatile uint32_t edscr_val;
167 while (!((edscr_val = *edscr) & EDSCR_ITE)) {
168 if (mach_absolute_time() > deadline) {
169 *status = DBGWRAP_ERR_INSTR_TIMEOUT;
170 return;
171 }
172 if (edscr_val & EDSCR_ERR) {
173 break;
174 }
175 }
176 if (edscr_val & EDSCR_ERR) {
177 /* If memory access mode was enable by a debugger, clear it.
178 * This will cause ERR to be set on any attempt to use EDITR. */
179 if (edscr_val & EDSCR_MA) {
180 *edscr = edscr_val & ~EDSCR_MA;
181 }
182 *edrcr = EDRCR_CSE;
183 ++retries;
184 } else {
185 break;
186 }
187 } while (retries < MAX_EDITR_RETRIES);
188
189 if (retries >= MAX_EDITR_RETRIES) {
190 *status = DBGWRAP_ERR_INSTR_ERROR;
191 return;
192 }
193}
194
195static uint64_t
196ml_dbgwrap_read_dtr(cpu_data_t *cdp, uint64_t timeout_ns, dbgwrap_status_t *status)
197{
198 if (*status < 0) {
199 return 0;
200 }
201
202 uint64_t interval;
203 nanoseconds_to_absolutetime(nanoseconds: timeout_ns, result: &interval);
204 uint64_t deadline = mach_absolute_time() + interval;
205
206 /* Per armv8 debug spec, writes to DBGDTR_EL0 on target cpu will set EDSCR.TXFull,
207 * with bits 63:32 available in EDDTRRX and bits 31:0 availabe in EDDTRTX. */
208 volatile uint32_t *edscr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDSCR_REG_OFFSET);
209
210 while (!(*edscr & EDSCR_TXFULL)) {
211 if (*edscr & EDSCR_ERR) {
212 *status = DBGWRAP_ERR_INSTR_ERROR;
213 return 0;
214 }
215 if (mach_absolute_time() > deadline) {
216 *status = DBGWRAP_ERR_INSTR_TIMEOUT;
217 return 0;
218 }
219 }
220
221 uint32_t dtrrx = *((volatile uint32_t*)(cdp->coresight_base[CORESIGHT_ED] + EDDTRRX_REG_OFFSET));
222 uint32_t dtrtx = *((volatile uint32_t*)(cdp->coresight_base[CORESIGHT_ED] + EDDTRTX_REG_OFFSET));
223
224 return ((uint64_t)dtrrx << 32) | dtrtx;
225}
226
227dbgwrap_status_t
228ml_dbgwrap_halt_cpu_with_state(int cpu_index, uint64_t timeout_ns, dbgwrap_thread_state_t *state)
229{
230 cpu_data_t *cdp = cpu_datap(cpu: cpu_index);
231 if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_ED] == 0)) {
232 return DBGWRAP_ERR_UNSUPPORTED;
233 }
234
235 /* Ensure memory-mapped coresight registers can be written */
236 *((volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR)) = ARM_DBG_LOCK_ACCESS_KEY;
237
238 dbgwrap_status_t status = ml_dbgwrap_halt_cpu(cpu_index, timeout_ns);
239
240 /* A core that is not fully powered (e.g. idling in wfi) can still be halted; the dbgwrap
241 * register and certain coresight registers such EDPRSR are in the always-on domain.
242 * However, EDSCR/EDITR are not in the always-on domain and will generate a parity abort
243 * on read. EDPRSR can be safely read in all cases, and the OS lock defaults to being set
244 * but we clear it first thing, so use that to detect the offline state. */
245 if (*((volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDPRSR_REG_OFFSET)) & EDPRSR_OSLK) {
246 bzero(s: state, n: sizeof(*state));
247 return DBGWRAP_WARN_CPU_OFFLINE;
248 }
249
250 uint32_t instr;
251
252 for (unsigned int i = 0; i < (sizeof(state->x) / sizeof(state->x[0])); ++i) {
253 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | i; // msr DBGDTR0, x<i>
254 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status);
255 state->x[i] = ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status);
256 }
257
258 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 29; // msr DBGDTR0, fp
259 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status);
260 state->fp = ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status);
261
262 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 30; // msr DBGDTR0, lr
263 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status);
264 state->lr = ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status);
265
266 /* Stack pointer (x31) can't be used as a register operand for msr; register 31 is treated as xzr
267 * rather than sp when used as the transfer operand there. Instead, load sp into a GPR
268 * we've already saved off and then store that register in the DTR. I've chosen x18
269 * as the temporary GPR since it's reserved by the arm64 ABI and unused by xnu, so overwriting
270 * it poses the least risk of causing trouble for external debuggers. */
271
272 instr = (0x91U << 24) | (31 << 5) | 18; // mov x18, sp
273 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status);
274 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18
275 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status);
276 state->sp = ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status);
277
278 /* reading PC (e.g. through adr) is undefined in debug state. Instead use DLR_EL0,
279 * which contains PC at time of entry into debug state.*/
280
281 instr = (0xD53U << 20) | (1 << 19) | (3 << 16) | (4 << 12) | (5 << 8) | (1 << 5) | 18; // mrs x18, DLR_EL0
282 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status);
283 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18
284 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status);
285 state->pc = ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status);
286
287 /* reading CPSR is undefined in debug state. Instead use DSPSR_EL0,
288 * which contains CPSR at time of entry into debug state.*/
289 instr = (0xD53U << 20) | (1 << 19) | (3 << 16) | (4 << 12) | (5 << 8) | 18; // mrs x18, DSPSR_EL0
290 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status);
291 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18
292 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, status: &status);
293 state->cpsr = (uint32_t)ml_dbgwrap_read_dtr(cdp, timeout_ns, status: &status);
294
295 return status;
296}
297