1 | /* |
2 | * Copyright (c) 2007-2023 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* CMU_ENDHIST */ |
32 | /* |
33 | * Mach Operating System |
34 | * Copyright (c) 1991,1990 Carnegie Mellon University |
35 | * All Rights Reserved. |
36 | * |
37 | * Permission to use, copy, modify and distribute this software and its |
38 | * documentation is hereby granted, provided that both the copyright |
39 | * notice and this permission notice appear in all copies of the |
40 | * software, derivative works or modified versions, and any portions |
41 | * thereof, and that both notices appear in supporting documentation. |
42 | * |
43 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
44 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
45 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
46 | * |
47 | * Carnegie Mellon requests users of this software to return to |
48 | * |
49 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
50 | * School of Computer Science |
51 | * Carnegie Mellon University |
52 | * Pittsburgh PA 15213-3890 |
53 | * |
54 | * any improvements or extensions that they make and grant Carnegie Mellon |
55 | * the rights to redistribute these changes. |
56 | */ |
57 | |
58 | /* |
59 | * Processor registers for ARM/ARM64 |
60 | */ |
61 | #ifndef _ARM64_PROC_REG_H_ |
62 | #define _ARM64_PROC_REG_H_ |
63 | |
64 | #if defined (__arm64__) |
65 | #include <pexpert/arm64/board_config.h> |
66 | #elif defined (__arm__) |
67 | #include <pexpert/arm/board_config.h> |
68 | #endif |
69 | |
70 | /* |
71 | * Processor registers for ARM |
72 | */ |
73 | #if __ARM_42BIT_PA_SPACE__ |
74 | /* For now, force the issue! */ |
75 | /* We need more VA space for the identity map to bootstrap the MMU */ |
76 | #undef __ARM64_PMAP_SUBPAGE_L1__ |
77 | #endif /* __ARM_42BIT_PA_SPACE__ */ |
78 | |
79 | /* For arm platforms, create one pset per cluster */ |
80 | #define MAX_PSETS MAX_CPU_CLUSTERS |
81 | |
82 | |
83 | /* Thread groups are enabled on all ARM platforms (irrespective of scheduler) */ |
84 | #define CONFIG_THREAD_GROUPS 1 |
85 | |
86 | #ifdef XNU_KERNEL_PRIVATE |
87 | |
88 | #if __ARM_VFP__ |
89 | #define ARM_VFP_DEBUG 0 |
90 | #endif /* __ARM_VFP__ */ |
91 | |
92 | #endif /* XNU_KERNEL_PRIVATE */ |
93 | |
94 | /* |
95 | * FSR registers |
96 | * |
97 | * CPSR: Current Program Status Register |
98 | * SPSR: Saved Program Status Registers |
99 | * |
100 | * 31 30 29 28 27 24 19 16 9 8 7 6 5 4 0 |
101 | * +-----------------------------------------------------------+ |
102 | * | N| Z| C| V| Q|...| J|...|GE[3:0]|...| E| A| I| F| T| MODE | |
103 | * +-----------------------------------------------------------+ |
104 | */ |
105 | |
106 | /* |
107 | * Flags |
108 | */ |
109 | #define PSR_NF 0x80000000 /* Negative/Less than */ |
110 | #define PSR_ZF 0x40000000 /* Zero */ |
111 | #define PSR_CF 0x20000000 /* Carry/Borrow/Extend */ |
112 | #define PSR_VF 0x10000000 /* Overflow */ |
113 | |
114 | /* |
115 | * Modified execution mode flags |
116 | */ |
117 | #define PSR_TF 0x00000020 /* thumb flag (BX ARMv4T) */ |
118 | |
119 | /* |
120 | * CPU mode |
121 | */ |
122 | #define PSR_USER_MODE 0x00000010 /* User mode */ |
123 | |
124 | #define PSR_MODE_MASK 0x0000001F |
125 | #define PSR_IS_KERNEL(psr) (((psr) & PSR_MODE_MASK) != PSR_USER_MODE) |
126 | #define PSR_IS_USER(psr) (((psr) & PSR_MODE_MASK) == PSR_USER_MODE) |
127 | |
128 | #define PSR_USERDFLT PSR_USER_MODE |
129 | |
130 | /* |
131 | * Cache configuration |
132 | */ |
133 | |
134 | #if defined (APPLETYPHOON) |
135 | |
136 | /* I-Cache */ |
137 | #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ |
138 | |
139 | /* D-Cache */ |
140 | #define MMU_CLINE 6 /* cache line size as 1<<MMU_CLINE (64) */ |
141 | |
142 | #elif defined (APPLETWISTER) |
143 | |
144 | /* I-Cache */ |
145 | #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ |
146 | |
147 | /* D-Cache */ |
148 | #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ |
149 | |
150 | #elif defined (APPLEHURRICANE) |
151 | |
152 | /* I-Cache */ |
153 | #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ |
154 | |
155 | /* D-Cache */ |
156 | #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ |
157 | |
158 | #elif defined (APPLEMONSOON) |
159 | |
160 | /* I-Cache, 96KB for Monsoon, 48KB for Mistral, 6-way. */ |
161 | #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ |
162 | |
163 | /* D-Cache, 64KB for Monsoon, 32KB for Mistral, 4-way. */ |
164 | #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ |
165 | |
166 | #elif defined (APPLEVORTEX) |
167 | |
168 | /* I-Cache, 128KB 8-way for Vortex, 48KB 6-way for Tempest. */ |
169 | #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ |
170 | |
171 | /* D-Cache, 128KB 8-way for Vortex, 32KB 4-way for Tempest. */ |
172 | #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ |
173 | |
174 | #elif defined (APPLELIGHTNING) |
175 | |
176 | /* I-Cache, 192KB for Lightning, 96KB for Thunder, 6-way. */ |
177 | #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ |
178 | |
179 | /* D-Cache, 128KB for Lightning, 8-way. 48KB for Thunder, 6-way. */ |
180 | #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ |
181 | |
182 | #elif defined (APPLEFIRESTORM) |
183 | |
184 | /* I-Cache, 256KB for Firestorm, 128KB for Icestorm, 6-way. */ |
185 | #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ |
186 | |
187 | /* D-Cache, 160KB for Firestorm, 8-way. 64KB for Icestorm, 6-way. */ |
188 | #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ |
189 | |
190 | #elif defined (APPLEAVALANCHE) |
191 | |
192 | /* I-Cache, 192KB for Avalanche, 128KB for Blizzard, 6-way. */ |
193 | #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ |
194 | |
195 | /* D-Cache, 128KB for Avalanche, 8-way. 64KB for Blizzard, 8-way. */ |
196 | #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ |
197 | |
198 | #elif defined (BCM2837) /* Raspberry Pi 3 */ |
199 | |
200 | /* I-Cache. We don't have detailed spec so we just follow the ARM technical reference. */ |
201 | #define MMU_I_CLINE 6 |
202 | |
203 | /* D-Cache. */ |
204 | #define MMU_CLINE 6 |
205 | |
206 | #elif defined (VMAPPLE) |
207 | |
208 | /* I-Cache. */ |
209 | #define MMU_I_CLINE 6 |
210 | |
211 | /* D-Cache. */ |
212 | #define MMU_CLINE 6 |
213 | |
214 | #else |
215 | #error processor not supported |
216 | #endif |
217 | |
218 | #define MAX_L2_CLINE_BYTES (1 << MAX_L2_CLINE) |
219 | |
220 | /* |
221 | * Format of the Debug & Watchpoint Breakpoint Value and Control Registers |
222 | */ |
223 | #define ARM_DBG_VR_ADDRESS_MASK 0xFFFFFFFC /* BVR & WVR */ |
224 | #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */ |
225 | |
226 | #define ARM_DBG_CR_ADDRESS_MASK_MASK 0x1F000000 /* BCR & WCR */ |
227 | #define ARM_DBGBCR_MATCH_MASK (1 << 22) /* BCR only */ |
228 | #define ARM_DBGBCR_TYPE_MASK (1 << 21) /* BCR only */ |
229 | #define ARM_DBGBCR_TYPE_IVA (0 << 21) |
230 | #define ARM_DBG_CR_LINKED_MASK (1 << 20) /* BCR & WCR */ |
231 | #define ARM_DBG_CR_LINKED_UNLINKED (0 << 20) |
232 | #define ARM_DBG_CR_SECURITY_STATE_BOTH (0 << 14) |
233 | #define ARM_DBG_CR_HIGHER_MODE_ENABLE (1 << 13) |
234 | #define ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK 0x00001FE0 /* WCR only */ |
235 | #define ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK 0x000001E0 /* BCR & WCR */ |
236 | #define ARM_DBGWCR_ACCESS_CONTROL_MASK (3 << 3) /* WCR only */ |
237 | #define ARM_DBG_CR_MODE_CONTROL_PRIVILEGED (1 << 1) /* BCR & WCR */ |
238 | #define ARM_DBG_CR_MODE_CONTROL_USER (2 << 1) /* BCR & WCR */ |
239 | #define ARM_DBG_CR_ENABLE_MASK (1 << 0) /* BCR & WCR */ |
240 | #define ARM_DBG_CR_ENABLE_ENABLE (1 << 0) |
241 | |
242 | /* |
243 | * Format of the OS Lock Access (DBGOSLAR) and Lock Access Registers (DBGLAR) |
244 | */ |
245 | #define ARM_DBG_LOCK_ACCESS_KEY 0xC5ACCE55 |
246 | |
247 | /* ARM Debug registers of interest */ |
248 | #define ARM_DEBUG_OFFSET_DBGPRCR (0x310) |
249 | #define ARM_DEBUG_OFFSET_DBGLAR (0xFB0) |
250 | |
251 | /* |
252 | * Main ID Register (MIDR) |
253 | * |
254 | * 31 24 23 20 19 16 15 4 3 0 |
255 | * +-----+-----+------+------+-----+ |
256 | * | IMP | VAR | ARCH | PNUM | REV | |
257 | * +-----+-----+------+------+-----+ |
258 | * |
259 | * where: |
260 | * IMP: Implementor code |
261 | * VAR: Variant number |
262 | * ARCH: Architecture code |
263 | * PNUM: Primary part number |
264 | * REV: Minor revision number |
265 | */ |
266 | #define MIDR_REV_SHIFT 0 |
267 | #define MIDR_REV_MASK (0xf << MIDR_REV_SHIFT) |
268 | #define MIDR_VAR_SHIFT 20 |
269 | #define MIDR_VAR_MASK (0xf << MIDR_VAR_SHIFT) |
270 | |
271 | |
272 | #if __ARM_KERNEL_PROTECT__ |
273 | /* |
274 | * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential |
275 | * architectural or microarchitectural vulnerabilities that could allow cores to |
276 | * read/access EL1-only mappings while in EL0 mode. This is achieved by |
277 | * removing as many mappings as possible when the core transitions to EL0 mode |
278 | * from EL1 mode, and restoring those mappings when the core transitions to EL1 |
279 | * mode from EL0 mode. |
280 | * |
281 | * At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is |
282 | * used to map and unmap the ordinary kernel mappings, by contracting and |
283 | * expanding translation zone size for TTBR1 when exiting and entering EL1, |
284 | * respectively: |
285 | * |
286 | * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is |
287 | * is in EL0. |
288 | * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in |
289 | * EL1. |
290 | * |
291 | * T1SZ_USER: T1SZ_BOOT + 1 |
292 | * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT) |
293 | * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER) |
294 | * TTBR1_EL1_MAX: (2^64) - 1 |
295 | * |
296 | * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the |
297 | * the following TTBR1 layout: |
298 | * |
299 | * TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX |
300 | * +---------------------------------------------------------+ |
301 | * | Kernel EL0 Mappings | Kernel EL1 Mappings | |
302 | * +---------------------------------------------------------+ |
303 | * |
304 | * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout: |
305 | * |
306 | * TTBR1_EL1_BASE_USER TTBR1_EL1_MAX |
307 | * +---------------------------------------------------------+ |
308 | * | Kernel EL0 Mappings | |
309 | * +---------------------------------------------------------+ |
310 | * |
311 | * With the current implementation, both the EL0 and EL1 mappings for the kernel |
312 | * use otherwise empty translation tables for mapping the exception vectors (so |
313 | * that we do not need to TLB flush the exception vector address when switching |
314 | * between EL0 and EL1). The rationale here is that the TLBI would require a |
315 | * DSB, and DSBs can be extremely expensive. |
316 | * |
317 | * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1 |
318 | * ASID. The core switches between ASIDs on EL transitions, so that the TLB |
319 | * does not need to be fully invalidated on an EL transition. |
320 | * |
321 | * Most kernel mappings will be marked non-global in this configuration, as |
322 | * global mappings would be visible to userspace unless we invalidate them on |
323 | * eret. |
324 | */ |
325 | #if XNU_MONITOR |
326 | /* |
327 | * Please note that because we indirect through the thread register in order to |
328 | * locate the kernel, and because we unmap most of the kernel, the security |
329 | * model of the PPL is undermined by __ARM_KERNEL_PROTECT__, as we rely on |
330 | * kernel controlled data to direct codeflow in the exception vectors. |
331 | * |
332 | * If we want to ship XNU_MONITOR paired with __ARM_KERNEL_PROTECT__, we will |
333 | * need to find a performant solution to this problem. |
334 | */ |
335 | #endif |
336 | #endif /* __ARM_KERNEL_PROTECT */ |
337 | |
338 | #if ARM_PARAMETERIZED_PMAP |
339 | /* |
340 | * ARM_PARAMETERIZED_PMAP configures the kernel to get the characteristics of |
341 | * the page tables (number of levels, size of the root allocation) from the |
342 | * pmap data structure, rather than treating them as compile-time constants. |
343 | * This allows the pmap code to dynamically adjust how it deals with page |
344 | * tables. |
345 | */ |
346 | #endif /* ARM_PARAMETERIZED_PMAP */ |
347 | |
348 | #if __ARM_MIXED_PAGE_SIZE__ |
349 | /* |
350 | * __ARM_MIXED_PAGE_SIZE__ configures the kernel to support page tables that do |
351 | * not use the kernel page size. This is primarily meant to support running |
352 | * 4KB page processes on a 16KB page kernel. |
353 | * |
354 | * This only covers support in the pmap/machine dependent layers. Any support |
355 | * elsewhere in the kernel must be managed separately. |
356 | */ |
357 | #if !ARM_PARAMETERIZED_PMAP |
358 | /* |
359 | * Page tables that use non-kernel page sizes require us to reprogram TCR based |
360 | * on the page tables we are switching to. This means that the parameterized |
361 | * pmap support is required. |
362 | */ |
363 | #error __ARM_MIXED_PAGE_SIZE__ requires ARM_PARAMETERIZED_PMAP |
364 | #endif /* !ARM_PARAMETERIZED_PMAP */ |
365 | #if __ARM_KERNEL_PROTECT__ |
366 | /* |
367 | * Because switching the page size requires updating TCR based on the pmap, and |
368 | * __ARM_KERNEL_PROTECT__ relies on TCR being programmed with constants, XNU |
369 | * does not currently support support configurations that use both |
370 | * __ARM_KERNEL_PROTECT__ and __ARM_MIXED_PAGE_SIZE__. |
371 | */ |
372 | #error __ARM_MIXED_PAGE_SIZE__ and __ARM_KERNEL_PROTECT__ are mutually exclusive |
373 | #endif /* __ARM_KERNEL_PROTECT__ */ |
374 | #endif /* __ARM_MIXED_PAGE_SIZE__ */ |
375 | |
376 | /* |
377 | * 64-bit Program Status Register (PSR64) |
378 | * |
379 | * 31 27 23 22 21 20 19 10 9 5 4 0 |
380 | * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+ |
381 | * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0| M | |
382 | * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+ |
383 | * |
384 | * where: |
385 | * NZCV: Comparison flags |
386 | * PAN: Privileged Access Never |
387 | * SS: Single step |
388 | * IL: Illegal state |
389 | * DAIF: Interrupt masks |
390 | * M: Mode field |
391 | */ |
392 | |
393 | #define PSR64_NZCV_SHIFT 28 |
394 | #define PSR64_NZCV_MASK (0xF << PSR64_NZCV_SHIFT) |
395 | |
396 | #define PSR64_N_SHIFT 31 |
397 | #define PSR64_N (1 << PSR64_N_SHIFT) |
398 | |
399 | #define PSR64_Z_SHIFT 30 |
400 | #define PSR64_Z (1 << PSR64_Z_SHIFT) |
401 | |
402 | #define PSR64_C_SHIFT 29 |
403 | #define PSR64_C (1 << PSR64_C_SHIFT) |
404 | |
405 | #define PSR64_V_SHIFT 28 |
406 | #define PSR64_V (1 << PSR64_V_SHIFT) |
407 | |
408 | #define PSR64_TCO_SHIFT 25 |
409 | #define PSR64_TCO (1 << PSR64_TCO_SHIFT) |
410 | |
411 | #define PSR64_DIT_SHIFT 24 |
412 | #define PSR64_DIT (1 << PSR64_DIT_SHIFT) |
413 | |
414 | #define PSR64_UAO_SHIFT 23 |
415 | #define PSR64_UAO (1 << PSR64_UAO_SHIFT) |
416 | |
417 | #define PSR64_PAN_SHIFT 22 |
418 | #define PSR64_PAN (1 << PSR64_PAN_SHIFT) |
419 | |
420 | #define PSR64_SS_SHIFT 21 |
421 | #define PSR64_SS (1 << PSR64_SS_SHIFT) |
422 | |
423 | #define PSR64_IL_SHIFT 20 |
424 | #define PSR64_IL (1 << PSR64_IL_SHIFT) |
425 | |
426 | /* |
427 | * SSBS is bit 12 for A64 SPSR and bit 23 for A32 SPSR |
428 | * I do not want to talk about it! |
429 | */ |
430 | #define PSR64_SSBS_SHIFT_32 23 |
431 | #define PSR64_SSBS_SHIFT_64 12 |
432 | #define PSR64_SSBS_32 (1 << PSR64_SSBS_SHIFT_32) |
433 | #define PSR64_SSBS_64 (1 << PSR64_SSBS_SHIFT_64) |
434 | |
435 | /* |
436 | * msr DAIF, Xn and mrs Xn, DAIF transfer into |
437 | * and out of bits 9:6 |
438 | */ |
439 | #define DAIF_DEBUG_SHIFT 9 |
440 | #define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT) |
441 | |
442 | #define DAIF_ASYNC_SHIFT 8 |
443 | #define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT) |
444 | |
445 | #define DAIF_IRQF_SHIFT 7 |
446 | #define DAIF_IRQF (1 << DAIF_IRQF_SHIFT) |
447 | |
448 | #define DAIF_FIQF_SHIFT 6 |
449 | #define DAIF_FIQF (1 << DAIF_FIQF_SHIFT) |
450 | |
451 | #define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) |
452 | #define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) |
453 | |
454 | #define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF)) |
455 | |
456 | #if HAS_ARM_FEAT_SSBS2 |
457 | #define PSR64_SSBS_U32_DEFAULT PSR64_SSBS_32 |
458 | #define PSR64_SSBS_U64_DEFAULT PSR64_SSBS_64 |
459 | #define PSR64_SSBS_KRN_DEFAULT PSR64_SSBS_64 |
460 | #else |
461 | #define PSR64_SSBS_U32_DEFAULT (0) |
462 | #define PSR64_SSBS_U64_DEFAULT (0) |
463 | #define PSR64_SSBS_KRN_DEFAULT (0) |
464 | #endif |
465 | |
466 | /* |
467 | * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer |
468 | * from bits 3:0. |
469 | */ |
470 | #define DAIFSC_DEBUGF (1 << 3) |
471 | #define DAIFSC_ASYNCF (1 << 2) |
472 | #define DAIFSC_IRQF (1 << 1) |
473 | #define DAIFSC_FIQF (1 << 0) |
474 | #define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) |
475 | #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) |
476 | #define DAIFSC_NOASYNC (DAIFSC_DEBUGF | DAIFSC_IRQF | DAIFSC_FIQF) |
477 | |
478 | /* |
479 | * ARM64_TODO: unify with ARM? |
480 | */ |
481 | #define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */ |
482 | |
483 | #define PSR64_MODE_MASK 0x1F |
484 | |
485 | #define PSR64_USER_MASK PSR64_NZCV_MASK |
486 | |
487 | #define PSR64_MODE_USER32_THUMB 0x20 |
488 | |
489 | #define PSR64_MODE_RW_SHIFT 4 |
490 | #define PSR64_MODE_RW_64 0 |
491 | #define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT) |
492 | |
493 | #define PSR64_MODE_EL_SHIFT 2 |
494 | #define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT) |
495 | #define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT) |
496 | #define PSR64_MODE_EL2 (0x2 << PSR64_MODE_EL_SHIFT) |
497 | #define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT) |
498 | #define PSR64_MODE_EL0 0 |
499 | |
500 | #define PSR64_MODE_EL_KERNEL (PSR64_MODE_EL1) |
501 | |
502 | #define PSR64_MODE_SPX 0x1 |
503 | #define PSR64_MODE_SP0 0 |
504 | |
505 | #define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U32_DEFAULT) |
506 | #define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U64_DEFAULT) |
507 | #define PSR64_KERNEL_STANDARD (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0 | PSR64_SSBS_KRN_DEFAULT) |
508 | #if __ARM_PAN_AVAILABLE__ |
509 | #define PSR64_KERNEL_DEFAULT (PSR64_KERNEL_STANDARD | PSR64_PAN) |
510 | #else |
511 | #define PSR64_KERNEL_DEFAULT PSR64_KERNEL_STANDARD |
512 | #endif |
513 | |
514 | #define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0) |
515 | #define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0) |
516 | |
517 | #define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32)) |
518 | #define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32)) |
519 | |
520 | |
521 | |
522 | /* |
523 | * System Control Register (SCTLR) |
524 | */ |
525 | |
526 | #define SCTLR_OTHER (0) |
527 | |
528 | #define SCTLR_DSSBS (1ULL << 44) |
529 | |
530 | |
531 | #define (0) |
532 | |
533 | |
534 | #define SCTLR_RESERVED ((3ULL << 28) | (1ULL << 20)) |
535 | #if defined(HAS_APPLE_PAC) |
536 | |
537 | // 31 PACIA_ENABLED AddPACIA and AuthIA functions enabled |
538 | #define SCTLR_PACIA_ENABLED_SHIFT 31 |
539 | #define SCTLR_PACIA_ENABLED (1ULL << SCTLR_PACIA_ENABLED_SHIFT) |
540 | // 30 PACIB_ENABLED AddPACIB and AuthIB functions enabled |
541 | #define SCTLR_PACIB_ENABLED (1ULL << 30) |
542 | // 29:28 RES1 11 |
543 | // 27 PACDA_ENABLED AddPACDA and AuthDA functions enabled |
544 | #define SCTLR_PACDA_ENABLED (1ULL << 27) |
545 | // 13 PACDB_ENABLED AddPACDB and AuthDB functions enabled |
546 | #define SCTLR_PACDB_ENABLED (1ULL << 13) |
547 | |
548 | #define SCTLR_JOP_KEYS_ENABLED (SCTLR_PACIA_ENABLED | SCTLR_PACDA_ENABLED | SCTLR_PACDB_ENABLED) |
549 | #endif /* defined(HAS_APPLE_PAC) */ |
550 | |
551 | // 26 UCI User Cache Instructions |
552 | #define SCTLR_UCI_ENABLED (1ULL << 26) |
553 | |
554 | // 25 EE Exception Endianness |
555 | #define SCTLR_EE_BIG_ENDIAN (1ULL << 25) |
556 | |
557 | // 24 E0E EL0 Endianness |
558 | #define SCTLR_E0E_BIG_ENDIAN (1ULL << 24) |
559 | |
560 | // 23 SPAN Set PAN |
561 | #define SCTLR_PAN_UNCHANGED (1ULL << 23) |
562 | |
563 | // 22 EIS Taking an exception is a context synchronization event |
564 | #define SCTLR_EIS (1ULL << 22) |
565 | |
566 | // 21 RES0 0 |
567 | // 20 RES1 1 |
568 | |
569 | // 19 WXN Writeable implies eXecute Never |
570 | #define SCTLR_WXN_ENABLED (1ULL << 19) |
571 | |
572 | // 18 nTWE Not trap WFE from EL0 |
573 | #define SCTLR_nTWE_WFE_ENABLED (1ULL << 18) |
574 | |
575 | // 17 RES0 0 |
576 | |
577 | // 16 nTWI Not trap WFI from EL0 |
578 | #define SCTRL_nTWI_WFI_ENABLED (1ULL << 16) |
579 | |
580 | // 15 UCT User Cache Type register (CTR_EL0) |
581 | #define SCTLR_UCT_ENABLED (1ULL << 15) |
582 | |
583 | // 14 DZE User Data Cache Zero (DC ZVA) |
584 | #define SCTLR_DZE_ENABLED (1ULL << 14) |
585 | |
586 | // 12 I Instruction cache enable |
587 | #define SCTLR_I_ENABLED (1ULL << 12) |
588 | |
589 | // 11 EOS Exception return is a context synchronization event |
590 | #define SCTLR_EOS (1ULL << 11) |
591 | |
592 | // 10 EnRCTX EL0 Access to FEAT_SPECRES speculation restriction instructions |
593 | #define SCTLR_EnRCTX (1ULL << 10) |
594 | |
595 | // 9 UMA User Mask Access |
596 | #define SCTLR_UMA_ENABLED (1ULL << 9) |
597 | |
598 | // 8 SED SETEND Disable |
599 | #define SCTLR_SED_DISABLED (1ULL << 8) |
600 | |
601 | // 7 ITD IT Disable |
602 | #define SCTLR_ITD_DISABLED (1ULL << 7) |
603 | |
604 | // 6 RES0 0 |
605 | |
606 | // 5 CP15BEN CP15 Barrier ENable |
607 | #define SCTLR_CP15BEN_ENABLED (1ULL << 5) |
608 | |
609 | // 4 SA0 Stack Alignment check for EL0 |
610 | #define SCTLR_SA0_ENABLED (1ULL << 4) |
611 | |
612 | // 3 SA Stack Alignment check |
613 | #define SCTLR_SA_ENABLED (1ULL << 3) |
614 | |
615 | // 2 C Cache enable |
616 | #define SCTLR_C_ENABLED (1ULL << 2) |
617 | |
618 | // 1 A Alignment check |
619 | #define SCTLR_A_ENABLED (1ULL << 1) |
620 | |
621 | // 0 M MMU enable |
622 | #define SCTLR_M_ENABLED (1ULL << 0) |
623 | |
624 | #if __ARM_ARCH_8_5__ |
625 | #define SCTLR_CSEH_DEFAULT (0) |
626 | #define SCTLR_DSSBS_DEFAULT SCTLR_DSSBS |
627 | #else |
628 | #define SCTLR_CSEH_DEFAULT (SCTLR_EIS | SCTLR_EOS) |
629 | #define SCTLR_DSSBS_DEFAULT (0) |
630 | #endif |
631 | |
632 | #if HAS_APPLE_PAC |
633 | #define SCTLR_ROP_KEYS_DEFAULT SCTLR_PACIB_ENABLED /* IB is ROP */ |
634 | #else /* !HAS_APPLE_PAC */ |
635 | #define SCTLR_ROP_KEYS_DEFAULT 0 |
636 | #endif /* HAS_APPLE_PAC */ |
637 | |
638 | #if HAS_APPLE_PAC |
639 | #define SCTLR_JOP_KEYS_DEFAULT SCTLR_JOP_KEYS_ENABLED |
640 | #else /* !HAS_APPLE_PAC */ |
641 | #define SCTLR_JOP_KEYS_DEFAULT 0 |
642 | #endif |
643 | |
644 | #define SCTLR_EL1_DEFAULT \ |
645 | (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \ |
646 | SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED | \ |
647 | SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED | \ |
648 | SCTLR_CSEH_DEFAULT | SCTLR_DSSBS_DEFAULT | SCTLR_ROP_KEYS_DEFAULT | \ |
649 | SCTLR_JOP_KEYS_DEFAULT | SCTLR_OTHER | SCTLR_EXTRA ) |
650 | |
651 | /* |
652 | * Coprocessor Access Control Register (CPACR) |
653 | * |
654 | * 31 28 27 22 21 20 19 0 |
655 | * +---+---+------+------+--------------------+ |
656 | * |000|TTA|000000| FPEN |00000000000000000000| |
657 | * +---+---+------+------+--------------------+ |
658 | * |
659 | * where: |
660 | * TTA: Trace trap |
661 | * FPEN: Floating point enable |
662 | */ |
663 | #define CPACR_TTA_SHIFT 28 |
664 | #define CPACR_TTA (1 << CPACR_TTA_SHIFT) |
665 | |
666 | |
667 | #define CPACR_FPEN_SHIFT 20 |
668 | #define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT) |
669 | #define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT) |
670 | |
671 | |
672 | /* |
673 | * FPSR: Floating Point Status Register |
674 | * |
675 | * 31 30 29 28 27 26 7 6 4 3 2 1 0 |
676 | * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ |
677 | * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC| |
678 | * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ |
679 | */ |
680 | |
681 | #define FPSR_N_SHIFT 31 |
682 | #define FPSR_Z_SHIFT 30 |
683 | #define FPSR_C_SHIFT 29 |
684 | #define FPSR_V_SHIFT 28 |
685 | #define FPSR_QC_SHIFT 27 |
686 | #define FPSR_IDC_SHIFT 7 |
687 | #define FPSR_IXC_SHIFT 4 |
688 | #define FPSR_UFC_SHIFT 3 |
689 | #define FPSR_OFC_SHIFT 2 |
690 | #define FPSR_DZC_SHIFT 1 |
691 | #define FPSR_IOC_SHIFT 0 |
692 | #define FPSR_N (1 << FPSR_N_SHIFT) |
693 | #define FPSR_Z (1 << FPSR_Z_SHIFT) |
694 | #define FPSR_C (1 << FPSR_C_SHIFT) |
695 | #define FPSR_V (1 << FPSR_V_SHIFT) |
696 | #define FPSR_QC (1 << FPSR_QC_SHIFT) |
697 | #define FPSR_IDC (1 << FPSR_IDC_SHIFT) |
698 | #define FPSR_IXC (1 << FPSR_IXC_SHIFT) |
699 | #define FPSR_UFC (1 << FPSR_UFC_SHIFT) |
700 | #define FPSR_OFC (1 << FPSR_OFC_SHIFT) |
701 | #define FPSR_DZC (1 << FPSR_DZC_SHIFT) |
702 | #define FPSR_IOC (1 << FPSR_IOC_SHIFT) |
703 | |
704 | /* |
705 | * A mask for all for all of the bits that are not RAZ for FPSR; this |
706 | * is primarily for converting between a 32-bit view of NEON state |
707 | * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). |
708 | */ |
709 | #define FPSR_MASK \ |
710 | (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | FPSR_IDC | FPSR_IXC | \ |
711 | FPSR_UFC | FPSR_OFC | FPSR_DZC | FPSR_IOC) |
712 | |
713 | /* |
714 | * FPCR: Floating Point Control Register |
715 | * |
716 | * 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0 |
717 | * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ |
718 | * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000| |
719 | * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ |
720 | */ |
721 | |
722 | #define FPCR_AHP_SHIFT 26 |
723 | #define FPCR_DN_SHIFT 25 |
724 | #define FPCR_FZ_SHIFT 24 |
725 | #define FPCR_RMODE_SHIFT 22 |
726 | #define FPCR_STRIDE_SHIFT 20 |
727 | #define FPCR_LEN_SHIFT 16 |
728 | #define FPCR_IDE_SHIFT 15 |
729 | #define FPCR_IXE_SHIFT 12 |
730 | #define FPCR_UFE_SHIFT 11 |
731 | #define FPCR_OFE_SHIFT 10 |
732 | #define FPCR_DZE_SHIFT 9 |
733 | #define FPCR_IOE_SHIFT 8 |
734 | #define FPCR_AHP (1 << FPCR_AHP_SHIFT) |
735 | #define FPCR_DN (1 << FPCR_DN_SHIFT) |
736 | #define FPCR_FZ (1 << FPCR_FZ_SHIFT) |
737 | #define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT) |
738 | #define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT) |
739 | #define FPCR_LEN (0x7 << FPCR_LEN_SHIFT) |
740 | #define FPCR_IDE (1 << FPCR_IDE_SHIFT) |
741 | #define FPCR_IXE (1 << FPCR_IXE_SHIFT) |
742 | #define FPCR_UFE (1 << FPCR_UFE_SHIFT) |
743 | #define FPCR_OFE (1 << FPCR_OFE_SHIFT) |
744 | #define FPCR_DZE (1 << FPCR_DZE_SHIFT) |
745 | #define FPCR_IOE (1 << FPCR_IOE_SHIFT) |
746 | #define FPCR_DEFAULT (0) |
747 | #define FPCR_DEFAULT_32 (FPCR_DN|FPCR_FZ) |
748 | |
749 | /* |
750 | * A mask for all for all of the bits that are not RAZ for FPCR; this |
751 | * is primarily for converting between a 32-bit view of NEON state |
752 | * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). |
753 | */ |
754 | #define FPCR_MASK \ |
755 | (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | FPCR_STRIDE | FPCR_LEN | \ |
756 | FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE) |
757 | |
758 | /* |
759 | * Translation Control Register (TCR) |
760 | * |
761 | * Legacy: |
762 | * |
763 | * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 |
764 | * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ |
765 | * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| |
766 | * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ |
767 | * |
768 | * Current (with 16KB granule support): |
769 | * |
770 | * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 |
771 | * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ |
772 | * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| |
773 | * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ |
774 | * |
775 | * TBI1: Top Byte Ignored for TTBR1 region |
776 | * TBI0: Top Byte Ignored for TTBR0 region |
777 | * AS: ASID Size |
778 | * IPS: Physical Address Size limit |
779 | * TG1: Granule Size for TTBR1 region |
780 | * SH1: Shareability for TTBR1 region |
781 | * ORGN1: Outer Cacheability for TTBR1 region |
782 | * IRGN1: Inner Cacheability for TTBR1 region |
783 | * EPD1: Translation table walk disable for TTBR1 |
784 | * A1: ASID selection from TTBR1 enable |
785 | * T1SZ: Virtual address size for TTBR1 |
786 | * TG0: Granule Size for TTBR0 region |
787 | * SH0: Shareability for TTBR0 region |
788 | * ORGN0: Outer Cacheability for TTBR0 region |
789 | * IRGN0: Inner Cacheability for TTBR0 region |
790 | * T0SZ: Virtual address size for TTBR0 |
791 | */ |
792 | |
793 | #define TCR_T0SZ_SHIFT 0ULL |
794 | #define TCR_TSZ_BITS 6ULL |
795 | #define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL) |
796 | |
797 | #define TCR_IRGN0_SHIFT 8ULL |
798 | #define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT) |
799 | #define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT) |
800 | #define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT) |
801 | #define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT) |
802 | |
803 | #define TCR_ORGN0_SHIFT 10ULL |
804 | #define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT) |
805 | #define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT) |
806 | #define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT) |
807 | #define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT) |
808 | |
809 | #define TCR_SH0_SHIFT 12ULL |
810 | #define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT) |
811 | #define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT) |
812 | #define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT) |
813 | |
814 | #define TCR_TG0_GRANULE_SHIFT (14ULL) |
815 | #define TCR_TG0_GRANULE_BITS (2ULL) |
816 | #define TCR_TG0_GRANULE_MASK ((1ULL << TCR_TG0_GRANULE_BITS) - 1ULL) |
817 | |
818 | #define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT) |
819 | #define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT) |
820 | #define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT) |
821 | |
822 | #if __ARM_16K_PG__ |
823 | #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB) |
824 | #else |
825 | #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB) |
826 | #endif |
827 | |
828 | #define TCR_T1SZ_SHIFT 16ULL |
829 | |
830 | #define TCR_A1_ASID1 (1ULL << 22ULL) |
831 | #define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL) |
832 | |
833 | #define TCR_IRGN1_SHIFT 24ULL |
834 | #define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT) |
835 | #define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT) |
836 | #define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT) |
837 | #define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT) |
838 | |
839 | #define TCR_ORGN1_SHIFT 26ULL |
840 | #define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT) |
841 | #define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT) |
842 | #define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT) |
843 | #define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT) |
844 | |
845 | #define TCR_SH1_SHIFT 28ULL |
846 | #define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT) |
847 | #define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT) |
848 | #define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT) |
849 | |
850 | #define TCR_TG1_GRANULE_SHIFT 30ULL |
851 | #define TCR_TG1_GRANULE_BITS (2ULL) |
852 | #define TCR_TG1_GRANULE_MASK ((1ULL << TCR_TG1_GRANULE_BITS) - 1ULL) |
853 | |
854 | #define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT) |
855 | #define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT) |
856 | #define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT) |
857 | |
858 | #if __ARM_16K_PG__ |
859 | #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB) |
860 | #else |
861 | #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB) |
862 | #endif |
863 | |
864 | #define TCR_IPS_SHIFT 32ULL |
865 | #define TCR_IPS_BITS 3ULL |
866 | #define TCR_IPS_MASK ((1ULL << TCR_IPS_BITS) - 1ULL) |
867 | #define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT) |
868 | #define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT) |
869 | #define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT) |
870 | #define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT) |
871 | #define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT) |
872 | #define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT) |
873 | |
874 | #define TCR_AS_16BIT_ASID (1ULL << 36) |
875 | #define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37) |
876 | #define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38) |
877 | #define TCR_TBID0_TBI_DATA_ONLY (1ULL << 51) |
878 | #define TCR_TBID1_TBI_DATA_ONLY (1ULL << 52) |
879 | |
880 | #if defined(HAS_APPLE_PAC) |
881 | #define TCR_TBID0_ENABLE TCR_TBID0_TBI_DATA_ONLY |
882 | #define TCR_TBID1_ENABLE TCR_TBID1_TBI_DATA_ONLY |
883 | #else |
884 | #define TCR_TBID0_ENABLE 0 |
885 | #define TCR_TBID1_ENABLE 0 |
886 | #endif |
887 | |
888 | #define TCR_E0PD0_BIT (1ULL << 55) |
889 | #define TCR_E0PD1_BIT (1ULL << 56) |
890 | |
891 | #if defined(HAS_E0PD) |
892 | #define TCR_E0PD_VALUE (TCR_E0PD1_BIT) |
893 | #else |
894 | #define TCR_E0PD_VALUE 0 |
895 | #endif |
896 | |
897 | |
898 | #define 0 |
899 | |
900 | |
901 | /* |
902 | * Multiprocessor Affinity Register (MPIDR_EL1) |
903 | * |
904 | * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+ |
905 | * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0| |
906 | * +---------------------------------+--+-----+--+-----+----+----+ |
907 | * |
908 | * where |
909 | * U: Uniprocessor |
910 | * MT: Multi-threading at lowest affinity level |
911 | * Aff2: "1" - PCORE, "0" - ECORE |
912 | * Aff1: Cluster ID |
913 | * Aff0: CPU ID |
914 | */ |
915 | #define MPIDR_AFF0_SHIFT 0 |
916 | #define MPIDR_AFF0_WIDTH 8 |
917 | #define MPIDR_AFF0_MASK (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT) |
918 | #define MPIDR_AFF1_SHIFT 8 |
919 | #define MPIDR_AFF1_WIDTH 8 |
920 | #define MPIDR_AFF1_MASK (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT) |
921 | #define MPIDR_AFF2_SHIFT 16 |
922 | #define MPIDR_AFF2_WIDTH 8 |
923 | #define MPIDR_AFF2_MASK (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT) |
924 | |
925 | /* |
926 | * TXSZ indicates the size of the range a TTBR covers. Currently, |
927 | * we support the following: |
928 | * |
929 | * 4KB pages, full page L1: 39 bit range. |
930 | * 4KB pages, sub-page L1: 38 bit range. |
931 | * 16KB pages, full page L1: 47 bit range. |
932 | * 16KB pages, sub-page L1: 39 bit range. |
933 | * 16KB pages, two level page tables: 36 bit range. |
934 | */ |
935 | #if __ARM_KERNEL_PROTECT__ |
936 | /* |
937 | * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the |
938 | * address space is used for the mappings that will remain in place when in EL0. |
939 | * As a result, 1 bit less of address space is available to the rest of the |
940 | * the kernel. |
941 | */ |
942 | #endif /* __ARM_KERNEL_PROTECT__ */ |
943 | #ifdef __ARM_16K_PG__ |
944 | #if __ARM64_PMAP_SUBPAGE_L1__ |
945 | #define T0SZ_BOOT 25ULL |
946 | #else /* !__ARM64_PMAP_SUBPAGE_L1__ */ |
947 | #define T0SZ_BOOT 17ULL |
948 | #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */ |
949 | #else /* __ARM_16K_PG__ */ |
950 | #if __ARM64_PMAP_SUBPAGE_L1__ |
951 | #define T0SZ_BOOT 26ULL |
952 | #else /* __ARM64_PMAP_SUBPAGE_L1__ */ |
953 | #define T0SZ_BOOT 25ULL |
954 | #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ |
955 | #endif /* __ARM_16K_PG__ */ |
956 | |
957 | #if defined(APPLE_ARM64_ARCH_FAMILY) |
958 | /* T0SZ must be the same as T1SZ */ |
959 | #define T1SZ_BOOT T0SZ_BOOT |
960 | #else /* defined(APPLE_ARM64_ARCH_FAMILY) */ |
961 | #ifdef __ARM_16K_PG__ |
962 | #if __ARM64_PMAP_SUBPAGE_L1__ |
963 | #define T1SZ_BOOT 25ULL |
964 | #else /* !__ARM64_PMAP_SUBPAGE_L1__ */ |
965 | #define T1SZ_BOOT 17ULL |
966 | #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */ |
967 | #else /* __ARM_16K_PG__ */ |
968 | #if __ARM64_PMAP_SUBPAGE_L1__ |
969 | #define T1SZ_BOOT 26ULL |
970 | #else /* __ARM64_PMAP_SUBPAGE_L1__ */ |
971 | #define T1SZ_BOOT 25ULL |
972 | #endif /*__ARM64_PMAP_SUBPAGE_L1__*/ |
973 | #endif /* __ARM_16K_PG__ */ |
974 | #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ |
975 | |
976 | #if __ARM_42BIT_PA_SPACE__ |
977 | #define TCR_IPS_VALUE TCR_IPS_42BITS |
978 | #else /* !__ARM_42BIT_PA_SPACE__ */ |
979 | #define TCR_IPS_VALUE TCR_IPS_40BITS |
980 | #endif /* !__ARM_42BIT_PA_SPACE__ */ |
981 | |
982 | #if CONFIG_KERNEL_TBI |
983 | #define TCR_EL1_DTBI (TCR_TBI1_TOPBYTE_IGNORED | TCR_TBID1_ENABLE) |
984 | #else /* CONFIG_KERNEL_TBI */ |
985 | #define TCR_EL1_DTBI 0 |
986 | #endif /* CONFIG_KERNEL_TBI */ |
987 | |
988 | #if HAS_16BIT_ASID |
989 | #define TCR_EL1_ASID TCR_AS_16BIT_ASID |
990 | #else /* HAS_16BIT_ASID */ |
991 | #define TCR_EL1_ASID 0 |
992 | #endif /* HAS_16BIT_ASID */ |
993 | |
994 | #define TCR_EL1_BASE \ |
995 | (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | \ |
996 | TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | \ |
997 | TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \ |
998 | TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) | \ |
999 | TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE) | TCR_E0PD_VALUE | \ |
1000 | TCR_EL1_DTBI | TCR_EL1_ASID | TCR_EL1_EXTRA) |
1001 | |
1002 | #if __ARM_KERNEL_PROTECT__ |
1003 | #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) |
1004 | #define T1SZ_USER (T1SZ_BOOT + 1) |
1005 | #define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) |
1006 | #else |
1007 | #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) |
1008 | #endif /* __ARM_KERNEL_PROTECT__ */ |
1009 | |
1010 | #define TCR_EL1_4KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_4KB)) |
1011 | #define TCR_EL1_16KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_16KB)) |
1012 | |
1013 | |
1014 | |
1015 | |
1016 | /* |
1017 | * Monitor Debug System Control Register (MDSCR) |
1018 | */ |
1019 | |
1020 | #define MDSCR_TFO_SHIFT 31 |
1021 | #define MDSCR_TFO (1ULL << MDSCR_TFO_SHIFT) |
1022 | #define MDSCR_RXFULL_SHIFT 30 |
1023 | #define MDSCR_RXFULL (1ULL << MDSCR_RXFULL_SHIFT) |
1024 | #define MDSCR_TXFULL_SHIFT 29 |
1025 | #define MDSCR_TXFULL (1ULL << MDSCR_TXFULL_SHIFT) |
1026 | #define MDSCR_RXO_SHIFT 27 |
1027 | #define MDSCR_RXO (1ULL << MDSCR_RXO_SHIFT) |
1028 | #define MDSCR_TXU_SHIFT 26 |
1029 | #define MDSCR_TXU (1ULL << MDSCR_TXU_SHIFT) |
1030 | #define MDSCR_INTDIS_SHIFT 22 |
1031 | #define MDSCR_INTDIS_MASK (0x2U << MDSCR_INTDIS_SHIFT) |
1032 | #define MDSCR_TDA_SHIFT 21 |
1033 | #define MDSCR_TDA (1ULL << MDSCR_TDA_SHIFT) |
1034 | #define MDSCR_SC2_SHIFT 19 |
1035 | #define MDSCR_SC2 (1ULL << MDSCR_SC2_SHIFT) |
1036 | #define MDSCR_MDE_SHIFT 15 |
1037 | #define MDSCR_MDE (1ULL << MDSCR_MDE_SHIFT) |
1038 | #define MDSCR_HDE_SHIFT 14 |
1039 | #define MDSCR_HDE (1ULL << MDSCR_HDE_SHIFT) |
1040 | #define MDSCR_KDE_SHIFT 13 |
1041 | #define MDSCR_KDE (1ULL << MDSCR_KDE_SHIFT) |
1042 | #define MDSCR_TDCC_SHIFT 12 |
1043 | #define MDSCR_TDCC (1ULL << MDSCR_TDCC_SHIFT) |
1044 | #define MDSCR_ERR_SHIFT 6 |
1045 | #define MDSCR_ERR (1ULL << MDSCR_ERR_SHIFT) |
1046 | #define MDSCR_SS_SHIFT 0 |
1047 | #define MDSCR_SS (1ULL << MDSCR_SS_SHIFT) |
1048 | |
1049 | /* |
1050 | * Translation Table Base Register (TTBR) |
1051 | * |
1052 | * 63 48 47 x x-1 0 |
1053 | * +--------+------------------+------+ |
1054 | * | ASID | Base Address | zero | |
1055 | * +--------+------------------+------+ |
1056 | * |
1057 | */ |
1058 | #define TTBR_ASID_SHIFT 48 |
1059 | #define TTBR_ASID_MASK 0xffff000000000000 |
1060 | |
1061 | #define TTBR_BADDR_MASK 0x0000ffffffffffff |
1062 | |
1063 | /* |
1064 | * Memory Attribute Indirection Register |
1065 | * |
1066 | * 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0 |
1067 | * +-------+-------+-------+-------+-------+-------+-------+-------+ |
1068 | * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 | |
1069 | * +-------+-------+-------+-------+-------+-------+-------+-------+ |
1070 | * |
1071 | */ |
1072 | |
1073 | #define MAIR_ATTR_SHIFT(x) (8*(x)) |
1074 | |
1075 | /* Strongly ordered or device memory attributes */ |
1076 | #define MAIR_OUTER_STRONGLY_ORDERED 0x0 |
1077 | #define MAIR_OUTER_DEVICE 0x0 |
1078 | |
1079 | #define MAIR_INNER_STRONGLY_ORDERED 0x0 |
1080 | #define MAIR_INNER_DEVICE 0x4 |
1081 | |
1082 | /* Normal memory attributes */ |
1083 | #define MAIR_OUTER_NON_CACHEABLE 0x40 |
1084 | #define MAIR_OUTER_WRITE_THROUGH 0x80 |
1085 | #define MAIR_OUTER_WRITE_BACK 0xc0 |
1086 | |
1087 | #define MAIR_INNER_NON_CACHEABLE 0x4 |
1088 | #define MAIR_INNER_WRITE_THROUGH 0x8 |
1089 | #define MAIR_INNER_WRITE_BACK 0xc |
1090 | |
1091 | /* Allocate policy for cacheable memory */ |
1092 | #define MAIR_OUTER_WRITE_ALLOCATE 0x10 |
1093 | #define MAIR_OUTER_READ_ALLOCATE 0x20 |
1094 | |
1095 | #define MAIR_INNER_WRITE_ALLOCATE 0x1 |
1096 | #define MAIR_INNER_READ_ALLOCATE 0x2 |
1097 | |
1098 | /* Memory Atribute Encoding */ |
1099 | |
1100 | /* |
1101 | * Device memory types: |
1102 | * G (gathering): multiple reads/writes can be combined |
1103 | * R (reordering): reads or writes may reach device out of program order |
1104 | * E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes) |
1105 | */ |
1106 | #if HAS_FEAT_XS |
1107 | |
1108 | #define MAIR_DISABLE 0x01 /* Device Memory, nGnRnE (strongly ordered), XS=0 */ |
1109 | #define MAIR_POSTED_XS 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes), XS=1 */ |
1110 | #define MAIR_POSTED_COMBINED_REORDERED_XS 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=1 */ |
1111 | #define MAIR_POSTED_COMBINED_REORDERED 0x0D /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=0 */ |
1112 | #define MAIR_WRITECOMB 0x40 /* Normal Memory, Non-Cacheable, XS=0 */ |
1113 | #define MAIR_WRITETHRU 0xA0 /* Normal Memory, Write-through, XS=0 */ |
1114 | #define MAIR_WRITEBACK 0xFF /* Normal Memory, Write-back, XS=0 */ |
1115 | |
1116 | /* |
1117 | * Memory Attribute Index. If these values change, please also update the pmap |
1118 | * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64). |
1119 | */ |
1120 | #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */ |
1121 | #define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */ |
1122 | #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */ |
1123 | #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */ |
1124 | #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */ |
1125 | #define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */ |
1126 | #define CACHE_ATTRINDX_POSTED_XS 0x5 /* no cache, no buffer, posted writes (device memory), XS=1 */ |
1127 | #define CACHE_ATTRINDX_POSTED CACHE_ATTRINDX_POSTED_XS /* posted mappings use XS by default */ |
1128 | #define CACHE_ATTRINDX_POSTED_REORDERED CACHE_ATTRINDX_DISABLE /* no need for device-nGRE on newer SoCs, fallback to nGnRnE */ |
1129 | #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x6 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=0 */ |
1130 | #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED_XS 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=1 */ |
1131 | #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK |
1132 | #define CACHE_ATTRINDX_N_INDICES (8ULL) |
1133 | |
1134 | #else |
1135 | |
1136 | #define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */ |
1137 | #define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */ |
1138 | #define MAIR_POSTED_REORDERED 0x08 /* Device Memory, nGRE (reorderable, posted writes) */ |
1139 | #define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */ |
1140 | #define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */ |
1141 | #define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */ |
1142 | #define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */ |
1143 | |
1144 | /* |
1145 | * Memory Attribute Index. If these values change, please also update the pmap |
1146 | * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64). |
1147 | */ |
1148 | #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */ |
1149 | #define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */ |
1150 | #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */ |
1151 | #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */ |
1152 | #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */ |
1153 | #define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */ |
1154 | #define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes (device memory) */ |
1155 | #define CACHE_ATTRINDX_POSTED_REORDERED 0x6 /* no cache, reorderable access, posted writes (device memory) */ |
1156 | #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */ |
1157 | #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK |
1158 | #define CACHE_ATTRINDX_N_INDICES (8ULL) |
1159 | |
1160 | #endif /* HAS_FEAT_XS */ |
1161 | |
1162 | #if HAS_UCNORMAL_MEM |
1163 | #define CACHE_ATTRINDX_RT CACHE_ATTRINDX_WRITECOMB |
1164 | #else |
1165 | #define CACHE_ATTRINDX_RT CACHE_ATTRINDX_DISABLE |
1166 | #endif /* HAS_UCNORMAL_MEM */ |
1167 | |
1168 | |
1169 | |
1170 | /* |
1171 | * Access protection bit values (TTEs and PTEs), stage 1 |
1172 | * |
1173 | * Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access) |
1174 | */ |
1175 | #define AP_RWNA 0x0 /* priv=read-write, user=no-access */ |
1176 | #define AP_RWRW 0x1 /* priv=read-write, user=read-write */ |
1177 | #define AP_RONA 0x2 /* priv=read-only, user=no-access */ |
1178 | #define AP_RORO 0x3 /* priv=read-only, user=read-only */ |
1179 | #define AP_MASK 0x3 /* mask to find ap bits */ |
1180 | |
1181 | /* |
1182 | * Shareability attributes |
1183 | */ |
1184 | #define SH_NONE 0x0 /* Non shareable */ |
1185 | #define SH_NONE 0x0 /* Device shareable */ |
1186 | #define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */ |
1187 | #define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */ |
1188 | #define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */ |
1189 | |
1190 | |
1191 | /* |
1192 | * ARM Page Granule |
1193 | */ |
1194 | #ifdef __ARM_16K_PG__ |
1195 | #define ARM_PGSHIFT 14 |
1196 | #else |
1197 | #define ARM_PGSHIFT 12 |
1198 | #endif |
1199 | #define ARM_PGBYTES (1 << ARM_PGSHIFT) |
1200 | #define ARM_PGMASK (ARM_PGBYTES-1) |
1201 | |
1202 | /* |
1203 | * L0 Translation table |
1204 | * |
1205 | * 4KB granule size: |
1206 | * Each translation table is 4KB |
1207 | * 512 64-bit entries of 512GB (2^39) of address space. |
1208 | * Covers 256TB (2^48) of address space. |
1209 | * |
1210 | * 16KB granule size: |
1211 | * Each translation table is 16KB |
1212 | * 2 64-bit entries of 128TB (2^47) of address space. |
1213 | * Covers 256TB (2^48) of address space. |
1214 | */ |
1215 | |
1216 | /* 16K L0 */ |
1217 | #define ARM_16K_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */ |
1218 | #define ARM_16K_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */ |
1219 | #define ARM_16K_TT_L0_SHIFT 47 /* page descriptor shift */ |
1220 | #define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */ |
1221 | |
1222 | /* 4K L0 */ |
1223 | #define ARM_4K_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */ |
1224 | #define ARM_4K_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */ |
1225 | #define ARM_4K_TT_L0_SHIFT 39 /* page descriptor shift */ |
1226 | #define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */ |
1227 | |
1228 | /* |
1229 | * L1 Translation table |
1230 | * |
1231 | * 4KB granule size: |
1232 | * Each translation table is 4KB |
1233 | * 512 64-bit entries of 1GB (2^30) of address space. |
1234 | * Covers 512GB (2^39) of address space. |
1235 | * |
1236 | * 16KB granule size: |
1237 | * Each translation table is 16KB |
1238 | * 2048 64-bit entries of 64GB (2^36) of address space. |
1239 | * Covers 128TB (2^47) of address space. |
1240 | */ |
1241 | |
1242 | /* 16K L1 */ |
1243 | #define ARM_16K_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */ |
1244 | #define ARM_16K_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */ |
1245 | #define ARM_16K_TT_L1_SHIFT 36 /* page descriptor shift */ |
1246 | #if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__ |
1247 | /* This config supports 512GB per TTBR. */ |
1248 | #define ARM_16K_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */ |
1249 | #else /* __ARM64_PMAP_SUBPAGE_L1__ */ |
1250 | #define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */ |
1251 | #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ |
1252 | |
1253 | /* 4K L1 */ |
1254 | #define ARM_4K_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */ |
1255 | #define ARM_4K_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */ |
1256 | #define ARM_4K_TT_L1_SHIFT 30 /* page descriptor shift */ |
1257 | #if __ARM64_PMAP_SUBPAGE_L1__ && !__ARM_16K_PG__ |
1258 | /* This config supports 256GB per TTBR. */ |
1259 | #define ARM_4K_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */ |
1260 | #else /* __ARM64_PMAP_SUBPAGE_L1__ */ |
1261 | #define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */ |
1262 | #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ |
1263 | |
1264 | /* some sugar for getting pointers to page tables and entries */ |
1265 | |
1266 | #define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT) |
1267 | #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT) |
1268 | #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT) |
1269 | |
1270 | #define L2_TABLE_VA(tte) ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK)) |
1271 | #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK)) |
1272 | |
1273 | /* |
1274 | * L2 Translation table |
1275 | * |
1276 | * 4KB granule size: |
1277 | * Each translation table is 4KB |
1278 | * 512 64-bit entries of 2MB (2^21) of address space. |
1279 | * Covers 1GB (2^30) of address space. |
1280 | * |
1281 | * 16KB granule size: |
1282 | * Each translation table is 16KB |
1283 | * 2048 64-bit entries of 32MB (2^25) of address space. |
1284 | * Covers 64GB (2^36) of address space. |
1285 | */ |
1286 | |
1287 | /* 16K L2 */ |
1288 | #define ARM_16K_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */ |
1289 | #define ARM_16K_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */ |
1290 | #define ARM_16K_TT_L2_SHIFT 25 /* page descriptor shift */ |
1291 | #define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */ |
1292 | |
1293 | /* 4K L2 */ |
1294 | #define ARM_4K_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */ |
1295 | #define ARM_4K_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */ |
1296 | #define ARM_4K_TT_L2_SHIFT 21 /* page descriptor shift */ |
1297 | #define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */ |
1298 | |
1299 | /* |
1300 | * L3 Translation table |
1301 | * |
1302 | * 4KB granule size: |
1303 | * Each translation table is 4KB |
1304 | * 512 64-bit entries of 4KB (2^12) of address space. |
1305 | * Covers 2MB (2^21) of address space. |
1306 | * |
1307 | * 16KB granule size: |
1308 | * Each translation table is 16KB |
1309 | * 2048 64-bit entries of 16KB (2^14) of address space. |
1310 | * Covers 32MB (2^25) of address space. |
1311 | */ |
1312 | |
1313 | /* 16K L3 */ |
1314 | #define ARM_16K_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ |
1315 | #define ARM_16K_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */ |
1316 | #define ARM_16K_TT_L3_SHIFT 14 /* page descriptor shift */ |
1317 | #define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */ |
1318 | |
1319 | /* 4K L3 */ |
1320 | #define ARM_4K_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ |
1321 | #define ARM_4K_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */ |
1322 | #define ARM_4K_TT_L3_SHIFT 12 /* page descriptor shift */ |
1323 | #define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */ |
1324 | |
1325 | #ifdef __ARM_16K_PG__ |
1326 | |
1327 | /* Native L0 defines */ |
1328 | #define ARM_TT_L0_SIZE ARM_16K_TT_L0_SIZE |
1329 | #define ARM_TT_L0_OFFMASK ARM_16K_TT_L0_OFFMASK |
1330 | #define ARM_TT_L0_SHIFT ARM_16K_TT_L0_SHIFT |
1331 | #define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK |
1332 | |
1333 | /* Native L1 defines */ |
1334 | #define ARM_TT_L1_SIZE ARM_16K_TT_L1_SIZE |
1335 | #define ARM_TT_L1_OFFMASK ARM_16K_TT_L1_OFFMASK |
1336 | #define ARM_TT_L1_SHIFT ARM_16K_TT_L1_SHIFT |
1337 | #define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK |
1338 | |
1339 | /* Native L2 defines */ |
1340 | #define ARM_TT_L2_SIZE ARM_16K_TT_L2_SIZE |
1341 | #define ARM_TT_L2_OFFMASK ARM_16K_TT_L2_OFFMASK |
1342 | #define ARM_TT_L2_SHIFT ARM_16K_TT_L2_SHIFT |
1343 | #define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK |
1344 | |
1345 | /* Native L3 defines */ |
1346 | #define ARM_TT_L3_SIZE ARM_16K_TT_L3_SIZE |
1347 | #define ARM_TT_L3_OFFMASK ARM_16K_TT_L3_OFFMASK |
1348 | #define ARM_TT_L3_SHIFT ARM_16K_TT_L3_SHIFT |
1349 | #define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK |
1350 | |
1351 | #else /* !__ARM_16K_PG__ */ |
1352 | |
1353 | /* Native L0 defines */ |
1354 | #define ARM_TT_L0_SIZE ARM_4K_TT_L0_SIZE |
1355 | #define ARM_TT_L0_OFFMASK ARM_4K_TT_L0_OFFMASK |
1356 | #define ARM_TT_L0_SHIFT ARM_4K_TT_L0_SHIFT |
1357 | #define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK |
1358 | |
1359 | /* Native L1 defines */ |
1360 | #define ARM_TT_L1_SIZE ARM_4K_TT_L1_SIZE |
1361 | #define ARM_TT_L1_OFFMASK ARM_4K_TT_L1_OFFMASK |
1362 | #define ARM_TT_L1_SHIFT ARM_4K_TT_L1_SHIFT |
1363 | #define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK |
1364 | |
1365 | /* Native L2 defines */ |
1366 | #define ARM_TT_L2_SIZE ARM_4K_TT_L2_SIZE |
1367 | #define ARM_TT_L2_OFFMASK ARM_4K_TT_L2_OFFMASK |
1368 | #define ARM_TT_L2_SHIFT ARM_4K_TT_L2_SHIFT |
1369 | #define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK |
1370 | |
1371 | /* Native L3 defines */ |
1372 | #define ARM_TT_L3_SIZE ARM_4K_TT_L3_SIZE |
1373 | #define ARM_TT_L3_OFFMASK ARM_4K_TT_L3_OFFMASK |
1374 | #define ARM_TT_L3_SHIFT ARM_4K_TT_L3_SHIFT |
1375 | #define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK |
1376 | |
1377 | #endif /* !__ARM_16K_PG__ */ |
1378 | |
1379 | /* |
1380 | * Convenience definitions for: |
1381 | * ARM_TT_LEAF: The last level of the configured page table format. |
1382 | * ARM_TT_TWIG: The second to last level of the configured page table format. |
1383 | * ARM_TT_ROOT: The first level of the configured page table format. |
1384 | * |
1385 | * My apologies to any botanists who may be reading this. |
1386 | */ |
1387 | #define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE |
1388 | #define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK |
1389 | #define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT |
1390 | #define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK |
1391 | |
1392 | #define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE |
1393 | #define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK |
1394 | #define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT |
1395 | #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK |
1396 | |
1397 | #define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE |
1398 | #define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK |
1399 | #define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT |
1400 | #define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK |
1401 | |
1402 | /* |
1403 | * 4KB granule size: |
1404 | * |
1405 | * Level 0 Translation Table Entry |
1406 | * |
1407 | * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 |
1408 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1409 | * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| |
1410 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1411 | * |
1412 | * Level 1 Translation Table Entry |
1413 | * |
1414 | * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 |
1415 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1416 | * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| |
1417 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1418 | * |
1419 | * Level 1 Translation Block Entry |
1420 | * |
1421 | * 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0 |
1422 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
1423 | * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| |
1424 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
1425 | * |
1426 | * Level 2 Translation Table Entry |
1427 | * |
1428 | * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 |
1429 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1430 | * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| |
1431 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1432 | * |
1433 | * Level 2 Translation Block Entry |
1434 | * |
1435 | * 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0 |
1436 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
1437 | * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| |
1438 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
1439 | * |
1440 | * 16KB granule size: |
1441 | * |
1442 | * Level 0 Translation Table Entry |
1443 | * |
1444 | * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 |
1445 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1446 | * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| |
1447 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1448 | * |
1449 | * Level 1 Translation Table Entry |
1450 | * |
1451 | * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 |
1452 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1453 | * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| |
1454 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1455 | * |
1456 | * Level 2 Translation Table Entry |
1457 | * |
1458 | * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 |
1459 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1460 | * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| |
1461 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
1462 | * |
1463 | * Level 2 Translation Block Entry |
1464 | * |
1465 | * 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0 |
1466 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
1467 | * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| |
1468 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
1469 | * |
1470 | * where: |
1471 | * nG: notGlobal bit |
1472 | * SH: Shareability field |
1473 | * AP: access protection |
1474 | * XN: eXecute Never bit |
1475 | * PXN: Privilege eXecute Never bit |
1476 | * NS: Non-Secure bit |
1477 | * HINT: 16 entry continuguous output hint |
1478 | * AttrIdx: Memory Attribute Index |
1479 | */ |
1480 | |
1481 | #define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */ |
1482 | #ifdef __ARM_16K_PG__ |
1483 | #define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */ |
1484 | #else |
1485 | #define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */ |
1486 | #endif |
1487 | |
1488 | #define ARM_TTE_MAX (TTE_PGENTRIES) |
1489 | |
1490 | #define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */ |
1491 | #define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */ |
1492 | |
1493 | #define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */ |
1494 | |
1495 | #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ |
1496 | #define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */ |
1497 | #define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */ |
1498 | #define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL |
1499 | #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ |
1500 | |
1501 | #ifdef __ARM_16K_PG__ |
1502 | /* |
1503 | * Note that L0/L1 block entries are disallowed for the 16KB granule size; what |
1504 | * are we doing with these? |
1505 | */ |
1506 | #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */ |
1507 | #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */ |
1508 | #define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */ |
1509 | #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */ |
1510 | #define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */ |
1511 | #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */ |
1512 | #else |
1513 | #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */ |
1514 | #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */ |
1515 | #define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */ |
1516 | #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */ |
1517 | #define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */ |
1518 | #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */ |
1519 | #endif |
1520 | |
1521 | #define ARM_TTE_BLOCK_APSHIFT 6 |
1522 | #define ARM_TTE_BLOCK_AP(x) ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */ |
1523 | #define ARM_TTE_BLOCK_APMASK (0x3 << ARM_TTE_BLOCK_APSHIFT) |
1524 | |
1525 | #define ARM_TTE_BLOCK_ATTRINDX(x) ((x) << 2) /* memory attributes index */ |
1526 | #define ARM_TTE_BLOCK_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ |
1527 | |
1528 | #define ARM_TTE_BLOCK_SH(x) ((x) << 8) /* access shared */ |
1529 | #define ARM_TTE_BLOCK_SHMASK (0x3ULL << 8) /* mask access shared */ |
1530 | |
1531 | #define ARM_TTE_BLOCK_AF 0x0000000000000400ULL /* value for access */ |
1532 | #define ARM_TTE_BLOCK_AFMASK 0x0000000000000400ULL /* access mask */ |
1533 | |
1534 | #define ARM_TTE_BLOCK_NG 0x0000000000000800ULL /* value for a global mapping */ |
1535 | #define ARM_TTE_BLOCK_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ |
1536 | |
1537 | #define ARM_TTE_BLOCK_NS 0x0000000000000020ULL /* value for a secure mapping */ |
1538 | #define ARM_TTE_BLOCK_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ |
1539 | |
1540 | #define ARM_TTE_BLOCK_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ |
1541 | #define ARM_TTE_BLOCK_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ |
1542 | |
1543 | #define ARM_TTE_BLOCK_NX 0x0040000000000000ULL /* value for no execute */ |
1544 | #define ARM_TTE_BLOCK_NXMASK 0x0040000000000000ULL /* no execute mask */ |
1545 | |
1546 | #define ARM_TTE_BLOCK_WIRED 0x0400000000000000ULL /* value for software wired bit */ |
1547 | #define ARM_TTE_BLOCK_WIREDMASK 0x0400000000000000ULL /* software wired mask */ |
1548 | |
1549 | #define ARM_TTE_BLOCK_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */ |
1550 | #define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL /* software writeable mask */ |
1551 | |
1552 | #define ARM_TTE_TABLE_MASK 0x0000fffffffff000ULL /* mask for extracting pointer to next table (works at any level) */ |
1553 | |
1554 | #define ARM_TTE_TABLE_APSHIFT 61 |
1555 | #define ARM_TTE_TABLE_AP_NO_EFFECT 0x0ULL |
1556 | #define ARM_TTE_TABLE_AP_USER_NA 0x1ULL |
1557 | #define ARM_TTE_TABLE_AP_RO 0x2ULL |
1558 | #define ARM_TTE_TABLE_AP_KERN_RO 0x3ULL |
1559 | #define ARM_TTE_TABLE_AP(x) ((x) << ARM_TTE_TABLE_APSHIFT) /* access protection */ |
1560 | |
1561 | #define ARM_TTE_TABLE_NS 0x8000000000000020ULL /* value for a secure mapping */ |
1562 | #define ARM_TTE_TABLE_NS_MASK 0x8000000000000020ULL /* notSecure mapping mask */ |
1563 | |
1564 | #define ARM_TTE_TABLE_XN 0x1000000000000000ULL /* value for no execute */ |
1565 | #define ARM_TTE_TABLE_XNMASK 0x1000000000000000ULL /* no execute mask */ |
1566 | |
1567 | #define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */ |
1568 | #define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */ |
1569 | |
1570 | #if __ARM_KERNEL_PROTECT__ |
1571 | #define ARM_TTE_BOOT_BLOCK \ |
1572 | (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \ |
1573 | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NG) |
1574 | #else /* __ARM_KERNEL_PROTECT__ */ |
1575 | #define ARM_TTE_BOOT_BLOCK \ |
1576 | (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \ |
1577 | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF) |
1578 | #endif /* __ARM_KERNEL_PROTECT__ */ |
1579 | |
1580 | #define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID ) |
1581 | /* |
1582 | * L3 Translation table |
1583 | * |
1584 | * 4KB granule size: |
1585 | * Each translation table is 4KB |
1586 | * 512 64-bit entries of 4KB (2^12) of address space. |
1587 | * Covers 2MB (2^21) of address space. |
1588 | * |
1589 | * 16KB granule size: |
1590 | * Each translation table is 16KB |
1591 | * 2048 64-bit entries of 16KB (2^14) of address space. |
1592 | * Covers 32MB (2^25) of address space. |
1593 | */ |
1594 | |
1595 | #ifdef __ARM_16K_PG__ |
1596 | #define ARM_PTE_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ |
1597 | #define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */ |
1598 | #define ARM_PTE_SHIFT 14 /* page descriptor shift */ |
1599 | #define ARM_PTE_MASK 0x0000ffffffffc000ULL /* mask for output address in PTE */ |
1600 | #else |
1601 | #define ARM_PTE_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ |
1602 | #define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */ |
1603 | #define ARM_PTE_SHIFT 12 /* page descriptor shift */ |
1604 | #define ARM_PTE_MASK 0x0000fffffffff000ULL /* mask for output address in PTE */ |
1605 | #endif |
1606 | |
1607 | #define ARM_TTE_PA_MASK 0x0000fffffffff000ULL |
1608 | |
1609 | /* |
1610 | * L3 Page table entries |
1611 | * |
1612 | * The following page table entry types are possible: |
1613 | * |
1614 | * fault page entry |
1615 | * 63 2 0 |
1616 | * +------------------------------+--+ |
1617 | * | ignored |00| |
1618 | * +------------------------------+--+ |
1619 | * |
1620 | * |
1621 | * 63 59 58 55 54 53 52 51 48 47 12 11 10 9 8 7 6 5 4 2 1 0 |
1622 | * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+ |
1623 | * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V| |
1624 | * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+ |
1625 | * |
1626 | * where: |
1627 | * nG: notGlobal bit |
1628 | * SH: Shareability field |
1629 | * AP: access protection |
1630 | * XN: eXecute Never bit |
1631 | * PXN: Privilege eXecute Never bit |
1632 | * NS: Non-Secure bit |
1633 | * HINT: 16 entry continuguous output hint |
1634 | * AttrIdx: Memory Attribute Index |
1635 | */ |
1636 | |
1637 | #define PTE_SHIFT 3 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */ |
1638 | #ifdef __ARM_16K_PG__ |
1639 | #define PTE_PGENTRIES (16384 >> PTE_SHIFT) /* number of ptes per page */ |
1640 | #else |
1641 | #define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */ |
1642 | #endif |
1643 | |
1644 | #define ARM_PTE_EMPTY 0x0000000000000000ULL /* unassigned - invalid entry */ |
1645 | |
1646 | /* markers for (invalid) PTE for a page sent to compressor */ |
1647 | #define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */ |
1648 | #define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */ |
1649 | #define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL |
1650 | |
1651 | #define ARM_PTE_TYPE 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ |
1652 | #define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ |
1653 | #define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */ |
1654 | #define ARM_PTE_TYPE_MASK 0x0000000000000002ULL /* mask to get pte type */ |
1655 | |
1656 | /* This mask works for both 16K and 4K pages because bits 12-13 will be zero in 16K pages */ |
1657 | #define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* output address mask for page */ |
1658 | #define ARM_PTE_PAGE_SHIFT 12 /* page shift for the output address in the entry */ |
1659 | |
1660 | #define ARM_PTE_AP(x) ((x) << 6) /* access protections */ |
1661 | #define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */ |
1662 | #define (x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */ |
1663 | |
1664 | #define ARM_PTE_ATTRINDX(x) ((x) << 2) /* memory attributes index */ |
1665 | #define ARM_PTE_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ |
1666 | #define (x) (((x) >> 2) & 0x7ULL) /* extract memory attributes index */ |
1667 | |
1668 | #define ARM_PTE_SH(x) ((x) << 8) /* access shared */ |
1669 | #define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */ |
1670 | |
1671 | #define ARM_PTE_AF 0x0000000000000400ULL /* value for access */ |
1672 | #define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */ |
1673 | |
1674 | #define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */ |
1675 | #define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ |
1676 | |
1677 | #define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */ |
1678 | #define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ |
1679 | |
1680 | #define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */ |
1681 | #define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */ |
1682 | |
1683 | #if __ARM_16K_PG__ |
1684 | #define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */ |
1685 | #define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */ |
1686 | #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */ |
1687 | #define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */ |
1688 | #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */ |
1689 | #else |
1690 | #define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */ |
1691 | #define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */ |
1692 | #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */ |
1693 | #define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */ |
1694 | #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */ |
1695 | #endif |
1696 | |
1697 | #define ARM_PTE_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ |
1698 | #define ARM_PTE_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ |
1699 | |
1700 | #define ARM_PTE_NX 0x0040000000000000ULL /* value for no execute bit */ |
1701 | #define ARM_PTE_NXMASK 0x0040000000000000ULL /* no execute mask */ |
1702 | |
1703 | #define ARM_PTE_XMASK (ARM_PTE_PNXMASK | ARM_PTE_NXMASK) |
1704 | |
1705 | #define ARM_PTE_WIRED 0x0400000000000000ULL /* value for software wired bit */ |
1706 | #define ARM_PTE_WIRED_MASK 0x0400000000000000ULL /* software wired mask */ |
1707 | |
1708 | #define ARM_PTE_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */ |
1709 | #define ARM_PTE_WRITEABLE_MASK 0x0800000000000000ULL /* software writeable mask */ |
1710 | |
1711 | #define ARM_PTE_BOOT_PAGE_BASE \ |
1712 | (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) | \ |
1713 | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF) |
1714 | |
1715 | #if __ARM_KERNEL_PROTECT__ |
1716 | #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG) |
1717 | #else /* __ARM_KERNEL_PROTECT__ */ |
1718 | #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE) |
1719 | #endif /* __ARM_KERNEL_PROTECT__ */ |
1720 | |
1721 | /* |
1722 | * TLBI appers to only deal in 4KB page addresses, so give |
1723 | * it an explicit shift of 12. |
1724 | */ |
1725 | #define TLBI_ADDR_SHIFT (0) |
1726 | #define TLBI_ADDR_SIZE (44) |
1727 | #define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1) |
1728 | #define TLBI_ASID_SHIFT (48) |
1729 | #define TLBI_ASID_SIZE (16) |
1730 | #define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1)) |
1731 | |
1732 | #define RTLBI_ADDR_SIZE (37) |
1733 | #define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1) |
1734 | #define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT |
1735 | #define RTLBI_TG(_page_shift_) ((uint64_t)((((_page_shift_) - 12) >> 1) + 1) << 46) |
1736 | #define RTLBI_SCALE_SHIFT (44) |
1737 | #define RTLBI_NUM_SHIFT (39) |
1738 | |
1739 | /* |
1740 | * Exception Syndrome Register |
1741 | * |
1742 | * 31 26 25 24 0 |
1743 | * +------+--+------------------+ |
1744 | * | EC |IL| ISS | |
1745 | * +------+--+------------------+ |
1746 | * |
1747 | * EC - Exception Class |
1748 | * IL - Instruction Length |
1749 | * ISS - Instruction Specific Syndrome |
1750 | * |
1751 | * Note: The ISS can have many forms. These are defined separately below. |
1752 | */ |
1753 | |
1754 | #define ESR_EC_SHIFT 26 |
1755 | #define ESR_EC_MASK (0x3FULL << ESR_EC_SHIFT) |
1756 | #define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT) |
1757 | |
1758 | #define ESR_IL_SHIFT 25 |
1759 | #define ESR_IL (1 << ESR_IL_SHIFT) |
1760 | |
1761 | #define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL)) |
1762 | |
1763 | #define ESR_ISS_MASK 0x01FFFFFF |
1764 | #define ESR_ISS(x) (x & ESR_ISS_MASK) |
1765 | |
1766 | #ifdef __ASSEMBLER__ |
1767 | /* Define only the classes we need to test in the exception vectors. */ |
1768 | #define ESR_EC_UNCATEGORIZED 0x00 |
1769 | #define ESR_EC_PAC_FAIL 0x1C |
1770 | #define ESR_EC_IABORT_EL1 0x21 |
1771 | #define ESR_EC_DABORT_EL1 0x25 |
1772 | #define ESR_EC_SP_ALIGN 0x26 |
1773 | #define ESR_EC_BRK_AARCH64 0x3C |
1774 | #else |
1775 | typedef enum { |
1776 | ESR_EC_UNCATEGORIZED = 0x00, |
1777 | ESR_EC_WFI_WFE = 0x01, |
1778 | ESR_EC_MCR_MRC_CP15_TRAP = 0x03, |
1779 | ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04, |
1780 | ESR_EC_MCR_MRC_CP14_TRAP = 0x05, |
1781 | ESR_EC_LDC_STC_CP14_TRAP = 0x06, |
1782 | ESR_EC_TRAP_SIMD_FP = 0x07, |
1783 | ESR_EC_PTRAUTH_INSTR_TRAP = 0x09, |
1784 | ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c, |
1785 | ESR_EC_ILLEGAL_INSTR_SET = 0x0e, |
1786 | ESR_EC_SVC_32 = 0x11, |
1787 | ESR_EC_HVC_32 = 0x12, |
1788 | ESR_EC_SVC_64 = 0x15, |
1789 | ESR_EC_HVC_64 = 0x16, |
1790 | ESR_EC_MSR_TRAP = 0x18, |
1791 | #if __has_feature(ptrauth_calls) |
1792 | ESR_EC_PAC_FAIL = 0x1C, |
1793 | #endif /* __has_feature(ptrauth_calls) */ |
1794 | ESR_EC_IABORT_EL0 = 0x20, |
1795 | ESR_EC_IABORT_EL1 = 0x21, |
1796 | ESR_EC_PC_ALIGN = 0x22, |
1797 | ESR_EC_DABORT_EL0 = 0x24, |
1798 | ESR_EC_DABORT_EL1 = 0x25, |
1799 | ESR_EC_SP_ALIGN = 0x26, |
1800 | ESR_EC_FLOATING_POINT_32 = 0x28, |
1801 | ESR_EC_FLOATING_POINT_64 = 0x2C, |
1802 | ESR_EC_SERROR_INTERRUPT = 0x2F, |
1803 | ESR_EC_BKPT_REG_MATCH_EL0 = 0x30, // Breakpoint Debug event taken to the EL from a lower EL. |
1804 | ESR_EC_BKPT_REG_MATCH_EL1 = 0x31, // Breakpoint Debug event taken to the EL from the EL. |
1805 | ESR_EC_SW_STEP_DEBUG_EL0 = 0x32, // Software Step Debug event taken to the EL from a lower EL. |
1806 | ESR_EC_SW_STEP_DEBUG_EL1 = 0x33, // Software Step Debug event taken to the EL from the EL. |
1807 | ESR_EC_WATCHPT_MATCH_EL0 = 0x34, // Watchpoint Debug event taken to the EL from a lower EL. |
1808 | ESR_EC_WATCHPT_MATCH_EL1 = 0x35, // Watchpoint Debug event taken to the EL from the EL. |
1809 | ESR_EC_BKPT_AARCH32 = 0x38, |
1810 | ESR_EC_BRK_AARCH64 = 0x3C, |
1811 | } esr_exception_class_t; |
1812 | |
1813 | typedef enum { |
1814 | FSC_TRANSLATION_FAULT_L0 = 0x04, |
1815 | FSC_TRANSLATION_FAULT_L1 = 0x05, |
1816 | FSC_TRANSLATION_FAULT_L2 = 0x06, |
1817 | FSC_TRANSLATION_FAULT_L3 = 0x07, |
1818 | FSC_ACCESS_FLAG_FAULT_L1 = 0x09, |
1819 | FSC_ACCESS_FLAG_FAULT_L2 = 0x0A, |
1820 | FSC_ACCESS_FLAG_FAULT_L3 = 0x0B, |
1821 | FSC_PERMISSION_FAULT_L1 = 0x0D, |
1822 | FSC_PERMISSION_FAULT_L2 = 0x0E, |
1823 | FSC_PERMISSION_FAULT_L3 = 0x0F, |
1824 | FSC_SYNC_EXT_ABORT = 0x10, |
1825 | FSC_SYNC_EXT_ABORT_TT_L1 = 0x15, |
1826 | FSC_SYNC_EXT_ABORT_TT_L2 = 0x16, |
1827 | FSC_SYNC_EXT_ABORT_TT_L3 = 0x17, |
1828 | FSC_SYNC_PARITY = 0x18, |
1829 | FSC_ASYNC_PARITY = 0x19, |
1830 | FSC_SYNC_PARITY_TT_L1 = 0x1D, |
1831 | FSC_SYNC_PARITY_TT_L2 = 0x1E, |
1832 | FSC_SYNC_PARITY_TT_L3 = 0x1F, |
1833 | FSC_ALIGNMENT_FAULT = 0x21, |
1834 | FSC_DEBUG_FAULT = 0x22, |
1835 | } fault_status_t; |
1836 | #endif /* ASSEMBLER */ |
1837 | |
1838 | /* |
1839 | * HVC event |
1840 | * 24 16 15 0 |
1841 | * +---------+-----+ |
1842 | * |000000000| IMM | |
1843 | * +---------+-----+ |
1844 | * |
1845 | * where: |
1846 | * IMM: Immediate value |
1847 | */ |
1848 | |
1849 | #define ISS_HVC_IMM_MASK 0xffff |
1850 | #define ISS_HVC_IMM(x) ((x) & ISS_HVC_IMM_MASK) |
1851 | |
1852 | /* |
1853 | * Software step debug event ISS (EL1) |
1854 | * 24 23 6 5 0 |
1855 | * +---+-----------------+--+------+ |
1856 | * |ISV|00000000000000000|EX| IFSC | |
1857 | * +---+-----------------+--+------+ |
1858 | * |
1859 | * where: |
1860 | * ISV: Instruction syndrome valid |
1861 | * EX: Exclusive access |
1862 | * IFSC: Instruction Fault Status Code |
1863 | */ |
1864 | |
1865 | #define ISS_SSDE_ISV_SHIFT 24 |
1866 | #define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT) |
1867 | |
1868 | #define ISS_SSDE_EX_SHIFT 6 |
1869 | #define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT) |
1870 | |
1871 | #define ISS_SSDE_FSC_MASK 0x3F |
1872 | #define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK) |
1873 | |
1874 | /* |
1875 | * Instruction Abort ISS (EL1) |
1876 | * 24 10 9 5 0 |
1877 | * +--------------+---+--+---+------+ |
1878 | * |00000000000000|FnV|EA|000| IFSC | |
1879 | * +--------------+---+--+---+------+ |
1880 | * |
1881 | * where: |
1882 | * FnV: FAR not Valid |
1883 | * EA: External Abort type |
1884 | * IFSC: Instruction Fault Status Code |
1885 | */ |
1886 | |
1887 | #define ISS_IA_FNV_SHIFT 10 |
1888 | #define ISS_IA_FNV (0x1 << ISS_IA_FNV_SHIFT) |
1889 | |
1890 | #define ISS_IA_EA_SHIFT 9 |
1891 | #define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT) |
1892 | |
1893 | #define ISS_IA_FSC_MASK 0x3F |
1894 | #define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK) |
1895 | |
1896 | |
1897 | /* |
1898 | * Data Abort ISS (EL1) |
1899 | * |
1900 | * 24 10 9 8 7 6 5 0 |
1901 | * +--------------+---+--+--+-----+---+----+ |
1902 | * |00000000000000|FnV|EA|CM|S1PTW|WnR|DFSC| |
1903 | * +--------------+---+--+--+-----+---+----+ |
1904 | * |
1905 | * where: |
1906 | * FnV: FAR not Valid |
1907 | * EA: External Abort type |
1908 | * CM: Cache Maintenance operation |
1909 | * WnR: Write not Read |
1910 | * S1PTW: Stage 2 exception on Stage 1 page table walk |
1911 | * DFSC: Data Fault Status Code |
1912 | */ |
1913 | #define ISS_DA_FNV_SHIFT 10 |
1914 | #define ISS_DA_FNV (0x1 << ISS_DA_FNV_SHIFT) |
1915 | |
1916 | #define ISS_DA_EA_SHIFT 9 |
1917 | #define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT) |
1918 | |
1919 | #define ISS_DA_CM_SHIFT 8 |
1920 | #define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT) |
1921 | |
1922 | #define ISS_DA_WNR_SHIFT 6 |
1923 | #define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT) |
1924 | |
1925 | #define ISS_DA_S1PTW_SHIFT 7 |
1926 | #define ISS_DA_S1PTW (0x1 << ISS_DA_S1PTW_SHIFT) |
1927 | |
1928 | #define ISS_DA_FSC_MASK 0x3F |
1929 | #define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK) |
1930 | |
1931 | /* |
1932 | * Floating Point Exception ISS (EL1) |
1933 | * |
1934 | * 24 23 22 8 7 4 3 2 1 0 |
1935 | * +-+---+---------------+---+--+---+---+---+---+---+ |
1936 | * |0|TFV|000000000000000|IDF|00|IXF|UFF|OFF|DZF|IOF| |
1937 | * +-+---+---------------+---+--+---+---+---+---+---+ |
1938 | * |
1939 | * where: |
1940 | * TFV: Trapped Fault Valid |
1941 | * IDF: Input Denormal Exception |
1942 | * IXF: Input Inexact Exception |
1943 | * UFF: Underflow Exception |
1944 | * OFF: Overflow Exception |
1945 | * DZF: Divide by Zero Exception |
1946 | * IOF: Invalid Operation Exception |
1947 | */ |
1948 | #define ISS_FP_TFV_SHIFT 23 |
1949 | #define ISS_FP_TFV (0x1 << ISS_FP_TFV_SHIFT) |
1950 | |
1951 | #define ISS_FP_IDF_SHIFT 7 |
1952 | #define ISS_FP_IDF (0x1 << ISS_FP_IDF_SHIFT) |
1953 | |
1954 | #define ISS_FP_IXF_SHIFT 4 |
1955 | #define ISS_FP_IXF (0x1 << ISS_FP_IXF_SHIFT) |
1956 | |
1957 | #define ISS_FP_UFF_SHIFT 3 |
1958 | #define ISS_FP_UFF (0x1 << ISS_FP_UFF_SHIFT) |
1959 | |
1960 | #define ISS_FP_OFF_SHIFT 2 |
1961 | #define ISS_FP_OFF (0x1 << ISS_FP_OFF_SHIFT) |
1962 | |
1963 | #define ISS_FP_DZF_SHIFT 1 |
1964 | #define ISS_FP_DZF (0x1 << ISS_FP_DZF_SHIFT) |
1965 | |
1966 | #define ISS_FP_IOF_SHIFT 0 |
1967 | #define ISS_FP_IOF (0x1 << ISS_FP_IOF_SHIFT) |
1968 | |
1969 | /* |
1970 | * Breakpoint Exception ISS (EL1) |
1971 | * 24 16 0 |
1972 | * +---------+---------+ |
1973 | * |000000000| Comment | |
1974 | * +---------+---------+ |
1975 | * |
1976 | * where: |
1977 | * Comment: Instruction Comment Field Value |
1978 | */ |
1979 | #define 0xFFFF |
1980 | #define (x) (x & ISS_BRK_COMMENT_MASK) |
1981 | |
1982 | |
1983 | /* |
1984 | * SError Interrupt, IDS=1 |
1985 | * 24 23 0 |
1986 | * +---+------------------------+ |
1987 | * |IDS| IMPLEMENTATION DEFINED | |
1988 | * +---+------------------------+ |
1989 | * |
1990 | * where: |
1991 | * IDS: Implementation-defined syndrome (1) |
1992 | */ |
1993 | |
1994 | #define ISS_SEI_IDS_SHIFT 24 |
1995 | #define ISS_SEI_IDS (0x1 << ISS_SEI_IDS_SHIFT) |
1996 | |
1997 | |
1998 | #if HAS_UCNORMAL_MEM |
1999 | #define ISS_UC 0x11 |
2000 | #endif /* HAS_UCNORMAL_MEM */ |
2001 | |
2002 | |
2003 | |
2004 | |
2005 | /* |
2006 | * Physical Address Register (EL1) |
2007 | */ |
2008 | #define PAR_F_SHIFT 0 |
2009 | #define PAR_F (0x1 << PAR_F_SHIFT) |
2010 | |
2011 | #define PLATFORM_SYSCALL_TRAP_NO 0x80000000 |
2012 | |
2013 | #define ARM64_SYSCALL_CODE_REG_NUM (16) |
2014 | |
2015 | #define ARM64_CLINE_SHIFT 6 |
2016 | |
2017 | #if defined(APPLE_ARM64_ARCH_FAMILY) |
2018 | #define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */ |
2019 | #define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */ |
2020 | #endif |
2021 | |
2022 | /* |
2023 | * Timer definitions. |
2024 | */ |
2025 | #define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */ |
2026 | #define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */ |
2027 | #define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */ |
2028 | #define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */ |
2029 | #define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */ |
2030 | #define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */ |
2031 | #define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to virtual timebase + frequency reg enabled */ |
2032 | #define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to physical timebase + frequency reg enabled */ |
2033 | |
2034 | #define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */ |
2035 | #define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */ |
2036 | #define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */ |
2037 | |
2038 | #define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS |
2039 | #define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED |
2040 | #define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE |
2041 | |
2042 | #define MIDR_EL1_REV_SHIFT 0 |
2043 | #define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT) |
2044 | #define MIDR_EL1_PNUM_SHIFT 4 |
2045 | #define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT) |
2046 | #define MIDR_EL1_ARCH_SHIFT 16 |
2047 | #define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT) |
2048 | #define MIDR_EL1_VAR_SHIFT 20 |
2049 | #define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT) |
2050 | #define MIDR_EL1_IMP_SHIFT 24 |
2051 | #define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT) |
2052 | |
2053 | #define MIDR_FIJI (0x002 << MIDR_EL1_PNUM_SHIFT) |
2054 | #define MIDR_CAPRI (0x003 << MIDR_EL1_PNUM_SHIFT) |
2055 | #define MIDR_MAUI (0x004 << MIDR_EL1_PNUM_SHIFT) |
2056 | #define MIDR_ELBA (0x005 << MIDR_EL1_PNUM_SHIFT) |
2057 | #define MIDR_CAYMAN (0x006 << MIDR_EL1_PNUM_SHIFT) |
2058 | #define MIDR_MYST (0x007 << MIDR_EL1_PNUM_SHIFT) |
2059 | #define MIDR_SKYE_MONSOON (0x008 << MIDR_EL1_PNUM_SHIFT) |
2060 | #define MIDR_SKYE_MISTRAL (0x009 << MIDR_EL1_PNUM_SHIFT) |
2061 | #define MIDR_CYPRUS_VORTEX (0x00B << MIDR_EL1_PNUM_SHIFT) |
2062 | #define MIDR_CYPRUS_TEMPEST (0x00C << MIDR_EL1_PNUM_SHIFT) |
2063 | #define MIDR_M9 (0x00F << MIDR_EL1_PNUM_SHIFT) |
2064 | #define MIDR_ARUBA_VORTEX (0x010 << MIDR_EL1_PNUM_SHIFT) |
2065 | #define MIDR_ARUBA_TEMPEST (0x011 << MIDR_EL1_PNUM_SHIFT) |
2066 | |
2067 | #ifdef APPLELIGHTNING |
2068 | #define MIDR_CEBU_LIGHTNING (0x012 << MIDR_EL1_PNUM_SHIFT) |
2069 | #define MIDR_CEBU_THUNDER (0x013 << MIDR_EL1_PNUM_SHIFT) |
2070 | #define MIDR_TURKS (0x026 << MIDR_EL1_PNUM_SHIFT) |
2071 | #endif |
2072 | |
2073 | #ifdef APPLEFIRESTORM |
2074 | #define MIDR_SICILY_ICESTORM (0x020 << MIDR_EL1_PNUM_SHIFT) |
2075 | #define MIDR_SICILY_FIRESTORM (0x021 << MIDR_EL1_PNUM_SHIFT) |
2076 | #define MIDR_TONGA_ICESTORM (0x022 << MIDR_EL1_PNUM_SHIFT) |
2077 | #define MIDR_TONGA_FIRESTORM (0x023 << MIDR_EL1_PNUM_SHIFT) |
2078 | #define MIDR_JADE_CHOP_ICESTORM (0x024 << MIDR_EL1_PNUM_SHIFT) |
2079 | #define MIDR_JADE_CHOP_FIRESTORM (0x025 << MIDR_EL1_PNUM_SHIFT) |
2080 | #define MIDR_JADE_DIE_ICESTORM (0x028 << MIDR_EL1_PNUM_SHIFT) |
2081 | #define MIDR_JADE_DIE_FIRESTORM (0x029 << MIDR_EL1_PNUM_SHIFT) |
2082 | #endif |
2083 | |
2084 | #ifdef APPLEAVALANCHE |
2085 | #define MIDR_ELLIS_BLIZZARD (0x030 << MIDR_EL1_PNUM_SHIFT) |
2086 | #define MIDR_ELLIS_AVALANCHE (0x031 << MIDR_EL1_PNUM_SHIFT) |
2087 | #endif |
2088 | |
2089 | |
2090 | |
2091 | |
2092 | /* |
2093 | * Apple-ISA-Extensions ID Register. |
2094 | */ |
2095 | #define AIDR_MUL53 (1 << 0) |
2096 | #define AIDR_WKDM (1 << 1) |
2097 | #define AIDR_ARCHRETENTION (1 << 2) |
2098 | |
2099 | |
2100 | |
2101 | /* |
2102 | * CoreSight debug registers |
2103 | */ |
2104 | #define CORESIGHT_ED 0 |
2105 | #define CORESIGHT_CTI 1 |
2106 | #define CORESIGHT_PMU 2 |
2107 | #define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */ |
2108 | |
2109 | #define CORESIGHT_OFFSET(x) ((x) * 0x10000) |
2110 | #define CORESIGHT_REGIONS 4 |
2111 | #define CORESIGHT_SIZE 0x1000 |
2112 | |
2113 | |
2114 | |
2115 | |
2116 | |
2117 | |
2118 | |
2119 | |
2120 | |
2121 | |
2122 | /* |
2123 | * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0 |
2124 | * |
2125 | * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 |
2126 | * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+ |
2127 | * | rndr | tlb | ts | fhm | dp | sm4 | sm3 | sha3 | rdm | res0 | atomic | crc32 | sha2 | sha1 | aes | res0 | |
2128 | * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+ |
2129 | */ |
2130 | |
2131 | #define ID_AA64ISAR0_EL1_TS_OFFSET 52 |
2132 | #define ID_AA64ISAR0_EL1_TS_MASK (0xfull << ID_AA64ISAR0_EL1_TS_OFFSET) |
2133 | #define ID_AA64ISAR0_EL1_TS_FLAGM_EN (1ull << ID_AA64ISAR0_EL1_TS_OFFSET) |
2134 | #define ID_AA64ISAR0_EL1_TS_FLAGM2_EN (2ull << ID_AA64ISAR0_EL1_TS_OFFSET) |
2135 | |
2136 | #define ID_AA64ISAR0_EL1_FHM_OFFSET 48 |
2137 | #define ID_AA64ISAR0_EL1_FHM_MASK (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET) |
2138 | #define ID_AA64ISAR0_EL1_FHM_8_2 (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET) |
2139 | |
2140 | #define ID_AA64ISAR0_EL1_DP_OFFSET 44 |
2141 | #define ID_AA64ISAR0_EL1_DP_MASK (0xfull << ID_AA64ISAR0_EL1_DP_OFFSET) |
2142 | #define ID_AA64ISAR0_EL1_DP_EN (1ull << ID_AA64ISAR0_EL1_DP_OFFSET) |
2143 | |
2144 | #define ID_AA64ISAR0_EL1_SHA3_OFFSET 32 |
2145 | #define ID_AA64ISAR0_EL1_SHA3_MASK (0xfull << ID_AA64ISAR0_EL1_SHA3_OFFSET) |
2146 | #define ID_AA64ISAR0_EL1_SHA3_EN (1ull << ID_AA64ISAR0_EL1_SHA3_OFFSET) |
2147 | |
2148 | #define ID_AA64ISAR0_EL1_RDM_OFFSET 28 |
2149 | #define ID_AA64ISAR0_EL1_RDM_MASK (0xfull << ID_AA64ISAR0_EL1_RDM_OFFSET) |
2150 | #define ID_AA64ISAR0_EL1_RDM_EN (1ull << ID_AA64ISAR0_EL1_RDM_OFFSET) |
2151 | |
2152 | #define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20 |
2153 | #define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) |
2154 | #define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) |
2155 | |
2156 | #define ID_AA64ISAR0_EL1_CRC32_OFFSET 16 |
2157 | #define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET) |
2158 | #define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET) |
2159 | |
2160 | #define ID_AA64ISAR0_EL1_SHA2_OFFSET 12 |
2161 | #define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET) |
2162 | #define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) |
2163 | #define ID_AA64ISAR0_EL1_SHA2_512_EN (2ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) |
2164 | |
2165 | #define ID_AA64ISAR0_EL1_SHA1_OFFSET 8 |
2166 | #define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET) |
2167 | #define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET) |
2168 | |
2169 | #define ID_AA64ISAR0_EL1_AES_OFFSET 4 |
2170 | #define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET) |
2171 | #define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET) |
2172 | #define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET) |
2173 | |
2174 | /* |
2175 | * ID_AA64ISAR1_EL1 - AArch64 Instruction Set Attribute Register 1 |
2176 | * |
2177 | * 63 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 |
2178 | * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+ |
2179 | * | res0 | i8mm | dgh | bf16 | specres | sb | frintts | gpi | gpa | lrcpc | fcma | jscvt | api | apa | dpb | |
2180 | * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+ |
2181 | */ |
2182 | |
2183 | #define ID_AA64ISAR1_EL1_I8MM_OFFSET 52 |
2184 | #define ID_AA64ISAR1_EL1_I8MM_MASK (0xfull << ID_AA64ISAR1_EL1_I8MM_OFFSET) |
2185 | #define ID_AA64ISAR1_EL1_I8MM_EN (1ull << ID_AA64ISAR1_EL1_I8MM_OFFSET) |
2186 | |
2187 | #define ID_AA64ISAR1_EL1_DGH_OFFSET 48 |
2188 | #define ID_AA64ISAR1_EL1_DGH_MASK (0xfull << ID_AA64ISAR1_EL1_DGH_OFFSET) |
2189 | |
2190 | #define ID_AA64ISAR1_EL1_BF16_OFFSET 44 |
2191 | #define ID_AA64ISAR1_EL1_BF16_MASK (0xfull << ID_AA64ISAR1_EL1_BF16_OFFSET) |
2192 | #define ID_AA64ISAR1_EL1_BF16_EN (1ull << ID_AA64ISAR1_EL1_BF16_OFFSET) |
2193 | |
2194 | #define ID_AA64ISAR1_EL1_SPECRES_OFFSET 40 |
2195 | #define ID_AA64ISAR1_EL1_SPECRES_MASK (0xfull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) |
2196 | #define ID_AA64ISAR1_EL1_SPECRES_EN (1ull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) |
2197 | |
2198 | #define ID_AA64ISAR1_EL1_SB_OFFSET 36 |
2199 | #define ID_AA64ISAR1_EL1_SB_MASK (0xfull << ID_AA64ISAR1_EL1_SB_OFFSET) |
2200 | #define ID_AA64ISAR1_EL1_SB_EN (1ull << ID_AA64ISAR1_EL1_SB_OFFSET) |
2201 | |
2202 | #define ID_AA64ISAR1_EL1_FRINTTS_OFFSET 32 |
2203 | #define ID_AA64ISAR1_EL1_FRINTTS_MASK (0xfull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET) |
2204 | #define ID_AA64ISAR1_EL1_FRINTTS_EN (1ull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET) |
2205 | |
2206 | #define ID_AA64ISAR1_EL1_GPI_OFFSET 28 |
2207 | #define ID_AA64ISAR1_EL1_GPI_MASK (0xfull << ID_AA64ISAR1_EL1_GPI_OFFSET) |
2208 | #define ID_AA64ISAR1_EL1_GPI_EN (1ull << ID_AA64ISAR1_EL1_GPI_OFFSET) |
2209 | |
2210 | #define ID_AA64ISAR1_EL1_GPA_OFFSET 24 |
2211 | #define ID_AA64ISAR1_EL1_GPA_MASK (0xfull << ID_AA64ISAR1_EL1_GPA_OFFSET) |
2212 | |
2213 | #define ID_AA64ISAR1_EL1_LRCPC_OFFSET 20 |
2214 | #define ID_AA64ISAR1_EL1_LRCPC_MASK (0xfull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) |
2215 | #define ID_AA64ISAR1_EL1_LRCPC_EN (1ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) |
2216 | #define ID_AA64ISAR1_EL1_LRCP2C_EN (2ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) |
2217 | |
2218 | #define ID_AA64ISAR1_EL1_FCMA_OFFSET 16 |
2219 | #define ID_AA64ISAR1_EL1_FCMA_MASK (0xfull << ID_AA64ISAR1_EL1_FCMA_OFFSET) |
2220 | #define ID_AA64ISAR1_EL1_FCMA_EN (1ull << ID_AA64ISAR1_EL1_FCMA_OFFSET) |
2221 | |
2222 | #define ID_AA64ISAR1_EL1_JSCVT_OFFSET 12 |
2223 | #define ID_AA64ISAR1_EL1_JSCVT_MASK (0xfull << ID_AA64ISAR1_EL1_JSCVT_OFFSET) |
2224 | #define ID_AA64ISAR1_EL1_JSCVT_EN (1ull << ID_AA64ISAR1_EL1_JSCVT_OFFSET) |
2225 | |
2226 | #define ID_AA64ISAR1_EL1_API_OFFSET 8 |
2227 | #define ID_AA64ISAR1_EL1_API_MASK (0xfull << ID_AA64ISAR1_EL1_API_OFFSET) |
2228 | #define ID_AA64ISAR1_EL1_API_PAuth_EN (1ull << ID_AA64ISAR1_EL1_API_OFFSET) |
2229 | #define ID_AA64ISAR1_EL1_API_PAuth2_EN (3ull << ID_AA64ISAR1_EL1_API_OFFSET) |
2230 | #define ID_AA64ISAR1_EL1_API_FPAC_EN (4ull << ID_AA64ISAR1_EL1_API_OFFSET) |
2231 | #define ID_AA64ISAR1_EL1_API_FPACCOMBINE (5ull << ID_AA64ISAR1_EL1_API_OFFSET) |
2232 | |
2233 | #define ID_AA64ISAR1_EL1_APA_OFFSET 4 |
2234 | #define ID_AA64ISAR1_EL1_APA_MASK (0xfull << ID_AA64ISAR1_EL1_APA_OFFSET) |
2235 | |
2236 | #define ID_AA64ISAR1_EL1_DPB_OFFSET 0 |
2237 | #define ID_AA64ISAR1_EL1_DPB_MASK (0xfull << ID_AA64ISAR1_EL1_DPB_OFFSET) |
2238 | #define ID_AA64ISAR1_EL1_DPB_EN (1ull << ID_AA64ISAR1_EL1_DPB_OFFSET) |
2239 | #define ID_AA64ISAR1_EL1_DPB2_EN (2ull << ID_AA64ISAR1_EL1_DPB_OFFSET) |
2240 | |
2241 | /* |
2242 | * ID_AA64ISAR2_EL1 - AArch64 Instruction Set Attribute Register 2 |
2243 | * |
2244 | * 63 8 7 4 3 0 |
2245 | * +------+-------+------+ |
2246 | * | res0 | RPRES | WFxT | |
2247 | * +------+-------+------+ |
2248 | */ |
2249 | |
2250 | #define ID_AA64ISAR2_EL1_RPRES_OFFSET 4 |
2251 | #define ID_AA64ISAR2_EL1_RPRES_MASK (0xfull << ID_AA64ISAR2_EL1_RPRES_OFFSET) |
2252 | #define ID_AA64ISAR2_EL1_RPRES_EN (1ull << ID_AA64ISAR2_EL1_RPRES_OFFSET) |
2253 | |
2254 | #define ID_AA64ISAR2_EL1_WFxT_OFFSET 0 |
2255 | #define ID_AA64ISAR2_EL1_WFxT_MASK (0xfull << ID_AA64ISAR2_EL1_WFxT_OFFSET) |
2256 | #define ID_AA64ISAR2_EL1_WFxT_EN (1ull << ID_AA64ISAR2_EL1_WFxT_OFFSET) |
2257 | |
2258 | /* |
2259 | * ID_AA64MMFR0_EL1 - AArch64 Memory Model Feature Register 0 |
2260 | * 63 60 59 56 55 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 |
2261 | * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+ |
2262 | * | ECV | FGT | RES0 | ExS | TGran4_2 | TGran64_2 | TGran16_2 | TGran4 | TGran64 | TGran16 | BigEndEL0 | SNSMem | BigEnd | ASIDBits | PARange | |
2263 | * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+ |
2264 | */ |
2265 | |
2266 | #define ID_AA64MMFR0_EL1_ECV_OFFSET 60 |
2267 | #define ID_AA64MMFR0_EL1_ECV_MASK (0xfull << ID_AA64MMFR0_EL1_ECV_OFFSET) |
2268 | #define ID_AA64MMFR0_EL1_ECV_EN (1ull << ID_AA64MMFR0_EL1_ECV_OFFSET) |
2269 | |
2270 | /* |
2271 | * ID_AA64MMFR2_EL1 - AArch64 Memory Model Feature Register 2 |
2272 | * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 14 8 7 4 3 0 |
2273 | * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+ |
2274 | * | E0PD | EVT | BBM | TTL | RES0 | FWB | IDS | AT | ST | NV | CCIDX | VARANGE | IESB | LSM | UAO | CnP | |
2275 | * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+ |
2276 | */ |
2277 | |
2278 | #define ID_AA64MMFR2_EL1_AT_OFFSET 32 |
2279 | #define ID_AA64MMFR2_EL1_AT_MASK (0xfull << ID_AA64MMFR2_EL1_AT_OFFSET) |
2280 | #define ID_AA64MMFR2_EL1_AT_LSE2_EN (1ull << ID_AA64MMFR2_EL1_AT_OFFSET) |
2281 | |
2282 | /* |
2283 | * ID_AA64PFR0_EL1 - AArch64 Processor Feature Register 0 |
2284 | * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 |
2285 | * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+ |
2286 | * | CSV3 | CSV2 | RES0 | DIT | AMU | MPAM | SEL2 | SVE | RAS | GIC | AdvSIMD | FP | EL3 | EL2 | EL1 | EL0 | |
2287 | * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+ |
2288 | */ |
2289 | |
2290 | #define ID_AA64PFR0_EL1_CSV3_OFFSET 60 |
2291 | #define ID_AA64PFR0_EL1_CSV3_MASK (0xfull << ID_AA64PFR0_EL1_CSV3_OFFSET) |
2292 | #define ID_AA64PFR0_EL1_CSV3_EN (1ull << ID_AA64PFR0_EL1_CSV3_OFFSET) |
2293 | |
2294 | #define ID_AA64PFR0_EL1_CSV2_OFFSET 56 |
2295 | #define ID_AA64PFR0_EL1_CSV2_MASK (0xfull << ID_AA64PFR0_EL1_CSV2_OFFSET) |
2296 | #define ID_AA64PFR0_EL1_CSV2_EN (1ull << ID_AA64PFR0_EL1_CSV2_OFFSET) |
2297 | |
2298 | #define ID_AA64PFR0_EL1_DIT_OFFSET 48 |
2299 | #define ID_AA64PFR0_EL1_DIT_MASK (0xfull << ID_AA64PFR0_EL1_DIT_OFFSET) |
2300 | #define ID_AA64PFR0_EL1_DIT_EN (1ull << ID_AA64PFR0_EL1_DIT_OFFSET) |
2301 | |
2302 | #define ID_AA64PFR0_EL1_AdvSIMD_OFFSET 20 |
2303 | #define ID_AA64PFR0_EL1_AdvSIMD_MASK (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) |
2304 | #define ID_AA64PFR0_EL1_AdvSIMD_HPFPCVT (0x0ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) |
2305 | #define ID_AA64PFR0_EL1_AdvSIMD_FP16 (0x1ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) |
2306 | #define ID_AA64PFR0_EL1_AdvSIMD_DIS (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) |
2307 | |
2308 | /* |
2309 | * ID_AA64PFR1_EL1 - AArch64 Processor Feature Register 1 |
2310 | * 63 20 19 16 15 12 11 8 7 4 3 0 |
2311 | * +----------------------------------+-----------+----------+-------+------+------+ |
2312 | * | RES0 | MPAM_frac | RAS_frac | MTE | SSBS | BT | |
2313 | * +----------------------------------+-----------+----------+-------+------+------+ |
2314 | */ |
2315 | |
2316 | |
2317 | #define ID_AA64PFR1_EL1_SSBS_OFFSET 4 |
2318 | #define ID_AA64PFR1_EL1_SSBS_MASK (0xfull << ID_AA64PFR1_EL1_SSBS_OFFSET) |
2319 | #define ID_AA64PFR1_EL1_SSBS_EN (1ull << ID_AA64PFR1_EL1_SSBS_OFFSET) |
2320 | |
2321 | #define ID_AA64PFR1_EL1_BT_OFFSET 0 |
2322 | #define ID_AA64PFR1_EL1_BT_MASK (0xfull << ID_AA64PFR1_EL1_BT_OFFSET) |
2323 | #define ID_AA64PFR1_EL1_BT_EN (1ull << ID_AA64PFR1_EL1_BT_OFFSET) |
2324 | |
2325 | /* |
2326 | * ID_AA64MMFR1_EL1 - AArch64 Memory Model Feature Register 1 |
2327 | * |
2328 | * 63 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 |
2329 | * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+ |
2330 | * | res0 | nTLBPA | AFP | HCX | ETS | TWED | XNX | SpecSEI | PAN | LO | HPDS | VH | VMIDBits | HAFDBS | |
2331 | * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+ |
2332 | */ |
2333 | |
2334 | #define ID_AA64MMFR1_EL1_AFP_OFFSET 44 |
2335 | #define ID_AA64MMFR1_EL1_AFP_MASK (0xfull << ID_AA64MMFR1_EL1_AFP_OFFSET) |
2336 | #define ID_AA64MMFR1_EL1_AFP_EN (1ull << ID_AA64MMFR1_EL1_AFP_OFFSET) |
2337 | |
2338 | |
2339 | |
2340 | |
2341 | |
2342 | #define APSTATE_G_SHIFT (0) |
2343 | #define APSTATE_P_SHIFT (1) |
2344 | #define APSTATE_A_SHIFT (2) |
2345 | #define APSTATE_AP_MASK ((1ULL << APSTATE_A_SHIFT) | (1ULL << APSTATE_P_SHIFT)) |
2346 | |
2347 | |
2348 | #define ACTLR_EL1_EnTSO (1ULL << 1) |
2349 | #define ACTLR_EL1_EnAPFLG (1ULL << 4) |
2350 | #define ACTLR_EL1_EnAFP (1ULL << 5) |
2351 | #define ACTLR_EL1_EnPRSV (1ULL << 6) |
2352 | |
2353 | |
2354 | #if HAS_USAT_BIT |
2355 | #define ACTLR_EL1_USAT_OFFSET 0 |
2356 | #define ACTLR_EL1_USAT_MASK (1ULL << ACTLR_EL1_USAT_OFFSET) |
2357 | #define ACTLR_EL1_USAT ACTLR_EL1_USAT_MASK |
2358 | #endif |
2359 | |
2360 | |
2361 | |
2362 | |
2363 | |
2364 | |
2365 | #if defined(HAS_APPLE_PAC) |
2366 | // The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code |
2367 | #define PAC_DISCRIMINATOR_RECOVER 0x1e02 |
2368 | #endif |
2369 | |
2370 | |
2371 | #define CTR_EL0_L1Ip_OFFSET 14 |
2372 | #define CTR_EL0_L1Ip_VIPT (2ULL << CTR_EL0_L1Ip_OFFSET) |
2373 | #define CTR_EL0_L1Ip_PIPT (3ULL << CTR_EL0_L1Ip_OFFSET) |
2374 | #define CTR_EL0_L1Ip_MASK (3ULL << CTR_EL0_L1Ip_OFFSET) |
2375 | |
2376 | |
2377 | #ifdef __ASSEMBLER__ |
2378 | |
2379 | /* |
2380 | * Conditionally write to system/special-purpose register. |
2381 | * The register is written to only when the first two arguments |
2382 | * do not match. If they do match, the macro jumps to a |
2383 | * caller-provided label. |
2384 | * The _ISB variant also conditionally issues an ISB after the MSR. |
2385 | * |
2386 | * $0 - System/special-purpose register to modify |
2387 | * $1 - Register containing current FPCR value |
2388 | * $2 - Register containing expected value |
2389 | * $3 - Label to jump to when register is already set to expected value |
2390 | */ |
2391 | .macro CMSR |
2392 | cmp $1, $2 |
2393 | |
2394 | /* Skip expensive MSR if not required */ |
2395 | b.eq $3f |
2396 | msr $0, $2 |
2397 | .endmacro |
2398 | |
2399 | .macro CMSR_ISB |
2400 | CMSR $0, $1, $2, $3 |
2401 | isb sy |
2402 | .endmacro |
2403 | |
2404 | /* |
2405 | * Modify FPCR only if it does not contain the XNU default value. |
2406 | * $0 - Register containing current FPCR value |
2407 | * $1 - Scratch register |
2408 | * $2 - Label to jump to when FPCR is already set to default value |
2409 | */ |
2410 | .macro SANITIZE_FPCR |
2411 | mov $1, #FPCR_DEFAULT |
2412 | CMSR FPCR, $0, $1, $2 |
2413 | .endmacro |
2414 | |
2415 | /* |
2416 | * Family of macros that can be used to protect code sections such that they |
2417 | * are only executed on a particular SoC/Revision/CPU, and skipped otherwise. |
2418 | * All macros will forward-jump to 1f when the condition is not matched. |
2419 | * This label may be defined manually, or implicitly through the use of |
2420 | * the EXEC_END macro. |
2421 | * For cores, XX can be: EQ (equal), ALL (don't care). |
2422 | * For revisions, XX can be: EQ (equal), LO (lower than), HS (higher or same), ALL (don't care). |
2423 | */ |
2424 | |
2425 | /* |
2426 | * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX |
2427 | * $1 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 |
2428 | * $2 - GPR containing MIDR_EL1 value |
2429 | * $3 - Scratch register |
2430 | */ |
2431 | .macro EXEC_COREEQ_REVEQ |
2432 | and $3, $2, #MIDR_EL1_PNUM_MASK |
2433 | cmp $3, $0 |
2434 | b.ne 1f |
2435 | |
2436 | mov $3, $2 |
2437 | bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2438 | ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2439 | cmp $3, $1 |
2440 | b.ne 1f |
2441 | .endmacro |
2442 | |
2443 | .macro EXEC_COREEQ_REVLO |
2444 | and $3, $2, #MIDR_EL1_PNUM_MASK |
2445 | cmp $3, $0 |
2446 | b.ne 1f |
2447 | |
2448 | mov $3, $2 |
2449 | bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2450 | ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2451 | cmp $3, $1 |
2452 | b.pl 1f |
2453 | .endmacro |
2454 | |
2455 | .macro EXEC_COREEQ_REVHS |
2456 | and $3, $2, #MIDR_EL1_PNUM_MASK |
2457 | cmp $3, $0 |
2458 | b.ne 1f |
2459 | |
2460 | mov $3, $2 |
2461 | bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2462 | ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2463 | cmp $3, $1 |
2464 | b.mi 1f |
2465 | .endmacro |
2466 | |
2467 | /* |
2468 | * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 |
2469 | * $1 - GPR containing MIDR_EL1 value |
2470 | * $2 - Scratch register |
2471 | */ |
2472 | .macro EXEC_COREALL_REVEQ |
2473 | mov $2, $1 |
2474 | bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2475 | ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2476 | cmp $2, $0 |
2477 | b.ne 1f |
2478 | .endmacro |
2479 | |
2480 | .macro EXEC_COREALL_REVLO |
2481 | mov $2, $1 |
2482 | bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2483 | ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2484 | cmp $2, $0 |
2485 | b.pl 1f |
2486 | .endmacro |
2487 | |
2488 | .macro EXEC_COREALL_REVHS |
2489 | mov $2, $1 |
2490 | bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2491 | ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2492 | cmp $2, $0 |
2493 | b.mi 1f |
2494 | .endmacro |
2495 | |
2496 | .macro CMP_FOREACH reg, cc, label, car, cdr:vararg |
2497 | cmp \reg, \car |
2498 | b.\cc \label |
2499 | .ifnb \cdr |
2500 | CMP_FOREACH \reg, \cc, \label, \cdr |
2501 | .endif |
2502 | .endm |
2503 | |
2504 | .macro EXEC_COREIN_REVALL midr_el1, scratch, midr_list:vararg |
2505 | and \scratch, \midr_el1, #MIDR_EL1_PNUM_MASK |
2506 | CMP_FOREACH \scratch, eq, Lmatch\@, \midr_list |
2507 | b 1f |
2508 | Lmatch\@: |
2509 | .endm |
2510 | |
2511 | /* |
2512 | * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX |
2513 | * $1 - GPR containing MIDR_EL1 value |
2514 | * $2 - Scratch register |
2515 | */ |
2516 | .macro EXEC_COREEQ_REVALL |
2517 | and $2, $1, #MIDR_EL1_PNUM_MASK |
2518 | cmp $2, $0 |
2519 | b.ne 1f |
2520 | .endmacro |
2521 | |
2522 | /* |
2523 | * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 |
2524 | * $1 - GPR containing MIDR_EL1 value |
2525 | * $2 - Scratch register |
2526 | */ |
2527 | .macro EXEC_PCORE_REVEQ |
2528 | mrs $2, MPIDR_EL1 |
2529 | and $2, $2, #(MPIDR_PNE) |
2530 | cmp $2, xzr |
2531 | b.eq 1f |
2532 | |
2533 | mov $2, $1 |
2534 | bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2535 | ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2536 | cmp $2, $0 |
2537 | b.ne 1f |
2538 | .endmacro |
2539 | |
2540 | .macro EXEC_PCORE_REVLO |
2541 | mrs $2, MPIDR_EL1 |
2542 | and $2, $2, #(MPIDR_PNE) |
2543 | cmp $2, xzr |
2544 | b.eq 1f |
2545 | |
2546 | mov $2, $1 |
2547 | bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2548 | ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2549 | cmp $2, $0 |
2550 | b.pl 1f |
2551 | .endmacro |
2552 | |
2553 | .macro EXEC_PCORE_REVHS |
2554 | mrs $2, MPIDR_EL1 |
2555 | and $2, $2, #(MPIDR_PNE) |
2556 | cmp $2, xzr |
2557 | b.eq 1f |
2558 | |
2559 | mov $2, $1 |
2560 | bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2561 | ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2562 | cmp $2, $0 |
2563 | b.mi 1f |
2564 | .endmacro |
2565 | |
2566 | .macro EXEC_ECORE_REVEQ |
2567 | mrs $2, MPIDR_EL1 |
2568 | and $2, $2, #(MPIDR_PNE) |
2569 | cmp $2, xzr |
2570 | b.ne 1f |
2571 | |
2572 | mov $2, $1 |
2573 | bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2574 | ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2575 | cmp $2, $0 |
2576 | b.ne 1f |
2577 | .endmacro |
2578 | |
2579 | .macro EXEC_ECORE_REVLO |
2580 | mrs $2, MPIDR_EL1 |
2581 | and $2, $2, #(MPIDR_PNE) |
2582 | cmp $2, xzr |
2583 | b.ne 1f |
2584 | |
2585 | mov $2, $1 |
2586 | bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2587 | ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2588 | cmp $2, $0 |
2589 | b.pl 1f |
2590 | .endmacro |
2591 | |
2592 | .macro EXEC_ECORE_REVHS |
2593 | mrs $2, MPIDR_EL1 |
2594 | and $2, $2, #(MPIDR_PNE) |
2595 | cmp $2, xzr |
2596 | b.ne 1f |
2597 | |
2598 | mov $2, $1 |
2599 | bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 |
2600 | ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 |
2601 | cmp $2, $0 |
2602 | b.mi 1f |
2603 | .endmacro |
2604 | |
2605 | /* |
2606 | * $0 - GPR containing MIDR_EL1 value |
2607 | * $1 - Scratch register |
2608 | */ |
2609 | .macro EXEC_PCORE_REVALL |
2610 | mrs $1, MPIDR_EL1 |
2611 | and $1, $1, #(MPIDR_PNE) |
2612 | cmp $1, xzr |
2613 | b.eq 1f |
2614 | .endmacro |
2615 | |
2616 | .macro EXEC_ECORE_REVALL |
2617 | mrs $1, MPIDR_EL1 |
2618 | and $1, $1, #(MPIDR_PNE) |
2619 | cmp $1, xzr |
2620 | b.ne 1f |
2621 | .endmacro |
2622 | |
2623 | |
2624 | |
2625 | /* |
2626 | * Macro that defines the label that all EXEC_COREXX_REVXX macros jump to. |
2627 | */ |
2628 | .macro EXEC_END |
2629 | 1: |
2630 | .endmacro |
2631 | |
2632 | /* |
2633 | * Wedges CPUs with a specified core that are below a specified revision. This |
2634 | * macro is intended for CPUs that have been deprecated in iBoot and may have |
2635 | * incorrect behavior if they continue running xnu. |
2636 | */ |
2637 | .macro DEPRECATE_COREEQ_REVLO core, rev, midr_el1, scratch |
2638 | EXEC_COREEQ_REVLO \core, \rev, \midr_el1, \scratch |
2639 | /* BEGIN IGNORE CODESTYLE */ |
2640 | b . |
2641 | /* END IGNORE CODESTYLE */ |
2642 | EXEC_END |
2643 | .endmacro |
2644 | |
2645 | /* |
2646 | * Sets bits in an SPR register. |
2647 | * arg0: Name of the register to be accessed. |
2648 | * arg1: Mask of bits to be set. |
2649 | * arg2: Scratch register |
2650 | */ |
2651 | .macro HID_SET_BITS |
2652 | mrs $2, $0 |
2653 | orr $2, $2, $1 |
2654 | msr $0, $2 |
2655 | .endmacro |
2656 | |
2657 | /* |
2658 | * Clears bits in an SPR register. |
2659 | * arg0: Name of the register to be accessed. |
2660 | * arg1: Mask of bits to be cleared. |
2661 | * arg2: Scratch register |
2662 | */ |
2663 | .macro HID_CLEAR_BITS |
2664 | mrs $2, $0 |
2665 | bic $2, $2, $1 |
2666 | msr $0, $2 |
2667 | .endmacro |
2668 | |
2669 | /* |
2670 | * Combines the functionality of HID_CLEAR_BITS followed by HID_SET_BITS into |
2671 | * a single read-modify-write sequence. |
2672 | * arg0: Name of the register to be accessed. |
2673 | * arg1: Mask of bits to be cleared. |
2674 | * arg2: Value to insert |
2675 | * arg3: Scratch register |
2676 | */ |
2677 | .macro HID_INSERT_BITS |
2678 | mrs $3, $0 |
2679 | bic $3, $3, $1 |
2680 | orr $3, $3, $2 |
2681 | msr $0, $3 |
2682 | .endmacro |
2683 | |
2684 | /* |
2685 | * Replaces the value of a field in an implementation-defined system register. |
2686 | * sreg: system register name |
2687 | * field: field name within the sysreg, where the assembler symbols |
2688 | * ARM64_REG_<field>_{shift,width} specify the bounds of the field |
2689 | * (note that preprocessor macros will not work here) |
2690 | * value: the value to insert |
2691 | * scr{1,2}: scratch regs |
2692 | */ |
2693 | .macro HID_WRITE_FIELD sreg, field, val, scr1, scr2 |
2694 | mrs \scr1, \sreg |
2695 | mov \scr2, \val |
2696 | bfi \scr1, \scr2, ARM64_REG_\sreg\()_\field\()_shift, ARM64_REG_\sreg\()_\field\()_width |
2697 | msr \sreg, \scr1 |
2698 | .endmacro |
2699 | |
2700 | /* |
2701 | * Macro intended to be used as a replacement for ERET. |
2702 | * It prevents speculation past ERET instructions by padding |
2703 | * up to the decoder width. |
2704 | */ |
2705 | .macro ERET_CONTEXT_SYNCHRONIZING |
2706 | eret |
2707 | #if __ARM_SB_AVAILABLE__ |
2708 | sb // Technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures |
2709 | #else /* __ARM_SB_AVAILABLE__ */ |
2710 | isb // ISB technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures |
2711 | nop // Sequence of six NOPs to pad out and terminate instruction decode group */ |
2712 | nop |
2713 | nop |
2714 | nop |
2715 | nop |
2716 | nop |
2717 | #endif /* !__ARM_SB_AVAILABLE__ */ |
2718 | .endmacro |
2719 | |
2720 | |
2721 | #endif /* __ASSEMBLER__ */ |
2722 | |
2723 | #define MSR(reg, src) __asm__ volatile ("msr " reg ", %0" :: "r" (src)) |
2724 | #define MRS(dest, reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest)) |
2725 | |
2726 | #if XNU_MONITOR |
2727 | #define __ARM_PTE_PHYSMAP__ 1 |
2728 | #define PPL_STATE_KERNEL 0 |
2729 | #define PPL_STATE_DISPATCH 1 |
2730 | #define PPL_STATE_PANIC 2 |
2731 | #define PPL_STATE_EXCEPTION 3 |
2732 | #endif |
2733 | |
2734 | |
2735 | #endif /* _ARM64_PROC_REG_H_ */ |
2736 | |