1/*
2 * Copyright (c) 2007-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/* CMU_ENDHIST */
32/*
33 * Mach Operating System
34 * Copyright (c) 1991,1990 Carnegie Mellon University
35 * All Rights Reserved.
36 *
37 * Permission to use, copy, modify and distribute this software and its
38 * documentation is hereby granted, provided that both the copyright
39 * notice and this permission notice appear in all copies of the
40 * software, derivative works or modified versions, and any portions
41 * thereof, and that both notices appear in supporting documentation.
42 *
43 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
44 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
45 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
46 *
47 * Carnegie Mellon requests users of this software to return to
48 *
49 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
50 * School of Computer Science
51 * Carnegie Mellon University
52 * Pittsburgh PA 15213-3890
53 *
54 * any improvements or extensions that they make and grant Carnegie Mellon
55 * the rights to redistribute these changes.
56 */
57
58/*
59 * Processor registers for ARM/ARM64
60 */
61#ifndef _ARM64_PROC_REG_H_
62#define _ARM64_PROC_REG_H_
63
64#if defined (__arm64__)
65#include <pexpert/arm64/board_config.h>
66#elif defined (__arm__)
67#include <pexpert/arm/board_config.h>
68#endif
69
70/*
71 * Processor registers for ARM
72 */
73#if __ARM_42BIT_PA_SPACE__
74/* For now, force the issue! */
75/* We need more VA space for the identity map to bootstrap the MMU */
76#undef __ARM64_PMAP_SUBPAGE_L1__
77#endif /* __ARM_42BIT_PA_SPACE__ */
78
79/* For arm platforms, create one pset per cluster */
80#define MAX_PSETS MAX_CPU_CLUSTERS
81
82
83/* Thread groups are enabled on all ARM platforms (irrespective of scheduler) */
84#define CONFIG_THREAD_GROUPS 1
85
86#ifdef XNU_KERNEL_PRIVATE
87
88#if __ARM_VFP__
89#define ARM_VFP_DEBUG 0
90#endif /* __ARM_VFP__ */
91
92#endif /* XNU_KERNEL_PRIVATE */
93
94/*
95 * FSR registers
96 *
97 * CPSR: Current Program Status Register
98 * SPSR: Saved Program Status Registers
99 *
100 * 31 30 29 28 27 24 19 16 9 8 7 6 5 4 0
101 * +-----------------------------------------------------------+
102 * | N| Z| C| V| Q|...| J|...|GE[3:0]|...| E| A| I| F| T| MODE |
103 * +-----------------------------------------------------------+
104 */
105
106/*
107 * Flags
108 */
109#define PSR_NF 0x80000000 /* Negative/Less than */
110#define PSR_ZF 0x40000000 /* Zero */
111#define PSR_CF 0x20000000 /* Carry/Borrow/Extend */
112#define PSR_VF 0x10000000 /* Overflow */
113
114/*
115 * Modified execution mode flags
116 */
117#define PSR_TF 0x00000020 /* thumb flag (BX ARMv4T) */
118
119/*
120 * CPU mode
121 */
122#define PSR_USER_MODE 0x00000010 /* User mode */
123
124#define PSR_MODE_MASK 0x0000001F
125#define PSR_IS_KERNEL(psr) (((psr) & PSR_MODE_MASK) != PSR_USER_MODE)
126#define PSR_IS_USER(psr) (((psr) & PSR_MODE_MASK) == PSR_USER_MODE)
127
128#define PSR_USERDFLT PSR_USER_MODE
129
130/*
131 * Cache configuration
132 */
133
134#if defined (APPLETYPHOON)
135
136/* I-Cache */
137#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
138
139/* D-Cache */
140#define MMU_CLINE 6 /* cache line size as 1<<MMU_CLINE (64) */
141
142#elif defined (APPLETWISTER)
143
144/* I-Cache */
145#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
146
147/* D-Cache */
148#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
149
150#elif defined (APPLEHURRICANE)
151
152/* I-Cache */
153#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
154
155/* D-Cache */
156#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
157
158#elif defined (APPLEMONSOON)
159
160/* I-Cache, 96KB for Monsoon, 48KB for Mistral, 6-way. */
161#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
162
163/* D-Cache, 64KB for Monsoon, 32KB for Mistral, 4-way. */
164#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
165
166#elif defined (APPLEVORTEX)
167
168/* I-Cache, 128KB 8-way for Vortex, 48KB 6-way for Tempest. */
169#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
170
171/* D-Cache, 128KB 8-way for Vortex, 32KB 4-way for Tempest. */
172#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
173
174#elif defined (APPLELIGHTNING)
175
176/* I-Cache, 192KB for Lightning, 96KB for Thunder, 6-way. */
177#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
178
179/* D-Cache, 128KB for Lightning, 8-way. 48KB for Thunder, 6-way. */
180#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
181
182#elif defined (APPLEFIRESTORM)
183
184/* I-Cache, 256KB for Firestorm, 128KB for Icestorm, 6-way. */
185#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
186
187/* D-Cache, 160KB for Firestorm, 8-way. 64KB for Icestorm, 6-way. */
188#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
189
190#elif defined (APPLEAVALANCHE)
191
192/* I-Cache, 192KB for Avalanche, 128KB for Blizzard, 6-way. */
193#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
194
195/* D-Cache, 128KB for Avalanche, 8-way. 64KB for Blizzard, 8-way. */
196#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
197
198#elif defined (BCM2837) /* Raspberry Pi 3 */
199
200/* I-Cache. We don't have detailed spec so we just follow the ARM technical reference. */
201#define MMU_I_CLINE 6
202
203/* D-Cache. */
204#define MMU_CLINE 6
205
206#elif defined (VMAPPLE)
207
208/* I-Cache. */
209#define MMU_I_CLINE 6
210
211/* D-Cache. */
212#define MMU_CLINE 6
213
214#else
215#error processor not supported
216#endif
217
218#define MAX_L2_CLINE_BYTES (1 << MAX_L2_CLINE)
219
220/*
221 * Format of the Debug & Watchpoint Breakpoint Value and Control Registers
222 */
223#define ARM_DBG_VR_ADDRESS_MASK 0xFFFFFFFC /* BVR & WVR */
224#define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */
225
226#define ARM_DBG_CR_ADDRESS_MASK_MASK 0x1F000000 /* BCR & WCR */
227#define ARM_DBGBCR_MATCH_MASK (1 << 22) /* BCR only */
228#define ARM_DBGBCR_TYPE_MASK (1 << 21) /* BCR only */
229#define ARM_DBGBCR_TYPE_IVA (0 << 21)
230#define ARM_DBG_CR_LINKED_MASK (1 << 20) /* BCR & WCR */
231#define ARM_DBG_CR_LINKED_UNLINKED (0 << 20)
232#define ARM_DBG_CR_SECURITY_STATE_BOTH (0 << 14)
233#define ARM_DBG_CR_HIGHER_MODE_ENABLE (1 << 13)
234#define ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK 0x00001FE0 /* WCR only */
235#define ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK 0x000001E0 /* BCR & WCR */
236#define ARM_DBGWCR_ACCESS_CONTROL_MASK (3 << 3) /* WCR only */
237#define ARM_DBG_CR_MODE_CONTROL_PRIVILEGED (1 << 1) /* BCR & WCR */
238#define ARM_DBG_CR_MODE_CONTROL_USER (2 << 1) /* BCR & WCR */
239#define ARM_DBG_CR_ENABLE_MASK (1 << 0) /* BCR & WCR */
240#define ARM_DBG_CR_ENABLE_ENABLE (1 << 0)
241
242/*
243 * Format of the OS Lock Access (DBGOSLAR) and Lock Access Registers (DBGLAR)
244 */
245#define ARM_DBG_LOCK_ACCESS_KEY 0xC5ACCE55
246
247/* ARM Debug registers of interest */
248#define ARM_DEBUG_OFFSET_DBGPRCR (0x310)
249#define ARM_DEBUG_OFFSET_DBGLAR (0xFB0)
250
251/*
252 * Main ID Register (MIDR)
253 *
254 * 31 24 23 20 19 16 15 4 3 0
255 * +-----+-----+------+------+-----+
256 * | IMP | VAR | ARCH | PNUM | REV |
257 * +-----+-----+------+------+-----+
258 *
259 * where:
260 * IMP: Implementor code
261 * VAR: Variant number
262 * ARCH: Architecture code
263 * PNUM: Primary part number
264 * REV: Minor revision number
265 */
266#define MIDR_REV_SHIFT 0
267#define MIDR_REV_MASK (0xf << MIDR_REV_SHIFT)
268#define MIDR_VAR_SHIFT 20
269#define MIDR_VAR_MASK (0xf << MIDR_VAR_SHIFT)
270
271
272#if __ARM_KERNEL_PROTECT__
273/*
274 * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential
275 * architectural or microarchitectural vulnerabilities that could allow cores to
276 * read/access EL1-only mappings while in EL0 mode. This is achieved by
277 * removing as many mappings as possible when the core transitions to EL0 mode
278 * from EL1 mode, and restoring those mappings when the core transitions to EL1
279 * mode from EL0 mode.
280 *
281 * At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is
282 * used to map and unmap the ordinary kernel mappings, by contracting and
283 * expanding translation zone size for TTBR1 when exiting and entering EL1,
284 * respectively:
285 *
286 * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is
287 * is in EL0.
288 * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in
289 * EL1.
290 *
291 * T1SZ_USER: T1SZ_BOOT + 1
292 * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT)
293 * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER)
294 * TTBR1_EL1_MAX: (2^64) - 1
295 *
296 * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the
297 * the following TTBR1 layout:
298 *
299 * TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX
300 * +---------------------------------------------------------+
301 * | Kernel EL0 Mappings | Kernel EL1 Mappings |
302 * +---------------------------------------------------------+
303 *
304 * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout:
305 *
306 * TTBR1_EL1_BASE_USER TTBR1_EL1_MAX
307 * +---------------------------------------------------------+
308 * | Kernel EL0 Mappings |
309 * +---------------------------------------------------------+
310 *
311 * With the current implementation, both the EL0 and EL1 mappings for the kernel
312 * use otherwise empty translation tables for mapping the exception vectors (so
313 * that we do not need to TLB flush the exception vector address when switching
314 * between EL0 and EL1). The rationale here is that the TLBI would require a
315 * DSB, and DSBs can be extremely expensive.
316 *
317 * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1
318 * ASID. The core switches between ASIDs on EL transitions, so that the TLB
319 * does not need to be fully invalidated on an EL transition.
320 *
321 * Most kernel mappings will be marked non-global in this configuration, as
322 * global mappings would be visible to userspace unless we invalidate them on
323 * eret.
324 */
325#if XNU_MONITOR
326/*
327 * Please note that because we indirect through the thread register in order to
328 * locate the kernel, and because we unmap most of the kernel, the security
329 * model of the PPL is undermined by __ARM_KERNEL_PROTECT__, as we rely on
330 * kernel controlled data to direct codeflow in the exception vectors.
331 *
332 * If we want to ship XNU_MONITOR paired with __ARM_KERNEL_PROTECT__, we will
333 * need to find a performant solution to this problem.
334 */
335#endif
336#endif /* __ARM_KERNEL_PROTECT */
337
338#if ARM_PARAMETERIZED_PMAP
339/*
340 * ARM_PARAMETERIZED_PMAP configures the kernel to get the characteristics of
341 * the page tables (number of levels, size of the root allocation) from the
342 * pmap data structure, rather than treating them as compile-time constants.
343 * This allows the pmap code to dynamically adjust how it deals with page
344 * tables.
345 */
346#endif /* ARM_PARAMETERIZED_PMAP */
347
348#if __ARM_MIXED_PAGE_SIZE__
349/*
350 * __ARM_MIXED_PAGE_SIZE__ configures the kernel to support page tables that do
351 * not use the kernel page size. This is primarily meant to support running
352 * 4KB page processes on a 16KB page kernel.
353 *
354 * This only covers support in the pmap/machine dependent layers. Any support
355 * elsewhere in the kernel must be managed separately.
356 */
357#if !ARM_PARAMETERIZED_PMAP
358/*
359 * Page tables that use non-kernel page sizes require us to reprogram TCR based
360 * on the page tables we are switching to. This means that the parameterized
361 * pmap support is required.
362 */
363#error __ARM_MIXED_PAGE_SIZE__ requires ARM_PARAMETERIZED_PMAP
364#endif /* !ARM_PARAMETERIZED_PMAP */
365#if __ARM_KERNEL_PROTECT__
366/*
367 * Because switching the page size requires updating TCR based on the pmap, and
368 * __ARM_KERNEL_PROTECT__ relies on TCR being programmed with constants, XNU
369 * does not currently support support configurations that use both
370 * __ARM_KERNEL_PROTECT__ and __ARM_MIXED_PAGE_SIZE__.
371 */
372#error __ARM_MIXED_PAGE_SIZE__ and __ARM_KERNEL_PROTECT__ are mutually exclusive
373#endif /* __ARM_KERNEL_PROTECT__ */
374#endif /* __ARM_MIXED_PAGE_SIZE__ */
375
376/*
377 * 64-bit Program Status Register (PSR64)
378 *
379 * 31 27 23 22 21 20 19 10 9 5 4 0
380 * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+
381 * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0| M |
382 * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+
383 *
384 * where:
385 * NZCV: Comparison flags
386 * PAN: Privileged Access Never
387 * SS: Single step
388 * IL: Illegal state
389 * DAIF: Interrupt masks
390 * M: Mode field
391 */
392
393#define PSR64_NZCV_SHIFT 28
394#define PSR64_NZCV_MASK (0xF << PSR64_NZCV_SHIFT)
395
396#define PSR64_N_SHIFT 31
397#define PSR64_N (1 << PSR64_N_SHIFT)
398
399#define PSR64_Z_SHIFT 30
400#define PSR64_Z (1 << PSR64_Z_SHIFT)
401
402#define PSR64_C_SHIFT 29
403#define PSR64_C (1 << PSR64_C_SHIFT)
404
405#define PSR64_V_SHIFT 28
406#define PSR64_V (1 << PSR64_V_SHIFT)
407
408#define PSR64_TCO_SHIFT 25
409#define PSR64_TCO (1 << PSR64_TCO_SHIFT)
410
411#define PSR64_DIT_SHIFT 24
412#define PSR64_DIT (1 << PSR64_DIT_SHIFT)
413
414#define PSR64_UAO_SHIFT 23
415#define PSR64_UAO (1 << PSR64_UAO_SHIFT)
416
417#define PSR64_PAN_SHIFT 22
418#define PSR64_PAN (1 << PSR64_PAN_SHIFT)
419
420#define PSR64_SS_SHIFT 21
421#define PSR64_SS (1 << PSR64_SS_SHIFT)
422
423#define PSR64_IL_SHIFT 20
424#define PSR64_IL (1 << PSR64_IL_SHIFT)
425
426/*
427 * SSBS is bit 12 for A64 SPSR and bit 23 for A32 SPSR
428 * I do not want to talk about it!
429 */
430#define PSR64_SSBS_SHIFT_32 23
431#define PSR64_SSBS_SHIFT_64 12
432#define PSR64_SSBS_32 (1 << PSR64_SSBS_SHIFT_32)
433#define PSR64_SSBS_64 (1 << PSR64_SSBS_SHIFT_64)
434
435/*
436 * msr DAIF, Xn and mrs Xn, DAIF transfer into
437 * and out of bits 9:6
438 */
439#define DAIF_DEBUG_SHIFT 9
440#define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT)
441
442#define DAIF_ASYNC_SHIFT 8
443#define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT)
444
445#define DAIF_IRQF_SHIFT 7
446#define DAIF_IRQF (1 << DAIF_IRQF_SHIFT)
447
448#define DAIF_FIQF_SHIFT 6
449#define DAIF_FIQF (1 << DAIF_FIQF_SHIFT)
450
451#define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
452#define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
453
454#define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF))
455
456#if HAS_ARM_FEAT_SSBS2
457#define PSR64_SSBS_U32_DEFAULT PSR64_SSBS_32
458#define PSR64_SSBS_U64_DEFAULT PSR64_SSBS_64
459#define PSR64_SSBS_KRN_DEFAULT PSR64_SSBS_64
460#else
461#define PSR64_SSBS_U32_DEFAULT (0)
462#define PSR64_SSBS_U64_DEFAULT (0)
463#define PSR64_SSBS_KRN_DEFAULT (0)
464#endif
465
466/*
467 * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer
468 * from bits 3:0.
469 */
470#define DAIFSC_DEBUGF (1 << 3)
471#define DAIFSC_ASYNCF (1 << 2)
472#define DAIFSC_IRQF (1 << 1)
473#define DAIFSC_FIQF (1 << 0)
474#define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
475#define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
476#define DAIFSC_NOASYNC (DAIFSC_DEBUGF | DAIFSC_IRQF | DAIFSC_FIQF)
477
478/*
479 * ARM64_TODO: unify with ARM?
480 */
481#define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */
482
483#define PSR64_MODE_MASK 0x1F
484
485#define PSR64_USER_MASK PSR64_NZCV_MASK
486
487#define PSR64_MODE_USER32_THUMB 0x20
488
489#define PSR64_MODE_RW_SHIFT 4
490#define PSR64_MODE_RW_64 0
491#define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT)
492
493#define PSR64_MODE_EL_SHIFT 2
494#define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT)
495#define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT)
496#define PSR64_MODE_EL2 (0x2 << PSR64_MODE_EL_SHIFT)
497#define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT)
498#define PSR64_MODE_EL0 0
499
500#define PSR64_MODE_EL_KERNEL (PSR64_MODE_EL1)
501
502#define PSR64_MODE_SPX 0x1
503#define PSR64_MODE_SP0 0
504
505#define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U32_DEFAULT)
506#define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U64_DEFAULT)
507#define PSR64_KERNEL_STANDARD (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0 | PSR64_SSBS_KRN_DEFAULT)
508#if __ARM_PAN_AVAILABLE__
509#define PSR64_KERNEL_DEFAULT (PSR64_KERNEL_STANDARD | PSR64_PAN)
510#else
511#define PSR64_KERNEL_DEFAULT PSR64_KERNEL_STANDARD
512#endif
513
514#define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0)
515#define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0)
516
517#define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32))
518#define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32))
519
520
521
522/*
523 * System Control Register (SCTLR)
524 */
525
526#define SCTLR_OTHER (0)
527
528#define SCTLR_DSSBS (1ULL << 44)
529
530
531#define SCTLR_EXTRA (0)
532
533
534#define SCTLR_RESERVED ((3ULL << 28) | (1ULL << 20))
535#if defined(HAS_APPLE_PAC)
536
537// 31 PACIA_ENABLED AddPACIA and AuthIA functions enabled
538#define SCTLR_PACIA_ENABLED_SHIFT 31
539#define SCTLR_PACIA_ENABLED (1ULL << SCTLR_PACIA_ENABLED_SHIFT)
540// 30 PACIB_ENABLED AddPACIB and AuthIB functions enabled
541#define SCTLR_PACIB_ENABLED (1ULL << 30)
542// 29:28 RES1 11
543// 27 PACDA_ENABLED AddPACDA and AuthDA functions enabled
544#define SCTLR_PACDA_ENABLED (1ULL << 27)
545// 13 PACDB_ENABLED AddPACDB and AuthDB functions enabled
546#define SCTLR_PACDB_ENABLED (1ULL << 13)
547
548#define SCTLR_JOP_KEYS_ENABLED (SCTLR_PACIA_ENABLED | SCTLR_PACDA_ENABLED | SCTLR_PACDB_ENABLED)
549#endif /* defined(HAS_APPLE_PAC) */
550
551// 26 UCI User Cache Instructions
552#define SCTLR_UCI_ENABLED (1ULL << 26)
553
554// 25 EE Exception Endianness
555#define SCTLR_EE_BIG_ENDIAN (1ULL << 25)
556
557// 24 E0E EL0 Endianness
558#define SCTLR_E0E_BIG_ENDIAN (1ULL << 24)
559
560// 23 SPAN Set PAN
561#define SCTLR_PAN_UNCHANGED (1ULL << 23)
562
563// 22 EIS Taking an exception is a context synchronization event
564#define SCTLR_EIS (1ULL << 22)
565
566// 21 RES0 0
567// 20 RES1 1
568
569// 19 WXN Writeable implies eXecute Never
570#define SCTLR_WXN_ENABLED (1ULL << 19)
571
572// 18 nTWE Not trap WFE from EL0
573#define SCTLR_nTWE_WFE_ENABLED (1ULL << 18)
574
575// 17 RES0 0
576
577// 16 nTWI Not trap WFI from EL0
578#define SCTRL_nTWI_WFI_ENABLED (1ULL << 16)
579
580// 15 UCT User Cache Type register (CTR_EL0)
581#define SCTLR_UCT_ENABLED (1ULL << 15)
582
583// 14 DZE User Data Cache Zero (DC ZVA)
584#define SCTLR_DZE_ENABLED (1ULL << 14)
585
586// 12 I Instruction cache enable
587#define SCTLR_I_ENABLED (1ULL << 12)
588
589// 11 EOS Exception return is a context synchronization event
590#define SCTLR_EOS (1ULL << 11)
591
592// 10 EnRCTX EL0 Access to FEAT_SPECRES speculation restriction instructions
593#define SCTLR_EnRCTX (1ULL << 10)
594
595// 9 UMA User Mask Access
596#define SCTLR_UMA_ENABLED (1ULL << 9)
597
598// 8 SED SETEND Disable
599#define SCTLR_SED_DISABLED (1ULL << 8)
600
601// 7 ITD IT Disable
602#define SCTLR_ITD_DISABLED (1ULL << 7)
603
604// 6 RES0 0
605
606// 5 CP15BEN CP15 Barrier ENable
607#define SCTLR_CP15BEN_ENABLED (1ULL << 5)
608
609// 4 SA0 Stack Alignment check for EL0
610#define SCTLR_SA0_ENABLED (1ULL << 4)
611
612// 3 SA Stack Alignment check
613#define SCTLR_SA_ENABLED (1ULL << 3)
614
615// 2 C Cache enable
616#define SCTLR_C_ENABLED (1ULL << 2)
617
618// 1 A Alignment check
619#define SCTLR_A_ENABLED (1ULL << 1)
620
621// 0 M MMU enable
622#define SCTLR_M_ENABLED (1ULL << 0)
623
624#if __ARM_ARCH_8_5__
625#define SCTLR_CSEH_DEFAULT (0)
626#define SCTLR_DSSBS_DEFAULT SCTLR_DSSBS
627#else
628#define SCTLR_CSEH_DEFAULT (SCTLR_EIS | SCTLR_EOS)
629#define SCTLR_DSSBS_DEFAULT (0)
630#endif
631
632#if HAS_APPLE_PAC
633#define SCTLR_ROP_KEYS_DEFAULT SCTLR_PACIB_ENABLED /* IB is ROP */
634#else /* !HAS_APPLE_PAC */
635#define SCTLR_ROP_KEYS_DEFAULT 0
636#endif /* HAS_APPLE_PAC */
637
638#if HAS_APPLE_PAC
639#define SCTLR_JOP_KEYS_DEFAULT SCTLR_JOP_KEYS_ENABLED
640#else /* !HAS_APPLE_PAC */
641#define SCTLR_JOP_KEYS_DEFAULT 0
642#endif
643
644#define SCTLR_EL1_DEFAULT \
645 (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \
646 SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED | \
647 SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED | \
648 SCTLR_CSEH_DEFAULT | SCTLR_DSSBS_DEFAULT | SCTLR_ROP_KEYS_DEFAULT | \
649 SCTLR_JOP_KEYS_DEFAULT | SCTLR_OTHER | SCTLR_EXTRA )
650
651/*
652 * Coprocessor Access Control Register (CPACR)
653 *
654 * 31 28 27 22 21 20 19 0
655 * +---+---+------+------+--------------------+
656 * |000|TTA|000000| FPEN |00000000000000000000|
657 * +---+---+------+------+--------------------+
658 *
659 * where:
660 * TTA: Trace trap
661 * FPEN: Floating point enable
662 */
663#define CPACR_TTA_SHIFT 28
664#define CPACR_TTA (1 << CPACR_TTA_SHIFT)
665
666
667#define CPACR_FPEN_SHIFT 20
668#define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT)
669#define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT)
670
671
672/*
673 * FPSR: Floating Point Status Register
674 *
675 * 31 30 29 28 27 26 7 6 4 3 2 1 0
676 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
677 * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC|
678 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
679 */
680
681#define FPSR_N_SHIFT 31
682#define FPSR_Z_SHIFT 30
683#define FPSR_C_SHIFT 29
684#define FPSR_V_SHIFT 28
685#define FPSR_QC_SHIFT 27
686#define FPSR_IDC_SHIFT 7
687#define FPSR_IXC_SHIFT 4
688#define FPSR_UFC_SHIFT 3
689#define FPSR_OFC_SHIFT 2
690#define FPSR_DZC_SHIFT 1
691#define FPSR_IOC_SHIFT 0
692#define FPSR_N (1 << FPSR_N_SHIFT)
693#define FPSR_Z (1 << FPSR_Z_SHIFT)
694#define FPSR_C (1 << FPSR_C_SHIFT)
695#define FPSR_V (1 << FPSR_V_SHIFT)
696#define FPSR_QC (1 << FPSR_QC_SHIFT)
697#define FPSR_IDC (1 << FPSR_IDC_SHIFT)
698#define FPSR_IXC (1 << FPSR_IXC_SHIFT)
699#define FPSR_UFC (1 << FPSR_UFC_SHIFT)
700#define FPSR_OFC (1 << FPSR_OFC_SHIFT)
701#define FPSR_DZC (1 << FPSR_DZC_SHIFT)
702#define FPSR_IOC (1 << FPSR_IOC_SHIFT)
703
704/*
705 * A mask for all for all of the bits that are not RAZ for FPSR; this
706 * is primarily for converting between a 32-bit view of NEON state
707 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
708 */
709#define FPSR_MASK \
710 (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | FPSR_IDC | FPSR_IXC | \
711 FPSR_UFC | FPSR_OFC | FPSR_DZC | FPSR_IOC)
712
713/*
714 * FPCR: Floating Point Control Register
715 *
716 * 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0
717 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
718 * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000|
719 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
720 */
721
722#define FPCR_AHP_SHIFT 26
723#define FPCR_DN_SHIFT 25
724#define FPCR_FZ_SHIFT 24
725#define FPCR_RMODE_SHIFT 22
726#define FPCR_STRIDE_SHIFT 20
727#define FPCR_LEN_SHIFT 16
728#define FPCR_IDE_SHIFT 15
729#define FPCR_IXE_SHIFT 12
730#define FPCR_UFE_SHIFT 11
731#define FPCR_OFE_SHIFT 10
732#define FPCR_DZE_SHIFT 9
733#define FPCR_IOE_SHIFT 8
734#define FPCR_AHP (1 << FPCR_AHP_SHIFT)
735#define FPCR_DN (1 << FPCR_DN_SHIFT)
736#define FPCR_FZ (1 << FPCR_FZ_SHIFT)
737#define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT)
738#define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT)
739#define FPCR_LEN (0x7 << FPCR_LEN_SHIFT)
740#define FPCR_IDE (1 << FPCR_IDE_SHIFT)
741#define FPCR_IXE (1 << FPCR_IXE_SHIFT)
742#define FPCR_UFE (1 << FPCR_UFE_SHIFT)
743#define FPCR_OFE (1 << FPCR_OFE_SHIFT)
744#define FPCR_DZE (1 << FPCR_DZE_SHIFT)
745#define FPCR_IOE (1 << FPCR_IOE_SHIFT)
746#define FPCR_DEFAULT (0)
747#define FPCR_DEFAULT_32 (FPCR_DN|FPCR_FZ)
748
749/*
750 * A mask for all for all of the bits that are not RAZ for FPCR; this
751 * is primarily for converting between a 32-bit view of NEON state
752 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
753 */
754#define FPCR_MASK \
755 (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | FPCR_STRIDE | FPCR_LEN | \
756 FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE)
757
758/*
759 * Translation Control Register (TCR)
760 *
761 * Legacy:
762 *
763 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0
764 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
765 * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
766 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
767 *
768 * Current (with 16KB granule support):
769 *
770 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0
771 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
772 * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
773 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
774 *
775 * TBI1: Top Byte Ignored for TTBR1 region
776 * TBI0: Top Byte Ignored for TTBR0 region
777 * AS: ASID Size
778 * IPS: Physical Address Size limit
779 * TG1: Granule Size for TTBR1 region
780 * SH1: Shareability for TTBR1 region
781 * ORGN1: Outer Cacheability for TTBR1 region
782 * IRGN1: Inner Cacheability for TTBR1 region
783 * EPD1: Translation table walk disable for TTBR1
784 * A1: ASID selection from TTBR1 enable
785 * T1SZ: Virtual address size for TTBR1
786 * TG0: Granule Size for TTBR0 region
787 * SH0: Shareability for TTBR0 region
788 * ORGN0: Outer Cacheability for TTBR0 region
789 * IRGN0: Inner Cacheability for TTBR0 region
790 * T0SZ: Virtual address size for TTBR0
791 */
792
793#define TCR_T0SZ_SHIFT 0ULL
794#define TCR_TSZ_BITS 6ULL
795#define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL)
796
797#define TCR_IRGN0_SHIFT 8ULL
798#define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT)
799#define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT)
800#define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT)
801#define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT)
802
803#define TCR_ORGN0_SHIFT 10ULL
804#define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT)
805#define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT)
806#define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT)
807#define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT)
808
809#define TCR_SH0_SHIFT 12ULL
810#define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT)
811#define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT)
812#define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT)
813
814#define TCR_TG0_GRANULE_SHIFT (14ULL)
815#define TCR_TG0_GRANULE_BITS (2ULL)
816#define TCR_TG0_GRANULE_MASK ((1ULL << TCR_TG0_GRANULE_BITS) - 1ULL)
817
818#define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT)
819#define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT)
820#define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT)
821
822#if __ARM_16K_PG__
823#define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB)
824#else
825#define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB)
826#endif
827
828#define TCR_T1SZ_SHIFT 16ULL
829
830#define TCR_A1_ASID1 (1ULL << 22ULL)
831#define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL)
832
833#define TCR_IRGN1_SHIFT 24ULL
834#define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT)
835#define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT)
836#define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT)
837#define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT)
838
839#define TCR_ORGN1_SHIFT 26ULL
840#define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT)
841#define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT)
842#define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT)
843#define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT)
844
845#define TCR_SH1_SHIFT 28ULL
846#define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT)
847#define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT)
848#define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT)
849
850#define TCR_TG1_GRANULE_SHIFT 30ULL
851#define TCR_TG1_GRANULE_BITS (2ULL)
852#define TCR_TG1_GRANULE_MASK ((1ULL << TCR_TG1_GRANULE_BITS) - 1ULL)
853
854#define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT)
855#define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT)
856#define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT)
857
858#if __ARM_16K_PG__
859#define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB)
860#else
861#define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB)
862#endif
863
864#define TCR_IPS_SHIFT 32ULL
865#define TCR_IPS_BITS 3ULL
866#define TCR_IPS_MASK ((1ULL << TCR_IPS_BITS) - 1ULL)
867#define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT)
868#define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT)
869#define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT)
870#define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT)
871#define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT)
872#define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT)
873
874#define TCR_AS_16BIT_ASID (1ULL << 36)
875#define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37)
876#define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38)
877#define TCR_TBID0_TBI_DATA_ONLY (1ULL << 51)
878#define TCR_TBID1_TBI_DATA_ONLY (1ULL << 52)
879
880#if defined(HAS_APPLE_PAC)
881#define TCR_TBID0_ENABLE TCR_TBID0_TBI_DATA_ONLY
882#define TCR_TBID1_ENABLE TCR_TBID1_TBI_DATA_ONLY
883#else
884#define TCR_TBID0_ENABLE 0
885#define TCR_TBID1_ENABLE 0
886#endif
887
888#define TCR_E0PD0_BIT (1ULL << 55)
889#define TCR_E0PD1_BIT (1ULL << 56)
890
891#if defined(HAS_E0PD)
892#define TCR_E0PD_VALUE (TCR_E0PD1_BIT)
893#else
894#define TCR_E0PD_VALUE 0
895#endif
896
897
898#define TCR_EL1_EXTRA 0
899
900
901/*
902 * Multiprocessor Affinity Register (MPIDR_EL1)
903 *
904 * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+
905 * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0|
906 * +---------------------------------+--+-----+--+-----+----+----+
907 *
908 * where
909 * U: Uniprocessor
910 * MT: Multi-threading at lowest affinity level
911 * Aff2: "1" - PCORE, "0" - ECORE
912 * Aff1: Cluster ID
913 * Aff0: CPU ID
914 */
915#define MPIDR_AFF0_SHIFT 0
916#define MPIDR_AFF0_WIDTH 8
917#define MPIDR_AFF0_MASK (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT)
918#define MPIDR_AFF1_SHIFT 8
919#define MPIDR_AFF1_WIDTH 8
920#define MPIDR_AFF1_MASK (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT)
921#define MPIDR_AFF2_SHIFT 16
922#define MPIDR_AFF2_WIDTH 8
923#define MPIDR_AFF2_MASK (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT)
924
925/*
926 * TXSZ indicates the size of the range a TTBR covers. Currently,
927 * we support the following:
928 *
929 * 4KB pages, full page L1: 39 bit range.
930 * 4KB pages, sub-page L1: 38 bit range.
931 * 16KB pages, full page L1: 47 bit range.
932 * 16KB pages, sub-page L1: 39 bit range.
933 * 16KB pages, two level page tables: 36 bit range.
934 */
935#if __ARM_KERNEL_PROTECT__
936/*
937 * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the
938 * address space is used for the mappings that will remain in place when in EL0.
939 * As a result, 1 bit less of address space is available to the rest of the
940 * the kernel.
941 */
942#endif /* __ARM_KERNEL_PROTECT__ */
943#ifdef __ARM_16K_PG__
944#if __ARM64_PMAP_SUBPAGE_L1__
945#define T0SZ_BOOT 25ULL
946#else /* !__ARM64_PMAP_SUBPAGE_L1__ */
947#define T0SZ_BOOT 17ULL
948#endif /* !__ARM64_PMAP_SUBPAGE_L1__ */
949#else /* __ARM_16K_PG__ */
950#if __ARM64_PMAP_SUBPAGE_L1__
951#define T0SZ_BOOT 26ULL
952#else /* __ARM64_PMAP_SUBPAGE_L1__ */
953#define T0SZ_BOOT 25ULL
954#endif /* __ARM64_PMAP_SUBPAGE_L1__ */
955#endif /* __ARM_16K_PG__ */
956
957#if defined(APPLE_ARM64_ARCH_FAMILY)
958/* T0SZ must be the same as T1SZ */
959#define T1SZ_BOOT T0SZ_BOOT
960#else /* defined(APPLE_ARM64_ARCH_FAMILY) */
961#ifdef __ARM_16K_PG__
962#if __ARM64_PMAP_SUBPAGE_L1__
963#define T1SZ_BOOT 25ULL
964#else /* !__ARM64_PMAP_SUBPAGE_L1__ */
965#define T1SZ_BOOT 17ULL
966#endif /* !__ARM64_PMAP_SUBPAGE_L1__ */
967#else /* __ARM_16K_PG__ */
968#if __ARM64_PMAP_SUBPAGE_L1__
969#define T1SZ_BOOT 26ULL
970#else /* __ARM64_PMAP_SUBPAGE_L1__ */
971#define T1SZ_BOOT 25ULL
972#endif /*__ARM64_PMAP_SUBPAGE_L1__*/
973#endif /* __ARM_16K_PG__ */
974#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
975
976#if __ARM_42BIT_PA_SPACE__
977#define TCR_IPS_VALUE TCR_IPS_42BITS
978#else /* !__ARM_42BIT_PA_SPACE__ */
979#define TCR_IPS_VALUE TCR_IPS_40BITS
980#endif /* !__ARM_42BIT_PA_SPACE__ */
981
982#if CONFIG_KERNEL_TBI
983#define TCR_EL1_DTBI (TCR_TBI1_TOPBYTE_IGNORED | TCR_TBID1_ENABLE)
984#else /* CONFIG_KERNEL_TBI */
985#define TCR_EL1_DTBI 0
986#endif /* CONFIG_KERNEL_TBI */
987
988#if HAS_16BIT_ASID
989#define TCR_EL1_ASID TCR_AS_16BIT_ASID
990#else /* HAS_16BIT_ASID */
991#define TCR_EL1_ASID 0
992#endif /* HAS_16BIT_ASID */
993
994#define TCR_EL1_BASE \
995 (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | \
996 TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | \
997 TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \
998 TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) | \
999 TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE) | TCR_E0PD_VALUE | \
1000 TCR_EL1_DTBI | TCR_EL1_ASID | TCR_EL1_EXTRA)
1001
1002#if __ARM_KERNEL_PROTECT__
1003#define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE))
1004#define T1SZ_USER (T1SZ_BOOT + 1)
1005#define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE))
1006#else
1007#define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE))
1008#endif /* __ARM_KERNEL_PROTECT__ */
1009
1010#define TCR_EL1_4KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_4KB))
1011#define TCR_EL1_16KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_16KB))
1012
1013
1014
1015
1016/*
1017 * Monitor Debug System Control Register (MDSCR)
1018 */
1019
1020#define MDSCR_TFO_SHIFT 31
1021#define MDSCR_TFO (1ULL << MDSCR_TFO_SHIFT)
1022#define MDSCR_RXFULL_SHIFT 30
1023#define MDSCR_RXFULL (1ULL << MDSCR_RXFULL_SHIFT)
1024#define MDSCR_TXFULL_SHIFT 29
1025#define MDSCR_TXFULL (1ULL << MDSCR_TXFULL_SHIFT)
1026#define MDSCR_RXO_SHIFT 27
1027#define MDSCR_RXO (1ULL << MDSCR_RXO_SHIFT)
1028#define MDSCR_TXU_SHIFT 26
1029#define MDSCR_TXU (1ULL << MDSCR_TXU_SHIFT)
1030#define MDSCR_INTDIS_SHIFT 22
1031#define MDSCR_INTDIS_MASK (0x2U << MDSCR_INTDIS_SHIFT)
1032#define MDSCR_TDA_SHIFT 21
1033#define MDSCR_TDA (1ULL << MDSCR_TDA_SHIFT)
1034#define MDSCR_SC2_SHIFT 19
1035#define MDSCR_SC2 (1ULL << MDSCR_SC2_SHIFT)
1036#define MDSCR_MDE_SHIFT 15
1037#define MDSCR_MDE (1ULL << MDSCR_MDE_SHIFT)
1038#define MDSCR_HDE_SHIFT 14
1039#define MDSCR_HDE (1ULL << MDSCR_HDE_SHIFT)
1040#define MDSCR_KDE_SHIFT 13
1041#define MDSCR_KDE (1ULL << MDSCR_KDE_SHIFT)
1042#define MDSCR_TDCC_SHIFT 12
1043#define MDSCR_TDCC (1ULL << MDSCR_TDCC_SHIFT)
1044#define MDSCR_ERR_SHIFT 6
1045#define MDSCR_ERR (1ULL << MDSCR_ERR_SHIFT)
1046#define MDSCR_SS_SHIFT 0
1047#define MDSCR_SS (1ULL << MDSCR_SS_SHIFT)
1048
1049/*
1050 * Translation Table Base Register (TTBR)
1051 *
1052 * 63 48 47 x x-1 0
1053 * +--------+------------------+------+
1054 * | ASID | Base Address | zero |
1055 * +--------+------------------+------+
1056 *
1057 */
1058#define TTBR_ASID_SHIFT 48
1059#define TTBR_ASID_MASK 0xffff000000000000
1060
1061#define TTBR_BADDR_MASK 0x0000ffffffffffff
1062
1063/*
1064 * Memory Attribute Indirection Register
1065 *
1066 * 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0
1067 * +-------+-------+-------+-------+-------+-------+-------+-------+
1068 * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 |
1069 * +-------+-------+-------+-------+-------+-------+-------+-------+
1070 *
1071 */
1072
1073#define MAIR_ATTR_SHIFT(x) (8*(x))
1074
1075/* Strongly ordered or device memory attributes */
1076#define MAIR_OUTER_STRONGLY_ORDERED 0x0
1077#define MAIR_OUTER_DEVICE 0x0
1078
1079#define MAIR_INNER_STRONGLY_ORDERED 0x0
1080#define MAIR_INNER_DEVICE 0x4
1081
1082/* Normal memory attributes */
1083#define MAIR_OUTER_NON_CACHEABLE 0x40
1084#define MAIR_OUTER_WRITE_THROUGH 0x80
1085#define MAIR_OUTER_WRITE_BACK 0xc0
1086
1087#define MAIR_INNER_NON_CACHEABLE 0x4
1088#define MAIR_INNER_WRITE_THROUGH 0x8
1089#define MAIR_INNER_WRITE_BACK 0xc
1090
1091/* Allocate policy for cacheable memory */
1092#define MAIR_OUTER_WRITE_ALLOCATE 0x10
1093#define MAIR_OUTER_READ_ALLOCATE 0x20
1094
1095#define MAIR_INNER_WRITE_ALLOCATE 0x1
1096#define MAIR_INNER_READ_ALLOCATE 0x2
1097
1098/* Memory Atribute Encoding */
1099
1100/*
1101 * Device memory types:
1102 * G (gathering): multiple reads/writes can be combined
1103 * R (reordering): reads or writes may reach device out of program order
1104 * E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes)
1105 */
1106#if HAS_FEAT_XS
1107
1108#define MAIR_DISABLE 0x01 /* Device Memory, nGnRnE (strongly ordered), XS=0 */
1109#define MAIR_POSTED_XS 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes), XS=1 */
1110#define MAIR_POSTED_COMBINED_REORDERED_XS 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=1 */
1111#define MAIR_POSTED_COMBINED_REORDERED 0x0D /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=0 */
1112#define MAIR_WRITECOMB 0x40 /* Normal Memory, Non-Cacheable, XS=0 */
1113#define MAIR_WRITETHRU 0xA0 /* Normal Memory, Write-through, XS=0 */
1114#define MAIR_WRITEBACK 0xFF /* Normal Memory, Write-back, XS=0 */
1115
1116/*
1117 * Memory Attribute Index. If these values change, please also update the pmap
1118 * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64).
1119 */
1120#define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */
1121#define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */
1122#define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */
1123#define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */
1124#define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */
1125#define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */
1126#define CACHE_ATTRINDX_POSTED_XS 0x5 /* no cache, no buffer, posted writes (device memory), XS=1 */
1127#define CACHE_ATTRINDX_POSTED CACHE_ATTRINDX_POSTED_XS /* posted mappings use XS by default */
1128#define CACHE_ATTRINDX_POSTED_REORDERED CACHE_ATTRINDX_DISABLE /* no need for device-nGRE on newer SoCs, fallback to nGnRnE */
1129#define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x6 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=0 */
1130#define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED_XS 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=1 */
1131#define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK
1132#define CACHE_ATTRINDX_N_INDICES (8ULL)
1133
1134#else
1135
1136#define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */
1137#define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */
1138#define MAIR_POSTED_REORDERED 0x08 /* Device Memory, nGRE (reorderable, posted writes) */
1139#define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */
1140#define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */
1141#define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */
1142#define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */
1143
1144/*
1145 * Memory Attribute Index. If these values change, please also update the pmap
1146 * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64).
1147 */
1148#define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */
1149#define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */
1150#define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */
1151#define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */
1152#define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */
1153#define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */
1154#define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes (device memory) */
1155#define CACHE_ATTRINDX_POSTED_REORDERED 0x6 /* no cache, reorderable access, posted writes (device memory) */
1156#define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */
1157#define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK
1158#define CACHE_ATTRINDX_N_INDICES (8ULL)
1159
1160#endif /* HAS_FEAT_XS */
1161
1162#if HAS_UCNORMAL_MEM
1163#define CACHE_ATTRINDX_RT CACHE_ATTRINDX_WRITECOMB
1164#else
1165#define CACHE_ATTRINDX_RT CACHE_ATTRINDX_DISABLE
1166#endif /* HAS_UCNORMAL_MEM */
1167
1168
1169
1170/*
1171 * Access protection bit values (TTEs and PTEs), stage 1
1172 *
1173 * Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access)
1174 */
1175#define AP_RWNA 0x0 /* priv=read-write, user=no-access */
1176#define AP_RWRW 0x1 /* priv=read-write, user=read-write */
1177#define AP_RONA 0x2 /* priv=read-only, user=no-access */
1178#define AP_RORO 0x3 /* priv=read-only, user=read-only */
1179#define AP_MASK 0x3 /* mask to find ap bits */
1180
1181/*
1182 * Shareability attributes
1183 */
1184#define SH_NONE 0x0 /* Non shareable */
1185#define SH_NONE 0x0 /* Device shareable */
1186#define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */
1187#define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */
1188#define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */
1189
1190
1191/*
1192 * ARM Page Granule
1193 */
1194#ifdef __ARM_16K_PG__
1195#define ARM_PGSHIFT 14
1196#else
1197#define ARM_PGSHIFT 12
1198#endif
1199#define ARM_PGBYTES (1 << ARM_PGSHIFT)
1200#define ARM_PGMASK (ARM_PGBYTES-1)
1201
1202/*
1203 * L0 Translation table
1204 *
1205 * 4KB granule size:
1206 * Each translation table is 4KB
1207 * 512 64-bit entries of 512GB (2^39) of address space.
1208 * Covers 256TB (2^48) of address space.
1209 *
1210 * 16KB granule size:
1211 * Each translation table is 16KB
1212 * 2 64-bit entries of 128TB (2^47) of address space.
1213 * Covers 256TB (2^48) of address space.
1214 */
1215
1216/* 16K L0 */
1217#define ARM_16K_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */
1218#define ARM_16K_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */
1219#define ARM_16K_TT_L0_SHIFT 47 /* page descriptor shift */
1220#define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */
1221
1222/* 4K L0 */
1223#define ARM_4K_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */
1224#define ARM_4K_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */
1225#define ARM_4K_TT_L0_SHIFT 39 /* page descriptor shift */
1226#define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */
1227
1228/*
1229 * L1 Translation table
1230 *
1231 * 4KB granule size:
1232 * Each translation table is 4KB
1233 * 512 64-bit entries of 1GB (2^30) of address space.
1234 * Covers 512GB (2^39) of address space.
1235 *
1236 * 16KB granule size:
1237 * Each translation table is 16KB
1238 * 2048 64-bit entries of 64GB (2^36) of address space.
1239 * Covers 128TB (2^47) of address space.
1240 */
1241
1242/* 16K L1 */
1243#define ARM_16K_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */
1244#define ARM_16K_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */
1245#define ARM_16K_TT_L1_SHIFT 36 /* page descriptor shift */
1246#if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__
1247/* This config supports 512GB per TTBR. */
1248#define ARM_16K_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */
1249#else /* __ARM64_PMAP_SUBPAGE_L1__ */
1250#define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */
1251#endif /* __ARM64_PMAP_SUBPAGE_L1__ */
1252
1253/* 4K L1 */
1254#define ARM_4K_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */
1255#define ARM_4K_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */
1256#define ARM_4K_TT_L1_SHIFT 30 /* page descriptor shift */
1257#if __ARM64_PMAP_SUBPAGE_L1__ && !__ARM_16K_PG__
1258/* This config supports 256GB per TTBR. */
1259#define ARM_4K_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */
1260#else /* __ARM64_PMAP_SUBPAGE_L1__ */
1261#define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */
1262#endif /* __ARM64_PMAP_SUBPAGE_L1__ */
1263
1264/* some sugar for getting pointers to page tables and entries */
1265
1266#define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)
1267#define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
1268#define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)
1269
1270#define L2_TABLE_VA(tte) ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK))
1271#define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK))
1272
1273/*
1274 * L2 Translation table
1275 *
1276 * 4KB granule size:
1277 * Each translation table is 4KB
1278 * 512 64-bit entries of 2MB (2^21) of address space.
1279 * Covers 1GB (2^30) of address space.
1280 *
1281 * 16KB granule size:
1282 * Each translation table is 16KB
1283 * 2048 64-bit entries of 32MB (2^25) of address space.
1284 * Covers 64GB (2^36) of address space.
1285 */
1286
1287/* 16K L2 */
1288#define ARM_16K_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */
1289#define ARM_16K_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */
1290#define ARM_16K_TT_L2_SHIFT 25 /* page descriptor shift */
1291#define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */
1292
1293/* 4K L2 */
1294#define ARM_4K_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */
1295#define ARM_4K_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */
1296#define ARM_4K_TT_L2_SHIFT 21 /* page descriptor shift */
1297#define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */
1298
1299/*
1300 * L3 Translation table
1301 *
1302 * 4KB granule size:
1303 * Each translation table is 4KB
1304 * 512 64-bit entries of 4KB (2^12) of address space.
1305 * Covers 2MB (2^21) of address space.
1306 *
1307 * 16KB granule size:
1308 * Each translation table is 16KB
1309 * 2048 64-bit entries of 16KB (2^14) of address space.
1310 * Covers 32MB (2^25) of address space.
1311 */
1312
1313/* 16K L3 */
1314#define ARM_16K_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */
1315#define ARM_16K_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */
1316#define ARM_16K_TT_L3_SHIFT 14 /* page descriptor shift */
1317#define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */
1318
1319/* 4K L3 */
1320#define ARM_4K_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */
1321#define ARM_4K_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */
1322#define ARM_4K_TT_L3_SHIFT 12 /* page descriptor shift */
1323#define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */
1324
1325#ifdef __ARM_16K_PG__
1326
1327/* Native L0 defines */
1328#define ARM_TT_L0_SIZE ARM_16K_TT_L0_SIZE
1329#define ARM_TT_L0_OFFMASK ARM_16K_TT_L0_OFFMASK
1330#define ARM_TT_L0_SHIFT ARM_16K_TT_L0_SHIFT
1331#define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK
1332
1333/* Native L1 defines */
1334#define ARM_TT_L1_SIZE ARM_16K_TT_L1_SIZE
1335#define ARM_TT_L1_OFFMASK ARM_16K_TT_L1_OFFMASK
1336#define ARM_TT_L1_SHIFT ARM_16K_TT_L1_SHIFT
1337#define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK
1338
1339/* Native L2 defines */
1340#define ARM_TT_L2_SIZE ARM_16K_TT_L2_SIZE
1341#define ARM_TT_L2_OFFMASK ARM_16K_TT_L2_OFFMASK
1342#define ARM_TT_L2_SHIFT ARM_16K_TT_L2_SHIFT
1343#define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK
1344
1345/* Native L3 defines */
1346#define ARM_TT_L3_SIZE ARM_16K_TT_L3_SIZE
1347#define ARM_TT_L3_OFFMASK ARM_16K_TT_L3_OFFMASK
1348#define ARM_TT_L3_SHIFT ARM_16K_TT_L3_SHIFT
1349#define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK
1350
1351#else /* !__ARM_16K_PG__ */
1352
1353/* Native L0 defines */
1354#define ARM_TT_L0_SIZE ARM_4K_TT_L0_SIZE
1355#define ARM_TT_L0_OFFMASK ARM_4K_TT_L0_OFFMASK
1356#define ARM_TT_L0_SHIFT ARM_4K_TT_L0_SHIFT
1357#define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK
1358
1359/* Native L1 defines */
1360#define ARM_TT_L1_SIZE ARM_4K_TT_L1_SIZE
1361#define ARM_TT_L1_OFFMASK ARM_4K_TT_L1_OFFMASK
1362#define ARM_TT_L1_SHIFT ARM_4K_TT_L1_SHIFT
1363#define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK
1364
1365/* Native L2 defines */
1366#define ARM_TT_L2_SIZE ARM_4K_TT_L2_SIZE
1367#define ARM_TT_L2_OFFMASK ARM_4K_TT_L2_OFFMASK
1368#define ARM_TT_L2_SHIFT ARM_4K_TT_L2_SHIFT
1369#define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK
1370
1371/* Native L3 defines */
1372#define ARM_TT_L3_SIZE ARM_4K_TT_L3_SIZE
1373#define ARM_TT_L3_OFFMASK ARM_4K_TT_L3_OFFMASK
1374#define ARM_TT_L3_SHIFT ARM_4K_TT_L3_SHIFT
1375#define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK
1376
1377#endif /* !__ARM_16K_PG__ */
1378
1379/*
1380 * Convenience definitions for:
1381 * ARM_TT_LEAF: The last level of the configured page table format.
1382 * ARM_TT_TWIG: The second to last level of the configured page table format.
1383 * ARM_TT_ROOT: The first level of the configured page table format.
1384 *
1385 * My apologies to any botanists who may be reading this.
1386 */
1387#define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE
1388#define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK
1389#define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT
1390#define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK
1391
1392#define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE
1393#define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK
1394#define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT
1395#define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK
1396
1397#define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE
1398#define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK
1399#define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT
1400#define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK
1401
1402/*
1403 * 4KB granule size:
1404 *
1405 * Level 0 Translation Table Entry
1406 *
1407 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
1408 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1409 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V|
1410 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1411 *
1412 * Level 1 Translation Table Entry
1413 *
1414 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
1415 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1416 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V|
1417 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1418 *
1419 * Level 1 Translation Block Entry
1420 *
1421 * 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0
1422 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1423 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
1424 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1425 *
1426 * Level 2 Translation Table Entry
1427 *
1428 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
1429 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1430 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V|
1431 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1432 *
1433 * Level 2 Translation Block Entry
1434 *
1435 * 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0
1436 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1437 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
1438 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1439 *
1440 * 16KB granule size:
1441 *
1442 * Level 0 Translation Table Entry
1443 *
1444 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
1445 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1446 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V|
1447 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1448 *
1449 * Level 1 Translation Table Entry
1450 *
1451 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
1452 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1453 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V|
1454 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1455 *
1456 * Level 2 Translation Table Entry
1457 *
1458 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
1459 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1460 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V|
1461 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
1462 *
1463 * Level 2 Translation Block Entry
1464 *
1465 * 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0
1466 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1467 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
1468 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1469 *
1470 * where:
1471 * nG: notGlobal bit
1472 * SH: Shareability field
1473 * AP: access protection
1474 * XN: eXecute Never bit
1475 * PXN: Privilege eXecute Never bit
1476 * NS: Non-Secure bit
1477 * HINT: 16 entry continuguous output hint
1478 * AttrIdx: Memory Attribute Index
1479 */
1480
1481#define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */
1482#ifdef __ARM_16K_PG__
1483#define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */
1484#else
1485#define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */
1486#endif
1487
1488#define ARM_TTE_MAX (TTE_PGENTRIES)
1489
1490#define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */
1491#define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */
1492
1493#define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */
1494
1495#define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */
1496#define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */
1497#define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */
1498#define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL
1499#define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */
1500
1501#ifdef __ARM_16K_PG__
1502/*
1503 * Note that L0/L1 block entries are disallowed for the 16KB granule size; what
1504 * are we doing with these?
1505 */
1506#define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */
1507#define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */
1508#define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */
1509#define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */
1510#define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */
1511#define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */
1512#else
1513#define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */
1514#define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */
1515#define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */
1516#define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */
1517#define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */
1518#define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */
1519#endif
1520
1521#define ARM_TTE_BLOCK_APSHIFT 6
1522#define ARM_TTE_BLOCK_AP(x) ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */
1523#define ARM_TTE_BLOCK_APMASK (0x3 << ARM_TTE_BLOCK_APSHIFT)
1524
1525#define ARM_TTE_BLOCK_ATTRINDX(x) ((x) << 2) /* memory attributes index */
1526#define ARM_TTE_BLOCK_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */
1527
1528#define ARM_TTE_BLOCK_SH(x) ((x) << 8) /* access shared */
1529#define ARM_TTE_BLOCK_SHMASK (0x3ULL << 8) /* mask access shared */
1530
1531#define ARM_TTE_BLOCK_AF 0x0000000000000400ULL /* value for access */
1532#define ARM_TTE_BLOCK_AFMASK 0x0000000000000400ULL /* access mask */
1533
1534#define ARM_TTE_BLOCK_NG 0x0000000000000800ULL /* value for a global mapping */
1535#define ARM_TTE_BLOCK_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */
1536
1537#define ARM_TTE_BLOCK_NS 0x0000000000000020ULL /* value for a secure mapping */
1538#define ARM_TTE_BLOCK_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */
1539
1540#define ARM_TTE_BLOCK_PNX 0x0020000000000000ULL /* value for privilege no execute bit */
1541#define ARM_TTE_BLOCK_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */
1542
1543#define ARM_TTE_BLOCK_NX 0x0040000000000000ULL /* value for no execute */
1544#define ARM_TTE_BLOCK_NXMASK 0x0040000000000000ULL /* no execute mask */
1545
1546#define ARM_TTE_BLOCK_WIRED 0x0400000000000000ULL /* value for software wired bit */
1547#define ARM_TTE_BLOCK_WIREDMASK 0x0400000000000000ULL /* software wired mask */
1548
1549#define ARM_TTE_BLOCK_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */
1550#define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL /* software writeable mask */
1551
1552#define ARM_TTE_TABLE_MASK 0x0000fffffffff000ULL /* mask for extracting pointer to next table (works at any level) */
1553
1554#define ARM_TTE_TABLE_APSHIFT 61
1555#define ARM_TTE_TABLE_AP_NO_EFFECT 0x0ULL
1556#define ARM_TTE_TABLE_AP_USER_NA 0x1ULL
1557#define ARM_TTE_TABLE_AP_RO 0x2ULL
1558#define ARM_TTE_TABLE_AP_KERN_RO 0x3ULL
1559#define ARM_TTE_TABLE_AP(x) ((x) << ARM_TTE_TABLE_APSHIFT) /* access protection */
1560
1561#define ARM_TTE_TABLE_NS 0x8000000000000020ULL /* value for a secure mapping */
1562#define ARM_TTE_TABLE_NS_MASK 0x8000000000000020ULL /* notSecure mapping mask */
1563
1564#define ARM_TTE_TABLE_XN 0x1000000000000000ULL /* value for no execute */
1565#define ARM_TTE_TABLE_XNMASK 0x1000000000000000ULL /* no execute mask */
1566
1567#define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */
1568#define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */
1569
1570#if __ARM_KERNEL_PROTECT__
1571#define ARM_TTE_BOOT_BLOCK \
1572 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \
1573 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NG)
1574#else /* __ARM_KERNEL_PROTECT__ */
1575#define ARM_TTE_BOOT_BLOCK \
1576 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \
1577 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF)
1578#endif /* __ARM_KERNEL_PROTECT__ */
1579
1580#define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID )
1581/*
1582 * L3 Translation table
1583 *
1584 * 4KB granule size:
1585 * Each translation table is 4KB
1586 * 512 64-bit entries of 4KB (2^12) of address space.
1587 * Covers 2MB (2^21) of address space.
1588 *
1589 * 16KB granule size:
1590 * Each translation table is 16KB
1591 * 2048 64-bit entries of 16KB (2^14) of address space.
1592 * Covers 32MB (2^25) of address space.
1593 */
1594
1595#ifdef __ARM_16K_PG__
1596#define ARM_PTE_SIZE 0x0000000000004000ULL /* size of area covered by a tte */
1597#define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */
1598#define ARM_PTE_SHIFT 14 /* page descriptor shift */
1599#define ARM_PTE_MASK 0x0000ffffffffc000ULL /* mask for output address in PTE */
1600#else
1601#define ARM_PTE_SIZE 0x0000000000001000ULL /* size of area covered by a tte */
1602#define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */
1603#define ARM_PTE_SHIFT 12 /* page descriptor shift */
1604#define ARM_PTE_MASK 0x0000fffffffff000ULL /* mask for output address in PTE */
1605#endif
1606
1607#define ARM_TTE_PA_MASK 0x0000fffffffff000ULL
1608
1609/*
1610 * L3 Page table entries
1611 *
1612 * The following page table entry types are possible:
1613 *
1614 * fault page entry
1615 * 63 2 0
1616 * +------------------------------+--+
1617 * | ignored |00|
1618 * +------------------------------+--+
1619 *
1620 *
1621 * 63 59 58 55 54 53 52 51 48 47 12 11 10 9 8 7 6 5 4 2 1 0
1622 * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1623 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V|
1624 * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1625 *
1626 * where:
1627 * nG: notGlobal bit
1628 * SH: Shareability field
1629 * AP: access protection
1630 * XN: eXecute Never bit
1631 * PXN: Privilege eXecute Never bit
1632 * NS: Non-Secure bit
1633 * HINT: 16 entry continuguous output hint
1634 * AttrIdx: Memory Attribute Index
1635 */
1636
1637#define PTE_SHIFT 3 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */
1638#ifdef __ARM_16K_PG__
1639#define PTE_PGENTRIES (16384 >> PTE_SHIFT) /* number of ptes per page */
1640#else
1641#define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */
1642#endif
1643
1644#define ARM_PTE_EMPTY 0x0000000000000000ULL /* unassigned - invalid entry */
1645
1646/* markers for (invalid) PTE for a page sent to compressor */
1647#define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */
1648#define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */
1649#define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL
1650
1651#define ARM_PTE_TYPE 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1652#define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1653#define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */
1654#define ARM_PTE_TYPE_MASK 0x0000000000000002ULL /* mask to get pte type */
1655
1656/* This mask works for both 16K and 4K pages because bits 12-13 will be zero in 16K pages */
1657#define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* output address mask for page */
1658#define ARM_PTE_PAGE_SHIFT 12 /* page shift for the output address in the entry */
1659
1660#define ARM_PTE_AP(x) ((x) << 6) /* access protections */
1661#define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */
1662#define ARM_PTE_EXTRACT_AP(x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */
1663
1664#define ARM_PTE_ATTRINDX(x) ((x) << 2) /* memory attributes index */
1665#define ARM_PTE_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */
1666#define ARM_PTE_EXTRACT_ATTRINDX(x) (((x) >> 2) & 0x7ULL) /* extract memory attributes index */
1667
1668#define ARM_PTE_SH(x) ((x) << 8) /* access shared */
1669#define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */
1670
1671#define ARM_PTE_AF 0x0000000000000400ULL /* value for access */
1672#define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */
1673
1674#define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */
1675#define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */
1676
1677#define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */
1678#define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */
1679
1680#define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */
1681#define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */
1682
1683#if __ARM_16K_PG__
1684#define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */
1685#define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */
1686#define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */
1687#define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */
1688#define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */
1689#else
1690#define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */
1691#define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */
1692#define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */
1693#define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */
1694#define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */
1695#endif
1696
1697#define ARM_PTE_PNX 0x0020000000000000ULL /* value for privilege no execute bit */
1698#define ARM_PTE_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */
1699
1700#define ARM_PTE_NX 0x0040000000000000ULL /* value for no execute bit */
1701#define ARM_PTE_NXMASK 0x0040000000000000ULL /* no execute mask */
1702
1703#define ARM_PTE_XMASK (ARM_PTE_PNXMASK | ARM_PTE_NXMASK)
1704
1705#define ARM_PTE_WIRED 0x0400000000000000ULL /* value for software wired bit */
1706#define ARM_PTE_WIRED_MASK 0x0400000000000000ULL /* software wired mask */
1707
1708#define ARM_PTE_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */
1709#define ARM_PTE_WRITEABLE_MASK 0x0800000000000000ULL /* software writeable mask */
1710
1711#define ARM_PTE_BOOT_PAGE_BASE \
1712 (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) | \
1713 ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF)
1714
1715#if __ARM_KERNEL_PROTECT__
1716#define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG)
1717#else /* __ARM_KERNEL_PROTECT__ */
1718#define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE)
1719#endif /* __ARM_KERNEL_PROTECT__ */
1720
1721/*
1722 * TLBI appers to only deal in 4KB page addresses, so give
1723 * it an explicit shift of 12.
1724 */
1725#define TLBI_ADDR_SHIFT (0)
1726#define TLBI_ADDR_SIZE (44)
1727#define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1)
1728#define TLBI_ASID_SHIFT (48)
1729#define TLBI_ASID_SIZE (16)
1730#define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1))
1731
1732#define RTLBI_ADDR_SIZE (37)
1733#define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1)
1734#define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT
1735#define RTLBI_TG(_page_shift_) ((uint64_t)((((_page_shift_) - 12) >> 1) + 1) << 46)
1736#define RTLBI_SCALE_SHIFT (44)
1737#define RTLBI_NUM_SHIFT (39)
1738
1739/*
1740 * Exception Syndrome Register
1741 *
1742 * 31 26 25 24 0
1743 * +------+--+------------------+
1744 * | EC |IL| ISS |
1745 * +------+--+------------------+
1746 *
1747 * EC - Exception Class
1748 * IL - Instruction Length
1749 * ISS - Instruction Specific Syndrome
1750 *
1751 * Note: The ISS can have many forms. These are defined separately below.
1752 */
1753
1754#define ESR_EC_SHIFT 26
1755#define ESR_EC_MASK (0x3FULL << ESR_EC_SHIFT)
1756#define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT)
1757
1758#define ESR_IL_SHIFT 25
1759#define ESR_IL (1 << ESR_IL_SHIFT)
1760
1761#define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL))
1762
1763#define ESR_ISS_MASK 0x01FFFFFF
1764#define ESR_ISS(x) (x & ESR_ISS_MASK)
1765
1766#ifdef __ASSEMBLER__
1767/* Define only the classes we need to test in the exception vectors. */
1768#define ESR_EC_UNCATEGORIZED 0x00
1769#define ESR_EC_PAC_FAIL 0x1C
1770#define ESR_EC_IABORT_EL1 0x21
1771#define ESR_EC_DABORT_EL1 0x25
1772#define ESR_EC_SP_ALIGN 0x26
1773#define ESR_EC_BRK_AARCH64 0x3C
1774#else
1775typedef enum {
1776 ESR_EC_UNCATEGORIZED = 0x00,
1777 ESR_EC_WFI_WFE = 0x01,
1778 ESR_EC_MCR_MRC_CP15_TRAP = 0x03,
1779 ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04,
1780 ESR_EC_MCR_MRC_CP14_TRAP = 0x05,
1781 ESR_EC_LDC_STC_CP14_TRAP = 0x06,
1782 ESR_EC_TRAP_SIMD_FP = 0x07,
1783 ESR_EC_PTRAUTH_INSTR_TRAP = 0x09,
1784 ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c,
1785 ESR_EC_ILLEGAL_INSTR_SET = 0x0e,
1786 ESR_EC_SVC_32 = 0x11,
1787 ESR_EC_HVC_32 = 0x12,
1788 ESR_EC_SVC_64 = 0x15,
1789 ESR_EC_HVC_64 = 0x16,
1790 ESR_EC_MSR_TRAP = 0x18,
1791#if __has_feature(ptrauth_calls)
1792 ESR_EC_PAC_FAIL = 0x1C,
1793#endif /* __has_feature(ptrauth_calls) */
1794 ESR_EC_IABORT_EL0 = 0x20,
1795 ESR_EC_IABORT_EL1 = 0x21,
1796 ESR_EC_PC_ALIGN = 0x22,
1797 ESR_EC_DABORT_EL0 = 0x24,
1798 ESR_EC_DABORT_EL1 = 0x25,
1799 ESR_EC_SP_ALIGN = 0x26,
1800 ESR_EC_FLOATING_POINT_32 = 0x28,
1801 ESR_EC_FLOATING_POINT_64 = 0x2C,
1802 ESR_EC_SERROR_INTERRUPT = 0x2F,
1803 ESR_EC_BKPT_REG_MATCH_EL0 = 0x30, // Breakpoint Debug event taken to the EL from a lower EL.
1804 ESR_EC_BKPT_REG_MATCH_EL1 = 0x31, // Breakpoint Debug event taken to the EL from the EL.
1805 ESR_EC_SW_STEP_DEBUG_EL0 = 0x32, // Software Step Debug event taken to the EL from a lower EL.
1806 ESR_EC_SW_STEP_DEBUG_EL1 = 0x33, // Software Step Debug event taken to the EL from the EL.
1807 ESR_EC_WATCHPT_MATCH_EL0 = 0x34, // Watchpoint Debug event taken to the EL from a lower EL.
1808 ESR_EC_WATCHPT_MATCH_EL1 = 0x35, // Watchpoint Debug event taken to the EL from the EL.
1809 ESR_EC_BKPT_AARCH32 = 0x38,
1810 ESR_EC_BRK_AARCH64 = 0x3C,
1811} esr_exception_class_t;
1812
1813typedef enum {
1814 FSC_TRANSLATION_FAULT_L0 = 0x04,
1815 FSC_TRANSLATION_FAULT_L1 = 0x05,
1816 FSC_TRANSLATION_FAULT_L2 = 0x06,
1817 FSC_TRANSLATION_FAULT_L3 = 0x07,
1818 FSC_ACCESS_FLAG_FAULT_L1 = 0x09,
1819 FSC_ACCESS_FLAG_FAULT_L2 = 0x0A,
1820 FSC_ACCESS_FLAG_FAULT_L3 = 0x0B,
1821 FSC_PERMISSION_FAULT_L1 = 0x0D,
1822 FSC_PERMISSION_FAULT_L2 = 0x0E,
1823 FSC_PERMISSION_FAULT_L3 = 0x0F,
1824 FSC_SYNC_EXT_ABORT = 0x10,
1825 FSC_SYNC_EXT_ABORT_TT_L1 = 0x15,
1826 FSC_SYNC_EXT_ABORT_TT_L2 = 0x16,
1827 FSC_SYNC_EXT_ABORT_TT_L3 = 0x17,
1828 FSC_SYNC_PARITY = 0x18,
1829 FSC_ASYNC_PARITY = 0x19,
1830 FSC_SYNC_PARITY_TT_L1 = 0x1D,
1831 FSC_SYNC_PARITY_TT_L2 = 0x1E,
1832 FSC_SYNC_PARITY_TT_L3 = 0x1F,
1833 FSC_ALIGNMENT_FAULT = 0x21,
1834 FSC_DEBUG_FAULT = 0x22,
1835} fault_status_t;
1836#endif /* ASSEMBLER */
1837
1838/*
1839 * HVC event
1840 * 24 16 15 0
1841 * +---------+-----+
1842 * |000000000| IMM |
1843 * +---------+-----+
1844 *
1845 * where:
1846 * IMM: Immediate value
1847 */
1848
1849#define ISS_HVC_IMM_MASK 0xffff
1850#define ISS_HVC_IMM(x) ((x) & ISS_HVC_IMM_MASK)
1851
1852/*
1853 * Software step debug event ISS (EL1)
1854 * 24 23 6 5 0
1855 * +---+-----------------+--+------+
1856 * |ISV|00000000000000000|EX| IFSC |
1857 * +---+-----------------+--+------+
1858 *
1859 * where:
1860 * ISV: Instruction syndrome valid
1861 * EX: Exclusive access
1862 * IFSC: Instruction Fault Status Code
1863 */
1864
1865#define ISS_SSDE_ISV_SHIFT 24
1866#define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT)
1867
1868#define ISS_SSDE_EX_SHIFT 6
1869#define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT)
1870
1871#define ISS_SSDE_FSC_MASK 0x3F
1872#define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK)
1873
1874/*
1875 * Instruction Abort ISS (EL1)
1876 * 24 10 9 5 0
1877 * +--------------+---+--+---+------+
1878 * |00000000000000|FnV|EA|000| IFSC |
1879 * +--------------+---+--+---+------+
1880 *
1881 * where:
1882 * FnV: FAR not Valid
1883 * EA: External Abort type
1884 * IFSC: Instruction Fault Status Code
1885 */
1886
1887#define ISS_IA_FNV_SHIFT 10
1888#define ISS_IA_FNV (0x1 << ISS_IA_FNV_SHIFT)
1889
1890#define ISS_IA_EA_SHIFT 9
1891#define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT)
1892
1893#define ISS_IA_FSC_MASK 0x3F
1894#define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK)
1895
1896
1897/*
1898 * Data Abort ISS (EL1)
1899 *
1900 * 24 10 9 8 7 6 5 0
1901 * +--------------+---+--+--+-----+---+----+
1902 * |00000000000000|FnV|EA|CM|S1PTW|WnR|DFSC|
1903 * +--------------+---+--+--+-----+---+----+
1904 *
1905 * where:
1906 * FnV: FAR not Valid
1907 * EA: External Abort type
1908 * CM: Cache Maintenance operation
1909 * WnR: Write not Read
1910 * S1PTW: Stage 2 exception on Stage 1 page table walk
1911 * DFSC: Data Fault Status Code
1912 */
1913#define ISS_DA_FNV_SHIFT 10
1914#define ISS_DA_FNV (0x1 << ISS_DA_FNV_SHIFT)
1915
1916#define ISS_DA_EA_SHIFT 9
1917#define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT)
1918
1919#define ISS_DA_CM_SHIFT 8
1920#define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT)
1921
1922#define ISS_DA_WNR_SHIFT 6
1923#define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT)
1924
1925#define ISS_DA_S1PTW_SHIFT 7
1926#define ISS_DA_S1PTW (0x1 << ISS_DA_S1PTW_SHIFT)
1927
1928#define ISS_DA_FSC_MASK 0x3F
1929#define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK)
1930
1931/*
1932 * Floating Point Exception ISS (EL1)
1933 *
1934 * 24 23 22 8 7 4 3 2 1 0
1935 * +-+---+---------------+---+--+---+---+---+---+---+
1936 * |0|TFV|000000000000000|IDF|00|IXF|UFF|OFF|DZF|IOF|
1937 * +-+---+---------------+---+--+---+---+---+---+---+
1938 *
1939 * where:
1940 * TFV: Trapped Fault Valid
1941 * IDF: Input Denormal Exception
1942 * IXF: Input Inexact Exception
1943 * UFF: Underflow Exception
1944 * OFF: Overflow Exception
1945 * DZF: Divide by Zero Exception
1946 * IOF: Invalid Operation Exception
1947 */
1948#define ISS_FP_TFV_SHIFT 23
1949#define ISS_FP_TFV (0x1 << ISS_FP_TFV_SHIFT)
1950
1951#define ISS_FP_IDF_SHIFT 7
1952#define ISS_FP_IDF (0x1 << ISS_FP_IDF_SHIFT)
1953
1954#define ISS_FP_IXF_SHIFT 4
1955#define ISS_FP_IXF (0x1 << ISS_FP_IXF_SHIFT)
1956
1957#define ISS_FP_UFF_SHIFT 3
1958#define ISS_FP_UFF (0x1 << ISS_FP_UFF_SHIFT)
1959
1960#define ISS_FP_OFF_SHIFT 2
1961#define ISS_FP_OFF (0x1 << ISS_FP_OFF_SHIFT)
1962
1963#define ISS_FP_DZF_SHIFT 1
1964#define ISS_FP_DZF (0x1 << ISS_FP_DZF_SHIFT)
1965
1966#define ISS_FP_IOF_SHIFT 0
1967#define ISS_FP_IOF (0x1 << ISS_FP_IOF_SHIFT)
1968
1969/*
1970 * Breakpoint Exception ISS (EL1)
1971 * 24 16 0
1972 * +---------+---------+
1973 * |000000000| Comment |
1974 * +---------+---------+
1975 *
1976 * where:
1977 * Comment: Instruction Comment Field Value
1978 */
1979#define ISS_BRK_COMMENT_MASK 0xFFFF
1980#define ISS_BRK_COMMENT(x) (x & ISS_BRK_COMMENT_MASK)
1981
1982
1983/*
1984 * SError Interrupt, IDS=1
1985 * 24 23 0
1986 * +---+------------------------+
1987 * |IDS| IMPLEMENTATION DEFINED |
1988 * +---+------------------------+
1989 *
1990 * where:
1991 * IDS: Implementation-defined syndrome (1)
1992 */
1993
1994#define ISS_SEI_IDS_SHIFT 24
1995#define ISS_SEI_IDS (0x1 << ISS_SEI_IDS_SHIFT)
1996
1997
1998#if HAS_UCNORMAL_MEM
1999#define ISS_UC 0x11
2000#endif /* HAS_UCNORMAL_MEM */
2001
2002
2003
2004
2005/*
2006 * Physical Address Register (EL1)
2007 */
2008#define PAR_F_SHIFT 0
2009#define PAR_F (0x1 << PAR_F_SHIFT)
2010
2011#define PLATFORM_SYSCALL_TRAP_NO 0x80000000
2012
2013#define ARM64_SYSCALL_CODE_REG_NUM (16)
2014
2015#define ARM64_CLINE_SHIFT 6
2016
2017#if defined(APPLE_ARM64_ARCH_FAMILY)
2018#define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */
2019#define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */
2020#endif
2021
2022/*
2023 * Timer definitions.
2024 */
2025#define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */
2026#define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */
2027#define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */
2028#define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */
2029#define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */
2030#define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */
2031#define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to virtual timebase + frequency reg enabled */
2032#define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to physical timebase + frequency reg enabled */
2033
2034#define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */
2035#define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */
2036#define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */
2037
2038#define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS
2039#define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED
2040#define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE
2041
2042#define MIDR_EL1_REV_SHIFT 0
2043#define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT)
2044#define MIDR_EL1_PNUM_SHIFT 4
2045#define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT)
2046#define MIDR_EL1_ARCH_SHIFT 16
2047#define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT)
2048#define MIDR_EL1_VAR_SHIFT 20
2049#define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT)
2050#define MIDR_EL1_IMP_SHIFT 24
2051#define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT)
2052
2053#define MIDR_FIJI (0x002 << MIDR_EL1_PNUM_SHIFT)
2054#define MIDR_CAPRI (0x003 << MIDR_EL1_PNUM_SHIFT)
2055#define MIDR_MAUI (0x004 << MIDR_EL1_PNUM_SHIFT)
2056#define MIDR_ELBA (0x005 << MIDR_EL1_PNUM_SHIFT)
2057#define MIDR_CAYMAN (0x006 << MIDR_EL1_PNUM_SHIFT)
2058#define MIDR_MYST (0x007 << MIDR_EL1_PNUM_SHIFT)
2059#define MIDR_SKYE_MONSOON (0x008 << MIDR_EL1_PNUM_SHIFT)
2060#define MIDR_SKYE_MISTRAL (0x009 << MIDR_EL1_PNUM_SHIFT)
2061#define MIDR_CYPRUS_VORTEX (0x00B << MIDR_EL1_PNUM_SHIFT)
2062#define MIDR_CYPRUS_TEMPEST (0x00C << MIDR_EL1_PNUM_SHIFT)
2063#define MIDR_M9 (0x00F << MIDR_EL1_PNUM_SHIFT)
2064#define MIDR_ARUBA_VORTEX (0x010 << MIDR_EL1_PNUM_SHIFT)
2065#define MIDR_ARUBA_TEMPEST (0x011 << MIDR_EL1_PNUM_SHIFT)
2066
2067#ifdef APPLELIGHTNING
2068#define MIDR_CEBU_LIGHTNING (0x012 << MIDR_EL1_PNUM_SHIFT)
2069#define MIDR_CEBU_THUNDER (0x013 << MIDR_EL1_PNUM_SHIFT)
2070#define MIDR_TURKS (0x026 << MIDR_EL1_PNUM_SHIFT)
2071#endif
2072
2073#ifdef APPLEFIRESTORM
2074#define MIDR_SICILY_ICESTORM (0x020 << MIDR_EL1_PNUM_SHIFT)
2075#define MIDR_SICILY_FIRESTORM (0x021 << MIDR_EL1_PNUM_SHIFT)
2076#define MIDR_TONGA_ICESTORM (0x022 << MIDR_EL1_PNUM_SHIFT)
2077#define MIDR_TONGA_FIRESTORM (0x023 << MIDR_EL1_PNUM_SHIFT)
2078#define MIDR_JADE_CHOP_ICESTORM (0x024 << MIDR_EL1_PNUM_SHIFT)
2079#define MIDR_JADE_CHOP_FIRESTORM (0x025 << MIDR_EL1_PNUM_SHIFT)
2080#define MIDR_JADE_DIE_ICESTORM (0x028 << MIDR_EL1_PNUM_SHIFT)
2081#define MIDR_JADE_DIE_FIRESTORM (0x029 << MIDR_EL1_PNUM_SHIFT)
2082#endif
2083
2084#ifdef APPLEAVALANCHE
2085#define MIDR_ELLIS_BLIZZARD (0x030 << MIDR_EL1_PNUM_SHIFT)
2086#define MIDR_ELLIS_AVALANCHE (0x031 << MIDR_EL1_PNUM_SHIFT)
2087#endif
2088
2089
2090
2091
2092/*
2093 * Apple-ISA-Extensions ID Register.
2094 */
2095#define AIDR_MUL53 (1 << 0)
2096#define AIDR_WKDM (1 << 1)
2097#define AIDR_ARCHRETENTION (1 << 2)
2098
2099
2100
2101/*
2102 * CoreSight debug registers
2103 */
2104#define CORESIGHT_ED 0
2105#define CORESIGHT_CTI 1
2106#define CORESIGHT_PMU 2
2107#define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */
2108
2109#define CORESIGHT_OFFSET(x) ((x) * 0x10000)
2110#define CORESIGHT_REGIONS 4
2111#define CORESIGHT_SIZE 0x1000
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122/*
2123 * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0
2124 *
2125 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0
2126 * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+
2127 * | rndr | tlb | ts | fhm | dp | sm4 | sm3 | sha3 | rdm | res0 | atomic | crc32 | sha2 | sha1 | aes | res0 |
2128 * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+
2129 */
2130
2131#define ID_AA64ISAR0_EL1_TS_OFFSET 52
2132#define ID_AA64ISAR0_EL1_TS_MASK (0xfull << ID_AA64ISAR0_EL1_TS_OFFSET)
2133#define ID_AA64ISAR0_EL1_TS_FLAGM_EN (1ull << ID_AA64ISAR0_EL1_TS_OFFSET)
2134#define ID_AA64ISAR0_EL1_TS_FLAGM2_EN (2ull << ID_AA64ISAR0_EL1_TS_OFFSET)
2135
2136#define ID_AA64ISAR0_EL1_FHM_OFFSET 48
2137#define ID_AA64ISAR0_EL1_FHM_MASK (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET)
2138#define ID_AA64ISAR0_EL1_FHM_8_2 (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET)
2139
2140#define ID_AA64ISAR0_EL1_DP_OFFSET 44
2141#define ID_AA64ISAR0_EL1_DP_MASK (0xfull << ID_AA64ISAR0_EL1_DP_OFFSET)
2142#define ID_AA64ISAR0_EL1_DP_EN (1ull << ID_AA64ISAR0_EL1_DP_OFFSET)
2143
2144#define ID_AA64ISAR0_EL1_SHA3_OFFSET 32
2145#define ID_AA64ISAR0_EL1_SHA3_MASK (0xfull << ID_AA64ISAR0_EL1_SHA3_OFFSET)
2146#define ID_AA64ISAR0_EL1_SHA3_EN (1ull << ID_AA64ISAR0_EL1_SHA3_OFFSET)
2147
2148#define ID_AA64ISAR0_EL1_RDM_OFFSET 28
2149#define ID_AA64ISAR0_EL1_RDM_MASK (0xfull << ID_AA64ISAR0_EL1_RDM_OFFSET)
2150#define ID_AA64ISAR0_EL1_RDM_EN (1ull << ID_AA64ISAR0_EL1_RDM_OFFSET)
2151
2152#define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20
2153#define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
2154#define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
2155
2156#define ID_AA64ISAR0_EL1_CRC32_OFFSET 16
2157#define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
2158#define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
2159
2160#define ID_AA64ISAR0_EL1_SHA2_OFFSET 12
2161#define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
2162#define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
2163#define ID_AA64ISAR0_EL1_SHA2_512_EN (2ull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
2164
2165#define ID_AA64ISAR0_EL1_SHA1_OFFSET 8
2166#define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
2167#define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
2168
2169#define ID_AA64ISAR0_EL1_AES_OFFSET 4
2170#define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET)
2171#define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET)
2172#define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET)
2173
2174/*
2175 * ID_AA64ISAR1_EL1 - AArch64 Instruction Set Attribute Register 1
2176 *
2177 * 63 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0
2178 * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+
2179 * | res0 | i8mm | dgh | bf16 | specres | sb | frintts | gpi | gpa | lrcpc | fcma | jscvt | api | apa | dpb |
2180 * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+
2181 */
2182
2183#define ID_AA64ISAR1_EL1_I8MM_OFFSET 52
2184#define ID_AA64ISAR1_EL1_I8MM_MASK (0xfull << ID_AA64ISAR1_EL1_I8MM_OFFSET)
2185#define ID_AA64ISAR1_EL1_I8MM_EN (1ull << ID_AA64ISAR1_EL1_I8MM_OFFSET)
2186
2187#define ID_AA64ISAR1_EL1_DGH_OFFSET 48
2188#define ID_AA64ISAR1_EL1_DGH_MASK (0xfull << ID_AA64ISAR1_EL1_DGH_OFFSET)
2189
2190#define ID_AA64ISAR1_EL1_BF16_OFFSET 44
2191#define ID_AA64ISAR1_EL1_BF16_MASK (0xfull << ID_AA64ISAR1_EL1_BF16_OFFSET)
2192#define ID_AA64ISAR1_EL1_BF16_EN (1ull << ID_AA64ISAR1_EL1_BF16_OFFSET)
2193
2194#define ID_AA64ISAR1_EL1_SPECRES_OFFSET 40
2195#define ID_AA64ISAR1_EL1_SPECRES_MASK (0xfull << ID_AA64ISAR1_EL1_SPECRES_OFFSET)
2196#define ID_AA64ISAR1_EL1_SPECRES_EN (1ull << ID_AA64ISAR1_EL1_SPECRES_OFFSET)
2197
2198#define ID_AA64ISAR1_EL1_SB_OFFSET 36
2199#define ID_AA64ISAR1_EL1_SB_MASK (0xfull << ID_AA64ISAR1_EL1_SB_OFFSET)
2200#define ID_AA64ISAR1_EL1_SB_EN (1ull << ID_AA64ISAR1_EL1_SB_OFFSET)
2201
2202#define ID_AA64ISAR1_EL1_FRINTTS_OFFSET 32
2203#define ID_AA64ISAR1_EL1_FRINTTS_MASK (0xfull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET)
2204#define ID_AA64ISAR1_EL1_FRINTTS_EN (1ull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET)
2205
2206#define ID_AA64ISAR1_EL1_GPI_OFFSET 28
2207#define ID_AA64ISAR1_EL1_GPI_MASK (0xfull << ID_AA64ISAR1_EL1_GPI_OFFSET)
2208#define ID_AA64ISAR1_EL1_GPI_EN (1ull << ID_AA64ISAR1_EL1_GPI_OFFSET)
2209
2210#define ID_AA64ISAR1_EL1_GPA_OFFSET 24
2211#define ID_AA64ISAR1_EL1_GPA_MASK (0xfull << ID_AA64ISAR1_EL1_GPA_OFFSET)
2212
2213#define ID_AA64ISAR1_EL1_LRCPC_OFFSET 20
2214#define ID_AA64ISAR1_EL1_LRCPC_MASK (0xfull << ID_AA64ISAR1_EL1_LRCPC_OFFSET)
2215#define ID_AA64ISAR1_EL1_LRCPC_EN (1ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET)
2216#define ID_AA64ISAR1_EL1_LRCP2C_EN (2ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET)
2217
2218#define ID_AA64ISAR1_EL1_FCMA_OFFSET 16
2219#define ID_AA64ISAR1_EL1_FCMA_MASK (0xfull << ID_AA64ISAR1_EL1_FCMA_OFFSET)
2220#define ID_AA64ISAR1_EL1_FCMA_EN (1ull << ID_AA64ISAR1_EL1_FCMA_OFFSET)
2221
2222#define ID_AA64ISAR1_EL1_JSCVT_OFFSET 12
2223#define ID_AA64ISAR1_EL1_JSCVT_MASK (0xfull << ID_AA64ISAR1_EL1_JSCVT_OFFSET)
2224#define ID_AA64ISAR1_EL1_JSCVT_EN (1ull << ID_AA64ISAR1_EL1_JSCVT_OFFSET)
2225
2226#define ID_AA64ISAR1_EL1_API_OFFSET 8
2227#define ID_AA64ISAR1_EL1_API_MASK (0xfull << ID_AA64ISAR1_EL1_API_OFFSET)
2228#define ID_AA64ISAR1_EL1_API_PAuth_EN (1ull << ID_AA64ISAR1_EL1_API_OFFSET)
2229#define ID_AA64ISAR1_EL1_API_PAuth2_EN (3ull << ID_AA64ISAR1_EL1_API_OFFSET)
2230#define ID_AA64ISAR1_EL1_API_FPAC_EN (4ull << ID_AA64ISAR1_EL1_API_OFFSET)
2231#define ID_AA64ISAR1_EL1_API_FPACCOMBINE (5ull << ID_AA64ISAR1_EL1_API_OFFSET)
2232
2233#define ID_AA64ISAR1_EL1_APA_OFFSET 4
2234#define ID_AA64ISAR1_EL1_APA_MASK (0xfull << ID_AA64ISAR1_EL1_APA_OFFSET)
2235
2236#define ID_AA64ISAR1_EL1_DPB_OFFSET 0
2237#define ID_AA64ISAR1_EL1_DPB_MASK (0xfull << ID_AA64ISAR1_EL1_DPB_OFFSET)
2238#define ID_AA64ISAR1_EL1_DPB_EN (1ull << ID_AA64ISAR1_EL1_DPB_OFFSET)
2239#define ID_AA64ISAR1_EL1_DPB2_EN (2ull << ID_AA64ISAR1_EL1_DPB_OFFSET)
2240
2241/*
2242 * ID_AA64ISAR2_EL1 - AArch64 Instruction Set Attribute Register 2
2243 *
2244 * 63 8 7 4 3 0
2245 * +------+-------+------+
2246 * | res0 | RPRES | WFxT |
2247 * +------+-------+------+
2248 */
2249
2250#define ID_AA64ISAR2_EL1_RPRES_OFFSET 4
2251#define ID_AA64ISAR2_EL1_RPRES_MASK (0xfull << ID_AA64ISAR2_EL1_RPRES_OFFSET)
2252#define ID_AA64ISAR2_EL1_RPRES_EN (1ull << ID_AA64ISAR2_EL1_RPRES_OFFSET)
2253
2254#define ID_AA64ISAR2_EL1_WFxT_OFFSET 0
2255#define ID_AA64ISAR2_EL1_WFxT_MASK (0xfull << ID_AA64ISAR2_EL1_WFxT_OFFSET)
2256#define ID_AA64ISAR2_EL1_WFxT_EN (1ull << ID_AA64ISAR2_EL1_WFxT_OFFSET)
2257
2258/*
2259 * ID_AA64MMFR0_EL1 - AArch64 Memory Model Feature Register 0
2260 * 63 60 59 56 55 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0
2261 * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+
2262 * | ECV | FGT | RES0 | ExS | TGran4_2 | TGran64_2 | TGran16_2 | TGran4 | TGran64 | TGran16 | BigEndEL0 | SNSMem | BigEnd | ASIDBits | PARange |
2263 * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+
2264 */
2265
2266#define ID_AA64MMFR0_EL1_ECV_OFFSET 60
2267#define ID_AA64MMFR0_EL1_ECV_MASK (0xfull << ID_AA64MMFR0_EL1_ECV_OFFSET)
2268#define ID_AA64MMFR0_EL1_ECV_EN (1ull << ID_AA64MMFR0_EL1_ECV_OFFSET)
2269
2270/*
2271 * ID_AA64MMFR2_EL1 - AArch64 Memory Model Feature Register 2
2272 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 14 8 7 4 3 0
2273 * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+
2274 * | E0PD | EVT | BBM | TTL | RES0 | FWB | IDS | AT | ST | NV | CCIDX | VARANGE | IESB | LSM | UAO | CnP |
2275 * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+
2276 */
2277
2278#define ID_AA64MMFR2_EL1_AT_OFFSET 32
2279#define ID_AA64MMFR2_EL1_AT_MASK (0xfull << ID_AA64MMFR2_EL1_AT_OFFSET)
2280#define ID_AA64MMFR2_EL1_AT_LSE2_EN (1ull << ID_AA64MMFR2_EL1_AT_OFFSET)
2281
2282/*
2283 * ID_AA64PFR0_EL1 - AArch64 Processor Feature Register 0
2284 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0
2285 * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+
2286 * | CSV3 | CSV2 | RES0 | DIT | AMU | MPAM | SEL2 | SVE | RAS | GIC | AdvSIMD | FP | EL3 | EL2 | EL1 | EL0 |
2287 * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+
2288 */
2289
2290#define ID_AA64PFR0_EL1_CSV3_OFFSET 60
2291#define ID_AA64PFR0_EL1_CSV3_MASK (0xfull << ID_AA64PFR0_EL1_CSV3_OFFSET)
2292#define ID_AA64PFR0_EL1_CSV3_EN (1ull << ID_AA64PFR0_EL1_CSV3_OFFSET)
2293
2294#define ID_AA64PFR0_EL1_CSV2_OFFSET 56
2295#define ID_AA64PFR0_EL1_CSV2_MASK (0xfull << ID_AA64PFR0_EL1_CSV2_OFFSET)
2296#define ID_AA64PFR0_EL1_CSV2_EN (1ull << ID_AA64PFR0_EL1_CSV2_OFFSET)
2297
2298#define ID_AA64PFR0_EL1_DIT_OFFSET 48
2299#define ID_AA64PFR0_EL1_DIT_MASK (0xfull << ID_AA64PFR0_EL1_DIT_OFFSET)
2300#define ID_AA64PFR0_EL1_DIT_EN (1ull << ID_AA64PFR0_EL1_DIT_OFFSET)
2301
2302#define ID_AA64PFR0_EL1_AdvSIMD_OFFSET 20
2303#define ID_AA64PFR0_EL1_AdvSIMD_MASK (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET)
2304#define ID_AA64PFR0_EL1_AdvSIMD_HPFPCVT (0x0ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET)
2305#define ID_AA64PFR0_EL1_AdvSIMD_FP16 (0x1ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET)
2306#define ID_AA64PFR0_EL1_AdvSIMD_DIS (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET)
2307
2308/*
2309 * ID_AA64PFR1_EL1 - AArch64 Processor Feature Register 1
2310 * 63 20 19 16 15 12 11 8 7 4 3 0
2311 * +----------------------------------+-----------+----------+-------+------+------+
2312 * | RES0 | MPAM_frac | RAS_frac | MTE | SSBS | BT |
2313 * +----------------------------------+-----------+----------+-------+------+------+
2314 */
2315
2316
2317#define ID_AA64PFR1_EL1_SSBS_OFFSET 4
2318#define ID_AA64PFR1_EL1_SSBS_MASK (0xfull << ID_AA64PFR1_EL1_SSBS_OFFSET)
2319#define ID_AA64PFR1_EL1_SSBS_EN (1ull << ID_AA64PFR1_EL1_SSBS_OFFSET)
2320
2321#define ID_AA64PFR1_EL1_BT_OFFSET 0
2322#define ID_AA64PFR1_EL1_BT_MASK (0xfull << ID_AA64PFR1_EL1_BT_OFFSET)
2323#define ID_AA64PFR1_EL1_BT_EN (1ull << ID_AA64PFR1_EL1_BT_OFFSET)
2324
2325/*
2326 * ID_AA64MMFR1_EL1 - AArch64 Memory Model Feature Register 1
2327 *
2328 * 63 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0
2329 * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+
2330 * | res0 | nTLBPA | AFP | HCX | ETS | TWED | XNX | SpecSEI | PAN | LO | HPDS | VH | VMIDBits | HAFDBS |
2331 * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+
2332 */
2333
2334#define ID_AA64MMFR1_EL1_AFP_OFFSET 44
2335#define ID_AA64MMFR1_EL1_AFP_MASK (0xfull << ID_AA64MMFR1_EL1_AFP_OFFSET)
2336#define ID_AA64MMFR1_EL1_AFP_EN (1ull << ID_AA64MMFR1_EL1_AFP_OFFSET)
2337
2338
2339
2340
2341
2342#define APSTATE_G_SHIFT (0)
2343#define APSTATE_P_SHIFT (1)
2344#define APSTATE_A_SHIFT (2)
2345#define APSTATE_AP_MASK ((1ULL << APSTATE_A_SHIFT) | (1ULL << APSTATE_P_SHIFT))
2346
2347
2348#define ACTLR_EL1_EnTSO (1ULL << 1)
2349#define ACTLR_EL1_EnAPFLG (1ULL << 4)
2350#define ACTLR_EL1_EnAFP (1ULL << 5)
2351#define ACTLR_EL1_EnPRSV (1ULL << 6)
2352
2353
2354#if HAS_USAT_BIT
2355#define ACTLR_EL1_USAT_OFFSET 0
2356#define ACTLR_EL1_USAT_MASK (1ULL << ACTLR_EL1_USAT_OFFSET)
2357#define ACTLR_EL1_USAT ACTLR_EL1_USAT_MASK
2358#endif
2359
2360
2361
2362
2363
2364
2365#if defined(HAS_APPLE_PAC)
2366// The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code
2367#define PAC_DISCRIMINATOR_RECOVER 0x1e02
2368#endif
2369
2370
2371#define CTR_EL0_L1Ip_OFFSET 14
2372#define CTR_EL0_L1Ip_VIPT (2ULL << CTR_EL0_L1Ip_OFFSET)
2373#define CTR_EL0_L1Ip_PIPT (3ULL << CTR_EL0_L1Ip_OFFSET)
2374#define CTR_EL0_L1Ip_MASK (3ULL << CTR_EL0_L1Ip_OFFSET)
2375
2376
2377#ifdef __ASSEMBLER__
2378
2379/*
2380 * Conditionally write to system/special-purpose register.
2381 * The register is written to only when the first two arguments
2382 * do not match. If they do match, the macro jumps to a
2383 * caller-provided label.
2384 * The _ISB variant also conditionally issues an ISB after the MSR.
2385 *
2386 * $0 - System/special-purpose register to modify
2387 * $1 - Register containing current FPCR value
2388 * $2 - Register containing expected value
2389 * $3 - Label to jump to when register is already set to expected value
2390 */
2391.macro CMSR
2392cmp $1, $2
2393
2394/* Skip expensive MSR if not required */
2395b.eq $3f
2396msr $0, $2
2397.endmacro
2398
2399.macro CMSR_ISB
2400CMSR $0, $1, $2, $3
2401isb sy
2402.endmacro
2403
2404/*
2405 * Modify FPCR only if it does not contain the XNU default value.
2406 * $0 - Register containing current FPCR value
2407 * $1 - Scratch register
2408 * $2 - Label to jump to when FPCR is already set to default value
2409 */
2410.macro SANITIZE_FPCR
2411mov $1, #FPCR_DEFAULT
2412CMSR FPCR, $0, $1, $2
2413.endmacro
2414
2415/*
2416 * Family of macros that can be used to protect code sections such that they
2417 * are only executed on a particular SoC/Revision/CPU, and skipped otherwise.
2418 * All macros will forward-jump to 1f when the condition is not matched.
2419 * This label may be defined manually, or implicitly through the use of
2420 * the EXEC_END macro.
2421 * For cores, XX can be: EQ (equal), ALL (don't care).
2422 * For revisions, XX can be: EQ (equal), LO (lower than), HS (higher or same), ALL (don't care).
2423 */
2424
2425/*
2426 * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX
2427 * $1 - CPU_VERSION_XX, e.g. CPU_VERSION_B1
2428 * $2 - GPR containing MIDR_EL1 value
2429 * $3 - Scratch register
2430 */
2431.macro EXEC_COREEQ_REVEQ
2432and $3, $2, #MIDR_EL1_PNUM_MASK
2433cmp $3, $0
2434b.ne 1f
2435
2436mov $3, $2
2437bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4
2438ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8
2439cmp $3, $1
2440b.ne 1f
2441.endmacro
2442
2443.macro EXEC_COREEQ_REVLO
2444and $3, $2, #MIDR_EL1_PNUM_MASK
2445cmp $3, $0
2446b.ne 1f
2447
2448mov $3, $2
2449bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4
2450ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8
2451cmp $3, $1
2452b.pl 1f
2453.endmacro
2454
2455.macro EXEC_COREEQ_REVHS
2456and $3, $2, #MIDR_EL1_PNUM_MASK
2457cmp $3, $0
2458b.ne 1f
2459
2460mov $3, $2
2461bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4
2462ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8
2463cmp $3, $1
2464b.mi 1f
2465.endmacro
2466
2467/*
2468 * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1
2469 * $1 - GPR containing MIDR_EL1 value
2470 * $2 - Scratch register
2471 */
2472.macro EXEC_COREALL_REVEQ
2473mov $2, $1
2474bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2475ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2476cmp $2, $0
2477b.ne 1f
2478.endmacro
2479
2480.macro EXEC_COREALL_REVLO
2481mov $2, $1
2482bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2483ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2484cmp $2, $0
2485b.pl 1f
2486.endmacro
2487
2488.macro EXEC_COREALL_REVHS
2489mov $2, $1
2490bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2491ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2492cmp $2, $0
2493b.mi 1f
2494.endmacro
2495
2496.macro CMP_FOREACH reg, cc, label, car, cdr:vararg
2497 cmp \reg, \car
2498 b.\cc \label
2499.ifnb \cdr
2500 CMP_FOREACH \reg, \cc, \label, \cdr
2501.endif
2502.endm
2503
2504.macro EXEC_COREIN_REVALL midr_el1, scratch, midr_list:vararg
2505and \scratch, \midr_el1, #MIDR_EL1_PNUM_MASK
2506 CMP_FOREACH \scratch, eq, Lmatch\@, \midr_list
2507 b 1f
2508Lmatch\@:
2509.endm
2510
2511/*
2512 * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX
2513 * $1 - GPR containing MIDR_EL1 value
2514 * $2 - Scratch register
2515 */
2516.macro EXEC_COREEQ_REVALL
2517and $2, $1, #MIDR_EL1_PNUM_MASK
2518cmp $2, $0
2519 b.ne 1f
2520.endmacro
2521
2522/*
2523 * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1
2524 * $1 - GPR containing MIDR_EL1 value
2525 * $2 - Scratch register
2526 */
2527.macro EXEC_PCORE_REVEQ
2528mrs $2, MPIDR_EL1
2529and $2, $2, #(MPIDR_PNE)
2530cmp $2, xzr
2531b.eq 1f
2532
2533mov $2, $1
2534bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2535ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2536cmp $2, $0
2537b.ne 1f
2538.endmacro
2539
2540.macro EXEC_PCORE_REVLO
2541mrs $2, MPIDR_EL1
2542and $2, $2, #(MPIDR_PNE)
2543cmp $2, xzr
2544b.eq 1f
2545
2546mov $2, $1
2547bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2548ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2549cmp $2, $0
2550b.pl 1f
2551.endmacro
2552
2553.macro EXEC_PCORE_REVHS
2554mrs $2, MPIDR_EL1
2555and $2, $2, #(MPIDR_PNE)
2556cmp $2, xzr
2557b.eq 1f
2558
2559mov $2, $1
2560bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2561ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2562cmp $2, $0
2563b.mi 1f
2564.endmacro
2565
2566.macro EXEC_ECORE_REVEQ
2567mrs $2, MPIDR_EL1
2568and $2, $2, #(MPIDR_PNE)
2569cmp $2, xzr
2570b.ne 1f
2571
2572mov $2, $1
2573bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2574ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2575cmp $2, $0
2576b.ne 1f
2577.endmacro
2578
2579.macro EXEC_ECORE_REVLO
2580mrs $2, MPIDR_EL1
2581and $2, $2, #(MPIDR_PNE)
2582cmp $2, xzr
2583b.ne 1f
2584
2585mov $2, $1
2586bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2587ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2588cmp $2, $0
2589b.pl 1f
2590.endmacro
2591
2592.macro EXEC_ECORE_REVHS
2593mrs $2, MPIDR_EL1
2594and $2, $2, #(MPIDR_PNE)
2595cmp $2, xzr
2596b.ne 1f
2597
2598mov $2, $1
2599bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4
2600ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8
2601cmp $2, $0
2602b.mi 1f
2603.endmacro
2604
2605/*
2606 * $0 - GPR containing MIDR_EL1 value
2607 * $1 - Scratch register
2608 */
2609.macro EXEC_PCORE_REVALL
2610mrs $1, MPIDR_EL1
2611and $1, $1, #(MPIDR_PNE)
2612cmp $1, xzr
2613b.eq 1f
2614.endmacro
2615
2616.macro EXEC_ECORE_REVALL
2617mrs $1, MPIDR_EL1
2618and $1, $1, #(MPIDR_PNE)
2619cmp $1, xzr
2620b.ne 1f
2621.endmacro
2622
2623
2624
2625/*
2626 * Macro that defines the label that all EXEC_COREXX_REVXX macros jump to.
2627 */
2628.macro EXEC_END
26291:
2630.endmacro
2631
2632/*
2633 * Wedges CPUs with a specified core that are below a specified revision. This
2634 * macro is intended for CPUs that have been deprecated in iBoot and may have
2635 * incorrect behavior if they continue running xnu.
2636 */
2637.macro DEPRECATE_COREEQ_REVLO core, rev, midr_el1, scratch
2638EXEC_COREEQ_REVLO \core, \rev, \midr_el1, \scratch
2639/* BEGIN IGNORE CODESTYLE */
2640b .
2641/* END IGNORE CODESTYLE */
2642EXEC_END
2643.endmacro
2644
2645/*
2646 * Sets bits in an SPR register.
2647 * arg0: Name of the register to be accessed.
2648 * arg1: Mask of bits to be set.
2649 * arg2: Scratch register
2650 */
2651.macro HID_SET_BITS
2652mrs $2, $0
2653orr $2, $2, $1
2654msr $0, $2
2655.endmacro
2656
2657/*
2658 * Clears bits in an SPR register.
2659 * arg0: Name of the register to be accessed.
2660 * arg1: Mask of bits to be cleared.
2661 * arg2: Scratch register
2662 */
2663.macro HID_CLEAR_BITS
2664mrs $2, $0
2665bic $2, $2, $1
2666msr $0, $2
2667.endmacro
2668
2669/*
2670 * Combines the functionality of HID_CLEAR_BITS followed by HID_SET_BITS into
2671 * a single read-modify-write sequence.
2672 * arg0: Name of the register to be accessed.
2673 * arg1: Mask of bits to be cleared.
2674 * arg2: Value to insert
2675 * arg3: Scratch register
2676 */
2677.macro HID_INSERT_BITS
2678mrs $3, $0
2679bic $3, $3, $1
2680orr $3, $3, $2
2681msr $0, $3
2682.endmacro
2683
2684/*
2685 * Replaces the value of a field in an implementation-defined system register.
2686 * sreg: system register name
2687 * field: field name within the sysreg, where the assembler symbols
2688 * ARM64_REG_<field>_{shift,width} specify the bounds of the field
2689 * (note that preprocessor macros will not work here)
2690 * value: the value to insert
2691 * scr{1,2}: scratch regs
2692 */
2693.macro HID_WRITE_FIELD sreg, field, val, scr1, scr2
2694mrs \scr1, \sreg
2695mov \scr2, \val
2696bfi \scr1, \scr2, ARM64_REG_\sreg\()_\field\()_shift, ARM64_REG_\sreg\()_\field\()_width
2697msr \sreg, \scr1
2698.endmacro
2699
2700/*
2701 * Macro intended to be used as a replacement for ERET.
2702 * It prevents speculation past ERET instructions by padding
2703 * up to the decoder width.
2704 */
2705.macro ERET_CONTEXT_SYNCHRONIZING
2706eret
2707#if __ARM_SB_AVAILABLE__
2708sb // Technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures
2709#else /* __ARM_SB_AVAILABLE__ */
2710isb // ISB technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures
2711nop // Sequence of six NOPs to pad out and terminate instruction decode group */
2712nop
2713nop
2714nop
2715nop
2716nop
2717#endif /* !__ARM_SB_AVAILABLE__ */
2718.endmacro
2719
2720
2721#endif /* __ASSEMBLER__ */
2722
2723#define MSR(reg, src) __asm__ volatile ("msr " reg ", %0" :: "r" (src))
2724#define MRS(dest, reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest))
2725
2726#if XNU_MONITOR
2727#define __ARM_PTE_PHYSMAP__ 1
2728#define PPL_STATE_KERNEL 0
2729#define PPL_STATE_DISPATCH 1
2730#define PPL_STATE_PANIC 2
2731#define PPL_STATE_EXCEPTION 3
2732#endif
2733
2734
2735#endif /* _ARM64_PROC_REG_H_ */
2736