proc_reg.h source code [xnu-build/EXPORT_HDRS/osfmk/arm64/proc_reg.h]

1	/*
2	* Copyright (c) 2007-2023 Apple Inc. All rights reserved.
3	*
4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5	*
6	* This file contains Original Code and/or Modifications of Original Code
7	* as defined in and that are subject to the Apple Public Source License
8	* Version 2.0 (the 'License'). You may not use this file except in
9	* compliance with the License. The rights granted to you under the License
10	* may not be used to create, or enable the creation or redistribution of,
11	* unlawful or unlicensed copies of an Apple operating system, or to
12	* circumvent, violate, or enable the circumvention or violation of, any
13	* terms of an Apple operating system software license agreement.
14	*
15	* Please obtain a copy of the License at
16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
17	*
18	* The Original Code and all software distributed under the License are
19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23	* Please see the License for the specific language governing rights and
24	* limitations under the License.
25	*
26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27	*/
28	/*
29	* @OSF_COPYRIGHT@
30	*/
31	/ CMU_ENDHIST /
32	/*
33	* Mach Operating System
34	* Copyright (c) 1991,1990 Carnegie Mellon University
35	* All Rights Reserved.
36	*
37	* Permission to use, copy, modify and distribute this software and its
38	* documentation is hereby granted, provided that both the copyright
39	* notice and this permission notice appear in all copies of the
40	* software, derivative works or modified versions, and any portions
41	* thereof, and that both notices appear in supporting documentation.
42	*
43	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
44	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
45	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
46	*
47	* Carnegie Mellon requests users of this software to return to
48	*
49	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
50	* School of Computer Science
51	* Carnegie Mellon University
52	* Pittsburgh PA 15213-3890
53	*
54	* any improvements or extensions that they make and grant Carnegie Mellon
55	* the rights to redistribute these changes.
56	*/
57
58	/*
59	* Processor registers for ARM/ARM64
60	*/
61	#ifndef _ARM64_PROC_REG_H_
62	#define _ARM64_PROC_REG_H_
63
64	#if defined (__arm64__)
65	#include <pexpert/arm64/board_config.h>
66	#elif defined (__arm__)
67	#include <pexpert/arm/board_config.h>
68	#endif
69
70	/*
71	* Processor registers for ARM
72	*/
73	#if __ARM_42BIT_PA_SPACE__
74	/ For now, force the issue! /
75	/ We need more VA space for the identity map to bootstrap the MMU /
76	#undef __ARM64_PMAP_SUBPAGE_L1__
77	#endif /* __ARM_42BIT_PA_SPACE__ */
78
79	/ For arm platforms, create one pset per cluster /
80	#define MAX_PSETS MAX_CPU_CLUSTERS
81
82
83	/ Thread groups are enabled on all ARM platforms (irrespective of scheduler) /
84	#define CONFIG_THREAD_GROUPS 1
85
86	#ifdef XNU_KERNEL_PRIVATE
87
88	#if __ARM_VFP__
89	#define ARM_VFP_DEBUG 0
90	#endif /* __ARM_VFP__ */
91
92	#endif /* XNU_KERNEL_PRIVATE */
93
94	/*
95	* FSR registers
96	*
97	* CPSR: Current Program Status Register
98	* SPSR: Saved Program Status Registers
99	*
100	* 31 30 29 28 27 24 19 16 9 8 7 6 5 4 0
101	* +-----------------------------------------------------------+
102	* \| N\| Z\| C\| V\| Q\|...\| J\|...\|GE[3:0]\|...\| E\| A\| I\| F\| T\| MODE \|
103	* +-----------------------------------------------------------+
104	*/
105
106	/*
107	* Flags
108	*/
109	#define PSR_NF 0x80000000 /* Negative/Less than */
110	#define PSR_ZF 0x40000000 /* Zero */
111	#define PSR_CF 0x20000000 /* Carry/Borrow/Extend */
112	#define PSR_VF 0x10000000 /* Overflow */
113
114	/*
115	* Modified execution mode flags
116	*/
117	#define PSR_TF 0x00000020 /* thumb flag (BX ARMv4T) */
118
119	/*
120	* CPU mode
121	*/
122	#define PSR_USER_MODE 0x00000010 /* User mode */
123
124	#define PSR_MODE_MASK 0x0000001F
125	#define PSR_IS_KERNEL(psr) (((psr) & PSR_MODE_MASK) != PSR_USER_MODE)
126	#define PSR_IS_USER(psr) (((psr) & PSR_MODE_MASK) == PSR_USER_MODE)
127
128	#define PSR_USERDFLT PSR_USER_MODE
129
130	/*
131	* Cache configuration
132	*/
133
134	#if defined (APPLETYPHOON)
135
136	/ I-Cache /
137	#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
138
139	/ D-Cache /
140	#define MMU_CLINE 6 /* cache line size as 1<<MMU_CLINE (64) */
141
142	#elif defined (APPLETWISTER)
143
144	/ I-Cache /
145	#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
146
147	/ D-Cache /
148	#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
149
150	#elif defined (APPLEHURRICANE)
151
152	/ I-Cache /
153	#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
154
155	/ D-Cache /
156	#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
157
158	#elif defined (APPLEMONSOON)
159
160	/ I-Cache, 96KB for Monsoon, 48KB for Mistral, 6-way. /
161	#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
162
163	/ D-Cache, 64KB for Monsoon, 32KB for Mistral, 4-way. /
164	#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
165
166	#elif defined (APPLEVORTEX)
167
168	/ I-Cache, 128KB 8-way for Vortex, 48KB 6-way for Tempest. /
169	#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
170
171	/ D-Cache, 128KB 8-way for Vortex, 32KB 4-way for Tempest. /
172	#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
173
174	#elif defined (APPLELIGHTNING)
175
176	/ I-Cache, 192KB for Lightning, 96KB for Thunder, 6-way. /
177	#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
178
179	/ D-Cache, 128KB for Lightning, 8-way. 48KB for Thunder, 6-way. /
180	#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
181
182	#elif defined (APPLEFIRESTORM)
183
184	/ I-Cache, 256KB for Firestorm, 128KB for Icestorm, 6-way. /
185	#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
186
187	/ D-Cache, 160KB for Firestorm, 8-way. 64KB for Icestorm, 6-way. /
188	#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
189
190	#elif defined (APPLEAVALANCHE)
191
192	/ I-Cache, 192KB for Avalanche, 128KB for Blizzard, 6-way. /
193	#define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */
194
195	/ D-Cache, 128KB for Avalanche, 8-way. 64KB for Blizzard, 8-way. /
196	#define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */
197
198	#elif defined (BCM2837) /* Raspberry Pi 3 */
199
200	/ I-Cache. We don't have detailed spec so we just follow the ARM technical reference. /
201	#define MMU_I_CLINE 6
202
203	/ D-Cache. /
204	#define MMU_CLINE 6
205
206	#elif defined (VMAPPLE)
207
208	/ I-Cache. /
209	#define MMU_I_CLINE 6
210
211	/ D-Cache. /
212	#define MMU_CLINE 6
213
214	#else
215	#error processor not supported
216	#endif
217
218	#define MAX_L2_CLINE_BYTES (1 << MAX_L2_CLINE)
219
220	/*
221	* Format of the Debug & Watchpoint Breakpoint Value and Control Registers
222	*/
223	#define ARM_DBG_VR_ADDRESS_MASK 0xFFFFFFFC /* BVR & WVR */
224	#define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */
225
226	#define ARM_DBG_CR_ADDRESS_MASK_MASK 0x1F000000 /* BCR & WCR */
227	#define ARM_DBGBCR_MATCH_MASK (1 << 22) /* BCR only */
228	#define ARM_DBGBCR_TYPE_MASK (1 << 21) /* BCR only */
229	#define ARM_DBGBCR_TYPE_IVA (0 << 21)
230	#define ARM_DBG_CR_LINKED_MASK (1 << 20) /* BCR & WCR */
231	#define ARM_DBG_CR_LINKED_UNLINKED (0 << 20)
232	#define ARM_DBG_CR_SECURITY_STATE_BOTH (0 << 14)
233	#define ARM_DBG_CR_HIGHER_MODE_ENABLE (1 << 13)
234	#define ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK 0x00001FE0 /* WCR only */
235	#define ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK 0x000001E0 /* BCR & WCR */
236	#define ARM_DBGWCR_ACCESS_CONTROL_MASK (3 << 3) /* WCR only */
237	#define ARM_DBG_CR_MODE_CONTROL_PRIVILEGED (1 << 1) /* BCR & WCR */
238	#define ARM_DBG_CR_MODE_CONTROL_USER (2 << 1) /* BCR & WCR */
239	#define ARM_DBG_CR_ENABLE_MASK (1 << 0) /* BCR & WCR */
240	#define ARM_DBG_CR_ENABLE_ENABLE (1 << 0)
241
242	/*
243	* Format of the OS Lock Access (DBGOSLAR) and Lock Access Registers (DBGLAR)
244	*/
245	#define ARM_DBG_LOCK_ACCESS_KEY 0xC5ACCE55
246
247	/ ARM Debug registers of interest /
248	#define ARM_DEBUG_OFFSET_DBGPRCR (0x310)
249	#define ARM_DEBUG_OFFSET_DBGLAR (0xFB0)
250
251	/*
252	* Main ID Register (MIDR)
253	*
254	* 31 24 23 20 19 16 15 4 3 0
255	* +-----+-----+------+------+-----+
256	* \| IMP \| VAR \| ARCH \| PNUM \| REV \|
257	* +-----+-----+------+------+-----+
258	*
259	* where:
260	* IMP: Implementor code
261	* VAR: Variant number
262	* ARCH: Architecture code
263	* PNUM: Primary part number
264	* REV: Minor revision number
265	*/
266	#define MIDR_REV_SHIFT 0
267	#define MIDR_REV_MASK (0xf << MIDR_REV_SHIFT)
268	#define MIDR_VAR_SHIFT 20
269	#define MIDR_VAR_MASK (0xf << MIDR_VAR_SHIFT)
270
271
272	#if __ARM_KERNEL_PROTECT__
273	/*
274	* __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential
275	* architectural or microarchitectural vulnerabilities that could allow cores to
276	* read/access EL1-only mappings while in EL0 mode. This is achieved by
277	* removing as many mappings as possible when the core transitions to EL0 mode
278	* from EL1 mode, and restoring those mappings when the core transitions to EL1
279	* mode from EL0 mode.
280	*
281	* At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is
282	* used to map and unmap the ordinary kernel mappings, by contracting and
283	* expanding translation zone size for TTBR1 when exiting and entering EL1,
284	* respectively:
285	*
286	* Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is
287	* is in EL0.
288	* Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in
289	* EL1.
290	*
291	* T1SZ_USER: T1SZ_BOOT + 1
292	* TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT)
293	* TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER)
294	* TTBR1_EL1_MAX: (2^64) - 1
295	*
296	* When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the
297	* the following TTBR1 layout:
298	*
299	* TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX
300	* +---------------------------------------------------------+
301	* \| Kernel EL0 Mappings \| Kernel EL1 Mappings \|
302	* +---------------------------------------------------------+
303	*
304	* And when in EL0, we program TCR_EL1 to give the following TTBR1 layout:
305	*
306	* TTBR1_EL1_BASE_USER TTBR1_EL1_MAX
307	* +---------------------------------------------------------+
308	* \| Kernel EL0 Mappings \|
309	* +---------------------------------------------------------+
310	*
311	* With the current implementation, both the EL0 and EL1 mappings for the kernel
312	* use otherwise empty translation tables for mapping the exception vectors (so
313	* that we do not need to TLB flush the exception vector address when switching
314	* between EL0 and EL1). The rationale here is that the TLBI would require a
315	* DSB, and DSBs can be extremely expensive.
316	*
317	* Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n \| 1) as an EL1
318	* ASID. The core switches between ASIDs on EL transitions, so that the TLB
319	* does not need to be fully invalidated on an EL transition.
320	*
321	* Most kernel mappings will be marked non-global in this configuration, as
322	* global mappings would be visible to userspace unless we invalidate them on
323	* eret.
324	*/
325	#if XNU_MONITOR
326	/*
327	* Please note that because we indirect through the thread register in order to
328	* locate the kernel, and because we unmap most of the kernel, the security
329	* model of the PPL is undermined by __ARM_KERNEL_PROTECT__, as we rely on
330	* kernel controlled data to direct codeflow in the exception vectors.
331	*
332	* If we want to ship XNU_MONITOR paired with __ARM_KERNEL_PROTECT__, we will
333	* need to find a performant solution to this problem.
334	*/
335	#endif
336	#endif /* __ARM_KERNEL_PROTECT */
337
338	#if ARM_PARAMETERIZED_PMAP
339	/*
340	* ARM_PARAMETERIZED_PMAP configures the kernel to get the characteristics of
341	* the page tables (number of levels, size of the root allocation) from the
342	* pmap data structure, rather than treating them as compile-time constants.
343	* This allows the pmap code to dynamically adjust how it deals with page
344	* tables.
345	*/
346	#endif /* ARM_PARAMETERIZED_PMAP */
347
348	#if __ARM_MIXED_PAGE_SIZE__
349	/*
350	* __ARM_MIXED_PAGE_SIZE__ configures the kernel to support page tables that do
351	* not use the kernel page size. This is primarily meant to support running
352	* 4KB page processes on a 16KB page kernel.
353	*
354	* This only covers support in the pmap/machine dependent layers. Any support
355	* elsewhere in the kernel must be managed separately.
356	*/
357	#if !ARM_PARAMETERIZED_PMAP
358	/*
359	* Page tables that use non-kernel page sizes require us to reprogram TCR based
360	* on the page tables we are switching to. This means that the parameterized
361	* pmap support is required.
362	*/
363	#error __ARM_MIXED_PAGE_SIZE__ requires ARM_PARAMETERIZED_PMAP
364	#endif /* !ARM_PARAMETERIZED_PMAP */
365	#if __ARM_KERNEL_PROTECT__
366	/*
367	* Because switching the page size requires updating TCR based on the pmap, and
368	* __ARM_KERNEL_PROTECT__ relies on TCR being programmed with constants, XNU
369	* does not currently support support configurations that use both
370	* __ARM_KERNEL_PROTECT__ and __ARM_MIXED_PAGE_SIZE__.
371	*/
372	#error __ARM_MIXED_PAGE_SIZE__ and __ARM_KERNEL_PROTECT__ are mutually exclusive
373	#endif /* __ARM_KERNEL_PROTECT__ */
374	#endif /* __ARM_MIXED_PAGE_SIZE__ */
375
376	/*
377	* 64-bit Program Status Register (PSR64)
378	*
379	* 31 27 23 22 21 20 19 10 9 5 4 0
380	* +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+
381	* \|N\|Z\|C\|V\|00000\|PAN\|SS\|IL\|0000000000\|D\|A\|I\|F\|0\| M \|
382	* +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+
383	*
384	* where:
385	* NZCV: Comparison flags
386	* PAN: Privileged Access Never
387	* SS: Single step
388	* IL: Illegal state
389	* DAIF: Interrupt masks
390	* M: Mode field
391	*/
392
393	#define PSR64_NZCV_SHIFT 28
394	#define PSR64_NZCV_MASK (0xF << PSR64_NZCV_SHIFT)
395
396	#define PSR64_N_SHIFT 31
397	#define PSR64_N (1 << PSR64_N_SHIFT)
398
399	#define PSR64_Z_SHIFT 30
400	#define PSR64_Z (1 << PSR64_Z_SHIFT)
401
402	#define PSR64_C_SHIFT 29
403	#define PSR64_C (1 << PSR64_C_SHIFT)
404
405	#define PSR64_V_SHIFT 28
406	#define PSR64_V (1 << PSR64_V_SHIFT)
407
408	#define PSR64_TCO_SHIFT 25
409	#define PSR64_TCO (1 << PSR64_TCO_SHIFT)
410
411	#define PSR64_DIT_SHIFT 24
412	#define PSR64_DIT (1 << PSR64_DIT_SHIFT)
413
414	#define PSR64_UAO_SHIFT 23
415	#define PSR64_UAO (1 << PSR64_UAO_SHIFT)
416
417	#define PSR64_PAN_SHIFT 22
418	#define PSR64_PAN (1 << PSR64_PAN_SHIFT)
419
420	#define PSR64_SS_SHIFT 21
421	#define PSR64_SS (1 << PSR64_SS_SHIFT)
422
423	#define PSR64_IL_SHIFT 20
424	#define PSR64_IL (1 << PSR64_IL_SHIFT)
425
426	/*
427	* SSBS is bit 12 for A64 SPSR and bit 23 for A32 SPSR
428	* I do not want to talk about it!
429	*/
430	#define PSR64_SSBS_SHIFT_32 23
431	#define PSR64_SSBS_SHIFT_64 12
432	#define PSR64_SSBS_32 (1 << PSR64_SSBS_SHIFT_32)
433	#define PSR64_SSBS_64 (1 << PSR64_SSBS_SHIFT_64)
434
435	/*
436	* msr DAIF, Xn and mrs Xn, DAIF transfer into
437	* and out of bits 9:6
438	*/
439	#define DAIF_DEBUG_SHIFT 9
440	#define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT)
441
442	#define DAIF_ASYNC_SHIFT 8
443	#define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT)
444
445	#define DAIF_IRQF_SHIFT 7
446	#define DAIF_IRQF (1 << DAIF_IRQF_SHIFT)
447
448	#define DAIF_FIQF_SHIFT 6
449	#define DAIF_FIQF (1 << DAIF_FIQF_SHIFT)
450
451	#define DAIF_ALL (DAIF_DEBUGF \| DAIF_ASYNCF \| DAIF_IRQF \| DAIF_FIQF)
452	#define DAIF_STANDARD_DISABLE (DAIF_ASYNCF \| DAIF_IRQF \| DAIF_FIQF)
453
454	#define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF))
455
456	#if HAS_ARM_FEAT_SSBS2
457	#define PSR64_SSBS_U32_DEFAULT PSR64_SSBS_32
458	#define PSR64_SSBS_U64_DEFAULT PSR64_SSBS_64
459	#define PSR64_SSBS_KRN_DEFAULT PSR64_SSBS_64
460	#else
461	#define PSR64_SSBS_U32_DEFAULT (0)
462	#define PSR64_SSBS_U64_DEFAULT (0)
463	#define PSR64_SSBS_KRN_DEFAULT (0)
464	#endif
465
466	/*
467	* msr DAIFSet, Xn, and msr DAIFClr, Xn transfer
468	* from bits 3:0.
469	*/
470	#define DAIFSC_DEBUGF (1 << 3)
471	#define DAIFSC_ASYNCF (1 << 2)
472	#define DAIFSC_IRQF (1 << 1)
473	#define DAIFSC_FIQF (1 << 0)
474	#define DAIFSC_ALL (DAIFSC_DEBUGF \| DAIFSC_ASYNCF \| DAIFSC_IRQF \| DAIFSC_FIQF)
475	#define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF \| DAIFSC_IRQF \| DAIFSC_FIQF)
476	#define DAIFSC_NOASYNC (DAIFSC_DEBUGF \| DAIFSC_IRQF \| DAIFSC_FIQF)
477
478	/*
479	* ARM64_TODO: unify with ARM?
480	*/
481	#define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */
482
483	#define PSR64_MODE_MASK 0x1F
484
485	#define PSR64_USER_MASK PSR64_NZCV_MASK
486
487	#define PSR64_MODE_USER32_THUMB 0x20
488
489	#define PSR64_MODE_RW_SHIFT 4
490	#define PSR64_MODE_RW_64 0
491	#define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT)
492
493	#define PSR64_MODE_EL_SHIFT 2
494	#define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT)
495	#define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT)
496	#define PSR64_MODE_EL2 (0x2 << PSR64_MODE_EL_SHIFT)
497	#define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT)
498	#define PSR64_MODE_EL0 0
499
500	#define PSR64_MODE_EL_KERNEL (PSR64_MODE_EL1)
501
502	#define PSR64_MODE_SPX 0x1
503	#define PSR64_MODE_SP0 0
504
505	#define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 \| PSR64_MODE_EL0 \| PSR64_MODE_SP0 \| PSR64_SSBS_U32_DEFAULT)
506	#define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 \| PSR64_MODE_EL0 \| PSR64_MODE_SP0 \| PSR64_SSBS_U64_DEFAULT)
507	#define PSR64_KERNEL_STANDARD (DAIF_STANDARD_DISABLE \| PSR64_MODE_RW_64 \| PSR64_MODE_EL1 \| PSR64_MODE_SP0 \| PSR64_SSBS_KRN_DEFAULT)
508	#if __ARM_PAN_AVAILABLE__
509	#define PSR64_KERNEL_DEFAULT (PSR64_KERNEL_STANDARD \| PSR64_PAN)
510	#else
511	#define PSR64_KERNEL_DEFAULT PSR64_KERNEL_STANDARD
512	#endif
513
514	#define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0)
515	#define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0)
516
517	#define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32))
518	#define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32))
519
520
521
522	/*
523	* System Control Register (SCTLR)
524	*/
525
526	#define SCTLR_OTHER (0)
527
528	#define SCTLR_DSSBS (1ULL << 44)
529
530
531	#define SCTLR_EXTRA (0)
532
533
534	#define SCTLR_RESERVED ((3ULL << 28) \| (1ULL << 20))
535	#if defined(HAS_APPLE_PAC)
536
537	// 31 PACIA_ENABLED AddPACIA and AuthIA functions enabled
538	#define SCTLR_PACIA_ENABLED_SHIFT 31
539	#define SCTLR_PACIA_ENABLED (1ULL << SCTLR_PACIA_ENABLED_SHIFT)
540	// 30 PACIB_ENABLED AddPACIB and AuthIB functions enabled
541	#define SCTLR_PACIB_ENABLED (1ULL << 30)
542	// 29:28 RES1 11
543	// 27 PACDA_ENABLED AddPACDA and AuthDA functions enabled
544	#define SCTLR_PACDA_ENABLED (1ULL << 27)
545	// 13 PACDB_ENABLED AddPACDB and AuthDB functions enabled
546	#define SCTLR_PACDB_ENABLED (1ULL << 13)
547
548	#define SCTLR_JOP_KEYS_ENABLED (SCTLR_PACIA_ENABLED \| SCTLR_PACDA_ENABLED \| SCTLR_PACDB_ENABLED)
549	#endif /* defined(HAS_APPLE_PAC) */
550
551	// 26 UCI User Cache Instructions
552	#define SCTLR_UCI_ENABLED (1ULL << 26)
553
554	// 25 EE Exception Endianness
555	#define SCTLR_EE_BIG_ENDIAN (1ULL << 25)
556
557	// 24 E0E EL0 Endianness
558	#define SCTLR_E0E_BIG_ENDIAN (1ULL << 24)
559
560	// 23 SPAN Set PAN
561	#define SCTLR_PAN_UNCHANGED (1ULL << 23)
562
563	// 22 EIS Taking an exception is a context synchronization event
564	#define SCTLR_EIS (1ULL << 22)
565
566	// 21 RES0 0
567	// 20 RES1 1
568
569	// 19 WXN Writeable implies eXecute Never
570	#define SCTLR_WXN_ENABLED (1ULL << 19)
571
572	// 18 nTWE Not trap WFE from EL0
573	#define SCTLR_nTWE_WFE_ENABLED (1ULL << 18)
574
575	// 17 RES0 0
576
577	// 16 nTWI Not trap WFI from EL0
578	#define SCTRL_nTWI_WFI_ENABLED (1ULL << 16)
579
580	// 15 UCT User Cache Type register (CTR_EL0)
581	#define SCTLR_UCT_ENABLED (1ULL << 15)
582
583	// 14 DZE User Data Cache Zero (DC ZVA)
584	#define SCTLR_DZE_ENABLED (1ULL << 14)
585
586	// 12 I Instruction cache enable
587	#define SCTLR_I_ENABLED (1ULL << 12)
588
589	// 11 EOS Exception return is a context synchronization event
590	#define SCTLR_EOS (1ULL << 11)
591
592	// 10 EnRCTX EL0 Access to FEAT_SPECRES speculation restriction instructions
593	#define SCTLR_EnRCTX (1ULL << 10)
594
595	// 9 UMA User Mask Access
596	#define SCTLR_UMA_ENABLED (1ULL << 9)
597
598	// 8 SED SETEND Disable
599	#define SCTLR_SED_DISABLED (1ULL << 8)
600
601	// 7 ITD IT Disable
602	#define SCTLR_ITD_DISABLED (1ULL << 7)
603
604	// 6 RES0 0
605
606	// 5 CP15BEN CP15 Barrier ENable
607	#define SCTLR_CP15BEN_ENABLED (1ULL << 5)
608
609	// 4 SA0 Stack Alignment check for EL0
610	#define SCTLR_SA0_ENABLED (1ULL << 4)
611
612	// 3 SA Stack Alignment check
613	#define SCTLR_SA_ENABLED (1ULL << 3)
614
615	// 2 C Cache enable
616	#define SCTLR_C_ENABLED (1ULL << 2)
617
618	// 1 A Alignment check
619	#define SCTLR_A_ENABLED (1ULL << 1)
620
621	// 0 M MMU enable
622	#define SCTLR_M_ENABLED (1ULL << 0)
623
624	#if __ARM_ARCH_8_5__
625	#define SCTLR_CSEH_DEFAULT (0)
626	#define SCTLR_DSSBS_DEFAULT SCTLR_DSSBS
627	#else
628	#define SCTLR_CSEH_DEFAULT (SCTLR_EIS \| SCTLR_EOS)
629	#define SCTLR_DSSBS_DEFAULT (0)
630	#endif
631
632	#if HAS_APPLE_PAC
633	#define SCTLR_ROP_KEYS_DEFAULT SCTLR_PACIB_ENABLED /* IB is ROP */
634	#else /* !HAS_APPLE_PAC */
635	#define SCTLR_ROP_KEYS_DEFAULT 0
636	#endif /* HAS_APPLE_PAC */
637
638	#if HAS_APPLE_PAC
639	#define SCTLR_JOP_KEYS_DEFAULT SCTLR_JOP_KEYS_ENABLED
640	#else /* !HAS_APPLE_PAC */
641	#define SCTLR_JOP_KEYS_DEFAULT 0
642	#endif
643
644	#define SCTLR_EL1_DEFAULT \
645	(SCTLR_RESERVED \| SCTLR_UCI_ENABLED \| SCTLR_nTWE_WFE_ENABLED \| SCTLR_DZE_ENABLED \| \
646	SCTLR_I_ENABLED \| SCTLR_SED_DISABLED \| SCTLR_CP15BEN_ENABLED \| \
647	SCTLR_SA0_ENABLED \| SCTLR_SA_ENABLED \| SCTLR_C_ENABLED \| SCTLR_M_ENABLED \| \
648	SCTLR_CSEH_DEFAULT \| SCTLR_DSSBS_DEFAULT \| SCTLR_ROP_KEYS_DEFAULT \| \
649	SCTLR_JOP_KEYS_DEFAULT \| SCTLR_OTHER \| SCTLR_EXTRA )
650
651	/*
652	* Coprocessor Access Control Register (CPACR)
653	*
654	* 31 28 27 22 21 20 19 0
655	* +---+---+------+------+--------------------+
656	* \|000\|TTA\|000000\| FPEN \|00000000000000000000\|
657	* +---+---+------+------+--------------------+
658	*
659	* where:
660	* TTA: Trace trap
661	* FPEN: Floating point enable
662	*/
663	#define CPACR_TTA_SHIFT 28
664	#define CPACR_TTA (1 << CPACR_TTA_SHIFT)
665
666
667	#define CPACR_FPEN_SHIFT 20
668	#define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT)
669	#define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT)
670
671
672	/*
673	* FPSR: Floating Point Status Register
674	*
675	* 31 30 29 28 27 26 7 6 4 3 2 1 0
676	* +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
677	* \| N\| Z\| C\| V\|QC\|0000000000000000000\|IDC\|00\|IXC\|UFC\|OFC\|DZC\|IOC\|
678	* +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
679	*/
680
681	#define FPSR_N_SHIFT 31
682	#define FPSR_Z_SHIFT 30
683	#define FPSR_C_SHIFT 29
684	#define FPSR_V_SHIFT 28
685	#define FPSR_QC_SHIFT 27
686	#define FPSR_IDC_SHIFT 7
687	#define FPSR_IXC_SHIFT 4
688	#define FPSR_UFC_SHIFT 3
689	#define FPSR_OFC_SHIFT 2
690	#define FPSR_DZC_SHIFT 1
691	#define FPSR_IOC_SHIFT 0
692	#define FPSR_N (1 << FPSR_N_SHIFT)
693	#define FPSR_Z (1 << FPSR_Z_SHIFT)
694	#define FPSR_C (1 << FPSR_C_SHIFT)
695	#define FPSR_V (1 << FPSR_V_SHIFT)
696	#define FPSR_QC (1 << FPSR_QC_SHIFT)
697	#define FPSR_IDC (1 << FPSR_IDC_SHIFT)
698	#define FPSR_IXC (1 << FPSR_IXC_SHIFT)
699	#define FPSR_UFC (1 << FPSR_UFC_SHIFT)
700	#define FPSR_OFC (1 << FPSR_OFC_SHIFT)
701	#define FPSR_DZC (1 << FPSR_DZC_SHIFT)
702	#define FPSR_IOC (1 << FPSR_IOC_SHIFT)
703
704	/*
705	* A mask for all for all of the bits that are not RAZ for FPSR; this
706	* is primarily for converting between a 32-bit view of NEON state
707	* (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
708	*/
709	#define FPSR_MASK \
710	(FPSR_N \| FPSR_Z \| FPSR_C \| FPSR_V \| FPSR_QC \| FPSR_IDC \| FPSR_IXC \| \
711	FPSR_UFC \| FPSR_OFC \| FPSR_DZC \| FPSR_IOC)
712
713	/*
714	* FPCR: Floating Point Control Register
715	*
716	* 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0
717	* +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
718	* \|00000\|AHP\|DN\|FZ\|RMODE\|STRIDE\| 0\|LEN\|IDE\|00\|IXE\|UFE\|OFE\|DZE\|IOE\|00000000\|
719	* +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
720	*/
721
722	#define FPCR_AHP_SHIFT 26
723	#define FPCR_DN_SHIFT 25
724	#define FPCR_FZ_SHIFT 24
725	#define FPCR_RMODE_SHIFT 22
726	#define FPCR_STRIDE_SHIFT 20
727	#define FPCR_LEN_SHIFT 16
728	#define FPCR_IDE_SHIFT 15
729	#define FPCR_IXE_SHIFT 12
730	#define FPCR_UFE_SHIFT 11
731	#define FPCR_OFE_SHIFT 10
732	#define FPCR_DZE_SHIFT 9
733	#define FPCR_IOE_SHIFT 8
734	#define FPCR_AHP (1 << FPCR_AHP_SHIFT)
735	#define FPCR_DN (1 << FPCR_DN_SHIFT)
736	#define FPCR_FZ (1 << FPCR_FZ_SHIFT)
737	#define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT)
738	#define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT)
739	#define FPCR_LEN (0x7 << FPCR_LEN_SHIFT)
740	#define FPCR_IDE (1 << FPCR_IDE_SHIFT)
741	#define FPCR_IXE (1 << FPCR_IXE_SHIFT)
742	#define FPCR_UFE (1 << FPCR_UFE_SHIFT)
743	#define FPCR_OFE (1 << FPCR_OFE_SHIFT)
744	#define FPCR_DZE (1 << FPCR_DZE_SHIFT)
745	#define FPCR_IOE (1 << FPCR_IOE_SHIFT)
746	#define FPCR_DEFAULT (0)
747	#define FPCR_DEFAULT_32 (FPCR_DN\|FPCR_FZ)
748
749	/*
750	* A mask for all for all of the bits that are not RAZ for FPCR; this
751	* is primarily for converting between a 32-bit view of NEON state
752	* (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
753	*/
754	#define FPCR_MASK \
755	(FPCR_AHP \| FPCR_DN \| FPCR_FZ \| FPCR_RMODE \| FPCR_STRIDE \| FPCR_LEN \| \
756	FPCR_IDE \| FPCR_IXE \| FPCR_UFE \| FPCR_OFE \| FPCR_DZE \| FPCR_IOE)
757
758	/*
759	* Translation Control Register (TCR)
760	*
761	* Legacy:
762	*
763	* 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0
764	* +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
765	* \| zero \|TBI1\|TBI0\|AS\|z\| IPS \|z\|TG1\| SH1 \|ORGN1\|IRGN1\|EPD1\|A1\| T1SZ \|z\|TG0\| SH0 \|ORGN0\|IRGN0\|EPD0\|z\|T0SZ\|
766	* +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
767	*
768	* Current (with 16KB granule support):
769	*
770	* 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0
771	* +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
772	* \| zero \|TBI1\|TBI0\|AS\|z\| IPS \| TG1 \| SH1 \|ORGN1\|IRGN1\|EPD1\|A1\| T1SZ \| TG0 \| SH0 \|ORGN0\|IRGN0\|EPD0\|z\|T0SZ\|
773	* +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
774	*
775	* TBI1: Top Byte Ignored for TTBR1 region
776	* TBI0: Top Byte Ignored for TTBR0 region
777	* AS: ASID Size
778	* IPS: Physical Address Size limit
779	* TG1: Granule Size for TTBR1 region
780	* SH1: Shareability for TTBR1 region
781	* ORGN1: Outer Cacheability for TTBR1 region
782	* IRGN1: Inner Cacheability for TTBR1 region
783	* EPD1: Translation table walk disable for TTBR1
784	* A1: ASID selection from TTBR1 enable
785	* T1SZ: Virtual address size for TTBR1
786	* TG0: Granule Size for TTBR0 region
787	* SH0: Shareability for TTBR0 region
788	* ORGN0: Outer Cacheability for TTBR0 region
789	* IRGN0: Inner Cacheability for TTBR0 region
790	* T0SZ: Virtual address size for TTBR0
791	*/
792
793	#define TCR_T0SZ_SHIFT 0ULL
794	#define TCR_TSZ_BITS 6ULL
795	#define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL)
796
797	#define TCR_IRGN0_SHIFT 8ULL
798	#define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT)
799	#define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT)
800	#define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT)
801	#define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT)
802
803	#define TCR_ORGN0_SHIFT 10ULL
804	#define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT)
805	#define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT)
806	#define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT)
807	#define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT)
808
809	#define TCR_SH0_SHIFT 12ULL
810	#define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT)
811	#define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT)
812	#define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT)
813
814	#define TCR_TG0_GRANULE_SHIFT (14ULL)
815	#define TCR_TG0_GRANULE_BITS (2ULL)
816	#define TCR_TG0_GRANULE_MASK ((1ULL << TCR_TG0_GRANULE_BITS) - 1ULL)
817
818	#define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT)
819	#define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT)
820	#define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT)
821
822	#if __ARM_16K_PG__
823	#define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB)
824	#else
825	#define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB)
826	#endif
827
828	#define TCR_T1SZ_SHIFT 16ULL
829
830	#define TCR_A1_ASID1 (1ULL << 22ULL)
831	#define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL)
832
833	#define TCR_IRGN1_SHIFT 24ULL
834	#define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT)
835	#define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT)
836	#define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT)
837	#define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT)
838
839	#define TCR_ORGN1_SHIFT 26ULL
840	#define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT)
841	#define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT)
842	#define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT)
843	#define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT)
844
845	#define TCR_SH1_SHIFT 28ULL
846	#define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT)
847	#define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT)
848	#define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT)
849
850	#define TCR_TG1_GRANULE_SHIFT 30ULL
851	#define TCR_TG1_GRANULE_BITS (2ULL)
852	#define TCR_TG1_GRANULE_MASK ((1ULL << TCR_TG1_GRANULE_BITS) - 1ULL)
853
854	#define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT)
855	#define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT)
856	#define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT)
857
858	#if __ARM_16K_PG__
859	#define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB)
860	#else
861	#define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB)
862	#endif
863
864	#define TCR_IPS_SHIFT 32ULL
865	#define TCR_IPS_BITS 3ULL
866	#define TCR_IPS_MASK ((1ULL << TCR_IPS_BITS) - 1ULL)
867	#define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT)
868	#define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT)
869	#define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT)
870	#define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT)
871	#define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT)
872	#define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT)
873
874	#define TCR_AS_16BIT_ASID (1ULL << 36)
875	#define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37)
876	#define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38)
877	#define TCR_TBID0_TBI_DATA_ONLY (1ULL << 51)
878	#define TCR_TBID1_TBI_DATA_ONLY (1ULL << 52)
879
880	#if defined(HAS_APPLE_PAC)
881	#define TCR_TBID0_ENABLE TCR_TBID0_TBI_DATA_ONLY
882	#define TCR_TBID1_ENABLE TCR_TBID1_TBI_DATA_ONLY
883	#else
884	#define TCR_TBID0_ENABLE 0
885	#define TCR_TBID1_ENABLE 0
886	#endif
887
888	#define TCR_E0PD0_BIT (1ULL << 55)
889	#define TCR_E0PD1_BIT (1ULL << 56)
890
891	#if defined(HAS_E0PD)
892	#define TCR_E0PD_VALUE (TCR_E0PD1_BIT)
893	#else
894	#define TCR_E0PD_VALUE 0
895	#endif
896
897
898	#define TCR_EL1_EXTRA 0
899
900
901	/*
902	* Multiprocessor Affinity Register (MPIDR_EL1)
903	*
904	* +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+
905	* \|000000000000000000000000000000001\| U\|00000\|MT\| Aff2\|Aff1\|Aff0\|
906	* +---------------------------------+--+-----+--+-----+----+----+
907	*
908	* where
909	* U: Uniprocessor
910	* MT: Multi-threading at lowest affinity level
911	* Aff2: "1" - PCORE, "0" - ECORE
912	* Aff1: Cluster ID
913	* Aff0: CPU ID
914	*/
915	#define MPIDR_AFF0_SHIFT 0
916	#define MPIDR_AFF0_WIDTH 8
917	#define MPIDR_AFF0_MASK (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT)
918	#define MPIDR_AFF1_SHIFT 8
919	#define MPIDR_AFF1_WIDTH 8
920	#define MPIDR_AFF1_MASK (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT)
921	#define MPIDR_AFF2_SHIFT 16
922	#define MPIDR_AFF2_WIDTH 8
923	#define MPIDR_AFF2_MASK (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT)
924
925	/*
926	* TXSZ indicates the size of the range a TTBR covers. Currently,
927	* we support the following:
928	*
929	* 4KB pages, full page L1: 39 bit range.
930	* 4KB pages, sub-page L1: 38 bit range.
931	* 16KB pages, full page L1: 47 bit range.
932	* 16KB pages, sub-page L1: 39 bit range.
933	* 16KB pages, two level page tables: 36 bit range.
934	*/
935	#if __ARM_KERNEL_PROTECT__
936	/*
937	* If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the
938	* address space is used for the mappings that will remain in place when in EL0.
939	* As a result, 1 bit less of address space is available to the rest of the
940	* the kernel.
941	*/
942	#endif /* __ARM_KERNEL_PROTECT__ */
943	#ifdef __ARM_16K_PG__
944	#if __ARM64_PMAP_SUBPAGE_L1__
945	#define T0SZ_BOOT 25ULL
946	#else /* !__ARM64_PMAP_SUBPAGE_L1__ */
947	#define T0SZ_BOOT 17ULL
948	#endif /* !__ARM64_PMAP_SUBPAGE_L1__ */
949	#else /* __ARM_16K_PG__ */
950	#if __ARM64_PMAP_SUBPAGE_L1__
951	#define T0SZ_BOOT 26ULL
952	#else /* __ARM64_PMAP_SUBPAGE_L1__ */
953	#define T0SZ_BOOT 25ULL
954	#endif /* __ARM64_PMAP_SUBPAGE_L1__ */
955	#endif /* __ARM_16K_PG__ */
956
957	#if defined(APPLE_ARM64_ARCH_FAMILY)
958	/ T0SZ must be the same as T1SZ /
959	#define T1SZ_BOOT T0SZ_BOOT
960	#else /* defined(APPLE_ARM64_ARCH_FAMILY) */
961	#ifdef __ARM_16K_PG__
962	#if __ARM64_PMAP_SUBPAGE_L1__
963	#define T1SZ_BOOT 25ULL
964	#else /* !__ARM64_PMAP_SUBPAGE_L1__ */
965	#define T1SZ_BOOT 17ULL
966	#endif /* !__ARM64_PMAP_SUBPAGE_L1__ */
967	#else /* __ARM_16K_PG__ */
968	#if __ARM64_PMAP_SUBPAGE_L1__
969	#define T1SZ_BOOT 26ULL
970	#else /* __ARM64_PMAP_SUBPAGE_L1__ */
971	#define T1SZ_BOOT 25ULL
972	#endif /__ARM64_PMAP_SUBPAGE_L1__/
973	#endif /* __ARM_16K_PG__ */
974	#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
975
976	#if __ARM_42BIT_PA_SPACE__
977	#define TCR_IPS_VALUE TCR_IPS_42BITS
978	#else /* !__ARM_42BIT_PA_SPACE__ */
979	#define TCR_IPS_VALUE TCR_IPS_40BITS
980	#endif /* !__ARM_42BIT_PA_SPACE__ */
981
982	#if CONFIG_KERNEL_TBI
983	#define TCR_EL1_DTBI (TCR_TBI1_TOPBYTE_IGNORED \| TCR_TBID1_ENABLE)
984	#else /* CONFIG_KERNEL_TBI */
985	#define TCR_EL1_DTBI 0
986	#endif /* CONFIG_KERNEL_TBI */
987
988	#if HAS_16BIT_ASID
989	#define TCR_EL1_ASID TCR_AS_16BIT_ASID
990	#else /* HAS_16BIT_ASID */
991	#define TCR_EL1_ASID 0
992	#endif /* HAS_16BIT_ASID */
993
994	#define TCR_EL1_BASE \
995	(TCR_IPS_VALUE \| TCR_SH0_OUTER \| TCR_ORGN0_WRITEBACK \| \
996	TCR_IRGN0_WRITEBACK \| (T0SZ_BOOT << TCR_T0SZ_SHIFT) \| \
997	TCR_SH1_OUTER \| TCR_ORGN1_WRITEBACK \| \
998	TCR_IRGN1_WRITEBACK \| (TCR_TG1_GRANULE_SIZE) \| \
999	TCR_TBI0_TOPBYTE_IGNORED \| (TCR_TBID0_ENABLE) \| TCR_E0PD_VALUE \| \
1000	TCR_EL1_DTBI \| TCR_EL1_ASID \| TCR_EL1_EXTRA)
1001
1002	#if __ARM_KERNEL_PROTECT__
1003	#define TCR_EL1_BOOT (TCR_EL1_BASE \| (T1SZ_BOOT << TCR_T1SZ_SHIFT) \| (TCR_TG0_GRANULE_SIZE))
1004	#define T1SZ_USER (T1SZ_BOOT + 1)
1005	#define TCR_EL1_USER (TCR_EL1_BASE \| (T1SZ_USER << TCR_T1SZ_SHIFT) \| (TCR_TG0_GRANULE_SIZE))
1006	#else
1007	#define TCR_EL1_BOOT (TCR_EL1_BASE \| (T1SZ_BOOT << TCR_T1SZ_SHIFT) \| (TCR_TG0_GRANULE_SIZE))
1008	#endif /* __ARM_KERNEL_PROTECT__ */
1009
1010	#define TCR_EL1_4KB (TCR_EL1_BASE \| (T1SZ_BOOT << TCR_T1SZ_SHIFT) \| (TCR_TG0_GRANULE_4KB))
1011	#define TCR_EL1_16KB (TCR_EL1_BASE \| (T1SZ_BOOT << TCR_T1SZ_SHIFT) \| (TCR_TG0_GRANULE_16KB))
1012
1013
1014
1015
1016	/*
1017	* Monitor Debug System Control Register (MDSCR)
1018	*/
1019
1020	#define MDSCR_TFO_SHIFT 31
1021	#define MDSCR_TFO (1ULL << MDSCR_TFO_SHIFT)
1022	#define MDSCR_RXFULL_SHIFT 30
1023	#define MDSCR_RXFULL (1ULL << MDSCR_RXFULL_SHIFT)
1024	#define MDSCR_TXFULL_SHIFT 29
1025	#define MDSCR_TXFULL (1ULL << MDSCR_TXFULL_SHIFT)
1026	#define MDSCR_RXO_SHIFT 27
1027	#define MDSCR_RXO (1ULL << MDSCR_RXO_SHIFT)
1028	#define MDSCR_TXU_SHIFT 26
1029	#define MDSCR_TXU (1ULL << MDSCR_TXU_SHIFT)
1030	#define MDSCR_INTDIS_SHIFT 22
1031	#define MDSCR_INTDIS_MASK (0x2U << MDSCR_INTDIS_SHIFT)
1032	#define MDSCR_TDA_SHIFT 21
1033	#define MDSCR_TDA (1ULL << MDSCR_TDA_SHIFT)
1034	#define MDSCR_SC2_SHIFT 19
1035	#define MDSCR_SC2 (1ULL << MDSCR_SC2_SHIFT)
1036	#define MDSCR_MDE_SHIFT 15
1037	#define MDSCR_MDE (1ULL << MDSCR_MDE_SHIFT)
1038	#define MDSCR_HDE_SHIFT 14
1039	#define MDSCR_HDE (1ULL << MDSCR_HDE_SHIFT)
1040	#define MDSCR_KDE_SHIFT 13
1041	#define MDSCR_KDE (1ULL << MDSCR_KDE_SHIFT)
1042	#define MDSCR_TDCC_SHIFT 12
1043	#define MDSCR_TDCC (1ULL << MDSCR_TDCC_SHIFT)
1044	#define MDSCR_ERR_SHIFT 6
1045	#define MDSCR_ERR (1ULL << MDSCR_ERR_SHIFT)
1046	#define MDSCR_SS_SHIFT 0
1047	#define MDSCR_SS (1ULL << MDSCR_SS_SHIFT)
1048
1049	/*
1050	* Translation Table Base Register (TTBR)
1051	*
1052	* 63 48 47 x x-1 0
1053	* +--------+------------------+------+
1054	* \| ASID \| Base Address \| zero \|
1055	* +--------+------------------+------+
1056	*
1057	*/
1058	#define TTBR_ASID_SHIFT 48
1059	#define TTBR_ASID_MASK 0xffff000000000000
1060
1061	#define TTBR_BADDR_MASK 0x0000ffffffffffff
1062
1063	/*
1064	* Memory Attribute Indirection Register
1065	*
1066	* 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0
1067	* +-------+-------+-------+-------+-------+-------+-------+-------+
1068	* \| Attr7 \| Attr6 \| Attr5 \| Attr4 \| Attr3 \| Attr2 \| Attr1 \| Attr0 \|
1069	* +-------+-------+-------+-------+-------+-------+-------+-------+
1070	*
1071	*/
1072
1073	#define MAIR_ATTR_SHIFT(x) (8*(x))
1074
1075	/ Strongly ordered or device memory attributes /
1076	#define MAIR_OUTER_STRONGLY_ORDERED 0x0
1077	#define MAIR_OUTER_DEVICE 0x0
1078
1079	#define MAIR_INNER_STRONGLY_ORDERED 0x0
1080	#define MAIR_INNER_DEVICE 0x4
1081
1082	/ Normal memory attributes /
1083	#define MAIR_OUTER_NON_CACHEABLE 0x40
1084	#define MAIR_OUTER_WRITE_THROUGH 0x80
1085	#define MAIR_OUTER_WRITE_BACK 0xc0
1086
1087	#define MAIR_INNER_NON_CACHEABLE 0x4
1088	#define MAIR_INNER_WRITE_THROUGH 0x8
1089	#define MAIR_INNER_WRITE_BACK 0xc
1090
1091	/ Allocate policy for cacheable memory /
1092	#define MAIR_OUTER_WRITE_ALLOCATE 0x10
1093	#define MAIR_OUTER_READ_ALLOCATE 0x20
1094
1095	#define MAIR_INNER_WRITE_ALLOCATE 0x1
1096	#define MAIR_INNER_READ_ALLOCATE 0x2
1097
1098	/ Memory Atribute Encoding /
1099
1100	/*
1101	* Device memory types:
1102	* G (gathering): multiple reads/writes can be combined
1103	* R (reordering): reads or writes may reach device out of program order
1104	* E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes)
1105	*/
1106	#if HAS_FEAT_XS
1107
1108	#define MAIR_DISABLE 0x01 /* Device Memory, nGnRnE (strongly ordered), XS=0 */
1109	#define MAIR_POSTED_XS 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes), XS=1 */
1110	#define MAIR_POSTED_COMBINED_REORDERED_XS 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=1 */
1111	#define MAIR_POSTED_COMBINED_REORDERED 0x0D /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=0 */
1112	#define MAIR_WRITECOMB 0x40 /* Normal Memory, Non-Cacheable, XS=0 */
1113	#define MAIR_WRITETHRU 0xA0 /* Normal Memory, Write-through, XS=0 */
1114	#define MAIR_WRITEBACK 0xFF /* Normal Memory, Write-back, XS=0 */
1115
1116	/*
1117	* Memory Attribute Index. If these values change, please also update the pmap
1118	* LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64).
1119	*/
1120	#define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */
1121	#define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */
1122	#define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */
1123	#define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */
1124	#define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */
1125	#define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */
1126	#define CACHE_ATTRINDX_POSTED_XS 0x5 /* no cache, no buffer, posted writes (device memory), XS=1 */
1127	#define CACHE_ATTRINDX_POSTED CACHE_ATTRINDX_POSTED_XS /* posted mappings use XS by default */
1128	#define CACHE_ATTRINDX_POSTED_REORDERED CACHE_ATTRINDX_DISABLE /* no need for device-nGRE on newer SoCs, fallback to nGnRnE */
1129	#define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x6 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=0 */
1130	#define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED_XS 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=1 */
1131	#define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK
1132	#define CACHE_ATTRINDX_N_INDICES (8ULL)
1133
1134	#else
1135
1136	#define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */
1137	#define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */
1138	#define MAIR_POSTED_REORDERED 0x08 /* Device Memory, nGRE (reorderable, posted writes) */
1139	#define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */
1140	#define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */
1141	#define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */
1142	#define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */
1143
1144	/*
1145	* Memory Attribute Index. If these values change, please also update the pmap
1146	* LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64).
1147	*/
1148	#define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */
1149	#define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */
1150	#define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */
1151	#define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */
1152	#define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */
1153	#define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */
1154	#define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes (device memory) */
1155	#define CACHE_ATTRINDX_POSTED_REORDERED 0x6 /* no cache, reorderable access, posted writes (device memory) */
1156	#define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */
1157	#define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK
1158	#define CACHE_ATTRINDX_N_INDICES (8ULL)
1159
1160	#endif /* HAS_FEAT_XS */
1161
1162	#if HAS_UCNORMAL_MEM
1163	#define CACHE_ATTRINDX_RT CACHE_ATTRINDX_WRITECOMB
1164	#else
1165	#define CACHE_ATTRINDX_RT CACHE_ATTRINDX_DISABLE
1166	#endif /* HAS_UCNORMAL_MEM */
1167
1168
1169
1170	/*
1171	* Access protection bit values (TTEs and PTEs), stage 1
1172	*
1173	* Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access)
1174	*/
1175	#define AP_RWNA 0x0 /* priv=read-write, user=no-access */
1176	#define AP_RWRW 0x1 /* priv=read-write, user=read-write */
1177	#define AP_RONA 0x2 /* priv=read-only, user=no-access */
1178	#define AP_RORO 0x3 /* priv=read-only, user=read-only */
1179	#define AP_MASK 0x3 /* mask to find ap bits */
1180
1181	/*
1182	* Shareability attributes
1183	*/
1184	#define SH_NONE 0x0 /* Non shareable */
1185	#define SH_NONE 0x0 /* Device shareable */
1186	#define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */
1187	#define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */
1188	#define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */
1189
1190
1191	/*
1192	* ARM Page Granule
1193	*/
1194	#ifdef __ARM_16K_PG__
1195	#define ARM_PGSHIFT 14
1196	#else
1197	#define ARM_PGSHIFT 12
1198	#endif
1199	#define ARM_PGBYTES (1 << ARM_PGSHIFT)
1200	#define ARM_PGMASK (ARM_PGBYTES-1)
1201
1202	/*
1203	* L0 Translation table
1204	*
1205	* 4KB granule size:
1206	* Each translation table is 4KB
1207	* 512 64-bit entries of 512GB (2^39) of address space.
1208	* Covers 256TB (2^48) of address space.
1209	*
1210	* 16KB granule size:
1211	* Each translation table is 16KB
1212	* 2 64-bit entries of 128TB (2^47) of address space.
1213	* Covers 256TB (2^48) of address space.
1214	*/
1215
1216	/ 16K L0 /
1217	#define ARM_16K_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */
1218	#define ARM_16K_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */
1219	#define ARM_16K_TT_L0_SHIFT 47 /* page descriptor shift */
1220	#define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */
1221
1222	/ 4K L0 /
1223	#define ARM_4K_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */
1224	#define ARM_4K_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */
1225	#define ARM_4K_TT_L0_SHIFT 39 /* page descriptor shift */
1226	#define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */
1227
1228	/*
1229	* L1 Translation table
1230	*
1231	* 4KB granule size:
1232	* Each translation table is 4KB
1233	* 512 64-bit entries of 1GB (2^30) of address space.
1234	* Covers 512GB (2^39) of address space.
1235	*
1236	* 16KB granule size:
1237	* Each translation table is 16KB
1238	* 2048 64-bit entries of 64GB (2^36) of address space.
1239	* Covers 128TB (2^47) of address space.
1240	*/
1241
1242	/ 16K L1 /
1243	#define ARM_16K_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */
1244	#define ARM_16K_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */
1245	#define ARM_16K_TT_L1_SHIFT 36 /* page descriptor shift */
1246	#if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__
1247	/ This config supports 512GB per TTBR. /
1248	#define ARM_16K_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */
1249	#else /* __ARM64_PMAP_SUBPAGE_L1__ */
1250	#define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */
1251	#endif /* __ARM64_PMAP_SUBPAGE_L1__ */
1252
1253	/ 4K L1 /
1254	#define ARM_4K_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */
1255	#define ARM_4K_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */
1256	#define ARM_4K_TT_L1_SHIFT 30 /* page descriptor shift */
1257	#if __ARM64_PMAP_SUBPAGE_L1__ && !__ARM_16K_PG__
1258	/ This config supports 256GB per TTBR. /
1259	#define ARM_4K_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */
1260	#else /* __ARM64_PMAP_SUBPAGE_L1__ */
1261	#define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */
1262	#endif /* __ARM64_PMAP_SUBPAGE_L1__ */
1263
1264	/ some sugar for getting pointers to page tables and entries /
1265
1266	#define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)
1267	#define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
1268	#define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)
1269
1270	#define L2_TABLE_VA(tte) ((tt_entry_t) phystokv(((tte)) & ARM_TTE_TABLE_MASK))
1271	#define L3_TABLE_VA(tte2) ((pt_entry_t) phystokv(((tte2)) & ARM_TTE_TABLE_MASK))
1272
1273	/*
1274	* L2 Translation table
1275	*
1276	* 4KB granule size:
1277	* Each translation table is 4KB
1278	* 512 64-bit entries of 2MB (2^21) of address space.
1279	* Covers 1GB (2^30) of address space.
1280	*
1281	* 16KB granule size:
1282	* Each translation table is 16KB
1283	* 2048 64-bit entries of 32MB (2^25) of address space.
1284	* Covers 64GB (2^36) of address space.
1285	*/
1286
1287	/ 16K L2 /
1288	#define ARM_16K_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */
1289	#define ARM_16K_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */
1290	#define ARM_16K_TT_L2_SHIFT 25 /* page descriptor shift */
1291	#define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */
1292
1293	/ 4K L2 /
1294	#define ARM_4K_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */
1295	#define ARM_4K_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */
1296	#define ARM_4K_TT_L2_SHIFT 21 /* page descriptor shift */
1297	#define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */
1298
1299	/*
1300	* L3 Translation table
1301	*
1302	* 4KB granule size:
1303	* Each translation table is 4KB
1304	* 512 64-bit entries of 4KB (2^12) of address space.
1305	* Covers 2MB (2^21) of address space.
1306	*
1307	* 16KB granule size:
1308	* Each translation table is 16KB
1309	* 2048 64-bit entries of 16KB (2^14) of address space.
1310	* Covers 32MB (2^25) of address space.
1311	*/
1312
1313	/ 16K L3 /
1314	#define ARM_16K_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */
1315	#define ARM_16K_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */
1316	#define ARM_16K_TT_L3_SHIFT 14 /* page descriptor shift */
1317	#define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */
1318
1319	/ 4K L3 /
1320	#define ARM_4K_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */
1321	#define ARM_4K_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */
1322	#define ARM_4K_TT_L3_SHIFT 12 /* page descriptor shift */
1323	#define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */
1324
1325	#ifdef __ARM_16K_PG__
1326
1327	/ Native L0 defines /
1328	#define ARM_TT_L0_SIZE ARM_16K_TT_L0_SIZE
1329	#define ARM_TT_L0_OFFMASK ARM_16K_TT_L0_OFFMASK
1330	#define ARM_TT_L0_SHIFT ARM_16K_TT_L0_SHIFT
1331	#define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK
1332
1333	/ Native L1 defines /
1334	#define ARM_TT_L1_SIZE ARM_16K_TT_L1_SIZE
1335	#define ARM_TT_L1_OFFMASK ARM_16K_TT_L1_OFFMASK
1336	#define ARM_TT_L1_SHIFT ARM_16K_TT_L1_SHIFT
1337	#define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK
1338
1339	/ Native L2 defines /
1340	#define ARM_TT_L2_SIZE ARM_16K_TT_L2_SIZE
1341	#define ARM_TT_L2_OFFMASK ARM_16K_TT_L2_OFFMASK
1342	#define ARM_TT_L2_SHIFT ARM_16K_TT_L2_SHIFT
1343	#define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK
1344
1345	/ Native L3 defines /
1346	#define ARM_TT_L3_SIZE ARM_16K_TT_L3_SIZE
1347	#define ARM_TT_L3_OFFMASK ARM_16K_TT_L3_OFFMASK
1348	#define ARM_TT_L3_SHIFT ARM_16K_TT_L3_SHIFT
1349	#define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK
1350
1351	#else /* !__ARM_16K_PG__ */
1352
1353	/ Native L0 defines /
1354	#define ARM_TT_L0_SIZE ARM_4K_TT_L0_SIZE
1355	#define ARM_TT_L0_OFFMASK ARM_4K_TT_L0_OFFMASK
1356	#define ARM_TT_L0_SHIFT ARM_4K_TT_L0_SHIFT
1357	#define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK
1358
1359	/ Native L1 defines /
1360	#define ARM_TT_L1_SIZE ARM_4K_TT_L1_SIZE
1361	#define ARM_TT_L1_OFFMASK ARM_4K_TT_L1_OFFMASK
1362	#define ARM_TT_L1_SHIFT ARM_4K_TT_L1_SHIFT
1363	#define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK
1364
1365	/ Native L2 defines /
1366	#define ARM_TT_L2_SIZE ARM_4K_TT_L2_SIZE
1367	#define ARM_TT_L2_OFFMASK ARM_4K_TT_L2_OFFMASK
1368	#define ARM_TT_L2_SHIFT ARM_4K_TT_L2_SHIFT
1369	#define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK
1370
1371	/ Native L3 defines /
1372	#define ARM_TT_L3_SIZE ARM_4K_TT_L3_SIZE
1373	#define ARM_TT_L3_OFFMASK ARM_4K_TT_L3_OFFMASK
1374	#define ARM_TT_L3_SHIFT ARM_4K_TT_L3_SHIFT
1375	#define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK
1376
1377	#endif /* !__ARM_16K_PG__ */
1378
1379	/*
1380	* Convenience definitions for:
1381	* ARM_TT_LEAF: The last level of the configured page table format.
1382	* ARM_TT_TWIG: The second to last level of the configured page table format.
1383	* ARM_TT_ROOT: The first level of the configured page table format.
1384	*
1385	* My apologies to any botanists who may be reading this.
1386	*/
1387	#define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE
1388	#define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK
1389	#define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT
1390	#define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK
1391
1392	#define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE
1393	#define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK
1394	#define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT
1395	#define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK
1396
1397	#define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE
1398	#define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK
1399	#define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT
1400	#define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK
1401
1402	/*
1403	* 4KB granule size:
1404	*
1405	* Level 0 Translation Table Entry
1406	*
1407	* 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
1408	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1409	* \|NS\| AP \|XN\|PXN\|ignored\| zero \| L1TableOutputAddress \|ignored\|1\|V\|
1410	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1411	*
1412	* Level 1 Translation Table Entry
1413	*
1414	* 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
1415	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1416	* \|NS\| AP \|XN\|PXN\|ignored\| zero \| L2TableOutputAddress \|ignored\|1\|V\|
1417	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1418	*
1419	* Level 1 Translation Block Entry
1420	*
1421	* 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0
1422	* +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1423	* \| ign \|sw use\|XN\|PXN\|HINT\| zero \| OutputAddress[47:30] \| zero \|nG\|AF\| SH \| AP \|NS\|AttrIdx\|0\|V\|
1424	* +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1425	*
1426	* Level 2 Translation Table Entry
1427	*
1428	* 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
1429	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1430	* \|NS\| AP \|XN\|PXN\|ignored\| zero \| L3TableOutputAddress \|ignored\|1\|V\|
1431	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1432	*
1433	* Level 2 Translation Block Entry
1434	*
1435	* 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0
1436	* +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1437	* \| ign \|sw use\|XN\|PXN\|HINT\| zero \| OutputAddress[47:21] \| zero \|nG\|AF\| SH \| AP \|NS\|AttrIdx\|0\|V\|
1438	* +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1439	*
1440	* 16KB granule size:
1441	*
1442	* Level 0 Translation Table Entry
1443	*
1444	* 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
1445	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1446	* \|NS\| AP \|XN\|PXN\|ignored\| zero \| L1TableOutputAddress \|ignored\|1\|V\|
1447	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1448	*
1449	* Level 1 Translation Table Entry
1450	*
1451	* 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
1452	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1453	* \|NS\| AP \|XN\|PXN\|ignored\| zero \| L2TableOutputAddress \|ignored\|1\|V\|
1454	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1455	*
1456	* Level 2 Translation Table Entry
1457	*
1458	* 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
1459	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1460	* \|NS\| AP \|XN\|PXN\|ignored\| zero \| L3TableOutputAddress \|ignored\|1\|V\|
1461	* +--+-----+--+---+-------+------+----------------------+-------+-+-+
1462	*
1463	* Level 2 Translation Block Entry
1464	*
1465	* 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0
1466	* +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1467	* \| ign \|sw use\|XN\|PXN\|HINT\| zero \| OutputAddress[47:25] \| zero \|nG\|AF\| SH \| AP \|NS\|AttrIdx\|0\|V\|
1468	* +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
1469	*
1470	* where:
1471	* nG: notGlobal bit
1472	* SH: Shareability field
1473	* AP: access protection
1474	* XN: eXecute Never bit
1475	* PXN: Privilege eXecute Never bit
1476	* NS: Non-Secure bit
1477	* HINT: 16 entry continuguous output hint
1478	* AttrIdx: Memory Attribute Index
1479	*/
1480
1481	#define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */
1482	#ifdef __ARM_16K_PG__
1483	#define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */
1484	#else
1485	#define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */
1486	#endif
1487
1488	#define ARM_TTE_MAX (TTE_PGENTRIES)
1489
1490	#define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */
1491	#define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */
1492
1493	#define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */
1494
1495	#define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */
1496	#define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */
1497	#define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */
1498	#define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL
1499	#define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */
1500
1501	#ifdef __ARM_16K_PG__
1502	/*
1503	* Note that L0/L1 block entries are disallowed for the 16KB granule size; what
1504	* are we doing with these?
1505	*/
1506	#define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */
1507	#define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */
1508	#define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */
1509	#define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */
1510	#define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */
1511	#define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */
1512	#else
1513	#define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */
1514	#define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */
1515	#define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */
1516	#define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */
1517	#define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */
1518	#define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */
1519	#endif
1520
1521	#define ARM_TTE_BLOCK_APSHIFT 6
1522	#define ARM_TTE_BLOCK_AP(x) ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */
1523	#define ARM_TTE_BLOCK_APMASK (0x3 << ARM_TTE_BLOCK_APSHIFT)
1524
1525	#define ARM_TTE_BLOCK_ATTRINDX(x) ((x) << 2) /* memory attributes index */
1526	#define ARM_TTE_BLOCK_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */
1527
1528	#define ARM_TTE_BLOCK_SH(x) ((x) << 8) /* access shared */
1529	#define ARM_TTE_BLOCK_SHMASK (0x3ULL << 8) /* mask access shared */
1530
1531	#define ARM_TTE_BLOCK_AF 0x0000000000000400ULL /* value for access */
1532	#define ARM_TTE_BLOCK_AFMASK 0x0000000000000400ULL /* access mask */
1533
1534	#define ARM_TTE_BLOCK_NG 0x0000000000000800ULL /* value for a global mapping */
1535	#define ARM_TTE_BLOCK_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */
1536
1537	#define ARM_TTE_BLOCK_NS 0x0000000000000020ULL /* value for a secure mapping */
1538	#define ARM_TTE_BLOCK_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */
1539
1540	#define ARM_TTE_BLOCK_PNX 0x0020000000000000ULL /* value for privilege no execute bit */
1541	#define ARM_TTE_BLOCK_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */
1542
1543	#define ARM_TTE_BLOCK_NX 0x0040000000000000ULL /* value for no execute */
1544	#define ARM_TTE_BLOCK_NXMASK 0x0040000000000000ULL /* no execute mask */
1545
1546	#define ARM_TTE_BLOCK_WIRED 0x0400000000000000ULL /* value for software wired bit */
1547	#define ARM_TTE_BLOCK_WIREDMASK 0x0400000000000000ULL /* software wired mask */
1548
1549	#define ARM_TTE_BLOCK_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */
1550	#define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL /* software writeable mask */
1551
1552	#define ARM_TTE_TABLE_MASK 0x0000fffffffff000ULL /* mask for extracting pointer to next table (works at any level) */
1553
1554	#define ARM_TTE_TABLE_APSHIFT 61
1555	#define ARM_TTE_TABLE_AP_NO_EFFECT 0x0ULL
1556	#define ARM_TTE_TABLE_AP_USER_NA 0x1ULL
1557	#define ARM_TTE_TABLE_AP_RO 0x2ULL
1558	#define ARM_TTE_TABLE_AP_KERN_RO 0x3ULL
1559	#define ARM_TTE_TABLE_AP(x) ((x) << ARM_TTE_TABLE_APSHIFT) /* access protection */
1560
1561	#define ARM_TTE_TABLE_NS 0x8000000000000020ULL /* value for a secure mapping */
1562	#define ARM_TTE_TABLE_NS_MASK 0x8000000000000020ULL /* notSecure mapping mask */
1563
1564	#define ARM_TTE_TABLE_XN 0x1000000000000000ULL /* value for no execute */
1565	#define ARM_TTE_TABLE_XNMASK 0x1000000000000000ULL /* no execute mask */
1566
1567	#define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */
1568	#define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */
1569
1570	#if __ARM_KERNEL_PROTECT__
1571	#define ARM_TTE_BOOT_BLOCK \
1572	(ARM_TTE_TYPE_BLOCK \| ARM_TTE_VALID \| ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) \| \
1573	ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) \| ARM_TTE_BLOCK_AF \| ARM_TTE_BLOCK_NG)
1574	#else /* __ARM_KERNEL_PROTECT__ */
1575	#define ARM_TTE_BOOT_BLOCK \
1576	(ARM_TTE_TYPE_BLOCK \| ARM_TTE_VALID \| ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) \| \
1577	ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) \| ARM_TTE_BLOCK_AF)
1578	#endif /* __ARM_KERNEL_PROTECT__ */
1579
1580	#define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE \| ARM_TTE_VALID )
1581	/*
1582	* L3 Translation table
1583	*
1584	* 4KB granule size:
1585	* Each translation table is 4KB
1586	* 512 64-bit entries of 4KB (2^12) of address space.
1587	* Covers 2MB (2^21) of address space.
1588	*
1589	* 16KB granule size:
1590	* Each translation table is 16KB
1591	* 2048 64-bit entries of 16KB (2^14) of address space.
1592	* Covers 32MB (2^25) of address space.
1593	*/
1594
1595	#ifdef __ARM_16K_PG__
1596	#define ARM_PTE_SIZE 0x0000000000004000ULL /* size of area covered by a tte */
1597	#define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */
1598	#define ARM_PTE_SHIFT 14 /* page descriptor shift */
1599	#define ARM_PTE_MASK 0x0000ffffffffc000ULL /* mask for output address in PTE */
1600	#else
1601	#define ARM_PTE_SIZE 0x0000000000001000ULL /* size of area covered by a tte */
1602	#define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */
1603	#define ARM_PTE_SHIFT 12 /* page descriptor shift */
1604	#define ARM_PTE_MASK 0x0000fffffffff000ULL /* mask for output address in PTE */
1605	#endif
1606
1607	#define ARM_TTE_PA_MASK 0x0000fffffffff000ULL
1608
1609	/*
1610	* L3 Page table entries
1611	*
1612	* The following page table entry types are possible:
1613	*
1614	* fault page entry
1615	* 63 2 0
1616	* +------------------------------+--+
1617	* \| ignored \|00\|
1618	* +------------------------------+--+
1619	*
1620	*
1621	* 63 59 58 55 54 53 52 51 48 47 12 11 10 9 8 7 6 5 4 2 1 0
1622	* +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1623	* \| ign \|sw use\|XN\|PXN\|HINT\| zero \| OutputAddress[47:12] \|nG\|AF\| SH \| AP \|NS\|AttrIdx\|1\|V\|
1624	* +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1625	*
1626	* where:
1627	* nG: notGlobal bit
1628	* SH: Shareability field
1629	* AP: access protection
1630	* XN: eXecute Never bit
1631	* PXN: Privilege eXecute Never bit
1632	* NS: Non-Secure bit
1633	* HINT: 16 entry continuguous output hint
1634	* AttrIdx: Memory Attribute Index
1635	*/
1636
1637	#define PTE_SHIFT 3 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */
1638	#ifdef __ARM_16K_PG__
1639	#define PTE_PGENTRIES (16384 >> PTE_SHIFT) /* number of ptes per page */
1640	#else
1641	#define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */
1642	#endif
1643
1644	#define ARM_PTE_EMPTY 0x0000000000000000ULL /* unassigned - invalid entry */
1645
1646	/ markers for (invalid) PTE for a page sent to compressor /
1647	#define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */
1648	#define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */
1649	#define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL
1650
1651	#define ARM_PTE_TYPE 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1652	#define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1653	#define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */
1654	#define ARM_PTE_TYPE_MASK 0x0000000000000002ULL /* mask to get pte type */
1655
1656	/ This mask works for both 16K and 4K pages because bits 12-13 will be zero in 16K pages /
1657	#define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* output address mask for page */
1658	#define ARM_PTE_PAGE_SHIFT 12 /* page shift for the output address in the entry */
1659
1660	#define ARM_PTE_AP(x) ((x) << 6) /* access protections */
1661	#define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */
1662	#define ARM_PTE_EXTRACT_AP(x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */
1663
1664	#define ARM_PTE_ATTRINDX(x) ((x) << 2) /* memory attributes index */
1665	#define ARM_PTE_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */
1666	#define ARM_PTE_EXTRACT_ATTRINDX(x) (((x) >> 2) & 0x7ULL) /* extract memory attributes index */
1667
1668	#define ARM_PTE_SH(x) ((x) << 8) /* access shared */
1669	#define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */
1670
1671	#define ARM_PTE_AF 0x0000000000000400ULL /* value for access */
1672	#define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */
1673
1674	#define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */
1675	#define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */
1676
1677	#define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */
1678	#define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */
1679
1680	#define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */
1681	#define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */
1682
1683	#if __ARM_16K_PG__
1684	#define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */
1685	#define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */
1686	#define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */
1687	#define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */
1688	#define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */
1689	#else
1690	#define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */
1691	#define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */
1692	#define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */
1693	#define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */
1694	#define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */
1695	#endif
1696
1697	#define ARM_PTE_PNX 0x0020000000000000ULL /* value for privilege no execute bit */
1698	#define ARM_PTE_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */
1699
1700	#define ARM_PTE_NX 0x0040000000000000ULL /* value for no execute bit */
1701	#define ARM_PTE_NXMASK 0x0040000000000000ULL /* no execute mask */
1702
1703	#define ARM_PTE_XMASK (ARM_PTE_PNXMASK \| ARM_PTE_NXMASK)
1704
1705	#define ARM_PTE_WIRED 0x0400000000000000ULL /* value for software wired bit */
1706	#define ARM_PTE_WIRED_MASK 0x0400000000000000ULL /* software wired mask */
1707
1708	#define ARM_PTE_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */
1709	#define ARM_PTE_WRITEABLE_MASK 0x0800000000000000ULL /* software writeable mask */
1710
1711	#define ARM_PTE_BOOT_PAGE_BASE \
1712	(ARM_PTE_TYPE_VALID \| ARM_PTE_SH(SH_OUTER_MEMORY) \| \
1713	ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) \| ARM_PTE_AF)
1714
1715	#if __ARM_KERNEL_PROTECT__
1716	#define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE \| ARM_PTE_NG)
1717	#else /* __ARM_KERNEL_PROTECT__ */
1718	#define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE)
1719	#endif /* __ARM_KERNEL_PROTECT__ */
1720
1721	/*
1722	* TLBI appers to only deal in 4KB page addresses, so give
1723	* it an explicit shift of 12.
1724	*/
1725	#define TLBI_ADDR_SHIFT (0)
1726	#define TLBI_ADDR_SIZE (44)
1727	#define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1)
1728	#define TLBI_ASID_SHIFT (48)
1729	#define TLBI_ASID_SIZE (16)
1730	#define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1))
1731
1732	#define RTLBI_ADDR_SIZE (37)
1733	#define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1)
1734	#define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT
1735	#define RTLBI_TG(_page_shift_) ((uint64_t)((((_page_shift_) - 12) >> 1) + 1) << 46)
1736	#define RTLBI_SCALE_SHIFT (44)
1737	#define RTLBI_NUM_SHIFT (39)
1738
1739	/*
1740	* Exception Syndrome Register
1741	*
1742	* 31 26 25 24 0
1743	* +------+--+------------------+
1744	* \| EC \|IL\| ISS \|
1745	* +------+--+------------------+
1746	*
1747	* EC - Exception Class
1748	* IL - Instruction Length
1749	* ISS - Instruction Specific Syndrome
1750	*
1751	* Note: The ISS can have many forms. These are defined separately below.
1752	*/
1753
1754	#define ESR_EC_SHIFT 26
1755	#define ESR_EC_MASK (0x3FULL << ESR_EC_SHIFT)
1756	#define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT)
1757
1758	#define ESR_IL_SHIFT 25
1759	#define ESR_IL (1 << ESR_IL_SHIFT)
1760
1761	#define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL))
1762
1763	#define ESR_ISS_MASK 0x01FFFFFF
1764	#define ESR_ISS(x) (x & ESR_ISS_MASK)
1765
1766	#ifdef __ASSEMBLER__
1767	/ Define only the classes we need to test in the exception vectors. /
1768	#define ESR_EC_UNCATEGORIZED 0x00
1769	#define ESR_EC_PAC_FAIL 0x1C
1770	#define ESR_EC_IABORT_EL1 0x21
1771	#define ESR_EC_DABORT_EL1 0x25
1772	#define ESR_EC_SP_ALIGN 0x26
1773	#define ESR_EC_BRK_AARCH64 0x3C
1774	#else
1775	typedef enum {
1776	ESR_EC_UNCATEGORIZED = `0x00`,
1777	ESR_EC_WFI_WFE = `0x01`,
1778	ESR_EC_MCR_MRC_CP15_TRAP = `0x03`,
1779	ESR_EC_MCRR_MRRC_CP15_TRAP = `0x04`,
1780	ESR_EC_MCR_MRC_CP14_TRAP = `0x05`,
1781	ESR_EC_LDC_STC_CP14_TRAP = `0x06`,
1782	ESR_EC_TRAP_SIMD_FP = `0x07`,
1783	ESR_EC_PTRAUTH_INSTR_TRAP = `0x09`,
1784	ESR_EC_MCRR_MRRC_CP14_TRAP = `0x0c`,
1785	ESR_EC_ILLEGAL_INSTR_SET = `0x0e`,
1786	ESR_EC_SVC_32 = `0x11`,
1787	ESR_EC_HVC_32 = `0x12`,
1788	ESR_EC_SVC_64 = `0x15`,
1789	ESR_EC_HVC_64 = `0x16`,
1790	ESR_EC_MSR_TRAP = `0x18`,
1791	#if __has_feature(ptrauth_calls)
1792	ESR_EC_PAC_FAIL = `0x1C`,
1793	#endif /* __has_feature(ptrauth_calls) */
1794	ESR_EC_IABORT_EL0 = `0x20`,
1795	ESR_EC_IABORT_EL1 = `0x21`,
1796	ESR_EC_PC_ALIGN = `0x22`,
1797	ESR_EC_DABORT_EL0 = `0x24`,
1798	ESR_EC_DABORT_EL1 = `0x25`,
1799	ESR_EC_SP_ALIGN = `0x26`,
1800	ESR_EC_FLOATING_POINT_32 = `0x28`,
1801	ESR_EC_FLOATING_POINT_64 = `0x2C`,
1802	ESR_EC_SERROR_INTERRUPT = `0x2F`,
1803	ESR_EC_BKPT_REG_MATCH_EL0 = `0x30`, // Breakpoint Debug event taken to the EL from a lower EL.
1804	ESR_EC_BKPT_REG_MATCH_EL1 = `0x31`, // Breakpoint Debug event taken to the EL from the EL.
1805	ESR_EC_SW_STEP_DEBUG_EL0 = `0x32`, // Software Step Debug event taken to the EL from a lower EL.
1806	ESR_EC_SW_STEP_DEBUG_EL1 = `0x33`, // Software Step Debug event taken to the EL from the EL.
1807	ESR_EC_WATCHPT_MATCH_EL0 = `0x34`, // Watchpoint Debug event taken to the EL from a lower EL.
1808	ESR_EC_WATCHPT_MATCH_EL1 = `0x35`, // Watchpoint Debug event taken to the EL from the EL.
1809	ESR_EC_BKPT_AARCH32 = `0x38`,
1810	ESR_EC_BRK_AARCH64 = `0x3C`,
1811	} esr_exception_class_t;
1812
1813	typedef enum {
1814	FSC_TRANSLATION_FAULT_L0 = `0x04`,
1815	FSC_TRANSLATION_FAULT_L1 = `0x05`,
1816	FSC_TRANSLATION_FAULT_L2 = `0x06`,
1817	FSC_TRANSLATION_FAULT_L3 = `0x07`,
1818	FSC_ACCESS_FLAG_FAULT_L1 = `0x09`,
1819	FSC_ACCESS_FLAG_FAULT_L2 = `0x0A`,
1820	FSC_ACCESS_FLAG_FAULT_L3 = `0x0B`,
1821	FSC_PERMISSION_FAULT_L1 = `0x0D`,
1822	FSC_PERMISSION_FAULT_L2 = `0x0E`,
1823	FSC_PERMISSION_FAULT_L3 = `0x0F`,
1824	FSC_SYNC_EXT_ABORT = `0x10`,
1825	FSC_SYNC_EXT_ABORT_TT_L1 = `0x15`,
1826	FSC_SYNC_EXT_ABORT_TT_L2 = `0x16`,
1827	FSC_SYNC_EXT_ABORT_TT_L3 = `0x17`,
1828	FSC_SYNC_PARITY = `0x18`,
1829	FSC_ASYNC_PARITY = `0x19`,
1830	FSC_SYNC_PARITY_TT_L1 = `0x1D`,
1831	FSC_SYNC_PARITY_TT_L2 = `0x1E`,
1832	FSC_SYNC_PARITY_TT_L3 = `0x1F`,
1833	FSC_ALIGNMENT_FAULT = `0x21`,
1834	FSC_DEBUG_FAULT = `0x22`,
1835	} fault_status_t;
1836	#endif /* ASSEMBLER */
1837
1838	/*
1839	* HVC event
1840	* 24 16 15 0
1841	* +---------+-----+
1842	* \|000000000\| IMM \|
1843	* +---------+-----+
1844	*
1845	* where:
1846	* IMM: Immediate value
1847	*/
1848
1849	#define ISS_HVC_IMM_MASK 0xffff
1850	#define ISS_HVC_IMM(x) ((x) & ISS_HVC_IMM_MASK)
1851
1852	/*
1853	* Software step debug event ISS (EL1)
1854	* 24 23 6 5 0
1855	* +---+-----------------+--+------+
1856	* \|ISV\|00000000000000000\|EX\| IFSC \|
1857	* +---+-----------------+--+------+
1858	*
1859	* where:
1860	* ISV: Instruction syndrome valid
1861	* EX: Exclusive access
1862	* IFSC: Instruction Fault Status Code
1863	*/
1864
1865	#define ISS_SSDE_ISV_SHIFT 24
1866	#define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT)
1867
1868	#define ISS_SSDE_EX_SHIFT 6
1869	#define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT)
1870
1871	#define ISS_SSDE_FSC_MASK 0x3F
1872	#define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK)
1873
1874	/*
1875	* Instruction Abort ISS (EL1)
1876	* 24 10 9 5 0
1877	* +--------------+---+--+---+------+
1878	* \|00000000000000\|FnV\|EA\|000\| IFSC \|
1879	* +--------------+---+--+---+------+
1880	*
1881	* where:
1882	* FnV: FAR not Valid
1883	* EA: External Abort type
1884	* IFSC: Instruction Fault Status Code
1885	*/
1886
1887	#define ISS_IA_FNV_SHIFT 10
1888	#define ISS_IA_FNV (0x1 << ISS_IA_FNV_SHIFT)
1889
1890	#define ISS_IA_EA_SHIFT 9
1891	#define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT)
1892
1893	#define ISS_IA_FSC_MASK 0x3F
1894	#define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK)
1895
1896
1897	/*
1898	* Data Abort ISS (EL1)
1899	*
1900	* 24 10 9 8 7 6 5 0
1901	* +--------------+---+--+--+-----+---+----+
1902	* \|00000000000000\|FnV\|EA\|CM\|S1PTW\|WnR\|DFSC\|
1903	* +--------------+---+--+--+-----+---+----+
1904	*
1905	* where:
1906	* FnV: FAR not Valid
1907	* EA: External Abort type
1908	* CM: Cache Maintenance operation
1909	* WnR: Write not Read
1910	* S1PTW: Stage 2 exception on Stage 1 page table walk
1911	* DFSC: Data Fault Status Code
1912	*/
1913	#define ISS_DA_FNV_SHIFT 10
1914	#define ISS_DA_FNV (0x1 << ISS_DA_FNV_SHIFT)
1915
1916	#define ISS_DA_EA_SHIFT 9
1917	#define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT)
1918
1919	#define ISS_DA_CM_SHIFT 8
1920	#define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT)
1921
1922	#define ISS_DA_WNR_SHIFT 6
1923	#define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT)
1924
1925	#define ISS_DA_S1PTW_SHIFT 7
1926	#define ISS_DA_S1PTW (0x1 << ISS_DA_S1PTW_SHIFT)
1927
1928	#define ISS_DA_FSC_MASK 0x3F
1929	#define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK)
1930
1931	/*
1932	* Floating Point Exception ISS (EL1)
1933	*
1934	* 24 23 22 8 7 4 3 2 1 0
1935	* +-+---+---------------+---+--+---+---+---+---+---+
1936	* \|0\|TFV\|000000000000000\|IDF\|00\|IXF\|UFF\|OFF\|DZF\|IOF\|
1937	* +-+---+---------------+---+--+---+---+---+---+---+
1938	*
1939	* where:
1940	* TFV: Trapped Fault Valid
1941	* IDF: Input Denormal Exception
1942	* IXF: Input Inexact Exception
1943	* UFF: Underflow Exception
1944	* OFF: Overflow Exception
1945	* DZF: Divide by Zero Exception
1946	* IOF: Invalid Operation Exception
1947	*/
1948	#define ISS_FP_TFV_SHIFT 23
1949	#define ISS_FP_TFV (0x1 << ISS_FP_TFV_SHIFT)
1950
1951	#define ISS_FP_IDF_SHIFT 7
1952	#define ISS_FP_IDF (0x1 << ISS_FP_IDF_SHIFT)
1953
1954	#define ISS_FP_IXF_SHIFT 4
1955	#define ISS_FP_IXF (0x1 << ISS_FP_IXF_SHIFT)
1956
1957	#define ISS_FP_UFF_SHIFT 3
1958	#define ISS_FP_UFF (0x1 << ISS_FP_UFF_SHIFT)
1959
1960	#define ISS_FP_OFF_SHIFT 2
1961	#define ISS_FP_OFF (0x1 << ISS_FP_OFF_SHIFT)
1962
1963	#define ISS_FP_DZF_SHIFT 1
1964	#define ISS_FP_DZF (0x1 << ISS_FP_DZF_SHIFT)
1965
1966	#define ISS_FP_IOF_SHIFT 0
1967	#define ISS_FP_IOF (0x1 << ISS_FP_IOF_SHIFT)
1968
1969	/*
1970	* Breakpoint Exception ISS (EL1)
1971	* 24 16 0
1972	* +---------+---------+
1973	* \|000000000\| Comment \|
1974	* +---------+---------+
1975	*
1976	* where:
1977	* Comment: Instruction Comment Field Value
1978	*/
1979	#define ISS_BRK_COMMENT_MASK 0xFFFF
1980	#define ISS_BRK_COMMENT(x) (x & ISS_BRK_COMMENT_MASK)
1981
1982
1983	/*
1984	* SError Interrupt, IDS=1
1985	* 24 23 0
1986	* +---+------------------------+
1987	* \|IDS\| IMPLEMENTATION DEFINED \|
1988	* +---+------------------------+
1989	*
1990	* where:
1991	* IDS: Implementation-defined syndrome (1)
1992	*/
1993
1994	#define ISS_SEI_IDS_SHIFT 24
1995	#define ISS_SEI_IDS (0x1 << ISS_SEI_IDS_SHIFT)
1996
1997
1998	#if HAS_UCNORMAL_MEM
1999	#define ISS_UC 0x11
2000	#endif /* HAS_UCNORMAL_MEM */
2001
2002
2003
2004
2005	/*
2006	* Physical Address Register (EL1)
2007	*/
2008	#define PAR_F_SHIFT 0
2009	#define PAR_F (0x1 << PAR_F_SHIFT)
2010
2011	#define PLATFORM_SYSCALL_TRAP_NO 0x80000000
2012
2013	#define ARM64_SYSCALL_CODE_REG_NUM (16)
2014
2015	#define ARM64_CLINE_SHIFT 6
2016
2017	#if defined(APPLE_ARM64_ARCH_FAMILY)
2018	#define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */
2019	#define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */
2020	#endif
2021
2022	/*
2023	* Timer definitions.
2024	*/
2025	#define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */
2026	#define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */
2027	#define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */
2028	#define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */
2029	#define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */
2030	#define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */
2031	#define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to virtual timebase + frequency reg enabled */
2032	#define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to physical timebase + frequency reg enabled */
2033
2034	#define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */
2035	#define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */
2036	#define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */
2037
2038	#define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS
2039	#define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED
2040	#define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE
2041
2042	#define MIDR_EL1_REV_SHIFT 0
2043	#define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT)
2044	#define MIDR_EL1_PNUM_SHIFT 4
2045	#define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT)
2046	#define MIDR_EL1_ARCH_SHIFT 16
2047	#define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT)
2048	#define MIDR_EL1_VAR_SHIFT 20
2049	#define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT)
2050	#define MIDR_EL1_IMP_SHIFT 24
2051	#define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT)
2052
2053	#define MIDR_FIJI (0x002 << MIDR_EL1_PNUM_SHIFT)
2054	#define MIDR_CAPRI (0x003 << MIDR_EL1_PNUM_SHIFT)
2055	#define MIDR_MAUI (0x004 << MIDR_EL1_PNUM_SHIFT)
2056	#define MIDR_ELBA (0x005 << MIDR_EL1_PNUM_SHIFT)
2057	#define MIDR_CAYMAN (0x006 << MIDR_EL1_PNUM_SHIFT)
2058	#define MIDR_MYST (0x007 << MIDR_EL1_PNUM_SHIFT)
2059	#define MIDR_SKYE_MONSOON (0x008 << MIDR_EL1_PNUM_SHIFT)
2060	#define MIDR_SKYE_MISTRAL (0x009 << MIDR_EL1_PNUM_SHIFT)
2061	#define MIDR_CYPRUS_VORTEX (0x00B << MIDR_EL1_PNUM_SHIFT)
2062	#define MIDR_CYPRUS_TEMPEST (0x00C << MIDR_EL1_PNUM_SHIFT)
2063	#define MIDR_M9 (0x00F << MIDR_EL1_PNUM_SHIFT)
2064	#define MIDR_ARUBA_VORTEX (0x010 << MIDR_EL1_PNUM_SHIFT)
2065	#define MIDR_ARUBA_TEMPEST (0x011 << MIDR_EL1_PNUM_SHIFT)
2066
2067	#ifdef APPLELIGHTNING
2068	#define MIDR_CEBU_LIGHTNING (0x012 << MIDR_EL1_PNUM_SHIFT)
2069	#define MIDR_CEBU_THUNDER (0x013 << MIDR_EL1_PNUM_SHIFT)
2070	#define MIDR_TURKS (0x026 << MIDR_EL1_PNUM_SHIFT)
2071	#endif
2072
2073	#ifdef APPLEFIRESTORM
2074	#define MIDR_SICILY_ICESTORM (0x020 << MIDR_EL1_PNUM_SHIFT)
2075	#define MIDR_SICILY_FIRESTORM (0x021 << MIDR_EL1_PNUM_SHIFT)
2076	#define MIDR_TONGA_ICESTORM (0x022 << MIDR_EL1_PNUM_SHIFT)
2077	#define MIDR_TONGA_FIRESTORM (0x023 << MIDR_EL1_PNUM_SHIFT)
2078	#define MIDR_JADE_CHOP_ICESTORM (0x024 << MIDR_EL1_PNUM_SHIFT)
2079	#define MIDR_JADE_CHOP_FIRESTORM (0x025 << MIDR_EL1_PNUM_SHIFT)
2080	#define MIDR_JADE_DIE_ICESTORM (0x028 << MIDR_EL1_PNUM_SHIFT)
2081	#define MIDR_JADE_DIE_FIRESTORM (0x029 << MIDR_EL1_PNUM_SHIFT)
2082	#endif
2083
2084	#ifdef APPLEAVALANCHE
2085	#define MIDR_ELLIS_BLIZZARD (0x030 << MIDR_EL1_PNUM_SHIFT)
2086	#define MIDR_ELLIS_AVALANCHE (0x031 << MIDR_EL1_PNUM_SHIFT)
2087	#endif
2088
2089
2090
2091
2092	/*
2093	* Apple-ISA-Extensions ID Register.
2094	*/
2095	#define AIDR_MUL53 (1 << 0)
2096	#define AIDR_WKDM (1 << 1)
2097	#define AIDR_ARCHRETENTION (1 << 2)
2098
2099
2100
2101	/*
2102	* CoreSight debug registers
2103	*/
2104	#define CORESIGHT_ED 0
2105	#define CORESIGHT_CTI 1
2106	#define CORESIGHT_PMU 2
2107	#define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */
2108
2109	#define CORESIGHT_OFFSET(x) ((x) * 0x10000)
2110	#define CORESIGHT_REGIONS 4
2111	#define CORESIGHT_SIZE 0x1000
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122	/*
2123	* ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0
2124	*
2125	* 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0
2126	* +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+
2127	* \| rndr \| tlb \| ts \| fhm \| dp \| sm4 \| sm3 \| sha3 \| rdm \| res0 \| atomic \| crc32 \| sha2 \| sha1 \| aes \| res0 \|
2128	* +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+
2129	*/
2130
2131	#define ID_AA64ISAR0_EL1_TS_OFFSET 52
2132	#define ID_AA64ISAR0_EL1_TS_MASK (0xfull << ID_AA64ISAR0_EL1_TS_OFFSET)
2133	#define ID_AA64ISAR0_EL1_TS_FLAGM_EN (1ull << ID_AA64ISAR0_EL1_TS_OFFSET)
2134	#define ID_AA64ISAR0_EL1_TS_FLAGM2_EN (2ull << ID_AA64ISAR0_EL1_TS_OFFSET)
2135
2136	#define ID_AA64ISAR0_EL1_FHM_OFFSET 48
2137	#define ID_AA64ISAR0_EL1_FHM_MASK (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET)
2138	#define ID_AA64ISAR0_EL1_FHM_8_2 (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET)
2139
2140	#define ID_AA64ISAR0_EL1_DP_OFFSET 44
2141	#define ID_AA64ISAR0_EL1_DP_MASK (0xfull << ID_AA64ISAR0_EL1_DP_OFFSET)
2142	#define ID_AA64ISAR0_EL1_DP_EN (1ull << ID_AA64ISAR0_EL1_DP_OFFSET)
2143
2144	#define ID_AA64ISAR0_EL1_SHA3_OFFSET 32
2145	#define ID_AA64ISAR0_EL1_SHA3_MASK (0xfull << ID_AA64ISAR0_EL1_SHA3_OFFSET)
2146	#define ID_AA64ISAR0_EL1_SHA3_EN (1ull << ID_AA64ISAR0_EL1_SHA3_OFFSET)
2147
2148	#define ID_AA64ISAR0_EL1_RDM_OFFSET 28
2149	#define ID_AA64ISAR0_EL1_RDM_MASK (0xfull << ID_AA64ISAR0_EL1_RDM_OFFSET)
2150	#define ID_AA64ISAR0_EL1_RDM_EN (1ull << ID_AA64ISAR0_EL1_RDM_OFFSET)
2151
2152	#define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20
2153	#define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
2154	#define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
2155
2156	#define ID_AA64ISAR0_EL1_CRC32_OFFSET 16
2157	#define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
2158	#define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
2159
2160	#define ID_AA64ISAR0_EL1_SHA2_OFFSET 12
2161	#define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
2162	#define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
2163	#define ID_AA64ISAR0_EL1_SHA2_512_EN (2ull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
2164
2165	#define ID_AA64ISAR0_EL1_SHA1_OFFSET 8
2166	#define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
2167	#define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
2168
2169	#define ID_AA64ISAR0_EL1_AES_OFFSET 4
2170	#define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET)
2171	#define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET)
2172	#define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET)
2173
2174	/*
2175	* ID_AA64ISAR1_EL1 - AArch64 Instruction Set Attribute Register 1
2176	*
2177	* 63 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0
2178	* +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+
2179	* \| res0 \| i8mm \| dgh \| bf16 \| specres \| sb \| frintts \| gpi \| gpa \| lrcpc \| fcma \| jscvt \| api \| apa \| dpb \|
2180	* +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+
2181	*/
2182
2183	#define ID_AA64ISAR1_EL1_I8MM_OFFSET 52
2184	#define ID_AA64ISAR1_EL1_I8MM_MASK (0xfull << ID_AA64ISAR1_EL1_I8MM_OFFSET)
2185	#define ID_AA64ISAR1_EL1_I8MM_EN (1ull << ID_AA64ISAR1_EL1_I8MM_OFFSET)
2186
2187	#define ID_AA64ISAR1_EL1_DGH_OFFSET 48
2188	#define ID_AA64ISAR1_EL1_DGH_MASK (0xfull << ID_AA64ISAR1_EL1_DGH_OFFSET)
2189
2190	#define ID_AA64ISAR1_EL1_BF16_OFFSET 44
2191	#define ID_AA64ISAR1_EL1_BF16_MASK (0xfull << ID_AA64ISAR1_EL1_BF16_OFFSET)
2192	#define ID_AA64ISAR1_EL1_BF16_EN (1ull << ID_AA64ISAR1_EL1_BF16_OFFSET)
2193
2194	#define ID_AA64ISAR1_EL1_SPECRES_OFFSET 40
2195	#define ID_AA64ISAR1_EL1_SPECRES_MASK (0xfull << ID_AA64ISAR1_EL1_SPECRES_OFFSET)
2196	#define ID_AA64ISAR1_EL1_SPECRES_EN (1ull << ID_AA64ISAR1_EL1_SPECRES_OFFSET)
2197
2198	#define ID_AA64ISAR1_EL1_SB_OFFSET 36
2199	#define ID_AA64ISAR1_EL1_SB_MASK (0xfull << ID_AA64ISAR1_EL1_SB_OFFSET)
2200	#define ID_AA64ISAR1_EL1_SB_EN (1ull << ID_AA64ISAR1_EL1_SB_OFFSET)
2201
2202	#define ID_AA64ISAR1_EL1_FRINTTS_OFFSET 32
2203	#define ID_AA64ISAR1_EL1_FRINTTS_MASK (0xfull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET)
2204	#define ID_AA64ISAR1_EL1_FRINTTS_EN (1ull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET)
2205
2206	#define ID_AA64ISAR1_EL1_GPI_OFFSET 28
2207	#define ID_AA64ISAR1_EL1_GPI_MASK (0xfull << ID_AA64ISAR1_EL1_GPI_OFFSET)
2208	#define ID_AA64ISAR1_EL1_GPI_EN (1ull << ID_AA64ISAR1_EL1_GPI_OFFSET)
2209
2210	#define ID_AA64ISAR1_EL1_GPA_OFFSET 24
2211	#define ID_AA64ISAR1_EL1_GPA_MASK (0xfull << ID_AA64ISAR1_EL1_GPA_OFFSET)
2212
2213	#define ID_AA64ISAR1_EL1_LRCPC_OFFSET 20
2214	#define ID_AA64ISAR1_EL1_LRCPC_MASK (0xfull << ID_AA64ISAR1_EL1_LRCPC_OFFSET)
2215	#define ID_AA64ISAR1_EL1_LRCPC_EN (1ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET)
2216	#define ID_AA64ISAR1_EL1_LRCP2C_EN (2ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET)
2217
2218	#define ID_AA64ISAR1_EL1_FCMA_OFFSET 16
2219	#define ID_AA64ISAR1_EL1_FCMA_MASK (0xfull << ID_AA64ISAR1_EL1_FCMA_OFFSET)
2220	#define ID_AA64ISAR1_EL1_FCMA_EN (1ull << ID_AA64ISAR1_EL1_FCMA_OFFSET)
2221
2222	#define ID_AA64ISAR1_EL1_JSCVT_OFFSET 12
2223	#define ID_AA64ISAR1_EL1_JSCVT_MASK (0xfull << ID_AA64ISAR1_EL1_JSCVT_OFFSET)
2224	#define ID_AA64ISAR1_EL1_JSCVT_EN (1ull << ID_AA64ISAR1_EL1_JSCVT_OFFSET)
2225
2226	#define ID_AA64ISAR1_EL1_API_OFFSET 8
2227	#define ID_AA64ISAR1_EL1_API_MASK (0xfull << ID_AA64ISAR1_EL1_API_OFFSET)
2228	#define ID_AA64ISAR1_EL1_API_PAuth_EN (1ull << ID_AA64ISAR1_EL1_API_OFFSET)
2229	#define ID_AA64ISAR1_EL1_API_PAuth2_EN (3ull << ID_AA64ISAR1_EL1_API_OFFSET)
2230	#define ID_AA64ISAR1_EL1_API_FPAC_EN (4ull << ID_AA64ISAR1_EL1_API_OFFSET)
2231	#define ID_AA64ISAR1_EL1_API_FPACCOMBINE (5ull << ID_AA64ISAR1_EL1_API_OFFSET)
2232
2233	#define ID_AA64ISAR1_EL1_APA_OFFSET 4
2234	#define ID_AA64ISAR1_EL1_APA_MASK (0xfull << ID_AA64ISAR1_EL1_APA_OFFSET)
2235
2236	#define ID_AA64ISAR1_EL1_DPB_OFFSET 0
2237	#define ID_AA64ISAR1_EL1_DPB_MASK (0xfull << ID_AA64ISAR1_EL1_DPB_OFFSET)
2238	#define ID_AA64ISAR1_EL1_DPB_EN (1ull << ID_AA64ISAR1_EL1_DPB_OFFSET)
2239	#define ID_AA64ISAR1_EL1_DPB2_EN (2ull << ID_AA64ISAR1_EL1_DPB_OFFSET)
2240
2241	/*
2242	* ID_AA64ISAR2_EL1 - AArch64 Instruction Set Attribute Register 2
2243	*
2244	* 63 8 7 4 3 0
2245	* +------+-------+------+
2246	* \| res0 \| RPRES \| WFxT \|
2247	* +------+-------+------+
2248	*/
2249
2250	#define ID_AA64ISAR2_EL1_RPRES_OFFSET 4
2251	#define ID_AA64ISAR2_EL1_RPRES_MASK (0xfull << ID_AA64ISAR2_EL1_RPRES_OFFSET)
2252	#define ID_AA64ISAR2_EL1_RPRES_EN (1ull << ID_AA64ISAR2_EL1_RPRES_OFFSET)
2253
2254	#define ID_AA64ISAR2_EL1_WFxT_OFFSET 0
2255	#define ID_AA64ISAR2_EL1_WFxT_MASK (0xfull << ID_AA64ISAR2_EL1_WFxT_OFFSET)
2256	#define ID_AA64ISAR2_EL1_WFxT_EN (1ull << ID_AA64ISAR2_EL1_WFxT_OFFSET)
2257
2258	/*
2259	* ID_AA64MMFR0_EL1 - AArch64 Memory Model Feature Register 0
2260	* 63 60 59 56 55 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0
2261	* +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+
2262	* \| ECV \| FGT \| RES0 \| ExS \| TGran4_2 \| TGran64_2 \| TGran16_2 \| TGran4 \| TGran64 \| TGran16 \| BigEndEL0 \| SNSMem \| BigEnd \| ASIDBits \| PARange \|
2263	* +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+
2264	*/
2265
2266	#define ID_AA64MMFR0_EL1_ECV_OFFSET 60
2267	#define ID_AA64MMFR0_EL1_ECV_MASK (0xfull << ID_AA64MMFR0_EL1_ECV_OFFSET)
2268	#define ID_AA64MMFR0_EL1_ECV_EN (1ull << ID_AA64MMFR0_EL1_ECV_OFFSET)
2269
2270	/*
2271	* ID_AA64MMFR2_EL1 - AArch64 Memory Model Feature Register 2
2272	* 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 14 8 7 4 3 0
2273	* +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+
2274	* \| E0PD \| EVT \| BBM \| TTL \| RES0 \| FWB \| IDS \| AT \| ST \| NV \| CCIDX \| VARANGE \| IESB \| LSM \| UAO \| CnP \|
2275	* +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+
2276	*/
2277
2278	#define ID_AA64MMFR2_EL1_AT_OFFSET 32
2279	#define ID_AA64MMFR2_EL1_AT_MASK (0xfull << ID_AA64MMFR2_EL1_AT_OFFSET)
2280	#define ID_AA64MMFR2_EL1_AT_LSE2_EN (1ull << ID_AA64MMFR2_EL1_AT_OFFSET)
2281
2282	/*
2283	* ID_AA64PFR0_EL1 - AArch64 Processor Feature Register 0
2284	* 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0
2285	* +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+
2286	* \| CSV3 \| CSV2 \| RES0 \| DIT \| AMU \| MPAM \| SEL2 \| SVE \| RAS \| GIC \| AdvSIMD \| FP \| EL3 \| EL2 \| EL1 \| EL0 \|
2287	* +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+
2288	*/
2289
2290	#define ID_AA64PFR0_EL1_CSV3_OFFSET 60
2291	#define ID_AA64PFR0_EL1_CSV3_MASK (0xfull << ID_AA64PFR0_EL1_CSV3_OFFSET)
2292	#define ID_AA64PFR0_EL1_CSV3_EN (1ull << ID_AA64PFR0_EL1_CSV3_OFFSET)
2293
2294	#define ID_AA64PFR0_EL1_CSV2_OFFSET 56
2295	#define ID_AA64PFR0_EL1_CSV2_MASK (0xfull << ID_AA64PFR0_EL1_CSV2_OFFSET)
2296	#define ID_AA64PFR0_EL1_CSV2_EN (1ull << ID_AA64PFR0_EL1_CSV2_OFFSET)
2297
2298	#define ID_AA64PFR0_EL1_DIT_OFFSET 48
2299	#define ID_AA64PFR0_EL1_DIT_MASK (0xfull << ID_AA64PFR0_EL1_DIT_OFFSET)
2300	#define ID_AA64PFR0_EL1_DIT_EN (1ull << ID_AA64PFR0_EL1_DIT_OFFSET)
2301
2302	#define ID_AA64PFR0_EL1_AdvSIMD_OFFSET 20
2303	#define ID_AA64PFR0_EL1_AdvSIMD_MASK (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET)
2304	#define ID_AA64PFR0_EL1_AdvSIMD_HPFPCVT (0x0ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET)
2305	#define ID_AA64PFR0_EL1_AdvSIMD_FP16 (0x1ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET)
2306	#define ID_AA64PFR0_EL1_AdvSIMD_DIS (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET)
2307
2308	/*
2309	* ID_AA64PFR1_EL1 - AArch64 Processor Feature Register 1
2310	* 63 20 19 16 15 12 11 8 7 4 3 0
2311	* +----------------------------------+-----------+----------+-------+------+------+
2312	* \| RES0 \| MPAM_frac \| RAS_frac \| MTE \| SSBS \| BT \|
2313	* +----------------------------------+-----------+----------+-------+------+------+
2314	*/
2315
2316
2317	#define ID_AA64PFR1_EL1_SSBS_OFFSET 4
2318	#define ID_AA64PFR1_EL1_SSBS_MASK (0xfull << ID_AA64PFR1_EL1_SSBS_OFFSET)
2319	#define ID_AA64PFR1_EL1_SSBS_EN (1ull << ID_AA64PFR1_EL1_SSBS_OFFSET)
2320
2321	#define ID_AA64PFR1_EL1_BT_OFFSET 0
2322	#define ID_AA64PFR1_EL1_BT_MASK (0xfull << ID_AA64PFR1_EL1_BT_OFFSET)
2323	#define ID_AA64PFR1_EL1_BT_EN (1ull << ID_AA64PFR1_EL1_BT_OFFSET)
2324
2325	/*
2326	* ID_AA64MMFR1_EL1 - AArch64 Memory Model Feature Register 1
2327	*
2328	* 63 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0
2329	* +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+
2330	* \| res0 \| nTLBPA \| AFP \| HCX \| ETS \| TWED \| XNX \| SpecSEI \| PAN \| LO \| HPDS \| VH \| VMIDBits \| HAFDBS \|
2331	* +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+
2332	*/
2333
2334	#define ID_AA64MMFR1_EL1_AFP_OFFSET 44
2335	#define ID_AA64MMFR1_EL1_AFP_MASK (0xfull << ID_AA64MMFR1_EL1_AFP_OFFSET)
2336	#define ID_AA64MMFR1_EL1_AFP_EN (1ull << ID_AA64MMFR1_EL1_AFP_OFFSET)
2337
2338
2339
2340
2341
2342	#define APSTATE_G_SHIFT (0)
2343	#define APSTATE_P_SHIFT (1)
2344	#define APSTATE_A_SHIFT (2)
2345	#define APSTATE_AP_MASK ((1ULL << APSTATE_A_SHIFT) \| (1ULL << APSTATE_P_SHIFT))
2346
2347
2348	#define ACTLR_EL1_EnTSO (1ULL << 1)
2349	#define ACTLR_EL1_EnAPFLG (1ULL << 4)
2350	#define ACTLR_EL1_EnAFP (1ULL << 5)
2351	#define ACTLR_EL1_EnPRSV (1ULL << 6)
2352
2353
2354	#if HAS_USAT_BIT
2355	#define ACTLR_EL1_USAT_OFFSET 0
2356	#define ACTLR_EL1_USAT_MASK (1ULL << ACTLR_EL1_USAT_OFFSET)
2357	#define ACTLR_EL1_USAT ACTLR_EL1_USAT_MASK
2358	#endif
2359
2360
2361
2362
2363
2364
2365	#if defined(HAS_APPLE_PAC)
2366	// The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code
2367	#define PAC_DISCRIMINATOR_RECOVER 0x1e02
2368	#endif
2369
2370
2371	#define CTR_EL0_L1Ip_OFFSET 14
2372	#define CTR_EL0_L1Ip_VIPT (2ULL << CTR_EL0_L1Ip_OFFSET)
2373	#define CTR_EL0_L1Ip_PIPT (3ULL << CTR_EL0_L1Ip_OFFSET)
2374	#define CTR_EL0_L1Ip_MASK (3ULL << CTR_EL0_L1Ip_OFFSET)
2375
2376
2377	#ifdef __ASSEMBLER__
2378
2379	/*
2380	* Conditionally write to system/special-purpose register.
2381	* The register is written to only when the first two arguments
2382	* do not match. If they do match, the macro jumps to a
2383	* caller-provided label.
2384	* The _ISB variant also conditionally issues an ISB after the MSR.
2385	*
2386	* $0 - System/special-purpose register to modify
2387	* $1 - Register containing current FPCR value
2388	* $2 - Register containing expected value
2389	* $3 - Label to jump to when register is already set to expected value
2390	*/
2391	.macro CMSR
2392	cmp $1, $2
2393
2394	/ Skip expensive MSR if not required /
2395	b.eq $3f
2396	msr $0, $2
2397	.endmacro
2398
2399	.macro CMSR_ISB
2400	CMSR $0, $1, $2, $3
2401	isb sy
2402	.endmacro
2403
2404	/*
2405	* Modify FPCR only if it does not contain the XNU default value.
2406	* $0 - Register containing current FPCR value
2407	* $1 - Scratch register
2408	* $2 - Label to jump to when FPCR is already set to default value
2409	*/
2410	.macro SANITIZE_FPCR
2411	mov $1, #FPCR_DEFAULT
2412	CMSR FPCR, $0, $1, $2
2413	.endmacro
2414
2415	/*
2416	* Family of macros that can be used to protect code sections such that they
2417	* are only executed on a particular SoC/Revision/CPU, and skipped otherwise.
2418	* All macros will forward-jump to 1f when the condition is not matched.
2419	* This label may be defined manually, or implicitly through the use of
2420	* the EXEC_END macro.
2421	* For cores, XX can be: EQ (equal), ALL (don't care).
2422	* For revisions, XX can be: EQ (equal), LO (lower than), HS (higher or same), ALL (don't care).
2423	*/
2424
2425	/*
2426	* $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX
2427	* $1 - CPU_VERSION_XX, e.g. CPU_VERSION_B1
2428	* $2 - GPR containing MIDR_EL1 value
2429	* $3 - Scratch register
2430	*/
2431	.macro EXEC_COREEQ_REVEQ
2432	and $3, $2, #MIDR_EL1_PNUM_MASK
2433	cmp $3, $0
2434	b.ne `1f`
2435
2436	mov $3, $2
2437	bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2438	ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2439	cmp $3, $1
2440	b.ne `1f`
2441	.endmacro
2442
2443	.macro EXEC_COREEQ_REVLO
2444	and $3, $2, #MIDR_EL1_PNUM_MASK
2445	cmp $3, $0
2446	b.ne `1f`
2447
2448	mov $3, $2
2449	bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2450	ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2451	cmp $3, $1
2452	b.pl `1f`
2453	.endmacro
2454
2455	.macro EXEC_COREEQ_REVHS
2456	and $3, $2, #MIDR_EL1_PNUM_MASK
2457	cmp $3, $0
2458	b.ne `1f`
2459
2460	mov $3, $2
2461	bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2462	ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2463	cmp $3, $1
2464	b.mi `1f`
2465	.endmacro
2466
2467	/*
2468	* $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1
2469	* $1 - GPR containing MIDR_EL1 value
2470	* $2 - Scratch register
2471	*/
2472	.macro EXEC_COREALL_REVEQ
2473	mov $2, $1
2474	bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2475	ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2476	cmp $2, $0
2477	b.ne `1f`
2478	.endmacro
2479
2480	.macro EXEC_COREALL_REVLO
2481	mov $2, $1
2482	bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2483	ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2484	cmp $2, $0
2485	b.pl `1f`
2486	.endmacro
2487
2488	.macro EXEC_COREALL_REVHS
2489	mov $2, $1
2490	bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2491	ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2492	cmp $2, $0
2493	b.mi `1f`
2494	.endmacro
2495
2496	.macro CMP_FOREACH reg, cc, label, car, cdr:vararg
2497	cmp \reg, \car
2498	b.\cc \label
2499	.ifnb \cdr
2500	CMP_FOREACH \reg, \cc, \label, \cdr
2501	.endif
2502	.endm
2503
2504	.macro EXEC_COREIN_REVALL midr_el1, scratch, midr_list:vararg
2505	and \scratch, \midr_el1, #MIDR_EL1_PNUM_MASK
2506	CMP_FOREACH \scratch, eq, Lmatch\@, \midr_list
2507	b `1f`
2508	Lmatch\@:
2509	.endm
2510
2511	/*
2512	* $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX
2513	* $1 - GPR containing MIDR_EL1 value
2514	* $2 - Scratch register
2515	*/
2516	.macro EXEC_COREEQ_REVALL
2517	and $2, $1, #MIDR_EL1_PNUM_MASK
2518	cmp $2, $0
2519	b.ne `1f`
2520	.endmacro
2521
2522	/*
2523	* $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1
2524	* $1 - GPR containing MIDR_EL1 value
2525	* $2 - Scratch register
2526	*/
2527	.macro EXEC_PCORE_REVEQ
2528	mrs $2, MPIDR_EL1
2529	and $2, $2, #(MPIDR_PNE)
2530	cmp $2, xzr
2531	b.eq `1f`
2532
2533	mov $2, $1
2534	bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2535	ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2536	cmp $2, $0
2537	b.ne `1f`
2538	.endmacro
2539
2540	.macro EXEC_PCORE_REVLO
2541	mrs $2, MPIDR_EL1
2542	and $2, $2, #(MPIDR_PNE)
2543	cmp $2, xzr
2544	b.eq `1f`
2545
2546	mov $2, $1
2547	bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2548	ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2549	cmp $2, $0
2550	b.pl `1f`
2551	.endmacro
2552
2553	.macro EXEC_PCORE_REVHS
2554	mrs $2, MPIDR_EL1
2555	and $2, $2, #(MPIDR_PNE)
2556	cmp $2, xzr
2557	b.eq `1f`
2558
2559	mov $2, $1
2560	bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2561	ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2562	cmp $2, $0
2563	b.mi `1f`
2564	.endmacro
2565
2566	.macro EXEC_ECORE_REVEQ
2567	mrs $2, MPIDR_EL1
2568	and $2, $2, #(MPIDR_PNE)
2569	cmp $2, xzr
2570	b.ne `1f`
2571
2572	mov $2, $1
2573	bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2574	ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2575	cmp $2, $0
2576	b.ne `1f`
2577	.endmacro
2578
2579	.macro EXEC_ECORE_REVLO
2580	mrs $2, MPIDR_EL1
2581	and $2, $2, #(MPIDR_PNE)
2582	cmp $2, xzr
2583	b.ne `1f`
2584
2585	mov $2, $1
2586	bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2587	ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2588	cmp $2, $0
2589	b.pl `1f`
2590	.endmacro
2591
2592	.macro EXEC_ECORE_REVHS
2593	mrs $2, MPIDR_EL1
2594	and $2, $2, #(MPIDR_PNE)
2595	cmp $2, xzr
2596	b.ne `1f`
2597
2598	mov $2, $1
2599	bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`4`
2600	ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - `4`), #`8`
2601	cmp $2, $0
2602	b.mi `1f`
2603	.endmacro
2604
2605	/*
2606	* $0 - GPR containing MIDR_EL1 value
2607	* $1 - Scratch register
2608	*/
2609	.macro EXEC_PCORE_REVALL
2610	mrs $1, MPIDR_EL1
2611	and $1, $1, #(MPIDR_PNE)
2612	cmp $1, xzr
2613	b.eq `1f`
2614	.endmacro
2615
2616	.macro EXEC_ECORE_REVALL
2617	mrs $1, MPIDR_EL1
2618	and $1, $1, #(MPIDR_PNE)
2619	cmp $1, xzr
2620	b.ne `1f`
2621	.endmacro
2622
2623
2624
2625	/*
2626	* Macro that defines the label that all EXEC_COREXX_REVXX macros jump to.
2627	*/
2628	.macro EXEC_END
2629	`1`:
2630	.endmacro
2631
2632	/*
2633	* Wedges CPUs with a specified core that are below a specified revision. This
2634	* macro is intended for CPUs that have been deprecated in iBoot and may have
2635	* incorrect behavior if they continue running xnu.
2636	*/
2637	.macro DEPRECATE_COREEQ_REVLO core, rev, midr_el1, scratch
2638	EXEC_COREEQ_REVLO \core, \rev, \midr_el1, \scratch
2639	/ BEGIN IGNORE CODESTYLE /
2640	b .
2641	/ END IGNORE CODESTYLE /
2642	EXEC_END
2643	.endmacro
2644
2645	/*
2646	* Sets bits in an SPR register.
2647	* arg0: Name of the register to be accessed.
2648	* arg1: Mask of bits to be set.
2649	* arg2: Scratch register
2650	*/
2651	.macro HID_SET_BITS
2652	mrs $2, $0
2653	orr $2, $2, $1
2654	msr $0, $2
2655	.endmacro
2656
2657	/*
2658	* Clears bits in an SPR register.
2659	* arg0: Name of the register to be accessed.
2660	* arg1: Mask of bits to be cleared.
2661	* arg2: Scratch register
2662	*/
2663	.macro HID_CLEAR_BITS
2664	mrs $2, $0
2665	bic $2, $2, $1
2666	msr $0, $2
2667	.endmacro
2668
2669	/*
2670	* Combines the functionality of HID_CLEAR_BITS followed by HID_SET_BITS into
2671	* a single read-modify-write sequence.
2672	* arg0: Name of the register to be accessed.
2673	* arg1: Mask of bits to be cleared.
2674	* arg2: Value to insert
2675	* arg3: Scratch register
2676	*/
2677	.macro HID_INSERT_BITS
2678	mrs $3, $0
2679	bic $3, $3, $1
2680	orr $3, $3, $2
2681	msr $0, $3
2682	.endmacro
2683
2684	/*
2685	* Replaces the value of a field in an implementation-defined system register.
2686	* sreg: system register name
2687	* field: field name within the sysreg, where the assembler symbols
2688	* ARM64_REG_<field>_{shift,width} specify the bounds of the field
2689	* (note that preprocessor macros will not work here)
2690	* value: the value to insert
2691	* scr{1,2}: scratch regs
2692	*/
2693	.macro HID_WRITE_FIELD sreg, field, val, scr1, scr2
2694	mrs \scr1, \sreg
2695	mov \scr2, \val
2696	bfi \scr1, \scr2, ARM64_REG_\sreg\()_\field\()_shift, ARM64_REG_\sreg\()_\field\()_width
2697	msr \sreg, \scr1
2698	.endmacro
2699
2700	/*
2701	* Macro intended to be used as a replacement for ERET.
2702	* It prevents speculation past ERET instructions by padding
2703	* up to the decoder width.
2704	*/
2705	.macro ERET_CONTEXT_SYNCHRONIZING
2706	eret
2707	#if __ARM_SB_AVAILABLE__
2708	sb // Technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures
2709	#else /* __ARM_SB_AVAILABLE__ */
2710	isb // ISB technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures
2711	nop // Sequence of six NOPs to pad out and terminate instruction decode group /*
2712	nop
2713	nop
2714	nop
2715	nop
2716	nop
2717	#endif /* !__ARM_SB_AVAILABLE__ */
2718	.endmacro
2719
2720
2721	#endif /* __ASSEMBLER__ */
2722
2723	#define MSR(reg, src) __asm__ volatile ("msr " reg ", %0" :: "r" (src))
2724	#define MRS(dest, reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest))
2725
2726	#if XNU_MONITOR
2727	#define __ARM_PTE_PHYSMAP__ 1
2728	#define PPL_STATE_KERNEL 0
2729	#define PPL_STATE_DISPATCH 1
2730	#define PPL_STATE_PANIC 2
2731	#define PPL_STATE_EXCEPTION 3
2732	#endif
2733
2734
2735	#endif /* _ARM64_PROC_REG_H_ */
2736

Browse the source code of xnu-build/EXPORT_HDRS/osfmk/arm64/proc_reg.h