1/*
2 * Copyright (c) 2007-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * Processor registers for ARM64
30 */
31#ifndef _ARM64_PROC_REG_H_
32#define _ARM64_PROC_REG_H_
33
34#include <arm/proc_reg.h>
35
36#if __ARM_KERNEL_PROTECT__
37/*
38 * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential
39 * architectural or microarchitectural vulnerabilities that could allow cores to
40 * read/access EL1-only mappings while in EL0 mode. This is achieved by
41 * removing as many mappings as possible when the core transitions to EL0 mode
42 * from EL1 mode, and restoring those mappings when the core transitions to EL1
43 * mode from EL0 mode.
44 *
45 * At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is
46 * used to map and unmap the ordinary kernel mappings, by contracting and
47 * expanding translation zone size for TTBR1 when exiting and entering EL1,
48 * respectively:
49 *
50 * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is
51 * is in EL0.
52 * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in
53 * EL1.
54 *
55 * T1SZ_USER: T1SZ_BOOT + 1
56 * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT)
57 * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER)
58 * TTBR1_EL1_MAX: (2^64) - 1
59 *
60 * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the
61 * the following TTBR1 layout:
62 *
63 * TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX
64 * +---------------------------------------------------------+
65 * | Kernel EL0 Mappings | Kernel EL1 Mappings |
66 * +---------------------------------------------------------+
67 *
68 * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout:
69 *
70 * TTBR1_EL1_BASE_USER TTBR1_EL1_MAX
71 * +---------------------------------------------------------+
72 * | Kernel EL0 Mappings |
73 * +---------------------------------------------------------+
74 *
75 * With the current implementation, both the EL0 and EL1 mappings for the kernel
76 * use otherwise empty translation tables for mapping the exception vectors (so
77 * that we do not need to TLB flush the exception vector address when switching
78 * between EL0 and EL1). The rationale here is that the TLBI would require a
79 * DSB, and DSBs can be extremely expensive.
80 *
81 * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1
82 * ASID. The core switches between ASIDs on EL transitions, so that the TLB
83 * does not need to be fully invalidated on an EL transition.
84 *
85 * Most kernel mappings will be marked non-global in this configuration, as
86 * global mappings would be visible to userspace unless we invalidate them on
87 * eret.
88 */
89#endif /* __ARM_KERNEL_PROTECT */
90
91/*
92 * 64-bit Program Status Register (PSR64)
93 *
94 * 31 27 23 22 21 20 19 10 9 5 4 0
95 * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+
96 * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0| M |
97 * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+
98 *
99 * where:
100 * NZCV Comparison flags
101 * PAN Privileged Access Never
102 * SS Single step
103 * IL Illegal state
104 * DAIF Interrupt masks
105 * M Mode field
106 */
107
108#define PSR64_NZCV_SHIFT 28
109#define PSR64_NZCV_MASK (1 << PSR64_NZCV_SHIFT)
110
111#define PSR64_N_SHIFT 31
112#define PSR64_N (1 << PSR64_N_SHIFT)
113
114#define PSR64_Z_SHIFT 30
115#define PSR64_Z (1 << PSR64_Z_SHIFT)
116
117#define PSR64_C_SHIFT 29
118#define PSR64_C (1 << PSR64_C_SHIFT)
119
120#define PSR64_V_SHIFT 28
121#define PSR64_V (1 << PSR64_V_SHIFT)
122
123#define PSR64_PAN_SHIFT 22
124#define PSR64_PAN (1 << PSR64_PAN_SHIFT)
125
126#define PSR64_SS_SHIFT 21
127#define PSR64_SS (1 << PSR64_SS_SHIFT)
128
129#define PSR64_IL_SHIFT 20
130#define PSR64_IL (1 << PSR64_IL_SHIFT)
131
132/*
133 * msr DAIF, Xn and mrs Xn, DAIF transfer into
134 * and out of bits 9:6
135 */
136#define DAIF_DEBUG_SHIFT 9
137#define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT)
138
139#define DAIF_ASYNC_SHIFT 8
140#define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT)
141
142#define DAIF_IRQF_SHIFT 7
143#define DAIF_IRQF (1 << DAIF_IRQF_SHIFT)
144
145#define DAIF_FIQF_SHIFT 6
146#define DAIF_FIQF (1 << DAIF_FIQF_SHIFT)
147
148#define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
149#define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
150
151#define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF))
152
153/*
154 * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer
155 * from bits 3:0.
156 */
157#define DAIFSC_DEBUGF (1 << 3)
158#define DAIFSC_ASYNCF (1 << 2)
159#define DAIFSC_IRQF (1 << 1)
160#define DAIFSC_FIQF (1 << 0)
161#define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
162#define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
163
164/*
165 * ARM64_TODO: unify with ARM?
166 */
167#define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */
168
169#define PSR64_MODE_MASK 0x1F
170
171#define PSR64_MODE_USER32_THUMB 0x20
172
173#define PSR64_MODE_RW_SHIFT 4
174#define PSR64_MODE_RW_64 0
175#define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT)
176
177#define PSR64_MODE_EL_SHIFT 2
178#define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT)
179#define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT)
180#define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT)
181#define PSR64_MODE_EL0 0
182
183#define PSR64_MODE_SPX 0x1
184#define PSR64_MODE_SP0 0
185
186#define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
187#define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
188#define PSR64_KERNEL_DEFAULT (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0)
189
190#define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL1)
191#define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0)
192
193#define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32))
194#define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32))
195
196
197
198/*
199 * System Control Register (SCTLR)
200 */
201
202#define SCTLR_RESERVED ((3 << 28) | (1 << 22) | (1 << 20) | (1 << 11))
203
204// 26 UCI User Cache Instructions
205#define SCTLR_UCI_ENABLED (1 << 26)
206
207// 25 EE Exception Endianness
208#define SCTLR_EE_BIG_ENDIAN (1 << 25)
209
210// 24 E0E EL0 Endianness
211#define SCTLR_E0E_BIG_ENDIAN (1 << 24)
212
213// 23 SPAN Set PAN
214#define SCTLR_PAN_UNCHANGED (1 << 23)
215
216// 22 RES1 1
217// 21 RES0 0
218// 20 RES1 1
219
220// 19 WXN Writeable implies eXecute Never
221#define SCTLR_WXN_ENABLED (1 << 19)
222
223// 18 nTWE Not trap WFE from EL0
224#define SCTLR_nTWE_WFE_ENABLED (1 << 18)
225
226// 17 RES0 0
227
228// 16 nTWI Not trap WFI from EL0
229#define SCTRL_nTWI_WFI_ENABLED (1 << 16)
230
231// 15 UCT User Cache Type register (CTR_EL0)
232#define SCTLR_UCT_ENABLED (1 << 15)
233
234// 14 DZE User Data Cache Zero (DC ZVA)
235#define SCTLR_DZE_ENABLED (1 << 14)
236
237// 13 PACDB_ENABLED AddPACDB and AuthDB functions enabled
238#define SCTLR_PACDB_ENABLED (1 << 13)
239
240// 12 I Instruction cache enable
241#define SCTLR_I_ENABLED (1 << 12)
242
243// 11 RES1 1
244// 10 RES0 0
245
246// 9 UMA User Mask Access
247#define SCTLR_UMA_ENABLED (1 << 9)
248
249// 8 SED SETEND Disable
250#define SCTLR_SED_DISABLED (1 << 8)
251
252// 7 ITD IT Disable
253#define SCTLR_ITD_DISABLED (1 << 7)
254
255// 6 RES0 0
256
257// 5 CP15BEN CP15 Barrier ENable
258#define SCTLR_CP15BEN_ENABLED (1 << 5)
259
260// 4 SA0 Stack Alignment check for EL0
261#define SCTLR_SA0_ENABLED (1 << 4)
262
263// 3 SA Stack Alignment check
264#define SCTLR_SA_ENABLED (1 << 3)
265
266// 2 C Cache enable
267#define SCTLR_C_ENABLED (1 << 2)
268
269// 1 A Alignment check
270#define SCTLR_A_ENABLED (1 << 1)
271
272// 0 M MMU enable
273#define SCTLR_M_ENABLED (1 << 0)
274
275#define SCTLR_EL1_DEFAULT (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \
276 SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED | \
277 SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED)
278
279/*
280 * Coprocessor Access Control Register (CPACR)
281 *
282 * 31 28 27 22 21 20 19 0
283 * +---+---+------+------+--------------------+
284 * |000|TTA|000000| FPEN |00000000000000000000|
285 * +---+---+------+------+--------------------+
286 *
287 * where:
288 * TTA Trace trap
289 * FPEN Floating point enable
290 */
291#define CPACR_TTA_SHIFT 28
292#define CPACR_TTA (1 << CPACR_TTA_SHIFT)
293
294#define CPACR_FPEN_SHIFT 20
295#define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT)
296#define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT)
297
298/*
299 * FPSR: Floating Point Status Register
300 *
301 * 31 30 29 28 27 26 7 6 4 3 2 1 0
302 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
303 * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC|
304 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+
305 */
306
307#define FPSR_N_SHIFT 31
308#define FPSR_Z_SHIFT 30
309#define FPSR_C_SHIFT 29
310#define FPSR_V_SHIFT 28
311#define FPSR_QC_SHIFT 27
312#define FPSR_IDC_SHIFT 7
313#define FPSR_IXC_SHIFT 4
314#define FPSR_UFC_SHIFT 3
315#define FPSR_OFC_SHIFT 2
316#define FPSR_DZC_SHIFT 1
317#define FPSR_IOC_SHIFT 0
318#define FPSR_N (1 << FPSR_N_SHIFT)
319#define FPSR_Z (1 << FPSR_Z_SHIFT)
320#define FPSR_C (1 << FPSR_C_SHIFT)
321#define FPSR_V (1 << FPSR_V_SHIFT)
322#define FPSR_QC (1 << FPSR_QC_SHIFT)
323#define FPSR_IDC (1 << FPSR_IDC_SHIFT)
324#define FPSR_IXC (1 << FPSR_IXC_SHIFT)
325#define FPSR_UFC (1 << FPSR_UFC_SHIFT)
326#define FPSR_OFC (1 << FPSR_OFC_SHIFT)
327#define FPSR_DZC (1 << FPSR_DZC_SHIFT)
328#define FPSR_IOC (1 << FPSR_IOC_SHIFT)
329
330/*
331 * A mask for all for all of the bits that are not RAZ for FPSR; this
332 * is primarily for converting between a 32-bit view of NEON state
333 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
334 */
335#define FPSR_MASK (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | \
336 FPSR_IDC | FPSR_IXC | FPSR_UFC | FPSR_OFC | \
337 FPSR_DZC | FPSR_IOC)
338
339/*
340 * FPCR: Floating Point Control Register
341 *
342 * 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0
343 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
344 * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000|
345 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+
346 */
347
348#define FPCR_AHP_SHIFT 26
349#define FPCR_DN_SHIFT 25
350#define FPCR_FZ_SHIFT 24
351#define FPCR_RMODE_SHIFT 22
352#define FPCR_STRIDE_SHIFT 20
353#define FPCR_LEN_SHIFT 16
354#define FPCR_IDE_SHIFT 15
355#define FPCR_IXE_SHIFT 12
356#define FPCR_UFE_SHIFT 11
357#define FPCR_OFE_SHIFT 10
358#define FPCR_DZE_SHIFT 9
359#define FPCR_IOE_SHIFT 8
360#define FPCR_AHP (1 << FPCR_AHP_SHIFT)
361#define FPCR_DN (1 << FPCR_DN_SHIFT)
362#define FPCR_FZ (1 << FPCR_FZ_SHIFT)
363#define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT)
364#define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT)
365#define FPCR_LEN (0x7 << FPCR_LEN_SHIFT)
366#define FPCR_IDE (1 << FPCR_IDE_SHIFT)
367#define FPCR_IXE (1 << FPCR_IXE_SHIFT)
368#define FPCR_UFE (1 << FPCR_UFE_SHIFT)
369#define FPCR_OFE (1 << FPCR_OFE_SHIFT)
370#define FPCR_DZE (1 << FPCR_DZE_SHIFT)
371#define FPCR_IOE (1 << FPCR_IOE_SHIFT)
372#define FPCR_DEFAULT (FPCR_DN)
373#define FPCR_DEFAULT_32 (FPCR_DN|FPCR_FZ)
374
375/*
376 * A mask for all for all of the bits that are not RAZ for FPCR; this
377 * is primarily for converting between a 32-bit view of NEON state
378 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR).
379 */
380#define FPCR_MASK (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | \
381 FPCR_STRIDE | FPCR_LEN | FPCR_IDE | FPCR_IXE | \
382 FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE)
383
384/*
385 * Translation Control Register (TCR)
386 *
387 * Legacy:
388 *
389 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0
390 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
391 * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
392 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
393 *
394 * Current (with 16KB granule support):
395 *
396 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0
397 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
398 * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ|
399 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+
400 *
401 * TBI1 Top Byte Ignored for TTBR1 region
402 * TBI0 Top Byte Ignored for TTBR0 region
403 * AS ASID Size
404 * IPS Physical Address Size limit
405 * TG1 Granule Size for TTBR1 region
406 * SH1 Shareability for TTBR1 region
407 * ORGN1 Outer Cacheability for TTBR1 region
408 * IRGN1 Inner Cacheability for TTBR1 region
409 * EPD1 Translation table walk disable for TTBR1
410 * A1 ASID selection from TTBR1 enable
411 * T1SZ Virtual address size for TTBR1
412 * TG0 Granule Size for TTBR0 region
413 * SH0 Shareability for TTBR0 region
414 * ORGN0 Outer Cacheability for TTBR0 region
415 * IRGN0 Inner Cacheability for TTBR0 region
416 * T0SZ Virtual address size for TTBR0
417 */
418
419#define TCR_T0SZ_SHIFT 0ULL
420#define TCR_TSZ_BITS 6ULL
421#define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL)
422
423#define TCR_IRGN0_SHIFT 8ULL
424#define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT)
425#define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT)
426#define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT)
427#define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT)
428
429#define TCR_ORGN0_SHIFT 10ULL
430#define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT)
431#define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT)
432#define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT)
433#define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT)
434
435#define TCR_SH0_SHIFT 12ULL
436#define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT)
437#define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT)
438#define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT)
439
440#define TCR_TG0_GRANULE_SHIFT (14ULL)
441
442#define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT)
443#define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT)
444#define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT)
445
446#if __ARM_16K_PG__
447#define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB)
448#else
449#define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB)
450#endif
451
452#define TCR_T1SZ_SHIFT 16ULL
453
454#define TCR_A1_ASID1 (1ULL << 22ULL)
455#define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL)
456
457#define TCR_IRGN1_SHIFT 24ULL
458#define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT)
459#define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT)
460#define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT)
461#define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT)
462
463#define TCR_ORGN1_SHIFT 26ULL
464#define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT)
465#define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT)
466#define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT)
467#define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT)
468
469#define TCR_SH1_SHIFT 28ULL
470#define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT)
471#define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT)
472#define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT)
473
474#define TCR_TG1_GRANULE_SHIFT 30ULL
475
476#define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT)
477#define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT)
478#define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT)
479
480#if __ARM_16K_PG__
481#define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB)
482#else
483#define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB)
484#endif
485
486#define TCR_IPS_SHIFT 32ULL
487#define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT)
488#define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT)
489#define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT)
490#define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT)
491#define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT)
492#define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT)
493
494#define TCR_AS_16BIT_ASID (1ULL << 36)
495#define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37)
496#define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38)
497
498/*
499 * Multiprocessor Affinity Register (MPIDR_EL1)
500 *
501 * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+
502 * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0|
503 * +---------------------------------+--+-----+--+-----+----+----+
504 *
505 * where
506 * U Uniprocessor
507 * MT Multi-threading at lowest affinity level
508 * Aff2 "1" - PCORE, "0" - ECORE
509 * Aff1 Cluster ID
510 * Aff0 CPU ID
511 */
512#define MPIDR_AFF0_MASK 0xFF
513#define MPIDR_AFF1_MASK 0xFF00
514#define MPIDR_AFF1_SHIFT 8
515#define MPIDR_AFF2_MASK 0xFF0000
516#define MPIDR_AFF2_SHIFT 16
517
518/*
519 * We currently use a 3 level page table (rather than the full 4
520 * level page table). As a result, we do not have the full 48-bits
521 * of address space per TTBR (although the 16KB granule size lets us
522 * get very close).
523 */
524#if __ARM64_TWO_LEVEL_PMAP__ && !__ARM_16K_PG__
525#error ARM64 does not currently support a 2 level page table with 4KB pages
526#endif /* __ARM64_TWO_LEVEL_PMAP__ */
527
528/*
529 * TXSZ indicates the size of the range a TTBR covers. Currently,
530 * we support the following:
531 *
532 * 4KB pages, full page L1: 39 bit range.
533 * 4KB pages, sub-page L1: 38 bit range.
534 * 16KB pages, full page L1: 47 bit range.
535 * 16KB pages, sub-page L1: 39 bit range.
536 * 16KB pages, two level page tables: 36 bit range.
537 */
538#if __ARM_KERNEL_PROTECT__
539/*
540 * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the
541 * address space is used for the mappings that will remain in place when in EL0.
542 * As a result, 1 bit less of address space is available to the rest of the
543 * the kernel.
544 */
545#endif /* __ARM_KERNEL_PROTECT__ */
546#ifdef __ARM_16K_PG__
547#if __ARM64_TWO_LEVEL_PMAP__
548#define T0SZ_BOOT 28ULL
549#elif __ARM64_PMAP_SUBPAGE_L1__
550#define T0SZ_BOOT 25ULL
551#else /* __ARM64_TWO_LEVEL_PMAP__ */
552#define T0SZ_BOOT 17ULL
553#endif /* __ARM64_TWO_LEVEL_PMAP__ */
554#else /* __ARM_16K_PG__ */
555#if __ARM64_PMAP_SUBPAGE_L1__
556#define T0SZ_BOOT 26ULL
557#else /* __ARM64_PMAP_SUBPAGE_L1__ */
558#define T0SZ_BOOT 25ULL
559#endif /* __ARM64_PMAP_SUBPAGE_L1__ */
560#endif /* __ARM_16K_PG__ */
561
562#if defined(APPLE_ARM64_ARCH_FAMILY)
563/* T0SZ must be the same as T1SZ */
564#define T1SZ_BOOT T0SZ_BOOT
565#else /* defined(APPLE_ARM64_ARCH_FAMILY) */
566#ifdef __ARM_16K_PG__
567#if __ARM64_TWO_LEVEL_PMAP__
568#define T1SZ_BOOT 28ULL
569#elif __ARM64_PMAP_SUBPAGE_L1__
570#define T1SZ_BOOT 25ULL
571#else /* __ARM64_TWO_LEVEL_PMAP__ */
572#define T1SZ_BOOT 17ULL
573#endif /* __ARM64_TWO_LEVEL_PMAP__ */
574#else /* __ARM_16K_PG__ */
575#if __ARM64_PMAP_SUBPAGE_L1__
576#define T1SZ_BOOT 26ULL
577#else /* __ARM64_PMAP_SUBPAGE_L1__ */
578#define T1SZ_BOOT 25ULL
579#endif /*__ARM64_PMAP_SUBPAGE_L1__*/
580#endif /* __ARM_16K_PG__ */
581#endif /* defined(APPLE_ARM64_ARCH_FAMILY) */
582
583#define TCR_EL1_BASE (TCR_IPS_40BITS | \
584 TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE) |\
585 TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE))
586
587#if __ARM_KERNEL_PROTECT__
588#define TCR_EL1_BOOT (TCR_EL1_BASE | \
589 (T1SZ_BOOT << TCR_T1SZ_SHIFT) | TCR_TBI0_TOPBYTE_IGNORED)
590#define T1SZ_USER (T1SZ_BOOT + 1)
591#define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | TCR_TBI0_TOPBYTE_IGNORED)
592#else
593#define TCR_EL1_BOOT (TCR_EL1_BASE | \
594 (T1SZ_BOOT << TCR_T1SZ_SHIFT))
595#endif /* __ARM_KERNEL_PROTECT__ */
596
597/*
598 * Translation Table Base Register (TTBR)
599 *
600 * 63 48 47 x x-1 0
601 * +--------+------------------+------+
602 * | ASID | Base Address | zero |
603 * +--------+------------------+------+
604 *
605 */
606#define TTBR_ASID_SHIFT 48
607#define TTBR_ASID_MASK 0xffff000000000000
608
609#define TTBR_BADDR_MASK 0x0000ffffffffffff
610
611/*
612 * Memory Attribute Indirection Register
613 *
614 * 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0
615 * +-------+-------+-------+-------+-------+-------+-------+-------+
616 * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 |
617 * +-------+-------+-------+-------+-------+-------+-------+-------+
618 *
619 */
620
621#define MAIR_ATTR_SHIFT(x) (8*(x))
622
623/* Strongly ordered or device memory attributes */
624#define MAIR_OUTER_STRONGLY_ORDERED 0x0
625#define MAIR_OUTER_DEVICE 0x0
626
627#define MAIR_INNER_STRONGLY_ORDERED 0x0
628#define MAIR_INNER_DEVICE 0x4
629
630/* Normal memory attributes */
631#define MAIR_OUTER_NON_CACHEABLE 0x40
632#define MAIR_OUTER_WRITE_THROUGH 0x80
633#define MAIR_OUTER_WRITE_BACK 0xc0
634
635#define MAIR_INNER_NON_CACHEABLE 0x4
636#define MAIR_INNER_WRITE_THROUGH 0x8
637#define MAIR_INNER_WRITE_BACK 0xc
638
639/* Allocate policy for cacheable memory */
640#define MAIR_OUTER_WRITE_ALLOCATE 0x10
641#define MAIR_OUTER_READ_ALLOCATE 0x20
642
643#define MAIR_INNER_WRITE_ALLOCATE 0x1
644#define MAIR_INNER_READ_ALLOCATE 0x2
645
646/* Memory Atribute Encoding */
647
648/* Device memory types:
649 G (gathering): multiple reads/writes can be combined
650 R (reordering): reads or writes may reach device out of program order
651 E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes)
652*/
653#define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */
654#define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */
655#define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */
656#define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */
657#define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */
658#define MAIR_INNERWRITEBACK 0x4F /* Normal Memory, Outer Non-Cacheable, Inner Write-back */
659
660
661/*
662 * ARM 4-level Page Table support - 2*1024TB (2^48) of address space
663 */
664
665
666/*
667 * Memory Attribute Index
668 */
669#define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled */
670#define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes */
671#define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled */
672#define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer */
673#define CACHE_ATTRINDX_INNERWRITEBACK 0x4 /* inner cache enabled, buffer enabled, write allocate */
674#define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes */
675#define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK
676
677/*
678 * Access protection bit values (TTEs and PTEs)
679 */
680#define AP_RWNA 0x0 /* priv=read-write, user=no-access */
681#define AP_RWRW 0x1 /* priv=read-write, user=read-write */
682#define AP_RONA 0x2 /* priv=read-only, user=no-access */
683#define AP_RORO 0x3 /* priv=read-only, user=read-only */
684#define AP_MASK 0x3 /* mask to find ap bits */
685
686/*
687 * Shareability attributes
688 */
689#define SH_NONE 0x0 /* Non shareable */
690#define SH_NONE 0x0 /* Device shareable */
691#define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */
692#define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */
693#define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */
694
695
696/*
697 * ARM Page Granule
698 */
699#ifdef __ARM_16K_PG__
700#define ARM_PGSHIFT 14
701#else
702#define ARM_PGSHIFT 12
703#endif
704#define ARM_PGBYTES (1 << ARM_PGSHIFT)
705#define ARM_PGMASK (ARM_PGBYTES-1)
706
707
708/*
709 * L0 Translation table
710 *
711 * 4KB granule size:
712 * Each translation table is 4KB
713 * 512 64-bit entries of 512GB (2^39) of address space.
714 * Covers 256TB (2^48) of address space.
715 *
716 * 16KB granule size:
717 * Each translation table is 16KB
718 * 2 64-bit entries of 128TB (2^47) of address space.
719 * Covers 256TB (2^48) of address space.
720 */
721
722#ifdef __ARM_16K_PG__
723#define ARM_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */
724#define ARM_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */
725#define ARM_TT_L0_SHIFT 47 /* page descriptor shift */
726#define ARM_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */
727#else
728#define ARM_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */
729#define ARM_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */
730#define ARM_TT_L0_SHIFT 39 /* page descriptor shift */
731#define ARM_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */
732#endif
733
734/*
735 * L1 Translation table
736 *
737 * 4KB granule size:
738 * Each translation table is 4KB
739 * 512 64-bit entries of 1GB (2^30) of address space.
740 * Covers 512GB (2^39) of address space.
741 *
742 * 16KB granule size:
743 * Each translation table is 16KB
744 * 2048 64-bit entries of 64GB (2^36) of address space.
745 * Covers 128TB (2^47) of address space.
746 */
747
748#ifdef __ARM_16K_PG__
749#define ARM_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */
750#define ARM_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */
751#define ARM_TT_L1_SHIFT 36 /* page descriptor shift */
752#ifdef __ARM64_PMAP_SUBPAGE_L1__
753/* This config supports 512GB per TTBR. */
754#define ARM_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */
755#else /* __ARM64_PMAP_SUBPAGE_L1__ */
756#define ARM_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */
757#endif /* __ARM64_PMAP_SUBPAGE_L1__ */
758#else /* __ARM_16K_PG__ */
759#define ARM_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */
760#define ARM_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */
761#define ARM_TT_L1_SHIFT 30 /* page descriptor shift */
762#ifdef __ARM64_PMAP_SUBPAGE_L1__
763/* This config supports 256GB per TTBR. */
764#define ARM_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */
765#else /* __ARM64_PMAP_SUBPAGE_L1__ */
766#define ARM_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */
767#endif /* __ARM64_PMAP_SUBPAGE_L1__ */
768#endif
769
770/* some sugar for getting pointers to page tables and entries */
771
772#define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT)
773#define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT)
774#define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT)
775
776#define L2_TABLE_VA(tte) ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK))
777#define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK))
778
779/*
780 * L2 Translation table
781 *
782 * 4KB granule size:
783 * Each translation table is 4KB
784 * 512 64-bit entries of 2MB (2^21) of address space.
785 * Covers 1GB (2^30) of address space.
786 *
787 * 16KB granule size:
788 * Each translation table is 16KB
789 * 2048 64-bit entries of 32MB (2^25) of address space.
790 * Covers 64GB (2^36) of address space.
791 */
792
793#ifdef __ARM_16K_PG__
794#define ARM_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */
795#define ARM_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */
796#define ARM_TT_L2_SHIFT 25 /* page descriptor shift */
797#define ARM_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */
798#else
799#define ARM_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */
800#define ARM_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */
801#define ARM_TT_L2_SHIFT 21 /* page descriptor shift */
802#define ARM_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */
803#endif
804
805/*
806 * L3 Translation table
807 *
808 * 4KB granule size:
809 * Each translation table is 4KB
810 * 512 64-bit entries of 4KB (2^12) of address space.
811 * Covers 2MB (2^21) of address space.
812 *
813 * 16KB granule size:
814 * Each translation table is 16KB
815 * 2048 64-bit entries of 16KB (2^14) of address space.
816 * Covers 32MB (2^25) of address space.
817 */
818
819#ifdef __ARM_16K_PG__
820#define ARM_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */
821#define ARM_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */
822#define ARM_TT_L3_SHIFT 14 /* page descriptor shift */
823#define ARM_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */
824#else
825#define ARM_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */
826#define ARM_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */
827#define ARM_TT_L3_SHIFT 12 /* page descriptor shift */
828#define ARM_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */
829#endif
830
831/*
832 * Convenience definitions for:
833 * ARM_TT_LEAF: The last level of the configured page table format.
834 * ARM_TT_TWIG: The second to last level of the configured page table format.
835 * ARM_TT_ROOT: The first level of the configured page table format.
836 *
837 * My apologies to any botanists who may be reading this.
838 */
839#define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE
840#define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK
841#define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT
842#define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK
843
844#define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE
845#define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK
846#define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT
847#define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK
848
849#if __ARM64_TWO_LEVEL_PMAP__
850#define ARM_TT_ROOT_SIZE ARM_TT_L2_SIZE
851#define ARM_TT_ROOT_OFFMASK ARM_TT_L2_OFFMASK
852#define ARM_TT_ROOT_SHIFT ARM_TT_L2_SHIFT
853#define ARM_TT_ROOT_INDEX_MASK ARM_TT_L2_INDEX_MASK
854#else
855#define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE
856#define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK
857#define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT
858#define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK
859#endif
860
861/*
862 * 4KB granule size:
863 *
864 * Level 0 Translation Table Entry
865 *
866 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
867 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
868 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V|
869 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
870 *
871 * Level 1 Translation Table Entry
872 *
873 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
874 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
875 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V|
876 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
877 *
878 * Level 1 Translation Block Entry
879 *
880 * 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0
881 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
882 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
883 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
884 *
885 * Level 2 Translation Table Entry
886 *
887 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
888 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
889 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V|
890 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
891 *
892 * Level 2 Translation Block Entry
893 *
894 * 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0
895 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
896 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
897 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
898 *
899 * 16KB granule size:
900 *
901 * Level 0 Translation Table Entry
902 *
903 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
904 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
905 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V|
906 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
907 *
908 * Level 1 Translation Table Entry
909 *
910 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
911 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
912 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V|
913 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
914 *
915 * Level 2 Translation Table Entry
916 *
917 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0
918 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
919 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V|
920 * +--+-----+--+---+-------+------+----------------------+-------+-+-+
921 *
922 * Level 2 Translation Block Entry
923 *
924 * 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0
925 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
926 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V|
927 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+
928 *
929 * where:
930 * 'nG' notGlobal bit
931 * 'SH' Shareability field
932 * 'AP' access protection
933 * 'XN' eXecute Never bit
934 * 'PXN' Privilege eXecute Never bit
935 * 'NS' Non-Secure bit
936 * 'HINT' 16 entry continuguous output hint
937 * 'AttrIdx' Memory Attribute Index
938 */
939
940#define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */
941#ifdef __ARM_16K_PG__
942#define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */
943#else
944#define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */
945#endif
946
947#define ARM_TTE_MAX (TTE_PGENTRIES)
948
949#define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */
950#define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */
951
952#define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */
953
954#define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */
955#define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */
956#define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */
957#define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL
958#define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */
959
960#ifdef __ARM_16K_PG__
961/* Note that L0/L1 block entries are disallowed for the 16KB granule size; what are we doing with these? */
962#define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */
963#define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */
964#define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */
965#define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */
966#define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */
967#define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */
968#else
969#define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */
970#define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */
971#define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */
972#define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */
973#define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */
974#define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */
975#endif
976
977#define ARM_TTE_BLOCK_APSHIFT 6
978#define ARM_TTE_BLOCK_AP(x) ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */
979#define ARM_TTE_BLOCK_APMASK (0x3 << ARM_TTE_BLOCK_APSHIFT)
980
981#define ARM_TTE_BLOCK_ATTRINDX(x) ((x) << 2) /* memory attributes index */
982#define ARM_TTE_BLOCK_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */
983
984#define ARM_TTE_BLOCK_SH(x) ((x) << 8) /* access shared */
985#define ARM_TTE_BLOCK_SHMASK (0x3ULL << 8) /* mask access shared */
986
987#define ARM_TTE_BLOCK_AF 0x0000000000000400ULL /* value for access */
988#define ARM_TTE_BLOCK_AFMASK 0x0000000000000400ULL /* access mask */
989
990#define ARM_TTE_BLOCK_NG 0x0000000000000800ULL /* value for a global mapping */
991#define ARM_TTE_BLOCK_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */
992
993#define ARM_TTE_BLOCK_NS 0x0000000000000020ULL /* value for a secure mapping */
994#define ARM_TTE_BLOCK_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */
995
996#define ARM_TTE_BLOCK_PNX 0x0020000000000000ULL /* value for privilege no execute bit */
997#define ARM_TTE_BLOCK_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */
998
999#define ARM_TTE_BLOCK_NX 0x0040000000000000ULL /* value for no execute */
1000#define ARM_TTE_BLOCK_NXMASK 0x0040000000000000ULL /* no execute mask */
1001
1002#define ARM_TTE_BLOCK_WIRED 0x0080000000000000ULL /* value for software wired bit */
1003#define ARM_TTE_BLOCK_WIREDMASK 0x0080000000000000ULL /* software wired mask */
1004
1005#define ARM_TTE_BLOCK_WRITEABLE 0x0100000000000000ULL /* value for software writeable bit */
1006#define ARM_TTE_BLOCK_WRITEABLEMASK 0x0100000000000000ULL /* software writeable mask */
1007
1008#ifdef __ARM_16K_PG__
1009/*
1010 * TODO: Do we care about the low bits being unused? It should technically work either way, but masking them out should be future proof;
1011 * it is only a matter of time before someone wants to shove something into the free bits.
1012 */
1013#define ARM_TTE_TABLE_MASK (0x0000ffffffffc000ULL) /* mask for extracting pointer to next table (works at any level) */
1014#else
1015#define ARM_TTE_TABLE_MASK (0x0000fffffffff000ULL) /* mask for extracting pointer to next table (works at any level) */
1016#endif
1017
1018#define ARM_TTE_TABLE_APSHIFT 61
1019#define ARM_TTE_TABLE_AP(x) ((x)<<TTE_BLOCK_APSHIFT) /* access protection */
1020
1021#define ARM_TTE_TABLE_NS 0x8000000000000020ULL /* value for a secure mapping */
1022#define ARM_TTE_TABLE_NS_MASK 0x8000000000000020ULL /* notSecure mapping mask */
1023
1024#define ARM_TTE_TABLE_XN 0x1000000000000000ULL /* value for no execute */
1025#define ARM_TTE_TABLE_XNMASK 0x1000000000000000ULL /* no execute mask */
1026
1027#define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */
1028#define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */
1029
1030#if __ARM_KERNEL_PROTECT__
1031#define ARM_TTE_BOOT_BLOCK (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) \
1032 | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF \
1033 | ARM_TTE_BLOCK_NG)
1034#else /* __ARM_KERNEL_PROTECT__ */
1035#define ARM_TTE_BOOT_BLOCK (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) \
1036 | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF)
1037#endif /* __ARM_KERNEL_PROTECT__ */
1038
1039#define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID )
1040/*
1041 * L3 Translation table
1042 *
1043 * 4KB granule size:
1044 * Each translation table is 4KB
1045 * 512 64-bit entries of 4KB (2^12) of address space.
1046 * Covers 2MB (2^21) of address space.
1047 *
1048 * 16KB granule size:
1049 * Each translation table is 16KB
1050 * 2048 64-bit entries of 16KB (2^14) of address space.
1051 * Covers 32MB (2^25) of address space.
1052 */
1053
1054#ifdef __ARM_16K_PG__
1055#define ARM_PTE_SIZE 0x0000000000004000ULL /* size of area covered by a tte */
1056#define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */
1057#define ARM_PTE_SHIFT 14 /* page descriptor shift */
1058#define ARM_PTE_MASK 0x0000ffffffffc000ULL /* mask for output address in PTE */
1059#else
1060#define ARM_PTE_SIZE 0x0000000000001000ULL /* size of area covered by a tte */
1061#define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */
1062#define ARM_PTE_SHIFT 12 /* page descriptor shift */
1063#define ARM_PTE_MASK 0x0000fffffffff000ULL /* mask for output address in PTE */
1064#endif
1065
1066/*
1067 * L3 Page table entries
1068 *
1069 * The following page table entry types are possible:
1070 *
1071 * fault page entry
1072 * 63 2 0
1073 * +------------------------------+--+
1074 * | ignored |00|
1075 * +------------------------------+--+
1076 *
1077 *
1078 * 63 59 58 55 54 53 52 51 48 47 12 11 10 9 8 7 6 5 4 2 1 0
1079 * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1080 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V|
1081 * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+
1082 *
1083 * where:
1084 * 'nG' notGlobal bit
1085 * 'SH' Shareability field
1086 * 'AP' access protection
1087 * 'XN' eXecute Never bit
1088 * 'PXN' Privilege eXecute Never bit
1089 * 'NS' Non-Secure bit
1090 * 'HINT' 16 entry continuguous output hint
1091 * 'AttrIdx' Memory Attribute Index
1092 */
1093
1094#define PTE_SHIFT 3 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */
1095#ifdef __ARM_16K_PG__
1096#define PTE_PGENTRIES (16384 >> PTE_SHIFT) /* number of ptes per page */
1097#else
1098#define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */
1099#endif
1100
1101#define ARM_PTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */
1102
1103/* markers for (invalid) PTE for a page sent to compressor */
1104#define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */
1105#define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */
1106#define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL
1107#define ARM_PTE_IS_COMPRESSED(x) \
1108 ((((x) & 0x3) == 0) && /* PTE is not valid... */ \
1109 ((x) & ARM_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \
1110 ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \
1111 (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \
1112 &(x), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE)))
1113
1114#define ARM_PTE_TYPE 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1115#define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */
1116#define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */
1117#define ARM_PTE_TYPE_MASK 0x0000000000000002ULL /* mask to get pte type */
1118
1119#ifdef __ARM_16K_PG__
1120/* TODO: What does the shift mean here? */
1121#define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFC000ULL /* mask for 16KB page */
1122#else
1123#define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* mask for 4KB page */
1124#define ARM_PTE_PAGE_SHIFT 12 /* page shift for 4KB page */
1125#endif
1126
1127#define ARM_PTE_AP(x) ((x) << 6) /* access protections */
1128#define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */
1129#define ARM_PTE_EXTRACT_AP(x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */
1130
1131#define ARM_PTE_ATTRINDX(x) ((x) << 2) /* memory attributes index */
1132#define ARM_PTE_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */
1133
1134#define ARM_PTE_SH(x) ((x) << 8) /* access shared */
1135#define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */
1136
1137#define ARM_PTE_AF 0x0000000000000400ULL /* value for access */
1138#define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */
1139
1140#define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */
1141#define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */
1142
1143#define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */
1144#define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */
1145
1146#define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */
1147#define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */
1148
1149#if __ARM_16K_PG__
1150#define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */
1151#define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */
1152#define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */
1153#define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */
1154#define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */
1155#else
1156#define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */
1157#define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */
1158#define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */
1159#define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */
1160#define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */
1161#endif
1162
1163#define ARM_PTE_PNX 0x0020000000000000ULL /* value for privilege no execute bit */
1164#define ARM_PTE_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */
1165
1166#define ARM_PTE_NX 0x0040000000000000ULL /* value for no execute bit */
1167#define ARM_PTE_NXMASK 0x0040000000000000ULL /* no execute mask */
1168
1169#define ARM_PTE_WIRED 0x0080000000000000ULL /* value for software wired bit */
1170#define ARM_PTE_WIRED_MASK 0x0080000000000000ULL /* software wired mask */
1171
1172#define ARM_PTE_WRITEABLE 0x0100000000000000ULL /* value for software writeable bit */
1173#define ARM_PTE_WRITEABLE_MASK 0x0100000000000000ULL /* software writeable mask */
1174
1175#if CONFIG_PGTRACE
1176#define ARM_PTE_PGTRACE 0x0200000000000000ULL /* value for software trace bit */
1177#define ARM_PTE_PGTRACE_MASK 0x0200000000000000ULL /* software trace mask */
1178#endif
1179
1180#define ARM_PTE_BOOT_PAGE_BASE (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) \
1181 | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF)
1182
1183#if __ARM_KERNEL_PROTECT__
1184#define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG)
1185#else /* __ARM_KERNEL_PROTECT__ */
1186#define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE)
1187#endif /* __ARM_KERNEL_PROTECT__ */
1188
1189/*
1190 * TLBI appers to only deal in 4KB page addresses, so give
1191 * it an explicit shift of 12.
1192 */
1193#define TLBI_ADDR_SIZE (44)
1194#define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1)
1195#define TLBI_ADDR_SHIFT (12)
1196#define TLBI_ASID_SHIFT (48)
1197#define TLBI_ASID_SIZE (16)
1198#define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1) << TLBI_ASID_SHIFT)
1199
1200/*
1201 * Exception Syndrome Register
1202 *
1203 * 31 26 25 24 0
1204 * +------+--+------------------+
1205 * | EC |IL| ISS |
1206 * +------+--+------------------+
1207 *
1208 * EC - Exception Class
1209 * IL - Instruction Length
1210 * ISS- Instruction Specific Syndrome
1211 *
1212 * Note: The ISS can have many forms. These are defined separately below.
1213 */
1214
1215#define ESR_EC_SHIFT 26
1216#define ESR_EC_MASK (0x3F << ESR_EC_SHIFT)
1217#define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT)
1218
1219#define ESR_IL_SHIFT 25
1220#define ESR_IL (1 << ESR_IL_SHIFT)
1221
1222#define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL))
1223
1224#define ESR_ISS_MASK 0x01FFFFFF
1225#define ESR_ISS(x) (x & ESR_ISS_MASK)
1226
1227#ifdef __ASSEMBLER__
1228/* Define only the classes we need to test in the exception vectors. */
1229#define ESR_EC_IABORT_EL1 0x21
1230#define ESR_EC_DABORT_EL1 0x25
1231#define ESR_EC_SP_ALIGN 0x26
1232#else
1233typedef enum {
1234 ESR_EC_UNCATEGORIZED = 0x00,
1235 ESR_EC_WFI_WFE = 0x01,
1236 ESR_EC_MCR_MRC_CP15_TRAP = 0x03,
1237 ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04,
1238 ESR_EC_MCR_MRC_CP14_TRAP = 0x05,
1239 ESR_EC_LDC_STC_CP14_TRAP = 0x06,
1240 ESR_EC_TRAP_SIMD_FP = 0x07,
1241 ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c,
1242 ESR_EC_ILLEGAL_INSTR_SET = 0x0e,
1243 ESR_EC_SVC_32 = 0x11,
1244 ESR_EC_SVC_64 = 0x15,
1245 ESR_EC_MSR_TRAP = 0x18,
1246 ESR_EC_IABORT_EL0 = 0x20,
1247 ESR_EC_IABORT_EL1 = 0x21,
1248 ESR_EC_PC_ALIGN = 0x22,
1249 ESR_EC_DABORT_EL0 = 0x24,
1250 ESR_EC_DABORT_EL1 = 0x25,
1251 ESR_EC_SP_ALIGN = 0x26,
1252 ESR_EC_FLOATING_POINT_32 = 0x28,
1253 ESR_EC_FLOATING_POINT_64 = 0x2C,
1254 ESR_EC_BKPT_REG_MATCH_EL0 = 0x30, // Breakpoint Debug event taken to the EL from a lower EL.
1255 ESR_EC_BKPT_REG_MATCH_EL1 = 0x31, // Breakpoint Debug event taken to the EL from the EL.
1256 ESR_EC_SW_STEP_DEBUG_EL0 = 0x32, // Software Step Debug event taken to the EL from a lower EL.
1257 ESR_EC_SW_STEP_DEBUG_EL1 = 0x33, // Software Step Debug event taken to the EL from the EL.
1258 ESR_EC_WATCHPT_MATCH_EL0 = 0x34, // Watchpoint Debug event taken to the EL from a lower EL.
1259 ESR_EC_WATCHPT_MATCH_EL1 = 0x35, // Watchpoint Debug event taken to the EL from the EL.
1260 ESR_EC_BKPT_AARCH32 = 0x38,
1261 ESR_EC_BRK_AARCH64 = 0x3C
1262} esr_exception_class_t;
1263
1264typedef enum {
1265 FSC_TRANSLATION_FAULT_L0 = 0x04,
1266 FSC_TRANSLATION_FAULT_L1 = 0x05,
1267 FSC_TRANSLATION_FAULT_L2 = 0x06,
1268 FSC_TRANSLATION_FAULT_L3 = 0x07,
1269 FSC_ACCESS_FLAG_FAULT_L1 = 0x09,
1270 FSC_ACCESS_FLAG_FAULT_L2 = 0x0A,
1271 FSC_ACCESS_FLAG_FAULT_L3 = 0x0B,
1272 FSC_PERMISSION_FAULT_L1 = 0x0D,
1273 FSC_PERMISSION_FAULT_L2 = 0x0E,
1274 FSC_PERMISSION_FAULT_L3 = 0x0F,
1275 FSC_SYNC_EXT_ABORT = 0x10,
1276 FSC_ASYNC_EXT_ABORT = 0x11,
1277 FSC_SYNC_EXT_ABORT_TT_L1 = 0x15,
1278 FSC_SYNC_EXT_ABORT_TT_L2 = 0x16,
1279 FSC_SYNC_EXT_ABORT_TT_L3 = 0x17,
1280 FSC_SYNC_PARITY = 0x18,
1281 FSC_ASYNC_PARITY = 0x19,
1282 FSC_SYNC_PARITY_TT_L1 = 0x1D,
1283 FSC_SYNC_PARITY_TT_L2 = 0x1E,
1284 FSC_SYNC_PARITY_TT_L3 = 0x1F,
1285 FSC_ALIGNMENT_FAULT = 0x21,
1286 FSC_DEBUG_FAULT = 0x22
1287} fault_status_t;
1288#endif /* ASSEMBLER */
1289
1290/*
1291 * Software step debug event ISS (EL1)
1292 * 24 23 6 5 0
1293 * +---+-----------------+--+------+
1294 * |ISV|00000000000000000|EX| IFSC |
1295 * +---+-----------------+--+------+
1296 *
1297 * where:
1298 * ISV Instruction syndrome valid
1299 * EX Exclusive access
1300 * IFSC Instruction Fault Status Code
1301 */
1302
1303#define ISS_SSDE_ISV_SHIFT 24
1304#define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT)
1305
1306#define ISS_SSDE_EX_SHIFT 6
1307#define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT)
1308
1309#define ISS_SSDE_FSC_MASK 0x3F
1310#define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK)
1311
1312/*
1313 * Instruction Abort ISS (EL1)
1314 * 24 10 9 5 0
1315 * +---------------+--+---+------+
1316 * |000000000000000|EA|000| IFSC |
1317 * +---------------+--+---+------+
1318 *
1319 * where:
1320 * EA External Abort type
1321 * IFSC Instruction Fault Status Code
1322 */
1323
1324#define ISS_IA_EA_SHIFT 9
1325#define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT)
1326
1327#define ISS_IA_FSC_MASK 0x3F
1328#define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK)
1329
1330
1331/*
1332 * Data Abort ISS (EL1)
1333 *
1334 * 24 9 8 7 6 5 0
1335 * +---------------+--+--+-+---+----+
1336 * |000000000000000|EA|CM|0|WnR|DFSC|
1337 * +---------------+--+--+-+---+----+
1338 *
1339 * where:
1340 * EA External Abort type
1341 * CM Cache Maintenance operation
1342 * WnR Write not Read
1343 * DFSC Data Fault Status Code
1344 */
1345#define ISS_DA_EA_SHIFT 9
1346#define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT)
1347
1348#define ISS_DA_CM_SHIFT 8
1349#define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT)
1350
1351#define ISS_DA_WNR_SHIFT 6
1352#define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT)
1353
1354#define ISS_DA_FSC_MASK 0x3F
1355#define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK)
1356
1357/*
1358 * Physical Address Register (EL1)
1359 */
1360#define PAR_F_SHIFT 0
1361#define PAR_F (0x1 << PAR_F_SHIFT)
1362
1363#define PLATFORM_SYSCALL_TRAP_NO 0x80000000
1364
1365#define ARM64_SYSCALL_CODE_REG_NUM (16)
1366
1367#define ARM64_CLINE_SHIFT 6
1368
1369#if defined(APPLE_ARM64_ARCH_FAMILY)
1370#define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */
1371#define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */
1372#endif
1373
1374/*
1375 * Timer definitions.
1376 */
1377#define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */
1378#define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */
1379#define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */
1380#define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */
1381#define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */
1382#define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */
1383#define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to physical timebase + frequency reg enabled */
1384#define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to virtual timebase + frequency reg enabled */
1385
1386#define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */
1387#define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */
1388#define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */
1389
1390#define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS
1391#define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED
1392#define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE
1393
1394/*
1395 * At present all other uses of ARM_DBG_* are shared bit compatibly with the 32bit definitons.
1396 * (cf. osfmk/arm/proc_reg.h)
1397 */
1398#define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */
1399
1400#define MIDR_EL1_REV_SHIFT 0
1401#define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT)
1402#define MIDR_EL1_PNUM_SHIFT 4
1403#define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT)
1404#define MIDR_EL1_ARCH_SHIFT 16
1405#define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT)
1406#define MIDR_EL1_VAR_SHIFT 20
1407#define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT)
1408#define MIDR_EL1_IMP_SHIFT 24
1409#define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT)
1410
1411/*
1412 * CoreSight debug registers
1413 */
1414#define CORESIGHT_ED 0
1415#define CORESIGHT_CTI 1
1416#define CORESIGHT_PMU 2
1417#define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */
1418
1419#define CORESIGHT_OFFSET(x) ((x) * 0x10000)
1420#define CORESIGHT_REGIONS 4
1421#define CORESIGHT_SIZE 0x1000
1422
1423
1424/*
1425 * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0
1426 *
1427 * 63 24 23 20 19 16 15 12 11 8 7 4 3 0
1428 * +----------+--------+------+------+------+-----+------+
1429 * | reserved | atomic |crc32 | sha2 | sha1 | aes | res0 |
1430 * +----------+--------+------+------+------+-----+------+
1431 */
1432
1433#define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20
1434#define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
1435#define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET)
1436
1437#define ID_AA64ISAR0_EL1_CRC32_OFFSET 16
1438#define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
1439#define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET)
1440
1441#define ID_AA64ISAR0_EL1_SHA2_OFFSET 12
1442#define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
1443#define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET)
1444
1445#define ID_AA64ISAR0_EL1_SHA1_OFFSET 8
1446#define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
1447#define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET)
1448
1449#define ID_AA64ISAR0_EL1_AES_OFFSET 4
1450#define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET)
1451#define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET)
1452#define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET)
1453
1454
1455
1456#ifdef __ASSEMBLER__
1457
1458/*
1459 * Compute CPU version:
1460 * Version is constructed as [4 bits of MIDR variant]:[4 bits of MIDR revision]
1461 *
1462 * Where the "variant" is the major number and the "revision" is the minor number.
1463 *
1464 * For example:
1465 * Cyclone A0 is variant 0, revision 0, i.e. 0.
1466 * Cyclone B0 is variant 1, revision 0, i.e. 0x10
1467 * $0 - register to place value in
1468 */
1469.macro GET_MIDR_CPU_VERSION
1470 mrs $0, MIDR_EL1 // Read MIDR_EL1 for CPUID
1471 bfi $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #4 // move bits 3:0 (revision) to 19:16 (below variant) to get values adjacent
1472 ubfx $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #8 // And extract the concatenated bitstring to beginning of register
1473.endmacro
1474
1475/*
1476 * To apply a workaround for CPU versions less than a given value
1477 * (e.g. earlier than when a fix arrived)
1478 *
1479 * $0 - scratch register1
1480 * $1 - version at which to stop applying workaround
1481 * $2 - label to branch to (at end of workaround)
1482 */
1483.macro SKIP_IF_CPU_VERSION_GREATER_OR_EQUAL
1484 GET_MIDR_CPU_VERSION $0
1485 cmp $0, $1
1486 b.pl $2 // Unsigned "greater or equal"
1487.endmacro
1488
1489/*
1490 * To apply a workaround for CPU versions greater than a given value
1491 * (e.g. starting when a bug was introduced)
1492 *
1493 * $0 - scratch register1
1494 * $1 - version at which to stop applying workaround
1495 * $2 - label to branch to (at end of workaround)
1496 */
1497.macro SKIP_IF_CPU_VERSION_LESS_THAN
1498 GET_MIDR_CPU_VERSION $0
1499 cmp $0, $1
1500 b.mi $2 // Unsigned "strictly less than"
1501.endmacro
1502
1503#endif /* __ASSEMBLER__ */
1504
1505#define MSR(reg,src) __asm__ volatile ("msr " reg ", %0" :: "r" (src))
1506#define MRS(dest,reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest))
1507
1508
1509#endif /* _ARM64_PROC_REG_H_ */
1510