1 | /* |
2 | * Copyright (c) 2007-2013 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * Processor registers for ARM64 |
30 | */ |
31 | #ifndef _ARM64_PROC_REG_H_ |
32 | #define _ARM64_PROC_REG_H_ |
33 | |
34 | #include <arm/proc_reg.h> |
35 | |
36 | #if __ARM_KERNEL_PROTECT__ |
37 | /* |
38 | * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential |
39 | * architectural or microarchitectural vulnerabilities that could allow cores to |
40 | * read/access EL1-only mappings while in EL0 mode. This is achieved by |
41 | * removing as many mappings as possible when the core transitions to EL0 mode |
42 | * from EL1 mode, and restoring those mappings when the core transitions to EL1 |
43 | * mode from EL0 mode. |
44 | * |
45 | * At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is |
46 | * used to map and unmap the ordinary kernel mappings, by contracting and |
47 | * expanding translation zone size for TTBR1 when exiting and entering EL1, |
48 | * respectively: |
49 | * |
50 | * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is |
51 | * is in EL0. |
52 | * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in |
53 | * EL1. |
54 | * |
55 | * T1SZ_USER: T1SZ_BOOT + 1 |
56 | * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT) |
57 | * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER) |
58 | * TTBR1_EL1_MAX: (2^64) - 1 |
59 | * |
60 | * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the |
61 | * the following TTBR1 layout: |
62 | * |
63 | * TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX |
64 | * +---------------------------------------------------------+ |
65 | * | Kernel EL0 Mappings | Kernel EL1 Mappings | |
66 | * +---------------------------------------------------------+ |
67 | * |
68 | * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout: |
69 | * |
70 | * TTBR1_EL1_BASE_USER TTBR1_EL1_MAX |
71 | * +---------------------------------------------------------+ |
72 | * | Kernel EL0 Mappings | |
73 | * +---------------------------------------------------------+ |
74 | * |
75 | * With the current implementation, both the EL0 and EL1 mappings for the kernel |
76 | * use otherwise empty translation tables for mapping the exception vectors (so |
77 | * that we do not need to TLB flush the exception vector address when switching |
78 | * between EL0 and EL1). The rationale here is that the TLBI would require a |
79 | * DSB, and DSBs can be extremely expensive. |
80 | * |
81 | * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1 |
82 | * ASID. The core switches between ASIDs on EL transitions, so that the TLB |
83 | * does not need to be fully invalidated on an EL transition. |
84 | * |
85 | * Most kernel mappings will be marked non-global in this configuration, as |
86 | * global mappings would be visible to userspace unless we invalidate them on |
87 | * eret. |
88 | */ |
89 | #endif /* __ARM_KERNEL_PROTECT */ |
90 | |
91 | /* |
92 | * 64-bit Program Status Register (PSR64) |
93 | * |
94 | * 31 27 23 22 21 20 19 10 9 5 4 0 |
95 | * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+ |
96 | * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0| M | |
97 | * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+ |
98 | * |
99 | * where: |
100 | * NZCV Comparison flags |
101 | * PAN Privileged Access Never |
102 | * SS Single step |
103 | * IL Illegal state |
104 | * DAIF Interrupt masks |
105 | * M Mode field |
106 | */ |
107 | |
108 | #define PSR64_NZCV_SHIFT 28 |
109 | #define PSR64_NZCV_MASK (1 << PSR64_NZCV_SHIFT) |
110 | |
111 | #define PSR64_N_SHIFT 31 |
112 | #define PSR64_N (1 << PSR64_N_SHIFT) |
113 | |
114 | #define PSR64_Z_SHIFT 30 |
115 | #define PSR64_Z (1 << PSR64_Z_SHIFT) |
116 | |
117 | #define PSR64_C_SHIFT 29 |
118 | #define PSR64_C (1 << PSR64_C_SHIFT) |
119 | |
120 | #define PSR64_V_SHIFT 28 |
121 | #define PSR64_V (1 << PSR64_V_SHIFT) |
122 | |
123 | #define PSR64_PAN_SHIFT 22 |
124 | #define PSR64_PAN (1 << PSR64_PAN_SHIFT) |
125 | |
126 | #define PSR64_SS_SHIFT 21 |
127 | #define PSR64_SS (1 << PSR64_SS_SHIFT) |
128 | |
129 | #define PSR64_IL_SHIFT 20 |
130 | #define PSR64_IL (1 << PSR64_IL_SHIFT) |
131 | |
132 | /* |
133 | * msr DAIF, Xn and mrs Xn, DAIF transfer into |
134 | * and out of bits 9:6 |
135 | */ |
136 | #define DAIF_DEBUG_SHIFT 9 |
137 | #define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT) |
138 | |
139 | #define DAIF_ASYNC_SHIFT 8 |
140 | #define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT) |
141 | |
142 | #define DAIF_IRQF_SHIFT 7 |
143 | #define DAIF_IRQF (1 << DAIF_IRQF_SHIFT) |
144 | |
145 | #define DAIF_FIQF_SHIFT 6 |
146 | #define DAIF_FIQF (1 << DAIF_FIQF_SHIFT) |
147 | |
148 | #define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) |
149 | #define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) |
150 | |
151 | #define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF)) |
152 | |
153 | /* |
154 | * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer |
155 | * from bits 3:0. |
156 | */ |
157 | #define DAIFSC_DEBUGF (1 << 3) |
158 | #define DAIFSC_ASYNCF (1 << 2) |
159 | #define DAIFSC_IRQF (1 << 1) |
160 | #define DAIFSC_FIQF (1 << 0) |
161 | #define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) |
162 | #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) |
163 | |
164 | /* |
165 | * ARM64_TODO: unify with ARM? |
166 | */ |
167 | #define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */ |
168 | |
169 | #define PSR64_MODE_MASK 0x1F |
170 | |
171 | #define PSR64_MODE_USER32_THUMB 0x20 |
172 | |
173 | #define PSR64_MODE_RW_SHIFT 4 |
174 | #define PSR64_MODE_RW_64 0 |
175 | #define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT) |
176 | |
177 | #define PSR64_MODE_EL_SHIFT 2 |
178 | #define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT) |
179 | #define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT) |
180 | #define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT) |
181 | #define PSR64_MODE_EL0 0 |
182 | |
183 | #define PSR64_MODE_SPX 0x1 |
184 | #define PSR64_MODE_SP0 0 |
185 | |
186 | #define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0) |
187 | #define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0) |
188 | #define PSR64_KERNEL_DEFAULT (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0) |
189 | |
190 | #define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL1) |
191 | #define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0) |
192 | |
193 | #define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32)) |
194 | #define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32)) |
195 | |
196 | |
197 | |
198 | /* |
199 | * System Control Register (SCTLR) |
200 | */ |
201 | |
202 | #define SCTLR_RESERVED ((3 << 28) | (1 << 22) | (1 << 20) | (1 << 11)) |
203 | |
204 | // 26 UCI User Cache Instructions |
205 | #define SCTLR_UCI_ENABLED (1 << 26) |
206 | |
207 | // 25 EE Exception Endianness |
208 | #define SCTLR_EE_BIG_ENDIAN (1 << 25) |
209 | |
210 | // 24 E0E EL0 Endianness |
211 | #define SCTLR_E0E_BIG_ENDIAN (1 << 24) |
212 | |
213 | // 23 SPAN Set PAN |
214 | #define SCTLR_PAN_UNCHANGED (1 << 23) |
215 | |
216 | // 22 RES1 1 |
217 | // 21 RES0 0 |
218 | // 20 RES1 1 |
219 | |
220 | // 19 WXN Writeable implies eXecute Never |
221 | #define SCTLR_WXN_ENABLED (1 << 19) |
222 | |
223 | // 18 nTWE Not trap WFE from EL0 |
224 | #define SCTLR_nTWE_WFE_ENABLED (1 << 18) |
225 | |
226 | // 17 RES0 0 |
227 | |
228 | // 16 nTWI Not trap WFI from EL0 |
229 | #define SCTRL_nTWI_WFI_ENABLED (1 << 16) |
230 | |
231 | // 15 UCT User Cache Type register (CTR_EL0) |
232 | #define SCTLR_UCT_ENABLED (1 << 15) |
233 | |
234 | // 14 DZE User Data Cache Zero (DC ZVA) |
235 | #define SCTLR_DZE_ENABLED (1 << 14) |
236 | |
237 | // 13 PACDB_ENABLED AddPACDB and AuthDB functions enabled |
238 | #define SCTLR_PACDB_ENABLED (1 << 13) |
239 | |
240 | // 12 I Instruction cache enable |
241 | #define SCTLR_I_ENABLED (1 << 12) |
242 | |
243 | // 11 RES1 1 |
244 | // 10 RES0 0 |
245 | |
246 | // 9 UMA User Mask Access |
247 | #define SCTLR_UMA_ENABLED (1 << 9) |
248 | |
249 | // 8 SED SETEND Disable |
250 | #define SCTLR_SED_DISABLED (1 << 8) |
251 | |
252 | // 7 ITD IT Disable |
253 | #define SCTLR_ITD_DISABLED (1 << 7) |
254 | |
255 | // 6 RES0 0 |
256 | |
257 | // 5 CP15BEN CP15 Barrier ENable |
258 | #define SCTLR_CP15BEN_ENABLED (1 << 5) |
259 | |
260 | // 4 SA0 Stack Alignment check for EL0 |
261 | #define SCTLR_SA0_ENABLED (1 << 4) |
262 | |
263 | // 3 SA Stack Alignment check |
264 | #define SCTLR_SA_ENABLED (1 << 3) |
265 | |
266 | // 2 C Cache enable |
267 | #define SCTLR_C_ENABLED (1 << 2) |
268 | |
269 | // 1 A Alignment check |
270 | #define SCTLR_A_ENABLED (1 << 1) |
271 | |
272 | // 0 M MMU enable |
273 | #define SCTLR_M_ENABLED (1 << 0) |
274 | |
275 | #define SCTLR_EL1_DEFAULT (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \ |
276 | SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED | \ |
277 | SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED) |
278 | |
279 | /* |
280 | * Coprocessor Access Control Register (CPACR) |
281 | * |
282 | * 31 28 27 22 21 20 19 0 |
283 | * +---+---+------+------+--------------------+ |
284 | * |000|TTA|000000| FPEN |00000000000000000000| |
285 | * +---+---+------+------+--------------------+ |
286 | * |
287 | * where: |
288 | * TTA Trace trap |
289 | * FPEN Floating point enable |
290 | */ |
291 | #define CPACR_TTA_SHIFT 28 |
292 | #define CPACR_TTA (1 << CPACR_TTA_SHIFT) |
293 | |
294 | #define CPACR_FPEN_SHIFT 20 |
295 | #define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT) |
296 | #define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT) |
297 | |
298 | /* |
299 | * FPSR: Floating Point Status Register |
300 | * |
301 | * 31 30 29 28 27 26 7 6 4 3 2 1 0 |
302 | * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ |
303 | * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC| |
304 | * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ |
305 | */ |
306 | |
307 | #define FPSR_N_SHIFT 31 |
308 | #define FPSR_Z_SHIFT 30 |
309 | #define FPSR_C_SHIFT 29 |
310 | #define FPSR_V_SHIFT 28 |
311 | #define FPSR_QC_SHIFT 27 |
312 | #define FPSR_IDC_SHIFT 7 |
313 | #define FPSR_IXC_SHIFT 4 |
314 | #define FPSR_UFC_SHIFT 3 |
315 | #define FPSR_OFC_SHIFT 2 |
316 | #define FPSR_DZC_SHIFT 1 |
317 | #define FPSR_IOC_SHIFT 0 |
318 | #define FPSR_N (1 << FPSR_N_SHIFT) |
319 | #define FPSR_Z (1 << FPSR_Z_SHIFT) |
320 | #define FPSR_C (1 << FPSR_C_SHIFT) |
321 | #define FPSR_V (1 << FPSR_V_SHIFT) |
322 | #define FPSR_QC (1 << FPSR_QC_SHIFT) |
323 | #define FPSR_IDC (1 << FPSR_IDC_SHIFT) |
324 | #define FPSR_IXC (1 << FPSR_IXC_SHIFT) |
325 | #define FPSR_UFC (1 << FPSR_UFC_SHIFT) |
326 | #define FPSR_OFC (1 << FPSR_OFC_SHIFT) |
327 | #define FPSR_DZC (1 << FPSR_DZC_SHIFT) |
328 | #define FPSR_IOC (1 << FPSR_IOC_SHIFT) |
329 | |
330 | /* |
331 | * A mask for all for all of the bits that are not RAZ for FPSR; this |
332 | * is primarily for converting between a 32-bit view of NEON state |
333 | * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). |
334 | */ |
335 | #define FPSR_MASK (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | \ |
336 | FPSR_IDC | FPSR_IXC | FPSR_UFC | FPSR_OFC | \ |
337 | FPSR_DZC | FPSR_IOC) |
338 | |
339 | /* |
340 | * FPCR: Floating Point Control Register |
341 | * |
342 | * 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0 |
343 | * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ |
344 | * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000| |
345 | * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ |
346 | */ |
347 | |
348 | #define FPCR_AHP_SHIFT 26 |
349 | #define FPCR_DN_SHIFT 25 |
350 | #define FPCR_FZ_SHIFT 24 |
351 | #define FPCR_RMODE_SHIFT 22 |
352 | #define FPCR_STRIDE_SHIFT 20 |
353 | #define FPCR_LEN_SHIFT 16 |
354 | #define FPCR_IDE_SHIFT 15 |
355 | #define FPCR_IXE_SHIFT 12 |
356 | #define FPCR_UFE_SHIFT 11 |
357 | #define FPCR_OFE_SHIFT 10 |
358 | #define FPCR_DZE_SHIFT 9 |
359 | #define FPCR_IOE_SHIFT 8 |
360 | #define FPCR_AHP (1 << FPCR_AHP_SHIFT) |
361 | #define FPCR_DN (1 << FPCR_DN_SHIFT) |
362 | #define FPCR_FZ (1 << FPCR_FZ_SHIFT) |
363 | #define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT) |
364 | #define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT) |
365 | #define FPCR_LEN (0x7 << FPCR_LEN_SHIFT) |
366 | #define FPCR_IDE (1 << FPCR_IDE_SHIFT) |
367 | #define FPCR_IXE (1 << FPCR_IXE_SHIFT) |
368 | #define FPCR_UFE (1 << FPCR_UFE_SHIFT) |
369 | #define FPCR_OFE (1 << FPCR_OFE_SHIFT) |
370 | #define FPCR_DZE (1 << FPCR_DZE_SHIFT) |
371 | #define FPCR_IOE (1 << FPCR_IOE_SHIFT) |
372 | #define FPCR_DEFAULT (FPCR_DN) |
373 | #define FPCR_DEFAULT_32 (FPCR_DN|FPCR_FZ) |
374 | |
375 | /* |
376 | * A mask for all for all of the bits that are not RAZ for FPCR; this |
377 | * is primarily for converting between a 32-bit view of NEON state |
378 | * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). |
379 | */ |
380 | #define FPCR_MASK (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | \ |
381 | FPCR_STRIDE | FPCR_LEN | FPCR_IDE | FPCR_IXE | \ |
382 | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE) |
383 | |
384 | /* |
385 | * Translation Control Register (TCR) |
386 | * |
387 | * Legacy: |
388 | * |
389 | * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 |
390 | * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ |
391 | * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| |
392 | * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ |
393 | * |
394 | * Current (with 16KB granule support): |
395 | * |
396 | * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 |
397 | * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ |
398 | * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| |
399 | * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ |
400 | * |
401 | * TBI1 Top Byte Ignored for TTBR1 region |
402 | * TBI0 Top Byte Ignored for TTBR0 region |
403 | * AS ASID Size |
404 | * IPS Physical Address Size limit |
405 | * TG1 Granule Size for TTBR1 region |
406 | * SH1 Shareability for TTBR1 region |
407 | * ORGN1 Outer Cacheability for TTBR1 region |
408 | * IRGN1 Inner Cacheability for TTBR1 region |
409 | * EPD1 Translation table walk disable for TTBR1 |
410 | * A1 ASID selection from TTBR1 enable |
411 | * T1SZ Virtual address size for TTBR1 |
412 | * TG0 Granule Size for TTBR0 region |
413 | * SH0 Shareability for TTBR0 region |
414 | * ORGN0 Outer Cacheability for TTBR0 region |
415 | * IRGN0 Inner Cacheability for TTBR0 region |
416 | * T0SZ Virtual address size for TTBR0 |
417 | */ |
418 | |
419 | #define TCR_T0SZ_SHIFT 0ULL |
420 | #define TCR_TSZ_BITS 6ULL |
421 | #define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL) |
422 | |
423 | #define TCR_IRGN0_SHIFT 8ULL |
424 | #define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT) |
425 | #define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT) |
426 | #define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT) |
427 | #define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT) |
428 | |
429 | #define TCR_ORGN0_SHIFT 10ULL |
430 | #define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT) |
431 | #define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT) |
432 | #define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT) |
433 | #define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT) |
434 | |
435 | #define TCR_SH0_SHIFT 12ULL |
436 | #define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT) |
437 | #define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT) |
438 | #define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT) |
439 | |
440 | #define TCR_TG0_GRANULE_SHIFT (14ULL) |
441 | |
442 | #define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT) |
443 | #define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT) |
444 | #define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT) |
445 | |
446 | #if __ARM_16K_PG__ |
447 | #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB) |
448 | #else |
449 | #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB) |
450 | #endif |
451 | |
452 | #define TCR_T1SZ_SHIFT 16ULL |
453 | |
454 | #define TCR_A1_ASID1 (1ULL << 22ULL) |
455 | #define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL) |
456 | |
457 | #define TCR_IRGN1_SHIFT 24ULL |
458 | #define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT) |
459 | #define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT) |
460 | #define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT) |
461 | #define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT) |
462 | |
463 | #define TCR_ORGN1_SHIFT 26ULL |
464 | #define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT) |
465 | #define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT) |
466 | #define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT) |
467 | #define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT) |
468 | |
469 | #define TCR_SH1_SHIFT 28ULL |
470 | #define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT) |
471 | #define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT) |
472 | #define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT) |
473 | |
474 | #define TCR_TG1_GRANULE_SHIFT 30ULL |
475 | |
476 | #define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT) |
477 | #define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT) |
478 | #define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT) |
479 | |
480 | #if __ARM_16K_PG__ |
481 | #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB) |
482 | #else |
483 | #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB) |
484 | #endif |
485 | |
486 | #define TCR_IPS_SHIFT 32ULL |
487 | #define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT) |
488 | #define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT) |
489 | #define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT) |
490 | #define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT) |
491 | #define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT) |
492 | #define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT) |
493 | |
494 | #define TCR_AS_16BIT_ASID (1ULL << 36) |
495 | #define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37) |
496 | #define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38) |
497 | |
498 | /* |
499 | * Multiprocessor Affinity Register (MPIDR_EL1) |
500 | * |
501 | * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+ |
502 | * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0| |
503 | * +---------------------------------+--+-----+--+-----+----+----+ |
504 | * |
505 | * where |
506 | * U Uniprocessor |
507 | * MT Multi-threading at lowest affinity level |
508 | * Aff2 "1" - PCORE, "0" - ECORE |
509 | * Aff1 Cluster ID |
510 | * Aff0 CPU ID |
511 | */ |
512 | #define MPIDR_AFF0_MASK 0xFF |
513 | #define MPIDR_AFF1_MASK 0xFF00 |
514 | #define MPIDR_AFF1_SHIFT 8 |
515 | #define MPIDR_AFF2_MASK 0xFF0000 |
516 | #define MPIDR_AFF2_SHIFT 16 |
517 | |
518 | /* |
519 | * We currently use a 3 level page table (rather than the full 4 |
520 | * level page table). As a result, we do not have the full 48-bits |
521 | * of address space per TTBR (although the 16KB granule size lets us |
522 | * get very close). |
523 | */ |
524 | #if __ARM64_TWO_LEVEL_PMAP__ && !__ARM_16K_PG__ |
525 | #error ARM64 does not currently support a 2 level page table with 4KB pages |
526 | #endif /* __ARM64_TWO_LEVEL_PMAP__ */ |
527 | |
528 | /* |
529 | * TXSZ indicates the size of the range a TTBR covers. Currently, |
530 | * we support the following: |
531 | * |
532 | * 4KB pages, full page L1: 39 bit range. |
533 | * 4KB pages, sub-page L1: 38 bit range. |
534 | * 16KB pages, full page L1: 47 bit range. |
535 | * 16KB pages, sub-page L1: 39 bit range. |
536 | * 16KB pages, two level page tables: 36 bit range. |
537 | */ |
538 | #if __ARM_KERNEL_PROTECT__ |
539 | /* |
540 | * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the |
541 | * address space is used for the mappings that will remain in place when in EL0. |
542 | * As a result, 1 bit less of address space is available to the rest of the |
543 | * the kernel. |
544 | */ |
545 | #endif /* __ARM_KERNEL_PROTECT__ */ |
546 | #ifdef __ARM_16K_PG__ |
547 | #if __ARM64_TWO_LEVEL_PMAP__ |
548 | #define T0SZ_BOOT 28ULL |
549 | #elif __ARM64_PMAP_SUBPAGE_L1__ |
550 | #define T0SZ_BOOT 25ULL |
551 | #else /* __ARM64_TWO_LEVEL_PMAP__ */ |
552 | #define T0SZ_BOOT 17ULL |
553 | #endif /* __ARM64_TWO_LEVEL_PMAP__ */ |
554 | #else /* __ARM_16K_PG__ */ |
555 | #if __ARM64_PMAP_SUBPAGE_L1__ |
556 | #define T0SZ_BOOT 26ULL |
557 | #else /* __ARM64_PMAP_SUBPAGE_L1__ */ |
558 | #define T0SZ_BOOT 25ULL |
559 | #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ |
560 | #endif /* __ARM_16K_PG__ */ |
561 | |
562 | #if defined(APPLE_ARM64_ARCH_FAMILY) |
563 | /* T0SZ must be the same as T1SZ */ |
564 | #define T1SZ_BOOT T0SZ_BOOT |
565 | #else /* defined(APPLE_ARM64_ARCH_FAMILY) */ |
566 | #ifdef __ARM_16K_PG__ |
567 | #if __ARM64_TWO_LEVEL_PMAP__ |
568 | #define T1SZ_BOOT 28ULL |
569 | #elif __ARM64_PMAP_SUBPAGE_L1__ |
570 | #define T1SZ_BOOT 25ULL |
571 | #else /* __ARM64_TWO_LEVEL_PMAP__ */ |
572 | #define T1SZ_BOOT 17ULL |
573 | #endif /* __ARM64_TWO_LEVEL_PMAP__ */ |
574 | #else /* __ARM_16K_PG__ */ |
575 | #if __ARM64_PMAP_SUBPAGE_L1__ |
576 | #define T1SZ_BOOT 26ULL |
577 | #else /* __ARM64_PMAP_SUBPAGE_L1__ */ |
578 | #define T1SZ_BOOT 25ULL |
579 | #endif /*__ARM64_PMAP_SUBPAGE_L1__*/ |
580 | #endif /* __ARM_16K_PG__ */ |
581 | #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ |
582 | |
583 | #define TCR_EL1_BASE (TCR_IPS_40BITS | \ |
584 | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE) |\ |
585 | TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE)) |
586 | |
587 | #if __ARM_KERNEL_PROTECT__ |
588 | #define TCR_EL1_BOOT (TCR_EL1_BASE | \ |
589 | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | TCR_TBI0_TOPBYTE_IGNORED) |
590 | #define T1SZ_USER (T1SZ_BOOT + 1) |
591 | #define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | TCR_TBI0_TOPBYTE_IGNORED) |
592 | #else |
593 | #define TCR_EL1_BOOT (TCR_EL1_BASE | \ |
594 | (T1SZ_BOOT << TCR_T1SZ_SHIFT)) |
595 | #endif /* __ARM_KERNEL_PROTECT__ */ |
596 | |
597 | /* |
598 | * Translation Table Base Register (TTBR) |
599 | * |
600 | * 63 48 47 x x-1 0 |
601 | * +--------+------------------+------+ |
602 | * | ASID | Base Address | zero | |
603 | * +--------+------------------+------+ |
604 | * |
605 | */ |
606 | #define TTBR_ASID_SHIFT 48 |
607 | #define TTBR_ASID_MASK 0xffff000000000000 |
608 | |
609 | #define TTBR_BADDR_MASK 0x0000ffffffffffff |
610 | |
611 | /* |
612 | * Memory Attribute Indirection Register |
613 | * |
614 | * 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0 |
615 | * +-------+-------+-------+-------+-------+-------+-------+-------+ |
616 | * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 | |
617 | * +-------+-------+-------+-------+-------+-------+-------+-------+ |
618 | * |
619 | */ |
620 | |
621 | #define MAIR_ATTR_SHIFT(x) (8*(x)) |
622 | |
623 | /* Strongly ordered or device memory attributes */ |
624 | #define MAIR_OUTER_STRONGLY_ORDERED 0x0 |
625 | #define MAIR_OUTER_DEVICE 0x0 |
626 | |
627 | #define MAIR_INNER_STRONGLY_ORDERED 0x0 |
628 | #define MAIR_INNER_DEVICE 0x4 |
629 | |
630 | /* Normal memory attributes */ |
631 | #define MAIR_OUTER_NON_CACHEABLE 0x40 |
632 | #define MAIR_OUTER_WRITE_THROUGH 0x80 |
633 | #define MAIR_OUTER_WRITE_BACK 0xc0 |
634 | |
635 | #define MAIR_INNER_NON_CACHEABLE 0x4 |
636 | #define MAIR_INNER_WRITE_THROUGH 0x8 |
637 | #define MAIR_INNER_WRITE_BACK 0xc |
638 | |
639 | /* Allocate policy for cacheable memory */ |
640 | #define MAIR_OUTER_WRITE_ALLOCATE 0x10 |
641 | #define MAIR_OUTER_READ_ALLOCATE 0x20 |
642 | |
643 | #define MAIR_INNER_WRITE_ALLOCATE 0x1 |
644 | #define MAIR_INNER_READ_ALLOCATE 0x2 |
645 | |
646 | /* Memory Atribute Encoding */ |
647 | |
648 | /* Device memory types: |
649 | G (gathering): multiple reads/writes can be combined |
650 | R (reordering): reads or writes may reach device out of program order |
651 | E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes) |
652 | */ |
653 | #define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */ |
654 | #define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */ |
655 | #define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */ |
656 | #define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */ |
657 | #define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */ |
658 | #define MAIR_INNERWRITEBACK 0x4F /* Normal Memory, Outer Non-Cacheable, Inner Write-back */ |
659 | |
660 | |
661 | /* |
662 | * ARM 4-level Page Table support - 2*1024TB (2^48) of address space |
663 | */ |
664 | |
665 | |
666 | /* |
667 | * Memory Attribute Index |
668 | */ |
669 | #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled */ |
670 | #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes */ |
671 | #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled */ |
672 | #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer */ |
673 | #define CACHE_ATTRINDX_INNERWRITEBACK 0x4 /* inner cache enabled, buffer enabled, write allocate */ |
674 | #define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes */ |
675 | #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK |
676 | |
677 | /* |
678 | * Access protection bit values (TTEs and PTEs) |
679 | */ |
680 | #define AP_RWNA 0x0 /* priv=read-write, user=no-access */ |
681 | #define AP_RWRW 0x1 /* priv=read-write, user=read-write */ |
682 | #define AP_RONA 0x2 /* priv=read-only, user=no-access */ |
683 | #define AP_RORO 0x3 /* priv=read-only, user=read-only */ |
684 | #define AP_MASK 0x3 /* mask to find ap bits */ |
685 | |
686 | /* |
687 | * Shareability attributes |
688 | */ |
689 | #define SH_NONE 0x0 /* Non shareable */ |
690 | #define SH_NONE 0x0 /* Device shareable */ |
691 | #define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */ |
692 | #define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */ |
693 | #define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */ |
694 | |
695 | |
696 | /* |
697 | * ARM Page Granule |
698 | */ |
699 | #ifdef __ARM_16K_PG__ |
700 | #define ARM_PGSHIFT 14 |
701 | #else |
702 | #define ARM_PGSHIFT 12 |
703 | #endif |
704 | #define ARM_PGBYTES (1 << ARM_PGSHIFT) |
705 | #define ARM_PGMASK (ARM_PGBYTES-1) |
706 | |
707 | |
708 | /* |
709 | * L0 Translation table |
710 | * |
711 | * 4KB granule size: |
712 | * Each translation table is 4KB |
713 | * 512 64-bit entries of 512GB (2^39) of address space. |
714 | * Covers 256TB (2^48) of address space. |
715 | * |
716 | * 16KB granule size: |
717 | * Each translation table is 16KB |
718 | * 2 64-bit entries of 128TB (2^47) of address space. |
719 | * Covers 256TB (2^48) of address space. |
720 | */ |
721 | |
722 | #ifdef __ARM_16K_PG__ |
723 | #define ARM_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */ |
724 | #define ARM_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */ |
725 | #define ARM_TT_L0_SHIFT 47 /* page descriptor shift */ |
726 | #define ARM_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */ |
727 | #else |
728 | #define ARM_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */ |
729 | #define ARM_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */ |
730 | #define ARM_TT_L0_SHIFT 39 /* page descriptor shift */ |
731 | #define ARM_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */ |
732 | #endif |
733 | |
734 | /* |
735 | * L1 Translation table |
736 | * |
737 | * 4KB granule size: |
738 | * Each translation table is 4KB |
739 | * 512 64-bit entries of 1GB (2^30) of address space. |
740 | * Covers 512GB (2^39) of address space. |
741 | * |
742 | * 16KB granule size: |
743 | * Each translation table is 16KB |
744 | * 2048 64-bit entries of 64GB (2^36) of address space. |
745 | * Covers 128TB (2^47) of address space. |
746 | */ |
747 | |
748 | #ifdef __ARM_16K_PG__ |
749 | #define ARM_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */ |
750 | #define ARM_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */ |
751 | #define ARM_TT_L1_SHIFT 36 /* page descriptor shift */ |
752 | #ifdef __ARM64_PMAP_SUBPAGE_L1__ |
753 | /* This config supports 512GB per TTBR. */ |
754 | #define ARM_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */ |
755 | #else /* __ARM64_PMAP_SUBPAGE_L1__ */ |
756 | #define ARM_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */ |
757 | #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ |
758 | #else /* __ARM_16K_PG__ */ |
759 | #define ARM_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */ |
760 | #define ARM_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */ |
761 | #define ARM_TT_L1_SHIFT 30 /* page descriptor shift */ |
762 | #ifdef __ARM64_PMAP_SUBPAGE_L1__ |
763 | /* This config supports 256GB per TTBR. */ |
764 | #define ARM_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */ |
765 | #else /* __ARM64_PMAP_SUBPAGE_L1__ */ |
766 | #define ARM_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */ |
767 | #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ |
768 | #endif |
769 | |
770 | /* some sugar for getting pointers to page tables and entries */ |
771 | |
772 | #define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT) |
773 | #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT) |
774 | #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT) |
775 | |
776 | #define L2_TABLE_VA(tte) ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK)) |
777 | #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK)) |
778 | |
779 | /* |
780 | * L2 Translation table |
781 | * |
782 | * 4KB granule size: |
783 | * Each translation table is 4KB |
784 | * 512 64-bit entries of 2MB (2^21) of address space. |
785 | * Covers 1GB (2^30) of address space. |
786 | * |
787 | * 16KB granule size: |
788 | * Each translation table is 16KB |
789 | * 2048 64-bit entries of 32MB (2^25) of address space. |
790 | * Covers 64GB (2^36) of address space. |
791 | */ |
792 | |
793 | #ifdef __ARM_16K_PG__ |
794 | #define ARM_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */ |
795 | #define ARM_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */ |
796 | #define ARM_TT_L2_SHIFT 25 /* page descriptor shift */ |
797 | #define ARM_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */ |
798 | #else |
799 | #define ARM_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */ |
800 | #define ARM_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */ |
801 | #define ARM_TT_L2_SHIFT 21 /* page descriptor shift */ |
802 | #define ARM_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */ |
803 | #endif |
804 | |
805 | /* |
806 | * L3 Translation table |
807 | * |
808 | * 4KB granule size: |
809 | * Each translation table is 4KB |
810 | * 512 64-bit entries of 4KB (2^12) of address space. |
811 | * Covers 2MB (2^21) of address space. |
812 | * |
813 | * 16KB granule size: |
814 | * Each translation table is 16KB |
815 | * 2048 64-bit entries of 16KB (2^14) of address space. |
816 | * Covers 32MB (2^25) of address space. |
817 | */ |
818 | |
819 | #ifdef __ARM_16K_PG__ |
820 | #define ARM_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ |
821 | #define ARM_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */ |
822 | #define ARM_TT_L3_SHIFT 14 /* page descriptor shift */ |
823 | #define ARM_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */ |
824 | #else |
825 | #define ARM_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ |
826 | #define ARM_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */ |
827 | #define ARM_TT_L3_SHIFT 12 /* page descriptor shift */ |
828 | #define ARM_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */ |
829 | #endif |
830 | |
831 | /* |
832 | * Convenience definitions for: |
833 | * ARM_TT_LEAF: The last level of the configured page table format. |
834 | * ARM_TT_TWIG: The second to last level of the configured page table format. |
835 | * ARM_TT_ROOT: The first level of the configured page table format. |
836 | * |
837 | * My apologies to any botanists who may be reading this. |
838 | */ |
839 | #define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE |
840 | #define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK |
841 | #define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT |
842 | #define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK |
843 | |
844 | #define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE |
845 | #define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK |
846 | #define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT |
847 | #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK |
848 | |
849 | #if __ARM64_TWO_LEVEL_PMAP__ |
850 | #define ARM_TT_ROOT_SIZE ARM_TT_L2_SIZE |
851 | #define ARM_TT_ROOT_OFFMASK ARM_TT_L2_OFFMASK |
852 | #define ARM_TT_ROOT_SHIFT ARM_TT_L2_SHIFT |
853 | #define ARM_TT_ROOT_INDEX_MASK ARM_TT_L2_INDEX_MASK |
854 | #else |
855 | #define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE |
856 | #define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK |
857 | #define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT |
858 | #define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK |
859 | #endif |
860 | |
861 | /* |
862 | * 4KB granule size: |
863 | * |
864 | * Level 0 Translation Table Entry |
865 | * |
866 | * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 |
867 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
868 | * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| |
869 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
870 | * |
871 | * Level 1 Translation Table Entry |
872 | * |
873 | * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 |
874 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
875 | * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| |
876 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
877 | * |
878 | * Level 1 Translation Block Entry |
879 | * |
880 | * 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0 |
881 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
882 | * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| |
883 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
884 | * |
885 | * Level 2 Translation Table Entry |
886 | * |
887 | * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 |
888 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
889 | * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| |
890 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
891 | * |
892 | * Level 2 Translation Block Entry |
893 | * |
894 | * 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0 |
895 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
896 | * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| |
897 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
898 | * |
899 | * 16KB granule size: |
900 | * |
901 | * Level 0 Translation Table Entry |
902 | * |
903 | * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 |
904 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
905 | * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| |
906 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
907 | * |
908 | * Level 1 Translation Table Entry |
909 | * |
910 | * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 |
911 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
912 | * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| |
913 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
914 | * |
915 | * Level 2 Translation Table Entry |
916 | * |
917 | * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 |
918 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
919 | * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| |
920 | * +--+-----+--+---+-------+------+----------------------+-------+-+-+ |
921 | * |
922 | * Level 2 Translation Block Entry |
923 | * |
924 | * 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0 |
925 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
926 | * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| |
927 | * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ |
928 | * |
929 | * where: |
930 | * 'nG' notGlobal bit |
931 | * 'SH' Shareability field |
932 | * 'AP' access protection |
933 | * 'XN' eXecute Never bit |
934 | * 'PXN' Privilege eXecute Never bit |
935 | * 'NS' Non-Secure bit |
936 | * 'HINT' 16 entry continuguous output hint |
937 | * 'AttrIdx' Memory Attribute Index |
938 | */ |
939 | |
940 | #define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */ |
941 | #ifdef __ARM_16K_PG__ |
942 | #define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */ |
943 | #else |
944 | #define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */ |
945 | #endif |
946 | |
947 | #define ARM_TTE_MAX (TTE_PGENTRIES) |
948 | |
949 | #define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */ |
950 | #define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */ |
951 | |
952 | #define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */ |
953 | |
954 | #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ |
955 | #define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */ |
956 | #define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */ |
957 | #define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL |
958 | #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ |
959 | |
960 | #ifdef __ARM_16K_PG__ |
961 | /* Note that L0/L1 block entries are disallowed for the 16KB granule size; what are we doing with these? */ |
962 | #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */ |
963 | #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */ |
964 | #define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */ |
965 | #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */ |
966 | #define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */ |
967 | #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */ |
968 | #else |
969 | #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */ |
970 | #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */ |
971 | #define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */ |
972 | #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */ |
973 | #define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */ |
974 | #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */ |
975 | #endif |
976 | |
977 | #define ARM_TTE_BLOCK_APSHIFT 6 |
978 | #define ARM_TTE_BLOCK_AP(x) ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */ |
979 | #define ARM_TTE_BLOCK_APMASK (0x3 << ARM_TTE_BLOCK_APSHIFT) |
980 | |
981 | #define ARM_TTE_BLOCK_ATTRINDX(x) ((x) << 2) /* memory attributes index */ |
982 | #define ARM_TTE_BLOCK_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ |
983 | |
984 | #define ARM_TTE_BLOCK_SH(x) ((x) << 8) /* access shared */ |
985 | #define ARM_TTE_BLOCK_SHMASK (0x3ULL << 8) /* mask access shared */ |
986 | |
987 | #define ARM_TTE_BLOCK_AF 0x0000000000000400ULL /* value for access */ |
988 | #define ARM_TTE_BLOCK_AFMASK 0x0000000000000400ULL /* access mask */ |
989 | |
990 | #define ARM_TTE_BLOCK_NG 0x0000000000000800ULL /* value for a global mapping */ |
991 | #define ARM_TTE_BLOCK_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ |
992 | |
993 | #define ARM_TTE_BLOCK_NS 0x0000000000000020ULL /* value for a secure mapping */ |
994 | #define ARM_TTE_BLOCK_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ |
995 | |
996 | #define ARM_TTE_BLOCK_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ |
997 | #define ARM_TTE_BLOCK_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ |
998 | |
999 | #define ARM_TTE_BLOCK_NX 0x0040000000000000ULL /* value for no execute */ |
1000 | #define ARM_TTE_BLOCK_NXMASK 0x0040000000000000ULL /* no execute mask */ |
1001 | |
1002 | #define ARM_TTE_BLOCK_WIRED 0x0080000000000000ULL /* value for software wired bit */ |
1003 | #define ARM_TTE_BLOCK_WIREDMASK 0x0080000000000000ULL /* software wired mask */ |
1004 | |
1005 | #define ARM_TTE_BLOCK_WRITEABLE 0x0100000000000000ULL /* value for software writeable bit */ |
1006 | #define ARM_TTE_BLOCK_WRITEABLEMASK 0x0100000000000000ULL /* software writeable mask */ |
1007 | |
1008 | #ifdef __ARM_16K_PG__ |
1009 | /* |
1010 | * TODO: Do we care about the low bits being unused? It should technically work either way, but masking them out should be future proof; |
1011 | * it is only a matter of time before someone wants to shove something into the free bits. |
1012 | */ |
1013 | #define ARM_TTE_TABLE_MASK (0x0000ffffffffc000ULL) /* mask for extracting pointer to next table (works at any level) */ |
1014 | #else |
1015 | #define ARM_TTE_TABLE_MASK (0x0000fffffffff000ULL) /* mask for extracting pointer to next table (works at any level) */ |
1016 | #endif |
1017 | |
1018 | #define ARM_TTE_TABLE_APSHIFT 61 |
1019 | #define ARM_TTE_TABLE_AP(x) ((x)<<TTE_BLOCK_APSHIFT) /* access protection */ |
1020 | |
1021 | #define ARM_TTE_TABLE_NS 0x8000000000000020ULL /* value for a secure mapping */ |
1022 | #define ARM_TTE_TABLE_NS_MASK 0x8000000000000020ULL /* notSecure mapping mask */ |
1023 | |
1024 | #define ARM_TTE_TABLE_XN 0x1000000000000000ULL /* value for no execute */ |
1025 | #define ARM_TTE_TABLE_XNMASK 0x1000000000000000ULL /* no execute mask */ |
1026 | |
1027 | #define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */ |
1028 | #define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */ |
1029 | |
1030 | #if __ARM_KERNEL_PROTECT__ |
1031 | #define ARM_TTE_BOOT_BLOCK (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) \ |
1032 | | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF \ |
1033 | | ARM_TTE_BLOCK_NG) |
1034 | #else /* __ARM_KERNEL_PROTECT__ */ |
1035 | #define ARM_TTE_BOOT_BLOCK (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) \ |
1036 | | ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF) |
1037 | #endif /* __ARM_KERNEL_PROTECT__ */ |
1038 | |
1039 | #define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID ) |
1040 | /* |
1041 | * L3 Translation table |
1042 | * |
1043 | * 4KB granule size: |
1044 | * Each translation table is 4KB |
1045 | * 512 64-bit entries of 4KB (2^12) of address space. |
1046 | * Covers 2MB (2^21) of address space. |
1047 | * |
1048 | * 16KB granule size: |
1049 | * Each translation table is 16KB |
1050 | * 2048 64-bit entries of 16KB (2^14) of address space. |
1051 | * Covers 32MB (2^25) of address space. |
1052 | */ |
1053 | |
1054 | #ifdef __ARM_16K_PG__ |
1055 | #define ARM_PTE_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ |
1056 | #define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */ |
1057 | #define ARM_PTE_SHIFT 14 /* page descriptor shift */ |
1058 | #define ARM_PTE_MASK 0x0000ffffffffc000ULL /* mask for output address in PTE */ |
1059 | #else |
1060 | #define ARM_PTE_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ |
1061 | #define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */ |
1062 | #define ARM_PTE_SHIFT 12 /* page descriptor shift */ |
1063 | #define ARM_PTE_MASK 0x0000fffffffff000ULL /* mask for output address in PTE */ |
1064 | #endif |
1065 | |
1066 | /* |
1067 | * L3 Page table entries |
1068 | * |
1069 | * The following page table entry types are possible: |
1070 | * |
1071 | * fault page entry |
1072 | * 63 2 0 |
1073 | * +------------------------------+--+ |
1074 | * | ignored |00| |
1075 | * +------------------------------+--+ |
1076 | * |
1077 | * |
1078 | * 63 59 58 55 54 53 52 51 48 47 12 11 10 9 8 7 6 5 4 2 1 0 |
1079 | * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+ |
1080 | * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V| |
1081 | * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+ |
1082 | * |
1083 | * where: |
1084 | * 'nG' notGlobal bit |
1085 | * 'SH' Shareability field |
1086 | * 'AP' access protection |
1087 | * 'XN' eXecute Never bit |
1088 | * 'PXN' Privilege eXecute Never bit |
1089 | * 'NS' Non-Secure bit |
1090 | * 'HINT' 16 entry continuguous output hint |
1091 | * 'AttrIdx' Memory Attribute Index |
1092 | */ |
1093 | |
1094 | #define PTE_SHIFT 3 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */ |
1095 | #ifdef __ARM_16K_PG__ |
1096 | #define PTE_PGENTRIES (16384 >> PTE_SHIFT) /* number of ptes per page */ |
1097 | #else |
1098 | #define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */ |
1099 | #endif |
1100 | |
1101 | #define ARM_PTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */ |
1102 | |
1103 | /* markers for (invalid) PTE for a page sent to compressor */ |
1104 | #define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */ |
1105 | #define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */ |
1106 | #define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL |
1107 | #define ARM_PTE_IS_COMPRESSED(x) \ |
1108 | ((((x) & 0x3) == 0) && /* PTE is not valid... */ \ |
1109 | ((x) & ARM_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \ |
1110 | ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \ |
1111 | (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \ |
1112 | &(x), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE))) |
1113 | |
1114 | #define ARM_PTE_TYPE 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ |
1115 | #define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ |
1116 | #define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */ |
1117 | #define ARM_PTE_TYPE_MASK 0x0000000000000002ULL /* mask to get pte type */ |
1118 | |
1119 | #ifdef __ARM_16K_PG__ |
1120 | /* TODO: What does the shift mean here? */ |
1121 | #define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFC000ULL /* mask for 16KB page */ |
1122 | #else |
1123 | #define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* mask for 4KB page */ |
1124 | #define ARM_PTE_PAGE_SHIFT 12 /* page shift for 4KB page */ |
1125 | #endif |
1126 | |
1127 | #define ARM_PTE_AP(x) ((x) << 6) /* access protections */ |
1128 | #define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */ |
1129 | #define (x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */ |
1130 | |
1131 | #define ARM_PTE_ATTRINDX(x) ((x) << 2) /* memory attributes index */ |
1132 | #define ARM_PTE_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ |
1133 | |
1134 | #define ARM_PTE_SH(x) ((x) << 8) /* access shared */ |
1135 | #define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */ |
1136 | |
1137 | #define ARM_PTE_AF 0x0000000000000400ULL /* value for access */ |
1138 | #define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */ |
1139 | |
1140 | #define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */ |
1141 | #define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ |
1142 | |
1143 | #define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */ |
1144 | #define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ |
1145 | |
1146 | #define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */ |
1147 | #define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */ |
1148 | |
1149 | #if __ARM_16K_PG__ |
1150 | #define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */ |
1151 | #define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */ |
1152 | #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */ |
1153 | #define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */ |
1154 | #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */ |
1155 | #else |
1156 | #define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */ |
1157 | #define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */ |
1158 | #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */ |
1159 | #define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */ |
1160 | #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */ |
1161 | #endif |
1162 | |
1163 | #define ARM_PTE_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ |
1164 | #define ARM_PTE_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ |
1165 | |
1166 | #define ARM_PTE_NX 0x0040000000000000ULL /* value for no execute bit */ |
1167 | #define ARM_PTE_NXMASK 0x0040000000000000ULL /* no execute mask */ |
1168 | |
1169 | #define ARM_PTE_WIRED 0x0080000000000000ULL /* value for software wired bit */ |
1170 | #define ARM_PTE_WIRED_MASK 0x0080000000000000ULL /* software wired mask */ |
1171 | |
1172 | #define ARM_PTE_WRITEABLE 0x0100000000000000ULL /* value for software writeable bit */ |
1173 | #define ARM_PTE_WRITEABLE_MASK 0x0100000000000000ULL /* software writeable mask */ |
1174 | |
1175 | #if CONFIG_PGTRACE |
1176 | #define ARM_PTE_PGTRACE 0x0200000000000000ULL /* value for software trace bit */ |
1177 | #define ARM_PTE_PGTRACE_MASK 0x0200000000000000ULL /* software trace mask */ |
1178 | #endif |
1179 | |
1180 | #define ARM_PTE_BOOT_PAGE_BASE (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) \ |
1181 | | ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF) |
1182 | |
1183 | #if __ARM_KERNEL_PROTECT__ |
1184 | #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG) |
1185 | #else /* __ARM_KERNEL_PROTECT__ */ |
1186 | #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE) |
1187 | #endif /* __ARM_KERNEL_PROTECT__ */ |
1188 | |
1189 | /* |
1190 | * TLBI appers to only deal in 4KB page addresses, so give |
1191 | * it an explicit shift of 12. |
1192 | */ |
1193 | #define TLBI_ADDR_SIZE (44) |
1194 | #define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1) |
1195 | #define TLBI_ADDR_SHIFT (12) |
1196 | #define TLBI_ASID_SHIFT (48) |
1197 | #define TLBI_ASID_SIZE (16) |
1198 | #define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1) << TLBI_ASID_SHIFT) |
1199 | |
1200 | /* |
1201 | * Exception Syndrome Register |
1202 | * |
1203 | * 31 26 25 24 0 |
1204 | * +------+--+------------------+ |
1205 | * | EC |IL| ISS | |
1206 | * +------+--+------------------+ |
1207 | * |
1208 | * EC - Exception Class |
1209 | * IL - Instruction Length |
1210 | * ISS- Instruction Specific Syndrome |
1211 | * |
1212 | * Note: The ISS can have many forms. These are defined separately below. |
1213 | */ |
1214 | |
1215 | #define ESR_EC_SHIFT 26 |
1216 | #define ESR_EC_MASK (0x3F << ESR_EC_SHIFT) |
1217 | #define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT) |
1218 | |
1219 | #define ESR_IL_SHIFT 25 |
1220 | #define ESR_IL (1 << ESR_IL_SHIFT) |
1221 | |
1222 | #define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL)) |
1223 | |
1224 | #define ESR_ISS_MASK 0x01FFFFFF |
1225 | #define ESR_ISS(x) (x & ESR_ISS_MASK) |
1226 | |
1227 | #ifdef __ASSEMBLER__ |
1228 | /* Define only the classes we need to test in the exception vectors. */ |
1229 | #define ESR_EC_IABORT_EL1 0x21 |
1230 | #define ESR_EC_DABORT_EL1 0x25 |
1231 | #define ESR_EC_SP_ALIGN 0x26 |
1232 | #else |
1233 | typedef enum { |
1234 | ESR_EC_UNCATEGORIZED = 0x00, |
1235 | ESR_EC_WFI_WFE = 0x01, |
1236 | ESR_EC_MCR_MRC_CP15_TRAP = 0x03, |
1237 | ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04, |
1238 | ESR_EC_MCR_MRC_CP14_TRAP = 0x05, |
1239 | ESR_EC_LDC_STC_CP14_TRAP = 0x06, |
1240 | ESR_EC_TRAP_SIMD_FP = 0x07, |
1241 | ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c, |
1242 | ESR_EC_ILLEGAL_INSTR_SET = 0x0e, |
1243 | ESR_EC_SVC_32 = 0x11, |
1244 | ESR_EC_SVC_64 = 0x15, |
1245 | ESR_EC_MSR_TRAP = 0x18, |
1246 | ESR_EC_IABORT_EL0 = 0x20, |
1247 | ESR_EC_IABORT_EL1 = 0x21, |
1248 | ESR_EC_PC_ALIGN = 0x22, |
1249 | ESR_EC_DABORT_EL0 = 0x24, |
1250 | ESR_EC_DABORT_EL1 = 0x25, |
1251 | ESR_EC_SP_ALIGN = 0x26, |
1252 | ESR_EC_FLOATING_POINT_32 = 0x28, |
1253 | ESR_EC_FLOATING_POINT_64 = 0x2C, |
1254 | ESR_EC_BKPT_REG_MATCH_EL0 = 0x30, // Breakpoint Debug event taken to the EL from a lower EL. |
1255 | ESR_EC_BKPT_REG_MATCH_EL1 = 0x31, // Breakpoint Debug event taken to the EL from the EL. |
1256 | ESR_EC_SW_STEP_DEBUG_EL0 = 0x32, // Software Step Debug event taken to the EL from a lower EL. |
1257 | ESR_EC_SW_STEP_DEBUG_EL1 = 0x33, // Software Step Debug event taken to the EL from the EL. |
1258 | ESR_EC_WATCHPT_MATCH_EL0 = 0x34, // Watchpoint Debug event taken to the EL from a lower EL. |
1259 | ESR_EC_WATCHPT_MATCH_EL1 = 0x35, // Watchpoint Debug event taken to the EL from the EL. |
1260 | ESR_EC_BKPT_AARCH32 = 0x38, |
1261 | ESR_EC_BRK_AARCH64 = 0x3C |
1262 | } esr_exception_class_t; |
1263 | |
1264 | typedef enum { |
1265 | FSC_TRANSLATION_FAULT_L0 = 0x04, |
1266 | FSC_TRANSLATION_FAULT_L1 = 0x05, |
1267 | FSC_TRANSLATION_FAULT_L2 = 0x06, |
1268 | FSC_TRANSLATION_FAULT_L3 = 0x07, |
1269 | FSC_ACCESS_FLAG_FAULT_L1 = 0x09, |
1270 | FSC_ACCESS_FLAG_FAULT_L2 = 0x0A, |
1271 | FSC_ACCESS_FLAG_FAULT_L3 = 0x0B, |
1272 | FSC_PERMISSION_FAULT_L1 = 0x0D, |
1273 | FSC_PERMISSION_FAULT_L2 = 0x0E, |
1274 | FSC_PERMISSION_FAULT_L3 = 0x0F, |
1275 | FSC_SYNC_EXT_ABORT = 0x10, |
1276 | FSC_ASYNC_EXT_ABORT = 0x11, |
1277 | FSC_SYNC_EXT_ABORT_TT_L1 = 0x15, |
1278 | FSC_SYNC_EXT_ABORT_TT_L2 = 0x16, |
1279 | FSC_SYNC_EXT_ABORT_TT_L3 = 0x17, |
1280 | FSC_SYNC_PARITY = 0x18, |
1281 | FSC_ASYNC_PARITY = 0x19, |
1282 | FSC_SYNC_PARITY_TT_L1 = 0x1D, |
1283 | FSC_SYNC_PARITY_TT_L2 = 0x1E, |
1284 | FSC_SYNC_PARITY_TT_L3 = 0x1F, |
1285 | FSC_ALIGNMENT_FAULT = 0x21, |
1286 | FSC_DEBUG_FAULT = 0x22 |
1287 | } fault_status_t; |
1288 | #endif /* ASSEMBLER */ |
1289 | |
1290 | /* |
1291 | * Software step debug event ISS (EL1) |
1292 | * 24 23 6 5 0 |
1293 | * +---+-----------------+--+------+ |
1294 | * |ISV|00000000000000000|EX| IFSC | |
1295 | * +---+-----------------+--+------+ |
1296 | * |
1297 | * where: |
1298 | * ISV Instruction syndrome valid |
1299 | * EX Exclusive access |
1300 | * IFSC Instruction Fault Status Code |
1301 | */ |
1302 | |
1303 | #define ISS_SSDE_ISV_SHIFT 24 |
1304 | #define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT) |
1305 | |
1306 | #define ISS_SSDE_EX_SHIFT 6 |
1307 | #define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT) |
1308 | |
1309 | #define ISS_SSDE_FSC_MASK 0x3F |
1310 | #define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK) |
1311 | |
1312 | /* |
1313 | * Instruction Abort ISS (EL1) |
1314 | * 24 10 9 5 0 |
1315 | * +---------------+--+---+------+ |
1316 | * |000000000000000|EA|000| IFSC | |
1317 | * +---------------+--+---+------+ |
1318 | * |
1319 | * where: |
1320 | * EA External Abort type |
1321 | * IFSC Instruction Fault Status Code |
1322 | */ |
1323 | |
1324 | #define ISS_IA_EA_SHIFT 9 |
1325 | #define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT) |
1326 | |
1327 | #define ISS_IA_FSC_MASK 0x3F |
1328 | #define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK) |
1329 | |
1330 | |
1331 | /* |
1332 | * Data Abort ISS (EL1) |
1333 | * |
1334 | * 24 9 8 7 6 5 0 |
1335 | * +---------------+--+--+-+---+----+ |
1336 | * |000000000000000|EA|CM|0|WnR|DFSC| |
1337 | * +---------------+--+--+-+---+----+ |
1338 | * |
1339 | * where: |
1340 | * EA External Abort type |
1341 | * CM Cache Maintenance operation |
1342 | * WnR Write not Read |
1343 | * DFSC Data Fault Status Code |
1344 | */ |
1345 | #define ISS_DA_EA_SHIFT 9 |
1346 | #define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT) |
1347 | |
1348 | #define ISS_DA_CM_SHIFT 8 |
1349 | #define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT) |
1350 | |
1351 | #define ISS_DA_WNR_SHIFT 6 |
1352 | #define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT) |
1353 | |
1354 | #define ISS_DA_FSC_MASK 0x3F |
1355 | #define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK) |
1356 | |
1357 | /* |
1358 | * Physical Address Register (EL1) |
1359 | */ |
1360 | #define PAR_F_SHIFT 0 |
1361 | #define PAR_F (0x1 << PAR_F_SHIFT) |
1362 | |
1363 | #define PLATFORM_SYSCALL_TRAP_NO 0x80000000 |
1364 | |
1365 | #define ARM64_SYSCALL_CODE_REG_NUM (16) |
1366 | |
1367 | #define ARM64_CLINE_SHIFT 6 |
1368 | |
1369 | #if defined(APPLE_ARM64_ARCH_FAMILY) |
1370 | #define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */ |
1371 | #define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */ |
1372 | #endif |
1373 | |
1374 | /* |
1375 | * Timer definitions. |
1376 | */ |
1377 | #define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */ |
1378 | #define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */ |
1379 | #define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */ |
1380 | #define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */ |
1381 | #define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */ |
1382 | #define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */ |
1383 | #define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to physical timebase + frequency reg enabled */ |
1384 | #define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to virtual timebase + frequency reg enabled */ |
1385 | |
1386 | #define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */ |
1387 | #define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */ |
1388 | #define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */ |
1389 | |
1390 | #define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS |
1391 | #define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED |
1392 | #define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE |
1393 | |
1394 | /* |
1395 | * At present all other uses of ARM_DBG_* are shared bit compatibly with the 32bit definitons. |
1396 | * (cf. osfmk/arm/proc_reg.h) |
1397 | */ |
1398 | #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */ |
1399 | |
1400 | #define MIDR_EL1_REV_SHIFT 0 |
1401 | #define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT) |
1402 | #define MIDR_EL1_PNUM_SHIFT 4 |
1403 | #define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT) |
1404 | #define MIDR_EL1_ARCH_SHIFT 16 |
1405 | #define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT) |
1406 | #define MIDR_EL1_VAR_SHIFT 20 |
1407 | #define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT) |
1408 | #define MIDR_EL1_IMP_SHIFT 24 |
1409 | #define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT) |
1410 | |
1411 | /* |
1412 | * CoreSight debug registers |
1413 | */ |
1414 | #define CORESIGHT_ED 0 |
1415 | #define CORESIGHT_CTI 1 |
1416 | #define CORESIGHT_PMU 2 |
1417 | #define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */ |
1418 | |
1419 | #define CORESIGHT_OFFSET(x) ((x) * 0x10000) |
1420 | #define CORESIGHT_REGIONS 4 |
1421 | #define CORESIGHT_SIZE 0x1000 |
1422 | |
1423 | |
1424 | /* |
1425 | * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0 |
1426 | * |
1427 | * 63 24 23 20 19 16 15 12 11 8 7 4 3 0 |
1428 | * +----------+--------+------+------+------+-----+------+ |
1429 | * | reserved | atomic |crc32 | sha2 | sha1 | aes | res0 | |
1430 | * +----------+--------+------+------+------+-----+------+ |
1431 | */ |
1432 | |
1433 | #define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20 |
1434 | #define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) |
1435 | #define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) |
1436 | |
1437 | #define ID_AA64ISAR0_EL1_CRC32_OFFSET 16 |
1438 | #define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET) |
1439 | #define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET) |
1440 | |
1441 | #define ID_AA64ISAR0_EL1_SHA2_OFFSET 12 |
1442 | #define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET) |
1443 | #define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) |
1444 | |
1445 | #define ID_AA64ISAR0_EL1_SHA1_OFFSET 8 |
1446 | #define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET) |
1447 | #define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET) |
1448 | |
1449 | #define ID_AA64ISAR0_EL1_AES_OFFSET 4 |
1450 | #define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET) |
1451 | #define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET) |
1452 | #define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET) |
1453 | |
1454 | |
1455 | |
1456 | #ifdef __ASSEMBLER__ |
1457 | |
1458 | /* |
1459 | * Compute CPU version: |
1460 | * Version is constructed as [4 bits of MIDR variant]:[4 bits of MIDR revision] |
1461 | * |
1462 | * Where the "variant" is the major number and the "revision" is the minor number. |
1463 | * |
1464 | * For example: |
1465 | * Cyclone A0 is variant 0, revision 0, i.e. 0. |
1466 | * Cyclone B0 is variant 1, revision 0, i.e. 0x10 |
1467 | * $0 - register to place value in |
1468 | */ |
1469 | .macro GET_MIDR_CPU_VERSION |
1470 | mrs $0, MIDR_EL1 // Read MIDR_EL1 for CPUID |
1471 | bfi $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #4 // move bits 3:0 (revision) to 19:16 (below variant) to get values adjacent |
1472 | ubfx $0, $0, #(MIDR_EL1_VAR_SHIFT - 4), #8 // And extract the concatenated bitstring to beginning of register |
1473 | .endmacro |
1474 | |
1475 | /* |
1476 | * To apply a workaround for CPU versions less than a given value |
1477 | * (e.g. earlier than when a fix arrived) |
1478 | * |
1479 | * $0 - scratch register1 |
1480 | * $1 - version at which to stop applying workaround |
1481 | * $2 - label to branch to (at end of workaround) |
1482 | */ |
1483 | .macro SKIP_IF_CPU_VERSION_GREATER_OR_EQUAL |
1484 | GET_MIDR_CPU_VERSION $0 |
1485 | cmp $0, $1 |
1486 | b.pl $2 // Unsigned "greater or equal" |
1487 | .endmacro |
1488 | |
1489 | /* |
1490 | * To apply a workaround for CPU versions greater than a given value |
1491 | * (e.g. starting when a bug was introduced) |
1492 | * |
1493 | * $0 - scratch register1 |
1494 | * $1 - version at which to stop applying workaround |
1495 | * $2 - label to branch to (at end of workaround) |
1496 | */ |
1497 | .macro SKIP_IF_CPU_VERSION_LESS_THAN |
1498 | GET_MIDR_CPU_VERSION $0 |
1499 | cmp $0, $1 |
1500 | b.mi $2 // Unsigned "strictly less than" |
1501 | .endmacro |
1502 | |
1503 | #endif /* __ASSEMBLER__ */ |
1504 | |
1505 | #define MSR(reg,src) __asm__ volatile ("msr " reg ", %0" :: "r" (src)) |
1506 | #define MRS(dest,reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest)) |
1507 | |
1508 | |
1509 | #endif /* _ARM64_PROC_REG_H_ */ |
1510 | |