1 | /* |
2 | * Copyright (c) 2007-2021 Apple Inc. All rights reserved. |
3 | * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. |
4 | * |
5 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
6 | * |
7 | * This file contains Original Code and/or Modifications of Original Code |
8 | * as defined in and that are subject to the Apple Public Source License |
9 | * Version 2.0 (the 'License'). You may not use this file except in |
10 | * compliance with the License. The rights granted to you under the License |
11 | * may not be used to create, or enable the creation or redistribution of, |
12 | * unlawful or unlicensed copies of an Apple operating system, or to |
13 | * circumvent, violate, or enable the circumvention or violation of, any |
14 | * terms of an Apple operating system software license agreement. |
15 | * |
16 | * Please obtain a copy of the License at |
17 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
18 | * |
19 | * The Original Code and all software distributed under the License are |
20 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
21 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
22 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
23 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
24 | * Please see the License for the specific language governing rights and |
25 | * limitations under the License. |
26 | * |
27 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
28 | */ |
29 | /* |
30 | * @OSF_COPYRIGHT@ |
31 | */ |
32 | /* |
33 | * @APPLE_FREE_COPYRIGHT@ |
34 | */ |
35 | /* |
36 | * File: arm/commpage/commpage.c |
37 | * Purpose: Set up and export a RO/RW page |
38 | */ |
39 | #include <libkern/section_keywords.h> |
40 | #include <mach/mach_types.h> |
41 | #include <mach/machine.h> |
42 | #include <mach/vm_map.h> |
43 | #include <machine/cpu_capabilities.h> |
44 | #include <machine/commpage.h> |
45 | #include <machine/config.h> |
46 | #include <machine/pmap.h> |
47 | #include <vm/vm_kern.h> |
48 | #include <vm/vm_map.h> |
49 | #include <vm/vm_protos.h> |
50 | #include <ipc/ipc_port.h> |
51 | #include <arm/cpuid.h> /* for cpuid_info() & cache_info() */ |
52 | #include <arm/misc_protos.h> |
53 | #include <arm/rtclock.h> |
54 | #include <libkern/OSAtomic.h> |
55 | #include <stdatomic.h> |
56 | #include <kern/remote_time.h> |
57 | #include <kern/smr.h> |
58 | #include <machine/atomic.h> |
59 | #include <machine/machine_remote_time.h> |
60 | #include <machine/machine_routines.h> |
61 | |
62 | #include <sys/kdebug.h> |
63 | #include <sys/random.h> |
64 | |
65 | #if CONFIG_ATM |
66 | #include <atm/atm_internal.h> |
67 | #endif |
68 | |
69 | static int commpage_cpus( void ); |
70 | |
71 | |
72 | static void commpage_init_cpu_capabilities( void ); |
73 | |
74 | SECURITY_READ_ONLY_LATE(vm_address_t) commPagePtr = 0; |
75 | SECURITY_READ_ONLY_LATE(vm_address_t) commpage_rw_addr = 0; |
76 | SECURITY_READ_ONLY_LATE(vm_address_t) commpage_kernel_ro_addr = 0; |
77 | SECURITY_READ_ONLY_LATE(uint64_t) _cpu_capabilities = 0; |
78 | SECURITY_READ_ONLY_LATE(vm_address_t) commpage_rw_text_addr = 0; |
79 | |
80 | extern user64_addr_t commpage_text64_location; |
81 | extern user32_addr_t commpage_text32_location; |
82 | |
83 | /* For sysctl access from BSD side */ |
84 | extern int gARMv8Crc32; |
85 | extern int gARMv8Gpi; |
86 | extern int gARM_FEAT_FlagM; |
87 | extern int gARM_FEAT_FlagM2; |
88 | extern int gARM_FEAT_FHM; |
89 | extern int gARM_FEAT_DotProd; |
90 | extern int gARM_FEAT_SHA3; |
91 | extern int gARM_FEAT_RDM; |
92 | extern int gARM_FEAT_LSE; |
93 | extern int gARM_FEAT_SHA256; |
94 | extern int gARM_FEAT_SHA512; |
95 | extern int gARM_FEAT_SHA1; |
96 | extern int gARM_FEAT_AES; |
97 | extern int gARM_FEAT_PMULL; |
98 | extern int gARM_FEAT_SPECRES; |
99 | extern int gARM_FEAT_SB; |
100 | extern int gARM_FEAT_FRINTTS; |
101 | extern int gARM_FEAT_LRCPC; |
102 | extern int gARM_FEAT_LRCPC2; |
103 | extern int gARM_FEAT_FCMA; |
104 | extern int gARM_FEAT_JSCVT; |
105 | extern int gARM_FEAT_PAuth; |
106 | extern int gARM_FEAT_PAuth2; |
107 | extern int gARM_FEAT_FPAC; |
108 | extern int gARM_FEAT_FPACCOMBINE; |
109 | extern int gARM_FEAT_DPB; |
110 | extern int gARM_FEAT_DPB2; |
111 | extern int gARM_FEAT_BF16; |
112 | extern int gARM_FEAT_I8MM; |
113 | extern int gARM_FEAT_RPRES; |
114 | extern int gARM_FEAT_ECV; |
115 | extern int gARM_FEAT_LSE2; |
116 | extern int gARM_FEAT_CSV2; |
117 | extern int gARM_FEAT_CSV3; |
118 | extern int gARM_FEAT_DIT; |
119 | extern int gARM_AdvSIMD; |
120 | extern int gARM_AdvSIMD_HPFPCvt; |
121 | extern int gARM_FEAT_FP16; |
122 | extern int gARM_FEAT_SSBS; |
123 | extern int gARM_FEAT_BTI; |
124 | extern int gARM_FP_SyncExceptions; |
125 | extern int gARM_FEAT_AFP; |
126 | |
127 | extern int gUCNormalMem; |
128 | |
129 | void |
130 | commpage_populate(void) |
131 | { |
132 | uint16_t c2; |
133 | int cpufamily; |
134 | |
135 | // Create the data and the text commpage |
136 | vm_map_address_t kernel_data_addr, kernel_text_addr, kernel_ro_data_addr, user_text_addr; |
137 | pmap_create_commpages(kernel_data_addr: &kernel_data_addr, kernel_text_addr: &kernel_text_addr, kernel_ro_data_addr: &kernel_ro_data_addr, user_text_addr: &user_text_addr); |
138 | |
139 | commpage_rw_addr = kernel_data_addr; |
140 | commpage_rw_text_addr = kernel_text_addr; |
141 | commpage_kernel_ro_addr = kernel_ro_data_addr; |
142 | commPagePtr = (vm_address_t) _COMM_PAGE_BASE_ADDRESS; |
143 | |
144 | #if __arm64__ |
145 | commpage_text64_location = user_text_addr; |
146 | bcopy(_COMM_PAGE64_SIGNATURE_STRING, dst: (void *)(_COMM_PAGE_SIGNATURE + _COMM_PAGE_RW_OFFSET), |
147 | MIN(_COMM_PAGE_SIGNATURELEN, strlen(_COMM_PAGE64_SIGNATURE_STRING))); |
148 | #endif |
149 | |
150 | *((uint16_t*)(_COMM_PAGE_VERSION + _COMM_PAGE_RW_OFFSET)) = (uint16_t) _COMM_PAGE_THIS_VERSION; |
151 | |
152 | commpage_init_cpu_capabilities(); |
153 | commpage_set_timestamp(tbr: 0, secs: 0, frac: 0, scale: 0, tick_per_sec: 0); |
154 | |
155 | if (_cpu_capabilities & kCache32) { |
156 | c2 = 32; |
157 | } else if (_cpu_capabilities & kCache64) { |
158 | c2 = 64; |
159 | } else if (_cpu_capabilities & kCache128) { |
160 | c2 = 128; |
161 | } else { |
162 | c2 = 0; |
163 | } |
164 | |
165 | *((uint16_t*)(_COMM_PAGE_CACHE_LINESIZE + _COMM_PAGE_RW_OFFSET)) = c2; |
166 | |
167 | commpage_update_active_cpus(); |
168 | cpufamily = cpuid_get_cpufamily(); |
169 | *((uint8_t*)(_COMM_PAGE_CPU_CLUSTERS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) ml_get_cluster_count(); |
170 | *((uint8_t*)(_COMM_PAGE_PHYSICAL_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.physical_cpu_max; |
171 | *((uint8_t*)(_COMM_PAGE_LOGICAL_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.logical_cpu_max; |
172 | *((uint64_t*)(_COMM_PAGE_MEMORY_SIZE + _COMM_PAGE_RW_OFFSET)) = machine_info.max_mem; |
173 | *((uint32_t*)(_COMM_PAGE_CPUFAMILY + _COMM_PAGE_RW_OFFSET)) = (uint32_t)cpufamily; |
174 | *((uint32_t*)(_COMM_PAGE_DEV_FIRM_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL); |
175 | *((uint32_t*)(_COMM_PAGE_DEV_FIRM + _COMM_PAGE_RO_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL); |
176 | *((uint8_t*)(_COMM_PAGE_USER_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = user_timebase_type(); |
177 | |
178 | // Populate logical CPU -> logical cluster table |
179 | ml_map_cpus_to_clusters(table: (uint8_t*)(_COMM_PAGE_CPU_TO_CLUSTER + _COMM_PAGE_RW_OFFSET)); |
180 | |
181 | *((uint8_t*)(_COMM_PAGE_CONT_HWCLOCK + _COMM_PAGE_RW_OFFSET)) = (uint8_t)user_cont_hwclock_allowed(); |
182 | *((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift; |
183 | *((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT + _COMM_PAGE_RO_OFFSET)) = (uint8_t) page_shift; |
184 | |
185 | #if __arm64__ |
186 | *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift_user32; |
187 | *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32 + _COMM_PAGE_RO_OFFSET)) = (uint8_t) page_shift_user32; |
188 | *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT; |
189 | *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64 + _COMM_PAGE_RO_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT; |
190 | #endif /* __arm64__ */ |
191 | |
192 | commpage_update_timebase(); |
193 | commpage_update_mach_continuous_time(sleeptime: 0); |
194 | |
195 | clock_sec_t secs; |
196 | clock_usec_t microsecs; |
197 | clock_get_boottime_microtime(secs: &secs, microsecs: µsecs); |
198 | commpage_update_boottime(boottime_usec: secs * USEC_PER_SEC + microsecs); |
199 | |
200 | /* |
201 | * set commpage approximate time to zero for initialization. |
202 | * scheduler shall populate correct value before running user thread |
203 | */ |
204 | *((uint64_t *)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET)) = 0; |
205 | #ifdef CONFIG_MACH_APPROXIMATE_TIME |
206 | *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED + _COMM_PAGE_RW_OFFSET)) = 1; |
207 | #else |
208 | *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED + _COMM_PAGE_RW_OFFSET)) = 0; |
209 | #endif |
210 | |
211 | commpage_update_kdebug_state(); |
212 | |
213 | #if CONFIG_ATM |
214 | commpage_update_atm_diagnostic_config(atm_get_diagnostic_config()); |
215 | #endif |
216 | |
217 | |
218 | *((uint64_t*)(_COMM_PAGE_REMOTETIME_PARAMS + _COMM_PAGE_RW_OFFSET)) = BT_RESET_SENTINEL_TS; |
219 | |
220 | #if CONFIG_QUIESCE_COUNTER |
221 | cpu_quiescent_set_storage(ptr: (_Atomic uint64_t *)(_COMM_PAGE_CPU_QUIESCENT_COUNTER + |
222 | _COMM_PAGE_RW_OFFSET)); |
223 | #endif /* CONFIG_QUIESCE_COUNTER */ |
224 | |
225 | /* |
226 | * Set random values for targets in Apple Security Bounty |
227 | * addr should be unmapped for userland processes |
228 | * kaddr should be unmapped for kernel |
229 | */ |
230 | uint64_t asb_value, asb_addr, asb_kvalue, asb_kaddr; |
231 | uint64_t asb_rand_vals[] = { |
232 | 0x93e78adcded4d3d5, 0xd16c5b76ad99bccf, 0x67dfbbd12c4a594e, 0x7365636e6f6f544f, |
233 | 0x239a974c9811e04b, 0xbf60e7fa45741446, 0x8acf5210b466b05, 0x67dfbbd12c4a594e |
234 | }; |
235 | const int nrandval = sizeof(asb_rand_vals) / sizeof(asb_rand_vals[0]); |
236 | uint8_t randidx; |
237 | |
238 | read_random(buffer: &randidx, numBytes: sizeof(uint8_t)); |
239 | asb_value = asb_rand_vals[randidx++ % nrandval]; |
240 | *((uint64_t*)(_COMM_PAGE_ASB_TARGET_VALUE + _COMM_PAGE_RW_OFFSET)) = asb_value; |
241 | |
242 | // userspace faulting address should be > MACH_VM_MAX_ADDRESS |
243 | asb_addr = asb_rand_vals[randidx++ % nrandval]; |
244 | uint64_t user_min = MACH_VM_MAX_ADDRESS; |
245 | uint64_t user_max = UINT64_MAX; |
246 | asb_addr %= (user_max - user_min); |
247 | asb_addr += user_min; |
248 | *((uint64_t*)(_COMM_PAGE_ASB_TARGET_ADDRESS + _COMM_PAGE_RW_OFFSET)) = asb_addr; |
249 | |
250 | asb_kvalue = asb_rand_vals[randidx++ % nrandval]; |
251 | *((uint64_t*)(_COMM_PAGE_ASB_TARGET_KERN_VALUE + _COMM_PAGE_RW_OFFSET)) = asb_kvalue; |
252 | |
253 | // kernel faulting address should be < VM_MIN_KERNEL_ADDRESS |
254 | asb_kaddr = asb_rand_vals[randidx++ % nrandval]; |
255 | uint64_t kernel_min = 0x0LL; |
256 | uint64_t kernel_max = VM_MIN_KERNEL_ADDRESS; |
257 | asb_kaddr %= (kernel_max - kernel_min); |
258 | asb_kaddr += kernel_min; |
259 | *((uint64_t*)(_COMM_PAGE_ASB_TARGET_KERN_ADDRESS + _COMM_PAGE_RW_OFFSET)) = asb_kaddr; |
260 | } |
261 | |
262 | #define COMMPAGE_TEXT_SEGMENT "__TEXT_EXEC" |
263 | #define COMMPAGE_TEXT_SECTION "__commpage_text" |
264 | |
265 | /* Get a pointer to the start of the ARM PFZ code section. This macro tell the |
266 | * linker that the storage for the variable here is at the start of the section */ |
267 | extern char commpage_text_start[] |
268 | __SECTION_START_SYM(COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION); |
269 | |
270 | /* Get a pointer to the end of the ARM PFZ code section. This macro tell the |
271 | * linker that the storage for the variable here is at the end of the section */ |
272 | extern char commpage_text_end[] |
273 | __SECTION_END_SYM(COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION); |
274 | |
275 | /* This is defined in the commpage text section as a symbol at the start of the preemptible |
276 | * functions */ |
277 | extern char commpage_text_preemptible_functions; |
278 | |
279 | #if CONFIG_ARM_PFZ |
280 | static size_t size_of_pfz = 0; |
281 | #endif |
282 | |
283 | /* This is the opcode for brk #666 */ |
284 | #define BRK_666_OPCODE 0xD4205340 |
285 | |
286 | void |
287 | commpage_text_populate(void) |
288 | { |
289 | #if CONFIG_ARM_PFZ |
290 | size_t size_of_commpage_text = commpage_text_end - commpage_text_start; |
291 | if (size_of_commpage_text == 0) { |
292 | panic("ARM comm page text section %s,%s missing" , COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION); |
293 | } |
294 | assert(size_of_commpage_text <= PAGE_SIZE); |
295 | assert(size_of_commpage_text > 0); |
296 | |
297 | /* Get the size of the PFZ half of the comm page text section. */ |
298 | size_of_pfz = &commpage_text_preemptible_functions - commpage_text_start; |
299 | |
300 | // Copy the code segment of comm page text section into the PFZ |
301 | memcpy(dst: (void *) _COMM_PAGE64_TEXT_START_ADDRESS, src: (void *) commpage_text_start, n: size_of_commpage_text); |
302 | |
303 | // Make sure to populate the rest of it with brk 666 so that undefined code |
304 | // doesn't get run |
305 | memset(s: (char *) _COMM_PAGE64_TEXT_START_ADDRESS + size_of_commpage_text, BRK_666_OPCODE, |
306 | PAGE_SIZE - size_of_commpage_text); |
307 | #endif |
308 | } |
309 | |
310 | uint32_t |
311 | commpage_is_in_pfz64(addr64_t addr64) |
312 | { |
313 | #if CONFIG_ARM_PFZ |
314 | if ((addr64 >= commpage_text64_location) && |
315 | (addr64 < (commpage_text64_location + size_of_pfz))) { |
316 | return 1; |
317 | } else { |
318 | return 0; |
319 | } |
320 | #else |
321 | #pragma unused (addr64) |
322 | return 0; |
323 | #endif |
324 | } |
325 | |
326 | |
327 | void |
328 | commpage_set_timestamp( |
329 | uint64_t tbr, |
330 | uint64_t secs, |
331 | uint64_t frac, |
332 | uint64_t scale, |
333 | uint64_t tick_per_sec) |
334 | { |
335 | new_commpage_timeofday_data_t *commpage_timeofday_datap; |
336 | |
337 | if (commPagePtr == 0) { |
338 | return; |
339 | } |
340 | |
341 | commpage_timeofday_datap = (new_commpage_timeofday_data_t *)(_COMM_PAGE_NEWTIMEOFDAY_DATA + _COMM_PAGE_RW_OFFSET); |
342 | |
343 | commpage_timeofday_datap->TimeStamp_tick = 0x0ULL; |
344 | |
345 | __builtin_arm_dmb(DMB_ISH); |
346 | |
347 | commpage_timeofday_datap->TimeStamp_sec = secs; |
348 | commpage_timeofday_datap->TimeStamp_frac = frac; |
349 | commpage_timeofday_datap->Ticks_scale = scale; |
350 | commpage_timeofday_datap->Ticks_per_sec = tick_per_sec; |
351 | |
352 | __builtin_arm_dmb(DMB_ISH); |
353 | |
354 | commpage_timeofday_datap->TimeStamp_tick = tbr; |
355 | |
356 | } |
357 | |
358 | /* |
359 | * Update _COMM_PAGE_MEMORY_PRESSURE. Called periodically from vm's compute_memory_pressure() |
360 | */ |
361 | |
362 | void |
363 | commpage_set_memory_pressure( |
364 | unsigned int pressure ) |
365 | { |
366 | if (commPagePtr == 0) { |
367 | return; |
368 | } |
369 | *((uint32_t *)(_COMM_PAGE_MEMORY_PRESSURE + _COMM_PAGE_RW_OFFSET)) = pressure; |
370 | } |
371 | |
372 | /* |
373 | * Determine number of CPUs on this system. |
374 | */ |
375 | static int |
376 | commpage_cpus( void ) |
377 | { |
378 | int cpus; |
379 | |
380 | cpus = machine_info.max_cpus; |
381 | |
382 | if (cpus == 0) { |
383 | panic("commpage cpus==0" ); |
384 | } |
385 | if (cpus > 0xFF) { |
386 | cpus = 0xFF; |
387 | } |
388 | |
389 | return cpus; |
390 | } |
391 | |
392 | uint64_t |
393 | _get_cpu_capabilities(void) |
394 | { |
395 | return _cpu_capabilities; |
396 | } |
397 | |
398 | vm_address_t |
399 | _get_commpage_priv_address(void) |
400 | { |
401 | return commpage_rw_addr; |
402 | } |
403 | |
404 | vm_address_t |
405 | _get_commpage_ro_address(void) |
406 | { |
407 | return commpage_kernel_ro_addr; |
408 | } |
409 | |
410 | vm_address_t |
411 | _get_commpage_text_priv_address(void) |
412 | { |
413 | return commpage_rw_text_addr; |
414 | } |
415 | |
416 | #if defined(__arm64__) |
417 | /** |
418 | * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64ISAR0_EL1 |
419 | */ |
420 | static void |
421 | commpage_init_arm_optional_features_isar0(uint64_t *commpage_bits) |
422 | { |
423 | uint64_t bits = 0; |
424 | uint64_t isar0 = __builtin_arm_rsr64("ID_AA64ISAR0_EL1" ); |
425 | |
426 | if ((isar0 & ID_AA64ISAR0_EL1_TS_MASK) >= ID_AA64ISAR0_EL1_TS_FLAGM_EN) { |
427 | gARM_FEAT_FlagM = 1; |
428 | bits |= kHasFEATFlagM; |
429 | } |
430 | if ((isar0 & ID_AA64ISAR0_EL1_TS_MASK) >= ID_AA64ISAR0_EL1_TS_FLAGM2_EN) { |
431 | gARM_FEAT_FlagM2 = 1; |
432 | bits |= kHasFEATFlagM2; |
433 | } |
434 | if ((isar0 & ID_AA64ISAR0_EL1_FHM_MASK) >= ID_AA64ISAR0_EL1_FHM_8_2) { |
435 | gARM_FEAT_FHM = 1; |
436 | bits |= kHasFeatFHM; |
437 | } |
438 | if ((isar0 & ID_AA64ISAR0_EL1_DP_MASK) >= ID_AA64ISAR0_EL1_DP_EN) { |
439 | gARM_FEAT_DotProd = 1; |
440 | bits |= kHasFeatDotProd; |
441 | } |
442 | if ((isar0 & ID_AA64ISAR0_EL1_SHA3_MASK) >= ID_AA64ISAR0_EL1_SHA3_EN) { |
443 | gARM_FEAT_SHA3 = 1; |
444 | bits |= kHasFeatSHA3; |
445 | } |
446 | if ((isar0 & ID_AA64ISAR0_EL1_RDM_MASK) >= ID_AA64ISAR0_EL1_RDM_EN) { |
447 | gARM_FEAT_RDM = 1; |
448 | bits |= kHasFeatRDM; |
449 | } |
450 | if ((isar0 & ID_AA64ISAR0_EL1_ATOMIC_MASK) >= ID_AA64ISAR0_EL1_ATOMIC_8_1) { |
451 | gARM_FEAT_LSE = 1; |
452 | bits |= kHasFeatLSE; |
453 | } |
454 | if ((isar0 & ID_AA64ISAR0_EL1_SHA2_MASK) >= ID_AA64ISAR0_EL1_SHA2_512_EN) { |
455 | gARM_FEAT_SHA512 = 1; |
456 | bits |= kHasFeatSHA512; |
457 | } |
458 | if ((isar0 & ID_AA64ISAR0_EL1_CRC32_MASK) == ID_AA64ISAR0_EL1_CRC32_EN) { |
459 | gARMv8Crc32 = 1; |
460 | bits |= kHasARMv8Crc32; |
461 | } |
462 | |
463 | #if __ARM_V8_CRYPTO_EXTENSIONS__ |
464 | /** |
465 | * T7000 has a bug in the ISAR0 register that reports that PMULL is not |
466 | * supported when it actually is. To work around this, for all of the crypto |
467 | * extensions, just check if they're supported using the board_config.h |
468 | * values. |
469 | */ |
470 | gARM_FEAT_PMULL = 1; |
471 | gARM_FEAT_SHA1 = 1; |
472 | gARM_FEAT_AES = 1; |
473 | gARM_FEAT_SHA256 = 1; |
474 | bits |= kHasARMv8Crypto; |
475 | #endif /* __ARM_V8_CRYPTO_EXTENSIONS__ */ |
476 | |
477 | *commpage_bits |= bits; |
478 | } |
479 | |
480 | /** |
481 | * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64ISAR1_EL1 |
482 | */ |
483 | static void |
484 | commpage_init_arm_optional_features_isar1(uint64_t *commpage_bits) |
485 | { |
486 | uint64_t bits = 0; |
487 | uint64_t isar1 = __builtin_arm_rsr64("ID_AA64ISAR1_EL1" ); |
488 | uint64_t sctlr = __builtin_arm_rsr64("SCTLR_EL1" ); |
489 | |
490 | if ((isar1 & ID_AA64ISAR1_EL1_SPECRES_MASK) >= ID_AA64ISAR1_EL1_SPECRES_EN && |
491 | sctlr & SCTLR_EnRCTX) { |
492 | gARM_FEAT_SPECRES = 1; |
493 | bits |= kHasFeatSPECRES; |
494 | } |
495 | if ((isar1 & ID_AA64ISAR1_EL1_SB_MASK) >= ID_AA64ISAR1_EL1_SB_EN) { |
496 | gARM_FEAT_SB = 1; |
497 | bits |= kHasFeatSB; |
498 | } |
499 | if ((isar1 & ID_AA64ISAR1_EL1_FRINTTS_MASK) >= ID_AA64ISAR1_EL1_FRINTTS_EN) { |
500 | gARM_FEAT_FRINTTS = 1; |
501 | bits |= kHasFeatFRINTTS; |
502 | } |
503 | if ((isar1 & ID_AA64ISAR1_EL1_GPI_MASK) >= ID_AA64ISAR1_EL1_GPI_EN) { |
504 | gARMv8Gpi = 1; |
505 | bits |= kHasArmv8GPI; |
506 | } |
507 | if ((isar1 & ID_AA64ISAR1_EL1_LRCPC_MASK) >= ID_AA64ISAR1_EL1_LRCPC_EN) { |
508 | gARM_FEAT_LRCPC = 1; |
509 | bits |= kHasFeatLRCPC; |
510 | } |
511 | if ((isar1 & ID_AA64ISAR1_EL1_LRCPC_MASK) >= ID_AA64ISAR1_EL1_LRCP2C_EN) { |
512 | gARM_FEAT_LRCPC2 = 1; |
513 | bits |= kHasFeatLRCPC2; |
514 | } |
515 | if ((isar1 & ID_AA64ISAR1_EL1_FCMA_MASK) >= ID_AA64ISAR1_EL1_FCMA_EN) { |
516 | gARM_FEAT_FCMA = 1; |
517 | bits |= kHasFeatFCMA; |
518 | } |
519 | if ((isar1 & ID_AA64ISAR1_EL1_JSCVT_MASK) >= ID_AA64ISAR1_EL1_JSCVT_EN) { |
520 | gARM_FEAT_JSCVT = 1; |
521 | bits |= kHasFeatJSCVT; |
522 | } |
523 | if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_PAuth_EN) { |
524 | gARM_FEAT_PAuth = 1; |
525 | bits |= kHasFeatPAuth; |
526 | } |
527 | if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_PAuth2_EN) { |
528 | gARM_FEAT_PAuth2 = 1; |
529 | } |
530 | if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_FPAC_EN) { |
531 | gARM_FEAT_FPAC = 1; |
532 | } |
533 | if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_FPACCOMBINE) { |
534 | gARM_FEAT_FPACCOMBINE = 1; |
535 | } |
536 | if ((isar1 & ID_AA64ISAR1_EL1_DPB_MASK) >= ID_AA64ISAR1_EL1_DPB_EN) { |
537 | gARM_FEAT_DPB = 1; |
538 | bits |= kHasFeatDPB; |
539 | } |
540 | if ((isar1 & ID_AA64ISAR1_EL1_DPB_MASK) >= ID_AA64ISAR1_EL1_DPB2_EN) { |
541 | gARM_FEAT_DPB2 = 1; |
542 | bits |= kHasFeatDPB2; |
543 | } |
544 | if ((isar1 & ID_AA64ISAR1_EL1_BF16_MASK) >= ID_AA64ISAR1_EL1_BF16_EN) { |
545 | gARM_FEAT_BF16 = 1; |
546 | } |
547 | if ((isar1 & ID_AA64ISAR1_EL1_I8MM_MASK) >= ID_AA64ISAR1_EL1_I8MM_EN) { |
548 | gARM_FEAT_I8MM = 1; |
549 | } |
550 | |
551 | *commpage_bits |= bits; |
552 | } |
553 | |
554 | /** |
555 | * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64ISAR2_EL1 |
556 | */ |
557 | static void |
558 | commpage_init_arm_optional_features_isar2(void) |
559 | { |
560 | uint64_t isar2 = __builtin_arm_rsr64("ID_AA64ISAR2_EL1" ); |
561 | |
562 | if ((isar2 & ID_AA64ISAR2_EL1_RPRES_MASK) >= ID_AA64ISAR2_EL1_RPRES_EN) { |
563 | gARM_FEAT_RPRES = 1; |
564 | } |
565 | } |
566 | |
567 | /** |
568 | * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64MMFR0_EL1 |
569 | */ |
570 | static void |
571 | commpage_init_arm_optional_features_mmfr0(uint64_t *commpage_bits) |
572 | { |
573 | uint64_t bits = 0; |
574 | uint64_t mmfr0 = __builtin_arm_rsr64("ID_AA64MMFR0_EL1" ); |
575 | |
576 | if ((mmfr0 & ID_AA64MMFR0_EL1_ECV_MASK) >= ID_AA64MMFR0_EL1_ECV_EN) { |
577 | gARM_FEAT_ECV = 1; |
578 | } |
579 | |
580 | *commpage_bits |= bits; |
581 | } |
582 | |
583 | /** |
584 | * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64MMFR2_EL1 |
585 | */ |
586 | static void |
587 | commpage_init_arm_optional_features_mmfr2(uint64_t *commpage_bits) |
588 | { |
589 | uint64_t bits = 0; |
590 | uint64_t mmfr2 = __builtin_arm_rsr64("ID_AA64MMFR2_EL1" ); |
591 | |
592 | if ((mmfr2 & ID_AA64MMFR2_EL1_AT_MASK) >= ID_AA64MMFR2_EL1_AT_LSE2_EN) { |
593 | gARM_FEAT_LSE2 = 1; |
594 | bits |= kHasFeatLSE2; |
595 | } |
596 | |
597 | *commpage_bits |= bits; |
598 | } |
599 | |
600 | /** |
601 | * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64PFR0_EL1 |
602 | */ |
603 | static void |
604 | commpage_init_arm_optional_features_pfr0(uint64_t *commpage_bits) |
605 | { |
606 | uint64_t bits = 0; |
607 | uint64_t pfr0 = __builtin_arm_rsr64("ID_AA64PFR0_EL1" ); |
608 | |
609 | if ((pfr0 & ID_AA64PFR0_EL1_CSV3_MASK) >= ID_AA64PFR0_EL1_CSV3_EN) { |
610 | gARM_FEAT_CSV3 = 1; |
611 | bits |= kHasFeatCSV3; |
612 | } |
613 | if ((pfr0 & ID_AA64PFR0_EL1_CSV2_MASK) >= ID_AA64PFR0_EL1_CSV2_EN) { |
614 | gARM_FEAT_CSV2 = 1; |
615 | bits |= kHasFeatCSV2; |
616 | } |
617 | if ((pfr0 & ID_AA64PFR0_EL1_DIT_MASK) >= ID_AA64PFR0_EL1_DIT_EN) { |
618 | gARM_FEAT_DIT = 1; |
619 | bits |= kHasFeatDIT; |
620 | } |
621 | if ((pfr0 & ID_AA64PFR0_EL1_AdvSIMD_MASK) != ID_AA64PFR0_EL1_AdvSIMD_DIS) { |
622 | gARM_AdvSIMD = 1; |
623 | bits |= kHasAdvSIMD; |
624 | if ((pfr0 & ID_AA64PFR0_EL1_AdvSIMD_MASK) >= ID_AA64PFR0_EL1_AdvSIMD_HPFPCVT) { |
625 | gARM_AdvSIMD_HPFPCvt = 1; |
626 | bits |= kHasAdvSIMD_HPFPCvt; |
627 | } |
628 | if ((pfr0 & ID_AA64PFR0_EL1_AdvSIMD_MASK) >= ID_AA64PFR0_EL1_AdvSIMD_FP16) { |
629 | gARM_FEAT_FP16 = 1; |
630 | bits |= kHasFeatFP16; |
631 | } |
632 | } |
633 | |
634 | *commpage_bits |= bits; |
635 | } |
636 | |
637 | /** |
638 | * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64PFR1_EL1 |
639 | */ |
640 | static void |
641 | commpage_init_arm_optional_features_pfr1(uint64_t *commpage_bits) |
642 | { |
643 | uint64_t pfr1 = __builtin_arm_rsr64("ID_AA64PFR1_EL1" ); |
644 | |
645 | if ((pfr1 & ID_AA64PFR1_EL1_SSBS_MASK) >= ID_AA64PFR1_EL1_SSBS_EN) { |
646 | gARM_FEAT_SSBS = 1; |
647 | } |
648 | |
649 | if ((pfr1 & ID_AA64PFR1_EL1_BT_MASK) >= ID_AA64PFR1_EL1_BT_EN) { |
650 | gARM_FEAT_BTI = 1; |
651 | } |
652 | |
653 | #pragma unused(commpage_bits) |
654 | } |
655 | |
656 | |
657 | static void |
658 | commpage_init_arm_optional_features_mmfr1(uint64_t *commpage_bits) |
659 | { |
660 | uint64_t bits = 0; |
661 | const uint64_t mmfr1 = __builtin_arm_rsr64("ID_AA64MMFR1_EL1" ); |
662 | |
663 | if ((mmfr1 & ID_AA64MMFR1_EL1_AFP_MASK) == ID_AA64MMFR1_EL1_AFP_EN) { |
664 | gARM_FEAT_AFP = 1; |
665 | bits |= kHasFeatAFP; |
666 | } |
667 | |
668 | *commpage_bits |= bits; |
669 | } |
670 | |
671 | /** |
672 | * Read the system register @name, attempt to set set bits of @mask if not |
673 | * already, test if bits were actually set, reset the register to its |
674 | * previous value if required, and 'return' @mask with only bits that |
675 | * were successfully set (or already set) in the system register. */ |
676 | #define _test_sys_bits(name, mask) ({ \ |
677 | const uint64_t src = __builtin_arm_rsr64(#name); \ |
678 | uint64_t test = src | mask; \ |
679 | if (test != src) { \ |
680 | __builtin_arm_wsr64(#name, test); \ |
681 | test = __builtin_arm_rsr64(#name); \ |
682 | if (test != src) { \ |
683 | __builtin_arm_wsr64(#name, src); \ |
684 | }\ |
685 | } \ |
686 | mask & test; \ |
687 | }) |
688 | |
689 | /** |
690 | * Reports whether FPU exceptions are supported. |
691 | * Possible FPU exceptions are : |
692 | * - input denormal; |
693 | * - inexact; |
694 | * - underflow; |
695 | * - overflow; |
696 | * - divide by 0; |
697 | * - invalid operation. |
698 | * |
699 | * Any of those can be supported or not but for now, we consider that |
700 | * it all or nothing : FPU exceptions support flag set <=> all 6 exceptions |
701 | * a supported. |
702 | */ |
703 | static void |
704 | commpage_init_arm_optional_features_fpcr(uint64_t *commpage_bits) |
705 | { |
706 | uint64_t support_mask = FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | |
707 | FPCR_DZE | FPCR_IOE; |
708 | uint64_t FPCR_bits = _test_sys_bits(FPCR, support_mask); |
709 | if (FPCR_bits == support_mask) { |
710 | gARM_FP_SyncExceptions = 1; |
711 | *commpage_bits |= kHasFP_SyncExceptions; |
712 | } |
713 | } |
714 | |
715 | /** |
716 | * Initializes all commpage entries and sysctls for ARM64 optional features accessible from EL0. |
717 | */ |
718 | static void |
719 | commpage_init_arm_optional_features(uint64_t *commpage_bits) |
720 | { |
721 | commpage_init_arm_optional_features_isar0(commpage_bits); |
722 | commpage_init_arm_optional_features_isar1(commpage_bits); |
723 | commpage_init_arm_optional_features_isar2(); |
724 | commpage_init_arm_optional_features_mmfr0(commpage_bits); |
725 | commpage_init_arm_optional_features_mmfr1(commpage_bits); |
726 | commpage_init_arm_optional_features_mmfr2(commpage_bits); |
727 | commpage_init_arm_optional_features_pfr0(commpage_bits); |
728 | commpage_init_arm_optional_features_pfr1(commpage_bits); |
729 | commpage_init_arm_optional_features_fpcr(commpage_bits); |
730 | } |
731 | #endif /* __arm64__ */ |
732 | |
733 | /* |
734 | * Initialize _cpu_capabilities vector |
735 | */ |
736 | static void |
737 | commpage_init_cpu_capabilities( void ) |
738 | { |
739 | uint64_t bits; |
740 | int cpus; |
741 | ml_cpu_info_t cpu_info; |
742 | |
743 | bits = 0; |
744 | ml_cpu_get_info(ml_cpu_info: &cpu_info); |
745 | |
746 | switch (cpu_info.cache_line_size) { |
747 | case 128: |
748 | bits |= kCache128; |
749 | break; |
750 | case 64: |
751 | bits |= kCache64; |
752 | break; |
753 | case 32: |
754 | bits |= kCache32; |
755 | break; |
756 | default: |
757 | break; |
758 | } |
759 | cpus = commpage_cpus(); |
760 | |
761 | if (cpus == 1) { |
762 | bits |= kUP; |
763 | } |
764 | |
765 | bits |= (cpus << kNumCPUsShift); |
766 | |
767 | bits |= kFastThreadLocalStorage; // TPIDRURO for TLS |
768 | |
769 | bits |= kHasVfp; |
770 | |
771 | #if defined(__arm64__) |
772 | bits |= kHasFMA; |
773 | #endif |
774 | bits |= kHasEvent; |
775 | #ifdef __arm64__ |
776 | commpage_init_arm_optional_features(commpage_bits: &bits); |
777 | #endif |
778 | |
779 | |
780 | |
781 | #if HAS_UCNORMAL_MEM |
782 | gUCNormalMem = 1; |
783 | bits |= kHasUCNormalMemory; |
784 | #endif |
785 | |
786 | _cpu_capabilities = bits; |
787 | |
788 | *((uint32_t *)(_COMM_PAGE_CPU_CAPABILITIES + _COMM_PAGE_RW_OFFSET)) = (uint32_t)_cpu_capabilities; |
789 | *((uint64_t *)(_COMM_PAGE_CPU_CAPABILITIES64 + _COMM_PAGE_RW_OFFSET)) = _cpu_capabilities; |
790 | |
791 | } |
792 | |
793 | /* |
794 | * Updated every time a logical CPU goes offline/online |
795 | */ |
796 | void |
797 | commpage_update_active_cpus(void) |
798 | { |
799 | if (!commPagePtr) { |
800 | return; |
801 | } |
802 | *((uint8_t *)(_COMM_PAGE_ACTIVE_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t)processor_avail_count; |
803 | |
804 | } |
805 | |
806 | /* |
807 | * Update the commpage bits for mach_absolute_time and mach_continuous_time (for userspace) |
808 | */ |
809 | void |
810 | commpage_update_timebase(void) |
811 | { |
812 | if (commPagePtr) { |
813 | *((uint64_t*)(_COMM_PAGE_TIMEBASE_OFFSET + _COMM_PAGE_RW_OFFSET)) = rtclock_base_abstime; |
814 | } |
815 | } |
816 | |
817 | /* |
818 | * Update the commpage with current kdebug state: whether tracing is enabled, a |
819 | * typefilter is present, and continuous time should be used for timestamps. |
820 | * |
821 | * Disregards configuration and set to 0 if tracing is disabled. |
822 | */ |
823 | void |
824 | commpage_update_kdebug_state(void) |
825 | { |
826 | if (commPagePtr) { |
827 | uint32_t state = kdebug_commpage_state(); |
828 | *((volatile uint32_t *)(_COMM_PAGE_KDEBUG_ENABLE + _COMM_PAGE_RW_OFFSET)) = state; |
829 | } |
830 | } |
831 | |
832 | /* Ditto for atm_diagnostic_config */ |
833 | void |
834 | commpage_update_atm_diagnostic_config(uint32_t diagnostic_config) |
835 | { |
836 | if (commPagePtr) { |
837 | *((volatile uint32_t*)(_COMM_PAGE_ATM_DIAGNOSTIC_CONFIG + _COMM_PAGE_RW_OFFSET)) = diagnostic_config; |
838 | } |
839 | } |
840 | |
841 | /* |
842 | * Update the commpage data with the state of multiuser mode for |
843 | * this device. Allowing various services in userspace to avoid |
844 | * IPC in the (more common) non-multiuser environment. |
845 | */ |
846 | void |
847 | commpage_update_multiuser_config(uint32_t multiuser_config) |
848 | { |
849 | if (commPagePtr) { |
850 | *((volatile uint32_t *)(_COMM_PAGE_MULTIUSER_CONFIG + _COMM_PAGE_RW_OFFSET)) = multiuser_config; |
851 | } |
852 | } |
853 | |
854 | /* |
855 | * update the commpage data for |
856 | * last known value of mach_absolute_time() |
857 | */ |
858 | |
859 | void |
860 | commpage_update_mach_approximate_time(uint64_t abstime) |
861 | { |
862 | #ifdef CONFIG_MACH_APPROXIMATE_TIME |
863 | if (!commPagePtr) { |
864 | return; |
865 | } |
866 | |
867 | uint64_t *approx_time_base = (uint64_t *)(uintptr_t)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET); |
868 | |
869 | uint64_t saved_data = os_atomic_load_wide(approx_time_base, relaxed); |
870 | if (saved_data < abstime) { |
871 | /* |
872 | * ignore the success/fail return value assuming that |
873 | * if the value has been updated since we last read it, |
874 | * someone else has written a timestamp that is new enough. |
875 | */ |
876 | __unused bool ret = os_atomic_cmpxchg(approx_time_base, |
877 | saved_data, abstime, relaxed); |
878 | } |
879 | |
880 | |
881 | #else /* CONFIG_MACH_APPROXIMATE_TIME */ |
882 | #pragma unused (abstime) |
883 | #endif |
884 | } |
885 | |
886 | /* |
887 | * update the commpage data's total system sleep time for |
888 | * userspace call to mach_continuous_time() |
889 | */ |
890 | void |
891 | commpage_update_mach_continuous_time(uint64_t sleeptime) |
892 | { |
893 | if (!commPagePtr) { |
894 | return; |
895 | } |
896 | |
897 | uint64_t *cont_time_base = (uint64_t *)(uintptr_t)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET); |
898 | |
899 | os_atomic_store_wide(cont_time_base, sleeptime, relaxed); |
900 | |
901 | } |
902 | |
903 | void |
904 | commpage_update_mach_continuous_time_hw_offset(uint64_t offset) |
905 | { |
906 | *((uint64_t *)(_COMM_PAGE_CONT_HW_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = offset; |
907 | } |
908 | |
909 | /* |
910 | * update the commpage's value for the boot time |
911 | */ |
912 | void |
913 | commpage_update_boottime(uint64_t value) |
914 | { |
915 | if (!commPagePtr) { |
916 | return; |
917 | } |
918 | |
919 | uint64_t *boottime_usec = (uint64_t *)(uintptr_t)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET); |
920 | |
921 | os_atomic_store_wide(boottime_usec, value, relaxed); |
922 | |
923 | } |
924 | |
925 | /* |
926 | * set the commpage's remote time params for |
927 | * userspace call to mach_bridge_remote_time() |
928 | */ |
929 | void |
930 | commpage_set_remotetime_params(double rate, uint64_t base_local_ts, uint64_t base_remote_ts) |
931 | { |
932 | if (commPagePtr) { |
933 | #ifdef __arm64__ |
934 | struct bt_params *paramsp = (struct bt_params *)(_COMM_PAGE_REMOTETIME_PARAMS + _COMM_PAGE_RW_OFFSET); |
935 | paramsp->base_local_ts = 0; |
936 | __builtin_arm_dmb(DMB_ISH); |
937 | paramsp->rate = rate; |
938 | paramsp->base_remote_ts = base_remote_ts; |
939 | __builtin_arm_dmb(DMB_ISH); |
940 | paramsp->base_local_ts = base_local_ts; //This will act as a generation count |
941 | #endif /* __arm64__ */ |
942 | } |
943 | } |
944 | |
945 | |
946 | /* |
947 | * update the commpage with if dtrace user land probes are enabled |
948 | */ |
949 | void |
950 | commpage_update_dof(boolean_t enabled) |
951 | { |
952 | #if CONFIG_DTRACE |
953 | *((uint8_t*)(_COMM_PAGE_DTRACE_DOF_ENABLED + _COMM_PAGE_RW_OFFSET)) = (enabled ? 1 : 0); |
954 | #else |
955 | (void)enabled; |
956 | #endif |
957 | } |
958 | |
959 | /* |
960 | * update the dyld global config flags |
961 | */ |
962 | void |
963 | commpage_update_dyld_flags(uint64_t value) |
964 | { |
965 | *((uint64_t*)(_COMM_PAGE_DYLD_FLAGS + _COMM_PAGE_RW_OFFSET)) = value; |
966 | |
967 | } |
968 | |