1/*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
4 *
5 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 *
7 * This file contains Original Code and/or Modifications of Original Code
8 * as defined in and that are subject to the Apple Public Source License
9 * Version 2.0 (the 'License'). You may not use this file except in
10 * compliance with the License. The rights granted to you under the License
11 * may not be used to create, or enable the creation or redistribution of,
12 * unlawful or unlicensed copies of an Apple operating system, or to
13 * circumvent, violate, or enable the circumvention or violation of, any
14 * terms of an Apple operating system software license agreement.
15 *
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 *
19 * The Original Code and all software distributed under the License are
20 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
21 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
22 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
24 * Please see the License for the specific language governing rights and
25 * limitations under the License.
26 *
27 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
28 */
29/*
30 * @OSF_COPYRIGHT@
31 */
32/*
33 * @APPLE_FREE_COPYRIGHT@
34 */
35/*
36 * File: arm/commpage/commpage.c
37 * Purpose: Set up and export a RO/RW page
38 */
39#include <libkern/section_keywords.h>
40#include <mach/mach_types.h>
41#include <mach/machine.h>
42#include <mach/vm_map.h>
43#include <machine/cpu_capabilities.h>
44#include <machine/commpage.h>
45#include <machine/config.h>
46#include <machine/pmap.h>
47#include <vm/vm_kern.h>
48#include <vm/vm_map.h>
49#include <vm/vm_protos.h>
50#include <ipc/ipc_port.h>
51#include <arm/cpuid.h> /* for cpuid_info() & cache_info() */
52#include <arm/misc_protos.h>
53#include <arm/rtclock.h>
54#include <libkern/OSAtomic.h>
55#include <stdatomic.h>
56#include <kern/remote_time.h>
57#include <kern/smr.h>
58#include <machine/atomic.h>
59#include <machine/machine_remote_time.h>
60#include <machine/machine_routines.h>
61
62#include <sys/kdebug.h>
63#include <sys/random.h>
64
65#if CONFIG_ATM
66#include <atm/atm_internal.h>
67#endif
68
69static int commpage_cpus( void );
70
71
72static void commpage_init_cpu_capabilities( void );
73
74SECURITY_READ_ONLY_LATE(vm_address_t) commPagePtr = 0;
75SECURITY_READ_ONLY_LATE(vm_address_t) commpage_rw_addr = 0;
76SECURITY_READ_ONLY_LATE(vm_address_t) commpage_kernel_ro_addr = 0;
77SECURITY_READ_ONLY_LATE(uint64_t) _cpu_capabilities = 0;
78SECURITY_READ_ONLY_LATE(vm_address_t) commpage_rw_text_addr = 0;
79
80extern user64_addr_t commpage_text64_location;
81extern user32_addr_t commpage_text32_location;
82
83/* For sysctl access from BSD side */
84extern int gARMv8Crc32;
85extern int gARMv8Gpi;
86extern int gARM_FEAT_FlagM;
87extern int gARM_FEAT_FlagM2;
88extern int gARM_FEAT_FHM;
89extern int gARM_FEAT_DotProd;
90extern int gARM_FEAT_SHA3;
91extern int gARM_FEAT_RDM;
92extern int gARM_FEAT_LSE;
93extern int gARM_FEAT_SHA256;
94extern int gARM_FEAT_SHA512;
95extern int gARM_FEAT_SHA1;
96extern int gARM_FEAT_AES;
97extern int gARM_FEAT_PMULL;
98extern int gARM_FEAT_SPECRES;
99extern int gARM_FEAT_SB;
100extern int gARM_FEAT_FRINTTS;
101extern int gARM_FEAT_LRCPC;
102extern int gARM_FEAT_LRCPC2;
103extern int gARM_FEAT_FCMA;
104extern int gARM_FEAT_JSCVT;
105extern int gARM_FEAT_PAuth;
106extern int gARM_FEAT_PAuth2;
107extern int gARM_FEAT_FPAC;
108extern int gARM_FEAT_FPACCOMBINE;
109extern int gARM_FEAT_DPB;
110extern int gARM_FEAT_DPB2;
111extern int gARM_FEAT_BF16;
112extern int gARM_FEAT_I8MM;
113extern int gARM_FEAT_RPRES;
114extern int gARM_FEAT_ECV;
115extern int gARM_FEAT_LSE2;
116extern int gARM_FEAT_CSV2;
117extern int gARM_FEAT_CSV3;
118extern int gARM_FEAT_DIT;
119extern int gARM_AdvSIMD;
120extern int gARM_AdvSIMD_HPFPCvt;
121extern int gARM_FEAT_FP16;
122extern int gARM_FEAT_SSBS;
123extern int gARM_FEAT_BTI;
124extern int gARM_FP_SyncExceptions;
125extern int gARM_FEAT_AFP;
126
127extern int gUCNormalMem;
128
129void
130commpage_populate(void)
131{
132 uint16_t c2;
133 int cpufamily;
134
135 // Create the data and the text commpage
136 vm_map_address_t kernel_data_addr, kernel_text_addr, kernel_ro_data_addr, user_text_addr;
137 pmap_create_commpages(kernel_data_addr: &kernel_data_addr, kernel_text_addr: &kernel_text_addr, kernel_ro_data_addr: &kernel_ro_data_addr, user_text_addr: &user_text_addr);
138
139 commpage_rw_addr = kernel_data_addr;
140 commpage_rw_text_addr = kernel_text_addr;
141 commpage_kernel_ro_addr = kernel_ro_data_addr;
142 commPagePtr = (vm_address_t) _COMM_PAGE_BASE_ADDRESS;
143
144#if __arm64__
145 commpage_text64_location = user_text_addr;
146 bcopy(_COMM_PAGE64_SIGNATURE_STRING, dst: (void *)(_COMM_PAGE_SIGNATURE + _COMM_PAGE_RW_OFFSET),
147 MIN(_COMM_PAGE_SIGNATURELEN, strlen(_COMM_PAGE64_SIGNATURE_STRING)));
148#endif
149
150 *((uint16_t*)(_COMM_PAGE_VERSION + _COMM_PAGE_RW_OFFSET)) = (uint16_t) _COMM_PAGE_THIS_VERSION;
151
152 commpage_init_cpu_capabilities();
153 commpage_set_timestamp(tbr: 0, secs: 0, frac: 0, scale: 0, tick_per_sec: 0);
154
155 if (_cpu_capabilities & kCache32) {
156 c2 = 32;
157 } else if (_cpu_capabilities & kCache64) {
158 c2 = 64;
159 } else if (_cpu_capabilities & kCache128) {
160 c2 = 128;
161 } else {
162 c2 = 0;
163 }
164
165 *((uint16_t*)(_COMM_PAGE_CACHE_LINESIZE + _COMM_PAGE_RW_OFFSET)) = c2;
166
167 commpage_update_active_cpus();
168 cpufamily = cpuid_get_cpufamily();
169 *((uint8_t*)(_COMM_PAGE_CPU_CLUSTERS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) ml_get_cluster_count();
170 *((uint8_t*)(_COMM_PAGE_PHYSICAL_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.physical_cpu_max;
171 *((uint8_t*)(_COMM_PAGE_LOGICAL_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.logical_cpu_max;
172 *((uint64_t*)(_COMM_PAGE_MEMORY_SIZE + _COMM_PAGE_RW_OFFSET)) = machine_info.max_mem;
173 *((uint32_t*)(_COMM_PAGE_CPUFAMILY + _COMM_PAGE_RW_OFFSET)) = (uint32_t)cpufamily;
174 *((uint32_t*)(_COMM_PAGE_DEV_FIRM_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL);
175 *((uint32_t*)(_COMM_PAGE_DEV_FIRM + _COMM_PAGE_RO_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL);
176 *((uint8_t*)(_COMM_PAGE_USER_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = user_timebase_type();
177
178 // Populate logical CPU -> logical cluster table
179 ml_map_cpus_to_clusters(table: (uint8_t*)(_COMM_PAGE_CPU_TO_CLUSTER + _COMM_PAGE_RW_OFFSET));
180
181 *((uint8_t*)(_COMM_PAGE_CONT_HWCLOCK + _COMM_PAGE_RW_OFFSET)) = (uint8_t)user_cont_hwclock_allowed();
182 *((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift;
183 *((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT + _COMM_PAGE_RO_OFFSET)) = (uint8_t) page_shift;
184
185#if __arm64__
186 *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift_user32;
187 *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32 + _COMM_PAGE_RO_OFFSET)) = (uint8_t) page_shift_user32;
188 *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
189 *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64 + _COMM_PAGE_RO_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
190#endif /* __arm64__ */
191
192 commpage_update_timebase();
193 commpage_update_mach_continuous_time(sleeptime: 0);
194
195 clock_sec_t secs;
196 clock_usec_t microsecs;
197 clock_get_boottime_microtime(secs: &secs, microsecs: &microsecs);
198 commpage_update_boottime(boottime_usec: secs * USEC_PER_SEC + microsecs);
199
200 /*
201 * set commpage approximate time to zero for initialization.
202 * scheduler shall populate correct value before running user thread
203 */
204 *((uint64_t *)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET)) = 0;
205#ifdef CONFIG_MACH_APPROXIMATE_TIME
206 *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED + _COMM_PAGE_RW_OFFSET)) = 1;
207#else
208 *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED + _COMM_PAGE_RW_OFFSET)) = 0;
209#endif
210
211 commpage_update_kdebug_state();
212
213#if CONFIG_ATM
214 commpage_update_atm_diagnostic_config(atm_get_diagnostic_config());
215#endif
216
217
218 *((uint64_t*)(_COMM_PAGE_REMOTETIME_PARAMS + _COMM_PAGE_RW_OFFSET)) = BT_RESET_SENTINEL_TS;
219
220#if CONFIG_QUIESCE_COUNTER
221 cpu_quiescent_set_storage(ptr: (_Atomic uint64_t *)(_COMM_PAGE_CPU_QUIESCENT_COUNTER +
222 _COMM_PAGE_RW_OFFSET));
223#endif /* CONFIG_QUIESCE_COUNTER */
224
225 /*
226 * Set random values for targets in Apple Security Bounty
227 * addr should be unmapped for userland processes
228 * kaddr should be unmapped for kernel
229 */
230 uint64_t asb_value, asb_addr, asb_kvalue, asb_kaddr;
231 uint64_t asb_rand_vals[] = {
232 0x93e78adcded4d3d5, 0xd16c5b76ad99bccf, 0x67dfbbd12c4a594e, 0x7365636e6f6f544f,
233 0x239a974c9811e04b, 0xbf60e7fa45741446, 0x8acf5210b466b05, 0x67dfbbd12c4a594e
234 };
235 const int nrandval = sizeof(asb_rand_vals) / sizeof(asb_rand_vals[0]);
236 uint8_t randidx;
237
238 read_random(buffer: &randidx, numBytes: sizeof(uint8_t));
239 asb_value = asb_rand_vals[randidx++ % nrandval];
240 *((uint64_t*)(_COMM_PAGE_ASB_TARGET_VALUE + _COMM_PAGE_RW_OFFSET)) = asb_value;
241
242 // userspace faulting address should be > MACH_VM_MAX_ADDRESS
243 asb_addr = asb_rand_vals[randidx++ % nrandval];
244 uint64_t user_min = MACH_VM_MAX_ADDRESS;
245 uint64_t user_max = UINT64_MAX;
246 asb_addr %= (user_max - user_min);
247 asb_addr += user_min;
248 *((uint64_t*)(_COMM_PAGE_ASB_TARGET_ADDRESS + _COMM_PAGE_RW_OFFSET)) = asb_addr;
249
250 asb_kvalue = asb_rand_vals[randidx++ % nrandval];
251 *((uint64_t*)(_COMM_PAGE_ASB_TARGET_KERN_VALUE + _COMM_PAGE_RW_OFFSET)) = asb_kvalue;
252
253 // kernel faulting address should be < VM_MIN_KERNEL_ADDRESS
254 asb_kaddr = asb_rand_vals[randidx++ % nrandval];
255 uint64_t kernel_min = 0x0LL;
256 uint64_t kernel_max = VM_MIN_KERNEL_ADDRESS;
257 asb_kaddr %= (kernel_max - kernel_min);
258 asb_kaddr += kernel_min;
259 *((uint64_t*)(_COMM_PAGE_ASB_TARGET_KERN_ADDRESS + _COMM_PAGE_RW_OFFSET)) = asb_kaddr;
260}
261
262#define COMMPAGE_TEXT_SEGMENT "__TEXT_EXEC"
263#define COMMPAGE_TEXT_SECTION "__commpage_text"
264
265/* Get a pointer to the start of the ARM PFZ code section. This macro tell the
266 * linker that the storage for the variable here is at the start of the section */
267extern char commpage_text_start[]
268__SECTION_START_SYM(COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION);
269
270/* Get a pointer to the end of the ARM PFZ code section. This macro tell the
271 * linker that the storage for the variable here is at the end of the section */
272extern char commpage_text_end[]
273__SECTION_END_SYM(COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION);
274
275/* This is defined in the commpage text section as a symbol at the start of the preemptible
276 * functions */
277extern char commpage_text_preemptible_functions;
278
279#if CONFIG_ARM_PFZ
280static size_t size_of_pfz = 0;
281#endif
282
283/* This is the opcode for brk #666 */
284#define BRK_666_OPCODE 0xD4205340
285
286void
287commpage_text_populate(void)
288{
289#if CONFIG_ARM_PFZ
290 size_t size_of_commpage_text = commpage_text_end - commpage_text_start;
291 if (size_of_commpage_text == 0) {
292 panic("ARM comm page text section %s,%s missing", COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION);
293 }
294 assert(size_of_commpage_text <= PAGE_SIZE);
295 assert(size_of_commpage_text > 0);
296
297 /* Get the size of the PFZ half of the comm page text section. */
298 size_of_pfz = &commpage_text_preemptible_functions - commpage_text_start;
299
300 // Copy the code segment of comm page text section into the PFZ
301 memcpy(dst: (void *) _COMM_PAGE64_TEXT_START_ADDRESS, src: (void *) commpage_text_start, n: size_of_commpage_text);
302
303 // Make sure to populate the rest of it with brk 666 so that undefined code
304 // doesn't get run
305 memset(s: (char *) _COMM_PAGE64_TEXT_START_ADDRESS + size_of_commpage_text, BRK_666_OPCODE,
306 PAGE_SIZE - size_of_commpage_text);
307#endif
308}
309
310uint32_t
311commpage_is_in_pfz64(addr64_t addr64)
312{
313#if CONFIG_ARM_PFZ
314 if ((addr64 >= commpage_text64_location) &&
315 (addr64 < (commpage_text64_location + size_of_pfz))) {
316 return 1;
317 } else {
318 return 0;
319 }
320#else
321#pragma unused (addr64)
322 return 0;
323#endif
324}
325
326
327void
328commpage_set_timestamp(
329 uint64_t tbr,
330 uint64_t secs,
331 uint64_t frac,
332 uint64_t scale,
333 uint64_t tick_per_sec)
334{
335 new_commpage_timeofday_data_t *commpage_timeofday_datap;
336
337 if (commPagePtr == 0) {
338 return;
339 }
340
341 commpage_timeofday_datap = (new_commpage_timeofday_data_t *)(_COMM_PAGE_NEWTIMEOFDAY_DATA + _COMM_PAGE_RW_OFFSET);
342
343 commpage_timeofday_datap->TimeStamp_tick = 0x0ULL;
344
345 __builtin_arm_dmb(DMB_ISH);
346
347 commpage_timeofday_datap->TimeStamp_sec = secs;
348 commpage_timeofday_datap->TimeStamp_frac = frac;
349 commpage_timeofday_datap->Ticks_scale = scale;
350 commpage_timeofday_datap->Ticks_per_sec = tick_per_sec;
351
352 __builtin_arm_dmb(DMB_ISH);
353
354 commpage_timeofday_datap->TimeStamp_tick = tbr;
355
356}
357
358/*
359 * Update _COMM_PAGE_MEMORY_PRESSURE. Called periodically from vm's compute_memory_pressure()
360 */
361
362void
363commpage_set_memory_pressure(
364 unsigned int pressure )
365{
366 if (commPagePtr == 0) {
367 return;
368 }
369 *((uint32_t *)(_COMM_PAGE_MEMORY_PRESSURE + _COMM_PAGE_RW_OFFSET)) = pressure;
370}
371
372/*
373 * Determine number of CPUs on this system.
374 */
375static int
376commpage_cpus( void )
377{
378 int cpus;
379
380 cpus = machine_info.max_cpus;
381
382 if (cpus == 0) {
383 panic("commpage cpus==0");
384 }
385 if (cpus > 0xFF) {
386 cpus = 0xFF;
387 }
388
389 return cpus;
390}
391
392uint64_t
393_get_cpu_capabilities(void)
394{
395 return _cpu_capabilities;
396}
397
398vm_address_t
399_get_commpage_priv_address(void)
400{
401 return commpage_rw_addr;
402}
403
404vm_address_t
405_get_commpage_ro_address(void)
406{
407 return commpage_kernel_ro_addr;
408}
409
410vm_address_t
411_get_commpage_text_priv_address(void)
412{
413 return commpage_rw_text_addr;
414}
415
416#if defined(__arm64__)
417/**
418 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64ISAR0_EL1
419 */
420static void
421commpage_init_arm_optional_features_isar0(uint64_t *commpage_bits)
422{
423 uint64_t bits = 0;
424 uint64_t isar0 = __builtin_arm_rsr64("ID_AA64ISAR0_EL1");
425
426 if ((isar0 & ID_AA64ISAR0_EL1_TS_MASK) >= ID_AA64ISAR0_EL1_TS_FLAGM_EN) {
427 gARM_FEAT_FlagM = 1;
428 bits |= kHasFEATFlagM;
429 }
430 if ((isar0 & ID_AA64ISAR0_EL1_TS_MASK) >= ID_AA64ISAR0_EL1_TS_FLAGM2_EN) {
431 gARM_FEAT_FlagM2 = 1;
432 bits |= kHasFEATFlagM2;
433 }
434 if ((isar0 & ID_AA64ISAR0_EL1_FHM_MASK) >= ID_AA64ISAR0_EL1_FHM_8_2) {
435 gARM_FEAT_FHM = 1;
436 bits |= kHasFeatFHM;
437 }
438 if ((isar0 & ID_AA64ISAR0_EL1_DP_MASK) >= ID_AA64ISAR0_EL1_DP_EN) {
439 gARM_FEAT_DotProd = 1;
440 bits |= kHasFeatDotProd;
441 }
442 if ((isar0 & ID_AA64ISAR0_EL1_SHA3_MASK) >= ID_AA64ISAR0_EL1_SHA3_EN) {
443 gARM_FEAT_SHA3 = 1;
444 bits |= kHasFeatSHA3;
445 }
446 if ((isar0 & ID_AA64ISAR0_EL1_RDM_MASK) >= ID_AA64ISAR0_EL1_RDM_EN) {
447 gARM_FEAT_RDM = 1;
448 bits |= kHasFeatRDM;
449 }
450 if ((isar0 & ID_AA64ISAR0_EL1_ATOMIC_MASK) >= ID_AA64ISAR0_EL1_ATOMIC_8_1) {
451 gARM_FEAT_LSE = 1;
452 bits |= kHasFeatLSE;
453 }
454 if ((isar0 & ID_AA64ISAR0_EL1_SHA2_MASK) >= ID_AA64ISAR0_EL1_SHA2_512_EN) {
455 gARM_FEAT_SHA512 = 1;
456 bits |= kHasFeatSHA512;
457 }
458 if ((isar0 & ID_AA64ISAR0_EL1_CRC32_MASK) == ID_AA64ISAR0_EL1_CRC32_EN) {
459 gARMv8Crc32 = 1;
460 bits |= kHasARMv8Crc32;
461 }
462
463#if __ARM_V8_CRYPTO_EXTENSIONS__
464 /**
465 * T7000 has a bug in the ISAR0 register that reports that PMULL is not
466 * supported when it actually is. To work around this, for all of the crypto
467 * extensions, just check if they're supported using the board_config.h
468 * values.
469 */
470 gARM_FEAT_PMULL = 1;
471 gARM_FEAT_SHA1 = 1;
472 gARM_FEAT_AES = 1;
473 gARM_FEAT_SHA256 = 1;
474 bits |= kHasARMv8Crypto;
475#endif /* __ARM_V8_CRYPTO_EXTENSIONS__ */
476
477 *commpage_bits |= bits;
478}
479
480/**
481 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64ISAR1_EL1
482 */
483static void
484commpage_init_arm_optional_features_isar1(uint64_t *commpage_bits)
485{
486 uint64_t bits = 0;
487 uint64_t isar1 = __builtin_arm_rsr64("ID_AA64ISAR1_EL1");
488 uint64_t sctlr = __builtin_arm_rsr64("SCTLR_EL1");
489
490 if ((isar1 & ID_AA64ISAR1_EL1_SPECRES_MASK) >= ID_AA64ISAR1_EL1_SPECRES_EN &&
491 sctlr & SCTLR_EnRCTX) {
492 gARM_FEAT_SPECRES = 1;
493 bits |= kHasFeatSPECRES;
494 }
495 if ((isar1 & ID_AA64ISAR1_EL1_SB_MASK) >= ID_AA64ISAR1_EL1_SB_EN) {
496 gARM_FEAT_SB = 1;
497 bits |= kHasFeatSB;
498 }
499 if ((isar1 & ID_AA64ISAR1_EL1_FRINTTS_MASK) >= ID_AA64ISAR1_EL1_FRINTTS_EN) {
500 gARM_FEAT_FRINTTS = 1;
501 bits |= kHasFeatFRINTTS;
502 }
503 if ((isar1 & ID_AA64ISAR1_EL1_GPI_MASK) >= ID_AA64ISAR1_EL1_GPI_EN) {
504 gARMv8Gpi = 1;
505 bits |= kHasArmv8GPI;
506 }
507 if ((isar1 & ID_AA64ISAR1_EL1_LRCPC_MASK) >= ID_AA64ISAR1_EL1_LRCPC_EN) {
508 gARM_FEAT_LRCPC = 1;
509 bits |= kHasFeatLRCPC;
510 }
511 if ((isar1 & ID_AA64ISAR1_EL1_LRCPC_MASK) >= ID_AA64ISAR1_EL1_LRCP2C_EN) {
512 gARM_FEAT_LRCPC2 = 1;
513 bits |= kHasFeatLRCPC2;
514 }
515 if ((isar1 & ID_AA64ISAR1_EL1_FCMA_MASK) >= ID_AA64ISAR1_EL1_FCMA_EN) {
516 gARM_FEAT_FCMA = 1;
517 bits |= kHasFeatFCMA;
518 }
519 if ((isar1 & ID_AA64ISAR1_EL1_JSCVT_MASK) >= ID_AA64ISAR1_EL1_JSCVT_EN) {
520 gARM_FEAT_JSCVT = 1;
521 bits |= kHasFeatJSCVT;
522 }
523 if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_PAuth_EN) {
524 gARM_FEAT_PAuth = 1;
525 bits |= kHasFeatPAuth;
526 }
527 if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_PAuth2_EN) {
528 gARM_FEAT_PAuth2 = 1;
529 }
530 if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_FPAC_EN) {
531 gARM_FEAT_FPAC = 1;
532 }
533 if ((isar1 & ID_AA64ISAR1_EL1_API_MASK) >= ID_AA64ISAR1_EL1_API_FPACCOMBINE) {
534 gARM_FEAT_FPACCOMBINE = 1;
535 }
536 if ((isar1 & ID_AA64ISAR1_EL1_DPB_MASK) >= ID_AA64ISAR1_EL1_DPB_EN) {
537 gARM_FEAT_DPB = 1;
538 bits |= kHasFeatDPB;
539 }
540 if ((isar1 & ID_AA64ISAR1_EL1_DPB_MASK) >= ID_AA64ISAR1_EL1_DPB2_EN) {
541 gARM_FEAT_DPB2 = 1;
542 bits |= kHasFeatDPB2;
543 }
544 if ((isar1 & ID_AA64ISAR1_EL1_BF16_MASK) >= ID_AA64ISAR1_EL1_BF16_EN) {
545 gARM_FEAT_BF16 = 1;
546 }
547 if ((isar1 & ID_AA64ISAR1_EL1_I8MM_MASK) >= ID_AA64ISAR1_EL1_I8MM_EN) {
548 gARM_FEAT_I8MM = 1;
549 }
550
551 *commpage_bits |= bits;
552}
553
554/**
555 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64ISAR2_EL1
556 */
557static void
558commpage_init_arm_optional_features_isar2(void)
559{
560 uint64_t isar2 = __builtin_arm_rsr64("ID_AA64ISAR2_EL1");
561
562 if ((isar2 & ID_AA64ISAR2_EL1_RPRES_MASK) >= ID_AA64ISAR2_EL1_RPRES_EN) {
563 gARM_FEAT_RPRES = 1;
564 }
565}
566
567/**
568 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64MMFR0_EL1
569 */
570static void
571commpage_init_arm_optional_features_mmfr0(uint64_t *commpage_bits)
572{
573 uint64_t bits = 0;
574 uint64_t mmfr0 = __builtin_arm_rsr64("ID_AA64MMFR0_EL1");
575
576 if ((mmfr0 & ID_AA64MMFR0_EL1_ECV_MASK) >= ID_AA64MMFR0_EL1_ECV_EN) {
577 gARM_FEAT_ECV = 1;
578 }
579
580 *commpage_bits |= bits;
581}
582
583/**
584 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64MMFR2_EL1
585 */
586static void
587commpage_init_arm_optional_features_mmfr2(uint64_t *commpage_bits)
588{
589 uint64_t bits = 0;
590 uint64_t mmfr2 = __builtin_arm_rsr64("ID_AA64MMFR2_EL1");
591
592 if ((mmfr2 & ID_AA64MMFR2_EL1_AT_MASK) >= ID_AA64MMFR2_EL1_AT_LSE2_EN) {
593 gARM_FEAT_LSE2 = 1;
594 bits |= kHasFeatLSE2;
595 }
596
597 *commpage_bits |= bits;
598}
599
600/**
601 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64PFR0_EL1
602 */
603static void
604commpage_init_arm_optional_features_pfr0(uint64_t *commpage_bits)
605{
606 uint64_t bits = 0;
607 uint64_t pfr0 = __builtin_arm_rsr64("ID_AA64PFR0_EL1");
608
609 if ((pfr0 & ID_AA64PFR0_EL1_CSV3_MASK) >= ID_AA64PFR0_EL1_CSV3_EN) {
610 gARM_FEAT_CSV3 = 1;
611 bits |= kHasFeatCSV3;
612 }
613 if ((pfr0 & ID_AA64PFR0_EL1_CSV2_MASK) >= ID_AA64PFR0_EL1_CSV2_EN) {
614 gARM_FEAT_CSV2 = 1;
615 bits |= kHasFeatCSV2;
616 }
617 if ((pfr0 & ID_AA64PFR0_EL1_DIT_MASK) >= ID_AA64PFR0_EL1_DIT_EN) {
618 gARM_FEAT_DIT = 1;
619 bits |= kHasFeatDIT;
620 }
621 if ((pfr0 & ID_AA64PFR0_EL1_AdvSIMD_MASK) != ID_AA64PFR0_EL1_AdvSIMD_DIS) {
622 gARM_AdvSIMD = 1;
623 bits |= kHasAdvSIMD;
624 if ((pfr0 & ID_AA64PFR0_EL1_AdvSIMD_MASK) >= ID_AA64PFR0_EL1_AdvSIMD_HPFPCVT) {
625 gARM_AdvSIMD_HPFPCvt = 1;
626 bits |= kHasAdvSIMD_HPFPCvt;
627 }
628 if ((pfr0 & ID_AA64PFR0_EL1_AdvSIMD_MASK) >= ID_AA64PFR0_EL1_AdvSIMD_FP16) {
629 gARM_FEAT_FP16 = 1;
630 bits |= kHasFeatFP16;
631 }
632 }
633
634 *commpage_bits |= bits;
635}
636
637/**
638 * Initializes all commpage entries and sysctls for EL0 visible features in ID_AA64PFR1_EL1
639 */
640static void
641commpage_init_arm_optional_features_pfr1(uint64_t *commpage_bits)
642{
643 uint64_t pfr1 = __builtin_arm_rsr64("ID_AA64PFR1_EL1");
644
645 if ((pfr1 & ID_AA64PFR1_EL1_SSBS_MASK) >= ID_AA64PFR1_EL1_SSBS_EN) {
646 gARM_FEAT_SSBS = 1;
647 }
648
649 if ((pfr1 & ID_AA64PFR1_EL1_BT_MASK) >= ID_AA64PFR1_EL1_BT_EN) {
650 gARM_FEAT_BTI = 1;
651 }
652
653#pragma unused(commpage_bits)
654}
655
656
657static void
658commpage_init_arm_optional_features_mmfr1(uint64_t *commpage_bits)
659{
660 uint64_t bits = 0;
661 const uint64_t mmfr1 = __builtin_arm_rsr64("ID_AA64MMFR1_EL1");
662
663 if ((mmfr1 & ID_AA64MMFR1_EL1_AFP_MASK) == ID_AA64MMFR1_EL1_AFP_EN) {
664 gARM_FEAT_AFP = 1;
665 bits |= kHasFeatAFP;
666 }
667
668 *commpage_bits |= bits;
669}
670
671/**
672 * Read the system register @name, attempt to set set bits of @mask if not
673 * already, test if bits were actually set, reset the register to its
674 * previous value if required, and 'return' @mask with only bits that
675 * were successfully set (or already set) in the system register. */
676#define _test_sys_bits(name, mask) ({ \
677 const uint64_t src = __builtin_arm_rsr64(#name); \
678 uint64_t test = src | mask; \
679 if (test != src) { \
680 __builtin_arm_wsr64(#name, test); \
681 test = __builtin_arm_rsr64(#name); \
682 if (test != src) { \
683 __builtin_arm_wsr64(#name, src); \
684 }\
685 } \
686 mask & test; \
687})
688
689/**
690 * Reports whether FPU exceptions are supported.
691 * Possible FPU exceptions are :
692 * - input denormal;
693 * - inexact;
694 * - underflow;
695 * - overflow;
696 * - divide by 0;
697 * - invalid operation.
698 *
699 * Any of those can be supported or not but for now, we consider that
700 * it all or nothing : FPU exceptions support flag set <=> all 6 exceptions
701 * a supported.
702 */
703static void
704commpage_init_arm_optional_features_fpcr(uint64_t *commpage_bits)
705{
706 uint64_t support_mask = FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE |
707 FPCR_DZE | FPCR_IOE;
708 uint64_t FPCR_bits = _test_sys_bits(FPCR, support_mask);
709 if (FPCR_bits == support_mask) {
710 gARM_FP_SyncExceptions = 1;
711 *commpage_bits |= kHasFP_SyncExceptions;
712 }
713}
714
715/**
716 * Initializes all commpage entries and sysctls for ARM64 optional features accessible from EL0.
717 */
718static void
719commpage_init_arm_optional_features(uint64_t *commpage_bits)
720{
721 commpage_init_arm_optional_features_isar0(commpage_bits);
722 commpage_init_arm_optional_features_isar1(commpage_bits);
723 commpage_init_arm_optional_features_isar2();
724 commpage_init_arm_optional_features_mmfr0(commpage_bits);
725 commpage_init_arm_optional_features_mmfr1(commpage_bits);
726 commpage_init_arm_optional_features_mmfr2(commpage_bits);
727 commpage_init_arm_optional_features_pfr0(commpage_bits);
728 commpage_init_arm_optional_features_pfr1(commpage_bits);
729 commpage_init_arm_optional_features_fpcr(commpage_bits);
730}
731#endif /* __arm64__ */
732
733/*
734 * Initialize _cpu_capabilities vector
735 */
736static void
737commpage_init_cpu_capabilities( void )
738{
739 uint64_t bits;
740 int cpus;
741 ml_cpu_info_t cpu_info;
742
743 bits = 0;
744 ml_cpu_get_info(ml_cpu_info: &cpu_info);
745
746 switch (cpu_info.cache_line_size) {
747 case 128:
748 bits |= kCache128;
749 break;
750 case 64:
751 bits |= kCache64;
752 break;
753 case 32:
754 bits |= kCache32;
755 break;
756 default:
757 break;
758 }
759 cpus = commpage_cpus();
760
761 if (cpus == 1) {
762 bits |= kUP;
763 }
764
765 bits |= (cpus << kNumCPUsShift);
766
767 bits |= kFastThreadLocalStorage; // TPIDRURO for TLS
768
769 bits |= kHasVfp;
770
771#if defined(__arm64__)
772 bits |= kHasFMA;
773#endif
774 bits |= kHasEvent;
775#ifdef __arm64__
776 commpage_init_arm_optional_features(commpage_bits: &bits);
777#endif
778
779
780
781#if HAS_UCNORMAL_MEM
782 gUCNormalMem = 1;
783 bits |= kHasUCNormalMemory;
784#endif
785
786 _cpu_capabilities = bits;
787
788 *((uint32_t *)(_COMM_PAGE_CPU_CAPABILITIES + _COMM_PAGE_RW_OFFSET)) = (uint32_t)_cpu_capabilities;
789 *((uint64_t *)(_COMM_PAGE_CPU_CAPABILITIES64 + _COMM_PAGE_RW_OFFSET)) = _cpu_capabilities;
790
791}
792
793/*
794 * Updated every time a logical CPU goes offline/online
795 */
796void
797commpage_update_active_cpus(void)
798{
799 if (!commPagePtr) {
800 return;
801 }
802 *((uint8_t *)(_COMM_PAGE_ACTIVE_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t)processor_avail_count;
803
804}
805
806/*
807 * Update the commpage bits for mach_absolute_time and mach_continuous_time (for userspace)
808 */
809void
810commpage_update_timebase(void)
811{
812 if (commPagePtr) {
813 *((uint64_t*)(_COMM_PAGE_TIMEBASE_OFFSET + _COMM_PAGE_RW_OFFSET)) = rtclock_base_abstime;
814 }
815}
816
817/*
818 * Update the commpage with current kdebug state: whether tracing is enabled, a
819 * typefilter is present, and continuous time should be used for timestamps.
820 *
821 * Disregards configuration and set to 0 if tracing is disabled.
822 */
823void
824commpage_update_kdebug_state(void)
825{
826 if (commPagePtr) {
827 uint32_t state = kdebug_commpage_state();
828 *((volatile uint32_t *)(_COMM_PAGE_KDEBUG_ENABLE + _COMM_PAGE_RW_OFFSET)) = state;
829 }
830}
831
832/* Ditto for atm_diagnostic_config */
833void
834commpage_update_atm_diagnostic_config(uint32_t diagnostic_config)
835{
836 if (commPagePtr) {
837 *((volatile uint32_t*)(_COMM_PAGE_ATM_DIAGNOSTIC_CONFIG + _COMM_PAGE_RW_OFFSET)) = diagnostic_config;
838 }
839}
840
841/*
842 * Update the commpage data with the state of multiuser mode for
843 * this device. Allowing various services in userspace to avoid
844 * IPC in the (more common) non-multiuser environment.
845 */
846void
847commpage_update_multiuser_config(uint32_t multiuser_config)
848{
849 if (commPagePtr) {
850 *((volatile uint32_t *)(_COMM_PAGE_MULTIUSER_CONFIG + _COMM_PAGE_RW_OFFSET)) = multiuser_config;
851 }
852}
853
854/*
855 * update the commpage data for
856 * last known value of mach_absolute_time()
857 */
858
859void
860commpage_update_mach_approximate_time(uint64_t abstime)
861{
862#ifdef CONFIG_MACH_APPROXIMATE_TIME
863 if (!commPagePtr) {
864 return;
865 }
866
867 uint64_t *approx_time_base = (uint64_t *)(uintptr_t)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET);
868
869 uint64_t saved_data = os_atomic_load_wide(approx_time_base, relaxed);
870 if (saved_data < abstime) {
871 /*
872 * ignore the success/fail return value assuming that
873 * if the value has been updated since we last read it,
874 * someone else has written a timestamp that is new enough.
875 */
876 __unused bool ret = os_atomic_cmpxchg(approx_time_base,
877 saved_data, abstime, relaxed);
878 }
879
880
881#else /* CONFIG_MACH_APPROXIMATE_TIME */
882#pragma unused (abstime)
883#endif
884}
885
886/*
887 * update the commpage data's total system sleep time for
888 * userspace call to mach_continuous_time()
889 */
890void
891commpage_update_mach_continuous_time(uint64_t sleeptime)
892{
893 if (!commPagePtr) {
894 return;
895 }
896
897 uint64_t *cont_time_base = (uint64_t *)(uintptr_t)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET);
898
899 os_atomic_store_wide(cont_time_base, sleeptime, relaxed);
900
901}
902
903void
904commpage_update_mach_continuous_time_hw_offset(uint64_t offset)
905{
906 *((uint64_t *)(_COMM_PAGE_CONT_HW_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = offset;
907}
908
909/*
910 * update the commpage's value for the boot time
911 */
912void
913commpage_update_boottime(uint64_t value)
914{
915 if (!commPagePtr) {
916 return;
917 }
918
919 uint64_t *boottime_usec = (uint64_t *)(uintptr_t)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET);
920
921 os_atomic_store_wide(boottime_usec, value, relaxed);
922
923}
924
925/*
926 * set the commpage's remote time params for
927 * userspace call to mach_bridge_remote_time()
928 */
929void
930commpage_set_remotetime_params(double rate, uint64_t base_local_ts, uint64_t base_remote_ts)
931{
932 if (commPagePtr) {
933#ifdef __arm64__
934 struct bt_params *paramsp = (struct bt_params *)(_COMM_PAGE_REMOTETIME_PARAMS + _COMM_PAGE_RW_OFFSET);
935 paramsp->base_local_ts = 0;
936 __builtin_arm_dmb(DMB_ISH);
937 paramsp->rate = rate;
938 paramsp->base_remote_ts = base_remote_ts;
939 __builtin_arm_dmb(DMB_ISH);
940 paramsp->base_local_ts = base_local_ts; //This will act as a generation count
941#endif /* __arm64__ */
942 }
943}
944
945
946/*
947 * update the commpage with if dtrace user land probes are enabled
948 */
949void
950commpage_update_dof(boolean_t enabled)
951{
952#if CONFIG_DTRACE
953 *((uint8_t*)(_COMM_PAGE_DTRACE_DOF_ENABLED + _COMM_PAGE_RW_OFFSET)) = (enabled ? 1 : 0);
954#else
955 (void)enabled;
956#endif
957}
958
959/*
960 * update the dyld global config flags
961 */
962void
963commpage_update_dyld_flags(uint64_t value)
964{
965 *((uint64_t*)(_COMM_PAGE_DYLD_FLAGS + _COMM_PAGE_RW_OFFSET)) = value;
966
967}
968