1/*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31
32#ifndef _ARM_MACHINE_ROUTINES_H_
33#define _ARM_MACHINE_ROUTINES_H_
34
35#include <mach/mach_types.h>
36#include <mach/vm_types.h>
37#include <mach/boolean.h>
38#include <kern/kern_types.h>
39#include <pexpert/pexpert.h>
40
41#include <sys/cdefs.h>
42#include <sys/appleapiopts.h>
43
44#include <stdarg.h>
45
46#ifdef XNU_KERNEL_PRIVATE
47#include <kern/sched_hygiene.h>
48#include <kern/startup.h>
49#endif /* XNU_KERNEL_PRIVATE */
50
51__BEGIN_DECLS
52#ifdef XNU_KERNEL_PRIVATE
53#ifdef __arm64__
54typedef bool (*expected_fault_handler_t)(arm_saved_state_t *);
55#endif /* __arm64__ */
56#endif /* XNU_KERNEL_PRIVATE */
57
58/* Interrupt handling */
59
60void ml_cpu_signal(unsigned int cpu_id);
61void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs);
62uint64_t ml_cpu_signal_deferred_get_timer(void);
63void ml_cpu_signal_deferred(unsigned int cpu_id);
64void ml_cpu_signal_retract(unsigned int cpu_id);
65bool ml_cpu_signal_is_enabled(void);
66
67/* Initialize Interrupts */
68void ml_init_interrupt(void);
69
70/* Get Interrupts Enabled */
71boolean_t ml_get_interrupts_enabled(void);
72
73/* Set Interrupts Enabled */
74#if __has_feature(ptrauth_calls)
75uint64_t ml_pac_safe_interrupts_disable(void);
76void ml_pac_safe_interrupts_restore(uint64_t);
77#endif /* __has_feature(ptrauth_calls) */
78boolean_t ml_set_interrupts_enabled_with_debug(boolean_t enable, boolean_t debug);
79boolean_t ml_set_interrupts_enabled(boolean_t enable);
80boolean_t ml_early_set_interrupts_enabled(boolean_t enable);
81
82/*
83 * Functions for disabling measurements for AppleCLPC only.
84 */
85boolean_t sched_perfcontrol_ml_set_interrupts_without_measurement(boolean_t enable);
86void sched_perfcontrol_abandon_preemption_disable_measurement(void);
87
88/* Check if running at interrupt context */
89boolean_t ml_at_interrupt_context(void);
90
91
92/* Generate a fake interrupt */
93void ml_cause_interrupt(void);
94
95
96#ifdef XNU_KERNEL_PRIVATE
97
98/* did this interrupt context interrupt userspace? */
99bool ml_did_interrupt_userspace(void);
100
101/* Clear interrupt spin debug state for thread */
102
103#if SCHED_HYGIENE_DEBUG
104void mt_cur_cpu_cycles_instrs_speculative(uint64_t *cycles, uint64_t *instrs);
105
106#if CONFIG_CPU_COUNTERS
107#define INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread) \
108 if (sched_hygiene_debug_pmc) { \
109 mt_cur_cpu_cycles_instrs_speculative(&thread->machine.intmask_cycles, \
110 &thread->machine.intmask_instr); \
111 }
112#else /* CONFIG_CPU_COUNTERS */
113#define INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread)
114#endif /* !CONFIG_CPU_COUNTERS */
115
116#define INTERRUPT_MASKED_DEBUG_START(handler_addr, type) \
117do { \
118 if ((interrupt_masked_debug_mode || sched_preemption_disable_debug_mode) && os_atomic_load(&interrupt_masked_timeout, relaxed) > 0) { \
119 thread_t thread = current_thread(); \
120 thread->machine.int_type = type; \
121 thread->machine.int_handler_addr = (uintptr_t)VM_KERNEL_STRIP_UPTR(handler_addr); \
122 thread->machine.inthandler_timestamp = ml_get_sched_hygiene_timebase(); \
123 INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread); \
124 thread->machine.int_vector = (uintptr_t)NULL; \
125 } \
126} while (0)
127
128#define INTERRUPT_MASKED_DEBUG_END() \
129do { \
130 if ((interrupt_masked_debug_mode || sched_preemption_disable_debug_mode) && os_atomic_load(&interrupt_masked_timeout, relaxed) > 0) { \
131 thread_t thread = current_thread(); \
132 ml_handle_interrupt_handler_duration(thread); \
133 thread->machine.inthandler_timestamp = 0; \
134 thread->machine.inthandler_abandon = false; \
135 } \
136} while (0)
137
138void ml_irq_debug_start(uintptr_t handler, uintptr_t vector);
139void ml_irq_debug_end(void);
140void ml_irq_debug_abandon(void);
141
142void ml_spin_debug_reset(thread_t thread);
143void ml_spin_debug_clear(thread_t thread);
144void ml_spin_debug_clear_self(void);
145void ml_handle_interrupts_disabled_duration(thread_t thread);
146void ml_handle_stackshot_interrupt_disabled_duration(thread_t thread);
147void ml_handle_interrupt_handler_duration(thread_t thread);
148
149#else /* SCHED_HYGIENE_DEBUG */
150
151#define INTERRUPT_MASKED_DEBUG_START(handler_addr, type)
152#define INTERRUPT_MASKED_DEBUG_END()
153
154#endif /* SCHED_HYGIENE_DEBUG */
155
156extern bool ml_snoop_thread_is_on_core(thread_t thread);
157extern boolean_t ml_is_quiescing(void);
158extern void ml_set_is_quiescing(boolean_t);
159extern uint64_t ml_get_booter_memory_size(void);
160#endif
161
162/* Type for the Time Base Enable function */
163typedef void (*time_base_enable_t)(cpu_id_t cpu_id, boolean_t enable);
164#if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE)
165/* Type for the Processor Cache Dispatch function */
166typedef void (*cache_dispatch_t)(cpu_id_t cpu_id, unsigned int select, unsigned int param0, unsigned int param1);
167
168typedef uint32_t (*get_decrementer_t)(void);
169typedef void (*set_decrementer_t)(uint32_t);
170typedef void (*fiq_handler_t)(void);
171
172#endif
173
174#define CacheConfig 0x00000000UL
175#define CacheControl 0x00000001UL
176#define CacheClean 0x00000002UL
177#define CacheCleanRegion 0x00000003UL
178#define CacheCleanFlush 0x00000004UL
179#define CacheCleanFlushRegion 0x00000005UL
180#define CacheShutdown 0x00000006UL
181
182#define CacheControlEnable 0x00000000UL
183
184#define CacheConfigCCSIDR 0x00000001UL
185#define CacheConfigSize 0x00000100UL
186
187/* Type for the Processor Idle function */
188typedef void (*processor_idle_t)(cpu_id_t cpu_id, boolean_t enter, uint64_t *new_timeout_ticks);
189
190/* Type for the Idle Tickle function */
191typedef void (*idle_tickle_t)(void);
192
193/* Type for the Idle Timer function */
194typedef void (*idle_timer_t)(void *refcon, uint64_t *new_timeout_ticks);
195
196/* Type for the IPI Hander */
197typedef void (*ipi_handler_t)(void);
198
199/* Type for the Lockdown Hander */
200typedef void (*lockdown_handler_t)(void *);
201
202/* Type for the Platform specific Error Handler */
203typedef void (*platform_error_handler_t)(void *refcon, vm_offset_t fault_addr);
204
205/*
206 * The exception callback (ex_cb) module is obsolete. Some definitions related
207 * to ex_cb were exported through the SDK, and are only left here for historical
208 * reasons.
209 */
210
211/* Unused. Left for historical reasons. */
212typedef enum{
213 EXCB_CLASS_ILLEGAL_INSTR_SET,
214#ifdef CONFIG_XNUPOST
215 EXCB_CLASS_TEST1,
216 EXCB_CLASS_TEST2,
217 EXCB_CLASS_TEST3,
218#endif
219 EXCB_CLASS_MAX
220}
221ex_cb_class_t;
222
223/* Unused. Left for historical reasons. */
224typedef enum{
225 EXCB_ACTION_RERUN,
226 EXCB_ACTION_NONE,
227#ifdef CONFIG_XNUPOST
228 EXCB_ACTION_TEST_FAIL,
229#endif
230}
231ex_cb_action_t;
232
233/* Unused. Left for historical reasons. */
234typedef struct{
235 vm_offset_t far;
236}
237ex_cb_state_t;
238
239/* Unused. Left for historical reasons. */
240typedef ex_cb_action_t (*ex_cb_t) (
241 ex_cb_class_t cb_class,
242 void *refcon,
243 const ex_cb_state_t *state
244 );
245
246/*
247 * This function is unimplemented. Its definition is left for historical
248 * reasons.
249 */
250kern_return_t ex_cb_register(
251 ex_cb_class_t cb_class,
252 ex_cb_t cb,
253 void *refcon );
254
255/*
256 * This function is unimplemented. Its definition is left for historical
257 * reasons.
258 */
259ex_cb_action_t ex_cb_invoke(
260 ex_cb_class_t cb_class,
261 vm_offset_t far);
262
263typedef enum {
264 CLUSTER_TYPE_SMP,
265 CLUSTER_TYPE_E,
266 CLUSTER_TYPE_P,
267 MAX_CPU_TYPES,
268} cluster_type_t;
269
270void ml_parse_cpu_topology(void);
271
272unsigned int ml_get_cpu_count(void);
273
274unsigned int ml_get_cpu_number_type(cluster_type_t cluster_type, bool logical, bool available);
275
276unsigned int ml_get_cluster_number_type(cluster_type_t cluster_type);
277
278unsigned int ml_cpu_cache_sharing(unsigned int level, cluster_type_t cluster_type, bool include_all_cpu_types);
279
280unsigned int ml_get_cpu_types(void);
281
282int ml_get_boot_cpu_number(void);
283
284int ml_get_cpu_number(uint32_t phys_id);
285
286unsigned int ml_get_cpu_number_local(void);
287
288int ml_get_cluster_number(uint32_t phys_id);
289
290int ml_get_max_cpu_number(void);
291
292int ml_get_max_cluster_number(void);
293
294/*
295 * Return the id of a cluster's first cpu.
296 */
297unsigned int ml_get_first_cpu_id(unsigned int cluster_id);
298
299/*
300 * Return the die id of a cluster.
301 */
302unsigned int ml_get_die_id(unsigned int cluster_id);
303
304/*
305 * Return the index of a cluster in its die.
306 */
307unsigned int ml_get_die_cluster_id(unsigned int cluster_id);
308
309/*
310 * Return the highest die id of the system.
311 */
312unsigned int ml_get_max_die_id(void);
313
314#ifdef __arm64__
315int ml_get_cluster_number_local(void);
316#endif /* __arm64__ */
317
318/* Struct for ml_cpu_get_info */
319struct ml_cpu_info {
320 unsigned long vector_unit;
321 unsigned long cache_line_size;
322 unsigned long l1_icache_size;
323 unsigned long l1_dcache_size;
324 unsigned long l2_settings;
325 unsigned long l2_cache_size;
326 unsigned long l3_settings;
327 unsigned long l3_cache_size;
328};
329typedef struct ml_cpu_info ml_cpu_info_t;
330
331cluster_type_t ml_get_boot_cluster_type(void);
332
333/*!
334 * @typedef ml_topology_cpu_t
335 * @brief Describes one CPU core in the topology.
336 *
337 * @field cpu_id Logical CPU ID: 0, 1, 2, 3, 4, ...
338 * Dynamically assigned by XNU so it might not match EDT. No holes.
339 * @field phys_id Physical CPU ID (EDT: reg). Same as MPIDR[15:0], i.e.
340 * (cluster_id << 8) | core_number_within_cluster
341 * @field cluster_id Logical Cluster ID: 0, 1, 2, 3, 4, ...
342 * Dynamically assigned by XNU so it might not match EDT. No holes.
343 * @field die_id Die ID (EDT: die-id)
344 * @field cluster_type The type of CPUs found in this cluster.
345 * @field l2_access_penalty Indicates that the scheduler should try to de-prioritize a core because
346 * L2 accesses are slower than on the boot processor.
347 * @field l2_cache_size Size of the L2 cache, in bytes. 0 if unknown or not present.
348 * @field l2_cache_id l2-cache-id property read from EDT.
349 * @field l3_cache_size Size of the L3 cache, in bytes. 0 if unknown or not present.
350 * @field l3_cache_id l3-cache-id property read from EDT.
351 * @field cpu_IMPL_regs IO-mapped virtual address of cpuX_IMPL (implementation-defined) register block.
352 * @field cpu_IMPL_pa Physical address of cpuX_IMPL register block.
353 * @field cpu_IMPL_len Length of cpuX_IMPL register block.
354 * @field cpu_UTTDBG_regs IO-mapped virtual address of cpuX_UTTDBG register block.
355 * @field cpu_UTTDBG_pa Physical address of cpuX_UTTDBG register block, if set in DT, else zero
356 * @field cpu_UTTDBG_len Length of cpuX_UTTDBG register block, if set in DT, else zero
357 * @field coresight_regs IO-mapped virtual address of CoreSight debug register block.
358 * @field coresight_pa Physical address of CoreSight register block.
359 * @field coresight_len Length of CoreSight register block.
360 * @field die_cluster_id Physical cluster ID within the local die (EDT: die-cluster-id)
361 * @field cluster_core_id Physical core ID within the local cluster (EDT: cluster-core-id)
362 */
363typedef struct ml_topology_cpu {
364 unsigned int cpu_id;
365 uint32_t phys_id;
366 unsigned int cluster_id;
367 unsigned int die_id;
368 cluster_type_t cluster_type;
369 uint32_t l2_access_penalty;
370 uint32_t l2_cache_size;
371 uint32_t l2_cache_id;
372 uint32_t l3_cache_size;
373 uint32_t l3_cache_id;
374 vm_offset_t cpu_IMPL_regs;
375 uint64_t cpu_IMPL_pa;
376 uint64_t cpu_IMPL_len;
377 vm_offset_t cpu_UTTDBG_regs;
378 uint64_t cpu_UTTDBG_pa;
379 uint64_t cpu_UTTDBG_len;
380 vm_offset_t coresight_regs;
381 uint64_t coresight_pa;
382 uint64_t coresight_len;
383 unsigned int die_cluster_id;
384 unsigned int cluster_core_id;
385} ml_topology_cpu_t;
386
387/*!
388 * @typedef ml_topology_cluster_t
389 * @brief Describes one cluster in the topology.
390 *
391 * @field cluster_id Cluster ID (EDT: cluster-id)
392 * @field cluster_type The type of CPUs found in this cluster.
393 * @field num_cpus Total number of usable CPU cores in this cluster.
394 * @field first_cpu_id The cpu_id of the first CPU in the cluster.
395 * @field cpu_mask A bitmask representing the cpu_id's that belong to the cluster. Example:
396 * If the cluster contains CPU4 and CPU5, cpu_mask will be 0x30.
397 * @field acc_IMPL_regs IO-mapped virtual address of acc_IMPL (implementation-defined) register block.
398 * @field acc_IMPL_pa Physical address of acc_IMPL register block.
399 * @field acc_IMPL_len Length of acc_IMPL register block.
400 * @field cpm_IMPL_regs IO-mapped virtual address of cpm_IMPL (implementation-defined) register block.
401 * @field cpm_IMPL_pa Physical address of cpm_IMPL register block.
402 * @field cpm_IMPL_len Length of cpm_IMPL register block.
403 */
404typedef struct ml_topology_cluster {
405 unsigned int cluster_id;
406 cluster_type_t cluster_type;
407 unsigned int num_cpus;
408 unsigned int first_cpu_id;
409 uint64_t cpu_mask;
410 vm_offset_t acc_IMPL_regs;
411 uint64_t acc_IMPL_pa;
412 uint64_t acc_IMPL_len;
413 vm_offset_t cpm_IMPL_regs;
414 uint64_t cpm_IMPL_pa;
415 uint64_t cpm_IMPL_len;
416} ml_topology_cluster_t;
417
418// Bump this version number any time any ml_topology_* struct changes, so
419// that KPI users can check whether their headers are compatible with
420// the running kernel.
421#define CPU_TOPOLOGY_VERSION 1
422
423/*!
424 * @typedef ml_topology_info_t
425 * @brief Describes the CPU topology for all APs in the system. Populated from EDT and read-only at runtime.
426 * @discussion This struct only lists CPU cores that are considered usable by both iBoot and XNU. Some
427 * physically present CPU cores may be considered unusable due to configuration options like
428 * the "cpus=" boot-arg. Cores that are disabled in hardware will not show up in EDT at all, so
429 * they also will not be present in this struct.
430 *
431 * @field version Version of the struct (set to CPU_TOPOLOGY_VERSION).
432 * @field num_cpus Total number of usable CPU cores.
433 * @field max_cpu_id The highest usable logical CPU ID.
434 * @field num_clusters Total number of AP CPU clusters on the system (usable or not).
435 * @field max_cluster_id The highest cluster ID found in EDT.
436 * @field cpus List of |num_cpus| entries.
437 * @field clusters List of |num_clusters| entries.
438 * @field boot_cpu Points to the |cpus| entry for the boot CPU.
439 * @field boot_cluster Points to the |clusters| entry which contains the boot CPU.
440 * @field chip_revision Silicon revision reported by iBoot, which comes from the
441 * SoC-specific fuse bits. See CPU_VERSION_xx macros for definitions.
442 */
443typedef struct ml_topology_info {
444 unsigned int version;
445 unsigned int num_cpus;
446 unsigned int max_cpu_id;
447 unsigned int num_clusters;
448 unsigned int max_cluster_id;
449 unsigned int max_die_id;
450 ml_topology_cpu_t *cpus;
451 ml_topology_cluster_t *clusters;
452 ml_topology_cpu_t *boot_cpu;
453 ml_topology_cluster_t *boot_cluster;
454 unsigned int chip_revision;
455 unsigned int cluster_types;
456 unsigned int cluster_type_num_cpus[MAX_CPU_TYPES];
457 unsigned int cluster_type_num_clusters[MAX_CPU_TYPES];
458} ml_topology_info_t;
459
460/*!
461 * @function ml_get_topology_info
462 * @result A pointer to the read-only topology struct. Does not need to be freed. Returns NULL
463 * if the struct hasn't been initialized or the feature is unsupported.
464 */
465const ml_topology_info_t *ml_get_topology_info(void);
466
467/*!
468 * @function ml_map_cpu_pio
469 * @brief Maps per-CPU and per-cluster PIO registers found in EDT. This needs to be
470 * called after arm_vm_init() so it can't be part of ml_parse_cpu_topology().
471 */
472void ml_map_cpu_pio(void);
473
474/* Struct for ml_processor_register */
475struct ml_processor_info {
476 cpu_id_t cpu_id;
477 vm_offset_t start_paddr;
478 boolean_t supports_nap;
479 void *platform_cache_dispatch;
480 time_base_enable_t time_base_enable;
481 processor_idle_t processor_idle;
482 idle_tickle_t *idle_tickle;
483 idle_timer_t idle_timer;
484 void *idle_timer_refcon;
485 vm_offset_t powergate_stub_addr;
486 uint32_t powergate_stub_length;
487 uint32_t powergate_latency;
488 platform_error_handler_t platform_error_handler;
489 uint64_t regmap_paddr;
490 uint32_t phys_id;
491 uint32_t log_id;
492 uint32_t l2_access_penalty;
493 uint32_t cluster_id;
494 cluster_type_t cluster_type;
495 uint32_t l2_cache_id;
496 uint32_t l2_cache_size;
497 uint32_t l3_cache_id;
498 uint32_t l3_cache_size;
499};
500typedef struct ml_processor_info ml_processor_info_t;
501
502#if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE)
503/* Struct for ml_init_timebase */
504struct tbd_ops {
505 fiq_handler_t tbd_fiq_handler;
506 get_decrementer_t tbd_get_decrementer;
507 set_decrementer_t tbd_set_decrementer;
508};
509typedef struct tbd_ops *tbd_ops_t;
510typedef struct tbd_ops tbd_ops_data_t;
511#endif
512
513
514/*!
515 * @function ml_processor_register
516 *
517 * @abstract callback from platform kext to register processor
518 *
519 * @discussion This function is called by the platform kext when a processor is
520 * being registered. This is called while running on the CPU itself, as part of
521 * its initialization.
522 *
523 * @param ml_processor_info provides machine-specific information about the
524 * processor to xnu.
525 *
526 * @param processor is set as an out-parameter to an opaque handle that should
527 * be used by the platform kext when referring to this processor in the future.
528 *
529 * @param ipi_handler is set as an out-parameter to the function that should be
530 * registered as the IPI handler.
531 *
532 * @param pmi_handler is set as an out-parameter to the function that should be
533 * registered as the PMI handler.
534 *
535 * @returns KERN_SUCCESS on success and an error code, otherwise.
536 */
537kern_return_t ml_processor_register(ml_processor_info_t *ml_processor_info,
538 processor_t *processor, ipi_handler_t *ipi_handler,
539 perfmon_interrupt_handler_func *pmi_handler);
540
541/* Register a lockdown handler */
542kern_return_t ml_lockdown_handler_register(lockdown_handler_t, void *);
543
544/* Register a M$ flushing */
545typedef kern_return_t (*mcache_flush_function)(void *service);
546kern_return_t ml_mcache_flush_callback_register(mcache_flush_function func, void *service);
547kern_return_t ml_mcache_flush(void);
548
549#if XNU_KERNEL_PRIVATE
550void ml_lockdown_init(void);
551
552/* Machine layer routine for intercepting panics */
553__printflike(1, 0)
554void ml_panic_trap_to_debugger(const char *panic_format_str,
555 va_list *panic_args,
556 unsigned int reason,
557 void *ctx,
558 uint64_t panic_options_mask,
559 unsigned long panic_caller);
560#endif /* XNU_KERNEL_PRIVATE */
561
562/* Initialize Interrupts */
563void ml_install_interrupt_handler(
564 void *nub,
565 int source,
566 void *target,
567 IOInterruptHandler handler,
568 void *refCon);
569
570vm_offset_t
571 ml_static_vtop(
572 vm_offset_t);
573
574kern_return_t
575ml_static_verify_page_protections(
576 uint64_t base, uint64_t size, vm_prot_t prot);
577
578vm_offset_t
579 ml_static_ptovirt(
580 vm_offset_t);
581
582/* Offset required to obtain absolute time value from tick counter */
583uint64_t ml_get_abstime_offset(void);
584
585/* Offset required to obtain continuous time value from tick counter */
586uint64_t ml_get_conttime_offset(void);
587
588#ifdef __APPLE_API_UNSTABLE
589/* PCI config cycle probing */
590boolean_t ml_probe_read(
591 vm_offset_t paddr,
592 unsigned int *val);
593boolean_t ml_probe_read_64(
594 addr64_t paddr,
595 unsigned int *val);
596
597/* Read physical address byte */
598unsigned int ml_phys_read_byte(
599 vm_offset_t paddr);
600unsigned int ml_phys_read_byte_64(
601 addr64_t paddr);
602
603/* Read physical address half word */
604unsigned int ml_phys_read_half(
605 vm_offset_t paddr);
606unsigned int ml_phys_read_half_64(
607 addr64_t paddr);
608
609/* Read physical address word*/
610unsigned int ml_phys_read(
611 vm_offset_t paddr);
612unsigned int ml_phys_read_64(
613 addr64_t paddr);
614unsigned int ml_phys_read_word(
615 vm_offset_t paddr);
616unsigned int ml_phys_read_word_64(
617 addr64_t paddr);
618
619/* Read physical address double word */
620unsigned long long ml_phys_read_double(
621 vm_offset_t paddr);
622unsigned long long ml_phys_read_double_64(
623 addr64_t paddr);
624
625/* Write physical address byte */
626void ml_phys_write_byte(
627 vm_offset_t paddr, unsigned int data);
628void ml_phys_write_byte_64(
629 addr64_t paddr, unsigned int data);
630
631/* Write physical address half word */
632void ml_phys_write_half(
633 vm_offset_t paddr, unsigned int data);
634void ml_phys_write_half_64(
635 addr64_t paddr, unsigned int data);
636
637/* Write physical address word */
638void ml_phys_write(
639 vm_offset_t paddr, unsigned int data);
640void ml_phys_write_64(
641 addr64_t paddr, unsigned int data);
642void ml_phys_write_word(
643 vm_offset_t paddr, unsigned int data);
644void ml_phys_write_word_64(
645 addr64_t paddr, unsigned int data);
646
647/* Write physical address double word */
648void ml_phys_write_double(
649 vm_offset_t paddr, unsigned long long data);
650void ml_phys_write_double_64(
651 addr64_t paddr, unsigned long long data);
652
653#if defined(__SIZEOF_INT128__) && APPLE_ARM64_ARCH_FAMILY
654/*
655 * Not all dependent projects consuming `machine_routines.h` are built using
656 * toolchains that support 128-bit integers.
657 */
658#define BUILD_QUAD_WORD_FUNCS 1
659#else
660#define BUILD_QUAD_WORD_FUNCS 0
661#endif /* defined(__SIZEOF_INT128__) && APPLE_ARM64_ARCH_FAMILY */
662
663#if BUILD_QUAD_WORD_FUNCS
664/*
665 * Not all dependent projects have their own typedef of `uint128_t` at the
666 * time they consume `machine_routines.h`.
667 */
668typedef unsigned __int128 uint128_t;
669
670/* Read physical address quad word */
671uint128_t ml_phys_read_quad(
672 vm_offset_t paddr);
673uint128_t ml_phys_read_quad_64(
674 addr64_t paddr);
675
676/* Write physical address quad word */
677void ml_phys_write_quad(
678 vm_offset_t paddr, uint128_t data);
679void ml_phys_write_quad_64(
680 addr64_t paddr, uint128_t data);
681#endif /* BUILD_QUAD_WORD_FUNCS */
682
683void ml_static_mfree(
684 vm_offset_t,
685 vm_size_t);
686
687kern_return_t
688ml_static_protect(
689 vm_offset_t start,
690 vm_size_t size,
691 vm_prot_t new_prot);
692
693/* virtual to physical on wired pages */
694vm_offset_t ml_vtophys(
695 vm_offset_t vaddr);
696
697/* Get processor cache info */
698void ml_cpu_get_info(ml_cpu_info_t *ml_cpu_info);
699void ml_cpu_get_info_type(ml_cpu_info_t * ml_cpu_info, cluster_type_t cluster_type);
700
701#endif /* __APPLE_API_UNSTABLE */
702
703typedef int ml_page_protection_t;
704
705/* Return the type of page protection supported */
706ml_page_protection_t ml_page_protection_type(void);
707
708#ifdef __APPLE_API_PRIVATE
709#ifdef XNU_KERNEL_PRIVATE
710vm_size_t ml_nofault_copy(
711 vm_offset_t virtsrc,
712 vm_offset_t virtdst,
713 vm_size_t size);
714boolean_t ml_validate_nofault(
715 vm_offset_t virtsrc, vm_size_t size);
716#endif /* XNU_KERNEL_PRIVATE */
717#if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE)
718/* IO memory map services */
719
720extern vm_offset_t io_map(
721 vm_map_offset_t phys_addr,
722 vm_size_t size,
723 unsigned int flags,
724 vm_prot_t prot,
725 bool unmappable);
726
727/* Map memory map IO space */
728vm_offset_t ml_io_map(
729 vm_offset_t phys_addr,
730 vm_size_t size);
731
732vm_offset_t ml_io_map_wcomb(
733 vm_offset_t phys_addr,
734 vm_size_t size);
735
736vm_offset_t ml_io_map_unmappable(
737 vm_offset_t phys_addr,
738 vm_size_t size,
739 uint32_t flags);
740
741vm_offset_t ml_io_map_with_prot(
742 vm_offset_t phys_addr,
743 vm_size_t size,
744 vm_prot_t prot);
745
746void ml_io_unmap(
747 vm_offset_t addr,
748 vm_size_t sz);
749
750void ml_get_bouncepool_info(
751 vm_offset_t *phys_addr,
752 vm_size_t *size);
753
754vm_map_address_t ml_map_high_window(
755 vm_offset_t phys_addr,
756 vm_size_t len);
757
758void ml_init_timebase(
759 void *args,
760 tbd_ops_t tbd_funcs,
761 vm_offset_t int_address,
762 vm_offset_t int_value);
763
764uint64_t ml_get_timebase(void);
765
766#if MACH_KERNEL_PRIVATE
767void ml_memory_to_timebase_fence(void);
768void ml_timebase_to_memory_fence(void);
769#endif /* MACH_KERNEL_PRIVATE */
770
771uint64_t ml_get_speculative_timebase(void);
772
773uint64_t ml_get_timebase_entropy(void);
774
775boolean_t ml_delay_should_spin(uint64_t interval);
776
777void ml_delay_on_yield(void);
778
779uint32_t ml_get_decrementer(void);
780
781#include <machine/config.h>
782
783uint64_t ml_get_hwclock(void);
784
785#ifdef __arm64__
786boolean_t ml_get_timer_pending(void);
787#endif
788
789void platform_syscall(
790 struct arm_saved_state *);
791
792void ml_set_decrementer(
793 uint32_t dec_value);
794
795boolean_t is_user_contex(
796 void);
797
798void ml_init_arm_debug_interface(void *args, vm_offset_t virt_address);
799
800/* These calls are only valid if __ARM_USER_PROTECT__ is defined */
801uintptr_t arm_user_protect_begin(
802 thread_t thread);
803
804void arm_user_protect_end(
805 thread_t thread,
806 uintptr_t up,
807 boolean_t disable_interrupts);
808
809#endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */
810
811/* Zero bytes starting at a physical address */
812void bzero_phys(
813 addr64_t phys_address,
814 vm_size_t length);
815
816void bzero_phys_nc(addr64_t src64, vm_size_t bytes);
817
818#if MACH_KERNEL_PRIVATE
819#ifdef __arm64__
820/* Pattern-fill buffer with zeros or a 32-bit pattern;
821 * target must be 128-byte aligned and sized a multiple of 128
822 * Both variants emit stores with non-temporal properties.
823 */
824void fill32_dczva(addr64_t, vm_size_t);
825void fill32_nt(addr64_t, vm_size_t, uint32_t);
826bool cpu_interrupt_is_pending(void);
827
828#endif // __arm64__
829#endif // MACH_KERNEL_PRIVATE
830
831void ml_thread_policy(
832 thread_t thread,
833 unsigned policy_id,
834 unsigned policy_info);
835
836#define MACHINE_GROUP 0x00000001
837#define MACHINE_NETWORK_GROUP 0x10000000
838#define MACHINE_NETWORK_WORKLOOP 0x00000001
839#define MACHINE_NETWORK_NETISR 0x00000002
840
841/* Set the maximum number of CPUs */
842void ml_set_max_cpus(
843 unsigned int max_cpus);
844
845/* Return the maximum number of CPUs set by ml_set_max_cpus(), waiting if necessary */
846unsigned int ml_wait_max_cpus(
847 void);
848
849/* Return the maximum memory size */
850unsigned int ml_get_machine_mem(void);
851
852#ifdef XNU_KERNEL_PRIVATE
853/* Return max offset */
854vm_map_offset_t ml_get_max_offset(
855 boolean_t is64,
856 unsigned int option);
857#define MACHINE_MAX_OFFSET_DEFAULT 0x01
858#define MACHINE_MAX_OFFSET_MIN 0x02
859#define MACHINE_MAX_OFFSET_MAX 0x04
860#define MACHINE_MAX_OFFSET_DEVICE 0x08
861#endif
862
863extern void ml_cpu_init_completed(void);
864extern void ml_cpu_up(void);
865extern void ml_cpu_down(void);
866/*
867 * The update to CPU counts needs to be separate from other actions
868 * in ml_cpu_up() and ml_cpu_down()
869 * because we don't update the counts when CLPC causes temporary
870 * cluster powerdown events, as these must be transparent to the user.
871 */
872extern void ml_cpu_up_update_counts(int cpu_id);
873extern void ml_cpu_down_update_counts(int cpu_id);
874extern void ml_arm_sleep(void);
875
876extern uint64_t ml_get_wake_timebase(void);
877extern uint64_t ml_get_conttime_wake_time(void);
878
879/* Time since the system was reset (as part of boot/wake) */
880uint64_t ml_get_time_since_reset(void);
881
882/*
883 * Called by ApplePMGR to set wake time. Units and epoch are identical
884 * to mach_continuous_time(). Has no effect on !HAS_CONTINUOUS_HWCLOCK
885 * chips. If wake_time == UINT64_MAX, that means the wake time is
886 * unknown and calls to ml_get_time_since_reset() will return UINT64_MAX.
887 */
888void ml_set_reset_time(uint64_t wake_time);
889
890#ifdef XNU_KERNEL_PRIVATE
891/* Just a stub on ARM */
892extern kern_return_t ml_interrupt_prewarm(uint64_t deadline);
893#define TCOAL_DEBUG(x, a, b, c, d, e) do { } while(0)
894#endif /* XNU_KERNEL_PRIVATE */
895
896/* Bytes available on current stack */
897vm_offset_t ml_stack_remaining(void);
898
899#ifdef MACH_KERNEL_PRIVATE
900uint32_t get_fpscr(void);
901void set_fpscr(uint32_t);
902void machine_conf(void);
903void machine_lockdown(void);
904
905#ifdef __arm64__
906unsigned long update_mdscr(unsigned long clear, unsigned long set);
907#endif /* __arm64__ */
908
909extern void arm_debug_set_cp14(arm_debug_state_t *debug_state);
910extern void fiq_context_init(boolean_t enable_fiq);
911
912extern void reenable_async_aborts(void);
913
914#ifdef __arm64__
915uint64_t ml_cluster_wfe_timeout(uint32_t wfe_cluster_id);
916#endif
917
918#ifdef MONITOR
919#define MONITOR_SET_ENTRY 0x800 /* Set kernel entry point from monitor */
920#define MONITOR_LOCKDOWN 0x801 /* Enforce kernel text/rodata integrity */
921unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1,
922 uintptr_t arg2, uintptr_t arg3);
923#endif /* MONITOR */
924
925#if __ARM_KERNEL_PROTECT__
926extern void set_vbar_el1(uint64_t);
927#endif /* __ARM_KERNEL_PROTECT__ */
928#endif /* MACH_KERNEL_PRIVATE */
929
930extern uint32_t arm_debug_read_dscr(void);
931
932extern int set_be_bit(void);
933extern int clr_be_bit(void);
934extern int be_tracing(void);
935
936/* Please note that cpu_broadcast_xcall is not as simple is you would like it to be.
937 * It will sometimes put the calling thread to sleep, and it is up to your callback
938 * to wake it up as needed, where "as needed" is defined as "all other CPUs have
939 * called the broadcast func". Look around the kernel for examples, or instead use
940 * cpu_broadcast_xcall_simple() which does indeed act like you would expect, given
941 * the prototype. cpu_broadcast_immediate_xcall has the same caveats and has a similar
942 * _simple() wrapper
943 */
944typedef void (*broadcastFunc) (void *);
945unsigned int cpu_broadcast_xcall(uint32_t *, boolean_t, broadcastFunc, void *);
946unsigned int cpu_broadcast_xcall_simple(boolean_t, broadcastFunc, void *);
947__result_use_check kern_return_t cpu_xcall(int, broadcastFunc, void *);
948unsigned int cpu_broadcast_immediate_xcall(uint32_t *, boolean_t, broadcastFunc, void *);
949unsigned int cpu_broadcast_immediate_xcall_simple(boolean_t, broadcastFunc, void *);
950__result_use_check kern_return_t cpu_immediate_xcall(int, broadcastFunc, void *);
951
952#ifdef KERNEL_PRIVATE
953
954/* Interface to be used by the perf. controller to register a callback, in a
955 * single-threaded fashion. The callback will receive notifications of
956 * processor performance quality-of-service changes from the scheduler.
957 */
958
959#ifdef __arm64__
960typedef void (*cpu_qos_update_t)(int throughput_qos, uint64_t qos_param1, uint64_t qos_param2);
961void cpu_qos_update_register(cpu_qos_update_t);
962#endif /* __arm64__ */
963
964struct going_on_core {
965 uint64_t thread_id;
966 uint16_t qos_class;
967 uint16_t urgency; /* XCPM compatibility */
968 uint32_t is_32_bit : 1; /* uses 32-bit ISA/register state in userspace (which may differ from address space size) */
969 uint32_t is_kernel_thread : 1;
970 uint64_t thread_group_id;
971 void *thread_group_data;
972 uint64_t scheduling_latency; /* absolute time between when thread was made runnable and this ctx switch */
973 uint64_t start_time;
974 uint64_t scheduling_latency_at_same_basepri;
975 uint32_t energy_estimate_nj; /* return: In nanojoules */
976 /* smaller of the time between last change to base priority and ctx switch and scheduling_latency */
977};
978typedef struct going_on_core *going_on_core_t;
979
980struct going_off_core {
981 uint64_t thread_id;
982 uint32_t energy_estimate_nj; /* return: In nanojoules */
983 uint32_t reserved;
984 uint64_t end_time;
985 uint64_t thread_group_id;
986 void *thread_group_data;
987};
988typedef struct going_off_core *going_off_core_t;
989
990struct thread_group_data {
991 uint64_t thread_group_id;
992 void *thread_group_data;
993 uint32_t thread_group_size;
994 uint32_t thread_group_flags;
995};
996typedef struct thread_group_data *thread_group_data_t;
997
998struct perfcontrol_max_runnable_latency {
999 uint64_t max_scheduling_latencies[4 /* THREAD_URGENCY_MAX */];
1000};
1001typedef struct perfcontrol_max_runnable_latency *perfcontrol_max_runnable_latency_t;
1002
1003struct perfcontrol_work_interval {
1004 uint64_t thread_id;
1005 uint16_t qos_class;
1006 uint16_t urgency;
1007 uint32_t flags; // notify
1008 uint64_t work_interval_id;
1009 uint64_t start;
1010 uint64_t finish;
1011 uint64_t deadline;
1012 uint64_t next_start;
1013 uint64_t thread_group_id;
1014 void *thread_group_data;
1015 uint32_t create_flags;
1016};
1017typedef struct perfcontrol_work_interval *perfcontrol_work_interval_t;
1018
1019typedef enum {
1020 WORK_INTERVAL_START,
1021 WORK_INTERVAL_UPDATE,
1022 WORK_INTERVAL_FINISH,
1023 WORK_INTERVAL_CREATE,
1024 WORK_INTERVAL_DEALLOCATE,
1025} work_interval_ctl_t;
1026
1027struct perfcontrol_work_interval_instance {
1028 work_interval_ctl_t ctl;
1029 uint32_t create_flags;
1030 uint64_t complexity;
1031 uint64_t thread_id;
1032 uint64_t work_interval_id;
1033 uint64_t instance_id; /* out: start, in: update/finish */
1034 uint64_t start;
1035 uint64_t finish;
1036 uint64_t deadline;
1037 uint64_t thread_group_id;
1038 void *thread_group_data;
1039};
1040typedef struct perfcontrol_work_interval_instance *perfcontrol_work_interval_instance_t;
1041
1042/*
1043 * Structure to export per-CPU counters as part of the CLPC callout.
1044 * Contains only the fixed CPU counters (instructions and cycles); CLPC
1045 * would call back into XNU to get the configurable counters if needed.
1046 */
1047struct perfcontrol_cpu_counters {
1048 uint64_t instructions;
1049 uint64_t cycles;
1050};
1051
1052__options_decl(perfcontrol_thread_flags_mask_t, uint64_t, {
1053 PERFCTL_THREAD_FLAGS_MASK_CLUSTER_SHARED_RSRC_RR = 1 << 0,
1054 PERFCTL_THREAD_FLAGS_MASK_CLUSTER_SHARED_RSRC_NATIVE_FIRST = 1 << 1,
1055});
1056
1057
1058/*
1059 * Structure used to pass information about a thread to CLPC
1060 */
1061struct perfcontrol_thread_data {
1062 /*
1063 * Energy estimate (return value)
1064 * The field is populated by CLPC and used to update the
1065 * energy estimate of the thread
1066 */
1067 uint32_t energy_estimate_nj;
1068 /* Perfcontrol class for thread */
1069 perfcontrol_class_t perfctl_class;
1070 /* Thread ID for the thread */
1071 uint64_t thread_id;
1072 /* Thread Group ID */
1073 uint64_t thread_group_id;
1074 /*
1075 * Scheduling latency for threads at the same base priority.
1076 * Calculated by the scheduler and passed into CLPC. The field is
1077 * populated only in the thread_data structure for the thread
1078 * going on-core.
1079 */
1080 uint64_t scheduling_latency_at_same_basepri;
1081 /* Thread Group data pointer */
1082 void *thread_group_data;
1083 /* perfctl state pointer */
1084 void *perfctl_state;
1085 /* Bitmask to indicate which thread flags have been updated as part of the callout */
1086 perfcontrol_thread_flags_mask_t thread_flags_mask;
1087 /* Actual values for the flags that are getting updated in the callout */
1088 perfcontrol_thread_flags_mask_t thread_flags;
1089};
1090
1091/*
1092 * All callouts from the scheduler are executed with interrupts
1093 * disabled. Callouts should be implemented in C with minimal
1094 * abstractions, and only use KPI exported by the mach/libkern
1095 * symbolset, restricted to routines like spinlocks and atomic
1096 * operations and scheduler routines as noted below. Spinlocks that
1097 * are used to synchronize data in the perfcontrol_state_t should only
1098 * ever be acquired with interrupts disabled, to avoid deadlocks where
1099 * an quantum expiration timer interrupt attempts to perform a callout
1100 * that attempts to lock a spinlock that is already held.
1101 */
1102
1103/*
1104 * When a processor is switching between two threads (after the
1105 * scheduler has chosen a new thread), the low-level platform layer
1106 * will call this routine, which should perform required timestamps,
1107 * MMIO register reads, or other state switching. No scheduler locks
1108 * are held during this callout.
1109 *
1110 * This function is called with interrupts ENABLED.
1111 */
1112typedef void (*sched_perfcontrol_context_switch_t)(perfcontrol_state_t, perfcontrol_state_t);
1113
1114/*
1115 * Once the processor has switched to the new thread, the offcore
1116 * callout will indicate the old thread that is no longer being
1117 * run. The thread's scheduler lock is held, so it will not begin
1118 * running on another processor (in the case of preemption where it
1119 * remains runnable) until it completes. If the "thread_terminating"
1120 * boolean is TRUE, this will be the last callout for this thread_id.
1121 */
1122typedef void (*sched_perfcontrol_offcore_t)(perfcontrol_state_t, going_off_core_t /* populated by callee */, boolean_t);
1123
1124/*
1125 * After the offcore callout and after the old thread can potentially
1126 * start running on another processor, the oncore callout will be
1127 * called with the thread's scheduler lock held. The oncore callout is
1128 * also called any time one of the parameters in the going_on_core_t
1129 * structure changes, like priority/QoS changes, and quantum
1130 * expiration, so the callout must not assume callouts are paired with
1131 * offcore callouts.
1132 */
1133typedef void (*sched_perfcontrol_oncore_t)(perfcontrol_state_t, going_on_core_t);
1134
1135/*
1136 * Periodically (on hundreds of ms scale), the scheduler will perform
1137 * maintenance and report the maximum latency for runnable (but not currently
1138 * running) threads for each urgency class.
1139 */
1140typedef void (*sched_perfcontrol_max_runnable_latency_t)(perfcontrol_max_runnable_latency_t);
1141
1142/*
1143 * When the kernel receives information about work intervals from userland,
1144 * it is passed along using this callback. No locks are held, although the state
1145 * object will not go away during the callout.
1146 */
1147typedef void (*sched_perfcontrol_work_interval_notify_t)(perfcontrol_state_t, perfcontrol_work_interval_t);
1148
1149/*
1150 * Start, update and finish work interval instance with optional complexity estimate.
1151 */
1152typedef void (*sched_perfcontrol_work_interval_ctl_t)(perfcontrol_state_t, perfcontrol_work_interval_instance_t);
1153
1154/*
1155 * These callbacks are used when thread groups are added, removed or properties
1156 * updated.
1157 * No blocking allocations (or anything else blocking) are allowed inside these
1158 * callbacks. No locks allowed in these callbacks as well since the kernel might
1159 * be holding the thread/task locks.
1160 */
1161typedef void (*sched_perfcontrol_thread_group_init_t)(thread_group_data_t);
1162typedef void (*sched_perfcontrol_thread_group_deinit_t)(thread_group_data_t);
1163typedef void (*sched_perfcontrol_thread_group_flags_update_t)(thread_group_data_t);
1164
1165/*
1166 * Sometime after the timeout set by sched_perfcontrol_update_callback_deadline has passed,
1167 * this function will be called, passing the timeout deadline that was previously armed as an argument.
1168 *
1169 * This is called inside context-switch/quantum-interrupt context and must follow the safety rules for that context.
1170 */
1171typedef void (*sched_perfcontrol_deadline_passed_t)(uint64_t deadline);
1172
1173/*
1174 * Context Switch Callout
1175 *
1176 * Parameters:
1177 * event - The perfcontrol_event for this callout
1178 * cpu_id - The CPU doing the context switch
1179 * timestamp - The timestamp for the context switch
1180 * flags - Flags for other relevant information
1181 * offcore - perfcontrol_data structure for thread going off-core
1182 * oncore - perfcontrol_data structure for thread going on-core
1183 * cpu_counters - perfcontrol_cpu_counters for the CPU doing the switch
1184 */
1185typedef void (*sched_perfcontrol_csw_t)(
1186 perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags,
1187 struct perfcontrol_thread_data *offcore, struct perfcontrol_thread_data *oncore,
1188 struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused);
1189
1190
1191/*
1192 * Thread State Update Callout
1193 *
1194 * Parameters:
1195 * event - The perfcontrol_event for this callout
1196 * cpu_id - The CPU doing the state update
1197 * timestamp - The timestamp for the state update
1198 * flags - Flags for other relevant information
1199 * thr_data - perfcontrol_data structure for the thread being updated
1200 */
1201typedef void (*sched_perfcontrol_state_update_t)(
1202 perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags,
1203 struct perfcontrol_thread_data *thr_data, __unused void *unused);
1204
1205/*
1206 * Thread Group Blocking Relationship Callout
1207 *
1208 * Parameters:
1209 * blocked_tg - Thread group blocking on progress of another thread group
1210 * blocking_tg - Thread group blocking progress of another thread group
1211 * flags - Flags for other relevant information
1212 * blocked_thr_state - Per-thread perfcontrol state for blocked thread
1213 */
1214typedef void (*sched_perfcontrol_thread_group_blocked_t)(
1215 thread_group_data_t blocked_tg, thread_group_data_t blocking_tg, uint32_t flags, perfcontrol_state_t blocked_thr_state);
1216
1217/*
1218 * Thread Group Unblocking Callout
1219 *
1220 * Parameters:
1221 * unblocked_tg - Thread group being unblocked from making forward progress
1222 * unblocking_tg - Thread group unblocking progress of another thread group
1223 * flags - Flags for other relevant information
1224 * unblocked_thr_state - Per-thread perfcontrol state for unblocked thread
1225 */
1226typedef void (*sched_perfcontrol_thread_group_unblocked_t)(
1227 thread_group_data_t unblocked_tg, thread_group_data_t unblocking_tg, uint32_t flags, perfcontrol_state_t unblocked_thr_state);
1228
1229/*
1230 * Callers should always use the CURRENT version so that the kernel can detect both older
1231 * and newer structure layouts. New callbacks should always be added at the end of the
1232 * structure, and xnu should expect existing source recompiled against newer headers
1233 * to pass NULL for unimplemented callbacks. Pass NULL as the as the callbacks parameter
1234 * to reset callbacks to their default in-kernel values.
1235 */
1236
1237#define SCHED_PERFCONTROL_CALLBACKS_VERSION_0 (0) /* up-to oncore */
1238#define SCHED_PERFCONTROL_CALLBACKS_VERSION_1 (1) /* up-to max_runnable_latency */
1239#define SCHED_PERFCONTROL_CALLBACKS_VERSION_2 (2) /* up-to work_interval_notify */
1240#define SCHED_PERFCONTROL_CALLBACKS_VERSION_3 (3) /* up-to thread_group_deinit */
1241#define SCHED_PERFCONTROL_CALLBACKS_VERSION_4 (4) /* up-to deadline_passed */
1242#define SCHED_PERFCONTROL_CALLBACKS_VERSION_5 (5) /* up-to state_update */
1243#define SCHED_PERFCONTROL_CALLBACKS_VERSION_6 (6) /* up-to thread_group_flags_update */
1244#define SCHED_PERFCONTROL_CALLBACKS_VERSION_7 (7) /* up-to work_interval_ctl */
1245#define SCHED_PERFCONTROL_CALLBACKS_VERSION_8 (8) /* up-to thread_group_unblocked */
1246#define SCHED_PERFCONTROL_CALLBACKS_VERSION_9 (9) /* allows CLPC to specify resource contention flags */
1247#define SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT SCHED_PERFCONTROL_CALLBACKS_VERSION_6
1248
1249struct sched_perfcontrol_callbacks {
1250 unsigned long version; /* Use SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT */
1251 sched_perfcontrol_offcore_t offcore;
1252 sched_perfcontrol_context_switch_t context_switch;
1253 sched_perfcontrol_oncore_t oncore;
1254 sched_perfcontrol_max_runnable_latency_t max_runnable_latency;
1255 sched_perfcontrol_work_interval_notify_t work_interval_notify;
1256 sched_perfcontrol_thread_group_init_t thread_group_init;
1257 sched_perfcontrol_thread_group_deinit_t thread_group_deinit;
1258 sched_perfcontrol_deadline_passed_t deadline_passed;
1259 sched_perfcontrol_csw_t csw;
1260 sched_perfcontrol_state_update_t state_update;
1261 sched_perfcontrol_thread_group_flags_update_t thread_group_flags_update;
1262 sched_perfcontrol_work_interval_ctl_t work_interval_ctl;
1263 sched_perfcontrol_thread_group_blocked_t thread_group_blocked;
1264 sched_perfcontrol_thread_group_unblocked_t thread_group_unblocked;
1265};
1266typedef struct sched_perfcontrol_callbacks *sched_perfcontrol_callbacks_t;
1267
1268extern void sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state);
1269extern void sched_perfcontrol_thread_group_recommend(void *data, cluster_type_t recommendation);
1270extern void sched_perfcontrol_inherit_recommendation_from_tg(perfcontrol_class_t perfctl_class, boolean_t inherit);
1271extern const char* sched_perfcontrol_thread_group_get_name(void *data);
1272
1273/*
1274 * Edge Scheduler-CLPC Interface
1275 *
1276 * sched_perfcontrol_thread_group_preferred_clusters_set()
1277 *
1278 * The Edge scheduler expects thread group recommendations to be specific clusters rather
1279 * than just E/P. In order to allow more fine grained control, CLPC can specify an override
1280 * preferred cluster per QoS bucket. CLPC passes a common preferred cluster `tg_preferred_cluster`
1281 * and an array of size [PERFCONTROL_CLASS_MAX] with overrides for specific perfctl classes.
1282 * The scheduler translates these preferences into sched_bucket
1283 * preferences and applies the changes.
1284 *
1285 */
1286/* Token to indicate a particular perfctl class is not overriden */
1287#define SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE ((uint32_t)~0)
1288
1289/*
1290 * CLPC can also indicate if there should be an immediate rebalancing of threads of this TG as
1291 * part of this preferred cluster change. It does that by specifying the following options.
1292 */
1293#define SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNING 0x1
1294#define SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNABLE 0x2
1295typedef uint64_t sched_perfcontrol_preferred_cluster_options_t;
1296
1297extern void sched_perfcontrol_thread_group_preferred_clusters_set(void *machine_data, uint32_t tg_preferred_cluster,
1298 uint32_t overrides[PERFCONTROL_CLASS_MAX], sched_perfcontrol_preferred_cluster_options_t options);
1299
1300/*
1301 * Edge Scheduler-CLPC Interface
1302 *
1303 * sched_perfcontrol_edge_matrix_get()/sched_perfcontrol_edge_matrix_set()
1304 *
1305 * The Edge scheduler uses edges between clusters to define the likelihood of migrating threads
1306 * across clusters. The edge config between any two clusters defines the edge weight and whether
1307 * migation and steal operations are allowed across that edge. The getter and setter allow CLPC
1308 * to query and configure edge properties between various clusters on the platform.
1309 */
1310
1311extern void sched_perfcontrol_edge_matrix_get(sched_clutch_edge *edge_matrix, bool *edge_request_bitmap, uint64_t flags, uint64_t matrix_order);
1312extern void sched_perfcontrol_edge_matrix_set(sched_clutch_edge *edge_matrix, bool *edge_changes_bitmap, uint64_t flags, uint64_t matrix_order);
1313
1314/*
1315 * sched_perfcontrol_edge_cpu_rotation_bitmasks_get()/sched_perfcontrol_edge_cpu_rotation_bitmasks_set()
1316 *
1317 * In order to drive intra-cluster core rotation CLPC supplies the edge scheduler with a pair of
1318 * per-cluster bitmasks. The preferred_bitmask is a bitmask of CPU cores where if a bit is set,
1319 * CLPC would prefer threads to be scheduled on that core if it is idle. The migration_bitmask
1320 * is a bitmask of CPU cores where if a bit is set, CLPC would prefer threads no longer continue
1321 * running on that core if there is any other non-avoided idle core in the cluster that is available.
1322 */
1323
1324extern void sched_perfcontrol_edge_cpu_rotation_bitmasks_set(uint32_t cluster_id, uint64_t preferred_bitmask, uint64_t migration_bitmask);
1325extern void sched_perfcontrol_edge_cpu_rotation_bitmasks_get(uint32_t cluster_id, uint64_t *preferred_bitmask, uint64_t *migration_bitmask);
1326
1327/*
1328 * Update the deadline after which sched_perfcontrol_deadline_passed will be called.
1329 * Returns TRUE if it successfully canceled a previously set callback,
1330 * and FALSE if it did not (i.e. one wasn't set, or callback already fired / is in flight).
1331 * The callback is automatically canceled when it fires, and does not repeat unless rearmed.
1332 *
1333 * This 'timer' executes as the scheduler switches between threads, on a non-idle core
1334 *
1335 * There can be only one outstanding timer globally.
1336 */
1337extern boolean_t sched_perfcontrol_update_callback_deadline(uint64_t deadline);
1338
1339/*
1340 * SFI configuration.
1341 */
1342extern kern_return_t sched_perfcontrol_sfi_set_window(uint64_t window_usecs);
1343extern kern_return_t sched_perfcontrol_sfi_set_bg_offtime(uint64_t offtime_usecs);
1344extern kern_return_t sched_perfcontrol_sfi_set_utility_offtime(uint64_t offtime_usecs);
1345
1346typedef enum perfcontrol_callout_type {
1347 PERFCONTROL_CALLOUT_ON_CORE,
1348 PERFCONTROL_CALLOUT_OFF_CORE,
1349 PERFCONTROL_CALLOUT_CONTEXT,
1350 PERFCONTROL_CALLOUT_STATE_UPDATE,
1351 /* Add other callout types here */
1352 PERFCONTROL_CALLOUT_MAX
1353} perfcontrol_callout_type_t;
1354
1355typedef enum perfcontrol_callout_stat {
1356 PERFCONTROL_STAT_INSTRS,
1357 PERFCONTROL_STAT_CYCLES,
1358 /* Add other stat types here */
1359 PERFCONTROL_STAT_MAX
1360} perfcontrol_callout_stat_t;
1361
1362uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type,
1363 perfcontrol_callout_stat_t stat);
1364
1365#ifdef __arm64__
1366/* The performance controller may use this interface to recommend
1367 * that CPUs in the designated cluster employ WFE rather than WFI
1368 * within the idle loop, falling back to WFI after the specified
1369 * timeout. The updates are expected to be serialized by the caller,
1370 * the implementation is not required to perform internal synchronization.
1371 */
1372uint32_t ml_update_cluster_wfe_recommendation(uint32_t wfe_cluster_id, uint64_t wfe_timeout_abstime_interval, uint64_t wfe_hint_flags);
1373#endif /* __arm64__ */
1374
1375#if defined(HAS_APPLE_PAC)
1376#define ONES(x) (BIT((x))-1)
1377#define PTR_MASK ONES(64-T1SZ_BOOT)
1378#define PAC_MASK ~PTR_MASK
1379#define SIGN(p) ((p) & BIT(55))
1380#define UNSIGN_PTR(p) \
1381 SIGN(p) ? ((p) | PAC_MASK) : ((p) & ~PAC_MASK)
1382
1383uint64_t ml_default_rop_pid(void);
1384uint64_t ml_default_jop_pid(void);
1385uint64_t ml_non_arm64e_user_jop_pid(void);
1386void ml_task_set_rop_pid(task_t task, task_t parent_task, boolean_t inherit);
1387void ml_task_set_jop_pid(task_t task, task_t parent_task, boolean_t inherit, boolean_t disable_user_jop);
1388void ml_task_set_jop_pid_from_shared_region(task_t task, boolean_t disable_user_jop);
1389uint8_t ml_task_get_disable_user_jop(task_t task);
1390void ml_task_set_disable_user_jop(task_t task, uint8_t disable_user_jop);
1391void ml_thread_set_disable_user_jop(thread_t thread, uint8_t disable_user_jop);
1392void ml_thread_set_jop_pid(thread_t thread, task_t task);
1393void *ml_auth_ptr_unchecked(void *ptr, unsigned key, uint64_t modifier);
1394
1395uint64_t ml_enable_user_jop_key(uint64_t user_jop_key);
1396
1397/**
1398 * Restores the previous JOP key state after a previous ml_enable_user_jop_key()
1399 * call.
1400 *
1401 * @param user_jop_key The userspace JOP key previously passed to
1402 * ml_enable_user_jop_key()
1403 * @param saved_jop_state The saved JOP state returned by
1404 * ml_enable_user_jop_key()
1405 */
1406void ml_disable_user_jop_key(uint64_t user_jop_key, uint64_t saved_jop_state);
1407#endif /* defined(HAS_APPLE_PAC) */
1408
1409void ml_enable_monitor(void);
1410
1411#endif /* KERNEL_PRIVATE */
1412
1413boolean_t machine_timeout_suspended(void);
1414void ml_get_power_state(boolean_t *, boolean_t *);
1415
1416uint32_t get_arm_cpu_version(void);
1417boolean_t user_cont_hwclock_allowed(void);
1418uint8_t user_timebase_type(void);
1419boolean_t ml_thread_is64bit(thread_t thread);
1420
1421#ifdef __arm64__
1422bool ml_feature_supported(uint32_t feature_bit);
1423void ml_set_align_checking(void);
1424extern void wfe_timeout_configure(void);
1425extern void wfe_timeout_init(void);
1426#endif /* __arm64__ */
1427
1428void ml_timer_evaluate(void);
1429boolean_t ml_timer_forced_evaluation(void);
1430void ml_gpu_stat_update(uint64_t);
1431uint64_t ml_gpu_stat(thread_t);
1432#endif /* __APPLE_API_PRIVATE */
1433
1434
1435
1436#if __arm64__ && defined(CONFIG_XNUPOST) && defined(XNU_KERNEL_PRIVATE)
1437extern void ml_expect_fault_begin(expected_fault_handler_t, uintptr_t);
1438extern void ml_expect_fault_pc_begin(expected_fault_handler_t, uintptr_t);
1439extern void ml_expect_fault_end(void);
1440#endif /* __arm64__ && defined(CONFIG_XNUPOST) && defined(XNU_KERNEL_PRIVATE) */
1441
1442
1443extern uint32_t phy_read_panic;
1444extern uint32_t phy_write_panic;
1445#if DEVELOPMENT || DEBUG
1446extern uint64_t simulate_stretched_io;
1447#endif
1448
1449void ml_hibernate_active_pre(void);
1450void ml_hibernate_active_post(void);
1451
1452void ml_report_minor_badness(uint32_t badness_id);
1453#define ML_MINOR_BADNESS_CONSOLE_BUFFER_FULL 0
1454#define ML_MINOR_BADNESS_MEMFAULT_REPORTING_NOT_ENABLED 1
1455#define ML_MINOR_BADNESS_PIO_WRITTEN_FROM_USERSPACE 2
1456
1457#ifdef XNU_KERNEL_PRIVATE
1458/**
1459 * Depending on the system, by the time a backtracer starts inspecting an
1460 * interrupted CPU's register state, the value of the PC might have been
1461 * modified. In those cases, the original PC value is placed into a different
1462 * register. This function abstracts out those differences for a backtracer
1463 * wanting the PC of an interrupted CPU.
1464 *
1465 * @param state The ARM register state to parse.
1466 *
1467 * @return The original PC of the interrupted CPU.
1468 */
1469uint64_t ml_get_backtrace_pc(struct arm_saved_state *state);
1470#endif /* XNU_KERNEL_PRIVATE */
1471
1472#ifdef KERNEL_PRIVATE
1473/**
1474 * Given a physical address, return whether that address is owned by the secure
1475 * world.
1476 *
1477 * @note This does not include memory shared between XNU and the secure world.
1478 *
1479 * @param paddr The physical address to check.
1480 *
1481 * @return True if the physical address is owned and being used exclusively by
1482 * the secure world, false otherwise.
1483 */
1484bool ml_paddr_is_exclaves_owned(vm_offset_t paddr);
1485#endif /* KERNEL_PRIVATE */
1486
1487__END_DECLS
1488
1489#endif /* _ARM_MACHINE_ROUTINES_H_ */
1490