1/*
2 * Copyright (c) 2007-2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31
32#ifndef _ARM_MACHINE_ROUTINES_H_
33#define _ARM_MACHINE_ROUTINES_H_
34
35#include <mach/mach_types.h>
36#include <mach/boolean.h>
37#include <kern/kern_types.h>
38#include <pexpert/pexpert.h>
39
40#include <sys/cdefs.h>
41#include <sys/appleapiopts.h>
42
43#include <stdarg.h>
44
45__BEGIN_DECLS
46
47/* Interrupt handling */
48
49void ml_cpu_signal(unsigned int cpu_id);
50void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs);
51uint64_t ml_cpu_signal_deferred_get_timer(void);
52void ml_cpu_signal_deferred(unsigned int cpu_id);
53void ml_cpu_signal_retract(unsigned int cpu_id);
54
55/* Initialize Interrupts */
56void ml_init_interrupt(void);
57
58/* Get Interrupts Enabled */
59boolean_t ml_get_interrupts_enabled(void);
60
61/* Set Interrupts Enabled */
62boolean_t ml_set_interrupts_enabled(boolean_t enable);
63
64/* Check if running at interrupt context */
65boolean_t ml_at_interrupt_context(void);
66
67/* Generate a fake interrupt */
68void ml_cause_interrupt(void);
69
70/* Clear interrupt spin debug state for thread */
71#if INTERRUPT_MASKED_DEBUG
72void ml_spin_debug_reset(thread_t thread);
73void ml_spin_debug_clear(thread_t thread);
74void ml_spin_debug_clear_self(void);
75void ml_check_interrupts_disabled_duration(thread_t thread);
76#endif
77
78#ifdef XNU_KERNEL_PRIVATE
79extern boolean_t ml_is_quiescing(void);
80extern void ml_set_is_quiescing(boolean_t);
81extern uint64_t ml_get_booter_memory_size(void);
82#endif
83
84/* Type for the Time Base Enable function */
85typedef void (*time_base_enable_t)(cpu_id_t cpu_id, boolean_t enable);
86#if MACH_KERNEL_PRIVATE
87/* Type for the Processor Cache Dispatch function */
88typedef void (*cache_dispatch_t)(cpu_id_t cpu_id, unsigned int select, unsigned int param0, unsigned int param1);
89#endif
90
91#define CacheConfig 0x00000000UL
92#define CacheControl 0x00000001UL
93#define CacheClean 0x00000002UL
94#define CacheCleanRegion 0x00000003UL
95#define CacheCleanFlush 0x00000004UL
96#define CacheCleanFlushRegion 0x00000005UL
97#define CacheShutdown 0x00000006UL
98
99#define CacheControlEnable 0x00000000UL
100
101#define CacheConfigCCSIDR 0x00000001UL
102#define CacheConfigSize 0x00000100UL
103
104/* Type for the Processor Idle function */
105typedef void (*processor_idle_t)(cpu_id_t cpu_id, boolean_t enter, uint64_t *new_timeout_ticks);
106
107/* Type for the Idle Tickle function */
108typedef void (*idle_tickle_t)(void);
109
110/* Type for the Idle Timer function */
111typedef void (*idle_timer_t)(void *refcon, uint64_t *new_timeout_ticks);
112
113/* Type for the IPI Hander */
114typedef void (*ipi_handler_t)(void);
115
116/* Type for the Lockdown Hander */
117typedef void (*lockdown_handler_t)(void *);
118
119/* Type for the Platform specific Error Handler */
120typedef void (*platform_error_handler_t)(void *refcon, vm_offset_t fault_addr);
121
122/*
123 * The exception callback (ex_cb) module allows kernel drivers to
124 * register and receive callbacks for exceptions, and indicate
125 * actions to be taken by the platform kernel
126 * Currently this is supported for ARM64 but extending support for ARM32
127 * should be straightforward
128 */
129
130/* Supported exception classes for callbacks */
131typedef enum
132{
133 EXCB_CLASS_ILLEGAL_INSTR_SET,
134#ifdef CONFIG_XNUPOST
135 EXCB_CLASS_TEST1,
136 EXCB_CLASS_TEST2,
137 EXCB_CLASS_TEST3,
138#endif
139 EXCB_CLASS_MAX // this must be last
140}
141ex_cb_class_t;
142
143/* Actions indicated by callbacks to be taken by platform kernel */
144typedef enum
145{
146 EXCB_ACTION_RERUN, // re-run the faulting instruction
147 EXCB_ACTION_NONE, // continue normal exception handling
148#ifdef CONFIG_XNUPOST
149 EXCB_ACTION_TEST_FAIL,
150#endif
151}
152ex_cb_action_t;
153
154/*
155 * Exception state
156 * We cannot use a private kernel data structure such as arm_saved_state_t
157 * The CPSR and ESR are not clobbered when the callback function is invoked so
158 * those registers can be examined by the callback function;
159 * the same is done in the platform error handlers
160 */
161typedef struct
162{
163 vm_offset_t far;
164}
165ex_cb_state_t;
166
167/* callback type definition */
168typedef ex_cb_action_t (*ex_cb_t) (
169 ex_cb_class_t cb_class,
170 void *refcon,// provided at registration
171 const ex_cb_state_t *state // exception state
172 );
173
174/*
175 * Callback registration
176 * Currently we support only one registered callback per class but
177 * it should be possible to support more callbacks
178 */
179kern_return_t ex_cb_register(
180 ex_cb_class_t cb_class,
181 ex_cb_t cb,
182 void *refcon );
183
184/*
185 * Called internally by platform kernel to invoke the registered callback for class
186 */
187ex_cb_action_t ex_cb_invoke(
188 ex_cb_class_t cb_class,
189 vm_offset_t far);
190
191
192void ml_parse_cpu_topology(void);
193
194unsigned int ml_get_cpu_count(void);
195
196int ml_get_boot_cpu_number(void);
197
198int ml_get_cpu_number(uint32_t phys_id);
199
200int ml_get_max_cpu_number(void);
201
202/* Struct for ml_cpu_get_info */
203struct ml_cpu_info {
204 unsigned long vector_unit;
205 unsigned long cache_line_size;
206 unsigned long l1_icache_size;
207 unsigned long l1_dcache_size;
208 unsigned long l2_settings;
209 unsigned long l2_cache_size;
210 unsigned long l3_settings;
211 unsigned long l3_cache_size;
212};
213typedef struct ml_cpu_info ml_cpu_info_t;
214
215typedef enum {
216 CLUSTER_TYPE_SMP,
217} cluster_type_t;
218
219cluster_type_t ml_get_boot_cluster(void);
220
221/* Struct for ml_processor_register */
222struct ml_processor_info {
223 cpu_id_t cpu_id;
224 vm_offset_t start_paddr;
225 boolean_t supports_nap;
226 void *platform_cache_dispatch;
227 time_base_enable_t time_base_enable;
228 processor_idle_t processor_idle;
229 idle_tickle_t *idle_tickle;
230 idle_timer_t idle_timer;
231 void *idle_timer_refcon;
232 vm_offset_t powergate_stub_addr;
233 uint32_t powergate_stub_length;
234 uint32_t powergate_latency;
235 platform_error_handler_t platform_error_handler;
236 uint64_t regmap_paddr;
237 uint32_t phys_id;
238 uint32_t log_id;
239 uint32_t l2_access_penalty;
240 uint32_t cluster_id;
241 cluster_type_t cluster_type;
242 uint32_t l2_cache_id;
243 uint32_t l2_cache_size;
244 uint32_t l3_cache_id;
245 uint32_t l3_cache_size;
246};
247typedef struct ml_processor_info ml_processor_info_t;
248
249#if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE)
250/* Struct for ml_init_timebase */
251struct tbd_ops {
252 void (*tbd_fiq_handler)(void);
253 uint32_t (*tbd_get_decrementer)(void);
254 void (*tbd_set_decrementer)(uint32_t dec_value);
255};
256typedef struct tbd_ops *tbd_ops_t;
257typedef struct tbd_ops tbd_ops_data_t;
258#endif
259
260/* Register a processor */
261kern_return_t ml_processor_register(
262 ml_processor_info_t *ml_processor_info,
263 processor_t *processor,
264 ipi_handler_t *ipi_handler);
265
266/* Register a lockdown handler */
267kern_return_t ml_lockdown_handler_register(lockdown_handler_t, void *);
268
269#if XNU_KERNEL_PRIVATE
270void ml_lockdown_init(void);
271
272/* Check if the machine layer wants to intercept a panic call */
273boolean_t ml_wants_panic_trap_to_debugger(void);
274
275/* Machine layer routine for intercepting panics */
276void ml_panic_trap_to_debugger(const char *panic_format_str,
277 va_list *panic_args,
278 unsigned int reason,
279 void *ctx,
280 uint64_t panic_options_mask,
281 unsigned long panic_caller);
282#endif /* XNU_KERNEL_PRIVATE */
283
284/* Initialize Interrupts */
285void ml_install_interrupt_handler(
286 void *nub,
287 int source,
288 void *target,
289 IOInterruptHandler handler,
290 void *refCon);
291
292vm_offset_t
293ml_static_vtop(
294 vm_offset_t);
295
296vm_offset_t
297ml_static_ptovirt(
298 vm_offset_t);
299
300vm_offset_t ml_static_slide(
301 vm_offset_t vaddr);
302
303vm_offset_t ml_static_unslide(
304 vm_offset_t vaddr);
305
306/* Offset required to obtain absolute time value from tick counter */
307uint64_t ml_get_abstime_offset(void);
308
309/* Offset required to obtain continuous time value from tick counter */
310uint64_t ml_get_conttime_offset(void);
311
312#ifdef __APPLE_API_UNSTABLE
313/* PCI config cycle probing */
314boolean_t ml_probe_read(
315 vm_offset_t paddr,
316 unsigned int *val);
317boolean_t ml_probe_read_64(
318 addr64_t paddr,
319 unsigned int *val);
320
321/* Read physical address byte */
322unsigned int ml_phys_read_byte(
323 vm_offset_t paddr);
324unsigned int ml_phys_read_byte_64(
325 addr64_t paddr);
326
327/* Read physical address half word */
328unsigned int ml_phys_read_half(
329 vm_offset_t paddr);
330unsigned int ml_phys_read_half_64(
331 addr64_t paddr);
332
333/* Read physical address word*/
334unsigned int ml_phys_read(
335 vm_offset_t paddr);
336unsigned int ml_phys_read_64(
337 addr64_t paddr);
338unsigned int ml_phys_read_word(
339 vm_offset_t paddr);
340unsigned int ml_phys_read_word_64(
341 addr64_t paddr);
342
343unsigned long long ml_io_read(uintptr_t iovaddr, int iovsz);
344unsigned int ml_io_read8(uintptr_t iovaddr);
345unsigned int ml_io_read16(uintptr_t iovaddr);
346unsigned int ml_io_read32(uintptr_t iovaddr);
347unsigned long long ml_io_read64(uintptr_t iovaddr);
348
349/* Read physical address double word */
350unsigned long long ml_phys_read_double(
351 vm_offset_t paddr);
352unsigned long long ml_phys_read_double_64(
353 addr64_t paddr);
354
355/* Write physical address byte */
356void ml_phys_write_byte(
357 vm_offset_t paddr, unsigned int data);
358void ml_phys_write_byte_64(
359 addr64_t paddr, unsigned int data);
360
361/* Write physical address half word */
362void ml_phys_write_half(
363 vm_offset_t paddr, unsigned int data);
364void ml_phys_write_half_64(
365 addr64_t paddr, unsigned int data);
366
367/* Write physical address word */
368void ml_phys_write(
369 vm_offset_t paddr, unsigned int data);
370void ml_phys_write_64(
371 addr64_t paddr, unsigned int data);
372void ml_phys_write_word(
373 vm_offset_t paddr, unsigned int data);
374void ml_phys_write_word_64(
375 addr64_t paddr, unsigned int data);
376
377/* Write physical address double word */
378void ml_phys_write_double(
379 vm_offset_t paddr, unsigned long long data);
380void ml_phys_write_double_64(
381 addr64_t paddr, unsigned long long data);
382
383void ml_static_mfree(
384 vm_offset_t,
385 vm_size_t);
386
387kern_return_t
388ml_static_protect(
389 vm_offset_t start,
390 vm_size_t size,
391 vm_prot_t new_prot);
392
393/* virtual to physical on wired pages */
394vm_offset_t ml_vtophys(
395 vm_offset_t vaddr);
396
397/* Get processor info */
398void ml_cpu_get_info(ml_cpu_info_t *ml_cpu_info);
399
400#endif /* __APPLE_API_UNSTABLE */
401
402#ifdef __APPLE_API_PRIVATE
403#ifdef XNU_KERNEL_PRIVATE
404vm_size_t ml_nofault_copy(
405 vm_offset_t virtsrc,
406 vm_offset_t virtdst,
407 vm_size_t size);
408boolean_t ml_validate_nofault(
409 vm_offset_t virtsrc, vm_size_t size);
410#endif /* XNU_KERNEL_PRIVATE */
411#if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE)
412/* IO memory map services */
413
414/* Map memory map IO space */
415vm_offset_t ml_io_map(
416 vm_offset_t phys_addr,
417 vm_size_t size);
418
419vm_offset_t ml_io_map_wcomb(
420 vm_offset_t phys_addr,
421 vm_size_t size);
422
423void ml_get_bouncepool_info(
424 vm_offset_t *phys_addr,
425 vm_size_t *size);
426
427vm_map_address_t ml_map_high_window(
428 vm_offset_t phys_addr,
429 vm_size_t len);
430
431/* boot memory allocation */
432vm_offset_t ml_static_malloc(
433 vm_size_t size);
434
435void ml_init_timebase(
436 void *args,
437 tbd_ops_t tbd_funcs,
438 vm_offset_t int_address,
439 vm_offset_t int_value);
440
441uint64_t ml_get_timebase(void);
442
443void ml_init_lock_timeout(void);
444
445boolean_t ml_delay_should_spin(uint64_t interval);
446
447uint32_t ml_get_decrementer(void);
448
449#if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME
450void timer_state_event_user_to_kernel(void);
451void timer_state_event_kernel_to_user(void);
452#endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */
453
454uint64_t ml_get_hwclock(void);
455
456#ifdef __arm64__
457boolean_t ml_get_timer_pending(void);
458#endif
459
460void platform_syscall(
461 struct arm_saved_state *);
462
463void ml_set_decrementer(
464 uint32_t dec_value);
465
466boolean_t is_user_contex(
467 void);
468
469void ml_init_arm_debug_interface(void *args, vm_offset_t virt_address);
470
471/* These calls are only valid if __ARM_USER_PROTECT__ is defined */
472uintptr_t arm_user_protect_begin(
473 thread_t thread);
474
475void arm_user_protect_end(
476 thread_t thread,
477 uintptr_t up,
478 boolean_t disable_interrupts);
479
480#endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */
481
482/* Zero bytes starting at a physical address */
483void bzero_phys(
484 addr64_t phys_address,
485 vm_size_t length);
486
487void bzero_phys_nc(addr64_t src64, vm_size_t bytes);
488
489void ml_thread_policy(
490 thread_t thread,
491 unsigned policy_id,
492 unsigned policy_info);
493
494#define MACHINE_GROUP 0x00000001
495#define MACHINE_NETWORK_GROUP 0x10000000
496#define MACHINE_NETWORK_WORKLOOP 0x00000001
497#define MACHINE_NETWORK_NETISR 0x00000002
498
499/* Initialize the maximum number of CPUs */
500void ml_init_max_cpus(
501 unsigned int max_cpus);
502
503/* Return the maximum number of CPUs set by ml_init_max_cpus() */
504unsigned int ml_get_max_cpus(
505 void);
506
507/* Return the maximum memory size */
508unsigned int ml_get_machine_mem(void);
509
510#ifdef XNU_KERNEL_PRIVATE
511/* Return max offset */
512vm_map_offset_t ml_get_max_offset(
513 boolean_t is64,
514 unsigned int option);
515#define MACHINE_MAX_OFFSET_DEFAULT 0x01
516#define MACHINE_MAX_OFFSET_MIN 0x02
517#define MACHINE_MAX_OFFSET_MAX 0x04
518#define MACHINE_MAX_OFFSET_DEVICE 0x08
519#endif
520
521extern void ml_cpu_up(void);
522extern void ml_cpu_down(void);
523extern void ml_arm_sleep(void);
524
525extern uint64_t ml_get_wake_timebase(void);
526extern uint64_t ml_get_conttime_wake_time(void);
527
528/* Time since the system was reset (as part of boot/wake) */
529uint64_t ml_get_time_since_reset(void);
530
531#ifdef XNU_KERNEL_PRIVATE
532/* Just a stub on ARM */
533extern kern_return_t ml_interrupt_prewarm(uint64_t deadline);
534#define TCOAL_DEBUG(x, a, b, c, d, e) do { } while(0)
535#endif /* XNU_KERNEL_PRIVATE */
536
537/* Bytes available on current stack */
538vm_offset_t ml_stack_remaining(void);
539
540#ifdef MACH_KERNEL_PRIVATE
541uint32_t get_fpscr(void);
542void set_fpscr(uint32_t);
543
544#ifdef __arm64__
545unsigned long update_mdscr(unsigned long clear, unsigned long set);
546#endif /* __arm64__ */
547
548extern void init_vfp(void);
549extern boolean_t get_vfp_enabled(void);
550extern void arm_debug_set_cp14(arm_debug_state_t *debug_state);
551extern void fiq_context_init(boolean_t enable_fiq);
552extern void fiq_context_bootstrap(boolean_t enable_fiq);
553
554extern void reenable_async_aborts(void);
555extern void cpu_idle_wfi(boolean_t wfi_fast);
556
557#ifdef MONITOR
558#define MONITOR_SET_ENTRY 0x800 /* Set kernel entry point from monitor */
559#define MONITOR_LOCKDOWN 0x801 /* Enforce kernel text/rodata integrity */
560unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1,
561 uintptr_t arg2, uintptr_t arg3);
562#endif /* MONITOR */
563
564#if defined(KERNEL_INTEGRITY_KTRR)
565void rorgn_stash_range(void);
566void rorgn_lockdown(void);
567#endif /* defined(KERNEL_INTEGRITY_KTRR)*/
568
569#if __ARM_KERNEL_PROTECT__
570extern void set_vbar_el1(uint64_t);
571#endif /* __ARM_KERNEL_PROTECT__ */
572#endif /* MACH_KERNEL_PRIVATE */
573
574extern uint32_t arm_debug_read_dscr(void);
575
576extern int set_be_bit(void);
577extern int clr_be_bit(void);
578extern int be_tracing(void);
579
580typedef void (*broadcastFunc) (void *);
581unsigned int cpu_broadcast_xcall(uint32_t *, boolean_t, broadcastFunc, void *);
582kern_return_t cpu_xcall(int, broadcastFunc, void *);
583
584#ifdef KERNEL_PRIVATE
585
586/* Interface to be used by the perf. controller to register a callback, in a
587 * single-threaded fashion. The callback will receive notifications of
588 * processor performance quality-of-service changes from the scheduler.
589 */
590
591#ifdef __arm64__
592typedef void (*cpu_qos_update_t)(int throughput_qos, uint64_t qos_param1, uint64_t qos_param2);
593void cpu_qos_update_register(cpu_qos_update_t);
594#endif /* __arm64__ */
595
596struct going_on_core {
597 uint64_t thread_id;
598 uint16_t qos_class;
599 uint16_t urgency; /* XCPM compatibility */
600 uint32_t is_32_bit : 1; /* uses 32-bit ISA/register state in userspace (which may differ from address space size) */
601 uint32_t is_kernel_thread : 1;
602 uint64_t thread_group_id;
603 void *thread_group_data;
604 uint64_t scheduling_latency; /* absolute time between when thread was made runnable and this ctx switch */
605 uint64_t start_time;
606 uint64_t scheduling_latency_at_same_basepri;
607 uint32_t energy_estimate_nj; /* return: In nanojoules */
608 /* smaller of the time between last change to base priority and ctx switch and scheduling_latency */
609};
610typedef struct going_on_core *going_on_core_t;
611
612struct going_off_core {
613 uint64_t thread_id;
614 uint32_t energy_estimate_nj; /* return: In nanojoules */
615 uint32_t reserved;
616 uint64_t end_time;
617 uint64_t thread_group_id;
618 void *thread_group_data;
619};
620typedef struct going_off_core *going_off_core_t;
621
622struct thread_group_data {
623 uint64_t thread_group_id;
624 void *thread_group_data;
625 uint32_t thread_group_size;
626 uint32_t thread_group_flags;
627};
628typedef struct thread_group_data *thread_group_data_t;
629
630struct perfcontrol_max_runnable_latency {
631 uint64_t max_scheduling_latencies[4 /* THREAD_URGENCY_MAX */];
632};
633typedef struct perfcontrol_max_runnable_latency *perfcontrol_max_runnable_latency_t;
634
635struct perfcontrol_work_interval {
636 uint64_t thread_id;
637 uint16_t qos_class;
638 uint16_t urgency;
639 uint32_t flags; // notify
640 uint64_t work_interval_id;
641 uint64_t start;
642 uint64_t finish;
643 uint64_t deadline;
644 uint64_t next_start;
645 uint64_t thread_group_id;
646 void *thread_group_data;
647 uint32_t create_flags;
648};
649typedef struct perfcontrol_work_interval *perfcontrol_work_interval_t;
650
651typedef enum {
652 WORK_INTERVAL_START,
653 WORK_INTERVAL_UPDATE,
654 WORK_INTERVAL_FINISH
655} work_interval_ctl_t;
656
657struct perfcontrol_work_interval_instance {
658 work_interval_ctl_t ctl;
659 uint32_t create_flags;
660 uint64_t complexity;
661 uint64_t thread_id;
662 uint64_t work_interval_id;
663 uint64_t instance_id; /* out: start, in: update/finish */
664 uint64_t start;
665 uint64_t finish;
666 uint64_t deadline;
667 uint64_t thread_group_id;
668 void *thread_group_data;
669};
670typedef struct perfcontrol_work_interval_instance *perfcontrol_work_interval_instance_t;
671
672/*
673 * Structure to export per-CPU counters as part of the CLPC callout.
674 * Contains only the fixed CPU counters (instructions and cycles); CLPC
675 * would call back into XNU to get the configurable counters if needed.
676 */
677struct perfcontrol_cpu_counters {
678 uint64_t instructions;
679 uint64_t cycles;
680};
681
682/*
683 * Structure used to pass information about a thread to CLPC
684 */
685struct perfcontrol_thread_data {
686 /*
687 * Energy estimate (return value)
688 * The field is populated by CLPC and used to update the
689 * energy estimate of the thread
690 */
691 uint32_t energy_estimate_nj;
692 /* Perfcontrol class for thread */
693 perfcontrol_class_t perfctl_class;
694 /* Thread ID for the thread */
695 uint64_t thread_id;
696 /* Thread Group ID */
697 uint64_t thread_group_id;
698 /*
699 * Scheduling latency for threads at the same base priority.
700 * Calculated by the scheduler and passed into CLPC. The field is
701 * populated only in the thread_data structure for the thread
702 * going on-core.
703 */
704 uint64_t scheduling_latency_at_same_basepri;
705 /* Thread Group data pointer */
706 void *thread_group_data;
707 /* perfctl state pointer */
708 void *perfctl_state;
709};
710
711/*
712 * All callouts from the scheduler are executed with interrupts
713 * disabled. Callouts should be implemented in C with minimal
714 * abstractions, and only use KPI exported by the mach/libkern
715 * symbolset, restricted to routines like spinlocks and atomic
716 * operations and scheduler routines as noted below. Spinlocks that
717 * are used to synchronize data in the perfcontrol_state_t should only
718 * ever be acquired with interrupts disabled, to avoid deadlocks where
719 * an quantum expiration timer interrupt attempts to perform a callout
720 * that attempts to lock a spinlock that is already held.
721 */
722
723/*
724 * When a processor is switching between two threads (after the
725 * scheduler has chosen a new thread), the low-level platform layer
726 * will call this routine, which should perform required timestamps,
727 * MMIO register reads, or other state switching. No scheduler locks
728 * are held during this callout.
729 *
730 * This function is called with interrupts ENABLED.
731 */
732typedef void (*sched_perfcontrol_context_switch_t)(perfcontrol_state_t, perfcontrol_state_t);
733
734/*
735 * Once the processor has switched to the new thread, the offcore
736 * callout will indicate the old thread that is no longer being
737 * run. The thread's scheduler lock is held, so it will not begin
738 * running on another processor (in the case of preemption where it
739 * remains runnable) until it completes. If the "thread_terminating"
740 * boolean is TRUE, this will be the last callout for this thread_id.
741 */
742typedef void (*sched_perfcontrol_offcore_t)(perfcontrol_state_t, going_off_core_t /* populated by callee */, boolean_t);
743
744/*
745 * After the offcore callout and after the old thread can potentially
746 * start running on another processor, the oncore callout will be
747 * called with the thread's scheduler lock held. The oncore callout is
748 * also called any time one of the parameters in the going_on_core_t
749 * structure changes, like priority/QoS changes, and quantum
750 * expiration, so the callout must not assume callouts are paired with
751 * offcore callouts.
752 */
753typedef void (*sched_perfcontrol_oncore_t)(perfcontrol_state_t, going_on_core_t);
754
755/*
756 * Periodically (on hundreds of ms scale), the scheduler will perform
757 * maintenance and report the maximum latency for runnable (but not currently
758 * running) threads for each urgency class.
759 */
760typedef void (*sched_perfcontrol_max_runnable_latency_t)(perfcontrol_max_runnable_latency_t);
761
762/*
763 * When the kernel receives information about work intervals from userland,
764 * it is passed along using this callback. No locks are held, although the state
765 * object will not go away during the callout.
766 */
767typedef void (*sched_perfcontrol_work_interval_notify_t)(perfcontrol_state_t, perfcontrol_work_interval_t);
768
769/*
770 * Start, update and finish work interval instance with optional complexity estimate.
771 */
772typedef void (*sched_perfcontrol_work_interval_ctl_t)(perfcontrol_state_t, perfcontrol_work_interval_instance_t);
773
774/*
775 * These callbacks are used when thread groups are added, removed or properties
776 * updated.
777 * No blocking allocations (or anything else blocking) are allowed inside these
778 * callbacks. No locks allowed in these callbacks as well since the kernel might
779 * be holding the thread/task locks.
780 */
781typedef void (*sched_perfcontrol_thread_group_init_t)(thread_group_data_t);
782typedef void (*sched_perfcontrol_thread_group_deinit_t)(thread_group_data_t);
783typedef void (*sched_perfcontrol_thread_group_flags_update_t)(thread_group_data_t);
784
785/*
786 * Sometime after the timeout set by sched_perfcontrol_update_callback_deadline has passed,
787 * this function will be called, passing the timeout deadline that was previously armed as an argument.
788 *
789 * This is called inside context-switch/quantum-interrupt context and must follow the safety rules for that context.
790 */
791typedef void (*sched_perfcontrol_deadline_passed_t)(uint64_t deadline);
792
793/*
794 * Context Switch Callout
795 *
796 * Parameters:
797 * event - The perfcontrol_event for this callout
798 * cpu_id - The CPU doing the context switch
799 * timestamp - The timestamp for the context switch
800 * flags - Flags for other relevant information
801 * offcore - perfcontrol_data structure for thread going off-core
802 * oncore - perfcontrol_data structure for thread going on-core
803 * cpu_counters - perfcontrol_cpu_counters for the CPU doing the switch
804 */
805typedef void (*sched_perfcontrol_csw_t)(
806 perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags,
807 struct perfcontrol_thread_data *offcore, struct perfcontrol_thread_data *oncore,
808 struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused);
809
810
811/*
812 * Thread State Update Callout
813 *
814 * Parameters:
815 * event - The perfcontrol_event for this callout
816 * cpu_id - The CPU doing the state update
817 * timestamp - The timestamp for the state update
818 * flags - Flags for other relevant information
819 * thr_data - perfcontrol_data structure for the thread being updated
820 */
821typedef void (*sched_perfcontrol_state_update_t)(
822 perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags,
823 struct perfcontrol_thread_data *thr_data, __unused void *unused);
824
825/*
826 * Callers should always use the CURRENT version so that the kernel can detect both older
827 * and newer structure layouts. New callbacks should always be added at the end of the
828 * structure, and xnu should expect existing source recompiled against newer headers
829 * to pass NULL for unimplemented callbacks. Pass NULL as the as the callbacks parameter
830 * to reset callbacks to their default in-kernel values.
831 */
832
833#define SCHED_PERFCONTROL_CALLBACKS_VERSION_0 (0) /* up-to oncore */
834#define SCHED_PERFCONTROL_CALLBACKS_VERSION_1 (1) /* up-to max_runnable_latency */
835#define SCHED_PERFCONTROL_CALLBACKS_VERSION_2 (2) /* up-to work_interval_notify */
836#define SCHED_PERFCONTROL_CALLBACKS_VERSION_3 (3) /* up-to thread_group_deinit */
837#define SCHED_PERFCONTROL_CALLBACKS_VERSION_4 (4) /* up-to deadline_passed */
838#define SCHED_PERFCONTROL_CALLBACKS_VERSION_5 (5) /* up-to state_update */
839#define SCHED_PERFCONTROL_CALLBACKS_VERSION_6 (6) /* up-to thread_group_flags_update */
840#define SCHED_PERFCONTROL_CALLBACKS_VERSION_7 (7) /* up-to work_interval_ctl */
841#define SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT SCHED_PERFCONTROL_CALLBACKS_VERSION_6
842
843struct sched_perfcontrol_callbacks {
844 unsigned long version; /* Use SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT */
845 sched_perfcontrol_offcore_t offcore;
846 sched_perfcontrol_context_switch_t context_switch;
847 sched_perfcontrol_oncore_t oncore;
848 sched_perfcontrol_max_runnable_latency_t max_runnable_latency;
849 sched_perfcontrol_work_interval_notify_t work_interval_notify;
850 sched_perfcontrol_thread_group_init_t thread_group_init;
851 sched_perfcontrol_thread_group_deinit_t thread_group_deinit;
852 sched_perfcontrol_deadline_passed_t deadline_passed;
853 sched_perfcontrol_csw_t csw;
854 sched_perfcontrol_state_update_t state_update;
855 sched_perfcontrol_thread_group_flags_update_t thread_group_flags_update;
856 sched_perfcontrol_work_interval_ctl_t work_interval_ctl;
857};
858typedef struct sched_perfcontrol_callbacks *sched_perfcontrol_callbacks_t;
859
860extern void sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state);
861
862/*
863 * Update the scheduler with the set of cores that should be used to dispatch new threads.
864 * Non-recommended cores can still be used to field interrupts or run bound threads.
865 * This should be called with interrupts enabled and no scheduler locks held.
866 */
867#define ALL_CORES_RECOMMENDED (~(uint32_t)0)
868
869extern void sched_perfcontrol_update_recommended_cores(uint32_t recommended_cores);
870extern void sched_perfcontrol_thread_group_recommend(void *data, cluster_type_t recommendation);
871extern void sched_override_recommended_cores_for_sleep(void);
872extern void sched_restore_recommended_cores_after_sleep(void);
873
874/*
875 * Update the deadline after which sched_perfcontrol_deadline_passed will be called.
876 * Returns TRUE if it successfully canceled a previously set callback,
877 * and FALSE if it did not (i.e. one wasn't set, or callback already fired / is in flight).
878 * The callback is automatically canceled when it fires, and does not repeat unless rearmed.
879 *
880 * This 'timer' executes as the scheduler switches between threads, on a non-idle core
881 *
882 * There can be only one outstanding timer globally.
883 */
884extern boolean_t sched_perfcontrol_update_callback_deadline(uint64_t deadline);
885
886typedef enum perfcontrol_callout_type {
887 PERFCONTROL_CALLOUT_ON_CORE,
888 PERFCONTROL_CALLOUT_OFF_CORE,
889 PERFCONTROL_CALLOUT_CONTEXT,
890 PERFCONTROL_CALLOUT_STATE_UPDATE,
891 /* Add other callout types here */
892 PERFCONTROL_CALLOUT_MAX
893} perfcontrol_callout_type_t;
894
895typedef enum perfcontrol_callout_stat {
896 PERFCONTROL_STAT_INSTRS,
897 PERFCONTROL_STAT_CYCLES,
898 /* Add other stat types here */
899 PERFCONTROL_STAT_MAX
900} perfcontrol_callout_stat_t;
901
902uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type,
903 perfcontrol_callout_stat_t stat);
904
905
906#endif /* KERNEL_PRIVATE */
907
908boolean_t machine_timeout_suspended(void);
909void ml_get_power_state(boolean_t *, boolean_t *);
910
911boolean_t user_cont_hwclock_allowed(void);
912boolean_t user_timebase_allowed(void);
913boolean_t ml_thread_is64bit(thread_t thread);
914
915#ifdef __arm64__
916void ml_set_align_checking(void);
917boolean_t arm64_wfe_allowed(void);
918#endif /* __arm64__ */
919
920void ml_timer_evaluate(void);
921boolean_t ml_timer_forced_evaluation(void);
922uint64_t ml_energy_stat(thread_t);
923void ml_gpu_stat_update(uint64_t);
924uint64_t ml_gpu_stat(thread_t);
925#endif /* __APPLE_API_PRIVATE */
926
927__END_DECLS
928
929#endif /* _ARM_MACHINE_ROUTINES_H_ */
930