1 | /* |
2 | * Copyright (c) 2007-2015 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | |
32 | #ifndef _ARM_MACHINE_ROUTINES_H_ |
33 | #define _ARM_MACHINE_ROUTINES_H_ |
34 | |
35 | #include <mach/mach_types.h> |
36 | #include <mach/boolean.h> |
37 | #include <kern/kern_types.h> |
38 | #include <pexpert/pexpert.h> |
39 | |
40 | #include <sys/cdefs.h> |
41 | #include <sys/appleapiopts.h> |
42 | |
43 | #include <stdarg.h> |
44 | |
45 | __BEGIN_DECLS |
46 | |
47 | /* Interrupt handling */ |
48 | |
49 | void ml_cpu_signal(unsigned int cpu_id); |
50 | void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs); |
51 | uint64_t ml_cpu_signal_deferred_get_timer(void); |
52 | void ml_cpu_signal_deferred(unsigned int cpu_id); |
53 | void ml_cpu_signal_retract(unsigned int cpu_id); |
54 | |
55 | /* Initialize Interrupts */ |
56 | void ml_init_interrupt(void); |
57 | |
58 | /* Get Interrupts Enabled */ |
59 | boolean_t ml_get_interrupts_enabled(void); |
60 | |
61 | /* Set Interrupts Enabled */ |
62 | boolean_t ml_set_interrupts_enabled(boolean_t enable); |
63 | |
64 | /* Check if running at interrupt context */ |
65 | boolean_t ml_at_interrupt_context(void); |
66 | |
67 | /* Generate a fake interrupt */ |
68 | void ml_cause_interrupt(void); |
69 | |
70 | /* Clear interrupt spin debug state for thread */ |
71 | #if INTERRUPT_MASKED_DEBUG |
72 | void ml_spin_debug_reset(thread_t thread); |
73 | void ml_spin_debug_clear(thread_t thread); |
74 | void ml_spin_debug_clear_self(void); |
75 | void ml_check_interrupts_disabled_duration(thread_t thread); |
76 | #endif |
77 | |
78 | #ifdef XNU_KERNEL_PRIVATE |
79 | extern boolean_t ml_is_quiescing(void); |
80 | extern void ml_set_is_quiescing(boolean_t); |
81 | extern uint64_t ml_get_booter_memory_size(void); |
82 | #endif |
83 | |
84 | /* Type for the Time Base Enable function */ |
85 | typedef void (*time_base_enable_t)(cpu_id_t cpu_id, boolean_t enable); |
86 | #if MACH_KERNEL_PRIVATE |
87 | /* Type for the Processor Cache Dispatch function */ |
88 | typedef void (*cache_dispatch_t)(cpu_id_t cpu_id, unsigned int select, unsigned int param0, unsigned int param1); |
89 | #endif |
90 | |
91 | #define CacheConfig 0x00000000UL |
92 | #define CacheControl 0x00000001UL |
93 | #define CacheClean 0x00000002UL |
94 | #define CacheCleanRegion 0x00000003UL |
95 | #define CacheCleanFlush 0x00000004UL |
96 | #define CacheCleanFlushRegion 0x00000005UL |
97 | #define CacheShutdown 0x00000006UL |
98 | |
99 | #define CacheControlEnable 0x00000000UL |
100 | |
101 | #define CacheConfigCCSIDR 0x00000001UL |
102 | #define CacheConfigSize 0x00000100UL |
103 | |
104 | /* Type for the Processor Idle function */ |
105 | typedef void (*processor_idle_t)(cpu_id_t cpu_id, boolean_t enter, uint64_t *new_timeout_ticks); |
106 | |
107 | /* Type for the Idle Tickle function */ |
108 | typedef void (*idle_tickle_t)(void); |
109 | |
110 | /* Type for the Idle Timer function */ |
111 | typedef void (*idle_timer_t)(void *refcon, uint64_t *new_timeout_ticks); |
112 | |
113 | /* Type for the IPI Hander */ |
114 | typedef void (*ipi_handler_t)(void); |
115 | |
116 | /* Type for the Lockdown Hander */ |
117 | typedef void (*lockdown_handler_t)(void *); |
118 | |
119 | /* Type for the Platform specific Error Handler */ |
120 | typedef void (*platform_error_handler_t)(void *refcon, vm_offset_t fault_addr); |
121 | |
122 | /* |
123 | * The exception callback (ex_cb) module allows kernel drivers to |
124 | * register and receive callbacks for exceptions, and indicate |
125 | * actions to be taken by the platform kernel |
126 | * Currently this is supported for ARM64 but extending support for ARM32 |
127 | * should be straightforward |
128 | */ |
129 | |
130 | /* Supported exception classes for callbacks */ |
131 | typedef enum |
132 | { |
133 | EXCB_CLASS_ILLEGAL_INSTR_SET, |
134 | #ifdef CONFIG_XNUPOST |
135 | EXCB_CLASS_TEST1, |
136 | EXCB_CLASS_TEST2, |
137 | EXCB_CLASS_TEST3, |
138 | #endif |
139 | EXCB_CLASS_MAX // this must be last |
140 | } |
141 | ex_cb_class_t; |
142 | |
143 | /* Actions indicated by callbacks to be taken by platform kernel */ |
144 | typedef enum |
145 | { |
146 | EXCB_ACTION_RERUN, // re-run the faulting instruction |
147 | EXCB_ACTION_NONE, // continue normal exception handling |
148 | #ifdef CONFIG_XNUPOST |
149 | EXCB_ACTION_TEST_FAIL, |
150 | #endif |
151 | } |
152 | ex_cb_action_t; |
153 | |
154 | /* |
155 | * Exception state |
156 | * We cannot use a private kernel data structure such as arm_saved_state_t |
157 | * The CPSR and ESR are not clobbered when the callback function is invoked so |
158 | * those registers can be examined by the callback function; |
159 | * the same is done in the platform error handlers |
160 | */ |
161 | typedef struct |
162 | { |
163 | vm_offset_t far; |
164 | } |
165 | ex_cb_state_t; |
166 | |
167 | /* callback type definition */ |
168 | typedef ex_cb_action_t (*ex_cb_t) ( |
169 | ex_cb_class_t cb_class, |
170 | void *refcon,// provided at registration |
171 | const ex_cb_state_t *state // exception state |
172 | ); |
173 | |
174 | /* |
175 | * Callback registration |
176 | * Currently we support only one registered callback per class but |
177 | * it should be possible to support more callbacks |
178 | */ |
179 | kern_return_t ex_cb_register( |
180 | ex_cb_class_t cb_class, |
181 | ex_cb_t cb, |
182 | void *refcon ); |
183 | |
184 | /* |
185 | * Called internally by platform kernel to invoke the registered callback for class |
186 | */ |
187 | ex_cb_action_t ex_cb_invoke( |
188 | ex_cb_class_t cb_class, |
189 | vm_offset_t far); |
190 | |
191 | |
192 | void ml_parse_cpu_topology(void); |
193 | |
194 | unsigned int ml_get_cpu_count(void); |
195 | |
196 | int ml_get_boot_cpu_number(void); |
197 | |
198 | int ml_get_cpu_number(uint32_t phys_id); |
199 | |
200 | int ml_get_max_cpu_number(void); |
201 | |
202 | /* Struct for ml_cpu_get_info */ |
203 | struct ml_cpu_info { |
204 | unsigned long vector_unit; |
205 | unsigned long cache_line_size; |
206 | unsigned long l1_icache_size; |
207 | unsigned long l1_dcache_size; |
208 | unsigned long l2_settings; |
209 | unsigned long l2_cache_size; |
210 | unsigned long l3_settings; |
211 | unsigned long l3_cache_size; |
212 | }; |
213 | typedef struct ml_cpu_info ml_cpu_info_t; |
214 | |
215 | typedef enum { |
216 | CLUSTER_TYPE_SMP, |
217 | } cluster_type_t; |
218 | |
219 | cluster_type_t ml_get_boot_cluster(void); |
220 | |
221 | /* Struct for ml_processor_register */ |
222 | struct ml_processor_info { |
223 | cpu_id_t cpu_id; |
224 | vm_offset_t start_paddr; |
225 | boolean_t supports_nap; |
226 | void *platform_cache_dispatch; |
227 | time_base_enable_t time_base_enable; |
228 | processor_idle_t processor_idle; |
229 | idle_tickle_t *idle_tickle; |
230 | idle_timer_t idle_timer; |
231 | void *idle_timer_refcon; |
232 | vm_offset_t powergate_stub_addr; |
233 | uint32_t powergate_stub_length; |
234 | uint32_t powergate_latency; |
235 | platform_error_handler_t platform_error_handler; |
236 | uint64_t regmap_paddr; |
237 | uint32_t phys_id; |
238 | uint32_t log_id; |
239 | uint32_t l2_access_penalty; |
240 | uint32_t cluster_id; |
241 | cluster_type_t cluster_type; |
242 | uint32_t l2_cache_id; |
243 | uint32_t l2_cache_size; |
244 | uint32_t l3_cache_id; |
245 | uint32_t l3_cache_size; |
246 | }; |
247 | typedef struct ml_processor_info ml_processor_info_t; |
248 | |
249 | #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) |
250 | /* Struct for ml_init_timebase */ |
251 | struct tbd_ops { |
252 | void (*tbd_fiq_handler)(void); |
253 | uint32_t (*tbd_get_decrementer)(void); |
254 | void (*tbd_set_decrementer)(uint32_t dec_value); |
255 | }; |
256 | typedef struct tbd_ops *tbd_ops_t; |
257 | typedef struct tbd_ops tbd_ops_data_t; |
258 | #endif |
259 | |
260 | /* Register a processor */ |
261 | kern_return_t ml_processor_register( |
262 | ml_processor_info_t *ml_processor_info, |
263 | processor_t *processor, |
264 | ipi_handler_t *ipi_handler); |
265 | |
266 | /* Register a lockdown handler */ |
267 | kern_return_t ml_lockdown_handler_register(lockdown_handler_t, void *); |
268 | |
269 | #if XNU_KERNEL_PRIVATE |
270 | void ml_lockdown_init(void); |
271 | |
272 | /* Check if the machine layer wants to intercept a panic call */ |
273 | boolean_t ml_wants_panic_trap_to_debugger(void); |
274 | |
275 | /* Machine layer routine for intercepting panics */ |
276 | void ml_panic_trap_to_debugger(const char *panic_format_str, |
277 | va_list *panic_args, |
278 | unsigned int reason, |
279 | void *ctx, |
280 | uint64_t panic_options_mask, |
281 | unsigned long panic_caller); |
282 | #endif /* XNU_KERNEL_PRIVATE */ |
283 | |
284 | /* Initialize Interrupts */ |
285 | void ml_install_interrupt_handler( |
286 | void *nub, |
287 | int source, |
288 | void *target, |
289 | IOInterruptHandler handler, |
290 | void *refCon); |
291 | |
292 | vm_offset_t |
293 | ml_static_vtop( |
294 | vm_offset_t); |
295 | |
296 | vm_offset_t |
297 | ml_static_ptovirt( |
298 | vm_offset_t); |
299 | |
300 | vm_offset_t ml_static_slide( |
301 | vm_offset_t vaddr); |
302 | |
303 | vm_offset_t ml_static_unslide( |
304 | vm_offset_t vaddr); |
305 | |
306 | /* Offset required to obtain absolute time value from tick counter */ |
307 | uint64_t ml_get_abstime_offset(void); |
308 | |
309 | /* Offset required to obtain continuous time value from tick counter */ |
310 | uint64_t ml_get_conttime_offset(void); |
311 | |
312 | #ifdef __APPLE_API_UNSTABLE |
313 | /* PCI config cycle probing */ |
314 | boolean_t ml_probe_read( |
315 | vm_offset_t paddr, |
316 | unsigned int *val); |
317 | boolean_t ml_probe_read_64( |
318 | addr64_t paddr, |
319 | unsigned int *val); |
320 | |
321 | /* Read physical address byte */ |
322 | unsigned int ml_phys_read_byte( |
323 | vm_offset_t paddr); |
324 | unsigned int ml_phys_read_byte_64( |
325 | addr64_t paddr); |
326 | |
327 | /* Read physical address half word */ |
328 | unsigned int ml_phys_read_half( |
329 | vm_offset_t paddr); |
330 | unsigned int ml_phys_read_half_64( |
331 | addr64_t paddr); |
332 | |
333 | /* Read physical address word*/ |
334 | unsigned int ml_phys_read( |
335 | vm_offset_t paddr); |
336 | unsigned int ml_phys_read_64( |
337 | addr64_t paddr); |
338 | unsigned int ml_phys_read_word( |
339 | vm_offset_t paddr); |
340 | unsigned int ml_phys_read_word_64( |
341 | addr64_t paddr); |
342 | |
343 | unsigned long long ml_io_read(uintptr_t iovaddr, int iovsz); |
344 | unsigned int ml_io_read8(uintptr_t iovaddr); |
345 | unsigned int ml_io_read16(uintptr_t iovaddr); |
346 | unsigned int ml_io_read32(uintptr_t iovaddr); |
347 | unsigned long long ml_io_read64(uintptr_t iovaddr); |
348 | |
349 | /* Read physical address double word */ |
350 | unsigned long long ml_phys_read_double( |
351 | vm_offset_t paddr); |
352 | unsigned long long ml_phys_read_double_64( |
353 | addr64_t paddr); |
354 | |
355 | /* Write physical address byte */ |
356 | void ml_phys_write_byte( |
357 | vm_offset_t paddr, unsigned int data); |
358 | void ml_phys_write_byte_64( |
359 | addr64_t paddr, unsigned int data); |
360 | |
361 | /* Write physical address half word */ |
362 | void ml_phys_write_half( |
363 | vm_offset_t paddr, unsigned int data); |
364 | void ml_phys_write_half_64( |
365 | addr64_t paddr, unsigned int data); |
366 | |
367 | /* Write physical address word */ |
368 | void ml_phys_write( |
369 | vm_offset_t paddr, unsigned int data); |
370 | void ml_phys_write_64( |
371 | addr64_t paddr, unsigned int data); |
372 | void ml_phys_write_word( |
373 | vm_offset_t paddr, unsigned int data); |
374 | void ml_phys_write_word_64( |
375 | addr64_t paddr, unsigned int data); |
376 | |
377 | /* Write physical address double word */ |
378 | void ml_phys_write_double( |
379 | vm_offset_t paddr, unsigned long long data); |
380 | void ml_phys_write_double_64( |
381 | addr64_t paddr, unsigned long long data); |
382 | |
383 | void ml_static_mfree( |
384 | vm_offset_t, |
385 | vm_size_t); |
386 | |
387 | kern_return_t |
388 | ml_static_protect( |
389 | vm_offset_t start, |
390 | vm_size_t size, |
391 | vm_prot_t new_prot); |
392 | |
393 | /* virtual to physical on wired pages */ |
394 | vm_offset_t ml_vtophys( |
395 | vm_offset_t vaddr); |
396 | |
397 | /* Get processor info */ |
398 | void ml_cpu_get_info(ml_cpu_info_t *ml_cpu_info); |
399 | |
400 | #endif /* __APPLE_API_UNSTABLE */ |
401 | |
402 | #ifdef __APPLE_API_PRIVATE |
403 | #ifdef XNU_KERNEL_PRIVATE |
404 | vm_size_t ml_nofault_copy( |
405 | vm_offset_t virtsrc, |
406 | vm_offset_t virtdst, |
407 | vm_size_t size); |
408 | boolean_t ml_validate_nofault( |
409 | vm_offset_t virtsrc, vm_size_t size); |
410 | #endif /* XNU_KERNEL_PRIVATE */ |
411 | #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) |
412 | /* IO memory map services */ |
413 | |
414 | /* Map memory map IO space */ |
415 | vm_offset_t ml_io_map( |
416 | vm_offset_t phys_addr, |
417 | vm_size_t size); |
418 | |
419 | vm_offset_t ml_io_map_wcomb( |
420 | vm_offset_t phys_addr, |
421 | vm_size_t size); |
422 | |
423 | void ml_get_bouncepool_info( |
424 | vm_offset_t *phys_addr, |
425 | vm_size_t *size); |
426 | |
427 | vm_map_address_t ml_map_high_window( |
428 | vm_offset_t phys_addr, |
429 | vm_size_t len); |
430 | |
431 | /* boot memory allocation */ |
432 | vm_offset_t ml_static_malloc( |
433 | vm_size_t size); |
434 | |
435 | void ml_init_timebase( |
436 | void *args, |
437 | tbd_ops_t tbd_funcs, |
438 | vm_offset_t int_address, |
439 | vm_offset_t int_value); |
440 | |
441 | uint64_t ml_get_timebase(void); |
442 | |
443 | void ml_init_lock_timeout(void); |
444 | |
445 | boolean_t ml_delay_should_spin(uint64_t interval); |
446 | |
447 | uint32_t ml_get_decrementer(void); |
448 | |
449 | #if !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME |
450 | void timer_state_event_user_to_kernel(void); |
451 | void timer_state_event_kernel_to_user(void); |
452 | #endif /* !CONFIG_SKIP_PRECISE_USER_KERNEL_TIME */ |
453 | |
454 | uint64_t ml_get_hwclock(void); |
455 | |
456 | #ifdef __arm64__ |
457 | boolean_t ml_get_timer_pending(void); |
458 | #endif |
459 | |
460 | void platform_syscall( |
461 | struct arm_saved_state *); |
462 | |
463 | void ml_set_decrementer( |
464 | uint32_t dec_value); |
465 | |
466 | boolean_t is_user_contex( |
467 | void); |
468 | |
469 | void ml_init_arm_debug_interface(void *args, vm_offset_t virt_address); |
470 | |
471 | /* These calls are only valid if __ARM_USER_PROTECT__ is defined */ |
472 | uintptr_t arm_user_protect_begin( |
473 | thread_t thread); |
474 | |
475 | void arm_user_protect_end( |
476 | thread_t thread, |
477 | uintptr_t up, |
478 | boolean_t disable_interrupts); |
479 | |
480 | #endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */ |
481 | |
482 | /* Zero bytes starting at a physical address */ |
483 | void bzero_phys( |
484 | addr64_t phys_address, |
485 | vm_size_t length); |
486 | |
487 | void bzero_phys_nc(addr64_t src64, vm_size_t bytes); |
488 | |
489 | void ml_thread_policy( |
490 | thread_t thread, |
491 | unsigned policy_id, |
492 | unsigned policy_info); |
493 | |
494 | #define MACHINE_GROUP 0x00000001 |
495 | #define MACHINE_NETWORK_GROUP 0x10000000 |
496 | #define MACHINE_NETWORK_WORKLOOP 0x00000001 |
497 | #define MACHINE_NETWORK_NETISR 0x00000002 |
498 | |
499 | /* Initialize the maximum number of CPUs */ |
500 | void ml_init_max_cpus( |
501 | unsigned int max_cpus); |
502 | |
503 | /* Return the maximum number of CPUs set by ml_init_max_cpus() */ |
504 | unsigned int ml_get_max_cpus( |
505 | void); |
506 | |
507 | /* Return the maximum memory size */ |
508 | unsigned int ml_get_machine_mem(void); |
509 | |
510 | #ifdef XNU_KERNEL_PRIVATE |
511 | /* Return max offset */ |
512 | vm_map_offset_t ml_get_max_offset( |
513 | boolean_t is64, |
514 | unsigned int option); |
515 | #define MACHINE_MAX_OFFSET_DEFAULT 0x01 |
516 | #define MACHINE_MAX_OFFSET_MIN 0x02 |
517 | #define MACHINE_MAX_OFFSET_MAX 0x04 |
518 | #define MACHINE_MAX_OFFSET_DEVICE 0x08 |
519 | #endif |
520 | |
521 | extern void ml_cpu_up(void); |
522 | extern void ml_cpu_down(void); |
523 | extern void ml_arm_sleep(void); |
524 | |
525 | extern uint64_t ml_get_wake_timebase(void); |
526 | extern uint64_t ml_get_conttime_wake_time(void); |
527 | |
528 | /* Time since the system was reset (as part of boot/wake) */ |
529 | uint64_t ml_get_time_since_reset(void); |
530 | |
531 | #ifdef XNU_KERNEL_PRIVATE |
532 | /* Just a stub on ARM */ |
533 | extern kern_return_t ml_interrupt_prewarm(uint64_t deadline); |
534 | #define TCOAL_DEBUG(x, a, b, c, d, e) do { } while(0) |
535 | #endif /* XNU_KERNEL_PRIVATE */ |
536 | |
537 | /* Bytes available on current stack */ |
538 | vm_offset_t ml_stack_remaining(void); |
539 | |
540 | #ifdef MACH_KERNEL_PRIVATE |
541 | uint32_t get_fpscr(void); |
542 | void set_fpscr(uint32_t); |
543 | |
544 | #ifdef __arm64__ |
545 | unsigned long update_mdscr(unsigned long clear, unsigned long set); |
546 | #endif /* __arm64__ */ |
547 | |
548 | extern void init_vfp(void); |
549 | extern boolean_t get_vfp_enabled(void); |
550 | extern void arm_debug_set_cp14(arm_debug_state_t *debug_state); |
551 | extern void fiq_context_init(boolean_t enable_fiq); |
552 | extern void fiq_context_bootstrap(boolean_t enable_fiq); |
553 | |
554 | extern void reenable_async_aborts(void); |
555 | extern void cpu_idle_wfi(boolean_t wfi_fast); |
556 | |
557 | #ifdef MONITOR |
558 | #define MONITOR_SET_ENTRY 0x800 /* Set kernel entry point from monitor */ |
559 | #define MONITOR_LOCKDOWN 0x801 /* Enforce kernel text/rodata integrity */ |
560 | unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1, |
561 | uintptr_t arg2, uintptr_t arg3); |
562 | #endif /* MONITOR */ |
563 | |
564 | #if defined(KERNEL_INTEGRITY_KTRR) |
565 | void rorgn_stash_range(void); |
566 | void rorgn_lockdown(void); |
567 | #endif /* defined(KERNEL_INTEGRITY_KTRR)*/ |
568 | |
569 | #if __ARM_KERNEL_PROTECT__ |
570 | extern void set_vbar_el1(uint64_t); |
571 | #endif /* __ARM_KERNEL_PROTECT__ */ |
572 | #endif /* MACH_KERNEL_PRIVATE */ |
573 | |
574 | extern uint32_t arm_debug_read_dscr(void); |
575 | |
576 | extern int set_be_bit(void); |
577 | extern int clr_be_bit(void); |
578 | extern int be_tracing(void); |
579 | |
580 | typedef void (*broadcastFunc) (void *); |
581 | unsigned int cpu_broadcast_xcall(uint32_t *, boolean_t, broadcastFunc, void *); |
582 | kern_return_t cpu_xcall(int, broadcastFunc, void *); |
583 | |
584 | #ifdef KERNEL_PRIVATE |
585 | |
586 | /* Interface to be used by the perf. controller to register a callback, in a |
587 | * single-threaded fashion. The callback will receive notifications of |
588 | * processor performance quality-of-service changes from the scheduler. |
589 | */ |
590 | |
591 | #ifdef __arm64__ |
592 | typedef void (*cpu_qos_update_t)(int throughput_qos, uint64_t qos_param1, uint64_t qos_param2); |
593 | void cpu_qos_update_register(cpu_qos_update_t); |
594 | #endif /* __arm64__ */ |
595 | |
596 | struct going_on_core { |
597 | uint64_t thread_id; |
598 | uint16_t qos_class; |
599 | uint16_t urgency; /* XCPM compatibility */ |
600 | uint32_t is_32_bit : 1; /* uses 32-bit ISA/register state in userspace (which may differ from address space size) */ |
601 | uint32_t is_kernel_thread : 1; |
602 | uint64_t thread_group_id; |
603 | void *thread_group_data; |
604 | uint64_t scheduling_latency; /* absolute time between when thread was made runnable and this ctx switch */ |
605 | uint64_t start_time; |
606 | uint64_t scheduling_latency_at_same_basepri; |
607 | uint32_t energy_estimate_nj; /* return: In nanojoules */ |
608 | /* smaller of the time between last change to base priority and ctx switch and scheduling_latency */ |
609 | }; |
610 | typedef struct going_on_core *going_on_core_t; |
611 | |
612 | struct going_off_core { |
613 | uint64_t thread_id; |
614 | uint32_t energy_estimate_nj; /* return: In nanojoules */ |
615 | uint32_t reserved; |
616 | uint64_t end_time; |
617 | uint64_t thread_group_id; |
618 | void *thread_group_data; |
619 | }; |
620 | typedef struct going_off_core *going_off_core_t; |
621 | |
622 | struct thread_group_data { |
623 | uint64_t thread_group_id; |
624 | void *thread_group_data; |
625 | uint32_t thread_group_size; |
626 | uint32_t thread_group_flags; |
627 | }; |
628 | typedef struct thread_group_data *thread_group_data_t; |
629 | |
630 | struct perfcontrol_max_runnable_latency { |
631 | uint64_t max_scheduling_latencies[4 /* THREAD_URGENCY_MAX */]; |
632 | }; |
633 | typedef struct perfcontrol_max_runnable_latency *perfcontrol_max_runnable_latency_t; |
634 | |
635 | struct perfcontrol_work_interval { |
636 | uint64_t thread_id; |
637 | uint16_t qos_class; |
638 | uint16_t urgency; |
639 | uint32_t flags; // notify |
640 | uint64_t work_interval_id; |
641 | uint64_t start; |
642 | uint64_t finish; |
643 | uint64_t deadline; |
644 | uint64_t next_start; |
645 | uint64_t thread_group_id; |
646 | void *thread_group_data; |
647 | uint32_t create_flags; |
648 | }; |
649 | typedef struct perfcontrol_work_interval *perfcontrol_work_interval_t; |
650 | |
651 | typedef enum { |
652 | WORK_INTERVAL_START, |
653 | WORK_INTERVAL_UPDATE, |
654 | WORK_INTERVAL_FINISH |
655 | } work_interval_ctl_t; |
656 | |
657 | struct perfcontrol_work_interval_instance { |
658 | work_interval_ctl_t ctl; |
659 | uint32_t create_flags; |
660 | uint64_t complexity; |
661 | uint64_t thread_id; |
662 | uint64_t work_interval_id; |
663 | uint64_t instance_id; /* out: start, in: update/finish */ |
664 | uint64_t start; |
665 | uint64_t finish; |
666 | uint64_t deadline; |
667 | uint64_t thread_group_id; |
668 | void *thread_group_data; |
669 | }; |
670 | typedef struct perfcontrol_work_interval_instance *perfcontrol_work_interval_instance_t; |
671 | |
672 | /* |
673 | * Structure to export per-CPU counters as part of the CLPC callout. |
674 | * Contains only the fixed CPU counters (instructions and cycles); CLPC |
675 | * would call back into XNU to get the configurable counters if needed. |
676 | */ |
677 | struct perfcontrol_cpu_counters { |
678 | uint64_t instructions; |
679 | uint64_t cycles; |
680 | }; |
681 | |
682 | /* |
683 | * Structure used to pass information about a thread to CLPC |
684 | */ |
685 | struct perfcontrol_thread_data { |
686 | /* |
687 | * Energy estimate (return value) |
688 | * The field is populated by CLPC and used to update the |
689 | * energy estimate of the thread |
690 | */ |
691 | uint32_t energy_estimate_nj; |
692 | /* Perfcontrol class for thread */ |
693 | perfcontrol_class_t perfctl_class; |
694 | /* Thread ID for the thread */ |
695 | uint64_t thread_id; |
696 | /* Thread Group ID */ |
697 | uint64_t thread_group_id; |
698 | /* |
699 | * Scheduling latency for threads at the same base priority. |
700 | * Calculated by the scheduler and passed into CLPC. The field is |
701 | * populated only in the thread_data structure for the thread |
702 | * going on-core. |
703 | */ |
704 | uint64_t scheduling_latency_at_same_basepri; |
705 | /* Thread Group data pointer */ |
706 | void *thread_group_data; |
707 | /* perfctl state pointer */ |
708 | void *perfctl_state; |
709 | }; |
710 | |
711 | /* |
712 | * All callouts from the scheduler are executed with interrupts |
713 | * disabled. Callouts should be implemented in C with minimal |
714 | * abstractions, and only use KPI exported by the mach/libkern |
715 | * symbolset, restricted to routines like spinlocks and atomic |
716 | * operations and scheduler routines as noted below. Spinlocks that |
717 | * are used to synchronize data in the perfcontrol_state_t should only |
718 | * ever be acquired with interrupts disabled, to avoid deadlocks where |
719 | * an quantum expiration timer interrupt attempts to perform a callout |
720 | * that attempts to lock a spinlock that is already held. |
721 | */ |
722 | |
723 | /* |
724 | * When a processor is switching between two threads (after the |
725 | * scheduler has chosen a new thread), the low-level platform layer |
726 | * will call this routine, which should perform required timestamps, |
727 | * MMIO register reads, or other state switching. No scheduler locks |
728 | * are held during this callout. |
729 | * |
730 | * This function is called with interrupts ENABLED. |
731 | */ |
732 | typedef void (*sched_perfcontrol_context_switch_t)(perfcontrol_state_t, perfcontrol_state_t); |
733 | |
734 | /* |
735 | * Once the processor has switched to the new thread, the offcore |
736 | * callout will indicate the old thread that is no longer being |
737 | * run. The thread's scheduler lock is held, so it will not begin |
738 | * running on another processor (in the case of preemption where it |
739 | * remains runnable) until it completes. If the "thread_terminating" |
740 | * boolean is TRUE, this will be the last callout for this thread_id. |
741 | */ |
742 | typedef void (*sched_perfcontrol_offcore_t)(perfcontrol_state_t, going_off_core_t /* populated by callee */, boolean_t); |
743 | |
744 | /* |
745 | * After the offcore callout and after the old thread can potentially |
746 | * start running on another processor, the oncore callout will be |
747 | * called with the thread's scheduler lock held. The oncore callout is |
748 | * also called any time one of the parameters in the going_on_core_t |
749 | * structure changes, like priority/QoS changes, and quantum |
750 | * expiration, so the callout must not assume callouts are paired with |
751 | * offcore callouts. |
752 | */ |
753 | typedef void (*sched_perfcontrol_oncore_t)(perfcontrol_state_t, going_on_core_t); |
754 | |
755 | /* |
756 | * Periodically (on hundreds of ms scale), the scheduler will perform |
757 | * maintenance and report the maximum latency for runnable (but not currently |
758 | * running) threads for each urgency class. |
759 | */ |
760 | typedef void (*sched_perfcontrol_max_runnable_latency_t)(perfcontrol_max_runnable_latency_t); |
761 | |
762 | /* |
763 | * When the kernel receives information about work intervals from userland, |
764 | * it is passed along using this callback. No locks are held, although the state |
765 | * object will not go away during the callout. |
766 | */ |
767 | typedef void (*sched_perfcontrol_work_interval_notify_t)(perfcontrol_state_t, perfcontrol_work_interval_t); |
768 | |
769 | /* |
770 | * Start, update and finish work interval instance with optional complexity estimate. |
771 | */ |
772 | typedef void (*sched_perfcontrol_work_interval_ctl_t)(perfcontrol_state_t, perfcontrol_work_interval_instance_t); |
773 | |
774 | /* |
775 | * These callbacks are used when thread groups are added, removed or properties |
776 | * updated. |
777 | * No blocking allocations (or anything else blocking) are allowed inside these |
778 | * callbacks. No locks allowed in these callbacks as well since the kernel might |
779 | * be holding the thread/task locks. |
780 | */ |
781 | typedef void (*sched_perfcontrol_thread_group_init_t)(thread_group_data_t); |
782 | typedef void (*sched_perfcontrol_thread_group_deinit_t)(thread_group_data_t); |
783 | typedef void (*sched_perfcontrol_thread_group_flags_update_t)(thread_group_data_t); |
784 | |
785 | /* |
786 | * Sometime after the timeout set by sched_perfcontrol_update_callback_deadline has passed, |
787 | * this function will be called, passing the timeout deadline that was previously armed as an argument. |
788 | * |
789 | * This is called inside context-switch/quantum-interrupt context and must follow the safety rules for that context. |
790 | */ |
791 | typedef void (*sched_perfcontrol_deadline_passed_t)(uint64_t deadline); |
792 | |
793 | /* |
794 | * Context Switch Callout |
795 | * |
796 | * Parameters: |
797 | * event - The perfcontrol_event for this callout |
798 | * cpu_id - The CPU doing the context switch |
799 | * timestamp - The timestamp for the context switch |
800 | * flags - Flags for other relevant information |
801 | * offcore - perfcontrol_data structure for thread going off-core |
802 | * oncore - perfcontrol_data structure for thread going on-core |
803 | * cpu_counters - perfcontrol_cpu_counters for the CPU doing the switch |
804 | */ |
805 | typedef void (*sched_perfcontrol_csw_t)( |
806 | perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, |
807 | struct perfcontrol_thread_data *offcore, struct perfcontrol_thread_data *oncore, |
808 | struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused); |
809 | |
810 | |
811 | /* |
812 | * Thread State Update Callout |
813 | * |
814 | * Parameters: |
815 | * event - The perfcontrol_event for this callout |
816 | * cpu_id - The CPU doing the state update |
817 | * timestamp - The timestamp for the state update |
818 | * flags - Flags for other relevant information |
819 | * thr_data - perfcontrol_data structure for the thread being updated |
820 | */ |
821 | typedef void (*sched_perfcontrol_state_update_t)( |
822 | perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, |
823 | struct perfcontrol_thread_data *thr_data, __unused void *unused); |
824 | |
825 | /* |
826 | * Callers should always use the CURRENT version so that the kernel can detect both older |
827 | * and newer structure layouts. New callbacks should always be added at the end of the |
828 | * structure, and xnu should expect existing source recompiled against newer headers |
829 | * to pass NULL for unimplemented callbacks. Pass NULL as the as the callbacks parameter |
830 | * to reset callbacks to their default in-kernel values. |
831 | */ |
832 | |
833 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_0 (0) /* up-to oncore */ |
834 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_1 (1) /* up-to max_runnable_latency */ |
835 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_2 (2) /* up-to work_interval_notify */ |
836 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_3 (3) /* up-to thread_group_deinit */ |
837 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_4 (4) /* up-to deadline_passed */ |
838 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_5 (5) /* up-to state_update */ |
839 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_6 (6) /* up-to thread_group_flags_update */ |
840 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_7 (7) /* up-to work_interval_ctl */ |
841 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT SCHED_PERFCONTROL_CALLBACKS_VERSION_6 |
842 | |
843 | struct sched_perfcontrol_callbacks { |
844 | unsigned long version; /* Use SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT */ |
845 | sched_perfcontrol_offcore_t offcore; |
846 | sched_perfcontrol_context_switch_t context_switch; |
847 | sched_perfcontrol_oncore_t oncore; |
848 | sched_perfcontrol_max_runnable_latency_t max_runnable_latency; |
849 | sched_perfcontrol_work_interval_notify_t work_interval_notify; |
850 | sched_perfcontrol_thread_group_init_t thread_group_init; |
851 | sched_perfcontrol_thread_group_deinit_t thread_group_deinit; |
852 | sched_perfcontrol_deadline_passed_t deadline_passed; |
853 | sched_perfcontrol_csw_t csw; |
854 | sched_perfcontrol_state_update_t state_update; |
855 | sched_perfcontrol_thread_group_flags_update_t thread_group_flags_update; |
856 | sched_perfcontrol_work_interval_ctl_t work_interval_ctl; |
857 | }; |
858 | typedef struct sched_perfcontrol_callbacks *sched_perfcontrol_callbacks_t; |
859 | |
860 | extern void sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state); |
861 | |
862 | /* |
863 | * Update the scheduler with the set of cores that should be used to dispatch new threads. |
864 | * Non-recommended cores can still be used to field interrupts or run bound threads. |
865 | * This should be called with interrupts enabled and no scheduler locks held. |
866 | */ |
867 | #define ALL_CORES_RECOMMENDED (~(uint32_t)0) |
868 | |
869 | extern void sched_perfcontrol_update_recommended_cores(uint32_t recommended_cores); |
870 | extern void sched_perfcontrol_thread_group_recommend(void *data, cluster_type_t recommendation); |
871 | extern void sched_override_recommended_cores_for_sleep(void); |
872 | extern void sched_restore_recommended_cores_after_sleep(void); |
873 | |
874 | /* |
875 | * Update the deadline after which sched_perfcontrol_deadline_passed will be called. |
876 | * Returns TRUE if it successfully canceled a previously set callback, |
877 | * and FALSE if it did not (i.e. one wasn't set, or callback already fired / is in flight). |
878 | * The callback is automatically canceled when it fires, and does not repeat unless rearmed. |
879 | * |
880 | * This 'timer' executes as the scheduler switches between threads, on a non-idle core |
881 | * |
882 | * There can be only one outstanding timer globally. |
883 | */ |
884 | extern boolean_t sched_perfcontrol_update_callback_deadline(uint64_t deadline); |
885 | |
886 | typedef enum perfcontrol_callout_type { |
887 | PERFCONTROL_CALLOUT_ON_CORE, |
888 | PERFCONTROL_CALLOUT_OFF_CORE, |
889 | PERFCONTROL_CALLOUT_CONTEXT, |
890 | PERFCONTROL_CALLOUT_STATE_UPDATE, |
891 | /* Add other callout types here */ |
892 | PERFCONTROL_CALLOUT_MAX |
893 | } perfcontrol_callout_type_t; |
894 | |
895 | typedef enum perfcontrol_callout_stat { |
896 | PERFCONTROL_STAT_INSTRS, |
897 | PERFCONTROL_STAT_CYCLES, |
898 | /* Add other stat types here */ |
899 | PERFCONTROL_STAT_MAX |
900 | } perfcontrol_callout_stat_t; |
901 | |
902 | uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type, |
903 | perfcontrol_callout_stat_t stat); |
904 | |
905 | |
906 | #endif /* KERNEL_PRIVATE */ |
907 | |
908 | boolean_t machine_timeout_suspended(void); |
909 | void ml_get_power_state(boolean_t *, boolean_t *); |
910 | |
911 | boolean_t user_cont_hwclock_allowed(void); |
912 | boolean_t user_timebase_allowed(void); |
913 | boolean_t ml_thread_is64bit(thread_t thread); |
914 | |
915 | #ifdef __arm64__ |
916 | void ml_set_align_checking(void); |
917 | boolean_t arm64_wfe_allowed(void); |
918 | #endif /* __arm64__ */ |
919 | |
920 | void ml_timer_evaluate(void); |
921 | boolean_t ml_timer_forced_evaluation(void); |
922 | uint64_t ml_energy_stat(thread_t); |
923 | void ml_gpu_stat_update(uint64_t); |
924 | uint64_t ml_gpu_stat(thread_t); |
925 | #endif /* __APPLE_API_PRIVATE */ |
926 | |
927 | __END_DECLS |
928 | |
929 | #endif /* _ARM_MACHINE_ROUTINES_H_ */ |
930 | |