1 | /* |
2 | * Copyright (c) 2007-2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | |
32 | #ifndef _ARM_MACHINE_ROUTINES_H_ |
33 | #define _ARM_MACHINE_ROUTINES_H_ |
34 | |
35 | #include <mach/mach_types.h> |
36 | #include <mach/vm_types.h> |
37 | #include <mach/boolean.h> |
38 | #include <kern/kern_types.h> |
39 | #include <pexpert/pexpert.h> |
40 | |
41 | #include <sys/cdefs.h> |
42 | #include <sys/appleapiopts.h> |
43 | |
44 | #include <stdarg.h> |
45 | |
46 | #ifdef XNU_KERNEL_PRIVATE |
47 | #include <kern/sched_hygiene.h> |
48 | #include <kern/startup.h> |
49 | #endif /* XNU_KERNEL_PRIVATE */ |
50 | |
51 | __BEGIN_DECLS |
52 | #ifdef XNU_KERNEL_PRIVATE |
53 | #ifdef __arm64__ |
54 | typedef bool (*expected_fault_handler_t)(arm_saved_state_t *); |
55 | #endif /* __arm64__ */ |
56 | #endif /* XNU_KERNEL_PRIVATE */ |
57 | |
58 | /* Interrupt handling */ |
59 | |
60 | void ml_cpu_signal(unsigned int cpu_id); |
61 | void ml_cpu_signal_deferred_adjust_timer(uint64_t nanosecs); |
62 | uint64_t ml_cpu_signal_deferred_get_timer(void); |
63 | void ml_cpu_signal_deferred(unsigned int cpu_id); |
64 | void ml_cpu_signal_retract(unsigned int cpu_id); |
65 | bool ml_cpu_signal_is_enabled(void); |
66 | |
67 | /* Initialize Interrupts */ |
68 | void ml_init_interrupt(void); |
69 | |
70 | /* Get Interrupts Enabled */ |
71 | boolean_t ml_get_interrupts_enabled(void); |
72 | |
73 | /* Set Interrupts Enabled */ |
74 | #if __has_feature(ptrauth_calls) |
75 | uint64_t ml_pac_safe_interrupts_disable(void); |
76 | void ml_pac_safe_interrupts_restore(uint64_t); |
77 | #endif /* __has_feature(ptrauth_calls) */ |
78 | boolean_t ml_set_interrupts_enabled_with_debug(boolean_t enable, boolean_t debug); |
79 | boolean_t ml_set_interrupts_enabled(boolean_t enable); |
80 | boolean_t ml_early_set_interrupts_enabled(boolean_t enable); |
81 | |
82 | /* |
83 | * Functions for disabling measurements for AppleCLPC only. |
84 | */ |
85 | boolean_t sched_perfcontrol_ml_set_interrupts_without_measurement(boolean_t enable); |
86 | void sched_perfcontrol_abandon_preemption_disable_measurement(void); |
87 | |
88 | /* Check if running at interrupt context */ |
89 | boolean_t ml_at_interrupt_context(void); |
90 | |
91 | |
92 | /* Generate a fake interrupt */ |
93 | void ml_cause_interrupt(void); |
94 | |
95 | |
96 | #ifdef XNU_KERNEL_PRIVATE |
97 | |
98 | /* did this interrupt context interrupt userspace? */ |
99 | bool ml_did_interrupt_userspace(void); |
100 | |
101 | /* Clear interrupt spin debug state for thread */ |
102 | |
103 | #if SCHED_HYGIENE_DEBUG |
104 | void mt_cur_cpu_cycles_instrs_speculative(uint64_t *cycles, uint64_t *instrs); |
105 | |
106 | #if CONFIG_CPU_COUNTERS |
107 | #define INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread) \ |
108 | if (sched_hygiene_debug_pmc) { \ |
109 | mt_cur_cpu_cycles_instrs_speculative(&thread->machine.intmask_cycles, \ |
110 | &thread->machine.intmask_instr); \ |
111 | } |
112 | #else /* CONFIG_CPU_COUNTERS */ |
113 | #define INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread) |
114 | #endif /* !CONFIG_CPU_COUNTERS */ |
115 | |
116 | #define INTERRUPT_MASKED_DEBUG_START(handler_addr, type) \ |
117 | do { \ |
118 | if ((interrupt_masked_debug_mode || sched_preemption_disable_debug_mode) && os_atomic_load(&interrupt_masked_timeout, relaxed) > 0) { \ |
119 | thread_t thread = current_thread(); \ |
120 | thread->machine.int_type = type; \ |
121 | thread->machine.int_handler_addr = (uintptr_t)VM_KERNEL_STRIP_UPTR(handler_addr); \ |
122 | thread->machine.inthandler_timestamp = ml_get_sched_hygiene_timebase(); \ |
123 | INTERRUPT_MASKED_DEBUG_CAPTURE_PMC(thread); \ |
124 | thread->machine.int_vector = (uintptr_t)NULL; \ |
125 | } \ |
126 | } while (0) |
127 | |
128 | #define INTERRUPT_MASKED_DEBUG_END() \ |
129 | do { \ |
130 | if ((interrupt_masked_debug_mode || sched_preemption_disable_debug_mode) && os_atomic_load(&interrupt_masked_timeout, relaxed) > 0) { \ |
131 | thread_t thread = current_thread(); \ |
132 | ml_handle_interrupt_handler_duration(thread); \ |
133 | thread->machine.inthandler_timestamp = 0; \ |
134 | thread->machine.inthandler_abandon = false; \ |
135 | } \ |
136 | } while (0) |
137 | |
138 | void ml_irq_debug_start(uintptr_t handler, uintptr_t vector); |
139 | void ml_irq_debug_end(void); |
140 | void ml_irq_debug_abandon(void); |
141 | |
142 | void ml_spin_debug_reset(thread_t thread); |
143 | void ml_spin_debug_clear(thread_t thread); |
144 | void ml_spin_debug_clear_self(void); |
145 | void ml_handle_interrupts_disabled_duration(thread_t thread); |
146 | void ml_handle_stackshot_interrupt_disabled_duration(thread_t thread); |
147 | void ml_handle_interrupt_handler_duration(thread_t thread); |
148 | |
149 | #else /* SCHED_HYGIENE_DEBUG */ |
150 | |
151 | #define INTERRUPT_MASKED_DEBUG_START(handler_addr, type) |
152 | #define INTERRUPT_MASKED_DEBUG_END() |
153 | |
154 | #endif /* SCHED_HYGIENE_DEBUG */ |
155 | |
156 | extern bool ml_snoop_thread_is_on_core(thread_t thread); |
157 | extern boolean_t ml_is_quiescing(void); |
158 | extern void ml_set_is_quiescing(boolean_t); |
159 | extern uint64_t ml_get_booter_memory_size(void); |
160 | #endif |
161 | |
162 | /* Type for the Time Base Enable function */ |
163 | typedef void (*time_base_enable_t)(cpu_id_t cpu_id, boolean_t enable); |
164 | #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) |
165 | /* Type for the Processor Cache Dispatch function */ |
166 | typedef void (*cache_dispatch_t)(cpu_id_t cpu_id, unsigned int select, unsigned int param0, unsigned int param1); |
167 | |
168 | typedef uint32_t (*get_decrementer_t)(void); |
169 | typedef void (*set_decrementer_t)(uint32_t); |
170 | typedef void (*fiq_handler_t)(void); |
171 | |
172 | #endif |
173 | |
174 | #define CacheConfig 0x00000000UL |
175 | #define CacheControl 0x00000001UL |
176 | #define CacheClean 0x00000002UL |
177 | #define CacheCleanRegion 0x00000003UL |
178 | #define CacheCleanFlush 0x00000004UL |
179 | #define CacheCleanFlushRegion 0x00000005UL |
180 | #define CacheShutdown 0x00000006UL |
181 | |
182 | #define CacheControlEnable 0x00000000UL |
183 | |
184 | #define CacheConfigCCSIDR 0x00000001UL |
185 | #define CacheConfigSize 0x00000100UL |
186 | |
187 | /* Type for the Processor Idle function */ |
188 | typedef void (*processor_idle_t)(cpu_id_t cpu_id, boolean_t enter, uint64_t *new_timeout_ticks); |
189 | |
190 | /* Type for the Idle Tickle function */ |
191 | typedef void (*idle_tickle_t)(void); |
192 | |
193 | /* Type for the Idle Timer function */ |
194 | typedef void (*idle_timer_t)(void *refcon, uint64_t *new_timeout_ticks); |
195 | |
196 | /* Type for the IPI Hander */ |
197 | typedef void (*ipi_handler_t)(void); |
198 | |
199 | /* Type for the Lockdown Hander */ |
200 | typedef void (*lockdown_handler_t)(void *); |
201 | |
202 | /* Type for the Platform specific Error Handler */ |
203 | typedef void (*platform_error_handler_t)(void *refcon, vm_offset_t fault_addr); |
204 | |
205 | /* |
206 | * The exception callback (ex_cb) module is obsolete. Some definitions related |
207 | * to ex_cb were exported through the SDK, and are only left here for historical |
208 | * reasons. |
209 | */ |
210 | |
211 | /* Unused. Left for historical reasons. */ |
212 | typedef enum{ |
213 | EXCB_CLASS_ILLEGAL_INSTR_SET, |
214 | #ifdef CONFIG_XNUPOST |
215 | EXCB_CLASS_TEST1, |
216 | EXCB_CLASS_TEST2, |
217 | EXCB_CLASS_TEST3, |
218 | #endif |
219 | EXCB_CLASS_MAX |
220 | } |
221 | ex_cb_class_t; |
222 | |
223 | /* Unused. Left for historical reasons. */ |
224 | typedef enum{ |
225 | EXCB_ACTION_RERUN, |
226 | EXCB_ACTION_NONE, |
227 | #ifdef CONFIG_XNUPOST |
228 | EXCB_ACTION_TEST_FAIL, |
229 | #endif |
230 | } |
231 | ex_cb_action_t; |
232 | |
233 | /* Unused. Left for historical reasons. */ |
234 | typedef struct{ |
235 | vm_offset_t far; |
236 | } |
237 | ex_cb_state_t; |
238 | |
239 | /* Unused. Left for historical reasons. */ |
240 | typedef ex_cb_action_t (*ex_cb_t) ( |
241 | ex_cb_class_t cb_class, |
242 | void *refcon, |
243 | const ex_cb_state_t *state |
244 | ); |
245 | |
246 | /* |
247 | * This function is unimplemented. Its definition is left for historical |
248 | * reasons. |
249 | */ |
250 | kern_return_t ex_cb_register( |
251 | ex_cb_class_t cb_class, |
252 | ex_cb_t cb, |
253 | void *refcon ); |
254 | |
255 | /* |
256 | * This function is unimplemented. Its definition is left for historical |
257 | * reasons. |
258 | */ |
259 | ex_cb_action_t ex_cb_invoke( |
260 | ex_cb_class_t cb_class, |
261 | vm_offset_t far); |
262 | |
263 | typedef enum { |
264 | CLUSTER_TYPE_SMP, |
265 | CLUSTER_TYPE_E, |
266 | CLUSTER_TYPE_P, |
267 | MAX_CPU_TYPES, |
268 | } cluster_type_t; |
269 | |
270 | void ml_parse_cpu_topology(void); |
271 | |
272 | unsigned int ml_get_cpu_count(void); |
273 | |
274 | unsigned int ml_get_cpu_number_type(cluster_type_t cluster_type, bool logical, bool available); |
275 | |
276 | unsigned int ml_get_cluster_number_type(cluster_type_t cluster_type); |
277 | |
278 | unsigned int ml_cpu_cache_sharing(unsigned int level, cluster_type_t cluster_type, bool include_all_cpu_types); |
279 | |
280 | unsigned int ml_get_cpu_types(void); |
281 | |
282 | int ml_get_boot_cpu_number(void); |
283 | |
284 | int ml_get_cpu_number(uint32_t phys_id); |
285 | |
286 | unsigned int ml_get_cpu_number_local(void); |
287 | |
288 | int ml_get_cluster_number(uint32_t phys_id); |
289 | |
290 | int ml_get_max_cpu_number(void); |
291 | |
292 | int ml_get_max_cluster_number(void); |
293 | |
294 | /* |
295 | * Return the id of a cluster's first cpu. |
296 | */ |
297 | unsigned int ml_get_first_cpu_id(unsigned int cluster_id); |
298 | |
299 | /* |
300 | * Return the die id of a cluster. |
301 | */ |
302 | unsigned int ml_get_die_id(unsigned int cluster_id); |
303 | |
304 | /* |
305 | * Return the index of a cluster in its die. |
306 | */ |
307 | unsigned int ml_get_die_cluster_id(unsigned int cluster_id); |
308 | |
309 | /* |
310 | * Return the highest die id of the system. |
311 | */ |
312 | unsigned int ml_get_max_die_id(void); |
313 | |
314 | #ifdef __arm64__ |
315 | int ml_get_cluster_number_local(void); |
316 | #endif /* __arm64__ */ |
317 | |
318 | /* Struct for ml_cpu_get_info */ |
319 | struct ml_cpu_info { |
320 | unsigned long vector_unit; |
321 | unsigned long cache_line_size; |
322 | unsigned long l1_icache_size; |
323 | unsigned long l1_dcache_size; |
324 | unsigned long l2_settings; |
325 | unsigned long l2_cache_size; |
326 | unsigned long l3_settings; |
327 | unsigned long l3_cache_size; |
328 | }; |
329 | typedef struct ml_cpu_info ml_cpu_info_t; |
330 | |
331 | cluster_type_t ml_get_boot_cluster_type(void); |
332 | |
333 | /*! |
334 | * @typedef ml_topology_cpu_t |
335 | * @brief Describes one CPU core in the topology. |
336 | * |
337 | * @field cpu_id Logical CPU ID: 0, 1, 2, 3, 4, ... |
338 | * Dynamically assigned by XNU so it might not match EDT. No holes. |
339 | * @field phys_id Physical CPU ID (EDT: reg). Same as MPIDR[15:0], i.e. |
340 | * (cluster_id << 8) | core_number_within_cluster |
341 | * @field cluster_id Logical Cluster ID: 0, 1, 2, 3, 4, ... |
342 | * Dynamically assigned by XNU so it might not match EDT. No holes. |
343 | * @field die_id Die ID (EDT: die-id) |
344 | * @field cluster_type The type of CPUs found in this cluster. |
345 | * @field l2_access_penalty Indicates that the scheduler should try to de-prioritize a core because |
346 | * L2 accesses are slower than on the boot processor. |
347 | * @field l2_cache_size Size of the L2 cache, in bytes. 0 if unknown or not present. |
348 | * @field l2_cache_id l2-cache-id property read from EDT. |
349 | * @field l3_cache_size Size of the L3 cache, in bytes. 0 if unknown or not present. |
350 | * @field l3_cache_id l3-cache-id property read from EDT. |
351 | * @field cpu_IMPL_regs IO-mapped virtual address of cpuX_IMPL (implementation-defined) register block. |
352 | * @field cpu_IMPL_pa Physical address of cpuX_IMPL register block. |
353 | * @field cpu_IMPL_len Length of cpuX_IMPL register block. |
354 | * @field cpu_UTTDBG_regs IO-mapped virtual address of cpuX_UTTDBG register block. |
355 | * @field cpu_UTTDBG_pa Physical address of cpuX_UTTDBG register block, if set in DT, else zero |
356 | * @field cpu_UTTDBG_len Length of cpuX_UTTDBG register block, if set in DT, else zero |
357 | * @field coresight_regs IO-mapped virtual address of CoreSight debug register block. |
358 | * @field coresight_pa Physical address of CoreSight register block. |
359 | * @field coresight_len Length of CoreSight register block. |
360 | * @field die_cluster_id Physical cluster ID within the local die (EDT: die-cluster-id) |
361 | * @field cluster_core_id Physical core ID within the local cluster (EDT: cluster-core-id) |
362 | */ |
363 | typedef struct ml_topology_cpu { |
364 | unsigned int cpu_id; |
365 | uint32_t phys_id; |
366 | unsigned int cluster_id; |
367 | unsigned int die_id; |
368 | cluster_type_t cluster_type; |
369 | uint32_t l2_access_penalty; |
370 | uint32_t l2_cache_size; |
371 | uint32_t l2_cache_id; |
372 | uint32_t l3_cache_size; |
373 | uint32_t l3_cache_id; |
374 | vm_offset_t cpu_IMPL_regs; |
375 | uint64_t cpu_IMPL_pa; |
376 | uint64_t cpu_IMPL_len; |
377 | vm_offset_t cpu_UTTDBG_regs; |
378 | uint64_t cpu_UTTDBG_pa; |
379 | uint64_t cpu_UTTDBG_len; |
380 | vm_offset_t coresight_regs; |
381 | uint64_t coresight_pa; |
382 | uint64_t coresight_len; |
383 | unsigned int die_cluster_id; |
384 | unsigned int cluster_core_id; |
385 | } ml_topology_cpu_t; |
386 | |
387 | /*! |
388 | * @typedef ml_topology_cluster_t |
389 | * @brief Describes one cluster in the topology. |
390 | * |
391 | * @field cluster_id Cluster ID (EDT: cluster-id) |
392 | * @field cluster_type The type of CPUs found in this cluster. |
393 | * @field num_cpus Total number of usable CPU cores in this cluster. |
394 | * @field first_cpu_id The cpu_id of the first CPU in the cluster. |
395 | * @field cpu_mask A bitmask representing the cpu_id's that belong to the cluster. Example: |
396 | * If the cluster contains CPU4 and CPU5, cpu_mask will be 0x30. |
397 | * @field acc_IMPL_regs IO-mapped virtual address of acc_IMPL (implementation-defined) register block. |
398 | * @field acc_IMPL_pa Physical address of acc_IMPL register block. |
399 | * @field acc_IMPL_len Length of acc_IMPL register block. |
400 | * @field cpm_IMPL_regs IO-mapped virtual address of cpm_IMPL (implementation-defined) register block. |
401 | * @field cpm_IMPL_pa Physical address of cpm_IMPL register block. |
402 | * @field cpm_IMPL_len Length of cpm_IMPL register block. |
403 | */ |
404 | typedef struct ml_topology_cluster { |
405 | unsigned int cluster_id; |
406 | cluster_type_t cluster_type; |
407 | unsigned int num_cpus; |
408 | unsigned int first_cpu_id; |
409 | uint64_t cpu_mask; |
410 | vm_offset_t acc_IMPL_regs; |
411 | uint64_t acc_IMPL_pa; |
412 | uint64_t acc_IMPL_len; |
413 | vm_offset_t cpm_IMPL_regs; |
414 | uint64_t cpm_IMPL_pa; |
415 | uint64_t cpm_IMPL_len; |
416 | } ml_topology_cluster_t; |
417 | |
418 | // Bump this version number any time any ml_topology_* struct changes, so |
419 | // that KPI users can check whether their headers are compatible with |
420 | // the running kernel. |
421 | #define CPU_TOPOLOGY_VERSION 1 |
422 | |
423 | /*! |
424 | * @typedef ml_topology_info_t |
425 | * @brief Describes the CPU topology for all APs in the system. Populated from EDT and read-only at runtime. |
426 | * @discussion This struct only lists CPU cores that are considered usable by both iBoot and XNU. Some |
427 | * physically present CPU cores may be considered unusable due to configuration options like |
428 | * the "cpus=" boot-arg. Cores that are disabled in hardware will not show up in EDT at all, so |
429 | * they also will not be present in this struct. |
430 | * |
431 | * @field version Version of the struct (set to CPU_TOPOLOGY_VERSION). |
432 | * @field num_cpus Total number of usable CPU cores. |
433 | * @field max_cpu_id The highest usable logical CPU ID. |
434 | * @field num_clusters Total number of AP CPU clusters on the system (usable or not). |
435 | * @field max_cluster_id The highest cluster ID found in EDT. |
436 | * @field cpus List of |num_cpus| entries. |
437 | * @field clusters List of |num_clusters| entries. |
438 | * @field boot_cpu Points to the |cpus| entry for the boot CPU. |
439 | * @field boot_cluster Points to the |clusters| entry which contains the boot CPU. |
440 | * @field chip_revision Silicon revision reported by iBoot, which comes from the |
441 | * SoC-specific fuse bits. See CPU_VERSION_xx macros for definitions. |
442 | */ |
443 | typedef struct ml_topology_info { |
444 | unsigned int version; |
445 | unsigned int num_cpus; |
446 | unsigned int max_cpu_id; |
447 | unsigned int num_clusters; |
448 | unsigned int max_cluster_id; |
449 | unsigned int max_die_id; |
450 | ml_topology_cpu_t *cpus; |
451 | ml_topology_cluster_t *clusters; |
452 | ml_topology_cpu_t *boot_cpu; |
453 | ml_topology_cluster_t *boot_cluster; |
454 | unsigned int chip_revision; |
455 | unsigned int cluster_types; |
456 | unsigned int cluster_type_num_cpus[MAX_CPU_TYPES]; |
457 | unsigned int cluster_type_num_clusters[MAX_CPU_TYPES]; |
458 | } ml_topology_info_t; |
459 | |
460 | /*! |
461 | * @function ml_get_topology_info |
462 | * @result A pointer to the read-only topology struct. Does not need to be freed. Returns NULL |
463 | * if the struct hasn't been initialized or the feature is unsupported. |
464 | */ |
465 | const ml_topology_info_t *ml_get_topology_info(void); |
466 | |
467 | /*! |
468 | * @function ml_map_cpu_pio |
469 | * @brief Maps per-CPU and per-cluster PIO registers found in EDT. This needs to be |
470 | * called after arm_vm_init() so it can't be part of ml_parse_cpu_topology(). |
471 | */ |
472 | void ml_map_cpu_pio(void); |
473 | |
474 | /* Struct for ml_processor_register */ |
475 | struct ml_processor_info { |
476 | cpu_id_t cpu_id; |
477 | vm_offset_t start_paddr; |
478 | boolean_t supports_nap; |
479 | void *platform_cache_dispatch; |
480 | time_base_enable_t time_base_enable; |
481 | processor_idle_t processor_idle; |
482 | idle_tickle_t *idle_tickle; |
483 | idle_timer_t idle_timer; |
484 | void *idle_timer_refcon; |
485 | vm_offset_t powergate_stub_addr; |
486 | uint32_t powergate_stub_length; |
487 | uint32_t powergate_latency; |
488 | platform_error_handler_t platform_error_handler; |
489 | uint64_t regmap_paddr; |
490 | uint32_t phys_id; |
491 | uint32_t log_id; |
492 | uint32_t l2_access_penalty; |
493 | uint32_t cluster_id; |
494 | cluster_type_t cluster_type; |
495 | uint32_t l2_cache_id; |
496 | uint32_t l2_cache_size; |
497 | uint32_t l3_cache_id; |
498 | uint32_t l3_cache_size; |
499 | }; |
500 | typedef struct ml_processor_info ml_processor_info_t; |
501 | |
502 | #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) |
503 | /* Struct for ml_init_timebase */ |
504 | struct tbd_ops { |
505 | fiq_handler_t tbd_fiq_handler; |
506 | get_decrementer_t tbd_get_decrementer; |
507 | set_decrementer_t tbd_set_decrementer; |
508 | }; |
509 | typedef struct tbd_ops *tbd_ops_t; |
510 | typedef struct tbd_ops tbd_ops_data_t; |
511 | #endif |
512 | |
513 | |
514 | /*! |
515 | * @function ml_processor_register |
516 | * |
517 | * @abstract callback from platform kext to register processor |
518 | * |
519 | * @discussion This function is called by the platform kext when a processor is |
520 | * being registered. This is called while running on the CPU itself, as part of |
521 | * its initialization. |
522 | * |
523 | * @param ml_processor_info provides machine-specific information about the |
524 | * processor to xnu. |
525 | * |
526 | * @param processor is set as an out-parameter to an opaque handle that should |
527 | * be used by the platform kext when referring to this processor in the future. |
528 | * |
529 | * @param ipi_handler is set as an out-parameter to the function that should be |
530 | * registered as the IPI handler. |
531 | * |
532 | * @param pmi_handler is set as an out-parameter to the function that should be |
533 | * registered as the PMI handler. |
534 | * |
535 | * @returns KERN_SUCCESS on success and an error code, otherwise. |
536 | */ |
537 | kern_return_t ml_processor_register(ml_processor_info_t *ml_processor_info, |
538 | processor_t *processor, ipi_handler_t *ipi_handler, |
539 | perfmon_interrupt_handler_func *pmi_handler); |
540 | |
541 | /* Register a lockdown handler */ |
542 | kern_return_t ml_lockdown_handler_register(lockdown_handler_t, void *); |
543 | |
544 | /* Register a M$ flushing */ |
545 | typedef kern_return_t (*mcache_flush_function)(void *service); |
546 | kern_return_t ml_mcache_flush_callback_register(mcache_flush_function func, void *service); |
547 | kern_return_t ml_mcache_flush(void); |
548 | |
549 | #if XNU_KERNEL_PRIVATE |
550 | void ml_lockdown_init(void); |
551 | |
552 | /* Machine layer routine for intercepting panics */ |
553 | __printflike(1, 0) |
554 | void ml_panic_trap_to_debugger(const char *panic_format_str, |
555 | va_list *panic_args, |
556 | unsigned int reason, |
557 | void *ctx, |
558 | uint64_t panic_options_mask, |
559 | unsigned long panic_caller); |
560 | #endif /* XNU_KERNEL_PRIVATE */ |
561 | |
562 | /* Initialize Interrupts */ |
563 | void ml_install_interrupt_handler( |
564 | void *nub, |
565 | int source, |
566 | void *target, |
567 | IOInterruptHandler handler, |
568 | void *refCon); |
569 | |
570 | vm_offset_t |
571 | ml_static_vtop( |
572 | vm_offset_t); |
573 | |
574 | kern_return_t |
575 | ml_static_verify_page_protections( |
576 | uint64_t base, uint64_t size, vm_prot_t prot); |
577 | |
578 | vm_offset_t |
579 | ml_static_ptovirt( |
580 | vm_offset_t); |
581 | |
582 | /* Offset required to obtain absolute time value from tick counter */ |
583 | uint64_t ml_get_abstime_offset(void); |
584 | |
585 | /* Offset required to obtain continuous time value from tick counter */ |
586 | uint64_t ml_get_conttime_offset(void); |
587 | |
588 | #ifdef __APPLE_API_UNSTABLE |
589 | /* PCI config cycle probing */ |
590 | boolean_t ml_probe_read( |
591 | vm_offset_t paddr, |
592 | unsigned int *val); |
593 | boolean_t ml_probe_read_64( |
594 | addr64_t paddr, |
595 | unsigned int *val); |
596 | |
597 | /* Read physical address byte */ |
598 | unsigned int ml_phys_read_byte( |
599 | vm_offset_t paddr); |
600 | unsigned int ml_phys_read_byte_64( |
601 | addr64_t paddr); |
602 | |
603 | /* Read physical address half word */ |
604 | unsigned int ml_phys_read_half( |
605 | vm_offset_t paddr); |
606 | unsigned int ml_phys_read_half_64( |
607 | addr64_t paddr); |
608 | |
609 | /* Read physical address word*/ |
610 | unsigned int ml_phys_read( |
611 | vm_offset_t paddr); |
612 | unsigned int ml_phys_read_64( |
613 | addr64_t paddr); |
614 | unsigned int ml_phys_read_word( |
615 | vm_offset_t paddr); |
616 | unsigned int ml_phys_read_word_64( |
617 | addr64_t paddr); |
618 | |
619 | /* Read physical address double word */ |
620 | unsigned long long ml_phys_read_double( |
621 | vm_offset_t paddr); |
622 | unsigned long long ml_phys_read_double_64( |
623 | addr64_t paddr); |
624 | |
625 | /* Write physical address byte */ |
626 | void ml_phys_write_byte( |
627 | vm_offset_t paddr, unsigned int data); |
628 | void ml_phys_write_byte_64( |
629 | addr64_t paddr, unsigned int data); |
630 | |
631 | /* Write physical address half word */ |
632 | void ml_phys_write_half( |
633 | vm_offset_t paddr, unsigned int data); |
634 | void ml_phys_write_half_64( |
635 | addr64_t paddr, unsigned int data); |
636 | |
637 | /* Write physical address word */ |
638 | void ml_phys_write( |
639 | vm_offset_t paddr, unsigned int data); |
640 | void ml_phys_write_64( |
641 | addr64_t paddr, unsigned int data); |
642 | void ml_phys_write_word( |
643 | vm_offset_t paddr, unsigned int data); |
644 | void ml_phys_write_word_64( |
645 | addr64_t paddr, unsigned int data); |
646 | |
647 | /* Write physical address double word */ |
648 | void ml_phys_write_double( |
649 | vm_offset_t paddr, unsigned long long data); |
650 | void ml_phys_write_double_64( |
651 | addr64_t paddr, unsigned long long data); |
652 | |
653 | #if defined(__SIZEOF_INT128__) && APPLE_ARM64_ARCH_FAMILY |
654 | /* |
655 | * Not all dependent projects consuming `machine_routines.h` are built using |
656 | * toolchains that support 128-bit integers. |
657 | */ |
658 | #define BUILD_QUAD_WORD_FUNCS 1 |
659 | #else |
660 | #define BUILD_QUAD_WORD_FUNCS 0 |
661 | #endif /* defined(__SIZEOF_INT128__) && APPLE_ARM64_ARCH_FAMILY */ |
662 | |
663 | #if BUILD_QUAD_WORD_FUNCS |
664 | /* |
665 | * Not all dependent projects have their own typedef of `uint128_t` at the |
666 | * time they consume `machine_routines.h`. |
667 | */ |
668 | typedef unsigned __int128 uint128_t; |
669 | |
670 | /* Read physical address quad word */ |
671 | uint128_t ml_phys_read_quad( |
672 | vm_offset_t paddr); |
673 | uint128_t ml_phys_read_quad_64( |
674 | addr64_t paddr); |
675 | |
676 | /* Write physical address quad word */ |
677 | void ml_phys_write_quad( |
678 | vm_offset_t paddr, uint128_t data); |
679 | void ml_phys_write_quad_64( |
680 | addr64_t paddr, uint128_t data); |
681 | #endif /* BUILD_QUAD_WORD_FUNCS */ |
682 | |
683 | void ml_static_mfree( |
684 | vm_offset_t, |
685 | vm_size_t); |
686 | |
687 | kern_return_t |
688 | ml_static_protect( |
689 | vm_offset_t start, |
690 | vm_size_t size, |
691 | vm_prot_t new_prot); |
692 | |
693 | /* virtual to physical on wired pages */ |
694 | vm_offset_t ml_vtophys( |
695 | vm_offset_t vaddr); |
696 | |
697 | /* Get processor cache info */ |
698 | void ml_cpu_get_info(ml_cpu_info_t *ml_cpu_info); |
699 | void ml_cpu_get_info_type(ml_cpu_info_t * ml_cpu_info, cluster_type_t cluster_type); |
700 | |
701 | #endif /* __APPLE_API_UNSTABLE */ |
702 | |
703 | typedef int ml_page_protection_t; |
704 | |
705 | /* Return the type of page protection supported */ |
706 | ml_page_protection_t ml_page_protection_type(void); |
707 | |
708 | #ifdef __APPLE_API_PRIVATE |
709 | #ifdef XNU_KERNEL_PRIVATE |
710 | vm_size_t ml_nofault_copy( |
711 | vm_offset_t virtsrc, |
712 | vm_offset_t virtdst, |
713 | vm_size_t size); |
714 | boolean_t ml_validate_nofault( |
715 | vm_offset_t virtsrc, vm_size_t size); |
716 | #endif /* XNU_KERNEL_PRIVATE */ |
717 | #if defined(PEXPERT_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE) |
718 | /* IO memory map services */ |
719 | |
720 | extern vm_offset_t io_map( |
721 | vm_map_offset_t phys_addr, |
722 | vm_size_t size, |
723 | unsigned int flags, |
724 | vm_prot_t prot, |
725 | bool unmappable); |
726 | |
727 | /* Map memory map IO space */ |
728 | vm_offset_t ml_io_map( |
729 | vm_offset_t phys_addr, |
730 | vm_size_t size); |
731 | |
732 | vm_offset_t ml_io_map_wcomb( |
733 | vm_offset_t phys_addr, |
734 | vm_size_t size); |
735 | |
736 | vm_offset_t ml_io_map_unmappable( |
737 | vm_offset_t phys_addr, |
738 | vm_size_t size, |
739 | uint32_t flags); |
740 | |
741 | vm_offset_t ml_io_map_with_prot( |
742 | vm_offset_t phys_addr, |
743 | vm_size_t size, |
744 | vm_prot_t prot); |
745 | |
746 | void ml_io_unmap( |
747 | vm_offset_t addr, |
748 | vm_size_t sz); |
749 | |
750 | void ml_get_bouncepool_info( |
751 | vm_offset_t *phys_addr, |
752 | vm_size_t *size); |
753 | |
754 | vm_map_address_t ml_map_high_window( |
755 | vm_offset_t phys_addr, |
756 | vm_size_t len); |
757 | |
758 | void ml_init_timebase( |
759 | void *args, |
760 | tbd_ops_t tbd_funcs, |
761 | vm_offset_t int_address, |
762 | vm_offset_t int_value); |
763 | |
764 | uint64_t ml_get_timebase(void); |
765 | |
766 | #if MACH_KERNEL_PRIVATE |
767 | void ml_memory_to_timebase_fence(void); |
768 | void ml_timebase_to_memory_fence(void); |
769 | #endif /* MACH_KERNEL_PRIVATE */ |
770 | |
771 | uint64_t ml_get_speculative_timebase(void); |
772 | |
773 | uint64_t ml_get_timebase_entropy(void); |
774 | |
775 | boolean_t ml_delay_should_spin(uint64_t interval); |
776 | |
777 | void ml_delay_on_yield(void); |
778 | |
779 | uint32_t ml_get_decrementer(void); |
780 | |
781 | #include <machine/config.h> |
782 | |
783 | uint64_t ml_get_hwclock(void); |
784 | |
785 | #ifdef __arm64__ |
786 | boolean_t ml_get_timer_pending(void); |
787 | #endif |
788 | |
789 | void platform_syscall( |
790 | struct arm_saved_state *); |
791 | |
792 | void ml_set_decrementer( |
793 | uint32_t dec_value); |
794 | |
795 | boolean_t is_user_contex( |
796 | void); |
797 | |
798 | void ml_init_arm_debug_interface(void *args, vm_offset_t virt_address); |
799 | |
800 | /* These calls are only valid if __ARM_USER_PROTECT__ is defined */ |
801 | uintptr_t arm_user_protect_begin( |
802 | thread_t thread); |
803 | |
804 | void arm_user_protect_end( |
805 | thread_t thread, |
806 | uintptr_t up, |
807 | boolean_t disable_interrupts); |
808 | |
809 | #endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */ |
810 | |
811 | /* Zero bytes starting at a physical address */ |
812 | void bzero_phys( |
813 | addr64_t phys_address, |
814 | vm_size_t length); |
815 | |
816 | void bzero_phys_nc(addr64_t src64, vm_size_t bytes); |
817 | |
818 | #if MACH_KERNEL_PRIVATE |
819 | #ifdef __arm64__ |
820 | /* Pattern-fill buffer with zeros or a 32-bit pattern; |
821 | * target must be 128-byte aligned and sized a multiple of 128 |
822 | * Both variants emit stores with non-temporal properties. |
823 | */ |
824 | void fill32_dczva(addr64_t, vm_size_t); |
825 | void fill32_nt(addr64_t, vm_size_t, uint32_t); |
826 | bool cpu_interrupt_is_pending(void); |
827 | |
828 | #endif // __arm64__ |
829 | #endif // MACH_KERNEL_PRIVATE |
830 | |
831 | void ml_thread_policy( |
832 | thread_t thread, |
833 | unsigned policy_id, |
834 | unsigned policy_info); |
835 | |
836 | #define MACHINE_GROUP 0x00000001 |
837 | #define MACHINE_NETWORK_GROUP 0x10000000 |
838 | #define MACHINE_NETWORK_WORKLOOP 0x00000001 |
839 | #define MACHINE_NETWORK_NETISR 0x00000002 |
840 | |
841 | /* Set the maximum number of CPUs */ |
842 | void ml_set_max_cpus( |
843 | unsigned int max_cpus); |
844 | |
845 | /* Return the maximum number of CPUs set by ml_set_max_cpus(), waiting if necessary */ |
846 | unsigned int ml_wait_max_cpus( |
847 | void); |
848 | |
849 | /* Return the maximum memory size */ |
850 | unsigned int ml_get_machine_mem(void); |
851 | |
852 | #ifdef XNU_KERNEL_PRIVATE |
853 | /* Return max offset */ |
854 | vm_map_offset_t ml_get_max_offset( |
855 | boolean_t is64, |
856 | unsigned int option); |
857 | #define MACHINE_MAX_OFFSET_DEFAULT 0x01 |
858 | #define MACHINE_MAX_OFFSET_MIN 0x02 |
859 | #define MACHINE_MAX_OFFSET_MAX 0x04 |
860 | #define MACHINE_MAX_OFFSET_DEVICE 0x08 |
861 | #endif |
862 | |
863 | extern void ml_cpu_init_completed(void); |
864 | extern void ml_cpu_up(void); |
865 | extern void ml_cpu_down(void); |
866 | /* |
867 | * The update to CPU counts needs to be separate from other actions |
868 | * in ml_cpu_up() and ml_cpu_down() |
869 | * because we don't update the counts when CLPC causes temporary |
870 | * cluster powerdown events, as these must be transparent to the user. |
871 | */ |
872 | extern void ml_cpu_up_update_counts(int cpu_id); |
873 | extern void ml_cpu_down_update_counts(int cpu_id); |
874 | extern void ml_arm_sleep(void); |
875 | |
876 | extern uint64_t ml_get_wake_timebase(void); |
877 | extern uint64_t ml_get_conttime_wake_time(void); |
878 | |
879 | /* Time since the system was reset (as part of boot/wake) */ |
880 | uint64_t ml_get_time_since_reset(void); |
881 | |
882 | /* |
883 | * Called by ApplePMGR to set wake time. Units and epoch are identical |
884 | * to mach_continuous_time(). Has no effect on !HAS_CONTINUOUS_HWCLOCK |
885 | * chips. If wake_time == UINT64_MAX, that means the wake time is |
886 | * unknown and calls to ml_get_time_since_reset() will return UINT64_MAX. |
887 | */ |
888 | void ml_set_reset_time(uint64_t wake_time); |
889 | |
890 | #ifdef XNU_KERNEL_PRIVATE |
891 | /* Just a stub on ARM */ |
892 | extern kern_return_t ml_interrupt_prewarm(uint64_t deadline); |
893 | #define TCOAL_DEBUG(x, a, b, c, d, e) do { } while(0) |
894 | #endif /* XNU_KERNEL_PRIVATE */ |
895 | |
896 | /* Bytes available on current stack */ |
897 | vm_offset_t ml_stack_remaining(void); |
898 | |
899 | #ifdef MACH_KERNEL_PRIVATE |
900 | uint32_t get_fpscr(void); |
901 | void set_fpscr(uint32_t); |
902 | void machine_conf(void); |
903 | void machine_lockdown(void); |
904 | |
905 | #ifdef __arm64__ |
906 | unsigned long update_mdscr(unsigned long clear, unsigned long set); |
907 | #endif /* __arm64__ */ |
908 | |
909 | extern void arm_debug_set_cp14(arm_debug_state_t *debug_state); |
910 | extern void fiq_context_init(boolean_t enable_fiq); |
911 | |
912 | extern void reenable_async_aborts(void); |
913 | |
914 | #ifdef __arm64__ |
915 | uint64_t ml_cluster_wfe_timeout(uint32_t wfe_cluster_id); |
916 | #endif |
917 | |
918 | #ifdef MONITOR |
919 | #define MONITOR_SET_ENTRY 0x800 /* Set kernel entry point from monitor */ |
920 | #define MONITOR_LOCKDOWN 0x801 /* Enforce kernel text/rodata integrity */ |
921 | unsigned long monitor_call(uintptr_t callnum, uintptr_t arg1, |
922 | uintptr_t arg2, uintptr_t arg3); |
923 | #endif /* MONITOR */ |
924 | |
925 | #if __ARM_KERNEL_PROTECT__ |
926 | extern void set_vbar_el1(uint64_t); |
927 | #endif /* __ARM_KERNEL_PROTECT__ */ |
928 | #endif /* MACH_KERNEL_PRIVATE */ |
929 | |
930 | extern uint32_t arm_debug_read_dscr(void); |
931 | |
932 | extern int set_be_bit(void); |
933 | extern int clr_be_bit(void); |
934 | extern int be_tracing(void); |
935 | |
936 | /* Please note that cpu_broadcast_xcall is not as simple is you would like it to be. |
937 | * It will sometimes put the calling thread to sleep, and it is up to your callback |
938 | * to wake it up as needed, where "as needed" is defined as "all other CPUs have |
939 | * called the broadcast func". Look around the kernel for examples, or instead use |
940 | * cpu_broadcast_xcall_simple() which does indeed act like you would expect, given |
941 | * the prototype. cpu_broadcast_immediate_xcall has the same caveats and has a similar |
942 | * _simple() wrapper |
943 | */ |
944 | typedef void (*broadcastFunc) (void *); |
945 | unsigned int cpu_broadcast_xcall(uint32_t *, boolean_t, broadcastFunc, void *); |
946 | unsigned int cpu_broadcast_xcall_simple(boolean_t, broadcastFunc, void *); |
947 | __result_use_check kern_return_t cpu_xcall(int, broadcastFunc, void *); |
948 | unsigned int cpu_broadcast_immediate_xcall(uint32_t *, boolean_t, broadcastFunc, void *); |
949 | unsigned int cpu_broadcast_immediate_xcall_simple(boolean_t, broadcastFunc, void *); |
950 | __result_use_check kern_return_t cpu_immediate_xcall(int, broadcastFunc, void *); |
951 | |
952 | #ifdef KERNEL_PRIVATE |
953 | |
954 | /* Interface to be used by the perf. controller to register a callback, in a |
955 | * single-threaded fashion. The callback will receive notifications of |
956 | * processor performance quality-of-service changes from the scheduler. |
957 | */ |
958 | |
959 | #ifdef __arm64__ |
960 | typedef void (*cpu_qos_update_t)(int throughput_qos, uint64_t qos_param1, uint64_t qos_param2); |
961 | void cpu_qos_update_register(cpu_qos_update_t); |
962 | #endif /* __arm64__ */ |
963 | |
964 | struct going_on_core { |
965 | uint64_t thread_id; |
966 | uint16_t qos_class; |
967 | uint16_t urgency; /* XCPM compatibility */ |
968 | uint32_t is_32_bit : 1; /* uses 32-bit ISA/register state in userspace (which may differ from address space size) */ |
969 | uint32_t is_kernel_thread : 1; |
970 | uint64_t thread_group_id; |
971 | void *thread_group_data; |
972 | uint64_t scheduling_latency; /* absolute time between when thread was made runnable and this ctx switch */ |
973 | uint64_t start_time; |
974 | uint64_t scheduling_latency_at_same_basepri; |
975 | uint32_t energy_estimate_nj; /* return: In nanojoules */ |
976 | /* smaller of the time between last change to base priority and ctx switch and scheduling_latency */ |
977 | }; |
978 | typedef struct going_on_core *going_on_core_t; |
979 | |
980 | struct going_off_core { |
981 | uint64_t thread_id; |
982 | uint32_t energy_estimate_nj; /* return: In nanojoules */ |
983 | uint32_t reserved; |
984 | uint64_t end_time; |
985 | uint64_t thread_group_id; |
986 | void *thread_group_data; |
987 | }; |
988 | typedef struct going_off_core *going_off_core_t; |
989 | |
990 | struct thread_group_data { |
991 | uint64_t thread_group_id; |
992 | void *thread_group_data; |
993 | uint32_t thread_group_size; |
994 | uint32_t thread_group_flags; |
995 | }; |
996 | typedef struct thread_group_data *thread_group_data_t; |
997 | |
998 | struct perfcontrol_max_runnable_latency { |
999 | uint64_t max_scheduling_latencies[4 /* THREAD_URGENCY_MAX */]; |
1000 | }; |
1001 | typedef struct perfcontrol_max_runnable_latency *perfcontrol_max_runnable_latency_t; |
1002 | |
1003 | struct perfcontrol_work_interval { |
1004 | uint64_t thread_id; |
1005 | uint16_t qos_class; |
1006 | uint16_t urgency; |
1007 | uint32_t flags; // notify |
1008 | uint64_t work_interval_id; |
1009 | uint64_t start; |
1010 | uint64_t finish; |
1011 | uint64_t deadline; |
1012 | uint64_t next_start; |
1013 | uint64_t thread_group_id; |
1014 | void *thread_group_data; |
1015 | uint32_t create_flags; |
1016 | }; |
1017 | typedef struct perfcontrol_work_interval *perfcontrol_work_interval_t; |
1018 | |
1019 | typedef enum { |
1020 | WORK_INTERVAL_START, |
1021 | WORK_INTERVAL_UPDATE, |
1022 | WORK_INTERVAL_FINISH, |
1023 | WORK_INTERVAL_CREATE, |
1024 | WORK_INTERVAL_DEALLOCATE, |
1025 | } work_interval_ctl_t; |
1026 | |
1027 | struct perfcontrol_work_interval_instance { |
1028 | work_interval_ctl_t ctl; |
1029 | uint32_t create_flags; |
1030 | uint64_t complexity; |
1031 | uint64_t thread_id; |
1032 | uint64_t work_interval_id; |
1033 | uint64_t instance_id; /* out: start, in: update/finish */ |
1034 | uint64_t start; |
1035 | uint64_t finish; |
1036 | uint64_t deadline; |
1037 | uint64_t thread_group_id; |
1038 | void *thread_group_data; |
1039 | }; |
1040 | typedef struct perfcontrol_work_interval_instance *perfcontrol_work_interval_instance_t; |
1041 | |
1042 | /* |
1043 | * Structure to export per-CPU counters as part of the CLPC callout. |
1044 | * Contains only the fixed CPU counters (instructions and cycles); CLPC |
1045 | * would call back into XNU to get the configurable counters if needed. |
1046 | */ |
1047 | struct perfcontrol_cpu_counters { |
1048 | uint64_t instructions; |
1049 | uint64_t cycles; |
1050 | }; |
1051 | |
1052 | __options_decl(perfcontrol_thread_flags_mask_t, uint64_t, { |
1053 | PERFCTL_THREAD_FLAGS_MASK_CLUSTER_SHARED_RSRC_RR = 1 << 0, |
1054 | PERFCTL_THREAD_FLAGS_MASK_CLUSTER_SHARED_RSRC_NATIVE_FIRST = 1 << 1, |
1055 | }); |
1056 | |
1057 | |
1058 | /* |
1059 | * Structure used to pass information about a thread to CLPC |
1060 | */ |
1061 | struct perfcontrol_thread_data { |
1062 | /* |
1063 | * Energy estimate (return value) |
1064 | * The field is populated by CLPC and used to update the |
1065 | * energy estimate of the thread |
1066 | */ |
1067 | uint32_t energy_estimate_nj; |
1068 | /* Perfcontrol class for thread */ |
1069 | perfcontrol_class_t perfctl_class; |
1070 | /* Thread ID for the thread */ |
1071 | uint64_t thread_id; |
1072 | /* Thread Group ID */ |
1073 | uint64_t thread_group_id; |
1074 | /* |
1075 | * Scheduling latency for threads at the same base priority. |
1076 | * Calculated by the scheduler and passed into CLPC. The field is |
1077 | * populated only in the thread_data structure for the thread |
1078 | * going on-core. |
1079 | */ |
1080 | uint64_t scheduling_latency_at_same_basepri; |
1081 | /* Thread Group data pointer */ |
1082 | void *thread_group_data; |
1083 | /* perfctl state pointer */ |
1084 | void *perfctl_state; |
1085 | /* Bitmask to indicate which thread flags have been updated as part of the callout */ |
1086 | perfcontrol_thread_flags_mask_t thread_flags_mask; |
1087 | /* Actual values for the flags that are getting updated in the callout */ |
1088 | perfcontrol_thread_flags_mask_t thread_flags; |
1089 | }; |
1090 | |
1091 | /* |
1092 | * All callouts from the scheduler are executed with interrupts |
1093 | * disabled. Callouts should be implemented in C with minimal |
1094 | * abstractions, and only use KPI exported by the mach/libkern |
1095 | * symbolset, restricted to routines like spinlocks and atomic |
1096 | * operations and scheduler routines as noted below. Spinlocks that |
1097 | * are used to synchronize data in the perfcontrol_state_t should only |
1098 | * ever be acquired with interrupts disabled, to avoid deadlocks where |
1099 | * an quantum expiration timer interrupt attempts to perform a callout |
1100 | * that attempts to lock a spinlock that is already held. |
1101 | */ |
1102 | |
1103 | /* |
1104 | * When a processor is switching between two threads (after the |
1105 | * scheduler has chosen a new thread), the low-level platform layer |
1106 | * will call this routine, which should perform required timestamps, |
1107 | * MMIO register reads, or other state switching. No scheduler locks |
1108 | * are held during this callout. |
1109 | * |
1110 | * This function is called with interrupts ENABLED. |
1111 | */ |
1112 | typedef void (*sched_perfcontrol_context_switch_t)(perfcontrol_state_t, perfcontrol_state_t); |
1113 | |
1114 | /* |
1115 | * Once the processor has switched to the new thread, the offcore |
1116 | * callout will indicate the old thread that is no longer being |
1117 | * run. The thread's scheduler lock is held, so it will not begin |
1118 | * running on another processor (in the case of preemption where it |
1119 | * remains runnable) until it completes. If the "thread_terminating" |
1120 | * boolean is TRUE, this will be the last callout for this thread_id. |
1121 | */ |
1122 | typedef void (*sched_perfcontrol_offcore_t)(perfcontrol_state_t, going_off_core_t /* populated by callee */, boolean_t); |
1123 | |
1124 | /* |
1125 | * After the offcore callout and after the old thread can potentially |
1126 | * start running on another processor, the oncore callout will be |
1127 | * called with the thread's scheduler lock held. The oncore callout is |
1128 | * also called any time one of the parameters in the going_on_core_t |
1129 | * structure changes, like priority/QoS changes, and quantum |
1130 | * expiration, so the callout must not assume callouts are paired with |
1131 | * offcore callouts. |
1132 | */ |
1133 | typedef void (*sched_perfcontrol_oncore_t)(perfcontrol_state_t, going_on_core_t); |
1134 | |
1135 | /* |
1136 | * Periodically (on hundreds of ms scale), the scheduler will perform |
1137 | * maintenance and report the maximum latency for runnable (but not currently |
1138 | * running) threads for each urgency class. |
1139 | */ |
1140 | typedef void (*sched_perfcontrol_max_runnable_latency_t)(perfcontrol_max_runnable_latency_t); |
1141 | |
1142 | /* |
1143 | * When the kernel receives information about work intervals from userland, |
1144 | * it is passed along using this callback. No locks are held, although the state |
1145 | * object will not go away during the callout. |
1146 | */ |
1147 | typedef void (*sched_perfcontrol_work_interval_notify_t)(perfcontrol_state_t, perfcontrol_work_interval_t); |
1148 | |
1149 | /* |
1150 | * Start, update and finish work interval instance with optional complexity estimate. |
1151 | */ |
1152 | typedef void (*sched_perfcontrol_work_interval_ctl_t)(perfcontrol_state_t, perfcontrol_work_interval_instance_t); |
1153 | |
1154 | /* |
1155 | * These callbacks are used when thread groups are added, removed or properties |
1156 | * updated. |
1157 | * No blocking allocations (or anything else blocking) are allowed inside these |
1158 | * callbacks. No locks allowed in these callbacks as well since the kernel might |
1159 | * be holding the thread/task locks. |
1160 | */ |
1161 | typedef void (*sched_perfcontrol_thread_group_init_t)(thread_group_data_t); |
1162 | typedef void (*sched_perfcontrol_thread_group_deinit_t)(thread_group_data_t); |
1163 | typedef void (*sched_perfcontrol_thread_group_flags_update_t)(thread_group_data_t); |
1164 | |
1165 | /* |
1166 | * Sometime after the timeout set by sched_perfcontrol_update_callback_deadline has passed, |
1167 | * this function will be called, passing the timeout deadline that was previously armed as an argument. |
1168 | * |
1169 | * This is called inside context-switch/quantum-interrupt context and must follow the safety rules for that context. |
1170 | */ |
1171 | typedef void (*sched_perfcontrol_deadline_passed_t)(uint64_t deadline); |
1172 | |
1173 | /* |
1174 | * Context Switch Callout |
1175 | * |
1176 | * Parameters: |
1177 | * event - The perfcontrol_event for this callout |
1178 | * cpu_id - The CPU doing the context switch |
1179 | * timestamp - The timestamp for the context switch |
1180 | * flags - Flags for other relevant information |
1181 | * offcore - perfcontrol_data structure for thread going off-core |
1182 | * oncore - perfcontrol_data structure for thread going on-core |
1183 | * cpu_counters - perfcontrol_cpu_counters for the CPU doing the switch |
1184 | */ |
1185 | typedef void (*sched_perfcontrol_csw_t)( |
1186 | perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, |
1187 | struct perfcontrol_thread_data *offcore, struct perfcontrol_thread_data *oncore, |
1188 | struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused); |
1189 | |
1190 | |
1191 | /* |
1192 | * Thread State Update Callout |
1193 | * |
1194 | * Parameters: |
1195 | * event - The perfcontrol_event for this callout |
1196 | * cpu_id - The CPU doing the state update |
1197 | * timestamp - The timestamp for the state update |
1198 | * flags - Flags for other relevant information |
1199 | * thr_data - perfcontrol_data structure for the thread being updated |
1200 | */ |
1201 | typedef void (*sched_perfcontrol_state_update_t)( |
1202 | perfcontrol_event event, uint32_t cpu_id, uint64_t timestamp, uint32_t flags, |
1203 | struct perfcontrol_thread_data *thr_data, __unused void *unused); |
1204 | |
1205 | /* |
1206 | * Thread Group Blocking Relationship Callout |
1207 | * |
1208 | * Parameters: |
1209 | * blocked_tg - Thread group blocking on progress of another thread group |
1210 | * blocking_tg - Thread group blocking progress of another thread group |
1211 | * flags - Flags for other relevant information |
1212 | * blocked_thr_state - Per-thread perfcontrol state for blocked thread |
1213 | */ |
1214 | typedef void (*sched_perfcontrol_thread_group_blocked_t)( |
1215 | thread_group_data_t blocked_tg, thread_group_data_t blocking_tg, uint32_t flags, perfcontrol_state_t blocked_thr_state); |
1216 | |
1217 | /* |
1218 | * Thread Group Unblocking Callout |
1219 | * |
1220 | * Parameters: |
1221 | * unblocked_tg - Thread group being unblocked from making forward progress |
1222 | * unblocking_tg - Thread group unblocking progress of another thread group |
1223 | * flags - Flags for other relevant information |
1224 | * unblocked_thr_state - Per-thread perfcontrol state for unblocked thread |
1225 | */ |
1226 | typedef void (*sched_perfcontrol_thread_group_unblocked_t)( |
1227 | thread_group_data_t unblocked_tg, thread_group_data_t unblocking_tg, uint32_t flags, perfcontrol_state_t unblocked_thr_state); |
1228 | |
1229 | /* |
1230 | * Callers should always use the CURRENT version so that the kernel can detect both older |
1231 | * and newer structure layouts. New callbacks should always be added at the end of the |
1232 | * structure, and xnu should expect existing source recompiled against newer headers |
1233 | * to pass NULL for unimplemented callbacks. Pass NULL as the as the callbacks parameter |
1234 | * to reset callbacks to their default in-kernel values. |
1235 | */ |
1236 | |
1237 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_0 (0) /* up-to oncore */ |
1238 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_1 (1) /* up-to max_runnable_latency */ |
1239 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_2 (2) /* up-to work_interval_notify */ |
1240 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_3 (3) /* up-to thread_group_deinit */ |
1241 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_4 (4) /* up-to deadline_passed */ |
1242 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_5 (5) /* up-to state_update */ |
1243 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_6 (6) /* up-to thread_group_flags_update */ |
1244 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_7 (7) /* up-to work_interval_ctl */ |
1245 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_8 (8) /* up-to thread_group_unblocked */ |
1246 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_9 (9) /* allows CLPC to specify resource contention flags */ |
1247 | #define SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT SCHED_PERFCONTROL_CALLBACKS_VERSION_6 |
1248 | |
1249 | struct sched_perfcontrol_callbacks { |
1250 | unsigned long version; /* Use SCHED_PERFCONTROL_CALLBACKS_VERSION_CURRENT */ |
1251 | sched_perfcontrol_offcore_t offcore; |
1252 | sched_perfcontrol_context_switch_t context_switch; |
1253 | sched_perfcontrol_oncore_t oncore; |
1254 | sched_perfcontrol_max_runnable_latency_t max_runnable_latency; |
1255 | sched_perfcontrol_work_interval_notify_t work_interval_notify; |
1256 | sched_perfcontrol_thread_group_init_t thread_group_init; |
1257 | sched_perfcontrol_thread_group_deinit_t thread_group_deinit; |
1258 | sched_perfcontrol_deadline_passed_t deadline_passed; |
1259 | sched_perfcontrol_csw_t csw; |
1260 | sched_perfcontrol_state_update_t state_update; |
1261 | sched_perfcontrol_thread_group_flags_update_t thread_group_flags_update; |
1262 | sched_perfcontrol_work_interval_ctl_t work_interval_ctl; |
1263 | sched_perfcontrol_thread_group_blocked_t thread_group_blocked; |
1264 | sched_perfcontrol_thread_group_unblocked_t thread_group_unblocked; |
1265 | }; |
1266 | typedef struct sched_perfcontrol_callbacks *sched_perfcontrol_callbacks_t; |
1267 | |
1268 | extern void sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state); |
1269 | extern void sched_perfcontrol_thread_group_recommend(void *data, cluster_type_t recommendation); |
1270 | extern void sched_perfcontrol_inherit_recommendation_from_tg(perfcontrol_class_t perfctl_class, boolean_t inherit); |
1271 | extern const char* sched_perfcontrol_thread_group_get_name(void *data); |
1272 | |
1273 | /* |
1274 | * Edge Scheduler-CLPC Interface |
1275 | * |
1276 | * sched_perfcontrol_thread_group_preferred_clusters_set() |
1277 | * |
1278 | * The Edge scheduler expects thread group recommendations to be specific clusters rather |
1279 | * than just E/P. In order to allow more fine grained control, CLPC can specify an override |
1280 | * preferred cluster per QoS bucket. CLPC passes a common preferred cluster `tg_preferred_cluster` |
1281 | * and an array of size [PERFCONTROL_CLASS_MAX] with overrides for specific perfctl classes. |
1282 | * The scheduler translates these preferences into sched_bucket |
1283 | * preferences and applies the changes. |
1284 | * |
1285 | */ |
1286 | /* Token to indicate a particular perfctl class is not overriden */ |
1287 | #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_OVERRIDE_NONE ((uint32_t)~0) |
1288 | |
1289 | /* |
1290 | * CLPC can also indicate if there should be an immediate rebalancing of threads of this TG as |
1291 | * part of this preferred cluster change. It does that by specifying the following options. |
1292 | */ |
1293 | #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNING 0x1 |
1294 | #define SCHED_PERFCONTROL_PREFERRED_CLUSTER_MIGRATE_RUNNABLE 0x2 |
1295 | typedef uint64_t sched_perfcontrol_preferred_cluster_options_t; |
1296 | |
1297 | extern void sched_perfcontrol_thread_group_preferred_clusters_set(void *machine_data, uint32_t tg_preferred_cluster, |
1298 | uint32_t overrides[PERFCONTROL_CLASS_MAX], sched_perfcontrol_preferred_cluster_options_t options); |
1299 | |
1300 | /* |
1301 | * Edge Scheduler-CLPC Interface |
1302 | * |
1303 | * sched_perfcontrol_edge_matrix_get()/sched_perfcontrol_edge_matrix_set() |
1304 | * |
1305 | * The Edge scheduler uses edges between clusters to define the likelihood of migrating threads |
1306 | * across clusters. The edge config between any two clusters defines the edge weight and whether |
1307 | * migation and steal operations are allowed across that edge. The getter and setter allow CLPC |
1308 | * to query and configure edge properties between various clusters on the platform. |
1309 | */ |
1310 | |
1311 | extern void sched_perfcontrol_edge_matrix_get(sched_clutch_edge *edge_matrix, bool *edge_request_bitmap, uint64_t flags, uint64_t matrix_order); |
1312 | extern void sched_perfcontrol_edge_matrix_set(sched_clutch_edge *edge_matrix, bool *edge_changes_bitmap, uint64_t flags, uint64_t matrix_order); |
1313 | |
1314 | /* |
1315 | * sched_perfcontrol_edge_cpu_rotation_bitmasks_get()/sched_perfcontrol_edge_cpu_rotation_bitmasks_set() |
1316 | * |
1317 | * In order to drive intra-cluster core rotation CLPC supplies the edge scheduler with a pair of |
1318 | * per-cluster bitmasks. The preferred_bitmask is a bitmask of CPU cores where if a bit is set, |
1319 | * CLPC would prefer threads to be scheduled on that core if it is idle. The migration_bitmask |
1320 | * is a bitmask of CPU cores where if a bit is set, CLPC would prefer threads no longer continue |
1321 | * running on that core if there is any other non-avoided idle core in the cluster that is available. |
1322 | */ |
1323 | |
1324 | extern void sched_perfcontrol_edge_cpu_rotation_bitmasks_set(uint32_t cluster_id, uint64_t preferred_bitmask, uint64_t migration_bitmask); |
1325 | extern void sched_perfcontrol_edge_cpu_rotation_bitmasks_get(uint32_t cluster_id, uint64_t *preferred_bitmask, uint64_t *migration_bitmask); |
1326 | |
1327 | /* |
1328 | * Update the deadline after which sched_perfcontrol_deadline_passed will be called. |
1329 | * Returns TRUE if it successfully canceled a previously set callback, |
1330 | * and FALSE if it did not (i.e. one wasn't set, or callback already fired / is in flight). |
1331 | * The callback is automatically canceled when it fires, and does not repeat unless rearmed. |
1332 | * |
1333 | * This 'timer' executes as the scheduler switches between threads, on a non-idle core |
1334 | * |
1335 | * There can be only one outstanding timer globally. |
1336 | */ |
1337 | extern boolean_t sched_perfcontrol_update_callback_deadline(uint64_t deadline); |
1338 | |
1339 | /* |
1340 | * SFI configuration. |
1341 | */ |
1342 | extern kern_return_t sched_perfcontrol_sfi_set_window(uint64_t window_usecs); |
1343 | extern kern_return_t sched_perfcontrol_sfi_set_bg_offtime(uint64_t offtime_usecs); |
1344 | extern kern_return_t sched_perfcontrol_sfi_set_utility_offtime(uint64_t offtime_usecs); |
1345 | |
1346 | typedef enum perfcontrol_callout_type { |
1347 | PERFCONTROL_CALLOUT_ON_CORE, |
1348 | PERFCONTROL_CALLOUT_OFF_CORE, |
1349 | PERFCONTROL_CALLOUT_CONTEXT, |
1350 | PERFCONTROL_CALLOUT_STATE_UPDATE, |
1351 | /* Add other callout types here */ |
1352 | PERFCONTROL_CALLOUT_MAX |
1353 | } perfcontrol_callout_type_t; |
1354 | |
1355 | typedef enum perfcontrol_callout_stat { |
1356 | PERFCONTROL_STAT_INSTRS, |
1357 | PERFCONTROL_STAT_CYCLES, |
1358 | /* Add other stat types here */ |
1359 | PERFCONTROL_STAT_MAX |
1360 | } perfcontrol_callout_stat_t; |
1361 | |
1362 | uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type, |
1363 | perfcontrol_callout_stat_t stat); |
1364 | |
1365 | #ifdef __arm64__ |
1366 | /* The performance controller may use this interface to recommend |
1367 | * that CPUs in the designated cluster employ WFE rather than WFI |
1368 | * within the idle loop, falling back to WFI after the specified |
1369 | * timeout. The updates are expected to be serialized by the caller, |
1370 | * the implementation is not required to perform internal synchronization. |
1371 | */ |
1372 | uint32_t ml_update_cluster_wfe_recommendation(uint32_t wfe_cluster_id, uint64_t wfe_timeout_abstime_interval, uint64_t wfe_hint_flags); |
1373 | #endif /* __arm64__ */ |
1374 | |
1375 | #if defined(HAS_APPLE_PAC) |
1376 | #define ONES(x) (BIT((x))-1) |
1377 | #define PTR_MASK ONES(64-T1SZ_BOOT) |
1378 | #define PAC_MASK ~PTR_MASK |
1379 | #define SIGN(p) ((p) & BIT(55)) |
1380 | #define UNSIGN_PTR(p) \ |
1381 | SIGN(p) ? ((p) | PAC_MASK) : ((p) & ~PAC_MASK) |
1382 | |
1383 | uint64_t ml_default_rop_pid(void); |
1384 | uint64_t ml_default_jop_pid(void); |
1385 | uint64_t ml_non_arm64e_user_jop_pid(void); |
1386 | void ml_task_set_rop_pid(task_t task, task_t parent_task, boolean_t inherit); |
1387 | void ml_task_set_jop_pid(task_t task, task_t parent_task, boolean_t inherit, boolean_t disable_user_jop); |
1388 | void ml_task_set_jop_pid_from_shared_region(task_t task, boolean_t disable_user_jop); |
1389 | uint8_t ml_task_get_disable_user_jop(task_t task); |
1390 | void ml_task_set_disable_user_jop(task_t task, uint8_t disable_user_jop); |
1391 | void ml_thread_set_disable_user_jop(thread_t thread, uint8_t disable_user_jop); |
1392 | void ml_thread_set_jop_pid(thread_t thread, task_t task); |
1393 | void *ml_auth_ptr_unchecked(void *ptr, unsigned key, uint64_t modifier); |
1394 | |
1395 | uint64_t ml_enable_user_jop_key(uint64_t user_jop_key); |
1396 | |
1397 | /** |
1398 | * Restores the previous JOP key state after a previous ml_enable_user_jop_key() |
1399 | * call. |
1400 | * |
1401 | * @param user_jop_key The userspace JOP key previously passed to |
1402 | * ml_enable_user_jop_key() |
1403 | * @param saved_jop_state The saved JOP state returned by |
1404 | * ml_enable_user_jop_key() |
1405 | */ |
1406 | void ml_disable_user_jop_key(uint64_t user_jop_key, uint64_t saved_jop_state); |
1407 | #endif /* defined(HAS_APPLE_PAC) */ |
1408 | |
1409 | void ml_enable_monitor(void); |
1410 | |
1411 | #endif /* KERNEL_PRIVATE */ |
1412 | |
1413 | boolean_t machine_timeout_suspended(void); |
1414 | void ml_get_power_state(boolean_t *, boolean_t *); |
1415 | |
1416 | uint32_t get_arm_cpu_version(void); |
1417 | boolean_t user_cont_hwclock_allowed(void); |
1418 | uint8_t user_timebase_type(void); |
1419 | boolean_t ml_thread_is64bit(thread_t thread); |
1420 | |
1421 | #ifdef __arm64__ |
1422 | bool ml_feature_supported(uint32_t feature_bit); |
1423 | void ml_set_align_checking(void); |
1424 | extern void wfe_timeout_configure(void); |
1425 | extern void wfe_timeout_init(void); |
1426 | #endif /* __arm64__ */ |
1427 | |
1428 | void ml_timer_evaluate(void); |
1429 | boolean_t ml_timer_forced_evaluation(void); |
1430 | void ml_gpu_stat_update(uint64_t); |
1431 | uint64_t ml_gpu_stat(thread_t); |
1432 | #endif /* __APPLE_API_PRIVATE */ |
1433 | |
1434 | |
1435 | |
1436 | #if __arm64__ && defined(CONFIG_XNUPOST) && defined(XNU_KERNEL_PRIVATE) |
1437 | extern void ml_expect_fault_begin(expected_fault_handler_t, uintptr_t); |
1438 | extern void ml_expect_fault_pc_begin(expected_fault_handler_t, uintptr_t); |
1439 | extern void ml_expect_fault_end(void); |
1440 | #endif /* __arm64__ && defined(CONFIG_XNUPOST) && defined(XNU_KERNEL_PRIVATE) */ |
1441 | |
1442 | |
1443 | extern uint32_t phy_read_panic; |
1444 | extern uint32_t phy_write_panic; |
1445 | #if DEVELOPMENT || DEBUG |
1446 | extern uint64_t simulate_stretched_io; |
1447 | #endif |
1448 | |
1449 | void ml_hibernate_active_pre(void); |
1450 | void ml_hibernate_active_post(void); |
1451 | |
1452 | void ml_report_minor_badness(uint32_t badness_id); |
1453 | #define ML_MINOR_BADNESS_CONSOLE_BUFFER_FULL 0 |
1454 | #define ML_MINOR_BADNESS_MEMFAULT_REPORTING_NOT_ENABLED 1 |
1455 | #define ML_MINOR_BADNESS_PIO_WRITTEN_FROM_USERSPACE 2 |
1456 | |
1457 | #ifdef XNU_KERNEL_PRIVATE |
1458 | /** |
1459 | * Depending on the system, by the time a backtracer starts inspecting an |
1460 | * interrupted CPU's register state, the value of the PC might have been |
1461 | * modified. In those cases, the original PC value is placed into a different |
1462 | * register. This function abstracts out those differences for a backtracer |
1463 | * wanting the PC of an interrupted CPU. |
1464 | * |
1465 | * @param state The ARM register state to parse. |
1466 | * |
1467 | * @return The original PC of the interrupted CPU. |
1468 | */ |
1469 | uint64_t ml_get_backtrace_pc(struct arm_saved_state *state); |
1470 | #endif /* XNU_KERNEL_PRIVATE */ |
1471 | |
1472 | #ifdef KERNEL_PRIVATE |
1473 | /** |
1474 | * Given a physical address, return whether that address is owned by the secure |
1475 | * world. |
1476 | * |
1477 | * @note This does not include memory shared between XNU and the secure world. |
1478 | * |
1479 | * @param paddr The physical address to check. |
1480 | * |
1481 | * @return True if the physical address is owned and being used exclusively by |
1482 | * the secure world, false otherwise. |
1483 | */ |
1484 | bool ml_paddr_is_exclaves_owned(vm_offset_t paddr); |
1485 | #endif /* KERNEL_PRIVATE */ |
1486 | |
1487 | __END_DECLS |
1488 | |
1489 | #endif /* _ARM_MACHINE_ROUTINES_H_ */ |
1490 | |