1 | /* |
2 | * Copyright (c) 2017-2019 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * File: arm/cpu_common.c |
30 | * |
31 | * cpu routines common to all supported arm variants |
32 | */ |
33 | |
34 | #include <kern/machine.h> |
35 | #include <kern/cpu_number.h> |
36 | #include <kern/thread.h> |
37 | #include <kern/percpu.h> |
38 | #include <kern/timer_queue.h> |
39 | #include <kern/locks.h> |
40 | #include <kern/clock.h> |
41 | #include <arm/cpu_data.h> |
42 | #include <arm/cpuid.h> |
43 | #include <arm/caches_internal.h> |
44 | #include <arm/cpu_data_internal.h> |
45 | #include <arm/cpu_internal.h> |
46 | #include <arm/misc_protos.h> |
47 | #include <arm/machine_cpu.h> |
48 | #include <arm/rtclock.h> |
49 | #include <mach/processor_info.h> |
50 | #include <machine/atomic.h> |
51 | #include <machine/config.h> |
52 | #include <vm/vm_kern.h> |
53 | #include <vm/vm_map.h> |
54 | #include <pexpert/arm/protos.h> |
55 | #include <pexpert/device_tree.h> |
56 | #include <sys/kdebug.h> |
57 | #include <arm/machine_routines.h> |
58 | #include <arm64/proc_reg.h> |
59 | #include <libkern/OSAtomic.h> |
60 | |
61 | SECURITY_READ_ONLY_LATE(struct percpu_base) percpu_base; |
62 | vm_address_t percpu_base_cur; |
63 | cpu_data_t PERCPU_DATA(cpu_data); |
64 | cpu_data_entry_t CpuDataEntries[MAX_CPUS]; |
65 | |
66 | static LCK_GRP_DECLARE(cpu_lck_grp, "cpu_lck_grp" ); |
67 | static LCK_RW_DECLARE(cpu_state_lock, &cpu_lck_grp); |
68 | |
69 | unsigned int real_ncpus = 1; |
70 | boolean_t idle_enable = FALSE; |
71 | uint64_t wake_abstime = 0x0ULL; |
72 | |
73 | extern uint64_t xcall_ack_timeout_abstime; |
74 | |
75 | #if defined(HAS_IPI) |
76 | extern unsigned int gFastIPI; |
77 | #endif /* defined(HAS_IPI) */ |
78 | |
79 | cpu_data_t * |
80 | cpu_datap(int cpu) |
81 | { |
82 | assert(cpu <= ml_get_max_cpu_number()); |
83 | return CpuDataEntries[cpu].cpu_data_vaddr; |
84 | } |
85 | |
86 | kern_return_t |
87 | cpu_control(int slot_num, |
88 | processor_info_t info, |
89 | unsigned int count) |
90 | { |
91 | printf(format: "cpu_control(%d,%p,%d) not implemented\n" , |
92 | slot_num, info, count); |
93 | return KERN_FAILURE; |
94 | } |
95 | |
96 | kern_return_t |
97 | cpu_info_count(processor_flavor_t flavor, |
98 | unsigned int *count) |
99 | { |
100 | switch (flavor) { |
101 | case PROCESSOR_CPU_STAT: |
102 | *count = PROCESSOR_CPU_STAT_COUNT; |
103 | return KERN_SUCCESS; |
104 | |
105 | case PROCESSOR_CPU_STAT64: |
106 | *count = PROCESSOR_CPU_STAT64_COUNT; |
107 | return KERN_SUCCESS; |
108 | |
109 | default: |
110 | *count = 0; |
111 | return KERN_FAILURE; |
112 | } |
113 | } |
114 | |
115 | kern_return_t |
116 | cpu_info(processor_flavor_t flavor, int slot_num, processor_info_t info, |
117 | unsigned int *count) |
118 | { |
119 | cpu_data_t *cpu_data_ptr = CpuDataEntries[slot_num].cpu_data_vaddr; |
120 | |
121 | switch (flavor) { |
122 | case PROCESSOR_CPU_STAT: |
123 | { |
124 | if (*count < PROCESSOR_CPU_STAT_COUNT) { |
125 | return KERN_FAILURE; |
126 | } |
127 | |
128 | processor_cpu_stat_t cpu_stat = (processor_cpu_stat_t)info; |
129 | cpu_stat->irq_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.irq_ex_cnt; |
130 | cpu_stat->ipi_cnt = (uint32_t)cpu_data_ptr->cpu_stat.ipi_cnt; |
131 | cpu_stat->timer_cnt = (uint32_t)cpu_data_ptr->cpu_stat.timer_cnt; |
132 | cpu_stat->undef_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.undef_ex_cnt; |
133 | cpu_stat->unaligned_cnt = (uint32_t)cpu_data_ptr->cpu_stat.unaligned_cnt; |
134 | cpu_stat->vfp_cnt = (uint32_t)cpu_data_ptr->cpu_stat.vfp_cnt; |
135 | cpu_stat->vfp_shortv_cnt = 0; |
136 | cpu_stat->data_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.data_ex_cnt; |
137 | cpu_stat->instr_ex_cnt = (uint32_t)cpu_data_ptr->cpu_stat.instr_ex_cnt; |
138 | |
139 | *count = PROCESSOR_CPU_STAT_COUNT; |
140 | |
141 | return KERN_SUCCESS; |
142 | } |
143 | |
144 | case PROCESSOR_CPU_STAT64: |
145 | { |
146 | if (*count < PROCESSOR_CPU_STAT64_COUNT) { |
147 | return KERN_FAILURE; |
148 | } |
149 | |
150 | processor_cpu_stat64_t cpu_stat = (processor_cpu_stat64_t)info; |
151 | cpu_stat->irq_ex_cnt = cpu_data_ptr->cpu_stat.irq_ex_cnt; |
152 | cpu_stat->ipi_cnt = cpu_data_ptr->cpu_stat.ipi_cnt; |
153 | cpu_stat->timer_cnt = cpu_data_ptr->cpu_stat.timer_cnt; |
154 | cpu_stat->undef_ex_cnt = cpu_data_ptr->cpu_stat.undef_ex_cnt; |
155 | cpu_stat->unaligned_cnt = cpu_data_ptr->cpu_stat.unaligned_cnt; |
156 | cpu_stat->vfp_cnt = cpu_data_ptr->cpu_stat.vfp_cnt; |
157 | cpu_stat->vfp_shortv_cnt = 0; |
158 | cpu_stat->data_ex_cnt = cpu_data_ptr->cpu_stat.data_ex_cnt; |
159 | cpu_stat->instr_ex_cnt = cpu_data_ptr->cpu_stat.instr_ex_cnt; |
160 | #if CONFIG_CPU_COUNTERS |
161 | cpu_stat->pmi_cnt = cpu_data_ptr->cpu_monotonic.mtc_npmis; |
162 | #endif /* CONFIG_CPU_COUNTERS */ |
163 | |
164 | *count = PROCESSOR_CPU_STAT64_COUNT; |
165 | |
166 | return KERN_SUCCESS; |
167 | } |
168 | |
169 | default: |
170 | return KERN_FAILURE; |
171 | } |
172 | } |
173 | |
174 | /* |
175 | * Routine: cpu_doshutdown |
176 | * Function: |
177 | */ |
178 | void |
179 | cpu_doshutdown(void (*doshutdown)(processor_t), |
180 | processor_t processor) |
181 | { |
182 | doshutdown(processor); |
183 | } |
184 | |
185 | /* |
186 | * Routine: cpu_idle_tickle |
187 | * |
188 | */ |
189 | void |
190 | cpu_idle_tickle(void) |
191 | { |
192 | boolean_t intr; |
193 | cpu_data_t *cpu_data_ptr; |
194 | uint64_t new_idle_timeout_ticks = 0x0ULL; |
195 | |
196 | intr = ml_set_interrupts_enabled(FALSE); |
197 | cpu_data_ptr = getCpuDatap(); |
198 | |
199 | if (cpu_data_ptr->idle_timer_notify != NULL) { |
200 | cpu_data_ptr->idle_timer_notify(cpu_data_ptr->idle_timer_refcon, &new_idle_timeout_ticks); |
201 | if (new_idle_timeout_ticks != 0x0ULL) { |
202 | /* if a new idle timeout was requested set the new idle timer deadline */ |
203 | clock_absolutetime_interval_to_deadline(abstime: new_idle_timeout_ticks, result: &cpu_data_ptr->idle_timer_deadline); |
204 | } else { |
205 | /* turn off the idle timer */ |
206 | cpu_data_ptr->idle_timer_deadline = 0x0ULL; |
207 | } |
208 | timer_resync_deadlines(); |
209 | } |
210 | (void) ml_set_interrupts_enabled(enable: intr); |
211 | } |
212 | |
213 | static void |
214 | cpu_handle_xcall(cpu_data_t *cpu_data_ptr) |
215 | { |
216 | broadcastFunc xfunc; |
217 | void *xparam; |
218 | |
219 | os_atomic_thread_fence(acquire); |
220 | /* Come back around if cpu_signal_internal is running on another CPU and has just |
221 | * added SIGPxcall to the pending mask, but hasn't yet assigned the call params.*/ |
222 | if (cpu_data_ptr->cpu_xcall_p0 != NULL && cpu_data_ptr->cpu_xcall_p1 != NULL) { |
223 | xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr); |
224 | INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI); |
225 | xparam = cpu_data_ptr->cpu_xcall_p1; |
226 | cpu_data_ptr->cpu_xcall_p0 = NULL; |
227 | cpu_data_ptr->cpu_xcall_p1 = NULL; |
228 | os_atomic_thread_fence(acq_rel); |
229 | os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcall, relaxed); |
230 | xfunc(xparam); |
231 | INTERRUPT_MASKED_DEBUG_END(); |
232 | } |
233 | if (cpu_data_ptr->cpu_imm_xcall_p0 != NULL && cpu_data_ptr->cpu_imm_xcall_p1 != NULL) { |
234 | xfunc = ptrauth_auth_function(cpu_data_ptr->cpu_imm_xcall_p0, ptrauth_key_function_pointer, cpu_data_ptr); |
235 | INTERRUPT_MASKED_DEBUG_START(xfunc, DBG_INTR_TYPE_IPI); |
236 | xparam = cpu_data_ptr->cpu_imm_xcall_p1; |
237 | cpu_data_ptr->cpu_imm_xcall_p0 = NULL; |
238 | cpu_data_ptr->cpu_imm_xcall_p1 = NULL; |
239 | os_atomic_thread_fence(acq_rel); |
240 | os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPxcallImm, relaxed); |
241 | xfunc(xparam); |
242 | INTERRUPT_MASKED_DEBUG_END(); |
243 | } |
244 | } |
245 | |
246 | static unsigned int |
247 | cpu_broadcast_xcall_internal(unsigned int signal, |
248 | uint32_t *synch, |
249 | boolean_t self_xcall, |
250 | broadcastFunc func, |
251 | void *parm) |
252 | { |
253 | boolean_t intr; |
254 | cpu_data_t *cpu_data_ptr; |
255 | cpu_data_t *target_cpu_datap; |
256 | unsigned int failsig; |
257 | int cpu; |
258 | int max_cpu = ml_get_max_cpu_number() + 1; |
259 | |
260 | //yes, param ALSO cannot be NULL |
261 | assert(func); |
262 | assert(parm); |
263 | |
264 | intr = ml_set_interrupts_enabled(FALSE); |
265 | cpu_data_ptr = getCpuDatap(); |
266 | |
267 | failsig = 0; |
268 | |
269 | if (synch != NULL) { |
270 | *synch = max_cpu; |
271 | assert_wait(event: (event_t)synch, THREAD_UNINT); |
272 | } |
273 | |
274 | for (cpu = 0; cpu < max_cpu; cpu++) { |
275 | target_cpu_datap = (cpu_data_t *)CpuDataEntries[cpu].cpu_data_vaddr; |
276 | |
277 | if (target_cpu_datap == cpu_data_ptr) { |
278 | continue; |
279 | } |
280 | |
281 | if ((target_cpu_datap == NULL) || |
282 | KERN_SUCCESS != cpu_signal(target: target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), p1: parm)) { |
283 | failsig++; |
284 | } |
285 | } |
286 | |
287 | if (self_xcall) { |
288 | func(parm); |
289 | } |
290 | |
291 | (void) ml_set_interrupts_enabled(enable: intr); |
292 | |
293 | if (synch != NULL) { |
294 | if (os_atomic_sub(synch, (!self_xcall) ? failsig + 1 : failsig, relaxed) == 0) { |
295 | clear_wait(thread: current_thread(), THREAD_AWAKENED); |
296 | } else { |
297 | thread_block(THREAD_CONTINUE_NULL); |
298 | } |
299 | } |
300 | |
301 | if (!self_xcall) { |
302 | return max_cpu - failsig - 1; |
303 | } else { |
304 | return max_cpu - failsig; |
305 | } |
306 | } |
307 | |
308 | unsigned int |
309 | cpu_broadcast_xcall(uint32_t *synch, |
310 | boolean_t self_xcall, |
311 | broadcastFunc func, |
312 | void *parm) |
313 | { |
314 | return cpu_broadcast_xcall_internal(SIGPxcall, synch, self_xcall, func, parm); |
315 | } |
316 | |
317 | struct cpu_broadcast_xcall_simple_data { |
318 | broadcastFunc func; |
319 | void* parm; |
320 | uint32_t sync; |
321 | }; |
322 | |
323 | static void |
324 | cpu_broadcast_xcall_simple_cbk(void *parm) |
325 | { |
326 | struct cpu_broadcast_xcall_simple_data *data = (struct cpu_broadcast_xcall_simple_data*)parm; |
327 | |
328 | data->func(data->parm); |
329 | |
330 | if (os_atomic_dec(&data->sync, relaxed) == 0) { |
331 | thread_wakeup((event_t)&data->sync); |
332 | } |
333 | } |
334 | |
335 | static unsigned int |
336 | cpu_xcall_simple(boolean_t self_xcall, |
337 | broadcastFunc func, |
338 | void *parm, |
339 | bool immediate) |
340 | { |
341 | struct cpu_broadcast_xcall_simple_data data = {}; |
342 | |
343 | data.func = func; |
344 | data.parm = parm; |
345 | |
346 | return cpu_broadcast_xcall_internal(signal: immediate ? SIGPxcallImm : SIGPxcall, synch: &data.sync, self_xcall, func: cpu_broadcast_xcall_simple_cbk, parm: &data); |
347 | } |
348 | |
349 | unsigned int |
350 | cpu_broadcast_immediate_xcall(uint32_t *synch, |
351 | boolean_t self_xcall, |
352 | broadcastFunc func, |
353 | void *parm) |
354 | { |
355 | return cpu_broadcast_xcall_internal(SIGPxcallImm, synch, self_xcall, func, parm); |
356 | } |
357 | |
358 | unsigned int |
359 | cpu_broadcast_xcall_simple(boolean_t self_xcall, |
360 | broadcastFunc func, |
361 | void *parm) |
362 | { |
363 | return cpu_xcall_simple(self_xcall, func, parm, false); |
364 | } |
365 | |
366 | unsigned int |
367 | cpu_broadcast_immediate_xcall_simple(boolean_t self_xcall, |
368 | broadcastFunc func, |
369 | void *parm) |
370 | { |
371 | return cpu_xcall_simple(self_xcall, func, parm, true); |
372 | } |
373 | |
374 | static kern_return_t |
375 | cpu_xcall_internal(unsigned int signal, int cpu_number, broadcastFunc func, void *param) |
376 | { |
377 | cpu_data_t *target_cpu_datap; |
378 | |
379 | if ((cpu_number < 0) || (cpu_number > ml_get_max_cpu_number())) { |
380 | panic("cpu_xcall_internal: invalid cpu_number %d" , cpu_number); |
381 | } |
382 | |
383 | if (func == NULL || param == NULL) { |
384 | // cpu_handle_xcall uses non-NULL-ness to tell when the value is ready |
385 | panic("cpu_xcall_internal: cannot have null func/param: %p %p" , func, param); |
386 | } |
387 | |
388 | target_cpu_datap = (cpu_data_t*)CpuDataEntries[cpu_number].cpu_data_vaddr; |
389 | if (target_cpu_datap == NULL) { |
390 | panic("cpu_xcall_internal: cpu %d not initialized" , cpu_number); |
391 | } |
392 | |
393 | return cpu_signal(target: target_cpu_datap, signal, ptrauth_nop_cast(void*, ptrauth_auth_and_resign(func, ptrauth_key_function_pointer, ptrauth_type_discriminator(broadcastFunc), ptrauth_key_function_pointer, target_cpu_datap)), p1: param); |
394 | } |
395 | |
396 | kern_return_t |
397 | cpu_xcall(int cpu_number, broadcastFunc func, void *param) |
398 | { |
399 | return cpu_xcall_internal(SIGPxcall, cpu_number, func, param); |
400 | } |
401 | |
402 | kern_return_t |
403 | cpu_immediate_xcall(int cpu_number, broadcastFunc func, void *param) |
404 | { |
405 | return cpu_xcall_internal(SIGPxcallImm, cpu_number, func, param); |
406 | } |
407 | |
408 | static kern_return_t |
409 | cpu_signal_internal(cpu_data_t *target_proc, |
410 | unsigned int signal, |
411 | void *p0, |
412 | void *p1, |
413 | boolean_t defer) |
414 | { |
415 | unsigned int Check_SIGPdisabled; |
416 | int current_signals; |
417 | bool swap_success; |
418 | boolean_t interruptible = ml_set_interrupts_enabled(FALSE); |
419 | cpu_data_t *current_proc = getCpuDatap(); |
420 | |
421 | /* We'll mandate that only IPIs meant to kick a core out of idle may ever be deferred. */ |
422 | if (defer) { |
423 | assert(signal == SIGPnop); |
424 | } |
425 | |
426 | if (current_proc != target_proc) { |
427 | Check_SIGPdisabled = SIGPdisabled; |
428 | } else { |
429 | Check_SIGPdisabled = 0; |
430 | } |
431 | |
432 | if ((signal == SIGPxcall) || (signal == SIGPxcallImm)) { |
433 | uint64_t start_mabs_time, max_mabs_time, current_mabs_time; |
434 | current_mabs_time = start_mabs_time = mach_absolute_time(); |
435 | max_mabs_time = xcall_ack_timeout_abstime + current_mabs_time; |
436 | assert(max_mabs_time > current_mabs_time); |
437 | |
438 | do { |
439 | current_signals = target_proc->cpu_signal; |
440 | if ((current_signals & SIGPdisabled) == SIGPdisabled) { |
441 | ml_set_interrupts_enabled(enable: interruptible); |
442 | return KERN_FAILURE; |
443 | } |
444 | swap_success = os_atomic_cmpxchg(&target_proc->cpu_signal, current_signals & (~signal), |
445 | current_signals | signal, release); |
446 | |
447 | if (!swap_success && (signal == SIGPxcallImm) && (target_proc->cpu_signal & SIGPxcallImm)) { |
448 | ml_set_interrupts_enabled(enable: interruptible); |
449 | return KERN_ALREADY_WAITING; |
450 | } |
451 | |
452 | /* Drain pending xcalls on this cpu; the CPU we're trying to xcall may in turn |
453 | * be trying to xcall us. Since we have interrupts disabled that can deadlock, |
454 | * so break the deadlock by draining pending xcalls. */ |
455 | if (!swap_success && (current_proc->cpu_signal & signal)) { |
456 | cpu_handle_xcall(cpu_data_ptr: current_proc); |
457 | } |
458 | } while (!swap_success && ((current_mabs_time = mach_absolute_time()) < max_mabs_time)); |
459 | |
460 | /* |
461 | * If we time out while waiting for the target CPU to respond, it's possible that no |
462 | * other CPU is available to handle the watchdog interrupt that would eventually trigger |
463 | * a panic. To prevent this from happening, we just panic here to flag this condition. |
464 | */ |
465 | if (__improbable(current_mabs_time >= max_mabs_time)) { |
466 | uint64_t end_time_ns, xcall_ack_timeout_ns; |
467 | absolutetime_to_nanoseconds(abstime: current_mabs_time - start_mabs_time, result: &end_time_ns); |
468 | absolutetime_to_nanoseconds(abstime: xcall_ack_timeout_abstime, result: &xcall_ack_timeout_ns); |
469 | panic("CPU%u has failed to respond to cross-call after %llu nanoseconds (timeout = %llu ns)" , |
470 | target_proc->cpu_number, end_time_ns, xcall_ack_timeout_ns); |
471 | } |
472 | |
473 | if (signal == SIGPxcallImm) { |
474 | target_proc->cpu_imm_xcall_p0 = p0; |
475 | target_proc->cpu_imm_xcall_p1 = p1; |
476 | } else { |
477 | target_proc->cpu_xcall_p0 = p0; |
478 | target_proc->cpu_xcall_p1 = p1; |
479 | } |
480 | } else { |
481 | do { |
482 | current_signals = target_proc->cpu_signal; |
483 | if ((Check_SIGPdisabled != 0) && (current_signals & Check_SIGPdisabled) == SIGPdisabled) { |
484 | ml_set_interrupts_enabled(enable: interruptible); |
485 | return KERN_FAILURE; |
486 | } |
487 | |
488 | swap_success = os_atomic_cmpxchg(&target_proc->cpu_signal, current_signals, |
489 | current_signals | signal, release); |
490 | } while (!swap_success); |
491 | } |
492 | |
493 | /* |
494 | * DSB is needed here to ensure prior stores to the pending signal mask and xcall params |
495 | * will be visible by the time the other cores are signaled. The IPI mechanism on any |
496 | * given platform will very likely use either an MSR or a non-coherent store that would |
497 | * not be ordered by a simple DMB. |
498 | */ |
499 | __builtin_arm_dsb(DSB_ISHST); |
500 | |
501 | if (!(target_proc->cpu_signal & SIGPdisabled)) { |
502 | if (defer) { |
503 | #if defined(HAS_IPI) |
504 | if (gFastIPI) { |
505 | ml_cpu_signal_deferred(target_proc->cpu_phys_id); |
506 | } else { |
507 | PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target_proc->cpu_id); |
508 | } |
509 | #else |
510 | PE_cpu_signal_deferred(getCpuDatap()->cpu_id, target: target_proc->cpu_id); |
511 | #endif /* defined(HAS_IPI) */ |
512 | } else { |
513 | #if defined(HAS_IPI) |
514 | if (gFastIPI) { |
515 | ml_cpu_signal(target_proc->cpu_phys_id); |
516 | } else { |
517 | PE_cpu_signal(getCpuDatap()->cpu_id, target_proc->cpu_id); |
518 | } |
519 | #else |
520 | PE_cpu_signal(getCpuDatap()->cpu_id, target: target_proc->cpu_id); |
521 | #endif /* defined(HAS_IPI) */ |
522 | } |
523 | } |
524 | |
525 | ml_set_interrupts_enabled(enable: interruptible); |
526 | return KERN_SUCCESS; |
527 | } |
528 | |
529 | kern_return_t |
530 | cpu_signal(cpu_data_t *target_proc, |
531 | unsigned int signal, |
532 | void *p0, |
533 | void *p1) |
534 | { |
535 | return cpu_signal_internal(target_proc, signal, p0, p1, FALSE); |
536 | } |
537 | |
538 | kern_return_t |
539 | cpu_signal_deferred(cpu_data_t *target_proc) |
540 | { |
541 | return cpu_signal_internal(target_proc, SIGPnop, NULL, NULL, TRUE); |
542 | } |
543 | |
544 | void |
545 | cpu_signal_cancel(cpu_data_t *target_proc) |
546 | { |
547 | /* TODO: Should we care about the state of a core as far as squashing deferred IPIs goes? */ |
548 | if (!(target_proc->cpu_signal & SIGPdisabled)) { |
549 | #if defined(HAS_IPI) |
550 | if (gFastIPI) { |
551 | ml_cpu_signal_retract(target_proc->cpu_phys_id); |
552 | } else { |
553 | PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target_proc->cpu_id); |
554 | } |
555 | #else |
556 | PE_cpu_signal_cancel(getCpuDatap()->cpu_id, target: target_proc->cpu_id); |
557 | #endif /* defined(HAS_IPI) */ |
558 | } |
559 | } |
560 | |
561 | void |
562 | cpu_signal_handler(void) |
563 | { |
564 | cpu_signal_handler_internal(FALSE); |
565 | } |
566 | |
567 | bool |
568 | cpu_has_SIGPdebug_pending(void) |
569 | { |
570 | cpu_data_t *cpu_data_ptr = getCpuDatap(); |
571 | |
572 | return cpu_data_ptr->cpu_signal & SIGPdebug; |
573 | } |
574 | |
575 | void |
576 | cpu_signal_handler_internal(boolean_t disable_signal) |
577 | { |
578 | cpu_data_t *cpu_data_ptr = getCpuDatap(); |
579 | unsigned int cpu_signal; |
580 | |
581 | cpu_data_ptr->cpu_stat.ipi_cnt++; |
582 | cpu_data_ptr->cpu_stat.ipi_cnt_wake++; |
583 | SCHED_STATS_INC(ipi_count); |
584 | |
585 | /* |
586 | * Employ an acquire barrier when loading cpu_signal to ensure that |
587 | * loads within individual signal handlers won't be speculated ahead |
588 | * of the load of cpu_signal. This pairs with the release barrier |
589 | * in cpu_signal_internal() to ensure that once a flag has been set in |
590 | * the cpu_signal mask, any prerequisite setup is also visible to signal |
591 | * handlers. |
592 | */ |
593 | cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, acquire); |
594 | |
595 | if ((!(cpu_signal & SIGPdisabled)) && (disable_signal == TRUE)) { |
596 | os_atomic_or(&cpu_data_ptr->cpu_signal, SIGPdisabled, relaxed); |
597 | } else if ((cpu_signal & SIGPdisabled) && (disable_signal == FALSE)) { |
598 | os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdisabled, relaxed); |
599 | } |
600 | |
601 | while (cpu_signal & ~SIGPdisabled) { |
602 | if (cpu_signal & SIGPdebug) { |
603 | os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdebug, relaxed); |
604 | INTERRUPT_MASKED_DEBUG_START(DebuggerXCall, DBG_INTR_TYPE_IPI); |
605 | DebuggerXCall(ctx: cpu_data_ptr->cpu_int_state); |
606 | INTERRUPT_MASKED_DEBUG_END(); |
607 | } |
608 | if (cpu_signal & SIGPdec) { |
609 | os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPdec, relaxed); |
610 | INTERRUPT_MASKED_DEBUG_START(rtclock_intr, DBG_INTR_TYPE_IPI); |
611 | rtclock_intr(FALSE); |
612 | INTERRUPT_MASKED_DEBUG_END(); |
613 | } |
614 | #if KPERF |
615 | if (cpu_signal & SIGPkppet) { |
616 | os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPkppet, relaxed); |
617 | extern void kperf_signal_handler(void); |
618 | INTERRUPT_MASKED_DEBUG_START(kperf_signal_handler, DBG_INTR_TYPE_IPI); |
619 | kperf_signal_handler(); |
620 | INTERRUPT_MASKED_DEBUG_END(); |
621 | } |
622 | #endif /* KPERF */ |
623 | if (cpu_signal & (SIGPxcall | SIGPxcallImm)) { |
624 | cpu_handle_xcall(cpu_data_ptr); |
625 | } |
626 | if (cpu_signal & SIGPast) { |
627 | os_atomic_andnot(&cpu_data_ptr->cpu_signal, SIGPast, acquire); |
628 | INTERRUPT_MASKED_DEBUG_START(ast_check, DBG_INTR_TYPE_IPI); |
629 | ast_check(processor: current_processor()); |
630 | INTERRUPT_MASKED_DEBUG_END(); |
631 | } |
632 | |
633 | cpu_signal = os_atomic_or(&cpu_data_ptr->cpu_signal, 0, acquire); |
634 | } |
635 | } |
636 | |
637 | void |
638 | cpu_exit_wait(int cpu_id) |
639 | { |
640 | #if USE_APPLEARMSMP |
641 | if (!ml_is_quiescing()) { |
642 | // For runtime disable (non S2R) the CPU will shut down immediately. |
643 | ml_topology_cpu_t *cpu = &ml_get_topology_info()->cpus[cpu_id]; |
644 | assert(cpu && cpu->cpu_IMPL_regs); |
645 | volatile uint64_t *cpu_sts = (void *)(cpu->cpu_IMPL_regs + CPU_PIO_CPU_STS_OFFSET); |
646 | |
647 | // Poll the "CPU running state" field until it is 0 (off) |
648 | // This loop typically finishes in about 600ns. Sometimes it takes as long as 10us. |
649 | // If it takes longer than 10s, assume something went horribly wrong and panic. |
650 | uint64_t start = mach_absolute_time(), interval; |
651 | nanoseconds_to_absolutetime(nanoseconds: 10 * NSEC_PER_SEC, result: &interval); |
652 | |
653 | while ((*cpu_sts & CPU_PIO_CPU_STS_cpuRunSt_mask) != 0x00) { |
654 | __builtin_arm_dsb(DSB_ISH); |
655 | if (mach_absolute_time() > start + interval) { |
656 | #if NO_CPU_OVRD |
657 | // On platforms where CPU_OVRD is unavailable, a core can get stuck |
658 | // in a loop where it tries to enter WFI but is constantly woken up |
659 | // by an IRQ or FIQ. This condition persists until the cluster-wide |
660 | // deep sleep bits are set. |
661 | // |
662 | // Making this a fatal condition would be a poor UX, but it's good to |
663 | // print a warning so we know how often it happens. |
664 | kprintf("CPU%d failed to shut down\n" , cpu_id); |
665 | #else |
666 | panic("CPU%d failed to shut down" , cpu_id); |
667 | #endif |
668 | } |
669 | } |
670 | return; |
671 | } |
672 | #endif /* USE_APPLEARMSMP */ |
673 | |
674 | if (cpu_id != master_cpu) { |
675 | // For S2R, ml_arm_sleep() will do some extra polling after setting ARM_CPU_ON_SLEEP_PATH. |
676 | cpu_data_t *cpu_data_ptr; |
677 | |
678 | cpu_data_ptr = CpuDataEntries[cpu_id].cpu_data_vaddr; |
679 | while (!((*(volatile unsigned int*)&cpu_data_ptr->cpu_sleep_token) == ARM_CPU_ON_SLEEP_PATH)) { |
680 | } |
681 | ; |
682 | } |
683 | } |
684 | |
685 | boolean_t |
686 | cpu_can_exit(__unused int cpu) |
687 | { |
688 | return TRUE; |
689 | } |
690 | |
691 | void |
692 | cpu_machine_init(void) |
693 | { |
694 | static boolean_t started = FALSE; |
695 | cpu_data_t *cpu_data_ptr; |
696 | |
697 | cpu_data_ptr = getCpuDatap(); |
698 | started = ((cpu_data_ptr->cpu_flags & StartedState) == StartedState); |
699 | if (cpu_data_ptr->cpu_cache_dispatch != NULL) { |
700 | platform_cache_init(); |
701 | } |
702 | |
703 | /* Note: this calls IOCPURunPlatformActiveActions when resuming on boot cpu */ |
704 | PE_cpu_machine_init(target: cpu_data_ptr->cpu_id, bootb: !started); |
705 | |
706 | cpu_data_ptr->cpu_flags |= StartedState; |
707 | ml_init_interrupt(); |
708 | } |
709 | |
710 | processor_t |
711 | current_processor(void) |
712 | { |
713 | return PERCPU_GET(processor); |
714 | } |
715 | |
716 | processor_t |
717 | cpu_to_processor(int cpu) |
718 | { |
719 | cpu_data_t *cpu_data = cpu_datap(cpu); |
720 | if (cpu_data != NULL) { |
721 | return PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data); |
722 | } else { |
723 | return NULL; |
724 | } |
725 | } |
726 | |
727 | cpu_data_t * |
728 | processor_to_cpu_datap(processor_t processor) |
729 | { |
730 | assert(processor->cpu_id <= ml_get_max_cpu_number()); |
731 | assert(CpuDataEntries[processor->cpu_id].cpu_data_vaddr != NULL); |
732 | |
733 | return PERCPU_GET_RELATIVE(cpu_data, processor, processor); |
734 | } |
735 | |
736 | __startup_func |
737 | static void |
738 | cpu_data_startup_init(void) |
739 | { |
740 | vm_size_t size = percpu_section_size() * (ml_get_cpu_count() - 1); |
741 | |
742 | percpu_base.size = percpu_section_size(); |
743 | if (ml_get_cpu_count() == 1) { |
744 | percpu_base.start = VM_MAX_KERNEL_ADDRESS; |
745 | return; |
746 | } |
747 | |
748 | /* |
749 | * The memory needs to be physically contiguous because it contains |
750 | * cpu_data_t structures sometimes accessed during reset |
751 | * with the MMU off. |
752 | * |
753 | * kmem_alloc_contig() can't be used early, at the time STARTUP_SUB_PERCPU |
754 | * normally runs, so we instead steal the memory for the PERCPU subsystem |
755 | * even earlier. |
756 | */ |
757 | percpu_base.start = (vm_offset_t)pmap_steal_memory(size, PAGE_SIZE); |
758 | bzero(s: (void *)percpu_base.start, n: size); |
759 | |
760 | percpu_base.start -= percpu_section_start(); |
761 | percpu_base.end = percpu_base.start + size - 1; |
762 | percpu_base_cur = percpu_base.start; |
763 | } |
764 | STARTUP(PMAP_STEAL, STARTUP_RANK_FIRST, cpu_data_startup_init); |
765 | |
766 | cpu_data_t * |
767 | cpu_data_alloc(boolean_t is_boot_cpu) |
768 | { |
769 | cpu_data_t *cpu_data_ptr = NULL; |
770 | vm_address_t base; |
771 | |
772 | if (is_boot_cpu) { |
773 | cpu_data_ptr = PERCPU_GET_MASTER(cpu_data); |
774 | } else { |
775 | base = os_atomic_add_orig(&percpu_base_cur, |
776 | percpu_section_size(), relaxed); |
777 | |
778 | cpu_data_ptr = PERCPU_GET_WITH_BASE(base, cpu_data); |
779 | cpu_stack_alloc(cpu_data_ptr); |
780 | } |
781 | |
782 | return cpu_data_ptr; |
783 | } |
784 | |
785 | ast_t * |
786 | ast_pending(void) |
787 | { |
788 | return &getCpuDatap()->cpu_pending_ast; |
789 | } |
790 | |
791 | cpu_type_t |
792 | slot_type(int slot_num) |
793 | { |
794 | return cpu_datap(cpu: slot_num)->cpu_type; |
795 | } |
796 | |
797 | cpu_subtype_t |
798 | slot_subtype(int slot_num) |
799 | { |
800 | return cpu_datap(cpu: slot_num)->cpu_subtype; |
801 | } |
802 | |
803 | cpu_threadtype_t |
804 | slot_threadtype(int slot_num) |
805 | { |
806 | return cpu_datap(cpu: slot_num)->cpu_threadtype; |
807 | } |
808 | |
809 | cpu_type_t |
810 | cpu_type(void) |
811 | { |
812 | return getCpuDatap()->cpu_type; |
813 | } |
814 | |
815 | cpu_subtype_t |
816 | cpu_subtype(void) |
817 | { |
818 | return getCpuDatap()->cpu_subtype; |
819 | } |
820 | |
821 | cpu_threadtype_t |
822 | cpu_threadtype(void) |
823 | { |
824 | return getCpuDatap()->cpu_threadtype; |
825 | } |
826 | |
827 | int |
828 | cpu_number(void) |
829 | { |
830 | return getCpuDatap()->cpu_number; |
831 | } |
832 | |
833 | vm_offset_t |
834 | current_percpu_base(void) |
835 | { |
836 | return current_thread()->machine.pcpu_data_base; |
837 | } |
838 | |
839 | vm_offset_t |
840 | other_percpu_base(int cpu) |
841 | { |
842 | return (char *)cpu_datap(cpu) - __PERCPU_ADDR(cpu_data); |
843 | } |
844 | |
845 | uint64_t |
846 | ml_get_wake_timebase(void) |
847 | { |
848 | return wake_abstime; |
849 | } |
850 | |
851 | bool |
852 | ml_cpu_signal_is_enabled(void) |
853 | { |
854 | return !(getCpuDatap()->cpu_signal & SIGPdisabled); |
855 | } |
856 | |
857 | bool |
858 | ml_cpu_can_exit(__unused int cpu_id, processor_reason_t reason) |
859 | { |
860 | /* processor_exit() is always allowed on the S2R path */ |
861 | if (ml_is_quiescing()) { |
862 | return true; |
863 | } |
864 | #if HAS_CLUSTER && USE_APPLEARMSMP |
865 | /* |
866 | * Until the feature is known to be stable, guard it with a boot-arg |
867 | */ |
868 | extern bool enable_processor_exit; |
869 | extern bool cluster_power_supported; |
870 | |
871 | if (!(enable_processor_exit || cluster_power_supported)) { |
872 | return false; |
873 | } |
874 | |
875 | if (!enable_processor_exit) { |
876 | /* Only cluster_power_supported so disallow any USER reason */ |
877 | if ((reason == REASON_USER) || (reason == REASON_CLPC_USER)) { |
878 | return false; |
879 | } |
880 | } |
881 | |
882 | /* |
883 | * Cyprus and newer chips can disable individual non-boot CPUs. The |
884 | * implementation polls cpuX_IMPL_CPU_STS, which differs on older chips. |
885 | */ |
886 | if (CpuDataEntries[cpu_id].cpu_data_vaddr != &BootCpuData) { |
887 | return true; |
888 | } |
889 | #else |
890 | (void)reason; |
891 | #endif |
892 | return false; |
893 | } |
894 | |
895 | #ifdef USE_APPLEARMSMP |
896 | |
897 | void |
898 | ml_cpu_begin_state_transition(int cpu_id) |
899 | { |
900 | lck_rw_lock_exclusive(lck: &cpu_state_lock); |
901 | CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = true; |
902 | lck_rw_unlock_exclusive(lck: &cpu_state_lock); |
903 | } |
904 | |
905 | void |
906 | ml_cpu_end_state_transition(int cpu_id) |
907 | { |
908 | lck_rw_lock_exclusive(lck: &cpu_state_lock); |
909 | CpuDataEntries[cpu_id].cpu_data_vaddr->in_state_transition = false; |
910 | lck_rw_unlock_exclusive(lck: &cpu_state_lock); |
911 | } |
912 | |
913 | void |
914 | ml_cpu_begin_loop(void) |
915 | { |
916 | lck_rw_lock_shared(lck: &cpu_state_lock); |
917 | } |
918 | |
919 | void |
920 | ml_cpu_end_loop(void) |
921 | { |
922 | lck_rw_unlock_shared(lck: &cpu_state_lock); |
923 | } |
924 | |
925 | void |
926 | ml_cpu_power_enable(int cpu_id) |
927 | { |
928 | PE_cpu_power_enable(cpu_id); |
929 | } |
930 | |
931 | void |
932 | ml_cpu_power_disable(int cpu_id) |
933 | { |
934 | PE_cpu_power_disable(cpu_id); |
935 | } |
936 | |
937 | #else /* USE_APPLEARMSMP */ |
938 | |
939 | void |
940 | ml_cpu_begin_state_transition(__unused int cpu_id) |
941 | { |
942 | } |
943 | |
944 | void |
945 | ml_cpu_end_state_transition(__unused int cpu_id) |
946 | { |
947 | } |
948 | |
949 | void |
950 | ml_cpu_begin_loop(void) |
951 | { |
952 | } |
953 | |
954 | void |
955 | ml_cpu_end_loop(void) |
956 | { |
957 | } |
958 | |
959 | void |
960 | ml_cpu_power_enable(__unused int cpu_id) |
961 | { |
962 | } |
963 | |
964 | void |
965 | ml_cpu_power_disable(__unused int cpu_id) |
966 | { |
967 | } |
968 | |
969 | #endif /* USE_APPLEARMSMP */ |
970 | |