1/*
2 * Copyright (c) 2019 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29extern "C" {
30#include <kern/debug.h>
31#include <pexpert/pexpert.h>
32#include <pexpert/arm64/board_config.h>
33};
34
35#include <kern/bits.h>
36#include <kern/processor.h>
37#include <kern/thread.h>
38#include <kperf/kperf.h>
39#include <machine/machine_routines.h>
40#include <libkern/OSAtomic.h>
41#include <libkern/c++/OSCollection.h>
42#include <IOKit/IODeviceTreeSupport.h>
43#include <IOKit/IOLib.h>
44#include <IOKit/IOPlatformActions.h>
45#include <IOKit/IOPMGR.h>
46#include <IOKit/IOReturn.h>
47#include <IOKit/IOService.h>
48#include <IOKit/PassthruInterruptController.h>
49#include <IOKit/pwr_mgt/RootDomain.h>
50#include <IOKit/pwr_mgt/IOPMPrivate.h>
51#include <Kernel/IOKitKernelInternal.h>
52
53#if USE_APPLEARMSMP
54
55// FIXME: These are in <kern/misc_protos.h> but that file has other deps that aren't being resolved
56extern "C" void console_suspend();
57extern "C" void console_resume();
58
59static PassthruInterruptController *gCPUIC;
60static IOPMGR *gPMGR;
61static IOInterruptController *gAIC;
62static bool aic_ipis = false;
63static const ml_topology_info *topology_info;
64
65// cpu_id of the boot processor
66static unsigned int boot_cpu;
67
68// array index is a cpu_id (so some elements may be NULL)
69static processor_t *machProcessors;
70
71bool cluster_power_supported = false;
72static uint64_t cpu_power_state_mask;
73static uint64_t all_clusters_mask;
74static uint64_t online_clusters_mask;
75
76static void
77processor_idle_wrapper(cpu_id_t /*cpu_id*/, boolean_t enter, uint64_t *new_timeout_ticks)
78{
79 if (enter) {
80 gPMGR->enterCPUIdle(newIdleTimeoutTicks: new_timeout_ticks);
81 } else {
82 gPMGR->exitCPUIdle(newIdleTimeoutTicks: new_timeout_ticks);
83 }
84}
85
86static void
87idle_timer_wrapper(void */*refCon*/, uint64_t *new_timeout_ticks)
88{
89 gPMGR->updateCPUIdle(newIdleTimeoutTicks: new_timeout_ticks);
90}
91
92static OSDictionary *
93matching_dict_for_cpu_id(unsigned int cpu_id)
94{
95 // The cpu-id property in EDT doesn't necessarily match the dynamically
96 // assigned logical ID in XNU, so look up the cpu node by the physical
97 // (cluster/core) ID instead.
98 OSSymbolConstPtr cpuTypeSymbol = OSSymbol::withCString(cString: "cpu");
99 OSSymbolConstPtr cpuIdSymbol = OSSymbol::withCString(cString: "reg");
100 OSDataPtr cpuId = OSData::withValue(value: topology_info->cpus[cpu_id].phys_id);
101
102 OSDictionary *propMatch = OSDictionary::withCapacity(capacity: 4);
103 propMatch->setObject(aKey: gIODTTypeKey, anObject: cpuTypeSymbol);
104 propMatch->setObject(aKey: cpuIdSymbol, anObject: cpuId);
105
106 OSDictionary *matching = IOService::serviceMatching(className: "IOPlatformDevice");
107 matching->setObject(aKey: gIOPropertyMatchKey, anObject: propMatch);
108
109 propMatch->release();
110 cpuTypeSymbol->release();
111 cpuIdSymbol->release();
112 cpuId->release();
113
114 return matching;
115}
116
117static void
118register_aic_handlers(const ml_topology_cpu *cpu_info,
119 ipi_handler_t ipi_handler,
120 perfmon_interrupt_handler_func pmi_handler)
121{
122 OSDictionary *matching = matching_dict_for_cpu_id(cpu_id: cpu_info->cpu_id);
123 IOService *cpu = IOService::waitForMatchingService(matching, UINT64_MAX);
124 matching->release();
125
126 OSArray *irqs = (OSArray *) cpu->getProperty(aKey: gIOInterruptSpecifiersKey);
127 if (!irqs) {
128 panic("Error finding interrupts for CPU %d", cpu_info->cpu_id);
129 }
130
131 unsigned int irqcount = irqs->getCount();
132
133 if (irqcount == 3) {
134 // Legacy configuration, for !HAS_IPI chips (pre-Skye).
135 if (cpu->registerInterrupt(source: 0, NULL, handler: (IOInterruptAction)ipi_handler, NULL) != kIOReturnSuccess ||
136 cpu->enableInterrupt(source: 0) != kIOReturnSuccess ||
137 cpu->registerInterrupt(source: 2, NULL, handler: (IOInterruptAction)ipi_handler, NULL) != kIOReturnSuccess ||
138 cpu->enableInterrupt(source: 2) != kIOReturnSuccess) {
139 panic("Error registering IPIs");
140 }
141#if !defined(HAS_IPI)
142 // Ideally this should be decided by EDT, but first we need to update EDT
143 // to default to fast IPIs on modern platforms.
144 aic_ipis = true;
145#endif
146 }
147
148 // Conditional, because on Skye and later, we use an FIQ instead of an external IRQ.
149 if (pmi_handler && irqcount == 1) {
150 if (cpu->registerInterrupt(source: 1, NULL, handler: (IOInterruptAction)(void (*)(void))pmi_handler, NULL) != kIOReturnSuccess ||
151 cpu->enableInterrupt(source: 1) != kIOReturnSuccess) {
152 panic("Error registering PMI");
153 }
154 }
155}
156
157static void
158cpu_boot_thread(void */*unused0*/, wait_result_t /*unused1*/)
159{
160 OSDictionary *matching = IOService::serviceMatching(className: "IOPlatformExpert");
161 IOService::waitForMatchingService(matching, UINT64_MAX);
162 matching->release();
163
164 gCPUIC = new PassthruInterruptController;
165 if (!gCPUIC || !gCPUIC->init()) {
166 panic("Can't initialize PassthruInterruptController");
167 }
168 gAIC = static_cast<IOInterruptController *>(gCPUIC->waitForChildController());
169
170 ml_set_max_cpus(max_cpus: topology_info->max_cpu_id + 1);
171
172#if XNU_CLUSTER_POWER_DOWN
173 cluster_power_supported = true;
174 /*
175 * If a boot-arg is set that allows threads to be bound
176 * to a cpu or cluster, cluster_power_supported must
177 * default to false.
178 */
179#ifdef CONFIG_XNUPOST
180 uint64_t kernel_post = 0;
181 PE_parse_boot_argn("kernPOST", &kernel_post, sizeof(kernel_post));
182 if (kernel_post != 0) {
183 cluster_power_supported = false;
184 }
185#endif
186 if (PE_parse_boot_argn("enable_skstb", NULL, 0)) {
187 cluster_power_supported = false;
188 }
189 if (PE_parse_boot_argn("enable_skstsct", NULL, 0)) {
190 cluster_power_supported = false;
191 }
192#endif
193 PE_parse_boot_argn(arg_string: "cluster_power", arg_ptr: &cluster_power_supported, max_arg: sizeof(cluster_power_supported));
194
195 matching = IOService::serviceMatching(className: "IOPMGR");
196 gPMGR = OSDynamicCast(IOPMGR,
197 IOService::waitForMatchingService(matching, UINT64_MAX));
198 matching->release();
199
200 const size_t array_size = (topology_info->max_cpu_id + 1) * sizeof(*machProcessors);
201 machProcessors = static_cast<processor_t *>(zalloc_permanent(array_size, ZALIGN_PTR));
202
203 for (unsigned int cpu = 0; cpu < topology_info->num_cpus; cpu++) {
204 const ml_topology_cpu *cpu_info = &topology_info->cpus[cpu];
205 const unsigned int cpu_id = cpu_info->cpu_id;
206 ml_processor_info_t this_processor_info;
207 ipi_handler_t ipi_handler;
208 perfmon_interrupt_handler_func pmi_handler;
209
210 memset(s: &this_processor_info, c: 0, n: sizeof(this_processor_info));
211 this_processor_info.cpu_id = reinterpret_cast<cpu_id_t>(cpu_id);
212 this_processor_info.phys_id = cpu_info->phys_id;
213 this_processor_info.log_id = cpu_id;
214 this_processor_info.cluster_id = cpu_info->cluster_id;
215 this_processor_info.cluster_type = cpu_info->cluster_type;
216 this_processor_info.l2_cache_size = cpu_info->l2_cache_size;
217 this_processor_info.l2_cache_id = cpu_info->l2_cache_id;
218 this_processor_info.l3_cache_size = cpu_info->l3_cache_size;
219 this_processor_info.l3_cache_id = cpu_info->l3_cache_id;
220
221 gPMGR->initCPUIdle(info: &this_processor_info);
222 this_processor_info.processor_idle = &processor_idle_wrapper;
223 this_processor_info.idle_timer = &idle_timer_wrapper;
224
225 kern_return_t result = ml_processor_register(ml_processor_info: &this_processor_info,
226 processor: &machProcessors[cpu_id], ipi_handler: &ipi_handler, pmi_handler: &pmi_handler);
227 if (result == KERN_FAILURE) {
228 panic("ml_processor_register failed: %d", result);
229 }
230 register_aic_handlers(cpu_info, ipi_handler, pmi_handler);
231
232 if (processor_start(processor: machProcessors[cpu_id]) != KERN_SUCCESS) {
233 panic("processor_start failed");
234 }
235 }
236 ml_cpu_init_completed();
237 IOService::publishResource(key: gIOAllCPUInitializedKey, value: kOSBooleanTrue);
238}
239
240void
241IOCPUInitialize(void)
242{
243 topology_info = ml_get_topology_info();
244 boot_cpu = topology_info->boot_cpu->cpu_id;
245
246 for (unsigned int i = 0; i < topology_info->num_clusters; i++) {
247 bit_set(all_clusters_mask, topology_info->clusters[i].cluster_id);
248 }
249 // iBoot powers up every cluster (at least for now)
250 online_clusters_mask = all_clusters_mask;
251
252 thread_t thread;
253 kernel_thread_start(continuation: &cpu_boot_thread, NULL, new_thread: &thread);
254 thread_set_thread_name(th: thread, name: "cpu_boot_thread");
255 thread_deallocate(thread);
256}
257
258static unsigned int
259target_to_cpu_id(cpu_id_t in)
260{
261 return (unsigned int)(uintptr_t)in;
262}
263
264// Release a secondary CPU from reset. Runs from a different CPU (obviously).
265kern_return_t
266PE_cpu_start(cpu_id_t target,
267 vm_offset_t /*start_paddr*/, vm_offset_t /*arg_paddr*/)
268{
269 unsigned int cpu_id = target_to_cpu_id(in: target);
270
271 if (cpu_id != boot_cpu) {
272#if APPLEVIRTUALPLATFORM
273 /* When running virtualized, the reset vector address must be passed to PMGR explicitly */
274 extern unsigned int LowResetVectorBase;
275 gPMGR->enableCPUCore(cpu_id, entry_pa: ml_vtophys(vaddr: (vm_offset_t)&LowResetVectorBase));
276#else
277 gPMGR->enableCPUCore(cpu_id, 0);
278#endif
279 }
280 return KERN_SUCCESS;
281}
282
283// Initialize a CPU when it first comes up. Runs on the target CPU.
284// |bootb| is true on the initial boot, false on S2R resume.
285void
286PE_cpu_machine_init(cpu_id_t target, boolean_t bootb)
287{
288 unsigned int cpu_id = target_to_cpu_id(in: target);
289
290 if (!bootb && cpu_id == boot_cpu && ml_is_quiescing()) {
291 IOCPURunPlatformActiveActions();
292 }
293
294 ml_broadcast_cpu_event(event: CPU_BOOTED, cpu_or_cluster: cpu_id);
295
296 // Send myself an IPI to clear SIGPdisabled. Hang here if IPIs are broken.
297 // (Probably only works on the boot CPU.)
298 PE_cpu_signal(source: target, target);
299 while (ml_get_interrupts_enabled() && !ml_cpu_signal_is_enabled()) {
300 OSMemoryBarrier();
301 }
302}
303
304void
305PE_cpu_halt(cpu_id_t target)
306{
307 unsigned int cpu_id = target_to_cpu_id(in: target);
308 processor_exit(processor: machProcessors[cpu_id]);
309}
310
311void
312PE_cpu_signal(cpu_id_t /*source*/, cpu_id_t target)
313{
314 struct ml_topology_cpu *cpu = &topology_info->cpus[target_to_cpu_id(in: target)];
315 if (aic_ipis) {
316 gAIC->sendIPI(cpu_id: cpu->cpu_id, deferred: false);
317 } else {
318 ml_cpu_signal(cpu_id: cpu->phys_id);
319 }
320}
321
322void
323PE_cpu_signal_deferred(cpu_id_t /*source*/, cpu_id_t target)
324{
325 struct ml_topology_cpu *cpu = &topology_info->cpus[target_to_cpu_id(in: target)];
326 if (aic_ipis) {
327 gAIC->sendIPI(cpu_id: cpu->cpu_id, deferred: true);
328 } else {
329 ml_cpu_signal_deferred(cpu_id: cpu->phys_id);
330 }
331}
332
333void
334PE_cpu_signal_cancel(cpu_id_t /*source*/, cpu_id_t target)
335{
336 struct ml_topology_cpu *cpu = &topology_info->cpus[target_to_cpu_id(in: target)];
337 if (aic_ipis) {
338 gAIC->cancelDeferredIPI(cpu_id: cpu->cpu_id);
339 } else {
340 ml_cpu_signal_retract(cpu_id: cpu->phys_id);
341 }
342}
343
344// Brings down one CPU core for S2R. Runs on the target CPU.
345void
346PE_cpu_machine_quiesce(cpu_id_t target)
347{
348 unsigned int cpu_id = target_to_cpu_id(in: target);
349
350 if (cpu_id == boot_cpu) {
351 IOCPURunPlatformQuiesceActions();
352 } else {
353 gPMGR->disableCPUCore(cpu_id);
354 }
355
356 ml_broadcast_cpu_event(event: CPU_DOWN, cpu_or_cluster: cpu_id);
357 ml_arm_sleep();
358}
359
360static bool
361is_cluster_powering_down(int cpu_id)
362{
363 // Don't kill the cluster power if any other CPUs in this cluster are still awake
364 unsigned int target_cluster_id = topology_info->cpus[cpu_id].cluster_id;
365 for (int i = 0; i < topology_info->num_cpus; i++) {
366 if (topology_info->cpus[i].cluster_id == target_cluster_id &&
367 cpu_id != i &&
368 bit_test(cpu_power_state_mask, i)) {
369 return false;
370 }
371 }
372 return true;
373}
374
375// Takes one secondary CPU core offline at runtime. Runs on the target CPU.
376// Returns true if the platform code should go into deep sleep WFI, false otherwise.
377bool
378PE_cpu_down(cpu_id_t target)
379{
380 unsigned int cpu_id = target_to_cpu_id(in: target);
381 assert(cpu_id != boot_cpu);
382 gPMGR->disableCPUCore(cpu_id);
383 ml_broadcast_cpu_event(event: CPU_DOWN, cpu_or_cluster: cpu_id);
384 return cluster_power_supported && is_cluster_powering_down(cpu_id);
385}
386
387void
388PE_handle_ext_interrupt(void)
389{
390 gCPUIC->externalInterrupt();
391}
392
393void
394PE_cpu_power_disable(int cpu_id)
395{
396 bit_clear(cpu_power_state_mask, cpu_id);
397 if (!cluster_power_supported || cpu_id == boot_cpu) {
398 return;
399 }
400
401 // Don't kill the cluster power if any other CPUs in this cluster are still awake
402 unsigned int target_cluster_id = topology_info->cpus[cpu_id].cluster_id;
403 if (!is_cluster_powering_down(cpu_id)) {
404 return;
405 }
406
407 if (processor_should_kprintf(processor: machProcessors[cpu_id], starting: false)) {
408 kprintf(fmt: "%s>turning off power to cluster %d\n", __FUNCTION__, target_cluster_id);
409 }
410 ml_broadcast_cpu_event(event: CLUSTER_EXIT_REQUESTED, cpu_or_cluster: target_cluster_id);
411 bit_clear(online_clusters_mask, target_cluster_id);
412 gPMGR->disableCPUCluster(cluster_id: target_cluster_id);
413}
414
415bool
416PE_cpu_power_check_kdp(int cpu_id)
417{
418 if (!cluster_power_supported) {
419 return true;
420 }
421
422 unsigned int cluster_id = topology_info->cpus[cpu_id].cluster_id;
423 return bit_test(online_clusters_mask, cluster_id);
424}
425
426void
427PE_cpu_power_enable(int cpu_id)
428{
429 bit_set(cpu_power_state_mask, cpu_id);
430 if (!cluster_power_supported || cpu_id == boot_cpu) {
431 return;
432 }
433
434 unsigned int cluster_id = topology_info->cpus[cpu_id].cluster_id;
435 if (!bit_test(online_clusters_mask, cluster_id)) {
436 if (processor_should_kprintf(processor: machProcessors[cpu_id], starting: true)) {
437 kprintf(fmt: "%s>turning on power to cluster %d\n", __FUNCTION__, cluster_id);
438 }
439 gPMGR->enableCPUCluster(cluster_id);
440 bit_set(online_clusters_mask, cluster_id);
441 ml_broadcast_cpu_event(event: CLUSTER_ACTIVE, cpu_or_cluster: cluster_id);
442 }
443}
444
445void
446IOCPUSleepKernel(void)
447{
448 IOPMrootDomain *rootDomain = IOService::getPMRootDomain();
449 unsigned int i;
450
451 printf("IOCPUSleepKernel enter\n");
452 sched_override_available_cores_for_sleep();
453
454 rootDomain->tracePoint( point: kIOPMTracePointSleepPlatformActions );
455 IOPlatformActionsPreSleep();
456 rootDomain->tracePoint( point: kIOPMTracePointSleepCPUs );
457
458 integer_t old_pri;
459 thread_t self = current_thread();
460
461 /*
462 * We need to boost this thread's priority to the maximum kernel priority to
463 * ensure we can urgently preempt ANY thread currently executing on the
464 * target CPU. Note that realtime threads have their own mechanism to eventually
465 * demote their priority below MAXPRI_KERNEL if they hog the CPU for too long.
466 */
467 old_pri = thread_kern_get_pri(thr: self);
468 thread_kern_set_pri(thr: self, pri: thread_kern_get_kernel_maxpri());
469
470 // Sleep the non-boot CPUs.
471 ml_set_is_quiescing(true);
472 for (i = 0; i < topology_info->num_cpus; i++) {
473 unsigned int cpu_id = topology_info->cpus[i].cpu_id;
474 if (cpu_id != boot_cpu) {
475 processor_exit(processor: machProcessors[cpu_id]);
476 }
477 }
478
479 console_suspend();
480
481 rootDomain->tracePoint( point: kIOPMTracePointSleepPlatformDriver );
482 rootDomain->stop_watchdog_timer();
483
484 /*
485 * Now sleep the boot CPU, including calling the kQueueQuiesce actions.
486 * The system sleeps here.
487 */
488 processor_exit(processor: machProcessors[boot_cpu]);
489
490 /*
491 * The system is now coming back from sleep on the boot CPU.
492 * The kQueueActive actions have already been called.
493 *
494 * The reconfig engine is programmed to power up all clusters on S2R resume.
495 */
496 online_clusters_mask = all_clusters_mask;
497
498 /*
499 * processor_start() never gets called for the boot CPU, so it needs to
500 * be explicitly marked as online here.
501 */
502 PE_cpu_power_enable(cpu_id: boot_cpu);
503
504 ml_set_is_quiescing(false);
505
506 rootDomain->start_watchdog_timer();
507
508 console_resume();
509
510 rootDomain->tracePoint( point: kIOPMTracePointWakeCPUs );
511
512 for (i = 0; i < topology_info->num_cpus; i++) {
513 unsigned int cpu_id = topology_info->cpus[i].cpu_id;
514 if (cpu_id != boot_cpu) {
515 processor_start(processor: machProcessors[cpu_id]);
516 }
517 }
518
519 rootDomain->tracePoint( point: kIOPMTracePointWakePlatformActions );
520 IOPlatformActionsPostResume();
521
522 sched_restore_available_cores_after_sleep();
523
524 thread_kern_set_pri(thr: self, pri: old_pri);
525 printf("IOCPUSleepKernel exit\n");
526}
527
528#endif /* USE_APPLEARMSMP */
529