1/*
2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * @APPLE_FREE_COPYRIGHT@
33 */
34/*
35 * File: arm/rtclock.c
36 * Purpose: Routines for handling the machine dependent
37 * real-time clock.
38 */
39
40#include <mach/mach_types.h>
41
42#include <kern/clock.h>
43#include <kern/thread.h>
44#include <kern/macro_help.h>
45#include <kern/spl.h>
46#include <kern/timer_queue.h>
47
48#include <kern/host_notify.h>
49
50#include <machine/commpage.h>
51#include <machine/machine_routines.h>
52#include <machine/config.h>
53#include <arm/exception.h>
54#include <arm/cpu_data_internal.h>
55#if __arm64__
56#include <arm64/proc_reg.h>
57#else
58#error Unsupported arch
59#endif
60#include <arm/rtclock.h>
61
62#include <IOKit/IOPlatformExpert.h>
63#include <libkern/OSAtomic.h>
64
65#include <sys/kdebug.h>
66
67#define MAX_TIMEBASE_TRIES 10
68
69int rtclock_init(void);
70
71static int
72deadline_to_decrementer(uint64_t deadline,
73 uint64_t now);
74static void
75timebase_callback(struct timebase_freq_t * freq);
76
77#if DEVELOPMENT || DEBUG
78uint32_t timebase_validation = 0;
79#endif
80
81/*
82 * Configure the real-time clock device at boot
83 */
84void
85rtclock_early_init(void)
86{
87 PE_register_timebase_callback(callback: timebase_callback);
88#if DEVELOPMENT || DEBUG
89 uint32_t tmp_mv = 1;
90
91#if defined(APPLE_ARM64_ARCH_FAMILY)
92 /* Enable MAT validation on A0 hardware by default. */
93 timebase_validation = ml_get_topology_info()->chip_revision == CPU_VERSION_A0;
94#endif
95
96 if (kern_feature_override(KF_MATV_OVRD)) {
97 timebase_validation = 0;
98 }
99 if (PE_parse_boot_argn("timebase_validation", &tmp_mv, sizeof(tmp_mv))) {
100 timebase_validation = tmp_mv;
101 }
102#endif
103}
104
105static void
106timebase_callback(struct timebase_freq_t * freq)
107{
108 unsigned long numer, denom;
109 uint64_t t64_1, t64_2;
110 uint32_t divisor;
111
112 if (freq->timebase_den < 1 || freq->timebase_den > 4 ||
113 freq->timebase_num < freq->timebase_den) {
114 panic("rtclock timebase_callback: invalid constant %ld / %ld",
115 freq->timebase_num, freq->timebase_den);
116 }
117
118 denom = freq->timebase_num;
119 numer = freq->timebase_den * NSEC_PER_SEC;
120 // reduce by the greatest common denominator to minimize overflow
121 if (numer > denom) {
122 t64_1 = numer;
123 t64_2 = denom;
124 } else {
125 t64_1 = denom;
126 t64_2 = numer;
127 }
128 while (t64_2 != 0) {
129 uint64_t temp = t64_2;
130 t64_2 = t64_1 % t64_2;
131 t64_1 = temp;
132 }
133 numer /= t64_1;
134 denom /= t64_1;
135
136 rtclock_timebase_const.numer = (uint32_t)numer;
137 rtclock_timebase_const.denom = (uint32_t)denom;
138 divisor = (uint32_t)(freq->timebase_num / freq->timebase_den);
139
140 rtclock_sec_divisor = divisor;
141 rtclock_usec_divisor = divisor / USEC_PER_SEC;
142}
143
144/*
145 * Initialize the system clock device for the current cpu
146 */
147int
148rtclock_init(void)
149{
150 uint64_t abstime;
151 cpu_data_t * cdp;
152
153 clock_timebase_init();
154
155 cdp = getCpuDatap();
156
157 abstime = mach_absolute_time();
158 cdp->rtcPop = EndOfAllTime; /* Init Pop time */
159 timer_resync_deadlines(); /* Start the timers going */
160
161 return 1;
162}
163
164
165uint64_t
166mach_absolute_time(void)
167{
168#if DEVELOPMENT || DEBUG
169 if (__improbable(timebase_validation)) {
170#if __ARM_ARCH_8_6__ || HAS_ACNTVCT
171 static _Atomic uint64_t s_last_absolute_time = 1;
172
173 uint64_t old_absolute_time = os_atomic_load(&s_last_absolute_time, relaxed);
174
175 /*
176 * Because this timebase read is nonspeculative, it cannot begin reading
177 * the timebase value until after the load of the old value completes.
178 */
179
180 if (old_absolute_time == 0) {
181 timebase_validation = 0; // we know it's bad, now prevent nested panics
182 panic("old_absolute_time was 0");
183 }
184
185 uint64_t new_absolute_time = ml_get_timebase();
186
187 if (old_absolute_time > new_absolute_time) {
188 timebase_validation = 0; // prevent nested panics
189 panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx)",
190 new_absolute_time, old_absolute_time);
191 }
192
193 if (old_absolute_time < new_absolute_time) {
194 /* read again, to pretest the atomic max */
195 uint64_t pretest_absolute_time = os_atomic_load(&s_last_absolute_time, relaxed);
196 if (pretest_absolute_time < new_absolute_time) {
197 uint64_t fresh_last_absolute_time = os_atomic_max_orig(&s_last_absolute_time, new_absolute_time, relaxed);
198
199 if (fresh_last_absolute_time != pretest_absolute_time) {
200 /*
201 * Someone else published a newer time after we loaded s_last_absolute_time.
202 * Enforce that our timebase is not behind this new one.
203 * We can't compare it with our previous timebase read, as it is too old.
204 */
205
206 uint64_t newest_absolute_time = ml_get_timebase();
207
208 if (fresh_last_absolute_time > newest_absolute_time) {
209 timebase_validation = 0; // prevent nested panics
210 panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old values 0x%llx, 0x%llx, 0x%llx, 0x%llx)\n",
211 newest_absolute_time, fresh_last_absolute_time, pretest_absolute_time, old_absolute_time, new_absolute_time);
212 }
213 }
214 }
215 }
216
217 return new_absolute_time;
218
219#else /* !(__ARM_ARCH_8_6__ || HAS_ACNTVCT) */
220 static volatile uint64_t s_last_absolute_time = 0;
221 uint64_t new_absolute_time, old_absolute_time;
222 int attempts = 0;
223
224 /* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time
225 * completes before the timebase read. Were the load to complete after the
226 * timebase read, there would be a window for another CPU to update
227 * s_last_absolute_time and leave us in an inconsistent state. Consider the
228 * following interleaving:
229 *
230 * Let s_last_absolute_time = t0
231 * CPU0: Read timebase at t1
232 * CPU1: Read timebase at t2
233 * CPU1: Update s_last_absolute_time to t2
234 * CPU0: Load completes
235 * CPU0: Update s_last_absolute_time to t1
236 *
237 * This would cause the assertion to fail even though time did not go
238 * backwards. Thus, we use a dsb to guarantee completion of the load before
239 * the timebase read.
240 */
241 do {
242 attempts++;
243 old_absolute_time = s_last_absolute_time;
244
245 __builtin_arm_dsb(DSB_ISHLD);
246
247 new_absolute_time = ml_get_timebase();
248 } while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time));
249
250 if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) {
251 timebase_validation = 0; // we know it's bad, now prevent nested panics
252 panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)",
253 new_absolute_time, old_absolute_time);
254 }
255 return new_absolute_time;
256#endif /* __ARM_ARCH_8_6__ || HAS_ACNTVCT */
257 }
258#endif /* DEVELOPMENT || DEBUG */
259
260 return ml_get_timebase();
261}
262
263uint64_t
264mach_approximate_time(void)
265{
266#if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__
267 /* Hardware supports a fast timestamp, so grab it without asserting monotonicity */
268 return ml_get_timebase();
269#else
270 processor_t processor;
271 uint64_t approx_time;
272
273 disable_preemption();
274 processor = current_processor();
275 approx_time = processor->last_dispatch;
276 enable_preemption();
277
278 return approx_time;
279#endif
280}
281
282void
283clock_get_system_microtime(clock_sec_t * secs,
284 clock_usec_t * microsecs)
285{
286 absolutetime_to_microtime(abstime: mach_absolute_time(), secs, microsecs);
287}
288
289void
290clock_get_system_nanotime(clock_sec_t * secs,
291 clock_nsec_t * nanosecs)
292{
293 uint64_t abstime;
294 uint64_t t64;
295
296 abstime = mach_absolute_time();
297 *secs = (t64 = abstime / rtclock_sec_divisor);
298 abstime -= (t64 * rtclock_sec_divisor);
299
300 *nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor);
301}
302
303void
304clock_gettimeofday_set_commpage(uint64_t abstime,
305 uint64_t sec,
306 uint64_t frac,
307 uint64_t scale,
308 uint64_t tick_per_sec)
309{
310 commpage_set_timestamp(tbr: abstime, secs: sec, frac, scale, tick_per_sec);
311}
312
313void
314clock_timebase_info(mach_timebase_info_t info)
315{
316 *info = rtclock_timebase_const;
317}
318
319/*
320 * Real-time clock device interrupt.
321 */
322void
323rtclock_intr(__unused unsigned int is_user_context)
324{
325 uint64_t abstime;
326 cpu_data_t * cdp;
327 struct arm_saved_state * regs;
328 unsigned int user_mode;
329 uintptr_t pc;
330
331 cdp = getCpuDatap();
332
333 cdp->cpu_stat.timer_cnt++;
334 SCHED_STATS_INC(timer_pop_count);
335
336 assert(!ml_get_interrupts_enabled());
337
338 abstime = mach_absolute_time();
339
340 if (cdp->cpu_idle_pop != 0x0ULL) {
341 if ((cdp->rtcPop - abstime) < cdp->cpu_idle_latency) {
342 cdp->cpu_idle_pop = 0x0ULL;
343 while (abstime < cdp->rtcPop) {
344 abstime = mach_absolute_time();
345 }
346 } else {
347 ClearIdlePop(FALSE);
348 }
349 }
350
351 if ((regs = cdp->cpu_int_state)) {
352 pc = get_saved_state_pc(iss: regs);
353
354#if __arm64__
355 user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs));
356#endif
357 } else {
358 pc = 0;
359 user_mode = 0;
360 }
361 if (abstime >= cdp->rtcPop) {
362 /* Log the interrupt service latency (-ve value expected by tool) */
363 KDBG_RELEASE(DECR_TRAP_LATENCY | DBG_FUNC_NONE,
364 -(abstime - cdp->rtcPop),
365 user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode);
366 }
367
368 /* call the generic etimer */
369 timer_intr(inuser: user_mode, iaddr: pc);
370}
371
372static int
373deadline_to_decrementer(uint64_t deadline,
374 uint64_t now)
375{
376 uint64_t delt;
377
378 if (deadline <= now) {
379 return DECREMENTER_MIN;
380 } else {
381 delt = deadline - now;
382
383 return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN);
384 }
385}
386
387/*
388 * Request a decrementer pop
389 */
390int
391setPop(uint64_t time)
392{
393 int delay_time;
394 uint64_t current_time;
395 cpu_data_t * cdp;
396
397 cdp = getCpuDatap();
398 current_time = mach_absolute_time();
399
400 delay_time = deadline_to_decrementer(deadline: time, now: current_time);
401 cdp->rtcPop = delay_time + current_time;
402
403 ml_set_decrementer(dec_value: (uint32_t) delay_time);
404
405 return delay_time;
406}
407
408/*
409 * Request decrementer Idle Pop. Return true if set
410 */
411boolean_t
412SetIdlePop(void)
413{
414 int delay_time;
415 uint64_t time;
416 uint64_t current_time;
417 cpu_data_t * cdp;
418
419 cdp = getCpuDatap();
420 current_time = mach_absolute_time();
421
422 if (((cdp->rtcPop < current_time) ||
423 (cdp->rtcPop - current_time) < cdp->cpu_idle_latency)) {
424 return FALSE;
425 }
426
427 time = cdp->rtcPop - cdp->cpu_idle_latency;
428
429 delay_time = deadline_to_decrementer(deadline: time, now: current_time);
430 cdp->cpu_idle_pop = delay_time + current_time;
431 ml_set_decrementer(dec_value: (uint32_t) delay_time);
432
433 return TRUE;
434}
435
436/*
437 * Clear decrementer Idle Pop
438 */
439void
440ClearIdlePop(
441 boolean_t wfi)
442{
443 cpu_data_t * cdp;
444
445 cdp = getCpuDatap();
446 cdp->cpu_idle_pop = 0x0ULL;
447
448 /*
449 * Don't update the HW timer if there's a pending
450 * interrupt (we can lose interrupt assertion);
451 * we want to take the interrupt right now and update
452 * the deadline from the handler).
453 *
454 * ARM64_TODO: consider this more carefully.
455 */
456 if (!(wfi && ml_get_timer_pending())) {
457 setPop(cdp->rtcPop);
458 }
459}
460
461void
462absolutetime_to_microtime(uint64_t abstime,
463 clock_sec_t * secs,
464 clock_usec_t * microsecs)
465{
466 uint64_t t64;
467
468 *secs = t64 = abstime / rtclock_sec_divisor;
469 abstime -= (t64 * rtclock_sec_divisor);
470
471 *microsecs = (uint32_t)(abstime / rtclock_usec_divisor);
472}
473
474void
475absolutetime_to_nanoseconds(uint64_t abstime,
476 uint64_t * result)
477{
478 uint64_t t64;
479
480 *result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC;
481 abstime -= (t64 * rtclock_sec_divisor);
482 *result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor;
483}
484
485void
486nanoseconds_to_absolutetime(uint64_t nanosecs,
487 uint64_t * result)
488{
489 uint64_t t64;
490
491 *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
492 nanosecs -= (t64 * NSEC_PER_SEC);
493 *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
494}
495
496void
497nanotime_to_absolutetime(clock_sec_t secs,
498 clock_nsec_t nanosecs,
499 uint64_t * result)
500{
501 *result = ((uint64_t) secs * rtclock_sec_divisor) +
502 ((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
503}
504
505void
506clock_interval_to_absolutetime_interval(uint32_t interval,
507 uint32_t scale_factor,
508 uint64_t * result)
509{
510 uint64_t nanosecs = (uint64_t) interval * scale_factor;
511 uint64_t t64;
512
513 *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
514 nanosecs -= (t64 * NSEC_PER_SEC);
515 *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
516}
517
518void
519machine_delay_until(uint64_t interval,
520 uint64_t deadline)
521{
522#pragma unused(interval)
523 uint64_t now;
524
525 do {
526 __builtin_arm_wfe();
527 now = mach_absolute_time();
528 } while (now < deadline);
529}
530