1/*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * @APPLE_FREE_COPYRIGHT@
33 */
34/*
35 * File: arm/rtclock.c
36 * Purpose: Routines for handling the machine dependent
37 * real-time clock.
38 */
39
40#include <mach/mach_types.h>
41
42#include <kern/clock.h>
43#include <kern/thread.h>
44#include <kern/macro_help.h>
45#include <kern/spl.h>
46#include <kern/timer_queue.h>
47
48#include <kern/host_notify.h>
49
50#include <machine/commpage.h>
51#include <machine/machine_routines.h>
52#include <arm/exception.h>
53#include <arm/cpu_data_internal.h>
54#if __arm64__
55#include <arm64/proc_reg.h>
56#elif __arm__
57#include <arm/proc_reg.h>
58#else
59#error Unsupported arch
60#endif
61#include <arm/rtclock.h>
62
63#include <IOKit/IOPlatformExpert.h>
64#include <libkern/OSAtomic.h>
65
66#include <sys/kdebug.h>
67
68#define MAX_TIMEBASE_TRIES 10
69
70int rtclock_init(void);
71
72static int
73deadline_to_decrementer(uint64_t deadline,
74 uint64_t now);
75static void
76timebase_callback(struct timebase_freq_t * freq);
77
78#if DEVELOPMENT || DEBUG
79uint32_t absolute_time_validation = 1;
80#endif
81
82/*
83 * Configure the real-time clock device at boot
84 */
85void
86rtclock_early_init(void)
87{
88 PE_register_timebase_callback(timebase_callback);
89#if DEVELOPMENT || DEBUG
90 uint32_t tmp_mv = 1;
91 if (kern_feature_override(KF_MATV_OVRD)) {
92 absolute_time_validation = 0;
93 }
94 if (PE_parse_boot_argn("timebase_validation", &tmp_mv, sizeof(tmp_mv))) {
95 if (tmp_mv == 0) {
96 absolute_time_validation = 0;
97 }
98 }
99#endif
100}
101
102static void
103timebase_callback(struct timebase_freq_t * freq)
104{
105 unsigned long numer, denom;
106 uint64_t t64_1, t64_2;
107 uint32_t divisor;
108
109 if (freq->timebase_den < 1 || freq->timebase_den > 4 ||
110 freq->timebase_num < freq->timebase_den)
111 panic("rtclock timebase_callback: invalid constant %ld / %ld",
112 freq->timebase_num, freq->timebase_den);
113
114 denom = freq->timebase_num;
115 numer = freq->timebase_den * NSEC_PER_SEC;
116 // reduce by the greatest common denominator to minimize overflow
117 if (numer > denom) {
118 t64_1 = numer;
119 t64_2 = denom;
120 } else {
121 t64_1 = denom;
122 t64_2 = numer;
123 }
124 while (t64_2 != 0) {
125 uint64_t temp = t64_2;
126 t64_2 = t64_1 % t64_2;
127 t64_1 = temp;
128 }
129 numer /= t64_1;
130 denom /= t64_1;
131
132 rtclock_timebase_const.numer = (uint32_t)numer;
133 rtclock_timebase_const.denom = (uint32_t)denom;
134 divisor = (uint32_t)(freq->timebase_num / freq->timebase_den);
135
136 rtclock_sec_divisor = divisor;
137 rtclock_usec_divisor = divisor / USEC_PER_SEC;
138}
139
140/*
141 * Initialize the system clock device for the current cpu
142 */
143int
144rtclock_init(void)
145{
146 uint64_t abstime;
147 cpu_data_t * cdp;
148
149 clock_timebase_init();
150 ml_init_lock_timeout();
151
152 cdp = getCpuDatap();
153
154 abstime = mach_absolute_time();
155 cdp->rtcPop = EndOfAllTime; /* Init Pop time */
156 timer_resync_deadlines(); /* Start the timers going */
157
158 return (1);
159}
160
161uint64_t
162mach_absolute_time(void)
163{
164#if DEVELOPMENT || DEBUG
165 if (__improbable(absolute_time_validation == 1)) {
166 static volatile uint64_t s_last_absolute_time = 0;
167 uint64_t new_absolute_time, old_absolute_time;
168 int attempts = 0;
169
170 /* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time
171 * completes before the timebase read. Were the load to complete after the
172 * timebase read, there would be a window for another CPU to update
173 * s_last_absolute_time and leave us in an inconsistent state. Consider the
174 * following interleaving:
175 *
176 * Let s_last_absolute_time = t0
177 * CPU0: Read timebase at t1
178 * CPU1: Read timebase at t2
179 * CPU1: Update s_last_absolute_time to t2
180 * CPU0: Load completes
181 * CPU0: Update s_last_absolute_time to t1
182 *
183 * This would cause the assertion to fail even though time did not go
184 * backwards. Thus, we use a dsb to guarantee completion of the load before
185 * the timebase read.
186 */
187 do {
188 attempts++;
189 old_absolute_time = s_last_absolute_time;
190
191#if __arm64__
192 __asm__ volatile("dsb ld" ::: "memory");
193#else
194 OSSynchronizeIO(); // See osfmk/arm64/rtclock.c
195#endif
196
197 new_absolute_time = ml_get_timebase();
198 } while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time));
199
200 if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) {
201 panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)\n",
202 new_absolute_time, old_absolute_time);
203 }
204 return new_absolute_time;
205 } else {
206 return ml_get_timebase();
207 }
208#else
209 return ml_get_timebase();
210#endif
211}
212
213uint64_t
214mach_approximate_time(void)
215{
216#if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__
217 /* Hardware supports a fast timestamp, so grab it without asserting monotonicity */
218 return ml_get_timebase();
219#else
220 processor_t processor;
221 uint64_t approx_time;
222
223 disable_preemption();
224 processor = current_processor();
225 approx_time = processor->last_dispatch;
226 enable_preemption();
227
228 return approx_time;
229#endif
230}
231
232void
233clock_get_system_microtime(clock_sec_t * secs,
234 clock_usec_t * microsecs)
235{
236 absolutetime_to_microtime(mach_absolute_time(), secs, microsecs);
237}
238
239void
240clock_get_system_nanotime(clock_sec_t * secs,
241 clock_nsec_t * nanosecs)
242{
243 uint64_t abstime;
244 uint64_t t64;
245
246 abstime = mach_absolute_time();
247 *secs = (t64 = abstime / rtclock_sec_divisor);
248 abstime -= (t64 * rtclock_sec_divisor);
249
250 *nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor);
251}
252
253void
254clock_gettimeofday_set_commpage(uint64_t abstime,
255 uint64_t sec,
256 uint64_t frac,
257 uint64_t scale,
258 uint64_t tick_per_sec)
259{
260 commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec);
261}
262
263void
264clock_timebase_info(mach_timebase_info_t info)
265{
266 *info = rtclock_timebase_const;
267}
268
269/*
270 * Real-time clock device interrupt.
271 */
272void
273rtclock_intr(__unused unsigned int is_user_context)
274{
275 uint64_t abstime;
276 cpu_data_t * cdp;
277 struct arm_saved_state * regs;
278 unsigned int user_mode;
279 uintptr_t pc;
280
281 cdp = getCpuDatap();
282
283 cdp->cpu_stat.timer_cnt++;
284 cdp->cpu_stat.timer_cnt_wake++;
285 SCHED_STATS_TIMER_POP(current_processor());
286
287 assert(!ml_get_interrupts_enabled());
288
289 abstime = mach_absolute_time();
290
291 if (cdp->cpu_idle_pop != 0x0ULL) {
292 if (( cdp->rtcPop-abstime) < cdp->cpu_idle_latency) {
293 cdp->cpu_idle_pop = 0x0ULL;
294 while (abstime < cdp->rtcPop)
295 abstime = mach_absolute_time();
296 } else {
297 ClearIdlePop(FALSE);
298 }
299 }
300
301 if ((regs = cdp->cpu_int_state)) {
302 pc = get_saved_state_pc(regs);
303
304#if __arm64__
305 user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs));
306#else
307 user_mode = (regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE;
308#endif
309 } else {
310 pc = 0;
311 user_mode = 0;
312 }
313 if (abstime >= cdp->rtcPop) {
314 /* Log the interrupt service latency (-ve value expected by tool) */
315 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
316 MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE,
317 -(abstime - cdp->rtcPop),
318 user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode, 0, 0);
319 }
320
321 /* call the generic etimer */
322 timer_intr(user_mode, pc);
323}
324
325static int
326deadline_to_decrementer(uint64_t deadline,
327 uint64_t now)
328{
329 uint64_t delt;
330
331 if (deadline <= now)
332 return DECREMENTER_MIN;
333 else {
334 delt = deadline - now;
335
336 return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN);
337 }
338}
339
340/*
341 * Request a decrementer pop
342 */
343int
344setPop(uint64_t time)
345{
346 int delay_time;
347 uint64_t current_time;
348 cpu_data_t * cdp;
349
350 cdp = getCpuDatap();
351 current_time = mach_absolute_time();
352
353 delay_time = deadline_to_decrementer(time, current_time);
354 cdp->rtcPop = delay_time + current_time;
355
356 ml_set_decrementer((uint32_t) delay_time);
357
358 return (delay_time);
359}
360
361/*
362 * Request decrementer Idle Pop. Return true if set
363 */
364boolean_t
365SetIdlePop(void)
366{
367 int delay_time;
368 uint64_t time;
369 uint64_t current_time;
370 cpu_data_t * cdp;
371
372 cdp = getCpuDatap();
373 current_time = mach_absolute_time();
374
375 if (((cdp->rtcPop < current_time) ||
376 (cdp->rtcPop - current_time) < cdp->cpu_idle_latency))
377 return FALSE;
378
379 time = cdp->rtcPop - cdp->cpu_idle_latency;
380
381 delay_time = deadline_to_decrementer(time, current_time);
382 cdp->cpu_idle_pop = delay_time + current_time;
383 ml_set_decrementer((uint32_t) delay_time);
384
385 return TRUE;
386}
387
388/*
389 * Clear decrementer Idle Pop
390 */
391void
392ClearIdlePop(
393 boolean_t wfi)
394{
395#if !__arm64__
396#pragma unused(wfi)
397#endif
398 cpu_data_t * cdp;
399
400 cdp = getCpuDatap();
401 cdp->cpu_idle_pop = 0x0ULL;
402
403#if __arm64__
404 /*
405 * Don't update the HW timer if there's a pending
406 * interrupt (we can lose interrupt assertion);
407 * we want to take the interrupt right now and update
408 * the deadline from the handler).
409 *
410 * ARM64_TODO: consider this more carefully.
411 */
412 if (!(wfi && ml_get_timer_pending()))
413#endif
414 {
415 setPop(cdp->rtcPop);
416 }
417}
418
419void
420absolutetime_to_microtime(uint64_t abstime,
421 clock_sec_t * secs,
422 clock_usec_t * microsecs)
423{
424 uint64_t t64;
425
426 *secs = t64 = abstime / rtclock_sec_divisor;
427 abstime -= (t64 * rtclock_sec_divisor);
428
429 *microsecs = (uint32_t)(abstime / rtclock_usec_divisor);
430}
431
432void
433absolutetime_to_nanoseconds(uint64_t abstime,
434 uint64_t * result)
435{
436 uint64_t t64;
437
438 *result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC;
439 abstime -= (t64 * rtclock_sec_divisor);
440 *result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor;
441}
442
443void
444nanoseconds_to_absolutetime(uint64_t nanosecs,
445 uint64_t * result)
446{
447 uint64_t t64;
448
449 *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
450 nanosecs -= (t64 * NSEC_PER_SEC);
451 *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
452}
453
454void
455nanotime_to_absolutetime(clock_sec_t secs,
456 clock_nsec_t nanosecs,
457 uint64_t * result)
458{
459 *result = ((uint64_t) secs * rtclock_sec_divisor) +
460 ((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
461}
462
463void
464clock_interval_to_absolutetime_interval(uint32_t interval,
465 uint32_t scale_factor,
466 uint64_t * result)
467{
468 uint64_t nanosecs = (uint64_t) interval * scale_factor;
469 uint64_t t64;
470
471 *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
472 nanosecs -= (t64 * NSEC_PER_SEC);
473 *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
474}
475
476void
477machine_delay_until(uint64_t interval,
478 uint64_t deadline)
479{
480#pragma unused(interval)
481 uint64_t now;
482
483 do {
484#if __ARM_ENABLE_WFE_
485#if __arm64__
486 if (arm64_wfe_allowed())
487#endif /* __arm64__ */
488 {
489 __builtin_arm_wfe();
490 }
491#endif /* __ARM_ENABLE_WFE_ */
492
493 now = mach_absolute_time();
494 } while (now < deadline);
495}
496