1 | /* |
2 | * Copyright (c) 2007-2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * @APPLE_FREE_COPYRIGHT@ |
33 | */ |
34 | /* |
35 | * File: arm/rtclock.c |
36 | * Purpose: Routines for handling the machine dependent |
37 | * real-time clock. |
38 | */ |
39 | |
40 | #include <mach/mach_types.h> |
41 | |
42 | #include <kern/clock.h> |
43 | #include <kern/thread.h> |
44 | #include <kern/macro_help.h> |
45 | #include <kern/spl.h> |
46 | #include <kern/timer_queue.h> |
47 | |
48 | #include <kern/host_notify.h> |
49 | |
50 | #include <machine/commpage.h> |
51 | #include <machine/machine_routines.h> |
52 | #include <machine/config.h> |
53 | #include <arm/exception.h> |
54 | #include <arm/cpu_data_internal.h> |
55 | #if __arm64__ |
56 | #include <arm64/proc_reg.h> |
57 | #else |
58 | #error Unsupported arch |
59 | #endif |
60 | #include <arm/rtclock.h> |
61 | |
62 | #include <IOKit/IOPlatformExpert.h> |
63 | #include <libkern/OSAtomic.h> |
64 | |
65 | #include <sys/kdebug.h> |
66 | |
67 | #define MAX_TIMEBASE_TRIES 10 |
68 | |
69 | int rtclock_init(void); |
70 | |
71 | static int |
72 | deadline_to_decrementer(uint64_t deadline, |
73 | uint64_t now); |
74 | static void |
75 | timebase_callback(struct timebase_freq_t * freq); |
76 | |
77 | #if DEVELOPMENT || DEBUG |
78 | uint32_t timebase_validation = 0; |
79 | #endif |
80 | |
81 | /* |
82 | * Configure the real-time clock device at boot |
83 | */ |
84 | void |
85 | rtclock_early_init(void) |
86 | { |
87 | PE_register_timebase_callback(callback: timebase_callback); |
88 | #if DEVELOPMENT || DEBUG |
89 | uint32_t tmp_mv = 1; |
90 | |
91 | #if defined(APPLE_ARM64_ARCH_FAMILY) |
92 | /* Enable MAT validation on A0 hardware by default. */ |
93 | timebase_validation = ml_get_topology_info()->chip_revision == CPU_VERSION_A0; |
94 | #endif |
95 | |
96 | if (kern_feature_override(KF_MATV_OVRD)) { |
97 | timebase_validation = 0; |
98 | } |
99 | if (PE_parse_boot_argn("timebase_validation" , &tmp_mv, sizeof(tmp_mv))) { |
100 | timebase_validation = tmp_mv; |
101 | } |
102 | #endif |
103 | } |
104 | |
105 | static void |
106 | timebase_callback(struct timebase_freq_t * freq) |
107 | { |
108 | unsigned long numer, denom; |
109 | uint64_t t64_1, t64_2; |
110 | uint32_t divisor; |
111 | |
112 | if (freq->timebase_den < 1 || freq->timebase_den > 4 || |
113 | freq->timebase_num < freq->timebase_den) { |
114 | panic("rtclock timebase_callback: invalid constant %ld / %ld" , |
115 | freq->timebase_num, freq->timebase_den); |
116 | } |
117 | |
118 | denom = freq->timebase_num; |
119 | numer = freq->timebase_den * NSEC_PER_SEC; |
120 | // reduce by the greatest common denominator to minimize overflow |
121 | if (numer > denom) { |
122 | t64_1 = numer; |
123 | t64_2 = denom; |
124 | } else { |
125 | t64_1 = denom; |
126 | t64_2 = numer; |
127 | } |
128 | while (t64_2 != 0) { |
129 | uint64_t temp = t64_2; |
130 | t64_2 = t64_1 % t64_2; |
131 | t64_1 = temp; |
132 | } |
133 | numer /= t64_1; |
134 | denom /= t64_1; |
135 | |
136 | rtclock_timebase_const.numer = (uint32_t)numer; |
137 | rtclock_timebase_const.denom = (uint32_t)denom; |
138 | divisor = (uint32_t)(freq->timebase_num / freq->timebase_den); |
139 | |
140 | rtclock_sec_divisor = divisor; |
141 | rtclock_usec_divisor = divisor / USEC_PER_SEC; |
142 | } |
143 | |
144 | /* |
145 | * Initialize the system clock device for the current cpu |
146 | */ |
147 | int |
148 | rtclock_init(void) |
149 | { |
150 | uint64_t abstime; |
151 | cpu_data_t * cdp; |
152 | |
153 | clock_timebase_init(); |
154 | |
155 | cdp = getCpuDatap(); |
156 | |
157 | abstime = mach_absolute_time(); |
158 | cdp->rtcPop = EndOfAllTime; /* Init Pop time */ |
159 | timer_resync_deadlines(); /* Start the timers going */ |
160 | |
161 | return 1; |
162 | } |
163 | |
164 | |
165 | uint64_t |
166 | mach_absolute_time(void) |
167 | { |
168 | #if DEVELOPMENT || DEBUG |
169 | if (__improbable(timebase_validation)) { |
170 | #if __ARM_ARCH_8_6__ || HAS_ACNTVCT |
171 | static _Atomic uint64_t s_last_absolute_time = 1; |
172 | |
173 | uint64_t old_absolute_time = os_atomic_load(&s_last_absolute_time, relaxed); |
174 | |
175 | /* |
176 | * Because this timebase read is nonspeculative, it cannot begin reading |
177 | * the timebase value until after the load of the old value completes. |
178 | */ |
179 | |
180 | if (old_absolute_time == 0) { |
181 | timebase_validation = 0; // we know it's bad, now prevent nested panics |
182 | panic("old_absolute_time was 0" ); |
183 | } |
184 | |
185 | uint64_t new_absolute_time = ml_get_timebase(); |
186 | |
187 | if (old_absolute_time > new_absolute_time) { |
188 | timebase_validation = 0; // prevent nested panics |
189 | panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx)" , |
190 | new_absolute_time, old_absolute_time); |
191 | } |
192 | |
193 | if (old_absolute_time < new_absolute_time) { |
194 | /* read again, to pretest the atomic max */ |
195 | uint64_t pretest_absolute_time = os_atomic_load(&s_last_absolute_time, relaxed); |
196 | if (pretest_absolute_time < new_absolute_time) { |
197 | uint64_t fresh_last_absolute_time = os_atomic_max_orig(&s_last_absolute_time, new_absolute_time, relaxed); |
198 | |
199 | if (fresh_last_absolute_time != pretest_absolute_time) { |
200 | /* |
201 | * Someone else published a newer time after we loaded s_last_absolute_time. |
202 | * Enforce that our timebase is not behind this new one. |
203 | * We can't compare it with our previous timebase read, as it is too old. |
204 | */ |
205 | |
206 | uint64_t newest_absolute_time = ml_get_timebase(); |
207 | |
208 | if (fresh_last_absolute_time > newest_absolute_time) { |
209 | timebase_validation = 0; // prevent nested panics |
210 | panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old values 0x%llx, 0x%llx, 0x%llx, 0x%llx)\n" , |
211 | newest_absolute_time, fresh_last_absolute_time, pretest_absolute_time, old_absolute_time, new_absolute_time); |
212 | } |
213 | } |
214 | } |
215 | } |
216 | |
217 | return new_absolute_time; |
218 | |
219 | #else /* !(__ARM_ARCH_8_6__ || HAS_ACNTVCT) */ |
220 | static volatile uint64_t s_last_absolute_time = 0; |
221 | uint64_t new_absolute_time, old_absolute_time; |
222 | int attempts = 0; |
223 | |
224 | /* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time |
225 | * completes before the timebase read. Were the load to complete after the |
226 | * timebase read, there would be a window for another CPU to update |
227 | * s_last_absolute_time and leave us in an inconsistent state. Consider the |
228 | * following interleaving: |
229 | * |
230 | * Let s_last_absolute_time = t0 |
231 | * CPU0: Read timebase at t1 |
232 | * CPU1: Read timebase at t2 |
233 | * CPU1: Update s_last_absolute_time to t2 |
234 | * CPU0: Load completes |
235 | * CPU0: Update s_last_absolute_time to t1 |
236 | * |
237 | * This would cause the assertion to fail even though time did not go |
238 | * backwards. Thus, we use a dsb to guarantee completion of the load before |
239 | * the timebase read. |
240 | */ |
241 | do { |
242 | attempts++; |
243 | old_absolute_time = s_last_absolute_time; |
244 | |
245 | __builtin_arm_dsb(DSB_ISHLD); |
246 | |
247 | new_absolute_time = ml_get_timebase(); |
248 | } while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time)); |
249 | |
250 | if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) { |
251 | timebase_validation = 0; // we know it's bad, now prevent nested panics |
252 | panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)" , |
253 | new_absolute_time, old_absolute_time); |
254 | } |
255 | return new_absolute_time; |
256 | #endif /* __ARM_ARCH_8_6__ || HAS_ACNTVCT */ |
257 | } |
258 | #endif /* DEVELOPMENT || DEBUG */ |
259 | |
260 | return ml_get_timebase(); |
261 | } |
262 | |
263 | uint64_t |
264 | mach_approximate_time(void) |
265 | { |
266 | #if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__ |
267 | /* Hardware supports a fast timestamp, so grab it without asserting monotonicity */ |
268 | return ml_get_timebase(); |
269 | #else |
270 | processor_t processor; |
271 | uint64_t approx_time; |
272 | |
273 | disable_preemption(); |
274 | processor = current_processor(); |
275 | approx_time = processor->last_dispatch; |
276 | enable_preemption(); |
277 | |
278 | return approx_time; |
279 | #endif |
280 | } |
281 | |
282 | void |
283 | clock_get_system_microtime(clock_sec_t * secs, |
284 | clock_usec_t * microsecs) |
285 | { |
286 | absolutetime_to_microtime(abstime: mach_absolute_time(), secs, microsecs); |
287 | } |
288 | |
289 | void |
290 | clock_get_system_nanotime(clock_sec_t * secs, |
291 | clock_nsec_t * nanosecs) |
292 | { |
293 | uint64_t abstime; |
294 | uint64_t t64; |
295 | |
296 | abstime = mach_absolute_time(); |
297 | *secs = (t64 = abstime / rtclock_sec_divisor); |
298 | abstime -= (t64 * rtclock_sec_divisor); |
299 | |
300 | *nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor); |
301 | } |
302 | |
303 | void |
304 | clock_gettimeofday_set_commpage(uint64_t abstime, |
305 | uint64_t sec, |
306 | uint64_t frac, |
307 | uint64_t scale, |
308 | uint64_t tick_per_sec) |
309 | { |
310 | commpage_set_timestamp(tbr: abstime, secs: sec, frac, scale, tick_per_sec); |
311 | } |
312 | |
313 | void |
314 | clock_timebase_info(mach_timebase_info_t info) |
315 | { |
316 | *info = rtclock_timebase_const; |
317 | } |
318 | |
319 | /* |
320 | * Real-time clock device interrupt. |
321 | */ |
322 | void |
323 | rtclock_intr(__unused unsigned int is_user_context) |
324 | { |
325 | uint64_t abstime; |
326 | cpu_data_t * cdp; |
327 | struct arm_saved_state * regs; |
328 | unsigned int user_mode; |
329 | uintptr_t pc; |
330 | |
331 | cdp = getCpuDatap(); |
332 | |
333 | cdp->cpu_stat.timer_cnt++; |
334 | SCHED_STATS_INC(timer_pop_count); |
335 | |
336 | assert(!ml_get_interrupts_enabled()); |
337 | |
338 | abstime = mach_absolute_time(); |
339 | |
340 | if (cdp->cpu_idle_pop != 0x0ULL) { |
341 | if ((cdp->rtcPop - abstime) < cdp->cpu_idle_latency) { |
342 | cdp->cpu_idle_pop = 0x0ULL; |
343 | while (abstime < cdp->rtcPop) { |
344 | abstime = mach_absolute_time(); |
345 | } |
346 | } else { |
347 | ClearIdlePop(FALSE); |
348 | } |
349 | } |
350 | |
351 | if ((regs = cdp->cpu_int_state)) { |
352 | pc = get_saved_state_pc(iss: regs); |
353 | |
354 | #if __arm64__ |
355 | user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs)); |
356 | #endif |
357 | } else { |
358 | pc = 0; |
359 | user_mode = 0; |
360 | } |
361 | if (abstime >= cdp->rtcPop) { |
362 | /* Log the interrupt service latency (-ve value expected by tool) */ |
363 | KDBG_RELEASE(DECR_TRAP_LATENCY | DBG_FUNC_NONE, |
364 | -(abstime - cdp->rtcPop), |
365 | user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode); |
366 | } |
367 | |
368 | /* call the generic etimer */ |
369 | timer_intr(inuser: user_mode, iaddr: pc); |
370 | } |
371 | |
372 | static int |
373 | deadline_to_decrementer(uint64_t deadline, |
374 | uint64_t now) |
375 | { |
376 | uint64_t delt; |
377 | |
378 | if (deadline <= now) { |
379 | return DECREMENTER_MIN; |
380 | } else { |
381 | delt = deadline - now; |
382 | |
383 | return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN); |
384 | } |
385 | } |
386 | |
387 | /* |
388 | * Request a decrementer pop |
389 | */ |
390 | int |
391 | setPop(uint64_t time) |
392 | { |
393 | int delay_time; |
394 | uint64_t current_time; |
395 | cpu_data_t * cdp; |
396 | |
397 | cdp = getCpuDatap(); |
398 | current_time = mach_absolute_time(); |
399 | |
400 | delay_time = deadline_to_decrementer(deadline: time, now: current_time); |
401 | cdp->rtcPop = delay_time + current_time; |
402 | |
403 | ml_set_decrementer(dec_value: (uint32_t) delay_time); |
404 | |
405 | return delay_time; |
406 | } |
407 | |
408 | /* |
409 | * Request decrementer Idle Pop. Return true if set |
410 | */ |
411 | boolean_t |
412 | SetIdlePop(void) |
413 | { |
414 | int delay_time; |
415 | uint64_t time; |
416 | uint64_t current_time; |
417 | cpu_data_t * cdp; |
418 | |
419 | cdp = getCpuDatap(); |
420 | current_time = mach_absolute_time(); |
421 | |
422 | if (((cdp->rtcPop < current_time) || |
423 | (cdp->rtcPop - current_time) < cdp->cpu_idle_latency)) { |
424 | return FALSE; |
425 | } |
426 | |
427 | time = cdp->rtcPop - cdp->cpu_idle_latency; |
428 | |
429 | delay_time = deadline_to_decrementer(deadline: time, now: current_time); |
430 | cdp->cpu_idle_pop = delay_time + current_time; |
431 | ml_set_decrementer(dec_value: (uint32_t) delay_time); |
432 | |
433 | return TRUE; |
434 | } |
435 | |
436 | /* |
437 | * Clear decrementer Idle Pop |
438 | */ |
439 | void |
440 | ClearIdlePop( |
441 | boolean_t wfi) |
442 | { |
443 | cpu_data_t * cdp; |
444 | |
445 | cdp = getCpuDatap(); |
446 | cdp->cpu_idle_pop = 0x0ULL; |
447 | |
448 | /* |
449 | * Don't update the HW timer if there's a pending |
450 | * interrupt (we can lose interrupt assertion); |
451 | * we want to take the interrupt right now and update |
452 | * the deadline from the handler). |
453 | * |
454 | * ARM64_TODO: consider this more carefully. |
455 | */ |
456 | if (!(wfi && ml_get_timer_pending())) { |
457 | setPop(cdp->rtcPop); |
458 | } |
459 | } |
460 | |
461 | void |
462 | absolutetime_to_microtime(uint64_t abstime, |
463 | clock_sec_t * secs, |
464 | clock_usec_t * microsecs) |
465 | { |
466 | uint64_t t64; |
467 | |
468 | *secs = t64 = abstime / rtclock_sec_divisor; |
469 | abstime -= (t64 * rtclock_sec_divisor); |
470 | |
471 | *microsecs = (uint32_t)(abstime / rtclock_usec_divisor); |
472 | } |
473 | |
474 | void |
475 | absolutetime_to_nanoseconds(uint64_t abstime, |
476 | uint64_t * result) |
477 | { |
478 | uint64_t t64; |
479 | |
480 | *result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC; |
481 | abstime -= (t64 * rtclock_sec_divisor); |
482 | *result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor; |
483 | } |
484 | |
485 | void |
486 | nanoseconds_to_absolutetime(uint64_t nanosecs, |
487 | uint64_t * result) |
488 | { |
489 | uint64_t t64; |
490 | |
491 | *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor; |
492 | nanosecs -= (t64 * NSEC_PER_SEC); |
493 | *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; |
494 | } |
495 | |
496 | void |
497 | nanotime_to_absolutetime(clock_sec_t secs, |
498 | clock_nsec_t nanosecs, |
499 | uint64_t * result) |
500 | { |
501 | *result = ((uint64_t) secs * rtclock_sec_divisor) + |
502 | ((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; |
503 | } |
504 | |
505 | void |
506 | clock_interval_to_absolutetime_interval(uint32_t interval, |
507 | uint32_t scale_factor, |
508 | uint64_t * result) |
509 | { |
510 | uint64_t nanosecs = (uint64_t) interval * scale_factor; |
511 | uint64_t t64; |
512 | |
513 | *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor; |
514 | nanosecs -= (t64 * NSEC_PER_SEC); |
515 | *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; |
516 | } |
517 | |
518 | void |
519 | machine_delay_until(uint64_t interval, |
520 | uint64_t deadline) |
521 | { |
522 | #pragma unused(interval) |
523 | uint64_t now; |
524 | |
525 | do { |
526 | __builtin_arm_wfe(); |
527 | now = mach_absolute_time(); |
528 | } while (now < deadline); |
529 | } |
530 | |