1 | /* |
2 | * Copyright (c) 2007 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * @APPLE_FREE_COPYRIGHT@ |
33 | */ |
34 | /* |
35 | * File: arm/rtclock.c |
36 | * Purpose: Routines for handling the machine dependent |
37 | * real-time clock. |
38 | */ |
39 | |
40 | #include <mach/mach_types.h> |
41 | |
42 | #include <kern/clock.h> |
43 | #include <kern/thread.h> |
44 | #include <kern/macro_help.h> |
45 | #include <kern/spl.h> |
46 | #include <kern/timer_queue.h> |
47 | |
48 | #include <kern/host_notify.h> |
49 | |
50 | #include <machine/commpage.h> |
51 | #include <machine/machine_routines.h> |
52 | #include <arm/exception.h> |
53 | #include <arm/cpu_data_internal.h> |
54 | #if __arm64__ |
55 | #include <arm64/proc_reg.h> |
56 | #elif __arm__ |
57 | #include <arm/proc_reg.h> |
58 | #else |
59 | #error Unsupported arch |
60 | #endif |
61 | #include <arm/rtclock.h> |
62 | |
63 | #include <IOKit/IOPlatformExpert.h> |
64 | #include <libkern/OSAtomic.h> |
65 | |
66 | #include <sys/kdebug.h> |
67 | |
68 | #define MAX_TIMEBASE_TRIES 10 |
69 | |
70 | int rtclock_init(void); |
71 | |
72 | static int |
73 | deadline_to_decrementer(uint64_t deadline, |
74 | uint64_t now); |
75 | static void |
76 | timebase_callback(struct timebase_freq_t * freq); |
77 | |
78 | #if DEVELOPMENT || DEBUG |
79 | uint32_t absolute_time_validation = 1; |
80 | #endif |
81 | |
82 | /* |
83 | * Configure the real-time clock device at boot |
84 | */ |
85 | void |
86 | rtclock_early_init(void) |
87 | { |
88 | PE_register_timebase_callback(timebase_callback); |
89 | #if DEVELOPMENT || DEBUG |
90 | uint32_t tmp_mv = 1; |
91 | if (kern_feature_override(KF_MATV_OVRD)) { |
92 | absolute_time_validation = 0; |
93 | } |
94 | if (PE_parse_boot_argn("timebase_validation" , &tmp_mv, sizeof(tmp_mv))) { |
95 | if (tmp_mv == 0) { |
96 | absolute_time_validation = 0; |
97 | } |
98 | } |
99 | #endif |
100 | } |
101 | |
102 | static void |
103 | timebase_callback(struct timebase_freq_t * freq) |
104 | { |
105 | unsigned long numer, denom; |
106 | uint64_t t64_1, t64_2; |
107 | uint32_t divisor; |
108 | |
109 | if (freq->timebase_den < 1 || freq->timebase_den > 4 || |
110 | freq->timebase_num < freq->timebase_den) |
111 | panic("rtclock timebase_callback: invalid constant %ld / %ld" , |
112 | freq->timebase_num, freq->timebase_den); |
113 | |
114 | denom = freq->timebase_num; |
115 | numer = freq->timebase_den * NSEC_PER_SEC; |
116 | // reduce by the greatest common denominator to minimize overflow |
117 | if (numer > denom) { |
118 | t64_1 = numer; |
119 | t64_2 = denom; |
120 | } else { |
121 | t64_1 = denom; |
122 | t64_2 = numer; |
123 | } |
124 | while (t64_2 != 0) { |
125 | uint64_t temp = t64_2; |
126 | t64_2 = t64_1 % t64_2; |
127 | t64_1 = temp; |
128 | } |
129 | numer /= t64_1; |
130 | denom /= t64_1; |
131 | |
132 | rtclock_timebase_const.numer = (uint32_t)numer; |
133 | rtclock_timebase_const.denom = (uint32_t)denom; |
134 | divisor = (uint32_t)(freq->timebase_num / freq->timebase_den); |
135 | |
136 | rtclock_sec_divisor = divisor; |
137 | rtclock_usec_divisor = divisor / USEC_PER_SEC; |
138 | } |
139 | |
140 | /* |
141 | * Initialize the system clock device for the current cpu |
142 | */ |
143 | int |
144 | rtclock_init(void) |
145 | { |
146 | uint64_t abstime; |
147 | cpu_data_t * cdp; |
148 | |
149 | clock_timebase_init(); |
150 | ml_init_lock_timeout(); |
151 | |
152 | cdp = getCpuDatap(); |
153 | |
154 | abstime = mach_absolute_time(); |
155 | cdp->rtcPop = EndOfAllTime; /* Init Pop time */ |
156 | timer_resync_deadlines(); /* Start the timers going */ |
157 | |
158 | return (1); |
159 | } |
160 | |
161 | uint64_t |
162 | mach_absolute_time(void) |
163 | { |
164 | #if DEVELOPMENT || DEBUG |
165 | if (__improbable(absolute_time_validation == 1)) { |
166 | static volatile uint64_t s_last_absolute_time = 0; |
167 | uint64_t new_absolute_time, old_absolute_time; |
168 | int attempts = 0; |
169 | |
170 | /* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time |
171 | * completes before the timebase read. Were the load to complete after the |
172 | * timebase read, there would be a window for another CPU to update |
173 | * s_last_absolute_time and leave us in an inconsistent state. Consider the |
174 | * following interleaving: |
175 | * |
176 | * Let s_last_absolute_time = t0 |
177 | * CPU0: Read timebase at t1 |
178 | * CPU1: Read timebase at t2 |
179 | * CPU1: Update s_last_absolute_time to t2 |
180 | * CPU0: Load completes |
181 | * CPU0: Update s_last_absolute_time to t1 |
182 | * |
183 | * This would cause the assertion to fail even though time did not go |
184 | * backwards. Thus, we use a dsb to guarantee completion of the load before |
185 | * the timebase read. |
186 | */ |
187 | do { |
188 | attempts++; |
189 | old_absolute_time = s_last_absolute_time; |
190 | |
191 | #if __arm64__ |
192 | __asm__ volatile("dsb ld" ::: "memory" ); |
193 | #else |
194 | OSSynchronizeIO(); // See osfmk/arm64/rtclock.c |
195 | #endif |
196 | |
197 | new_absolute_time = ml_get_timebase(); |
198 | } while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time)); |
199 | |
200 | if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) { |
201 | panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)\n" , |
202 | new_absolute_time, old_absolute_time); |
203 | } |
204 | return new_absolute_time; |
205 | } else { |
206 | return ml_get_timebase(); |
207 | } |
208 | #else |
209 | return ml_get_timebase(); |
210 | #endif |
211 | } |
212 | |
213 | uint64_t |
214 | mach_approximate_time(void) |
215 | { |
216 | #if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__ |
217 | /* Hardware supports a fast timestamp, so grab it without asserting monotonicity */ |
218 | return ml_get_timebase(); |
219 | #else |
220 | processor_t processor; |
221 | uint64_t approx_time; |
222 | |
223 | disable_preemption(); |
224 | processor = current_processor(); |
225 | approx_time = processor->last_dispatch; |
226 | enable_preemption(); |
227 | |
228 | return approx_time; |
229 | #endif |
230 | } |
231 | |
232 | void |
233 | clock_get_system_microtime(clock_sec_t * secs, |
234 | clock_usec_t * microsecs) |
235 | { |
236 | absolutetime_to_microtime(mach_absolute_time(), secs, microsecs); |
237 | } |
238 | |
239 | void |
240 | clock_get_system_nanotime(clock_sec_t * secs, |
241 | clock_nsec_t * nanosecs) |
242 | { |
243 | uint64_t abstime; |
244 | uint64_t t64; |
245 | |
246 | abstime = mach_absolute_time(); |
247 | *secs = (t64 = abstime / rtclock_sec_divisor); |
248 | abstime -= (t64 * rtclock_sec_divisor); |
249 | |
250 | *nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor); |
251 | } |
252 | |
253 | void |
254 | clock_gettimeofday_set_commpage(uint64_t abstime, |
255 | uint64_t sec, |
256 | uint64_t frac, |
257 | uint64_t scale, |
258 | uint64_t tick_per_sec) |
259 | { |
260 | commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec); |
261 | } |
262 | |
263 | void |
264 | clock_timebase_info(mach_timebase_info_t info) |
265 | { |
266 | *info = rtclock_timebase_const; |
267 | } |
268 | |
269 | /* |
270 | * Real-time clock device interrupt. |
271 | */ |
272 | void |
273 | rtclock_intr(__unused unsigned int is_user_context) |
274 | { |
275 | uint64_t abstime; |
276 | cpu_data_t * cdp; |
277 | struct arm_saved_state * regs; |
278 | unsigned int user_mode; |
279 | uintptr_t pc; |
280 | |
281 | cdp = getCpuDatap(); |
282 | |
283 | cdp->cpu_stat.timer_cnt++; |
284 | cdp->cpu_stat.timer_cnt_wake++; |
285 | SCHED_STATS_TIMER_POP(current_processor()); |
286 | |
287 | assert(!ml_get_interrupts_enabled()); |
288 | |
289 | abstime = mach_absolute_time(); |
290 | |
291 | if (cdp->cpu_idle_pop != 0x0ULL) { |
292 | if (( cdp->rtcPop-abstime) < cdp->cpu_idle_latency) { |
293 | cdp->cpu_idle_pop = 0x0ULL; |
294 | while (abstime < cdp->rtcPop) |
295 | abstime = mach_absolute_time(); |
296 | } else { |
297 | ClearIdlePop(FALSE); |
298 | } |
299 | } |
300 | |
301 | if ((regs = cdp->cpu_int_state)) { |
302 | pc = get_saved_state_pc(regs); |
303 | |
304 | #if __arm64__ |
305 | user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs)); |
306 | #else |
307 | user_mode = (regs->cpsr & PSR_MODE_MASK) == PSR_USER_MODE; |
308 | #endif |
309 | } else { |
310 | pc = 0; |
311 | user_mode = 0; |
312 | } |
313 | if (abstime >= cdp->rtcPop) { |
314 | /* Log the interrupt service latency (-ve value expected by tool) */ |
315 | KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, |
316 | MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE, |
317 | -(abstime - cdp->rtcPop), |
318 | user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode, 0, 0); |
319 | } |
320 | |
321 | /* call the generic etimer */ |
322 | timer_intr(user_mode, pc); |
323 | } |
324 | |
325 | static int |
326 | deadline_to_decrementer(uint64_t deadline, |
327 | uint64_t now) |
328 | { |
329 | uint64_t delt; |
330 | |
331 | if (deadline <= now) |
332 | return DECREMENTER_MIN; |
333 | else { |
334 | delt = deadline - now; |
335 | |
336 | return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN); |
337 | } |
338 | } |
339 | |
340 | /* |
341 | * Request a decrementer pop |
342 | */ |
343 | int |
344 | setPop(uint64_t time) |
345 | { |
346 | int delay_time; |
347 | uint64_t current_time; |
348 | cpu_data_t * cdp; |
349 | |
350 | cdp = getCpuDatap(); |
351 | current_time = mach_absolute_time(); |
352 | |
353 | delay_time = deadline_to_decrementer(time, current_time); |
354 | cdp->rtcPop = delay_time + current_time; |
355 | |
356 | ml_set_decrementer((uint32_t) delay_time); |
357 | |
358 | return (delay_time); |
359 | } |
360 | |
361 | /* |
362 | * Request decrementer Idle Pop. Return true if set |
363 | */ |
364 | boolean_t |
365 | SetIdlePop(void) |
366 | { |
367 | int delay_time; |
368 | uint64_t time; |
369 | uint64_t current_time; |
370 | cpu_data_t * cdp; |
371 | |
372 | cdp = getCpuDatap(); |
373 | current_time = mach_absolute_time(); |
374 | |
375 | if (((cdp->rtcPop < current_time) || |
376 | (cdp->rtcPop - current_time) < cdp->cpu_idle_latency)) |
377 | return FALSE; |
378 | |
379 | time = cdp->rtcPop - cdp->cpu_idle_latency; |
380 | |
381 | delay_time = deadline_to_decrementer(time, current_time); |
382 | cdp->cpu_idle_pop = delay_time + current_time; |
383 | ml_set_decrementer((uint32_t) delay_time); |
384 | |
385 | return TRUE; |
386 | } |
387 | |
388 | /* |
389 | * Clear decrementer Idle Pop |
390 | */ |
391 | void |
392 | ClearIdlePop( |
393 | boolean_t wfi) |
394 | { |
395 | #if !__arm64__ |
396 | #pragma unused(wfi) |
397 | #endif |
398 | cpu_data_t * cdp; |
399 | |
400 | cdp = getCpuDatap(); |
401 | cdp->cpu_idle_pop = 0x0ULL; |
402 | |
403 | #if __arm64__ |
404 | /* |
405 | * Don't update the HW timer if there's a pending |
406 | * interrupt (we can lose interrupt assertion); |
407 | * we want to take the interrupt right now and update |
408 | * the deadline from the handler). |
409 | * |
410 | * ARM64_TODO: consider this more carefully. |
411 | */ |
412 | if (!(wfi && ml_get_timer_pending())) |
413 | #endif |
414 | { |
415 | setPop(cdp->rtcPop); |
416 | } |
417 | } |
418 | |
419 | void |
420 | absolutetime_to_microtime(uint64_t abstime, |
421 | clock_sec_t * secs, |
422 | clock_usec_t * microsecs) |
423 | { |
424 | uint64_t t64; |
425 | |
426 | *secs = t64 = abstime / rtclock_sec_divisor; |
427 | abstime -= (t64 * rtclock_sec_divisor); |
428 | |
429 | *microsecs = (uint32_t)(abstime / rtclock_usec_divisor); |
430 | } |
431 | |
432 | void |
433 | absolutetime_to_nanoseconds(uint64_t abstime, |
434 | uint64_t * result) |
435 | { |
436 | uint64_t t64; |
437 | |
438 | *result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC; |
439 | abstime -= (t64 * rtclock_sec_divisor); |
440 | *result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor; |
441 | } |
442 | |
443 | void |
444 | nanoseconds_to_absolutetime(uint64_t nanosecs, |
445 | uint64_t * result) |
446 | { |
447 | uint64_t t64; |
448 | |
449 | *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor; |
450 | nanosecs -= (t64 * NSEC_PER_SEC); |
451 | *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; |
452 | } |
453 | |
454 | void |
455 | nanotime_to_absolutetime(clock_sec_t secs, |
456 | clock_nsec_t nanosecs, |
457 | uint64_t * result) |
458 | { |
459 | *result = ((uint64_t) secs * rtclock_sec_divisor) + |
460 | ((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; |
461 | } |
462 | |
463 | void |
464 | clock_interval_to_absolutetime_interval(uint32_t interval, |
465 | uint32_t scale_factor, |
466 | uint64_t * result) |
467 | { |
468 | uint64_t nanosecs = (uint64_t) interval * scale_factor; |
469 | uint64_t t64; |
470 | |
471 | *result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor; |
472 | nanosecs -= (t64 * NSEC_PER_SEC); |
473 | *result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC; |
474 | } |
475 | |
476 | void |
477 | machine_delay_until(uint64_t interval, |
478 | uint64_t deadline) |
479 | { |
480 | #pragma unused(interval) |
481 | uint64_t now; |
482 | |
483 | do { |
484 | #if __ARM_ENABLE_WFE_ |
485 | #if __arm64__ |
486 | if (arm64_wfe_allowed()) |
487 | #endif /* __arm64__ */ |
488 | { |
489 | __builtin_arm_wfe(); |
490 | } |
491 | #endif /* __ARM_ENABLE_WFE_ */ |
492 | |
493 | now = mach_absolute_time(); |
494 | } while (now < deadline); |
495 | } |
496 | |