1/*
2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 */
33/*-
34 * Copyright (c) 1982, 1986, 1993
35 * The Regents of the University of California. All rights reserved.
36 *
37 * Redistribution and use in source and binary forms, with or without
38 * modification, are permitted provided that the following conditions
39 * are met:
40 * 1. Redistributions of source code must retain the above copyright
41 * notice, this list of conditions and the following disclaimer.
42 * 2. Redistributions in binary form must reproduce the above copyright
43 * notice, this list of conditions and the following disclaimer in the
44 * documentation and/or other materials provided with the distribution.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)time.h 8.5 (Berkeley) 5/4/95
62 * $FreeBSD$
63 */
64
65#include <mach/mach_types.h>
66
67#include <kern/spl.h>
68#include <kern/sched_prim.h>
69#include <kern/thread.h>
70#include <kern/clock.h>
71#include <kern/host_notify.h>
72#include <kern/thread_call.h>
73#include <libkern/OSAtomic.h>
74
75#include <IOKit/IOPlatformExpert.h>
76
77#include <machine/commpage.h>
78#include <machine/config.h>
79#include <machine/machine_routines.h>
80
81#include <mach/mach_traps.h>
82#include <mach/mach_time.h>
83
84#include <sys/kdebug.h>
85#include <sys/timex.h>
86#include <kern/arithmetic_128.h>
87#include <os/log.h>
88
89uint32_t hz_tick_interval = 1;
90static uint64_t has_monotonic_clock = 0;
91
92decl_simple_lock_data(,clock_lock)
93lck_grp_attr_t * settime_lock_grp_attr;
94lck_grp_t * settime_lock_grp;
95lck_attr_t * settime_lock_attr;
96lck_mtx_t settime_lock;
97
98#define clock_lock() \
99 simple_lock(&clock_lock)
100
101#define clock_unlock() \
102 simple_unlock(&clock_lock)
103
104#define clock_lock_init() \
105 simple_lock_init(&clock_lock, 0)
106
107#ifdef kdp_simple_lock_is_acquired
108boolean_t kdp_clock_is_locked()
109{
110 return kdp_simple_lock_is_acquired(&clock_lock);
111}
112#endif
113
114struct bintime {
115 time_t sec;
116 uint64_t frac;
117};
118
119static __inline void
120bintime_addx(struct bintime *_bt, uint64_t _x)
121{
122 uint64_t _u;
123
124 _u = _bt->frac;
125 _bt->frac += _x;
126 if (_u > _bt->frac)
127 _bt->sec++;
128}
129
130static __inline void
131bintime_subx(struct bintime *_bt, uint64_t _x)
132{
133 uint64_t _u;
134
135 _u = _bt->frac;
136 _bt->frac -= _x;
137 if (_u < _bt->frac)
138 _bt->sec--;
139}
140
141static __inline void
142bintime_addns(struct bintime *bt, uint64_t ns)
143{
144 bt->sec += ns/ (uint64_t)NSEC_PER_SEC;
145 ns = ns % (uint64_t)NSEC_PER_SEC;
146 if (ns) {
147 /* 18446744073 = int(2^64 / NSEC_PER_SEC) */
148 ns = ns * (uint64_t)18446744073LL;
149 bintime_addx(bt, ns);
150 }
151}
152
153static __inline void
154bintime_subns(struct bintime *bt, uint64_t ns)
155{
156 bt->sec -= ns/ (uint64_t)NSEC_PER_SEC;
157 ns = ns % (uint64_t)NSEC_PER_SEC;
158 if (ns) {
159 /* 18446744073 = int(2^64 / NSEC_PER_SEC) */
160 ns = ns * (uint64_t)18446744073LL;
161 bintime_subx(bt, ns);
162 }
163}
164
165static __inline void
166bintime_addxns(struct bintime *bt, uint64_t a, int64_t xns)
167{
168 uint64_t uxns = (xns > 0)?(uint64_t )xns:(uint64_t)-xns;
169 uint64_t ns = multi_overflow(a, uxns);
170 if (xns > 0) {
171 if (ns)
172 bintime_addns(bt, ns);
173 ns = (a * uxns) / (uint64_t)NSEC_PER_SEC;
174 bintime_addx(bt, ns);
175 }
176 else{
177 if (ns)
178 bintime_subns(bt, ns);
179 ns = (a * uxns) / (uint64_t)NSEC_PER_SEC;
180 bintime_subx(bt,ns);
181 }
182}
183
184
185static __inline void
186bintime_add(struct bintime *_bt, const struct bintime *_bt2)
187{
188 uint64_t _u;
189
190 _u = _bt->frac;
191 _bt->frac += _bt2->frac;
192 if (_u > _bt->frac)
193 _bt->sec++;
194 _bt->sec += _bt2->sec;
195}
196
197static __inline void
198bintime_sub(struct bintime *_bt, const struct bintime *_bt2)
199{
200 uint64_t _u;
201
202 _u = _bt->frac;
203 _bt->frac -= _bt2->frac;
204 if (_u < _bt->frac)
205 _bt->sec--;
206 _bt->sec -= _bt2->sec;
207}
208
209static __inline void
210clock2bintime(const clock_sec_t *secs, const clock_usec_t *microsecs, struct bintime *_bt)
211{
212
213 _bt->sec = *secs;
214 /* 18446744073709 = int(2^64 / 1000000) */
215 _bt->frac = *microsecs * (uint64_t)18446744073709LL;
216}
217
218static __inline void
219bintime2usclock(const struct bintime *_bt, clock_sec_t *secs, clock_usec_t *microsecs)
220{
221
222 *secs = _bt->sec;
223 *microsecs = ((uint64_t)USEC_PER_SEC * (uint32_t)(_bt->frac >> 32)) >> 32;
224}
225
226static __inline void
227bintime2nsclock(const struct bintime *_bt, clock_sec_t *secs, clock_usec_t *nanosecs)
228{
229
230 *secs = _bt->sec;
231 *nanosecs = ((uint64_t)NSEC_PER_SEC * (uint32_t)(_bt->frac >> 32)) >> 32;
232}
233
234static __inline void
235bintime2absolutetime(const struct bintime *_bt, uint64_t *abs)
236{
237 uint64_t nsec;
238 nsec = (uint64_t) _bt->sec * (uint64_t)NSEC_PER_SEC + (((uint64_t)NSEC_PER_SEC * (uint32_t)(_bt->frac >> 32)) >> 32);
239 nanoseconds_to_absolutetime(nsec, abs);
240}
241
242struct latched_time {
243 uint64_t monotonic_time_usec;
244 uint64_t mach_time;
245};
246
247extern int
248kernel_sysctlbyname(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
249
250/*
251 * Time of day (calendar) variables.
252 *
253 * Algorithm:
254 *
255 * TOD <- bintime + delta*scale
256 *
257 * where :
258 * bintime is a cumulative offset that includes bootime and scaled time elapsed betweed bootime and last scale update.
259 * delta is ticks elapsed since last scale update.
260 * scale is computed according to an adjustment provided by ntp_kern.
261 */
262static struct clock_calend {
263 uint64_t s_scale_ns; /* scale to apply for each second elapsed, it converts in ns */
264 int64_t s_adj_nsx; /* additional adj to apply for each second elapsed, it is expressed in 64 bit frac of ns */
265 uint64_t tick_scale_x; /* scale to apply for each tick elapsed, it converts in 64 bit frac of s */
266 uint64_t offset_count; /* abs time from which apply current scales */
267 struct bintime offset; /* cumulative offset expressed in (sec, 64 bits frac of a second) */
268 struct bintime bintime; /* cumulative offset (it includes bootime) expressed in (sec, 64 bits frac of a second) */
269 struct bintime boottime; /* boot time expressed in (sec, 64 bits frac of a second) */
270 struct bintime basesleep;
271} clock_calend;
272
273static uint64_t ticks_per_sec; /* ticks in a second (expressed in abs time) */
274
275#if DEVELOPMENT || DEBUG
276extern int g_should_log_clock_adjustments;
277
278static void print_all_clock_variables(const char*, clock_sec_t* pmu_secs, clock_usec_t* pmu_usec, clock_sec_t* sys_secs, clock_usec_t* sys_usec, struct clock_calend* calend_cp);
279static void print_all_clock_variables_internal(const char *, struct clock_calend* calend_cp);
280#else
281#define print_all_clock_variables(...) do { } while (0)
282#define print_all_clock_variables_internal(...) do { } while (0)
283#endif
284
285#if CONFIG_DTRACE
286
287
288/*
289 * Unlocked calendar flipflop; this is used to track a clock_calend such
290 * that we can safely access a snapshot of a valid clock_calend structure
291 * without needing to take any locks to do it.
292 *
293 * The trick is to use a generation count and set the low bit when it is
294 * being updated/read; by doing this, we guarantee, through use of the
295 * hw_atomic functions, that the generation is incremented when the bit
296 * is cleared atomically (by using a 1 bit add).
297 */
298static struct unlocked_clock_calend {
299 struct clock_calend calend; /* copy of calendar */
300 uint32_t gen; /* generation count */
301} flipflop[ 2];
302
303static void clock_track_calend_nowait(void);
304
305#endif
306
307void _clock_delay_until_deadline(uint64_t interval, uint64_t deadline);
308void _clock_delay_until_deadline_with_leeway(uint64_t interval, uint64_t deadline, uint64_t leeway);
309
310/* Boottime variables*/
311static uint64_t clock_boottime;
312static uint32_t clock_boottime_usec;
313
314#define TIME_ADD(rsecs, secs, rfrac, frac, unit) \
315MACRO_BEGIN \
316 if (((rfrac) += (frac)) >= (unit)) { \
317 (rfrac) -= (unit); \
318 (rsecs) += 1; \
319 } \
320 (rsecs) += (secs); \
321MACRO_END
322
323#define TIME_SUB(rsecs, secs, rfrac, frac, unit) \
324MACRO_BEGIN \
325 if ((int)((rfrac) -= (frac)) < 0) { \
326 (rfrac) += (unit); \
327 (rsecs) -= 1; \
328 } \
329 (rsecs) -= (secs); \
330MACRO_END
331
332/*
333 * clock_config:
334 *
335 * Called once at boot to configure the clock subsystem.
336 */
337void
338clock_config(void)
339{
340
341 clock_lock_init();
342
343 settime_lock_grp_attr = lck_grp_attr_alloc_init();
344 settime_lock_grp = lck_grp_alloc_init("settime grp", settime_lock_grp_attr);
345 settime_lock_attr = lck_attr_alloc_init();
346 lck_mtx_init(&settime_lock, settime_lock_grp, settime_lock_attr);
347
348 clock_oldconfig();
349
350 ntp_init();
351
352 nanoseconds_to_absolutetime((uint64_t)NSEC_PER_SEC, &ticks_per_sec);
353}
354
355/*
356 * clock_init:
357 *
358 * Called on a processor each time started.
359 */
360void
361clock_init(void)
362{
363 clock_oldinit();
364}
365
366/*
367 * clock_timebase_init:
368 *
369 * Called by machine dependent code
370 * to initialize areas dependent on the
371 * timebase value. May be called multiple
372 * times during start up.
373 */
374void
375clock_timebase_init(void)
376{
377 uint64_t abstime;
378
379 nanoseconds_to_absolutetime(NSEC_PER_SEC / 100, &abstime);
380 hz_tick_interval = (uint32_t)abstime;
381
382 sched_timebase_init();
383}
384
385/*
386 * mach_timebase_info_trap:
387 *
388 * User trap returns timebase constant.
389 */
390kern_return_t
391mach_timebase_info_trap(
392 struct mach_timebase_info_trap_args *args)
393{
394 mach_vm_address_t out_info_addr = args->info;
395 mach_timebase_info_data_t info = {};
396
397 clock_timebase_info(&info);
398
399 copyout((void *)&info, out_info_addr, sizeof (info));
400
401 return (KERN_SUCCESS);
402}
403
404/*
405 * Calendar routines.
406 */
407
408/*
409 * clock_get_calendar_microtime:
410 *
411 * Returns the current calendar value,
412 * microseconds as the fraction.
413 */
414void
415clock_get_calendar_microtime(
416 clock_sec_t *secs,
417 clock_usec_t *microsecs)
418{
419 clock_get_calendar_absolute_and_microtime(secs, microsecs, NULL);
420}
421
422/*
423 * get_scale_factors_from_adj:
424 *
425 * computes scale factors from the value given in adjustment.
426 *
427 * Part of the code has been taken from tc_windup of FreeBSD
428 * written by Poul-Henning Kamp <phk@FreeBSD.ORG>, Julien Ridoux and
429 * Konstantin Belousov.
430 * https://github.com/freebsd/freebsd/blob/master/sys/kern/kern_tc.c
431 */
432static void
433get_scale_factors_from_adj(int64_t adjustment, uint64_t* tick_scale_x, uint64_t* s_scale_ns, int64_t* s_adj_nsx)
434{
435 uint64_t scale;
436 int64_t nano, frac;
437
438 /*-
439 * Calculating the scaling factor. We want the number of 1/2^64
440 * fractions of a second per period of the hardware counter, taking
441 * into account the th_adjustment factor which the NTP PLL/adjtime(2)
442 * processing provides us with.
443 *
444 * The th_adjustment is nanoseconds per second with 32 bit binary
445 * fraction and we want 64 bit binary fraction of second:
446 *
447 * x = a * 2^32 / 10^9 = a * 4.294967296
448 *
449 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int
450 * we can only multiply by about 850 without overflowing, that
451 * leaves no suitably precise fractions for multiply before divide.
452 *
453 * Divide before multiply with a fraction of 2199/512 results in a
454 * systematic undercompensation of 10PPM of th_adjustment. On a
455 * 5000PPM adjustment this is a 0.05PPM error. This is acceptable.
456 *
457 * We happily sacrifice the lowest of the 64 bits of our result
458 * to the goddess of code clarity.
459 *
460 */
461 scale = (uint64_t)1 << 63;
462 scale += (adjustment / 1024) * 2199;
463 scale /= ticks_per_sec;
464 *tick_scale_x = scale * 2;
465
466 /*
467 * hi part of adj
468 * it contains ns (without fraction) to add to the next sec.
469 * Get ns scale factor for the next sec.
470 */
471 nano = (adjustment > 0)? adjustment >> 32 : -((-adjustment) >> 32);
472 scale = (uint64_t) NSEC_PER_SEC;
473 scale += nano;
474 *s_scale_ns = scale;
475
476 /*
477 * lo part of adj
478 * it contains 32 bit frac of ns to add to the next sec.
479 * Keep it as additional adjustment for the next sec.
480 */
481 frac = (adjustment > 0)? ((uint32_t) adjustment) : -((uint32_t) (-adjustment));
482 *s_adj_nsx = (frac>0)? frac << 32 : -( (-frac) << 32);
483
484 return;
485}
486
487/*
488 * scale_delta:
489 *
490 * returns a bintime struct representing delta scaled accordingly to the
491 * scale factors provided to this function.
492 */
493static struct bintime
494scale_delta(uint64_t delta, uint64_t tick_scale_x, uint64_t s_scale_ns, int64_t s_adj_nsx)
495{
496 uint64_t sec, new_ns, over;
497 struct bintime bt;
498
499 bt.sec = 0;
500 bt.frac = 0;
501
502 /*
503 * If more than one second is elapsed,
504 * scale fully elapsed seconds using scale factors for seconds.
505 * s_scale_ns -> scales sec to ns.
506 * s_adj_nsx -> additional adj expressed in 64 bit frac of ns to apply to each sec.
507 */
508 if (delta > ticks_per_sec) {
509 sec = (delta/ticks_per_sec);
510 new_ns = sec * s_scale_ns;
511 bintime_addns(&bt, new_ns);
512 if (s_adj_nsx) {
513 if (sec == 1) {
514 /* shortcut, no overflow can occur */
515 if (s_adj_nsx > 0)
516 bintime_addx(&bt, (uint64_t)s_adj_nsx/ (uint64_t)NSEC_PER_SEC);
517 else
518 bintime_subx(&bt, (uint64_t)-s_adj_nsx/ (uint64_t)NSEC_PER_SEC);
519 }
520 else{
521 /*
522 * s_adj_nsx is 64 bit frac of ns.
523 * sec*s_adj_nsx might overflow in int64_t.
524 * use bintime_addxns to not lose overflowed ns.
525 */
526 bintime_addxns(&bt, sec, s_adj_nsx);
527 }
528 }
529 delta = (delta % ticks_per_sec);
530 }
531
532 over = multi_overflow(tick_scale_x, delta);
533 if(over){
534 bt.sec += over;
535 }
536
537 /*
538 * scale elapsed ticks using the scale factor for ticks.
539 */
540 bintime_addx(&bt, delta * tick_scale_x);
541
542 return bt;
543}
544
545/*
546 * get_scaled_time:
547 *
548 * returns the scaled time of the time elapsed from the last time
549 * scale factors were updated to now.
550 */
551static struct bintime
552get_scaled_time(uint64_t now)
553{
554 uint64_t delta;
555
556 /*
557 * Compute ticks elapsed since last scale update.
558 * This time will be scaled according to the value given by ntp kern.
559 */
560 delta = now - clock_calend.offset_count;
561
562 return scale_delta(delta, clock_calend.tick_scale_x, clock_calend.s_scale_ns, clock_calend.s_adj_nsx);
563}
564
565static void
566clock_get_calendar_absolute_and_microtime_locked(
567 clock_sec_t *secs,
568 clock_usec_t *microsecs,
569 uint64_t *abstime)
570{
571 uint64_t now;
572 struct bintime bt;
573
574 now = mach_absolute_time();
575 if (abstime)
576 *abstime = now;
577
578 bt = get_scaled_time(now);
579 bintime_add(&bt, &clock_calend.bintime);
580 bintime2usclock(&bt, secs, microsecs);
581}
582
583static void
584clock_get_calendar_absolute_and_nanotime_locked(
585 clock_sec_t *secs,
586 clock_usec_t *nanosecs,
587 uint64_t *abstime)
588{
589 uint64_t now;
590 struct bintime bt;
591
592 now = mach_absolute_time();
593 if (abstime)
594 *abstime = now;
595
596 bt = get_scaled_time(now);
597 bintime_add(&bt, &clock_calend.bintime);
598 bintime2nsclock(&bt, secs, nanosecs);
599}
600
601/*
602 * clock_get_calendar_absolute_and_microtime:
603 *
604 * Returns the current calendar value,
605 * microseconds as the fraction. Also
606 * returns mach_absolute_time if abstime
607 * is not NULL.
608 */
609void
610clock_get_calendar_absolute_and_microtime(
611 clock_sec_t *secs,
612 clock_usec_t *microsecs,
613 uint64_t *abstime)
614{
615 spl_t s;
616
617 s = splclock();
618 clock_lock();
619
620 clock_get_calendar_absolute_and_microtime_locked(secs, microsecs, abstime);
621
622 clock_unlock();
623 splx(s);
624}
625
626/*
627 * clock_get_calendar_nanotime:
628 *
629 * Returns the current calendar value,
630 * nanoseconds as the fraction.
631 *
632 * Since we do not have an interface to
633 * set the calendar with resolution greater
634 * than a microsecond, we honor that here.
635 */
636void
637clock_get_calendar_nanotime(
638 clock_sec_t *secs,
639 clock_nsec_t *nanosecs)
640{
641 spl_t s;
642
643 s = splclock();
644 clock_lock();
645
646 clock_get_calendar_absolute_and_nanotime_locked(secs, nanosecs, NULL);
647
648 clock_unlock();
649 splx(s);
650}
651
652/*
653 * clock_gettimeofday:
654 *
655 * Kernel interface for commpage implementation of
656 * gettimeofday() syscall.
657 *
658 * Returns the current calendar value, and updates the
659 * commpage info as appropriate. Because most calls to
660 * gettimeofday() are handled in user mode by the commpage,
661 * this routine should be used infrequently.
662 */
663void
664clock_gettimeofday(
665 clock_sec_t *secs,
666 clock_usec_t *microsecs)
667{
668 clock_gettimeofday_and_absolute_time(secs, microsecs, NULL);
669}
670
671void
672clock_gettimeofday_and_absolute_time(
673 clock_sec_t *secs,
674 clock_usec_t *microsecs,
675 uint64_t *mach_time)
676{
677 uint64_t now;
678 spl_t s;
679 struct bintime bt;
680
681 s = splclock();
682 clock_lock();
683
684 now = mach_absolute_time();
685 bt = get_scaled_time(now);
686 bintime_add(&bt, &clock_calend.bintime);
687 bintime2usclock(&bt, secs, microsecs);
688
689 clock_gettimeofday_set_commpage(now, bt.sec, bt.frac, clock_calend.tick_scale_x, ticks_per_sec);
690
691 clock_unlock();
692 splx(s);
693
694 if (mach_time) {
695 *mach_time = now;
696 }
697}
698
699/*
700 * clock_set_calendar_microtime:
701 *
702 * Sets the current calendar value by
703 * recalculating the epoch and offset
704 * from the system clock.
705 *
706 * Also adjusts the boottime to keep the
707 * value consistent, writes the new
708 * calendar value to the platform clock,
709 * and sends calendar change notifications.
710 */
711void
712clock_set_calendar_microtime(
713 clock_sec_t secs,
714 clock_usec_t microsecs)
715{
716 uint64_t absolutesys;
717 clock_sec_t newsecs;
718 clock_sec_t oldsecs;
719 clock_usec_t newmicrosecs;
720 clock_usec_t oldmicrosecs;
721 uint64_t commpage_value;
722 spl_t s;
723 struct bintime bt;
724 clock_sec_t deltasecs;
725 clock_usec_t deltamicrosecs;
726
727 newsecs = secs;
728 newmicrosecs = microsecs;
729
730 /*
731 * settime_lock mtx is used to avoid that racing settimeofdays update the wall clock and
732 * the platform clock concurrently.
733 *
734 * clock_lock cannot be used for this race because it is acquired from interrupt context
735 * and it needs interrupts disabled while instead updating the platform clock needs to be
736 * called with interrupts enabled.
737 */
738 lck_mtx_lock(&settime_lock);
739
740 s = splclock();
741 clock_lock();
742
743#if DEVELOPMENT || DEBUG
744 struct clock_calend clock_calend_cp = clock_calend;
745#endif
746 commpage_disable_timestamp();
747
748 /*
749 * Adjust the boottime based on the delta.
750 */
751 clock_get_calendar_absolute_and_microtime_locked(&oldsecs, &oldmicrosecs, &absolutesys);
752
753#if DEVELOPMENT || DEBUG
754 if (g_should_log_clock_adjustments) {
755 os_log(OS_LOG_DEFAULT, "%s wall %lu s %d u computed with %llu abs\n",
756 __func__, (unsigned long)oldsecs, oldmicrosecs, absolutesys);
757 os_log(OS_LOG_DEFAULT, "%s requested %lu s %d u\n",
758 __func__, (unsigned long)secs, microsecs );
759 }
760#endif
761
762 if (oldsecs < secs || (oldsecs == secs && oldmicrosecs < microsecs)) {
763 // moving forwards
764 deltasecs = secs;
765 deltamicrosecs = microsecs;
766
767 TIME_SUB(deltasecs, oldsecs, deltamicrosecs, oldmicrosecs, USEC_PER_SEC);
768
769 TIME_ADD(clock_boottime, deltasecs, clock_boottime_usec, deltamicrosecs, USEC_PER_SEC);
770 clock2bintime(&deltasecs, &deltamicrosecs, &bt);
771 bintime_add(&clock_calend.boottime, &bt);
772 } else {
773 // moving backwards
774 deltasecs = oldsecs;
775 deltamicrosecs = oldmicrosecs;
776
777 TIME_SUB(deltasecs, secs, deltamicrosecs, microsecs, USEC_PER_SEC);
778
779 TIME_SUB(clock_boottime, deltasecs, clock_boottime_usec, deltamicrosecs, USEC_PER_SEC);
780 clock2bintime(&deltasecs, &deltamicrosecs, &bt);
781 bintime_sub(&clock_calend.boottime, &bt);
782 }
783
784 clock_calend.bintime = clock_calend.boottime;
785 bintime_add(&clock_calend.bintime, &clock_calend.offset);
786
787 clock2bintime((clock_sec_t *) &secs, (clock_usec_t *) &microsecs, &bt);
788
789 clock_gettimeofday_set_commpage(absolutesys, bt.sec, bt.frac, clock_calend.tick_scale_x, ticks_per_sec);
790
791#if DEVELOPMENT || DEBUG
792 struct clock_calend clock_calend_cp1 = clock_calend;
793#endif
794
795 commpage_value = clock_boottime * USEC_PER_SEC + clock_boottime_usec;
796
797 clock_unlock();
798 splx(s);
799
800 /*
801 * Set the new value for the platform clock.
802 * This call might block, so interrupts must be enabled.
803 */
804#if DEVELOPMENT || DEBUG
805 uint64_t now_b = mach_absolute_time();
806#endif
807
808 PESetUTCTimeOfDay(newsecs, newmicrosecs);
809
810#if DEVELOPMENT || DEBUG
811 uint64_t now_a = mach_absolute_time();
812 if (g_should_log_clock_adjustments) {
813 os_log(OS_LOG_DEFAULT, "%s mach bef PESet %llu mach aft %llu \n", __func__, now_b, now_a);
814 }
815#endif
816
817 print_all_clock_variables_internal(__func__, &clock_calend_cp);
818 print_all_clock_variables_internal(__func__, &clock_calend_cp1);
819
820 commpage_update_boottime(commpage_value);
821
822 /*
823 * Send host notifications.
824 */
825 host_notify_calendar_change();
826 host_notify_calendar_set();
827
828#if CONFIG_DTRACE
829 clock_track_calend_nowait();
830#endif
831
832 lck_mtx_unlock(&settime_lock);
833}
834
835uint64_t mach_absolutetime_asleep = 0;
836uint64_t mach_absolutetime_last_sleep = 0;
837
838void
839clock_get_calendar_uptime(clock_sec_t *secs)
840{
841 uint64_t now;
842 spl_t s;
843 struct bintime bt;
844
845 s = splclock();
846 clock_lock();
847
848 now = mach_absolute_time();
849
850 bt = get_scaled_time(now);
851 bintime_add(&bt, &clock_calend.offset);
852
853 *secs = bt.sec;
854
855 clock_unlock();
856 splx(s);
857}
858
859
860/*
861 * clock_update_calendar:
862 *
863 * called by ntp timer to update scale factors.
864 */
865void
866clock_update_calendar(void)
867{
868
869 uint64_t now, delta;
870 struct bintime bt;
871 spl_t s;
872 int64_t adjustment;
873
874 s = splclock();
875 clock_lock();
876
877 now = mach_absolute_time();
878
879 /*
880 * scale the time elapsed since the last update and
881 * add it to offset.
882 */
883 bt = get_scaled_time(now);
884 bintime_add(&clock_calend.offset, &bt);
885
886 /*
887 * update the base from which apply next scale factors.
888 */
889 delta = now - clock_calend.offset_count;
890 clock_calend.offset_count += delta;
891
892 clock_calend.bintime = clock_calend.offset;
893 bintime_add(&clock_calend.bintime, &clock_calend.boottime);
894
895 /*
896 * recompute next adjustment.
897 */
898 ntp_update_second(&adjustment, clock_calend.bintime.sec);
899
900#if DEVELOPMENT || DEBUG
901 if (g_should_log_clock_adjustments) {
902 os_log(OS_LOG_DEFAULT, "%s adjustment %lld\n", __func__, adjustment);
903 }
904#endif
905
906 /*
907 * recomputing scale factors.
908 */
909 get_scale_factors_from_adj(adjustment, &clock_calend.tick_scale_x, &clock_calend.s_scale_ns, &clock_calend.s_adj_nsx);
910
911 clock_gettimeofday_set_commpage(now, clock_calend.bintime.sec, clock_calend.bintime.frac, clock_calend.tick_scale_x, ticks_per_sec);
912
913#if DEVELOPMENT || DEBUG
914 struct clock_calend calend_cp = clock_calend;
915#endif
916
917 clock_unlock();
918 splx(s);
919
920 print_all_clock_variables(__func__, NULL,NULL,NULL,NULL, &calend_cp);
921}
922
923
924#if DEVELOPMENT || DEBUG
925
926void print_all_clock_variables_internal(const char* func, struct clock_calend* clock_calend_cp)
927{
928 clock_sec_t offset_secs;
929 clock_usec_t offset_microsecs;
930 clock_sec_t bintime_secs;
931 clock_usec_t bintime_microsecs;
932 clock_sec_t bootime_secs;
933 clock_usec_t bootime_microsecs;
934
935 if (!g_should_log_clock_adjustments)
936 return;
937
938 bintime2usclock(&clock_calend_cp->offset, &offset_secs, &offset_microsecs);
939 bintime2usclock(&clock_calend_cp->bintime, &bintime_secs, &bintime_microsecs);
940 bintime2usclock(&clock_calend_cp->boottime, &bootime_secs, &bootime_microsecs);
941
942 os_log(OS_LOG_DEFAULT, "%s s_scale_ns %llu s_adj_nsx %lld tick_scale_x %llu offset_count %llu\n",
943 func , clock_calend_cp->s_scale_ns, clock_calend_cp->s_adj_nsx,
944 clock_calend_cp->tick_scale_x, clock_calend_cp->offset_count);
945 os_log(OS_LOG_DEFAULT, "%s offset.sec %ld offset.frac %llu offset_secs %lu offset_microsecs %d\n",
946 func, clock_calend_cp->offset.sec, clock_calend_cp->offset.frac,
947 (unsigned long)offset_secs, offset_microsecs);
948 os_log(OS_LOG_DEFAULT, "%s bintime.sec %ld bintime.frac %llu bintime_secs %lu bintime_microsecs %d\n",
949 func, clock_calend_cp->bintime.sec, clock_calend_cp->bintime.frac,
950 (unsigned long)bintime_secs, bintime_microsecs);
951 os_log(OS_LOG_DEFAULT, "%s bootime.sec %ld bootime.frac %llu bootime_secs %lu bootime_microsecs %d\n",
952 func, clock_calend_cp->boottime.sec, clock_calend_cp->boottime.frac,
953 (unsigned long)bootime_secs, bootime_microsecs);
954
955 clock_sec_t basesleep_secs;
956 clock_usec_t basesleep_microsecs;
957
958 bintime2usclock(&clock_calend_cp->basesleep, &basesleep_secs, &basesleep_microsecs);
959 os_log(OS_LOG_DEFAULT, "%s basesleep.sec %ld basesleep.frac %llu basesleep_secs %lu basesleep_microsecs %d\n",
960 func, clock_calend_cp->basesleep.sec, clock_calend_cp->basesleep.frac,
961 (unsigned long)basesleep_secs, basesleep_microsecs);
962
963}
964
965
966void print_all_clock_variables(const char* func, clock_sec_t* pmu_secs, clock_usec_t* pmu_usec, clock_sec_t* sys_secs, clock_usec_t* sys_usec, struct clock_calend* clock_calend_cp)
967{
968 if (!g_should_log_clock_adjustments)
969 return;
970
971 struct bintime bt;
972 clock_sec_t wall_secs;
973 clock_usec_t wall_microsecs;
974 uint64_t now;
975 uint64_t delta;
976
977 if (pmu_secs) {
978 os_log(OS_LOG_DEFAULT, "%s PMU %lu s %d u \n", func, (unsigned long)*pmu_secs, *pmu_usec);
979 }
980 if (sys_secs) {
981 os_log(OS_LOG_DEFAULT, "%s sys %lu s %d u \n", func, (unsigned long)*sys_secs, *sys_usec);
982 }
983
984 print_all_clock_variables_internal(func, clock_calend_cp);
985
986 now = mach_absolute_time();
987 delta = now - clock_calend_cp->offset_count;
988
989 bt = scale_delta(delta, clock_calend_cp->tick_scale_x, clock_calend_cp->s_scale_ns, clock_calend_cp->s_adj_nsx);
990 bintime_add(&bt, &clock_calend_cp->bintime);
991 bintime2usclock(&bt, &wall_secs, &wall_microsecs);
992
993 os_log(OS_LOG_DEFAULT, "%s wall %lu s %d u computed with %llu abs\n",
994 func, (unsigned long)wall_secs, wall_microsecs, now);
995}
996
997
998#endif /* DEVELOPMENT || DEBUG */
999
1000
1001/*
1002 * clock_initialize_calendar:
1003 *
1004 * Set the calendar and related clocks
1005 * from the platform clock at boot.
1006 *
1007 * Also sends host notifications.
1008 */
1009void
1010clock_initialize_calendar(void)
1011{
1012 clock_sec_t sys; // sleepless time since boot in seconds
1013 clock_sec_t secs; // Current UTC time
1014 clock_sec_t utc_offset_secs; // Difference in current UTC time and sleepless time since boot
1015 clock_usec_t microsys;
1016 clock_usec_t microsecs;
1017 clock_usec_t utc_offset_microsecs;
1018 spl_t s;
1019 struct bintime bt;
1020 struct bintime monotonic_bt;
1021 struct latched_time monotonic_time;
1022 uint64_t monotonic_usec_total;
1023 clock_sec_t sys2, monotonic_sec;
1024 clock_usec_t microsys2, monotonic_usec;
1025 size_t size;
1026
1027 //Get the UTC time and corresponding sys time
1028 PEGetUTCTimeOfDay(&secs, &microsecs);
1029 clock_get_system_microtime(&sys, &microsys);
1030
1031 /*
1032 * If the platform has a monotonic clock, use kern.monotonicclock_usecs
1033 * to estimate the sleep/wake time, otherwise use the UTC time to estimate
1034 * the sleep time.
1035 */
1036 size = sizeof(monotonic_time);
1037 if (kernel_sysctlbyname("kern.monotonicclock_usecs", &monotonic_time, &size, NULL, 0) != 0) {
1038 has_monotonic_clock = 0;
1039 os_log(OS_LOG_DEFAULT, "%s system does not have monotonic clock\n", __func__);
1040 } else {
1041 has_monotonic_clock = 1;
1042 monotonic_usec_total = monotonic_time.monotonic_time_usec;
1043 absolutetime_to_microtime(monotonic_time.mach_time, &sys2, &microsys2);
1044 os_log(OS_LOG_DEFAULT, "%s system has monotonic clock\n", __func__);
1045 }
1046
1047 s = splclock();
1048 clock_lock();
1049
1050 commpage_disable_timestamp();
1051
1052 utc_offset_secs = secs;
1053 utc_offset_microsecs = microsecs;
1054
1055 /*
1056 * We normally expect the UTC clock to be always-on and produce
1057 * greater readings than the tick counter. There may be corner cases
1058 * due to differing clock resolutions (UTC clock is likely lower) and
1059 * and errors reading the UTC clock (some implementations return 0
1060 * on error) in which that doesn't hold true. Bring the UTC measurements
1061 * in-line with the tick counter measurements as a best effort in that case.
1062 */
1063 if ((sys > secs) || ((sys == secs) && (microsys > microsecs))) {
1064 os_log(OS_LOG_DEFAULT, "%s WARNING: UTC time is less then sys time, (%lu s %d u) UTC (%lu s %d u) sys\n",
1065 __func__, (unsigned long) secs, microsecs, (unsigned long)sys, microsys);
1066 secs = utc_offset_secs = sys;
1067 microsecs = utc_offset_microsecs = microsys;
1068 }
1069
1070 // UTC - sys
1071 // This macro stores the subtraction result in utc_offset_secs and utc_offset_microsecs
1072 TIME_SUB(utc_offset_secs, sys, utc_offset_microsecs, microsys, USEC_PER_SEC);
1073 // This function converts utc_offset_secs and utc_offset_microsecs in bintime
1074 clock2bintime(&utc_offset_secs, &utc_offset_microsecs, &bt);
1075
1076 /*
1077 * Initialize the boot time based on the platform clock.
1078 */
1079 clock_boottime = secs;
1080 clock_boottime_usec = microsecs;
1081 commpage_update_boottime(clock_boottime * USEC_PER_SEC + clock_boottime_usec);
1082
1083 nanoseconds_to_absolutetime((uint64_t)NSEC_PER_SEC, &ticks_per_sec);
1084 clock_calend.boottime = bt;
1085 clock_calend.bintime = bt;
1086 clock_calend.offset.sec = 0;
1087 clock_calend.offset.frac = 0;
1088
1089 clock_calend.tick_scale_x = (uint64_t)1 << 63;
1090 clock_calend.tick_scale_x /= ticks_per_sec;
1091 clock_calend.tick_scale_x *= 2;
1092
1093 clock_calend.s_scale_ns = NSEC_PER_SEC;
1094 clock_calend.s_adj_nsx = 0;
1095
1096 if (has_monotonic_clock) {
1097
1098 monotonic_sec = monotonic_usec_total / (clock_sec_t)USEC_PER_SEC;
1099 monotonic_usec = monotonic_usec_total % (clock_usec_t)USEC_PER_SEC;
1100
1101 // monotonic clock - sys
1102 // This macro stores the subtraction result in monotonic_sec and monotonic_usec
1103 TIME_SUB(monotonic_sec, sys2, monotonic_usec, microsys2, USEC_PER_SEC);
1104 clock2bintime(&monotonic_sec, &monotonic_usec, &monotonic_bt);
1105
1106 // set the baseleep as the difference between monotonic clock - sys
1107 clock_calend.basesleep = monotonic_bt;
1108 }
1109 commpage_update_mach_continuous_time(mach_absolutetime_asleep);
1110
1111#if DEVELOPMENT || DEBUG
1112 struct clock_calend clock_calend_cp = clock_calend;
1113#endif
1114
1115 clock_unlock();
1116 splx(s);
1117
1118 print_all_clock_variables(__func__, &secs, &microsecs, &sys, &microsys, &clock_calend_cp);
1119
1120 /*
1121 * Send host notifications.
1122 */
1123 host_notify_calendar_change();
1124
1125#if CONFIG_DTRACE
1126 clock_track_calend_nowait();
1127#endif
1128}
1129
1130
1131void
1132clock_wakeup_calendar(void)
1133{
1134 clock_sec_t wake_sys_sec;
1135 clock_usec_t wake_sys_usec;
1136 clock_sec_t wake_sec;
1137 clock_usec_t wake_usec;
1138 clock_sec_t wall_time_sec;
1139 clock_usec_t wall_time_usec;
1140 clock_sec_t diff_sec;
1141 clock_usec_t diff_usec;
1142 clock_sec_t var_s;
1143 clock_usec_t var_us;
1144 spl_t s;
1145 struct bintime bt, last_sleep_bt;
1146 struct latched_time monotonic_time;
1147 uint64_t monotonic_usec_total;
1148 uint64_t wake_abs;
1149 size_t size;
1150
1151 /*
1152 * If the platform has the monotonic clock use that to
1153 * compute the sleep time. The monotonic clock does not have an offset
1154 * that can be modified, so nor kernel or userspace can change the time
1155 * of this clock, it can only monotonically increase over time.
1156 * During sleep mach_absolute_time (sys time) does not tick,
1157 * so the sleep time is the difference between the current monotonic time
1158 * less the absolute time and the previous difference stored at wake time.
1159 *
1160 * basesleep = (monotonic - sys) ---> computed at last wake
1161 * sleep_time = (monotonic - sys) - basesleep
1162 *
1163 * If the platform does not support monotonic clock we set the wall time to what the
1164 * UTC clock returns us.
1165 * Setting the wall time to UTC time implies that we loose all the adjustments
1166 * done during wake time through adjtime/ntp_adjustime.
1167 * The UTC time is the monotonic clock + an offset that can be set
1168 * by kernel.
1169 * The time slept in this case is the difference between wall time and UTC
1170 * at wake.
1171 *
1172 * IMPORTANT:
1173 * We assume that only the kernel is setting the offset of the PMU/RTC and that
1174 * it is doing it only througth the settimeofday interface.
1175 */
1176 if (has_monotonic_clock) {
1177
1178#if DEVELOPMENT || DEBUG
1179 /*
1180 * Just for debugging, get the wake UTC time.
1181 */
1182 PEGetUTCTimeOfDay(&var_s, &var_us);
1183#endif
1184 /*
1185 * Get monotonic time with corresponding sys time
1186 */
1187 size = sizeof(monotonic_time);
1188 if (kernel_sysctlbyname("kern.monotonicclock_usecs", &monotonic_time, &size, NULL, 0) != 0) {
1189 panic("%s: could not call kern.monotonicclock_usecs", __func__);
1190 }
1191 wake_abs = monotonic_time.mach_time;
1192 absolutetime_to_microtime(wake_abs, &wake_sys_sec, &wake_sys_usec);
1193
1194 monotonic_usec_total = monotonic_time.monotonic_time_usec;
1195 wake_sec = monotonic_usec_total / (clock_sec_t)USEC_PER_SEC;
1196 wake_usec = monotonic_usec_total % (clock_usec_t)USEC_PER_SEC;
1197 } else {
1198 /*
1199 * Get UTC time and corresponding sys time
1200 */
1201 PEGetUTCTimeOfDay(&wake_sec, &wake_usec);
1202 wake_abs = mach_absolute_time();
1203 absolutetime_to_microtime(wake_abs, &wake_sys_sec, &wake_sys_usec);
1204 }
1205
1206#if DEVELOPMENT || DEBUG
1207 os_log(OS_LOG_DEFAULT, "time at wake %lu s %d u from %s clock, abs %llu\n", (unsigned long)wake_sec, wake_usec, (has_monotonic_clock)?"monotonic":"UTC", wake_abs);
1208 if (has_monotonic_clock) {
1209 os_log(OS_LOG_DEFAULT, "UTC time %lu s %d u\n", (unsigned long)var_s, var_us);
1210 }
1211#endif /* DEVELOPMENT || DEBUG */
1212
1213 s = splclock();
1214 clock_lock();
1215
1216 commpage_disable_timestamp();
1217
1218#if DEVELOPMENT || DEBUG
1219 struct clock_calend clock_calend_cp1 = clock_calend;
1220#endif /* DEVELOPMENT || DEBUG */
1221
1222 /*
1223 * We normally expect the UTC/monotonic clock to be always-on and produce
1224 * greater readings than the sys counter. There may be corner cases
1225 * due to differing clock resolutions (UTC/monotonic clock is likely lower) and
1226 * and errors reading the UTC/monotonic clock (some implementations return 0
1227 * on error) in which that doesn't hold true.
1228 */
1229 if ((wake_sys_sec > wake_sec) || ((wake_sys_sec == wake_sec) && (wake_sys_usec > wake_usec))) {
1230 os_log_error(OS_LOG_DEFAULT, "WARNING: %s clock is less then sys clock at wake: %lu s %d u vs %lu s %d u, defaulting sleep time to zero\n", (has_monotonic_clock)?"monotonic":"UTC", (unsigned long)wake_sec, wake_usec, (unsigned long)wake_sys_sec, wake_sys_usec);
1231 mach_absolutetime_last_sleep = 0;
1232 goto done;
1233 }
1234
1235 if (has_monotonic_clock) {
1236 /*
1237 * computer the difference monotonic - sys
1238 * we already checked that monotonic time is
1239 * greater than sys.
1240 */
1241 diff_sec = wake_sec;
1242 diff_usec = wake_usec;
1243 // This macro stores the subtraction result in diff_sec and diff_usec
1244 TIME_SUB(diff_sec, wake_sys_sec, diff_usec, wake_sys_usec, USEC_PER_SEC);
1245 //This function converts diff_sec and diff_usec in bintime
1246 clock2bintime(&diff_sec, &diff_usec, &bt);
1247
1248 /*
1249 * Safety belt: the monotonic clock will likely have a lower resolution than the sys counter.
1250 * It's also possible that the device didn't fully transition to the powered-off state on
1251 * the most recent sleep, so the sys counter may not have reset or may have only briefly
1252 * turned off. In that case it's possible for the difference between the monotonic clock and the
1253 * sys counter to be less than the previously recorded value in clock.calend.basesleep.
1254 * In that case simply record that we slept for 0 ticks.
1255 */
1256 if ((bt.sec > clock_calend.basesleep.sec) ||
1257 ((bt.sec == clock_calend.basesleep.sec) && (bt.frac > clock_calend.basesleep.frac))) {
1258
1259 //last_sleep is the difference between (current monotonic - abs) and (last wake monotonic - abs)
1260 last_sleep_bt = bt;
1261 bintime_sub(&last_sleep_bt, &clock_calend.basesleep);
1262
1263 bintime2absolutetime(&last_sleep_bt, &mach_absolutetime_last_sleep);
1264 mach_absolutetime_asleep += mach_absolutetime_last_sleep;
1265
1266 //set basesleep to current monotonic - abs
1267 clock_calend.basesleep = bt;
1268
1269 //update wall time
1270 bintime_add(&clock_calend.offset, &last_sleep_bt);
1271 bintime_add(&clock_calend.bintime, &last_sleep_bt);
1272
1273 bintime2usclock(&last_sleep_bt, &var_s, &var_us);
1274 os_log(OS_LOG_DEFAULT, "time_slept (%lu s %d u)\n", (unsigned long) var_s, var_us);
1275
1276 } else {
1277 bintime2usclock(&clock_calend.basesleep, &var_s, &var_us);
1278 os_log_error(OS_LOG_DEFAULT, "WARNING: last wake monotonic-sys time (%lu s %d u) is greater then current monotonic-sys time(%lu s %d u), defaulting sleep time to zero\n", (unsigned long) var_s, var_us, (unsigned long) diff_sec, diff_usec);
1279
1280 mach_absolutetime_last_sleep = 0;
1281 }
1282 } else {
1283 /*
1284 * set the wall time to UTC value
1285 */
1286 bt = get_scaled_time(wake_abs);
1287 bintime_add(&bt, &clock_calend.bintime);
1288 bintime2usclock(&bt, &wall_time_sec, &wall_time_usec);
1289
1290 if (wall_time_sec > wake_sec || (wall_time_sec == wake_sec && wall_time_usec > wake_usec) ) {
1291 os_log(OS_LOG_DEFAULT, "WARNING: wall time (%lu s %d u) is greater than current UTC time (%lu s %d u), defaulting sleep time to zero\n", (unsigned long) wall_time_sec, wall_time_usec, (unsigned long) wake_sec, wake_usec);
1292
1293 mach_absolutetime_last_sleep = 0;
1294 } else {
1295 diff_sec = wake_sec;
1296 diff_usec = wake_usec;
1297 // This macro stores the subtraction result in diff_sec and diff_usec
1298 TIME_SUB(diff_sec, wall_time_sec, diff_usec, wall_time_usec, USEC_PER_SEC);
1299 //This function converts diff_sec and diff_usec in bintime
1300 clock2bintime(&diff_sec, &diff_usec, &bt);
1301
1302 //time slept in this case is the difference between PMU/RTC and wall time
1303 last_sleep_bt = bt;
1304
1305 bintime2absolutetime(&last_sleep_bt, &mach_absolutetime_last_sleep);
1306 mach_absolutetime_asleep += mach_absolutetime_last_sleep;
1307
1308 //update wall time
1309 bintime_add(&clock_calend.offset, &last_sleep_bt);
1310 bintime_add(&clock_calend.bintime, &last_sleep_bt);
1311
1312 bintime2usclock(&last_sleep_bt, &var_s, &var_us);
1313 os_log(OS_LOG_DEFAULT, "time_slept (%lu s %d u)\n", (unsigned long)var_s, var_us);
1314 }
1315 }
1316done:
1317 KERNEL_DEBUG_CONSTANT(
1318 MACHDBG_CODE(DBG_MACH_CLOCK,MACH_EPOCH_CHANGE) | DBG_FUNC_NONE,
1319 (uintptr_t) mach_absolutetime_last_sleep,
1320 (uintptr_t) mach_absolutetime_asleep,
1321 (uintptr_t) (mach_absolutetime_last_sleep >> 32),
1322 (uintptr_t) (mach_absolutetime_asleep >> 32),
1323 0);
1324
1325 commpage_update_mach_continuous_time(mach_absolutetime_asleep);
1326 adjust_cont_time_thread_calls();
1327
1328#if DEVELOPMENT || DEBUG
1329 struct clock_calend clock_calend_cp = clock_calend;
1330#endif
1331
1332 clock_unlock();
1333 splx(s);
1334
1335#if DEVELOPMENT || DEBUG
1336 if (g_should_log_clock_adjustments) {
1337 print_all_clock_variables("clock_wakeup_calendar: BEFORE", NULL, NULL, NULL, NULL, &clock_calend_cp1);
1338 print_all_clock_variables("clock_wakeup_calendar: AFTER", NULL, NULL, NULL, NULL, &clock_calend_cp);
1339 }
1340#endif /* DEVELOPMENT || DEBUG */
1341
1342 host_notify_calendar_change();
1343
1344#if CONFIG_DTRACE
1345 clock_track_calend_nowait();
1346#endif
1347}
1348
1349
1350/*
1351 * clock_get_boottime_nanotime:
1352 *
1353 * Return the boottime, used by sysctl.
1354 */
1355void
1356clock_get_boottime_nanotime(
1357 clock_sec_t *secs,
1358 clock_nsec_t *nanosecs)
1359{
1360 spl_t s;
1361
1362 s = splclock();
1363 clock_lock();
1364
1365 *secs = (clock_sec_t)clock_boottime;
1366 *nanosecs = (clock_nsec_t)clock_boottime_usec * NSEC_PER_USEC;
1367
1368 clock_unlock();
1369 splx(s);
1370}
1371
1372/*
1373 * clock_get_boottime_nanotime:
1374 *
1375 * Return the boottime, used by sysctl.
1376 */
1377void
1378clock_get_boottime_microtime(
1379 clock_sec_t *secs,
1380 clock_usec_t *microsecs)
1381{
1382 spl_t s;
1383
1384 s = splclock();
1385 clock_lock();
1386
1387 *secs = (clock_sec_t)clock_boottime;
1388 *microsecs = (clock_nsec_t)clock_boottime_usec;
1389
1390 clock_unlock();
1391 splx(s);
1392}
1393
1394
1395/*
1396 * Wait / delay routines.
1397 */
1398static void
1399mach_wait_until_continue(
1400 __unused void *parameter,
1401 wait_result_t wresult)
1402{
1403 thread_syscall_return((wresult == THREAD_INTERRUPTED)? KERN_ABORTED: KERN_SUCCESS);
1404 /*NOTREACHED*/
1405}
1406
1407/*
1408 * mach_wait_until_trap: Suspend execution of calling thread until the specified time has passed
1409 *
1410 * Parameters: args->deadline Amount of time to wait
1411 *
1412 * Returns: 0 Success
1413 * !0 Not success
1414 *
1415 */
1416kern_return_t
1417mach_wait_until_trap(
1418 struct mach_wait_until_trap_args *args)
1419{
1420 uint64_t deadline = args->deadline;
1421 wait_result_t wresult;
1422
1423 wresult = assert_wait_deadline_with_leeway((event_t)mach_wait_until_trap, THREAD_ABORTSAFE,
1424 TIMEOUT_URGENCY_USER_NORMAL, deadline, 0);
1425 if (wresult == THREAD_WAITING)
1426 wresult = thread_block(mach_wait_until_continue);
1427
1428 return ((wresult == THREAD_INTERRUPTED)? KERN_ABORTED: KERN_SUCCESS);
1429}
1430
1431void
1432clock_delay_until(
1433 uint64_t deadline)
1434{
1435 uint64_t now = mach_absolute_time();
1436
1437 if (now >= deadline)
1438 return;
1439
1440 _clock_delay_until_deadline(deadline - now, deadline);
1441}
1442
1443/*
1444 * Preserve the original precise interval that the client
1445 * requested for comparison to the spin threshold.
1446 */
1447void
1448_clock_delay_until_deadline(
1449 uint64_t interval,
1450 uint64_t deadline)
1451{
1452 _clock_delay_until_deadline_with_leeway(interval, deadline, 0);
1453}
1454
1455/*
1456 * Like _clock_delay_until_deadline, but it accepts a
1457 * leeway value.
1458 */
1459void
1460_clock_delay_until_deadline_with_leeway(
1461 uint64_t interval,
1462 uint64_t deadline,
1463 uint64_t leeway)
1464{
1465
1466 if (interval == 0)
1467 return;
1468
1469 if ( ml_delay_should_spin(interval) ||
1470 get_preemption_level() != 0 ||
1471 ml_get_interrupts_enabled() == FALSE ) {
1472 machine_delay_until(interval, deadline);
1473 } else {
1474 /*
1475 * For now, assume a leeway request of 0 means the client does not want a leeway
1476 * value. We may want to change this interpretation in the future.
1477 */
1478
1479 if (leeway) {
1480 assert_wait_deadline_with_leeway((event_t)clock_delay_until, THREAD_UNINT, TIMEOUT_URGENCY_LEEWAY, deadline, leeway);
1481 } else {
1482 assert_wait_deadline((event_t)clock_delay_until, THREAD_UNINT, deadline);
1483 }
1484
1485 thread_block(THREAD_CONTINUE_NULL);
1486 }
1487}
1488
1489void
1490delay_for_interval(
1491 uint32_t interval,
1492 uint32_t scale_factor)
1493{
1494 uint64_t abstime;
1495
1496 clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime);
1497
1498 _clock_delay_until_deadline(abstime, mach_absolute_time() + abstime);
1499}
1500
1501void
1502delay_for_interval_with_leeway(
1503 uint32_t interval,
1504 uint32_t leeway,
1505 uint32_t scale_factor)
1506{
1507 uint64_t abstime_interval;
1508 uint64_t abstime_leeway;
1509
1510 clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime_interval);
1511 clock_interval_to_absolutetime_interval(leeway, scale_factor, &abstime_leeway);
1512
1513 _clock_delay_until_deadline_with_leeway(abstime_interval, mach_absolute_time() + abstime_interval, abstime_leeway);
1514}
1515
1516void
1517delay(
1518 int usec)
1519{
1520 delay_for_interval((usec < 0)? -usec: usec, NSEC_PER_USEC);
1521}
1522
1523/*
1524 * Miscellaneous routines.
1525 */
1526void
1527clock_interval_to_deadline(
1528 uint32_t interval,
1529 uint32_t scale_factor,
1530 uint64_t *result)
1531{
1532 uint64_t abstime;
1533
1534 clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime);
1535
1536 *result = mach_absolute_time() + abstime;
1537}
1538
1539void
1540clock_absolutetime_interval_to_deadline(
1541 uint64_t abstime,
1542 uint64_t *result)
1543{
1544 *result = mach_absolute_time() + abstime;
1545}
1546
1547void
1548clock_continuoustime_interval_to_deadline(
1549 uint64_t conttime,
1550 uint64_t *result)
1551{
1552 *result = mach_continuous_time() + conttime;
1553}
1554
1555void
1556clock_get_uptime(
1557 uint64_t *result)
1558{
1559 *result = mach_absolute_time();
1560}
1561
1562void
1563clock_deadline_for_periodic_event(
1564 uint64_t interval,
1565 uint64_t abstime,
1566 uint64_t *deadline)
1567{
1568 assert(interval != 0);
1569
1570 *deadline += interval;
1571
1572 if (*deadline <= abstime) {
1573 *deadline = abstime + interval;
1574 abstime = mach_absolute_time();
1575
1576 if (*deadline <= abstime)
1577 *deadline = abstime + interval;
1578 }
1579}
1580
1581uint64_t
1582mach_continuous_time(void)
1583{
1584 while(1) {
1585 uint64_t read1 = mach_absolutetime_asleep;
1586 uint64_t absolute = mach_absolute_time();
1587 OSMemoryBarrier();
1588 uint64_t read2 = mach_absolutetime_asleep;
1589
1590 if(__builtin_expect(read1 == read2, 1)) {
1591 return absolute + read1;
1592 }
1593 }
1594}
1595
1596uint64_t
1597mach_continuous_approximate_time(void)
1598{
1599 while(1) {
1600 uint64_t read1 = mach_absolutetime_asleep;
1601 uint64_t absolute = mach_approximate_time();
1602 OSMemoryBarrier();
1603 uint64_t read2 = mach_absolutetime_asleep;
1604
1605 if(__builtin_expect(read1 == read2, 1)) {
1606 return absolute + read1;
1607 }
1608 }
1609}
1610
1611/*
1612 * continuoustime_to_absolutetime
1613 * Must be called with interrupts disabled
1614 * Returned value is only valid until the next update to
1615 * mach_continuous_time
1616 */
1617uint64_t
1618continuoustime_to_absolutetime(uint64_t conttime) {
1619 if (conttime <= mach_absolutetime_asleep)
1620 return 0;
1621 else
1622 return conttime - mach_absolutetime_asleep;
1623}
1624
1625/*
1626 * absolutetime_to_continuoustime
1627 * Must be called with interrupts disabled
1628 * Returned value is only valid until the next update to
1629 * mach_continuous_time
1630 */
1631uint64_t
1632absolutetime_to_continuoustime(uint64_t abstime) {
1633 return abstime + mach_absolutetime_asleep;
1634}
1635
1636#if CONFIG_DTRACE
1637
1638/*
1639 * clock_get_calendar_nanotime_nowait
1640 *
1641 * Description: Non-blocking version of clock_get_calendar_nanotime()
1642 *
1643 * Notes: This function operates by separately tracking calendar time
1644 * updates using a two element structure to copy the calendar
1645 * state, which may be asynchronously modified. It utilizes
1646 * barrier instructions in the tracking process and in the local
1647 * stable snapshot process in order to ensure that a consistent
1648 * snapshot is used to perform the calculation.
1649 */
1650void
1651clock_get_calendar_nanotime_nowait(
1652 clock_sec_t *secs,
1653 clock_nsec_t *nanosecs)
1654{
1655 int i = 0;
1656 uint64_t now;
1657 struct unlocked_clock_calend stable;
1658 struct bintime bt;
1659
1660 for (;;) {
1661 stable = flipflop[i]; /* take snapshot */
1662
1663 /*
1664 * Use a barrier instructions to ensure atomicity. We AND
1665 * off the "in progress" bit to get the current generation
1666 * count.
1667 */
1668 (void)hw_atomic_and(&stable.gen, ~(uint32_t)1);
1669
1670 /*
1671 * If an update _is_ in progress, the generation count will be
1672 * off by one, if it _was_ in progress, it will be off by two,
1673 * and if we caught it at a good time, it will be equal (and
1674 * our snapshot is threfore stable).
1675 */
1676 if (flipflop[i].gen == stable.gen)
1677 break;
1678
1679 /* Switch to the other element of the flipflop, and try again. */
1680 i ^= 1;
1681 }
1682
1683 now = mach_absolute_time();
1684
1685 bt = get_scaled_time(now);
1686
1687 bintime_add(&bt, &clock_calend.bintime);
1688
1689 bintime2nsclock(&bt, secs, nanosecs);
1690}
1691
1692static void
1693clock_track_calend_nowait(void)
1694{
1695 int i;
1696
1697 for (i = 0; i < 2; i++) {
1698 struct clock_calend tmp = clock_calend;
1699
1700 /*
1701 * Set the low bit if the generation count; since we use a
1702 * barrier instruction to do this, we are guaranteed that this
1703 * will flag an update in progress to an async caller trying
1704 * to examine the contents.
1705 */
1706 (void)hw_atomic_or(&flipflop[i].gen, 1);
1707
1708 flipflop[i].calend = tmp;
1709
1710 /*
1711 * Increment the generation count to clear the low bit to
1712 * signal completion. If a caller compares the generation
1713 * count after taking a copy while in progress, the count
1714 * will be off by two.
1715 */
1716 (void)hw_atomic_add(&flipflop[i].gen, 1);
1717 }
1718}
1719
1720#endif /* CONFIG_DTRACE */
1721
1722