1 | /* |
2 | * Copyright (c) 2017-2020 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | #include <mach/mach_time.h> |
29 | #include <mach/clock_types.h> |
30 | #include <kern/misc_protos.h> |
31 | #include <kern/clock.h> |
32 | #include <kern/remote_time.h> |
33 | #include <kern/spl.h> |
34 | #include <kern/locks.h> |
35 | #include <sys/kdebug.h> |
36 | #include <machine/machine_routines.h> |
37 | #include <kern/assert.h> |
38 | #include <kern/kern_types.h> |
39 | #include <kern/thread.h> |
40 | #include <machine/commpage.h> |
41 | #include <machine/atomic.h> |
42 | |
43 | LCK_GRP_DECLARE(bt_lck_grp, "bridge timestamp" ); |
44 | LCK_SPIN_DECLARE(bt_spin_lock, &bt_lck_grp); |
45 | LCK_SPIN_DECLARE(bt_ts_conversion_lock, &bt_lck_grp); |
46 | LCK_SPIN_DECLARE(bt_maintenance_lock, &bt_lck_grp); |
47 | |
48 | #if CONFIG_MACH_BRIDGE_SEND_TIME |
49 | |
50 | uint32_t bt_enable_flag = 0; |
51 | _Atomic uint32_t bt_init_flag = 0; |
52 | |
53 | void mach_bridge_timer_maintenance(void); |
54 | uint32_t mach_bridge_timer_enable(uint32_t new_value, int change); |
55 | |
56 | /* |
57 | * When CONFIG_MACH_BRIDGE_SEND_TIME is defined, it is expected |
58 | * that a machine-specific timestamp sending routine such as |
59 | * void mach_bridge_send_timestamp(uint64_t); has also been defined. |
60 | */ |
61 | extern void mach_bridge_send_timestamp(uint64_t); |
62 | |
63 | void |
64 | mach_bridge_timer_maintenance(void) |
65 | { |
66 | if (!os_atomic_load(&bt_init_flag, acquire)) { |
67 | return; |
68 | } |
69 | |
70 | lck_spin_lock(&bt_maintenance_lock); |
71 | if (!bt_enable_flag) { |
72 | goto done; |
73 | } |
74 | mach_bridge_send_timestamp(0); |
75 | |
76 | done: |
77 | lck_spin_unlock(&bt_maintenance_lock); |
78 | } |
79 | |
80 | /* |
81 | * If change = 0, return the current value of bridge_timer_enable |
82 | * If change = 1, update bridge_timer_enable and return the updated |
83 | * value |
84 | */ |
85 | uint32_t |
86 | mach_bridge_timer_enable(uint32_t new_value, int change) |
87 | { |
88 | uint32_t current_value = 0; |
89 | assert(os_atomic_load(&bt_init_flag, relaxed)); |
90 | lck_spin_lock(&bt_maintenance_lock); |
91 | if (change) { |
92 | bt_enable_flag = new_value; |
93 | } |
94 | current_value = bt_enable_flag; |
95 | lck_spin_unlock(&bt_maintenance_lock); |
96 | return current_value; |
97 | } |
98 | |
99 | #endif /* CONFIG_MACH_BRIDGE_SEND_TIME */ |
100 | |
101 | #if CONFIG_MACH_BRIDGE_RECV_TIME |
102 | #include <machine/machine_remote_time.h> |
103 | |
104 | /* |
105 | * functions used by machine-specific code |
106 | * that implements CONFIG_MACH_BRIDGE_RECV_TIME |
107 | */ |
108 | void mach_bridge_add_timestamp(uint64_t remote_timestamp, uint64_t local_timestamp); |
109 | void bt_calibration_thread_start(void); |
110 | void bt_params_add(struct bt_params *params); |
111 | |
112 | /* function called by sysctl */ |
113 | struct bt_params bt_params_get_latest(void); |
114 | |
115 | /* |
116 | * Platform specific bridge time receiving interface. |
117 | * These variables should be exported by the platform specific time receiving code. |
118 | */ |
119 | extern _Atomic uint32_t bt_init_flag; |
120 | |
121 | static uint64_t received_local_timestamp = 0; |
122 | static uint64_t received_remote_timestamp = 0; |
123 | /* |
124 | * Buffer the previous timestamp pairs and rate |
125 | * It is protected by the bt_ts_conversion_lock |
126 | */ |
127 | #define BT_PARAMS_COUNT 10 |
128 | static struct bt_params bt_params_hist[BT_PARAMS_COUNT] = {}; |
129 | static int bt_params_idx = -1; |
130 | |
131 | void |
132 | bt_params_add(struct bt_params *params) |
133 | { |
134 | lck_spin_assert(lck: &bt_ts_conversion_lock, LCK_ASSERT_OWNED); |
135 | |
136 | bt_params_idx = (bt_params_idx + 1) % BT_PARAMS_COUNT; |
137 | bt_params_hist[bt_params_idx] = *params; |
138 | } |
139 | |
140 | #if defined(XNU_TARGET_OS_BRIDGE) |
141 | static inline struct bt_params* |
142 | bt_params_find(uint64_t local_ts) |
143 | { |
144 | lck_spin_assert(&bt_ts_conversion_lock, LCK_ASSERT_OWNED); |
145 | |
146 | int idx = bt_params_idx; |
147 | if (idx < 0) { |
148 | return NULL; |
149 | } |
150 | do { |
151 | if (local_ts >= bt_params_hist[idx].base_local_ts) { |
152 | return &bt_params_hist[idx]; |
153 | } |
154 | if (--idx < 0) { |
155 | idx = BT_PARAMS_COUNT - 1; |
156 | } |
157 | } while (idx != bt_params_idx); |
158 | |
159 | return NULL; |
160 | } |
161 | #endif /* defined(XNU_TARGET_OS_BRIDGE) */ |
162 | |
163 | static inline struct bt_params |
164 | bt_params_get_latest_locked(void) |
165 | { |
166 | lck_spin_assert(lck: &bt_ts_conversion_lock, LCK_ASSERT_OWNED); |
167 | |
168 | struct bt_params latest_params = {}; |
169 | if (bt_params_idx >= 0) { |
170 | latest_params = bt_params_hist[bt_params_idx]; |
171 | } |
172 | |
173 | return latest_params; |
174 | } |
175 | |
176 | struct bt_params |
177 | bt_params_get_latest(void) |
178 | { |
179 | struct bt_params latest_params = {}; |
180 | |
181 | /* Check if ts_converison_lock has been initialized */ |
182 | if (os_atomic_load(&bt_init_flag, acquire)) { |
183 | lck_spin_lock(lck: &bt_ts_conversion_lock); |
184 | latest_params = bt_params_get_latest_locked(); |
185 | lck_spin_unlock(lck: &bt_ts_conversion_lock); |
186 | } |
187 | return latest_params; |
188 | } |
189 | |
190 | /* |
191 | * Conditions: bt_spin_lock held and called from primary interrupt context |
192 | */ |
193 | void |
194 | mach_bridge_add_timestamp(uint64_t remote_timestamp, uint64_t local_timestamp) |
195 | { |
196 | lck_spin_assert(lck: &bt_spin_lock, LCK_ASSERT_OWNED); |
197 | |
198 | /* sleep/wake might return the same mach_absolute_time as the previous timestamp pair */ |
199 | if ((received_local_timestamp == local_timestamp) || |
200 | (received_remote_timestamp == remote_timestamp)) { |
201 | return; |
202 | } |
203 | |
204 | received_local_timestamp = local_timestamp; |
205 | received_remote_timestamp = remote_timestamp; |
206 | thread_wakeup((event_t)bt_params_hist); |
207 | } |
208 | |
209 | static double |
210 | mach_bridge_compute_rate(uint64_t new_local_ts, uint64_t new_remote_ts, |
211 | uint64_t old_local_ts, uint64_t old_remote_ts) |
212 | { |
213 | int64_t rdiff = (int64_t)new_remote_ts - (int64_t)old_remote_ts; |
214 | int64_t ldiff = (int64_t)new_local_ts - (int64_t)old_local_ts; |
215 | double calc_rate = ((double)rdiff) / (double)ldiff; |
216 | return calc_rate; |
217 | } |
218 | |
219 | #define MAX_RECALCULATE_COUNT 8 |
220 | #define CUMULATIVE_RATE_DECAY_CONSTANT 0.01 |
221 | #define CUMULATIVE_RATE_WEIGHT 0.99 |
222 | #define INITIAL_RATE 1.0 |
223 | #define MIN_INITIAL_SAMPLE_COUNT 10 |
224 | #define MAX_INITIAL_SAMPLE_COUNT 50 |
225 | #define MAX_SKIP_RESET_COUNT 2 |
226 | #define MIN_LOCAL_TS_DISTANCE_NS 100000000 /* 100 ms */ |
227 | #define MAX_LOCAL_TS_DISTANCE_NS 350000000 /* 350 ms */ |
228 | #define TS_PAIR_MISMATCH_THRESHOLD_NS 50000000 /* 50 ms */ |
229 | #define MAX_TS_PAIR_MISMATCHES 5 |
230 | #define MAX_TS_PAIR_MISMATCH_RESET_COUNT 3 |
231 | #define MIN_OBSERVED_RATE 0.8 |
232 | #define MAX_OBSERVED_RATE 1.2 |
233 | |
234 | static void |
235 | bt_calibration_thread(void) |
236 | { |
237 | static uint64_t prev_local_ts = 0, prev_remote_ts = 0, curr_local_ts = 0, curr_remote_ts = 0; |
238 | static uint64_t prev_received_local_ts = 0, prev_received_remote_ts = 0; |
239 | static double cumulative_rate = INITIAL_RATE; |
240 | static uint32_t initial_sample_count = 1; |
241 | static uint32_t max_initial_sample_count = MAX_INITIAL_SAMPLE_COUNT; |
242 | static uint32_t skip_reset_count = MAX_SKIP_RESET_COUNT; |
243 | int recalculate_count = 1; |
244 | static bool reset = false; |
245 | bool sleep = false; |
246 | static bool skip_rcv_ts = false; |
247 | static uint64_t ts_pair_mismatch = 0; |
248 | static uint32_t ts_pair_mismatch_reset_count = 0; |
249 | spl_t s = splsched(); |
250 | lck_spin_lock(lck: &bt_spin_lock); |
251 | if (!received_remote_timestamp) { |
252 | if (PE_parse_boot_argn(arg_string: "rt_ini_count" , arg_ptr: &max_initial_sample_count, |
253 | max_arg: sizeof(uint32_t)) == TRUE) { |
254 | if (max_initial_sample_count < MIN_INITIAL_SAMPLE_COUNT) { |
255 | max_initial_sample_count = MIN_INITIAL_SAMPLE_COUNT; |
256 | } |
257 | } |
258 | /* Nothing to do the first time */ |
259 | goto block; |
260 | } |
261 | /* |
262 | * The values in bt_params are recalculated every time a new timestamp |
263 | * pair is received. Firstly, both timestamps are converted to nanoseconds. |
264 | * The current and previous timestamp pairs are used to compute the |
265 | * observed_rate of the two clocks w.r.t each other. For the first |
266 | * MIN_INITIAL_SAMPLE_COUNT number of pairs, the cumulative_rate is a simple |
267 | * average of the observed_rate. For the later pairs, the cumulative_rate |
268 | * is updated using exponential moving average of the observed_rate. |
269 | * The current and bt_params' base timestamp pairs are used to compute |
270 | * the rate_from_base. This value ensures that the bt_params base |
271 | * timestamp pair curve doesn't stay parallel to the observed timestamp |
272 | * pair curve, rather moves in the direction of the observed timestamp curve. |
273 | * The bt_params.rate is computed as a weighted average of the cumulative_rate |
274 | * and the rate_from_base. For each current local timestamp, the remote_time |
275 | * is predicted using the previous values of bt_params. After computing the new |
276 | * bt_params.rate, bt_params.base_remote_time is set to this predicted value |
277 | * and bt_params.base_local_time is set to the current local timestamp. |
278 | */ |
279 | recalculate: |
280 | assertf(recalculate_count <= MAX_RECALCULATE_COUNT, "bt_caliberation_thread: recalculate \ |
281 | invocation exceeds MAX_RECALCULATE_COUNT" ); |
282 | |
283 | if ((received_remote_timestamp == BT_RESET_SENTINEL_TS) || (received_remote_timestamp == BT_WAKE_SENTINEL_TS)) { |
284 | KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), received_local_timestamp, received_remote_timestamp, 1); |
285 | reset = true; |
286 | skip_reset_count = MAX_SKIP_RESET_COUNT; |
287 | ts_pair_mismatch_reset_count = 0; |
288 | goto block; |
289 | } else if (received_remote_timestamp == BT_SLEEP_SENTINEL_TS) { |
290 | sleep = true; |
291 | } else if (!received_local_timestamp) { |
292 | /* If the local timestamp isn't accurately captured, the received value will be ignored */ |
293 | skip_rcv_ts = true; |
294 | goto block; |
295 | } |
296 | |
297 | /* Keep a copy of the prev timestamps to compute distance */ |
298 | prev_received_local_ts = curr_local_ts; |
299 | prev_received_remote_ts = curr_remote_ts; |
300 | |
301 | uint64_t curr_local_abs = received_local_timestamp; |
302 | absolutetime_to_nanoseconds(abstime: curr_local_abs, result: &curr_local_ts); |
303 | curr_remote_ts = received_remote_timestamp; |
304 | |
305 | /* Prevent unusual rate changes caused by delayed timestamps */ |
306 | uint64_t local_diff = curr_local_ts - prev_received_local_ts; |
307 | if (!(reset || sleep) && ((local_diff < MIN_LOCAL_TS_DISTANCE_NS) || |
308 | (!skip_rcv_ts && (local_diff > MAX_LOCAL_TS_DISTANCE_NS)))) { |
309 | /* Skip the current timestamp */ |
310 | KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_SKIP_TS), curr_local_ts, curr_remote_ts, |
311 | prev_received_local_ts); |
312 | goto block; |
313 | } else { |
314 | skip_rcv_ts = false; |
315 | /* Use the prev copy of timestamps only if the distance is acceptable */ |
316 | prev_local_ts = prev_received_local_ts; |
317 | prev_remote_ts = prev_received_remote_ts; |
318 | } |
319 | lck_spin_unlock(lck: &bt_spin_lock); |
320 | splx(s); |
321 | |
322 | struct bt_params bt_params = {}; |
323 | |
324 | lck_spin_lock(lck: &bt_ts_conversion_lock); |
325 | if (reset) { |
326 | if (skip_reset_count > 0) { |
327 | KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_SKIP_TS), curr_local_ts, curr_remote_ts, |
328 | prev_local_ts, skip_reset_count); |
329 | skip_reset_count--; |
330 | goto skip_reset; |
331 | } |
332 | bt_params.base_local_ts = curr_local_ts; |
333 | bt_params.base_remote_ts = curr_remote_ts; |
334 | bt_params.rate = cumulative_rate; |
335 | prev_local_ts = 0; |
336 | prev_remote_ts = 0; |
337 | ts_pair_mismatch = 0; |
338 | initial_sample_count = 1; |
339 | reset = false; |
340 | KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), curr_local_ts, curr_remote_ts, 2); |
341 | } else if (sleep) { |
342 | absolutetime_to_nanoseconds(abstime: mach_absolute_time(), result: &bt_params.base_local_ts); |
343 | bt_params.base_remote_ts = 0; |
344 | bt_params.rate = 0; |
345 | sleep = false; |
346 | } else { |
347 | struct bt_params bt_params_snapshot = {}; |
348 | if (bt_params_idx >= 0) { |
349 | bt_params_snapshot = bt_params_hist[bt_params_idx]; |
350 | } |
351 | lck_spin_unlock(lck: &bt_ts_conversion_lock); |
352 | if (bt_params_snapshot.rate == 0.0) { |
353 | /* |
354 | * The rate should never be 0 because we always expect a reset/wake |
355 | * sentinel after sleep, followed by valid timestamp pair data that |
356 | * will be handled by the reset clause (above). However, we should |
357 | * not rely on a paired version of the remote OS - we could actually |
358 | * be running a completely different OS! Treat a timestamp after |
359 | * a sleep as a reset condition. |
360 | */ |
361 | reset = true; |
362 | skip_reset_count = MAX_SKIP_RESET_COUNT; |
363 | ts_pair_mismatch_reset_count = 0; |
364 | KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), curr_local_ts, curr_remote_ts, 3); |
365 | s = splsched(); |
366 | lck_spin_lock(lck: &bt_spin_lock); |
367 | goto block; |
368 | } |
369 | |
370 | /* Check if the predicted remote timestamp is within the expected current remote timestamp range */ |
371 | uint64_t pred_remote_ts = mach_bridge_compute_timestamp(local_ts_ns: curr_local_ts, params: &bt_params_snapshot); |
372 | uint64_t diff = 0; |
373 | if (initial_sample_count >= max_initial_sample_count) { |
374 | if (pred_remote_ts > curr_remote_ts) { |
375 | diff = pred_remote_ts - curr_remote_ts; |
376 | } else { |
377 | diff = curr_remote_ts - pred_remote_ts; |
378 | } |
379 | if (diff > TS_PAIR_MISMATCH_THRESHOLD_NS) { |
380 | ts_pair_mismatch++; |
381 | KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_TS_MISMATCH), curr_local_ts, |
382 | curr_remote_ts, pred_remote_ts, ts_pair_mismatch); |
383 | } else { |
384 | ts_pair_mismatch = 0; |
385 | } |
386 | if (ts_pair_mismatch > MAX_TS_PAIR_MISMATCHES) { |
387 | #if (DEVELOPMENT || DEBUG) |
388 | if (ts_pair_mismatch_reset_count == MAX_TS_PAIR_MISMATCH_RESET_COUNT) { |
389 | panic("remote_time: timestamp pair mismatch exceeded limit" ); |
390 | } |
391 | #endif /* (DEVELOPMENT || DEBUG) */ |
392 | reset = true; |
393 | ts_pair_mismatch_reset_count++; |
394 | KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_RESET_TS), curr_local_ts, curr_remote_ts, 4); |
395 | s = splsched(); |
396 | lck_spin_lock(lck: &bt_spin_lock); |
397 | goto block; |
398 | } |
399 | } |
400 | double observed_rate, rate_from_base, new_rate; |
401 | observed_rate = mach_bridge_compute_rate(new_local_ts: curr_local_ts, new_remote_ts: curr_remote_ts, old_local_ts: prev_local_ts, old_remote_ts: prev_remote_ts); |
402 | /* Log bad observed rates and skip the timestamp pair */ |
403 | if ((observed_rate < MIN_OBSERVED_RATE) || (observed_rate > MAX_OBSERVED_RATE)) { |
404 | KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_OBSV_RATE), *(uint64_t *)((void *)&observed_rate)); |
405 | ts_pair_mismatch = ts_pair_mismatch > 0 ? (ts_pair_mismatch - 1) : 0; |
406 | s = splsched(); |
407 | lck_spin_lock(lck: &bt_spin_lock); |
408 | goto block; |
409 | } |
410 | if (initial_sample_count <= MIN_INITIAL_SAMPLE_COUNT) { |
411 | initial_sample_count++; |
412 | cumulative_rate = cumulative_rate + (observed_rate - cumulative_rate) / initial_sample_count; |
413 | } else { |
414 | if (initial_sample_count < max_initial_sample_count) { |
415 | initial_sample_count++; |
416 | } |
417 | cumulative_rate = cumulative_rate + CUMULATIVE_RATE_DECAY_CONSTANT * (observed_rate - cumulative_rate); |
418 | } |
419 | rate_from_base = mach_bridge_compute_rate(new_local_ts: curr_local_ts, new_remote_ts: curr_remote_ts, old_local_ts: bt_params_snapshot.base_local_ts, |
420 | old_remote_ts: bt_params_snapshot.base_remote_ts); |
421 | new_rate = CUMULATIVE_RATE_WEIGHT * cumulative_rate + (1 - CUMULATIVE_RATE_WEIGHT) * rate_from_base; |
422 | /* |
423 | * Acquire the lock first to ensure that bt_params.base_local_ts is always |
424 | * greater than the last value of now captured by mach_bridge_remote_time. |
425 | * This ensures that we always use the same parameters to compute remote |
426 | * timestamp for a given local timestamp. |
427 | */ |
428 | lck_spin_lock(lck: &bt_ts_conversion_lock); |
429 | absolutetime_to_nanoseconds(abstime: mach_absolute_time(), result: &bt_params.base_local_ts); |
430 | bt_params.base_remote_ts = mach_bridge_compute_timestamp(local_ts_ns: bt_params.base_local_ts, params: &bt_params_snapshot); |
431 | bt_params.rate = new_rate; |
432 | } |
433 | bt_params_add(params: &bt_params); |
434 | commpage_set_remotetime_params(rate: bt_params.rate, base_local_ts: bt_params.base_local_ts, base_remote_ts: bt_params.base_remote_ts); |
435 | KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_TS_PARAMS), bt_params.base_local_ts, |
436 | bt_params.base_remote_ts, *(uint64_t *)((void *)&bt_params.rate)); |
437 | |
438 | skip_reset: |
439 | lck_spin_unlock(lck: &bt_ts_conversion_lock); |
440 | |
441 | s = splsched(); |
442 | lck_spin_lock(lck: &bt_spin_lock); |
443 | /* Check if a new timestamp pair was received */ |
444 | if (received_local_timestamp != curr_local_abs) { |
445 | recalculate_count++; |
446 | goto recalculate; |
447 | } |
448 | block: |
449 | assert_wait(event: (event_t)bt_params_hist, THREAD_UNINT); |
450 | lck_spin_unlock(lck: &bt_spin_lock); |
451 | splx(s); |
452 | thread_block(continuation: (thread_continue_t)bt_calibration_thread); |
453 | } |
454 | |
455 | void |
456 | bt_calibration_thread_start(void) |
457 | { |
458 | thread_t thread; |
459 | kern_return_t result = kernel_thread_start_priority(continuation: (thread_continue_t)bt_calibration_thread, |
460 | NULL, BASEPRI_KERNEL, new_thread: &thread); |
461 | if (result != KERN_SUCCESS) { |
462 | panic("mach_bridge_add_timestamp: thread_timestamp_calibration" ); |
463 | } |
464 | thread_deallocate(thread); |
465 | } |
466 | |
467 | #endif /* CONFIG_MACH_BRIDGE_RECV_TIME */ |
468 | |
469 | /** |
470 | * mach_bridge_remote_time |
471 | * |
472 | * This function is used to predict the remote CPU's clock time, given |
473 | * the local time. |
474 | * |
475 | * If local_timestamp = 0, then the remote_timestamp is calculated |
476 | * corresponding to the current mach_absolute_time. |
477 | * |
478 | * If XNU_TARGET_OS_BRIDGE is defined, then monotonicity of |
479 | * predicted time is guaranteed only for recent local_timestamp values |
480 | * lesser than the current mach_absolute_time upto 1 second. |
481 | * |
482 | * If CONFIG_MACH_BRIDGE_SEND_TIME is true, then the function is compiled |
483 | * for the remote CPU. If CONFIG_MACH_BRIDGE_RECV_TIME is true, then the |
484 | * the function is compiled for the local CPU. Both config options cannot |
485 | * be true simultaneously. |
486 | */ |
487 | uint64_t |
488 | mach_bridge_remote_time(uint64_t local_timestamp) |
489 | { |
490 | #if defined(CONFIG_MACH_BRIDGE_SEND_TIME) |
491 | #if !defined(CONFIG_MACH_BRIDGE_RECV_TIME) |
492 | /* only send side of the bridge is defined: no translation needed */ |
493 | if (!local_timestamp) { |
494 | return mach_absolute_time(); |
495 | } |
496 | return 0; |
497 | #else |
498 | #error "You cannot define both sides of the bridge!" |
499 | #endif /* !defined(CONFIG_MACH_BRIDGE_RECV_TIME) */ |
500 | #else |
501 | #if !defined(CONFIG_MACH_BRIDGE_RECV_TIME) |
502 | /* neither the send or receive side of the bridge is defined: echo the input */ |
503 | return local_timestamp; |
504 | #else |
505 | if (!os_atomic_load(&bt_init_flag, acquire)) { |
506 | return 0; |
507 | } |
508 | |
509 | uint64_t remote_timestamp = 0; |
510 | |
511 | lck_spin_lock(lck: &bt_ts_conversion_lock); |
512 | uint64_t now = mach_absolute_time(); |
513 | if (!local_timestamp) { |
514 | local_timestamp = now; |
515 | } |
516 | #if defined(XNU_TARGET_OS_BRIDGE) |
517 | uint64_t local_timestamp_ns = 0; |
518 | if (local_timestamp < now) { |
519 | absolutetime_to_nanoseconds(local_timestamp, &local_timestamp_ns); |
520 | struct bt_params *params = bt_params_find(local_timestamp_ns); |
521 | remote_timestamp = mach_bridge_compute_timestamp(local_timestamp_ns, params); |
522 | } |
523 | #else |
524 | struct bt_params params = bt_params_get_latest_locked(); |
525 | remote_timestamp = mach_bridge_compute_timestamp(local_ts_ns: local_timestamp, params: ¶ms); |
526 | #endif /* defined(XNU_TARGET_OS_BRIDGE) */ |
527 | lck_spin_unlock(lck: &bt_ts_conversion_lock); |
528 | KDBG(MACHDBG_CODE(DBG_MACH_CLOCK, MACH_BRIDGE_REMOTE_TIME), local_timestamp, remote_timestamp, now); |
529 | |
530 | return remote_timestamp; |
531 | #endif /* !defined(CONFIG_MACH_BRIDGE_RECV_TIME) */ |
532 | #endif /* defined(CONFIG_MACH_BRIDGE_SEND_TIME) */ |
533 | } |
534 | |