1 | /* |
2 | * Copyright (c) 1998-2000, 2009-2010 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <ptrauth.h> |
30 | #include <sys/cdefs.h> |
31 | |
32 | __BEGIN_DECLS |
33 | #include <kern/thread_call.h> |
34 | __END_DECLS |
35 | |
36 | #include <IOKit/assert.h> |
37 | #include <IOKit/system.h> |
38 | |
39 | #include <IOKit/IOLib.h> |
40 | #include <IOKit/IOTimerEventSource.h> |
41 | #include <IOKit/IOWorkLoop.h> |
42 | |
43 | #include <IOKit/IOTimeStamp.h> |
44 | #include <IOKit/IOKitDebug.h> |
45 | #if CONFIG_DTRACE |
46 | #include <mach/sdt.h> |
47 | #endif |
48 | |
49 | #include <libkern/Block.h> |
50 | #include <libkern/Block_private.h> |
51 | |
52 | |
53 | #define super IOEventSource |
54 | OSDefineMetaClassAndStructors(IOTimerEventSource, IOEventSource) |
55 | OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 0); |
56 | OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 1); |
57 | OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 2); |
58 | OSMetaClassDefineReservedUnused(IOTimerEventSource, 3); |
59 | OSMetaClassDefineReservedUnused(IOTimerEventSource, 4); |
60 | OSMetaClassDefineReservedUnused(IOTimerEventSource, 5); |
61 | OSMetaClassDefineReservedUnused(IOTimerEventSource, 6); |
62 | OSMetaClassDefineReservedUnused(IOTimerEventSource, 7); |
63 | |
64 | #if IOKITSTATS |
65 | |
66 | #define IOStatisticsInitializeCounter() \ |
67 | do { \ |
68 | IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsTimerEventSourceCounter); \ |
69 | } while (0) |
70 | |
71 | #define IOStatisticsOpenGate() \ |
72 | do { \ |
73 | IOStatistics::countOpenGate(me->IOEventSource::reserved->counter); \ |
74 | } while (0) |
75 | |
76 | #define IOStatisticsCloseGate() \ |
77 | do { \ |
78 | IOStatistics::countCloseGate(me->IOEventSource::reserved->counter); \ |
79 | } while (0) |
80 | |
81 | #define IOStatisticsTimeout() \ |
82 | do { \ |
83 | IOStatistics::countTimerTimeout(me->IOEventSource::reserved->counter); \ |
84 | } while (0) |
85 | |
86 | #else |
87 | |
88 | #define IOStatisticsInitializeCounter() |
89 | #define IOStatisticsOpenGate() |
90 | #define IOStatisticsCloseGate() |
91 | #define IOStatisticsTimeout() |
92 | |
93 | #endif /* IOKITSTATS */ |
94 | |
95 | // |
96 | // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used, |
97 | // not a subclassed implementation. |
98 | // |
99 | |
100 | // Timeout handler function. This function is called by the kernel when |
101 | // the timeout interval expires. |
102 | // |
103 | |
104 | __inline__ void |
105 | IOTimerEventSource::invokeAction(IOEventSource::Action _action, IOTimerEventSource * ts, |
106 | OSObject * _owner, IOWorkLoop * _workLoop) |
107 | { |
108 | bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false; |
109 | void * address; |
110 | |
111 | if (kActionBlock & flags) { |
112 | address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *) actionBlock)); |
113 | } else { |
114 | address = ptrauth_nop_cast(void *, _action); |
115 | } |
116 | |
117 | if (trace) { |
118 | IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION), |
119 | VM_KERNEL_ADDRHIDE(address), |
120 | VM_KERNEL_ADDRHIDE(_owner)); |
121 | } |
122 | |
123 | if (kActionBlock & flags) { |
124 | ((IOTimerEventSource::ActionBlock) actionBlock)(ts); |
125 | } else { |
126 | ((IOTimerEventSource::Action)_action)(_owner, ts); |
127 | } |
128 | |
129 | #if CONFIG_DTRACE |
130 | DTRACE_TMR3(iotescallout__expire, Action, address, OSObject, _owner, void, _workLoop); |
131 | #endif |
132 | |
133 | if (trace) { |
134 | IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION), |
135 | VM_KERNEL_UNSLIDE(address), |
136 | VM_KERNEL_ADDRHIDE(_owner)); |
137 | } |
138 | } |
139 | |
140 | void |
141 | IOTimerEventSource::timeout(void *self) |
142 | { |
143 | IOTimerEventSource *me = (IOTimerEventSource *) self; |
144 | |
145 | IOStatisticsTimeout(); |
146 | |
147 | if (me->enabled && me->action) { |
148 | IOWorkLoop * |
149 | wl = me->workLoop; |
150 | if (wl) { |
151 | IOEventSource::Action doit; |
152 | wl->closeGate(); |
153 | IOStatisticsCloseGate(); |
154 | doit = me->action; |
155 | if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) { |
156 | me->invokeAction(action: doit, ts: me, owner: me->owner, workLoop: me->workLoop); |
157 | } |
158 | IOStatisticsOpenGate(); |
159 | wl->openGate(); |
160 | } |
161 | } |
162 | } |
163 | |
164 | void |
165 | IOTimerEventSource::timeoutAndRelease(void * self, void * c) |
166 | { |
167 | IOTimerEventSource *me = (IOTimerEventSource *) self; |
168 | /* The second parameter (a pointer) gets abused to carry an SInt32, so on LP64, "count" |
169 | * must be cast to "long" before, in order to tell GCC we're not truncating a pointer. */ |
170 | SInt32 count = (SInt32) (long) c; |
171 | |
172 | IOStatisticsTimeout(); |
173 | |
174 | if (me->enabled && me->action) { |
175 | IOWorkLoop * |
176 | wl = me->reserved->workLoop; |
177 | if (wl) { |
178 | IOEventSource::Action doit; |
179 | wl->closeGate(); |
180 | IOStatisticsCloseGate(); |
181 | doit = me->action; |
182 | if (doit && (me->reserved->calloutGeneration == count)) { |
183 | thread_call_start_iotes_invocation(call: (thread_call_t)me->calloutEntry); |
184 | me->invokeAction(action: doit, ts: me, owner: me->owner, workLoop: me->workLoop); |
185 | } |
186 | IOStatisticsOpenGate(); |
187 | wl->openGate(); |
188 | } |
189 | } |
190 | if (me->reserved->workLoop) { |
191 | me->reserved->workLoop->release(); |
192 | } |
193 | me->release(); |
194 | } |
195 | |
196 | // -- work loop delivery |
197 | |
198 | bool |
199 | IOTimerEventSource::checkForWork() |
200 | { |
201 | IOEventSource::Action doit; |
202 | |
203 | if (reserved |
204 | && (reserved->calloutGenerationSignaled == reserved->calloutGeneration) |
205 | && enabled && (doit = action)) { |
206 | reserved->calloutGenerationSignaled = ~reserved->calloutGeneration; |
207 | invokeAction(action: doit, ts: this, owner: owner, workLoop: workLoop); |
208 | } |
209 | |
210 | return false; |
211 | } |
212 | |
213 | void |
214 | IOTimerEventSource::timeoutSignaled(void * self, void * c) |
215 | { |
216 | IOTimerEventSource *me = (IOTimerEventSource *) self; |
217 | |
218 | me->reserved->calloutGenerationSignaled = (SInt32)(long) c; |
219 | if (me->enabled) { |
220 | me->signalWorkAvailable(); |
221 | } |
222 | } |
223 | |
224 | // -- |
225 | |
226 | void |
227 | IOTimerEventSource::setTimeoutFunc() |
228 | { |
229 | thread_call_priority_t pri; |
230 | uint32_t options; |
231 | |
232 | if (reserved) { |
233 | panic("setTimeoutFunc already %p, %p" , this, reserved); |
234 | } |
235 | |
236 | // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used, |
237 | // not a subclassed implementation |
238 | reserved = IOMallocType(ExpansionData); |
239 | |
240 | reserved->calloutGenerationSignaled = ~reserved->calloutGeneration; |
241 | // make use of an existing ivar for parameter passing |
242 | options = (uint32_t) abstime; |
243 | abstime = 0; |
244 | |
245 | thread_call_options_t tcoptions = 0; |
246 | thread_call_func_t func = NULL; |
247 | |
248 | switch (kIOTimerEventSourceOptionsPriorityMask & options) { |
249 | case kIOTimerEventSourceOptionsPriorityHigh: |
250 | pri = THREAD_CALL_PRIORITY_HIGH; |
251 | func = &IOTimerEventSource::timeoutAndRelease; |
252 | break; |
253 | |
254 | case kIOTimerEventSourceOptionsPriorityKernel: |
255 | pri = THREAD_CALL_PRIORITY_KERNEL; |
256 | func = &IOTimerEventSource::timeoutAndRelease; |
257 | break; |
258 | |
259 | case kIOTimerEventSourceOptionsPriorityKernelHigh: |
260 | pri = THREAD_CALL_PRIORITY_KERNEL_HIGH; |
261 | func = &IOTimerEventSource::timeoutAndRelease; |
262 | break; |
263 | |
264 | case kIOTimerEventSourceOptionsPriorityUser: |
265 | pri = THREAD_CALL_PRIORITY_USER; |
266 | func = &IOTimerEventSource::timeoutAndRelease; |
267 | break; |
268 | |
269 | case kIOTimerEventSourceOptionsPriorityLow: |
270 | pri = THREAD_CALL_PRIORITY_LOW; |
271 | func = &IOTimerEventSource::timeoutAndRelease; |
272 | break; |
273 | |
274 | case kIOTimerEventSourceOptionsPriorityWorkLoop: |
275 | pri = THREAD_CALL_PRIORITY_KERNEL; |
276 | tcoptions |= THREAD_CALL_OPTIONS_SIGNAL; |
277 | if (kIOTimerEventSourceOptionsAllowReenter & options) { |
278 | break; |
279 | } |
280 | func = &IOTimerEventSource::timeoutSignaled; |
281 | break; |
282 | |
283 | default: |
284 | break; |
285 | } |
286 | |
287 | assertf(func, "IOTimerEventSource options 0x%x" , options); |
288 | if (!func) { |
289 | return; // init will fail |
290 | } |
291 | if (THREAD_CALL_OPTIONS_SIGNAL & tcoptions) { |
292 | flags |= kActive; |
293 | } else { |
294 | flags |= kPassive; |
295 | } |
296 | |
297 | if (!(kIOTimerEventSourceOptionsAllowReenter & options)) { |
298 | tcoptions |= THREAD_CALL_OPTIONS_ONCE; |
299 | } |
300 | |
301 | calloutEntry = (void *) thread_call_allocate_with_options(func, |
302 | param0: (thread_call_param_t) this, pri, options: tcoptions); |
303 | assert(calloutEntry); |
304 | } |
305 | |
306 | bool |
307 | IOTimerEventSource::init(OSObject *inOwner, Action inAction) |
308 | { |
309 | if (!super::init(owner: inOwner, action: (IOEventSource::Action) inAction)) { |
310 | return false; |
311 | } |
312 | |
313 | setTimeoutFunc(); |
314 | if (!calloutEntry) { |
315 | return false; |
316 | } |
317 | |
318 | IOStatisticsInitializeCounter(); |
319 | |
320 | return true; |
321 | } |
322 | |
323 | bool |
324 | IOTimerEventSource::init(uint32_t options, OSObject *inOwner, Action inAction) |
325 | { |
326 | // make use of an existing ivar for parameter passing |
327 | abstime = options; |
328 | return init(inOwner, inAction); |
329 | } |
330 | |
331 | IOTimerEventSource * |
332 | IOTimerEventSource::timerEventSource(uint32_t inOptions, OSObject *inOwner, Action inAction) |
333 | { |
334 | IOTimerEventSource *me = new IOTimerEventSource; |
335 | |
336 | if (me && !me->init(options: inOptions, inOwner, inAction)) { |
337 | me->release(); |
338 | return NULL; |
339 | } |
340 | |
341 | return me; |
342 | } |
343 | |
344 | IOTimerEventSource * |
345 | IOTimerEventSource::timerEventSource(uint32_t options, OSObject *inOwner, ActionBlock _action) |
346 | { |
347 | IOTimerEventSource * tes; |
348 | tes = IOTimerEventSource::timerEventSource(inOptions: options, inOwner, inAction: (Action) NULL); |
349 | if (tes) { |
350 | tes->setActionBlock((IOEventSource::ActionBlock) _action); |
351 | } |
352 | |
353 | return tes; |
354 | } |
355 | |
356 | #define _thread_call_cancel(tc) ((kActive & flags) ? thread_call_cancel_wait((tc)) : thread_call_cancel((tc))) |
357 | |
358 | IOTimerEventSource * |
359 | IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction) |
360 | { |
361 | return IOTimerEventSource::timerEventSource( |
362 | inOptions: kIOTimerEventSourceOptionsPriorityKernelHigh, |
363 | inOwner, inAction); |
364 | } |
365 | |
366 | void |
367 | IOTimerEventSource::free() |
368 | { |
369 | if (calloutEntry) { |
370 | __assert_only bool freed; |
371 | |
372 | cancelTimeout(); |
373 | |
374 | freed = thread_call_free(call: (thread_call_t) calloutEntry); |
375 | assert(freed); |
376 | } |
377 | |
378 | if (reserved) { |
379 | IOFreeType(reserved, ExpansionData); |
380 | } |
381 | |
382 | super::free(); |
383 | } |
384 | |
385 | void |
386 | IOTimerEventSource::cancelTimeout() |
387 | { |
388 | if (reserved) { |
389 | reserved->calloutGeneration++; |
390 | } |
391 | bool active = _thread_call_cancel((thread_call_t) calloutEntry); |
392 | AbsoluteTime_to_scalar(&abstime) = 0; |
393 | if (active && reserved && (kPassive & flags)) { |
394 | release(); |
395 | workLoop->release(); |
396 | } |
397 | } |
398 | |
399 | void |
400 | IOTimerEventSource::enable() |
401 | { |
402 | super::enable(); |
403 | if (kIOReturnSuccess != wakeAtTime(abstime)) { |
404 | super::disable(); // Problem re-scheduling timeout ignore enable |
405 | } |
406 | } |
407 | |
408 | void |
409 | IOTimerEventSource::disable() |
410 | { |
411 | if (reserved) { |
412 | reserved->calloutGeneration++; |
413 | } |
414 | bool active = _thread_call_cancel((thread_call_t) calloutEntry); |
415 | super::disable(); |
416 | if (active && reserved && (kPassive & flags)) { |
417 | release(); |
418 | workLoop->release(); |
419 | } |
420 | } |
421 | |
422 | IOReturn |
423 | IOTimerEventSource::setTimeoutTicks(UInt32 ticks) |
424 | { |
425 | return setTimeout(interval: ticks, scale_factor: kTickScale); |
426 | } |
427 | |
428 | IOReturn |
429 | IOTimerEventSource::setTimeoutMS(UInt32 ms) |
430 | { |
431 | return setTimeout(interval: ms, scale_factor: kMillisecondScale); |
432 | } |
433 | |
434 | IOReturn |
435 | IOTimerEventSource::setTimeoutUS(UInt32 us) |
436 | { |
437 | return setTimeout(interval: us, scale_factor: kMicrosecondScale); |
438 | } |
439 | |
440 | IOReturn |
441 | IOTimerEventSource::setTimeout(UInt32 interval, UInt32 scale_factor) |
442 | { |
443 | AbsoluteTime end; |
444 | |
445 | clock_interval_to_deadline(interval, scale_factor, result: &end); |
446 | return wakeAtTime(abstime: end); |
447 | } |
448 | |
449 | #if !defined(__LP64__) |
450 | IOReturn |
451 | IOTimerEventSource::setTimeout(mach_timespec_t interval) |
452 | { |
453 | AbsoluteTime end, nsecs; |
454 | |
455 | clock_interval_to_absolutetime_interval |
456 | (interval.tv_nsec, kNanosecondScale, &nsecs); |
457 | clock_interval_to_deadline |
458 | (interval.tv_sec, NSEC_PER_SEC, &end); |
459 | ADD_ABSOLUTETIME(&end, &nsecs); |
460 | |
461 | return wakeAtTime(end); |
462 | } |
463 | #endif |
464 | |
465 | IOReturn |
466 | IOTimerEventSource::setTimeout(AbsoluteTime interval) |
467 | { |
468 | AbsoluteTime end; |
469 | clock_absolutetime_interval_to_deadline(abstime: interval, result: &end); |
470 | return wakeAtTime(abstime: end); |
471 | } |
472 | |
473 | IOReturn |
474 | IOTimerEventSource::setTimeout(uint32_t options, |
475 | AbsoluteTime abstime, AbsoluteTime leeway) |
476 | { |
477 | AbsoluteTime end; |
478 | if (options & kIOTimeOptionsContinuous) { |
479 | clock_continuoustime_interval_to_deadline(abstime, result: &end); |
480 | } else { |
481 | clock_absolutetime_interval_to_deadline(abstime, result: &end); |
482 | } |
483 | |
484 | return wakeAtTime(options, abstime: end, leeway); |
485 | } |
486 | |
487 | IOReturn |
488 | IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks) |
489 | { |
490 | return wakeAtTime(abstime: ticks, scale_factor: kTickScale); |
491 | } |
492 | |
493 | IOReturn |
494 | IOTimerEventSource::wakeAtTimeMS(UInt32 ms) |
495 | { |
496 | return wakeAtTime(abstime: ms, scale_factor: kMillisecondScale); |
497 | } |
498 | |
499 | IOReturn |
500 | IOTimerEventSource::wakeAtTimeUS(UInt32 us) |
501 | { |
502 | return wakeAtTime(abstime: us, scale_factor: kMicrosecondScale); |
503 | } |
504 | |
505 | IOReturn |
506 | IOTimerEventSource::wakeAtTime(UInt32 inAbstime, UInt32 scale_factor) |
507 | { |
508 | AbsoluteTime end; |
509 | clock_interval_to_absolutetime_interval(interval: inAbstime, scale_factor, result: &end); |
510 | |
511 | return wakeAtTime(abstime: end); |
512 | } |
513 | |
514 | #if !defined(__LP64__) |
515 | IOReturn |
516 | IOTimerEventSource::wakeAtTime(mach_timespec_t inAbstime) |
517 | { |
518 | AbsoluteTime end, nsecs; |
519 | |
520 | clock_interval_to_absolutetime_interval |
521 | (inAbstime.tv_nsec, kNanosecondScale, &nsecs); |
522 | clock_interval_to_absolutetime_interval |
523 | (inAbstime.tv_sec, kSecondScale, &end); |
524 | ADD_ABSOLUTETIME(&end, &nsecs); |
525 | |
526 | return wakeAtTime(end); |
527 | } |
528 | #endif |
529 | |
530 | void |
531 | IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop) |
532 | { |
533 | super::setWorkLoop(inWorkLoop); |
534 | if (enabled && AbsoluteTime_to_scalar(&abstime) && workLoop) { |
535 | wakeAtTime(abstime); |
536 | } |
537 | } |
538 | |
539 | IOReturn |
540 | IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime) |
541 | { |
542 | return wakeAtTime(options: 0, abstime: inAbstime, leeway: 0); |
543 | } |
544 | |
545 | IOReturn |
546 | IOTimerEventSource::wakeAtTime(uint32_t options, AbsoluteTime inAbstime, AbsoluteTime leeway) |
547 | { |
548 | if (!action) { |
549 | return kIOReturnNoResources; |
550 | } |
551 | |
552 | abstime = inAbstime; |
553 | if (enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop) { |
554 | uint32_t tcoptions = 0; |
555 | |
556 | if (kIOTimeOptionsWithLeeway & options) { |
557 | tcoptions |= THREAD_CALL_DELAY_LEEWAY; |
558 | } |
559 | if (kIOTimeOptionsContinuous & options) { |
560 | tcoptions |= THREAD_CALL_CONTINUOUS; |
561 | } |
562 | |
563 | if (reserved) { |
564 | if (kPassive & flags) { |
565 | retain(); |
566 | workLoop->retain(); |
567 | } |
568 | reserved->workLoop = workLoop; |
569 | reserved->calloutGeneration++; |
570 | if (thread_call_enter_delayed_with_leeway(call: (thread_call_t) calloutEntry, |
571 | param1: (void *)(uintptr_t) reserved->calloutGeneration, deadline: inAbstime, leeway, flags: tcoptions) |
572 | && (kPassive & flags)) { |
573 | release(); |
574 | workLoop->release(); |
575 | } |
576 | } else { |
577 | thread_call_enter_delayed_with_leeway(call: (thread_call_t) calloutEntry, |
578 | NULL, deadline: inAbstime, leeway, flags: tcoptions); |
579 | } |
580 | } |
581 | |
582 | return kIOReturnSuccess; |
583 | } |
584 | |