1/*
2 * Copyright (c) 1998-2000, 2009-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <ptrauth.h>
30#include <sys/cdefs.h>
31
32__BEGIN_DECLS
33#include <kern/thread_call.h>
34__END_DECLS
35
36#include <IOKit/assert.h>
37#include <IOKit/system.h>
38
39#include <IOKit/IOLib.h>
40#include <IOKit/IOTimerEventSource.h>
41#include <IOKit/IOWorkLoop.h>
42
43#include <IOKit/IOTimeStamp.h>
44#include <IOKit/IOKitDebug.h>
45#if CONFIG_DTRACE
46#include <mach/sdt.h>
47#endif
48
49#include <libkern/Block.h>
50#include <libkern/Block_private.h>
51
52
53#define super IOEventSource
54OSDefineMetaClassAndStructors(IOTimerEventSource, IOEventSource)
55OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 0);
56OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 1);
57OSMetaClassDefineReservedUsedX86(IOTimerEventSource, 2);
58OSMetaClassDefineReservedUnused(IOTimerEventSource, 3);
59OSMetaClassDefineReservedUnused(IOTimerEventSource, 4);
60OSMetaClassDefineReservedUnused(IOTimerEventSource, 5);
61OSMetaClassDefineReservedUnused(IOTimerEventSource, 6);
62OSMetaClassDefineReservedUnused(IOTimerEventSource, 7);
63
64#if IOKITSTATS
65
66#define IOStatisticsInitializeCounter() \
67do { \
68 IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsTimerEventSourceCounter); \
69} while (0)
70
71#define IOStatisticsOpenGate() \
72do { \
73 IOStatistics::countOpenGate(me->IOEventSource::reserved->counter); \
74} while (0)
75
76#define IOStatisticsCloseGate() \
77do { \
78 IOStatistics::countCloseGate(me->IOEventSource::reserved->counter); \
79} while (0)
80
81#define IOStatisticsTimeout() \
82do { \
83 IOStatistics::countTimerTimeout(me->IOEventSource::reserved->counter); \
84} while (0)
85
86#else
87
88#define IOStatisticsInitializeCounter()
89#define IOStatisticsOpenGate()
90#define IOStatisticsCloseGate()
91#define IOStatisticsTimeout()
92
93#endif /* IOKITSTATS */
94
95//
96// reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
97// not a subclassed implementation.
98//
99
100// Timeout handler function. This function is called by the kernel when
101// the timeout interval expires.
102//
103
104__inline__ void
105IOTimerEventSource::invokeAction(IOEventSource::Action _action, IOTimerEventSource * ts,
106 OSObject * _owner, IOWorkLoop * _workLoop)
107{
108 bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false;
109 void * address;
110
111 if (kActionBlock & flags) {
112 address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *) actionBlock));
113 } else {
114 address = ptrauth_nop_cast(void *, _action);
115 }
116
117 if (trace) {
118 IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION),
119 VM_KERNEL_ADDRHIDE(address),
120 VM_KERNEL_ADDRHIDE(_owner));
121 }
122
123 if (kActionBlock & flags) {
124 ((IOTimerEventSource::ActionBlock) actionBlock)(ts);
125 } else {
126 ((IOTimerEventSource::Action)_action)(_owner, ts);
127 }
128
129#if CONFIG_DTRACE
130 DTRACE_TMR3(iotescallout__expire, Action, address, OSObject, _owner, void, _workLoop);
131#endif
132
133 if (trace) {
134 IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION),
135 VM_KERNEL_UNSLIDE(address),
136 VM_KERNEL_ADDRHIDE(_owner));
137 }
138}
139
140void
141IOTimerEventSource::timeout(void *self)
142{
143 IOTimerEventSource *me = (IOTimerEventSource *) self;
144
145 IOStatisticsTimeout();
146
147 if (me->enabled && me->action) {
148 IOWorkLoop *
149 wl = me->workLoop;
150 if (wl) {
151 IOEventSource::Action doit;
152 wl->closeGate();
153 IOStatisticsCloseGate();
154 doit = me->action;
155 if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime)) {
156 me->invokeAction(action: doit, ts: me, owner: me->owner, workLoop: me->workLoop);
157 }
158 IOStatisticsOpenGate();
159 wl->openGate();
160 }
161 }
162}
163
164void
165IOTimerEventSource::timeoutAndRelease(void * self, void * c)
166{
167 IOTimerEventSource *me = (IOTimerEventSource *) self;
168 /* The second parameter (a pointer) gets abused to carry an SInt32, so on LP64, "count"
169 * must be cast to "long" before, in order to tell GCC we're not truncating a pointer. */
170 SInt32 count = (SInt32) (long) c;
171
172 IOStatisticsTimeout();
173
174 if (me->enabled && me->action) {
175 IOWorkLoop *
176 wl = me->reserved->workLoop;
177 if (wl) {
178 IOEventSource::Action doit;
179 wl->closeGate();
180 IOStatisticsCloseGate();
181 doit = me->action;
182 if (doit && (me->reserved->calloutGeneration == count)) {
183 thread_call_start_iotes_invocation(call: (thread_call_t)me->calloutEntry);
184 me->invokeAction(action: doit, ts: me, owner: me->owner, workLoop: me->workLoop);
185 }
186 IOStatisticsOpenGate();
187 wl->openGate();
188 }
189 }
190 if (me->reserved->workLoop) {
191 me->reserved->workLoop->release();
192 }
193 me->release();
194}
195
196// -- work loop delivery
197
198bool
199IOTimerEventSource::checkForWork()
200{
201 IOEventSource::Action doit;
202
203 if (reserved
204 && (reserved->calloutGenerationSignaled == reserved->calloutGeneration)
205 && enabled && (doit = action)) {
206 reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
207 invokeAction(action: doit, ts: this, owner: owner, workLoop: workLoop);
208 }
209
210 return false;
211}
212
213void
214IOTimerEventSource::timeoutSignaled(void * self, void * c)
215{
216 IOTimerEventSource *me = (IOTimerEventSource *) self;
217
218 me->reserved->calloutGenerationSignaled = (SInt32)(long) c;
219 if (me->enabled) {
220 me->signalWorkAvailable();
221 }
222}
223
224// --
225
226void
227IOTimerEventSource::setTimeoutFunc()
228{
229 thread_call_priority_t pri;
230 uint32_t options;
231
232 if (reserved) {
233 panic("setTimeoutFunc already %p, %p", this, reserved);
234 }
235
236 // reserved != 0 means IOTimerEventSource::timeoutAndRelease is being used,
237 // not a subclassed implementation
238 reserved = IOMallocType(ExpansionData);
239
240 reserved->calloutGenerationSignaled = ~reserved->calloutGeneration;
241 // make use of an existing ivar for parameter passing
242 options = (uint32_t) abstime;
243 abstime = 0;
244
245 thread_call_options_t tcoptions = 0;
246 thread_call_func_t func = NULL;
247
248 switch (kIOTimerEventSourceOptionsPriorityMask & options) {
249 case kIOTimerEventSourceOptionsPriorityHigh:
250 pri = THREAD_CALL_PRIORITY_HIGH;
251 func = &IOTimerEventSource::timeoutAndRelease;
252 break;
253
254 case kIOTimerEventSourceOptionsPriorityKernel:
255 pri = THREAD_CALL_PRIORITY_KERNEL;
256 func = &IOTimerEventSource::timeoutAndRelease;
257 break;
258
259 case kIOTimerEventSourceOptionsPriorityKernelHigh:
260 pri = THREAD_CALL_PRIORITY_KERNEL_HIGH;
261 func = &IOTimerEventSource::timeoutAndRelease;
262 break;
263
264 case kIOTimerEventSourceOptionsPriorityUser:
265 pri = THREAD_CALL_PRIORITY_USER;
266 func = &IOTimerEventSource::timeoutAndRelease;
267 break;
268
269 case kIOTimerEventSourceOptionsPriorityLow:
270 pri = THREAD_CALL_PRIORITY_LOW;
271 func = &IOTimerEventSource::timeoutAndRelease;
272 break;
273
274 case kIOTimerEventSourceOptionsPriorityWorkLoop:
275 pri = THREAD_CALL_PRIORITY_KERNEL;
276 tcoptions |= THREAD_CALL_OPTIONS_SIGNAL;
277 if (kIOTimerEventSourceOptionsAllowReenter & options) {
278 break;
279 }
280 func = &IOTimerEventSource::timeoutSignaled;
281 break;
282
283 default:
284 break;
285 }
286
287 assertf(func, "IOTimerEventSource options 0x%x", options);
288 if (!func) {
289 return; // init will fail
290 }
291 if (THREAD_CALL_OPTIONS_SIGNAL & tcoptions) {
292 flags |= kActive;
293 } else {
294 flags |= kPassive;
295 }
296
297 if (!(kIOTimerEventSourceOptionsAllowReenter & options)) {
298 tcoptions |= THREAD_CALL_OPTIONS_ONCE;
299 }
300
301 calloutEntry = (void *) thread_call_allocate_with_options(func,
302 param0: (thread_call_param_t) this, pri, options: tcoptions);
303 assert(calloutEntry);
304}
305
306bool
307IOTimerEventSource::init(OSObject *inOwner, Action inAction)
308{
309 if (!super::init(owner: inOwner, action: (IOEventSource::Action) inAction)) {
310 return false;
311 }
312
313 setTimeoutFunc();
314 if (!calloutEntry) {
315 return false;
316 }
317
318 IOStatisticsInitializeCounter();
319
320 return true;
321}
322
323bool
324IOTimerEventSource::init(uint32_t options, OSObject *inOwner, Action inAction)
325{
326 // make use of an existing ivar for parameter passing
327 abstime = options;
328 return init(inOwner, inAction);
329}
330
331IOTimerEventSource *
332IOTimerEventSource::timerEventSource(uint32_t inOptions, OSObject *inOwner, Action inAction)
333{
334 IOTimerEventSource *me = new IOTimerEventSource;
335
336 if (me && !me->init(options: inOptions, inOwner, inAction)) {
337 me->release();
338 return NULL;
339 }
340
341 return me;
342}
343
344IOTimerEventSource *
345IOTimerEventSource::timerEventSource(uint32_t options, OSObject *inOwner, ActionBlock _action)
346{
347 IOTimerEventSource * tes;
348 tes = IOTimerEventSource::timerEventSource(inOptions: options, inOwner, inAction: (Action) NULL);
349 if (tes) {
350 tes->setActionBlock((IOEventSource::ActionBlock) _action);
351 }
352
353 return tes;
354}
355
356#define _thread_call_cancel(tc) ((kActive & flags) ? thread_call_cancel_wait((tc)) : thread_call_cancel((tc)))
357
358IOTimerEventSource *
359IOTimerEventSource::timerEventSource(OSObject *inOwner, Action inAction)
360{
361 return IOTimerEventSource::timerEventSource(
362 inOptions: kIOTimerEventSourceOptionsPriorityKernelHigh,
363 inOwner, inAction);
364}
365
366void
367IOTimerEventSource::free()
368{
369 if (calloutEntry) {
370 __assert_only bool freed;
371
372 cancelTimeout();
373
374 freed = thread_call_free(call: (thread_call_t) calloutEntry);
375 assert(freed);
376 }
377
378 if (reserved) {
379 IOFreeType(reserved, ExpansionData);
380 }
381
382 super::free();
383}
384
385void
386IOTimerEventSource::cancelTimeout()
387{
388 if (reserved) {
389 reserved->calloutGeneration++;
390 }
391 bool active = _thread_call_cancel((thread_call_t) calloutEntry);
392 AbsoluteTime_to_scalar(&abstime) = 0;
393 if (active && reserved && (kPassive & flags)) {
394 release();
395 workLoop->release();
396 }
397}
398
399void
400IOTimerEventSource::enable()
401{
402 super::enable();
403 if (kIOReturnSuccess != wakeAtTime(abstime)) {
404 super::disable(); // Problem re-scheduling timeout ignore enable
405 }
406}
407
408void
409IOTimerEventSource::disable()
410{
411 if (reserved) {
412 reserved->calloutGeneration++;
413 }
414 bool active = _thread_call_cancel((thread_call_t) calloutEntry);
415 super::disable();
416 if (active && reserved && (kPassive & flags)) {
417 release();
418 workLoop->release();
419 }
420}
421
422IOReturn
423IOTimerEventSource::setTimeoutTicks(UInt32 ticks)
424{
425 return setTimeout(interval: ticks, scale_factor: kTickScale);
426}
427
428IOReturn
429IOTimerEventSource::setTimeoutMS(UInt32 ms)
430{
431 return setTimeout(interval: ms, scale_factor: kMillisecondScale);
432}
433
434IOReturn
435IOTimerEventSource::setTimeoutUS(UInt32 us)
436{
437 return setTimeout(interval: us, scale_factor: kMicrosecondScale);
438}
439
440IOReturn
441IOTimerEventSource::setTimeout(UInt32 interval, UInt32 scale_factor)
442{
443 AbsoluteTime end;
444
445 clock_interval_to_deadline(interval, scale_factor, result: &end);
446 return wakeAtTime(abstime: end);
447}
448
449#if !defined(__LP64__)
450IOReturn
451IOTimerEventSource::setTimeout(mach_timespec_t interval)
452{
453 AbsoluteTime end, nsecs;
454
455 clock_interval_to_absolutetime_interval
456 (interval.tv_nsec, kNanosecondScale, &nsecs);
457 clock_interval_to_deadline
458 (interval.tv_sec, NSEC_PER_SEC, &end);
459 ADD_ABSOLUTETIME(&end, &nsecs);
460
461 return wakeAtTime(end);
462}
463#endif
464
465IOReturn
466IOTimerEventSource::setTimeout(AbsoluteTime interval)
467{
468 AbsoluteTime end;
469 clock_absolutetime_interval_to_deadline(abstime: interval, result: &end);
470 return wakeAtTime(abstime: end);
471}
472
473IOReturn
474IOTimerEventSource::setTimeout(uint32_t options,
475 AbsoluteTime abstime, AbsoluteTime leeway)
476{
477 AbsoluteTime end;
478 if (options & kIOTimeOptionsContinuous) {
479 clock_continuoustime_interval_to_deadline(abstime, result: &end);
480 } else {
481 clock_absolutetime_interval_to_deadline(abstime, result: &end);
482 }
483
484 return wakeAtTime(options, abstime: end, leeway);
485}
486
487IOReturn
488IOTimerEventSource::wakeAtTimeTicks(UInt32 ticks)
489{
490 return wakeAtTime(abstime: ticks, scale_factor: kTickScale);
491}
492
493IOReturn
494IOTimerEventSource::wakeAtTimeMS(UInt32 ms)
495{
496 return wakeAtTime(abstime: ms, scale_factor: kMillisecondScale);
497}
498
499IOReturn
500IOTimerEventSource::wakeAtTimeUS(UInt32 us)
501{
502 return wakeAtTime(abstime: us, scale_factor: kMicrosecondScale);
503}
504
505IOReturn
506IOTimerEventSource::wakeAtTime(UInt32 inAbstime, UInt32 scale_factor)
507{
508 AbsoluteTime end;
509 clock_interval_to_absolutetime_interval(interval: inAbstime, scale_factor, result: &end);
510
511 return wakeAtTime(abstime: end);
512}
513
514#if !defined(__LP64__)
515IOReturn
516IOTimerEventSource::wakeAtTime(mach_timespec_t inAbstime)
517{
518 AbsoluteTime end, nsecs;
519
520 clock_interval_to_absolutetime_interval
521 (inAbstime.tv_nsec, kNanosecondScale, &nsecs);
522 clock_interval_to_absolutetime_interval
523 (inAbstime.tv_sec, kSecondScale, &end);
524 ADD_ABSOLUTETIME(&end, &nsecs);
525
526 return wakeAtTime(end);
527}
528#endif
529
530void
531IOTimerEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
532{
533 super::setWorkLoop(inWorkLoop);
534 if (enabled && AbsoluteTime_to_scalar(&abstime) && workLoop) {
535 wakeAtTime(abstime);
536 }
537}
538
539IOReturn
540IOTimerEventSource::wakeAtTime(AbsoluteTime inAbstime)
541{
542 return wakeAtTime(options: 0, abstime: inAbstime, leeway: 0);
543}
544
545IOReturn
546IOTimerEventSource::wakeAtTime(uint32_t options, AbsoluteTime inAbstime, AbsoluteTime leeway)
547{
548 if (!action) {
549 return kIOReturnNoResources;
550 }
551
552 abstime = inAbstime;
553 if (enabled && AbsoluteTime_to_scalar(&inAbstime) && AbsoluteTime_to_scalar(&abstime) && workLoop) {
554 uint32_t tcoptions = 0;
555
556 if (kIOTimeOptionsWithLeeway & options) {
557 tcoptions |= THREAD_CALL_DELAY_LEEWAY;
558 }
559 if (kIOTimeOptionsContinuous & options) {
560 tcoptions |= THREAD_CALL_CONTINUOUS;
561 }
562
563 if (reserved) {
564 if (kPassive & flags) {
565 retain();
566 workLoop->retain();
567 }
568 reserved->workLoop = workLoop;
569 reserved->calloutGeneration++;
570 if (thread_call_enter_delayed_with_leeway(call: (thread_call_t) calloutEntry,
571 param1: (void *)(uintptr_t) reserved->calloutGeneration, deadline: inAbstime, leeway, flags: tcoptions)
572 && (kPassive & flags)) {
573 release();
574 workLoop->release();
575 }
576 } else {
577 thread_call_enter_delayed_with_leeway(call: (thread_call_t) calloutEntry,
578 NULL, deadline: inAbstime, leeway, flags: tcoptions);
579 }
580 }
581
582 return kIOReturnSuccess;
583}
584