1 | /* |
2 | * Copyright (c) 1998-2014 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #define IOKIT_ENABLE_SHARED_PTR |
30 | |
31 | #include <ptrauth.h> |
32 | #include <IOKit/IOInterruptEventSource.h> |
33 | #include <IOKit/IOKitDebug.h> |
34 | #include <IOKit/IOLib.h> |
35 | #include <IOKit/IOService.h> |
36 | #include <IOKit/IOInterrupts.h> |
37 | #include <IOKit/IOTimeStamp.h> |
38 | #include <IOKit/IOWorkLoop.h> |
39 | #include <IOKit/IOInterruptAccountingPrivate.h> |
40 | #include <libkern/Block_private.h> |
41 | |
42 | #if IOKITSTATS |
43 | |
44 | #define IOStatisticsInitializeCounter() \ |
45 | do { \ |
46 | IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsInterruptEventSourceCounter); \ |
47 | } while (0) |
48 | |
49 | #define IOStatisticsCheckForWork() \ |
50 | do { \ |
51 | IOStatistics::countInterruptCheckForWork(IOEventSource::reserved->counter); \ |
52 | } while (0) |
53 | |
54 | #define IOStatisticsInterrupt() \ |
55 | do { \ |
56 | IOStatistics::countInterrupt(IOEventSource::reserved->counter); \ |
57 | } while (0) |
58 | |
59 | #else |
60 | |
61 | #define IOStatisticsInitializeCounter() |
62 | #define IOStatisticsCheckForWork() |
63 | #define IOStatisticsInterrupt() |
64 | |
65 | #endif // IOKITSTATS |
66 | |
67 | #define super IOEventSource |
68 | |
69 | OSDefineMetaClassAndStructors(IOInterruptEventSource, IOEventSource) |
70 | OSMetaClassDefineReservedUnused(IOInterruptEventSource, 0); |
71 | OSMetaClassDefineReservedUnused(IOInterruptEventSource, 1); |
72 | OSMetaClassDefineReservedUnused(IOInterruptEventSource, 2); |
73 | OSMetaClassDefineReservedUnused(IOInterruptEventSource, 3); |
74 | OSMetaClassDefineReservedUnused(IOInterruptEventSource, 4); |
75 | OSMetaClassDefineReservedUnused(IOInterruptEventSource, 5); |
76 | OSMetaClassDefineReservedUnused(IOInterruptEventSource, 6); |
77 | OSMetaClassDefineReservedUnused(IOInterruptEventSource, 7); |
78 | |
79 | bool |
80 | IOInterruptEventSource::init(OSObject *inOwner, |
81 | Action inAction, |
82 | IOService *inProvider, |
83 | int inIntIndex) |
84 | { |
85 | bool res = true; |
86 | |
87 | if (inIntIndex < 0) { |
88 | return false; |
89 | } |
90 | |
91 | if (!super::init(owner: inOwner, action: (IOEventSourceAction) inAction)) { |
92 | return false; |
93 | } |
94 | |
95 | reserved = IOMallocType(ExpansionData); |
96 | |
97 | provider = inProvider; |
98 | producerCount = consumerCount = 0; |
99 | autoDisable = explicitDisable = false; |
100 | intIndex = ~inIntIndex; |
101 | |
102 | // Assumes inOwner holds a reference(retain) on the provider |
103 | if (inProvider) { |
104 | if (IA_ANY_STATISTICS_ENABLED) { |
105 | /* |
106 | * We only treat this as an "interrupt" if it has a provider; if it does, |
107 | * set up the objects necessary to track interrupt statistics. Interrupt |
108 | * event sources without providers are most likely being used as simple |
109 | * event source in order to poke at workloops and kick off work. |
110 | * |
111 | * We also avoid try to avoid interrupt accounting overhead if none of |
112 | * the statistics are enabled. |
113 | */ |
114 | reserved->statistics = IOMallocType(IOInterruptAccountingData); |
115 | |
116 | reserved->statistics->owner = this; |
117 | } |
118 | |
119 | res = (kIOReturnSuccess == registerInterruptHandler(inProvider, inIntIndex)); |
120 | |
121 | if (res) { |
122 | intIndex = inIntIndex; |
123 | } |
124 | } |
125 | |
126 | IOStatisticsInitializeCounter(); |
127 | |
128 | return res; |
129 | } |
130 | |
131 | IOReturn |
132 | IOInterruptEventSource::registerInterruptHandler(IOService *inProvider, |
133 | int inIntIndex) |
134 | { |
135 | IOReturn ret; |
136 | int intType; |
137 | IOInterruptAction intHandler; |
138 | |
139 | ret = inProvider->getInterruptType(source: inIntIndex, interruptType: &intType); |
140 | if (kIOReturnSuccess != ret) { |
141 | return ret; |
142 | } |
143 | |
144 | autoDisable = (intType == kIOInterruptTypeLevel); |
145 | if (autoDisable) { |
146 | intHandler = OSMemberFunctionCast(IOInterruptAction, |
147 | this, &IOInterruptEventSource::disableInterruptOccurred); |
148 | } else { |
149 | intHandler = OSMemberFunctionCast(IOInterruptAction, |
150 | this, &IOInterruptEventSource::normalInterruptOccurred); |
151 | } |
152 | |
153 | ret = provider->registerInterrupt(source: inIntIndex, target: this, handler: intHandler); |
154 | |
155 | /* |
156 | * Add statistics to the provider. The setWorkLoop convention should ensure |
157 | * that we always go down the unregister path before we register (outside of |
158 | * init()), so we don't have to worry that we will invoke addInterruptStatistics |
159 | * erroneously. |
160 | */ |
161 | if ((ret == kIOReturnSuccess) && (reserved->statistics)) { |
162 | /* |
163 | * Stash the normal index value, for the sake of debugging. |
164 | */ |
165 | reserved->statistics->interruptIndex = inIntIndex; |
166 | |
167 | /* |
168 | * We need to hook the interrupt information up to the provider so that it |
169 | * can find the statistics for this interrupt when desired. The provider is |
170 | * responsible for maintaining the reporter for a particular interrupt, and |
171 | * needs a handle on the statistics so that it can request that the reporter |
172 | * be updated as needed. Errors are considered "soft" for the moment (it |
173 | * will either panic, or fail in a way such that we can still service the |
174 | * interrupt). |
175 | */ |
176 | provider->addInterruptStatistics(statistics: reserved->statistics, source: inIntIndex); |
177 | |
178 | /* |
179 | * Add the statistics object to the global list of statistics objects; this |
180 | * is an aid to debugging (we can trivially find statistics for all eligible |
181 | * interrupts, and dump them; potentially helpful if the system is wedged |
182 | * due to interrupt activity). |
183 | */ |
184 | interruptAccountingDataAddToList(data: reserved->statistics); |
185 | } |
186 | |
187 | return ret; |
188 | } |
189 | |
190 | void |
191 | IOInterruptEventSource::unregisterInterruptHandler(IOService *inProvider, |
192 | int inIntIndex) |
193 | { |
194 | if (reserved->statistics) { |
195 | interruptAccountingDataRemoveFromList(data: reserved->statistics); |
196 | provider->removeInterruptStatistics(source: reserved->statistics->interruptIndex); |
197 | } |
198 | |
199 | provider->unregisterInterrupt(source: inIntIndex); |
200 | } |
201 | |
202 | |
203 | OSSharedPtr<IOInterruptEventSource> |
204 | IOInterruptEventSource::interruptEventSource(OSObject *inOwner, |
205 | Action inAction, |
206 | IOService *inProvider, |
207 | int inIntIndex) |
208 | { |
209 | OSSharedPtr<IOInterruptEventSource> me = OSMakeShared<IOInterruptEventSource>(); |
210 | |
211 | if (me && !me->init(inOwner, inAction, inProvider, inIntIndex)) { |
212 | return nullptr; |
213 | } |
214 | |
215 | return me; |
216 | } |
217 | |
218 | OSSharedPtr<IOInterruptEventSource> |
219 | IOInterruptEventSource::interruptEventSource(OSObject *inOwner, |
220 | IOService *inProvider, |
221 | int inIntIndex, |
222 | ActionBlock inAction) |
223 | { |
224 | OSSharedPtr<IOInterruptEventSource> ies; |
225 | ies = IOInterruptEventSource::interruptEventSource(inOwner, inAction: (Action) NULL, inProvider, inIntIndex); |
226 | if (ies) { |
227 | ies->setActionBlock((IOEventSource::ActionBlock) inAction); |
228 | } |
229 | |
230 | return ies; |
231 | } |
232 | |
233 | void |
234 | IOInterruptEventSource::free() |
235 | { |
236 | if (provider && intIndex >= 0) { |
237 | unregisterInterruptHandler(inProvider: provider, inIntIndex: intIndex); |
238 | } |
239 | |
240 | if (reserved) { |
241 | if (reserved->statistics) { |
242 | IOFreeType(reserved->statistics, IOInterruptAccountingData); |
243 | } |
244 | |
245 | IOFreeType(reserved, ExpansionData); |
246 | } |
247 | |
248 | super::free(); |
249 | } |
250 | |
251 | void |
252 | IOInterruptEventSource::enable() |
253 | { |
254 | if (provider && intIndex >= 0) { |
255 | provider->enableInterrupt(source: intIndex); |
256 | explicitDisable = false; |
257 | enabled = true; |
258 | } |
259 | } |
260 | |
261 | void |
262 | IOInterruptEventSource::disable() |
263 | { |
264 | if (provider && intIndex >= 0) { |
265 | provider->disableInterrupt(source: intIndex); |
266 | explicitDisable = true; |
267 | enabled = false; |
268 | } |
269 | } |
270 | |
271 | void |
272 | IOInterruptEventSource::setWorkLoop(IOWorkLoop *inWorkLoop) |
273 | { |
274 | if (inWorkLoop) { |
275 | super::setWorkLoop(inWorkLoop); |
276 | } |
277 | |
278 | if (provider) { |
279 | if (!inWorkLoop) { |
280 | if (intIndex >= 0) { |
281 | /* |
282 | * It isn't necessarily safe to wait until free() to unregister the interrupt; |
283 | * our provider may disappear. |
284 | */ |
285 | unregisterInterruptHandler(inProvider: provider, inIntIndex: intIndex); |
286 | intIndex = ~intIndex; |
287 | } |
288 | } else if ((intIndex < 0) && (kIOReturnSuccess == registerInterruptHandler(inProvider: provider, inIntIndex: ~intIndex))) { |
289 | intIndex = ~intIndex; |
290 | } |
291 | } |
292 | |
293 | if (!inWorkLoop) { |
294 | super::setWorkLoop(inWorkLoop); |
295 | } |
296 | } |
297 | |
298 | const IOService * |
299 | IOInterruptEventSource::getProvider() const |
300 | { |
301 | return provider; |
302 | } |
303 | |
304 | int |
305 | IOInterruptEventSource::getIntIndex() const |
306 | { |
307 | return intIndex; |
308 | } |
309 | |
310 | bool |
311 | IOInterruptEventSource::getAutoDisable() const |
312 | { |
313 | return autoDisable; |
314 | } |
315 | |
316 | bool |
317 | IOInterruptEventSource::checkForWork() |
318 | { |
319 | uint64_t startSystemTime = 0; |
320 | uint64_t endSystemTime = 0; |
321 | uint64_t startCPUTime = 0; |
322 | uint64_t endCPUTime = 0; |
323 | unsigned int cacheProdCount = producerCount; |
324 | int numInts = cacheProdCount - consumerCount; |
325 | IOEventSource::Action intAction = action; |
326 | ActionBlock intActionBlock = (ActionBlock) actionBlock; |
327 | void *address; |
328 | bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false; |
329 | |
330 | if (kActionBlock & flags) { |
331 | address = ptrauth_nop_cast(void *, _Block_get_invoke_fn((struct Block_layout *)intActionBlock)); |
332 | } else { |
333 | address = ptrauth_nop_cast(void *, intAction); |
334 | } |
335 | |
336 | IOStatisticsCheckForWork(); |
337 | |
338 | if (numInts > 0) { |
339 | if (trace) { |
340 | IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION), |
341 | VM_KERNEL_ADDRHIDE(address), |
342 | VM_KERNEL_ADDRHIDE(owner), |
343 | VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); |
344 | } |
345 | |
346 | if (reserved->statistics) { |
347 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { |
348 | startSystemTime = mach_absolute_time(); |
349 | } |
350 | |
351 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { |
352 | startCPUTime = thread_get_runtime_self(); |
353 | } |
354 | } |
355 | |
356 | // Call the handler |
357 | if (kActionBlock & flags) { |
358 | (intActionBlock)(this, numInts); |
359 | } else { |
360 | ((IOInterruptEventAction)intAction)(owner, this, numInts); |
361 | } |
362 | |
363 | if (reserved->statistics) { |
364 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) { |
365 | IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1); |
366 | } |
367 | |
368 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { |
369 | endCPUTime = thread_get_runtime_self(); |
370 | IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime); |
371 | } |
372 | |
373 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { |
374 | endSystemTime = mach_absolute_time(); |
375 | IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime); |
376 | } |
377 | } |
378 | |
379 | if (trace) { |
380 | IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION), |
381 | VM_KERNEL_ADDRHIDE(address), |
382 | VM_KERNEL_ADDRHIDE(owner), |
383 | VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); |
384 | } |
385 | |
386 | consumerCount = cacheProdCount; |
387 | if (autoDisable && !explicitDisable) { |
388 | enable(); |
389 | } |
390 | } else if (numInts < 0) { |
391 | if (trace) { |
392 | IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION), |
393 | VM_KERNEL_ADDRHIDE(address), |
394 | VM_KERNEL_ADDRHIDE(owner), |
395 | VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); |
396 | } |
397 | |
398 | if (reserved->statistics) { |
399 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { |
400 | startSystemTime = mach_absolute_time(); |
401 | } |
402 | |
403 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { |
404 | startCPUTime = thread_get_runtime_self(); |
405 | } |
406 | } |
407 | |
408 | // Call the handler |
409 | if (kActionBlock & flags) { |
410 | (intActionBlock)(this, numInts); |
411 | } else { |
412 | ((IOInterruptEventAction)intAction)(owner, this, numInts); |
413 | } |
414 | |
415 | if (reserved->statistics) { |
416 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) { |
417 | endSystemTime = mach_absolute_time(); |
418 | IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime); |
419 | } |
420 | |
421 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) { |
422 | endCPUTime = thread_get_runtime_self(); |
423 | IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime); |
424 | } |
425 | |
426 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) { |
427 | IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1); |
428 | } |
429 | } |
430 | |
431 | if (trace) { |
432 | IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION), |
433 | VM_KERNEL_ADDRHIDE(address), |
434 | VM_KERNEL_ADDRHIDE(owner), |
435 | VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(workLoop)); |
436 | } |
437 | |
438 | consumerCount = cacheProdCount; |
439 | if (autoDisable && !explicitDisable) { |
440 | enable(); |
441 | } |
442 | } |
443 | |
444 | return false; |
445 | } |
446 | |
447 | void |
448 | IOInterruptEventSource::normalInterruptOccurred |
449 | (void */*refcon*/, IOService */*prov*/, int /*source*/) |
450 | { |
451 | bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false; |
452 | |
453 | IOStatisticsInterrupt(); |
454 | producerCount++; |
455 | |
456 | if (trace) { |
457 | IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); |
458 | } |
459 | |
460 | if (reserved->statistics) { |
461 | if (reserved->statistics->enablePrimaryTimestamp) { |
462 | reserved->statistics->primaryTimestamp = mach_absolute_time(); |
463 | } |
464 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) { |
465 | IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1); |
466 | } |
467 | } |
468 | |
469 | signalWorkAvailable(); |
470 | |
471 | if (trace) { |
472 | IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); |
473 | } |
474 | } |
475 | |
476 | void |
477 | IOInterruptEventSource::disableInterruptOccurred |
478 | (void */*refcon*/, IOService *prov, int source) |
479 | { |
480 | bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false; |
481 | |
482 | prov->disableInterrupt(source); /* disable the interrupt */ |
483 | |
484 | IOStatisticsInterrupt(); |
485 | producerCount++; |
486 | |
487 | if (trace) { |
488 | IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); |
489 | } |
490 | |
491 | if (reserved->statistics) { |
492 | if (reserved->statistics->enablePrimaryTimestamp) { |
493 | reserved->statistics->primaryTimestamp = mach_absolute_time(); |
494 | } |
495 | if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) { |
496 | IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1); |
497 | } |
498 | } |
499 | |
500 | signalWorkAvailable(); |
501 | |
502 | if (trace) { |
503 | IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), VM_KERNEL_ADDRHIDE(this), VM_KERNEL_ADDRHIDE(owner)); |
504 | } |
505 | } |
506 | |
507 | void |
508 | IOInterruptEventSource::interruptOccurred |
509 | (void *_refcon, IOService *prov, int source) |
510 | { |
511 | if (autoDisable && prov) { |
512 | disableInterruptOccurred(_refcon, prov, source); |
513 | } else { |
514 | normalInterruptOccurred(_refcon, prov, source); |
515 | } |
516 | } |
517 | |
518 | IOReturn |
519 | IOInterruptEventSource::warmCPU |
520 | (uint64_t abstime) |
521 | { |
522 | return ml_interrupt_prewarm(deadline: abstime); |
523 | } |
524 | |
525 | void |
526 | IOInterruptEventSource::enablePrimaryInterruptTimestamp(bool enable) |
527 | { |
528 | if (reserved->statistics) { |
529 | reserved->statistics->enablePrimaryTimestamp = enable; |
530 | } |
531 | } |
532 | |
533 | uint64_t |
534 | IOInterruptEventSource::getPrimaryInterruptTimestamp() |
535 | { |
536 | if (reserved->statistics && reserved->statistics->enablePrimaryTimestamp) { |
537 | return reserved->statistics->primaryTimestamp; |
538 | } |
539 | return -1ULL; |
540 | } |
541 | |