1/*
2 * Copyright (c) 1998-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <IOKit/system.h>
30#include <IOKit/IOService.h>
31#include <libkern/OSDebug.h>
32#include <libkern/c++/OSAllocation.h>
33#include <libkern/c++/OSContainers.h>
34#include <libkern/c++/OSKext.h>
35#include <libkern/c++/OSUnserialize.h>
36#include <libkern/c++/OSKext.h>
37#include <libkern/c++/OSSharedPtr.h>
38#include <libkern/Block.h>
39#include <IOKit/IOCatalogue.h>
40#include <IOKit/IOCommand.h>
41#include <IOKit/IODeviceTreeSupport.h>
42#include <IOKit/IODeviceMemory.h>
43#include <IOKit/IOInterrupts.h>
44#include <IOKit/IOInterruptController.h>
45#include <IOKit/IOPlatformExpert.h>
46#include <IOKit/IOMessage.h>
47#include <IOKit/IOLib.h>
48#include <IOKit/IOKitKeysPrivate.h>
49#include <IOKit/IOBSD.h>
50#include <IOKit/IOUserClient.h>
51#include <IOKit/IOUserServer.h>
52#include <IOKit/IOWorkLoop.h>
53#include <IOKit/IOTimeStamp.h>
54#include <IOKit/IOHibernatePrivate.h>
55#include <IOKit/IOInterruptAccountingPrivate.h>
56#include <IOKit/IOKernelReporters.h>
57#include <IOKit/AppleKeyStoreInterface.h>
58#include <IOKit/pwr_mgt/RootDomain.h>
59#include <IOKit/IOCPU.h>
60#include <Exclaves/Exclaves.h>
61#include <kern/cs_blobs.h>
62#include <mach/sync_policy.h>
63#include <mach/thread_info.h>
64#include <IOKit/assert.h>
65#include <sys/errno.h>
66#include <sys/kdebug.h>
67#include <string.h>
68
69#include <machine/pal_routines.h>
70
71#define LOG kprintf
72//#define LOG IOLog
73#define MATCH_DEBUG 0
74#define IOSERVICE_OBFUSCATE(x) ((void *)(VM_KERNEL_ADDRPERM(x)))
75
76// disabled since lockForArbitration() can be held externally
77#define DEBUG_NOTIFIER_LOCKED 0
78
79enum{
80 kIOUserServerCheckInTimeoutSecs = 120ULL
81};
82
83#include "IOServicePrivate.h"
84#include "IOKitKernelInternal.h"
85
86// take lockForArbitration before LOCKNOTIFY
87
88/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
89
90#define super IORegistryEntry
91
92OSDefineMetaClassAndStructors(IOService, IORegistryEntry)
93
94OSDefineMetaClassAndStructors(_IOServiceNotifier, IONotifier)
95OSDefineMetaClassAndStructors(_IOServiceNullNotifier, IONotifier)
96
97OSDefineMetaClassAndStructors(_IOServiceInterestNotifier, IONotifier)
98
99OSDefineMetaClassAndStructors(_IOConfigThread, OSObject)
100
101OSDefineMetaClassAndStructors(_IOServiceJob, OSObject)
102
103OSDefineMetaClassAndStructors(IOResources, IOService)
104OSDefineMetaClassAndStructors(IOUserResources, IOService)
105OSDefineMetaClassAndStructors(IOExclaveProxy, IOService)
106
107OSDefineMetaClassAndStructors(_IOOpenServiceIterator, OSIterator)
108
109OSDefineMetaClassAndStructors(_IOServiceStateNotification, IOService)
110
111OSDefineMetaClassAndAbstractStructors(IONotifier, OSObject)
112
113OSDefineMetaClassAndStructors(IOServiceCompatibility, IOService)
114
115/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
116
117static IOPlatformExpert * gIOPlatform;
118static class IOPMrootDomain * gIOPMRootDomain;
119const IORegistryPlane * gIOServicePlane;
120const IORegistryPlane * gIOPowerPlane;
121const OSSymbol * gIODeviceMemoryKey;
122const OSSymbol * gIOInterruptControllersKey;
123const OSSymbol * gIOInterruptSpecifiersKey;
124
125const OSSymbol * gIOResourcesKey;
126const OSSymbol * gIOUserResourcesKey;
127const OSSymbol * gIOResourceMatchKey;
128const OSSymbol * gIOResourceMatchedKey;
129const OSSymbol * gIOResourceIOKitKey;
130
131const OSSymbol * gIOProviderClassKey;
132const OSSymbol * gIONameMatchKey;
133const OSSymbol * gIONameMatchedKey;
134const OSSymbol * gIOPropertyMatchKey;
135const OSSymbol * gIOPropertyExistsMatchKey;
136const OSSymbol * gIOLocationMatchKey;
137const OSSymbol * gIOParentMatchKey;
138const OSSymbol * gIOPathMatchKey;
139const OSSymbol * gIOMatchCategoryKey;
140const OSSymbol * gIODefaultMatchCategoryKey;
141const OSSymbol * gIOMatchedAtBootKey;
142const OSSymbol * gIOMatchedServiceCountKey;
143const OSSymbol * gIOMatchedPersonalityKey;
144const OSSymbol * gIORematchPersonalityKey;
145const OSSymbol * gIORematchCountKey;
146const OSSymbol * gIODEXTMatchCountKey;
147const OSSymbol * gIOSupportedPropertiesKey;
148const OSSymbol * gIOUserServicePropertiesKey;
149#if defined(XNU_TARGET_OS_OSX)
150const OSSymbol * gIOServiceLegacyMatchingRegistryIDKey;
151#endif /* defined(XNU_TARGET_OS_OSX) */
152
153const OSSymbol * gIOCompatibilityMatchKey;
154const OSSymbol * gIOCompatibilityPropertiesKey;
155const OSSymbol * gIOPathKey;
156
157const OSSymbol * gIOMapperIDKey;
158const OSSymbol * gIOUserClientClassKey;
159
160const OSSymbol * gIOUserClassKey;
161const OSSymbol * gIOUserClassesKey;
162const OSSymbol * gIOUserServerClassKey;
163const OSSymbol * gIOUserServerNameKey;
164const OSSymbol * gIOUserServerTagKey;
165const OSSymbol * gIOUserUserClientKey;
166const OSSymbol * gIOUserServerOneProcessKey;
167const OSSymbol * gIOUserServerPreserveUserspaceRebootKey;
168
169const OSSymbol * gIOKitDebugKey;
170
171const OSSymbol * gIOCommandPoolSizeKey;
172
173const OSSymbol * gIOConsoleLockedKey;
174const OSSymbol * gIOConsoleUsersKey;
175const OSSymbol * gIOConsoleSessionUIDKey;
176const OSSymbol * gIOConsoleSessionAuditIDKey;
177const OSSymbol * gIOConsoleUsersSeedKey;
178const OSSymbol * gIOConsoleSessionOnConsoleKey;
179const OSSymbol * gIOConsoleSessionLoginDoneKey;
180const OSSymbol * gIOConsoleSessionSecureInputPIDKey;
181const OSSymbol * gIOConsoleSessionScreenLockedTimeKey;
182const OSSymbol * gIOConsoleSessionScreenIsLockedKey;
183clock_sec_t gIOConsoleLockTime;
184static bool gIOConsoleLoggedIn;
185#if HIBERNATION
186static OSBoolean * gIOConsoleBooterLockState;
187static uint32_t gIOScreenLockState;
188#endif
189static IORegistryEntry * gIOChosenEntry;
190
191static int gIOResourceGenerationCount;
192
193const OSSymbol * gIOServiceKey;
194const OSSymbol * gIOPublishNotification;
195const OSSymbol * gIOFirstPublishNotification;
196const OSSymbol * gIOMatchedNotification;
197const OSSymbol * gIOFirstMatchNotification;
198const OSSymbol * gIOTerminatedNotification;
199const OSSymbol * gIOWillTerminateNotification;
200
201const OSSymbol * gIOUserClientEntitlementsKey;
202const OSSymbol * gIOServiceDEXTEntitlementsKey;
203const OSSymbol * gIODriverKitEntitlementKey;
204const OSSymbol * gIODriverKitUserClientEntitlementsKey;
205const OSSymbol * gIODriverKitUserClientEntitlementAllowAnyKey;
206const OSSymbol * gIODriverKitRequiredEntitlementsKey;
207const OSSymbol * gIODriverKitTestDriverEntitlementKey;
208const OSSymbol * gIODriverKitUserClientEntitlementCommunicatesWithDriversKey;
209const OSSymbol * gIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey;
210const OSSymbol * gIOMatchDeferKey;
211const OSSymbol * gIOServiceMatchDeferredKey;
212const OSSymbol * gIOServiceNotificationUserKey;
213
214const OSSymbol * gIOExclaveAssignedKey;
215const OSSymbol * gIOExclaveProxyKey;
216
217const OSSymbol * gIOPrimaryDriverTerminateOptionsKey;
218const OSSymbol * gIOMediaKey;
219const OSSymbol * gIOBlockStorageDriverKey;
220static const OSSymbol * gPhysicalInterconnectKey;
221static const OSSymbol * gVirtualInterfaceKey;
222
223const OSSymbol * gIOAllCPUInitializedKey;
224
225const OSSymbol * gIOGeneralInterest;
226const OSSymbol * gIOBusyInterest;
227const OSSymbol * gIOAppPowerStateInterest;
228const OSSymbol * gIOPriorityPowerStateInterest;
229const OSSymbol * gIOConsoleSecurityInterest;
230
231const OSSymbol * gIOBSDKey;
232const OSSymbol * gIOBSDNameKey;
233const OSSymbol * gIOBSDMajorKey;
234const OSSymbol * gIOBSDMinorKey;
235const OSSymbol * gIOBSDUnitKey;
236
237const OSSymbol * gAKSGetKey;
238#if defined(__i386__) || defined(__x86_64__)
239const OSSymbol * gIOCreateEFIDevicePathSymbol;
240#endif
241
242static OSDictionary * gNotifications;
243static IORecursiveLock * gNotificationLock;
244
245static IOService * gIOResources;
246static IOService * gIOUserResources;
247static IOService * gIOServiceRoot;
248
249static OSOrderedSet * gJobs;
250static semaphore_port_t gJobsSemaphore;
251static IOLock * gJobsLock;
252static int gOutstandingJobs;
253static int gNumConfigThreads;
254static int gHighNumConfigThreads;
255static int gMaxConfigThreads = kMaxConfigThreads;
256static int gNumWaitingThreads;
257static IOLock * gIOServiceBusyLock;
258bool gCPUsRunning;
259bool gIOKitWillTerminate;
260bool gInUserspaceReboot;
261
262#define kIOServiceRootMediaParentInvalid ((IOService *) -1UL)
263#if NO_KEXTD
264static bool gIOServiceHideIOMedia = false;
265static IOService * gIOServiceRootMediaParent = NULL;
266#else /* NO_KEXTD */
267static bool gIOServiceHideIOMedia = true;
268static IOService * gIOServiceRootMediaParent = kIOServiceRootMediaParentInvalid;
269#endif /* !NO_KEXTD */
270
271static thread_t gIOTerminateThread;
272static thread_t gIOTerminateWorkerThread;
273static UInt32 gIOTerminateWork;
274static OSArray * gIOTerminatePhase2List;
275static OSArray * gIOStopList;
276static OSArray * gIOStopProviderList;
277static OSArray * gIOFinalizeList;
278
279#if !NO_KEXTD
280static OSArray * gIOMatchDeferList;
281#endif
282
283static SInt32 gIOConsoleUsersSeed;
284static OSData * gIOConsoleUsersSeedValue;
285
286extern const OSSymbol * gIODTPHandleKey;
287
288const OSSymbol * gIOPlatformFunctionHandlerSet;
289
290
291static IOLock * gIOConsoleUsersLock;
292static thread_call_t gIOConsoleLockCallout;
293static IONotifier * gIOServiceNullNotifier;
294
295static uint32_t gIODextRelaunchMax = 1000;
296
297#if DEVELOPMENT || DEBUG
298uint64_t driverkit_checkin_timed_out = 0;
299#endif
300
301IORecursiveLock * gDriverKitLaunchLock;
302OSSet * gDriverKitLaunches;
303const OSSymbol * gIOAssociatedServicesKey;
304
305/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
306
307#define LOCKREADNOTIFY() \
308 IORecursiveLockLock( gNotificationLock )
309#define LOCKWRITENOTIFY() \
310 IORecursiveLockLock( gNotificationLock )
311#define LOCKWRITE2READNOTIFY()
312#define UNLOCKNOTIFY() \
313 IORecursiveLockUnlock( gNotificationLock )
314#define SLEEPNOTIFY(event) \
315 IORecursiveLockSleep( gNotificationLock, (void *)(event), THREAD_UNINT )
316#define SLEEPNOTIFYTO(event, deadline) \
317 IORecursiveLockSleepDeadline( gNotificationLock, (void *)(event), deadline, THREAD_UNINT )
318#define WAKEUPNOTIFY(event) \
319 IORecursiveLockWakeup( gNotificationLock, (void *)(event), /* wake one */ false )
320
321#define randomDelay() \
322 int del = read_processor_clock(); \
323 del = (((int)IOThreadSelf()) ^ del ^ (del >> 10)) & 0x3ff; \
324 IOSleep( del );
325
326/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
327
328#define queue_element(entry, element, type, field) do { \
329 vm_address_t __ele = (vm_address_t) (entry); \
330 __ele -= -4 + ((size_t)(&((type) 4)->field)); \
331 (element) = (type) __ele; \
332 } while(0)
333
334#define iterqueue(que, elt) \
335 for (queue_entry_t elt = queue_first(que); \
336 !queue_end(que, elt); \
337 elt = queue_next(elt))
338
339/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
340
341struct IOInterruptAccountingReporter {
342 IOSimpleReporter * reporter; /* Reporter responsible for communicating the statistics */
343 IOInterruptAccountingData * statistics; /* The live statistics values, if any */
344};
345
346struct ArbitrationLockQueueElement {
347 queue_chain_t link;
348 IOThread thread;
349 IOService * service;
350 unsigned count;
351 bool required;
352 bool aborted;
353};
354
355static queue_head_t gArbitrationLockQueueActive;
356static queue_head_t gArbitrationLockQueueWaiting;
357static queue_head_t gArbitrationLockQueueFree;
358static IOLock * gArbitrationLockQueueLock;
359
360bool
361IOService::isInactive( void ) const
362{
363 return 0 != (kIOServiceInactiveState & getState());
364}
365
366/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
367
368// Only used by the intel implementation of
369// IOService::requireMaxBusStall(UInt32 ns)
370// IOService::requireMaxInterruptDelay(uint32_t ns)
371struct CpuDelayEntry {
372 IOService * fService;
373 UInt32 fMaxDelay;
374 UInt32 fDelayType;
375};
376
377enum {
378 kCpuDelayBusStall,
379#if defined(__x86_64__)
380 kCpuDelayInterrupt,
381#endif /* defined(__x86_64__) */
382 kCpuNumDelayTypes
383};
384
385static OSData *sCpuDelayData = OSData::withCapacity(capacity: 8 * sizeof(CpuDelayEntry));
386static IORecursiveLock *sCpuDelayLock = IORecursiveLockAlloc();
387static OSArray *sCpuLatencyHandlers[kCpuNumDelayTypes];
388const OSSymbol *sCPULatencyFunctionName[kCpuNumDelayTypes];
389static OSNumber * sCPULatencyHolder[kCpuNumDelayTypes];
390static char sCPULatencyHolderName[kCpuNumDelayTypes][128];
391static OSNumber * sCPULatencySet[kCpuNumDelayTypes];
392
393static void
394requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType);
395static IOReturn
396setLatencyHandler(UInt32 delayType, IOService * target, bool enable);
397
398/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
399
400IOCoreAnalyticsSendEventProc gIOCoreAnalyticsSendEventProc;
401
402kern_return_t
403IOSetCoreAnalyticsSendEventProc(IOCoreAnalyticsSendEventProc proc)
404{
405 if (gIOCoreAnalyticsSendEventProc) {
406 return kIOReturnNotPermitted;
407 }
408 gIOCoreAnalyticsSendEventProc = proc;
409
410 return kIOReturnSuccess;
411}
412
413/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
414
415
416static IOMessage sSystemPower;
417
418namespace IOServicePH
419{
420IONotifier * fRootNotifier;
421OSArray * fUserServers;
422OSArray * fUserServersWait;
423OSArray * fMatchingWork;
424OSArray * fMatchingDelayed;
425IOService * fSystemPowerAckTo;
426uint32_t fSystemPowerAckRef;
427uint8_t fSystemOff;
428uint8_t fUserServerOff;
429uint8_t fWaitingUserServers;
430thread_call_t fUserServerAckTimer;
431
432void lock();
433void unlock();
434
435void init(IOPMrootDomain * root);
436
437IOReturn systemPowerChange(
438 void * target,
439 void * refCon,
440 UInt32 messageType, IOService * service,
441 void * messageArgument, vm_size_t argSize);
442
443bool matchingStart(IOService * service);
444void matchingEnd(IOService * service);
445void userServerAckTimerExpired(void *, void *);
446};
447
448/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
449
450void
451IOService::initialize( void )
452{
453 kern_return_t err;
454
455 gIOServicePlane = IORegistryEntry::makePlane( kIOServicePlane );
456 gIOPowerPlane = IORegistryEntry::makePlane( kIOPowerPlane );
457
458 gIOProviderClassKey = OSSymbol::withCStringNoCopy( kIOProviderClassKey );
459 gIONameMatchKey = OSSymbol::withCStringNoCopy( kIONameMatchKey );
460 gIONameMatchedKey = OSSymbol::withCStringNoCopy( kIONameMatchedKey );
461 gIOPropertyMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyMatchKey );
462 gIOPropertyExistsMatchKey = OSSymbol::withCStringNoCopy( kIOPropertyExistsMatchKey );
463 gIOPathMatchKey = OSSymbol::withCStringNoCopy( kIOPathMatchKey );
464 gIOLocationMatchKey = OSSymbol::withCStringNoCopy( kIOLocationMatchKey );
465 gIOParentMatchKey = OSSymbol::withCStringNoCopy( kIOParentMatchKey );
466
467 gIOMatchCategoryKey = OSSymbol::withCStringNoCopy( kIOMatchCategoryKey );
468 gIODefaultMatchCategoryKey = OSSymbol::withCStringNoCopy(
469 kIODefaultMatchCategoryKey );
470 gIOMatchedAtBootKey = OSSymbol::withCStringNoCopy(
471 kIOMatchedAtBootKey );
472
473 gIOMatchedServiceCountKey = OSSymbol::withCStringNoCopy(
474 kIOMatchedServiceCountKey );
475 gIOMatchedPersonalityKey = OSSymbol::withCStringNoCopy(
476 kIOMatchedPersonalityKey );
477 gIORematchPersonalityKey = OSSymbol::withCStringNoCopy(
478 kIORematchPersonalityKey );
479 gIORematchCountKey = OSSymbol::withCStringNoCopy(
480 kIORematchCountKey );
481 gIODEXTMatchCountKey = OSSymbol::withCStringNoCopy(
482 kIODEXTMatchCountKey );
483
484#if defined(XNU_TARGET_OS_OSX)
485 gIOServiceLegacyMatchingRegistryIDKey = OSSymbol::withCStringNoCopy(
486 kIOServiceLegacyMatchingRegistryIDKey );
487#endif /* defined(XNU_TARGET_OS_OSX) */
488
489 PE_parse_boot_argn(arg_string: "dextrelaunch", arg_ptr: &gIODextRelaunchMax, max_arg: sizeof(gIODextRelaunchMax));
490 PE_parse_boot_argn(arg_string: "iocthreads", arg_ptr: &gMaxConfigThreads, max_arg: sizeof(gMaxConfigThreads));
491
492 gIOUserClientClassKey = OSSymbol::withCStringNoCopy( kIOUserClientClassKey );
493
494 gIOUserClassKey = OSSymbol::withCStringNoCopy(kIOUserClassKey);
495 gIOUserClassesKey = OSSymbol::withCStringNoCopy(kIOUserClassesKey);
496
497 gIOUserServerClassKey = OSSymbol::withCStringNoCopy(kIOUserServerClassKey);
498 gIOUserServerNameKey = OSSymbol::withCStringNoCopy(kIOUserServerNameKey);
499 gIOUserServerTagKey = OSSymbol::withCStringNoCopy(kIOUserServerTagKey);
500 gIOUserUserClientKey = OSSymbol::withCStringNoCopy(kIOUserUserClientKey);
501
502 gIOUserServerOneProcessKey = OSSymbol::withCStringNoCopy(kIOUserServerOneProcessKey);
503 gIOUserServerPreserveUserspaceRebootKey = OSSymbol::withCStringNoCopy(kIOUserServerPreserveUserspaceRebootKey);
504
505 gIOResourcesKey = OSSymbol::withCStringNoCopy( kIOResourcesClass );
506 gIOResourceMatchKey = OSSymbol::withCStringNoCopy( kIOResourceMatchKey );
507 gIOResourceMatchedKey = OSSymbol::withCStringNoCopy( kIOResourceMatchedKey );
508 gIOResourceIOKitKey = OSSymbol::withCStringNoCopy(cString: "IOKit");
509
510 gIODeviceMemoryKey = OSSymbol::withCStringNoCopy( cString: "IODeviceMemory" );
511 gIOInterruptControllersKey
512 = OSSymbol::withCStringNoCopy(cString: "IOInterruptControllers");
513 gIOInterruptSpecifiersKey
514 = OSSymbol::withCStringNoCopy(cString: "IOInterruptSpecifiers");
515
516 gIOCompatibilityMatchKey = OSSymbol::withCStringNoCopy(kIOCompatibilityMatchKey);
517 gIOCompatibilityPropertiesKey = OSSymbol::withCStringNoCopy(kIOCompatibilityPropertiesKey);
518 gIOPathKey = OSSymbol::withCStringNoCopy(kIOPathKey);
519 gIOSupportedPropertiesKey = OSSymbol::withCStringNoCopy(kIOSupportedPropertiesKey);
520 gIOUserServicePropertiesKey = OSSymbol::withCStringNoCopy(kIOUserServicePropertiesKey);
521
522 gIOMapperIDKey = OSSymbol::withCStringNoCopy(kIOMapperIDKey);
523
524 gIOKitDebugKey = OSSymbol::withCStringNoCopy( kIOKitDebugKey );
525
526 gIOCommandPoolSizeKey = OSSymbol::withCStringNoCopy( kIOCommandPoolSizeKey );
527
528 gIOGeneralInterest = OSSymbol::withCStringNoCopy( kIOGeneralInterest );
529 gIOBusyInterest = OSSymbol::withCStringNoCopy( kIOBusyInterest );
530 gIOAppPowerStateInterest = OSSymbol::withCStringNoCopy( kIOAppPowerStateInterest );
531 gIOPriorityPowerStateInterest = OSSymbol::withCStringNoCopy( kIOPriorityPowerStateInterest );
532 gIOConsoleSecurityInterest = OSSymbol::withCStringNoCopy( kIOConsoleSecurityInterest );
533
534 gIOBSDKey = OSSymbol::withCStringNoCopy(kIOBSDKey);
535 gIOBSDNameKey = OSSymbol::withCStringNoCopy(kIOBSDNameKey);
536 gIOBSDMajorKey = OSSymbol::withCStringNoCopy(kIOBSDMajorKey);
537 gIOBSDMinorKey = OSSymbol::withCStringNoCopy(kIOBSDMinorKey);
538 gIOBSDUnitKey = OSSymbol::withCStringNoCopy(kIOBSDUnitKey);
539
540 gNotifications = OSDictionary::withCapacity( capacity: 1 );
541 gIOPublishNotification = OSSymbol::withCStringNoCopy(
542 kIOPublishNotification );
543 gIOFirstPublishNotification = OSSymbol::withCStringNoCopy(
544 kIOFirstPublishNotification );
545 gIOMatchedNotification = OSSymbol::withCStringNoCopy(
546 kIOMatchedNotification );
547 gIOFirstMatchNotification = OSSymbol::withCStringNoCopy(
548 kIOFirstMatchNotification );
549 gIOTerminatedNotification = OSSymbol::withCStringNoCopy(
550 kIOTerminatedNotification );
551 gIOWillTerminateNotification = OSSymbol::withCStringNoCopy(
552 kIOWillTerminateNotification );
553 gIOServiceKey = OSSymbol::withCStringNoCopy( kIOServiceClass);
554
555
556 gIOConsoleLockedKey = OSSymbol::withCStringNoCopy( kIOConsoleLockedKey);
557 gIOConsoleUsersKey = OSSymbol::withCStringNoCopy( kIOConsoleUsersKey);
558 gIOConsoleSessionUIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionUIDKey);
559 gIOConsoleSessionAuditIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionAuditIDKey);
560
561 gIOConsoleUsersSeedKey = OSSymbol::withCStringNoCopy(kIOConsoleUsersSeedKey);
562 gIOConsoleSessionOnConsoleKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionOnConsoleKey);
563 gIOConsoleSessionLoginDoneKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionLoginDoneKey);
564 gIOConsoleSessionSecureInputPIDKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionSecureInputPIDKey);
565 gIOConsoleSessionScreenLockedTimeKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenLockedTimeKey);
566 gIOConsoleSessionScreenIsLockedKey = OSSymbol::withCStringNoCopy(kIOConsoleSessionScreenIsLockedKey);
567
568 gIOConsoleUsersSeedValue = OSData::withValueNoCopy(value&: gIOConsoleUsersSeed);
569
570 gIOUserClientEntitlementsKey = OSSymbol::withCStringNoCopy( kIOUserClientEntitlementsKey );
571 gIOServiceDEXTEntitlementsKey = OSSymbol::withCStringNoCopy( kIOServiceDEXTEntitlementsKey );
572 gIODriverKitEntitlementKey = OSSymbol::withCStringNoCopy( kIODriverKitEntitlementKey );
573 gIODriverKitUserClientEntitlementsKey = OSSymbol::withCStringNoCopy( kIODriverKitUserClientEntitlementsKey );
574#if XNU_TARGET_OS_OSX
575 gIODriverKitUserClientEntitlementAllowAnyKey = OSSymbol::withCStringNoCopy( kIODriverKitUserClientEntitlementAllowAnyKey );
576#else
577 gIODriverKitUserClientEntitlementAllowAnyKey = NULL;
578#endif
579 gIODriverKitRequiredEntitlementsKey = OSSymbol::withCStringNoCopy( kIODriverKitRequiredEntitlementsKey );
580 gIODriverKitTestDriverEntitlementKey = OSSymbol::withCStringNoCopy( kIODriverKitTestDriverEntitlementKey );
581 gIODriverKitUserClientEntitlementCommunicatesWithDriversKey = OSSymbol::withCStringNoCopy(kIODriverKitUserClientEntitlementCommunicatesWithDriversKey);
582 gIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey = OSSymbol::withCStringNoCopy(kIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey);
583
584 gIOMatchDeferKey = OSSymbol::withCStringNoCopy( kIOMatchDeferKey );
585 gIOServiceMatchDeferredKey = OSSymbol::withCStringNoCopy( kIOServiceMatchDeferredKey );
586 gIOServiceNotificationUserKey = OSSymbol::withCStringNoCopy( kIOServiceNotificationUserKey );
587 gIOExclaveAssignedKey = OSSymbol::withCStringNoCopy(kIOExclaveAssignedKey);
588 gIOExclaveProxyKey = OSSymbol::withCStringNoCopy(kIOExclaveProxyKey);
589
590 gIOPrimaryDriverTerminateOptionsKey = OSSymbol::withCStringNoCopy(kIOPrimaryDriverTerminateOptionsKey);
591 gIOMediaKey = OSSymbol::withCStringNoCopy(cString: "IOMedia");
592 gIOBlockStorageDriverKey = OSSymbol::withCStringNoCopy(cString: "IOBlockStorageDriver");
593 gPhysicalInterconnectKey = OSSymbol::withCStringNoCopy(cString: "Physical Interconnect");
594 gVirtualInterfaceKey = OSSymbol::withCStringNoCopy(cString: "Virtual Interface");
595
596 gIOAllCPUInitializedKey = OSSymbol::withCStringNoCopy( kIOAllCPUInitializedKey );
597
598 gIOPlatformFunctionHandlerSet = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerSet);
599 sCPULatencyFunctionName[kCpuDelayBusStall] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxBusDelay);
600#if defined(__x86_64__)
601 sCPULatencyFunctionName[kCpuDelayInterrupt] = OSSymbol::withCStringNoCopy(kIOPlatformFunctionHandlerMaxInterruptDelay);
602#endif /* defined(__x86_64__) */
603 uint32_t idx;
604 for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
605 sCPULatencySet[idx] = OSNumber::withNumber(UINT_MAX, numberOfBits: 32);
606 sCPULatencyHolder[idx] = OSNumber::withNumber(value: 0ULL, numberOfBits: 64);
607 assert(sCPULatencySet[idx] && sCPULatencyHolder[idx]);
608 }
609
610#if defined(__x86_64__)
611 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
612#endif /* defined(__x86_64__) */
613
614 gNotificationLock = IORecursiveLockAlloc();
615
616 gAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
617
618#if CONFIG_EXCLAVES
619 gExclaveProxyStates = OSDictionary::withCapacity( 1 );
620
621 gExclaveProxyStateLock = IORecursiveLockAlloc();
622 assert( gExclaveProxyStates && gExclaveProxyStateLock);
623#endif /* CONFIG_EXCLAVES */
624
625 assert( gIOServicePlane && gIODeviceMemoryKey
626 && gIOInterruptControllersKey && gIOInterruptSpecifiersKey
627 && gIOResourcesKey && gNotifications && gNotificationLock
628 && gIOProviderClassKey && gIONameMatchKey && gIONameMatchedKey
629 && gIOMatchCategoryKey && gIODefaultMatchCategoryKey
630 && gIOPublishNotification && gIOMatchedNotification
631 && gIOTerminatedNotification && gIOServiceKey
632 && gIOConsoleUsersKey && gIOConsoleSessionUIDKey
633 && gIOConsoleSessionOnConsoleKey && gIOConsoleSessionSecureInputPIDKey
634 && gIOConsoleUsersSeedKey && gIOConsoleUsersSeedValue);
635
636 gJobsLock = IOLockAlloc();
637 gJobs = OSOrderedSet::withCapacity( capacity: 10 );
638
639 gIOServiceBusyLock = IOLockAlloc();
640
641 gIOConsoleUsersLock = IOLockAlloc();
642
643 err = semaphore_create(task: kernel_task, semaphore: &gJobsSemaphore, SYNC_POLICY_FIFO, value: 0);
644
645 gIOConsoleLockCallout = thread_call_allocate(func: &IOService::consoleLockTimer, NULL);
646
647 IORegistryEntry::getRegistryRoot()->setProperty(aKey: gIOConsoleLockedKey, anObject: kOSBooleanTrue);
648
649 assert( gIOServiceBusyLock && gJobs && gJobsLock && gIOConsoleUsersLock
650 && gIOConsoleLockCallout && (err == KERN_SUCCESS));
651
652 gIOResources = IOResources::resources();
653 gIOUserResources = IOUserResources::resources();
654 assert( gIOResources && gIOUserResources );
655
656 gIOServiceNullNotifier = OSTypeAlloc(_IOServiceNullNotifier);
657 assert(gIOServiceNullNotifier);
658
659 gArbitrationLockQueueLock = IOLockAlloc();
660 queue_init(&gArbitrationLockQueueActive);
661 queue_init(&gArbitrationLockQueueWaiting);
662 queue_init(&gArbitrationLockQueueFree);
663
664 assert( gArbitrationLockQueueLock );
665
666 allocPMInitLock();
667
668 gIOTerminatePhase2List = OSArray::withCapacity( capacity: 2 );
669 gIOStopList = OSArray::withCapacity( capacity: 16 );
670 gIOStopProviderList = OSArray::withCapacity( capacity: 16 );
671 gIOFinalizeList = OSArray::withCapacity( capacity: 16 );
672#if !NO_KEXTD
673 if (OSKext::iokitDaemonAvailable()) {
674 gIOMatchDeferList = OSArray::withCapacity( capacity: 16 );
675 } else {
676 gIOMatchDeferList = NULL;
677 }
678#endif
679 assert( gIOTerminatePhase2List && gIOStopList && gIOStopProviderList && gIOFinalizeList );
680
681 gDriverKitLaunches = OSSet::withCapacity(capacity: 0);
682 gDriverKitLaunchLock = IORecursiveLockAlloc();
683 gIOAssociatedServicesKey = OSSymbol::withCStringNoCopy( cString: "IOAssociatedServices" );
684#if CONFIG_EXCLAVES
685 gDARTMapperFunctionSetActive = OSSymbol::withCStringNoCopy("setActive");
686#endif /* CONFIG_EXCLAVES */
687
688 // worker thread that is responsible for terminating / cleaning up threads
689 kernel_thread_start(continuation: &terminateThread, NULL, new_thread: &gIOTerminateWorkerThread);
690 assert(gIOTerminateWorkerThread);
691 thread_set_thread_name(th: gIOTerminateWorkerThread, name: "IOServiceTerminateThread");
692}
693
694/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
695
696#if defined(__x86_64__)
697extern "C" {
698const char *getCpuDelayBusStallHolderName(void);
699const char *
700getCpuDelayBusStallHolderName(void)
701{
702 return sCPULatencyHolderName[kCpuDelayBusStall];
703}
704
705const char *getCpuInterruptDelayHolderName(void);
706const char *
707getCpuInterruptDelayHolderName(void)
708{
709 return sCPULatencyHolderName[kCpuDelayInterrupt];
710}
711}
712#endif /* defined(__x86_64__) */
713
714
715
716#if IOMATCHDEBUG
717static UInt64
718getDebugFlags( OSDictionary * props )
719{
720 OSNumber * debugProp;
721 UInt64 debugFlags;
722
723 debugProp = OSDynamicCast( OSNumber,
724 props->getObject( gIOKitDebugKey ));
725 if (debugProp) {
726 debugFlags = debugProp->unsigned64BitValue();
727 } else {
728 debugFlags = gIOKitDebug;
729 }
730
731 return debugFlags;
732}
733
734static UInt64
735getDebugFlags( IOService * inst )
736{
737 OSObject * prop;
738 OSNumber * debugProp;
739 UInt64 debugFlags;
740
741 prop = inst->copyProperty(aKey: gIOKitDebugKey);
742 debugProp = OSDynamicCast(OSNumber, prop);
743 if (debugProp) {
744 debugFlags = debugProp->unsigned64BitValue();
745 } else {
746 debugFlags = gIOKitDebug;
747 }
748
749 OSSafeReleaseNULL(prop);
750
751 return debugFlags;
752}
753#endif
754
755/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
756
757// Probe a matched service and return an instance to be started.
758// The default score is from the property table, & may be altered
759// during probe to change the start order.
760
761IOService *
762IOService::probe( IOService * provider,
763 SInt32 * score )
764{
765 return this;
766}
767
768bool
769IOService::start( IOService * provider )
770{
771 return true;
772}
773
774void
775IOService::stop( IOService * provider )
776{
777 if (reserved->uvars && reserved->uvars->started && reserved->uvars->userServer) {
778 reserved->uvars->userServer->serviceStop(service: this, provider);
779 }
780}
781
782bool
783IOService::init( OSDictionary * dictionary )
784{
785 bool ret;
786
787 ret = super::init(dictionary);
788 if (!ret) {
789 return false;
790 }
791 if (reserved) {
792 return true;
793 }
794
795 reserved = IOMallocType(ExpansionData);
796 IOLockInlineInit(&reserved->interruptStatisticsLock);
797 return true;
798}
799
800bool
801IOService::init( IORegistryEntry * from,
802 const IORegistryPlane * inPlane )
803{
804 bool ret;
805
806 ret = super::init(from, inPlane);
807 if (!ret) {
808 return false;
809 }
810 if (reserved) {
811 return true;
812 }
813
814 reserved = IOMallocType(ExpansionData);
815 IOLockInlineInit(&reserved->interruptStatisticsLock);
816
817 return true;
818}
819
820void
821IOService::free( void )
822{
823 IOInterruptSourcePrivate *sourcesPrivate = NULL;
824 int i = 0;
825 requireMaxBusStall(ns: 0);
826#if defined(__x86_64__)
827 requireMaxInterruptDelay(0);
828#endif /* defined(__x86_64__) */
829 if (getPropertyTable()) {
830 unregisterAllInterest();
831 }
832 PMfree();
833
834 if (reserved) {
835 if (reserved->interruptStatisticsArray) {
836 for (i = 0; i < reserved->interruptStatisticsArrayCount; i++) {
837 if (reserved->interruptStatisticsArray[i].reporter) {
838 reserved->interruptStatisticsArray[i].reporter->release();
839 }
840 }
841
842 IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
843 }
844
845 if (reserved->uvars && reserved->uvars->userServer) {
846 reserved->uvars->userServer->serviceFree(service: this);
847 }
848 sourcesPrivate = reserved->interruptSourcesPrivate;
849 IOLockInlineDestroy(&reserved->interruptStatisticsLock);
850 IOFreeType(reserved, ExpansionData);
851 }
852
853 if (_numInterruptSources && _interruptSources) {
854 assert(sourcesPrivate);
855 for (i = 0; i < _numInterruptSources; i++) {
856 void * block = sourcesPrivate[i].vectorBlock;
857 if (block) {
858 Block_release(block);
859 }
860 }
861 IODelete(_interruptSources, IOInterruptSource, _numInterruptSources);
862 _interruptSources = NULL;
863 IODelete(sourcesPrivate, IOInterruptSourcePrivate, _numInterruptSources);
864 }
865
866 super::free();
867}
868
869/*
870 * Attach in service plane
871 */
872bool
873IOService::attach( IOService * provider )
874{
875 bool ok;
876 uint32_t count;
877 AbsoluteTime deadline;
878 int waitResult = THREAD_AWAKENED;
879 bool wait, computeDeadline = true;
880
881 if (provider) {
882 if (gIOKitDebug & kIOLogAttach) {
883 LOG( fmt: "%s::attach(%s)\n", getName(),
884 provider->getName());
885 }
886
887 ok = false;
888 do{
889 wait = false;
890 provider->lockForArbitration();
891 if (provider->__state[0] & kIOServiceInactiveState) {
892 ok = false;
893 } else {
894 count = provider->getChildCount(plane: gIOServicePlane);
895 wait = (count > (kIOServiceBusyMax - 4));
896 if (!wait) {
897 ok = attachToParent(parent: provider, plane: gIOServicePlane);
898 } else {
899 IOLog(format: "stalling for detach from %s\n", provider->getName());
900 IOLockLock( gIOServiceBusyLock );
901 provider->__state[1] |= kIOServiceWaitDetachState;
902 }
903 }
904 provider->unlockForArbitration();
905 if (wait) {
906 if (computeDeadline) {
907 clock_interval_to_deadline(interval: 15, scale_factor: kSecondScale, result: &deadline);
908 computeDeadline = false;
909 }
910 assert_wait_deadline(event: (event_t)&provider->__provider, THREAD_UNINT, deadline);
911 IOLockUnlock( gIOServiceBusyLock );
912 waitResult = thread_block(THREAD_CONTINUE_NULL);
913 wait = (waitResult != THREAD_TIMED_OUT);
914 }
915 }while (wait);
916 } else {
917 gIOServiceRoot = this;
918 ok = attachToParent( parent: getRegistryRoot(), plane: gIOServicePlane);
919 }
920
921 if (ok && !__provider) {
922 (void) getProvider();
923 }
924
925 return ok;
926}
927
928IOService *
929IOService::getServiceRoot( void )
930{
931 return gIOServiceRoot;
932}
933
934void
935IOService::detach( IOService * provider )
936{
937 IOService * newProvider = NULL;
938 SInt32 busy;
939 bool adjParent;
940
941 if (gIOKitDebug & kIOLogAttach) {
942 LOG(fmt: "%s::detach(%s)\n", getName(), provider->getName());
943 }
944
945#if !NO_KEXTD
946 IOLockLock(gJobsLock);
947 if (gIOMatchDeferList) {
948 auto idx = gIOMatchDeferList->getNextIndexOfObject(anObject: this, index: 0);
949 if (-1U != idx) {
950 gIOMatchDeferList->removeObject(index: idx);
951 }
952 }
953 if (IOServicePH::fMatchingDelayed) {
954 auto idx = IOServicePH::fMatchingDelayed->getNextIndexOfObject(anObject: this, index: 0);
955 if (-1U != idx) {
956 IOServicePH::fMatchingDelayed->removeObject(index: idx);
957 }
958 }
959 IOLockUnlock(gJobsLock);
960#endif /* NO_KEXTD */
961
962 lockForArbitration();
963
964 uint64_t regID1 = provider->getRegistryEntryID();
965 uint64_t regID2 = getRegistryEntryID();
966 IOServiceTrace(
967 IOSERVICE_DETACH,
968 (uintptr_t) regID1,
969 (uintptr_t) (regID1 >> 32),
970 (uintptr_t) regID2,
971 (uintptr_t) (regID2 >> 32));
972
973 adjParent = ((busy = (__state[1] & kIOServiceBusyStateMask))
974 && (provider == getProvider()));
975
976 detachFromParent( parent: provider, plane: gIOServicePlane );
977
978 if (busy) {
979 newProvider = getProvider();
980 if (busy && (__state[1] & kIOServiceTermPhase3State) && (NULL == newProvider)) {
981 _adjustBusy( delta: -busy );
982 }
983 }
984
985 if (kIOServiceInactiveState & __state[0]) {
986 getMetaClass()->removeInstance(instance: this);
987 IORemoveServicePlatformActions(service: this);
988 }
989
990 unlockForArbitration();
991
992 if (newProvider && adjParent) {
993 newProvider->lockForArbitration();
994 newProvider->_adjustBusy(delta: 1);
995 newProvider->unlockForArbitration();
996 }
997
998 // check for last client detach from a terminated service
999 if (provider->lockForArbitration( isSuccessRequired: true )) {
1000 if (kIOServiceStartState & __state[1]) {
1001 provider->scheduleTerminatePhase2();
1002 }
1003 if (adjParent) {
1004 provider->_adjustBusy( delta: -1 );
1005 }
1006 if ((provider->__state[1] & kIOServiceTermPhase3State)
1007 && (NULL == provider->getClient())) {
1008 provider->scheduleFinalize(now: false);
1009 }
1010
1011 IOLockLock( gIOServiceBusyLock );
1012 if (kIOServiceWaitDetachState & provider->__state[1]) {
1013 provider->__state[1] &= ~kIOServiceWaitDetachState;
1014 thread_wakeup(&provider->__provider);
1015 }
1016 IOLockUnlock( gIOServiceBusyLock );
1017
1018 provider->unlockForArbitration();
1019 }
1020
1021 if (kIOServiceRematchOnDetach & __state[1]) {
1022 provider->registerService();
1023 }
1024}
1025
1026/*
1027 * Register instance - publish it for matching
1028 */
1029
1030void
1031IOService::registerService( IOOptionBits options )
1032{
1033 OSDataAllocation<char> pathBuf;
1034 const char * path;
1035 const char * skip;
1036 int len;
1037 enum { kMaxPathLen = 256 };
1038 enum { kMaxChars = 63 };
1039
1040 IORegistryEntry * parent = this;
1041 IORegistryEntry * root = getRegistryRoot();
1042 while (parent && (parent != root)) {
1043 parent = parent->getParentEntry( plane: gIOServicePlane);
1044 }
1045
1046 if (parent != root) {
1047 IOLog(format: "%s: not registry member at registerService()\n", getName());
1048 return;
1049 }
1050
1051 // Allow the Platform Expert to adjust this node.
1052 if (gIOPlatform && (!gIOPlatform->platformAdjustService(service: this))) {
1053 return;
1054 }
1055
1056 IOInstallServicePlatformActions(service: this);
1057 IOInstallServiceSleepPlatformActions(service: this);
1058
1059 if ((this != gIOResources)
1060 && (kIOLogRegister & gIOKitDebug)) {
1061 pathBuf = OSDataAllocation<char>( kMaxPathLen, OSAllocateMemory );
1062
1063 IOLog( format: "Registering: " );
1064
1065 len = kMaxPathLen;
1066 if (pathBuf && getPath( path: pathBuf.data(), length: &len, plane: gIOServicePlane)) {
1067 path = pathBuf.data();
1068 if (len > kMaxChars) {
1069 IOLog(format: "..");
1070 len -= kMaxChars;
1071 path += len;
1072 if ((skip = strchr( s: path, c: '/'))) {
1073 path = skip;
1074 }
1075 }
1076 } else {
1077 path = getName();
1078 }
1079
1080 IOLog( format: "%s\n", path );
1081 }
1082
1083 startMatching( options );
1084}
1085
1086void
1087IOService::startMatching( IOOptionBits options )
1088{
1089 IOService * provider;
1090 UInt32 prevBusy = 0;
1091 bool needConfig;
1092 bool needWake = false;
1093 bool sync;
1094 bool waitAgain;
1095 bool releaseAssertion = false;
1096
1097 if (options & kIOServiceDextRequirePowerForMatching) {
1098 bool ok = gIOPMRootDomain->acquireDriverKitMatchingAssertion() == kIOReturnSuccess;
1099 if (!ok) {
1100 panic("%s: Failed to acquire power assertion for matching", getName());
1101 }
1102 releaseAssertion = true;
1103 }
1104
1105 lockForArbitration();
1106
1107 sync = (options & kIOServiceSynchronous)
1108 || ((provider = getProvider())
1109 && (provider->__state[1] & kIOServiceSynchronousState));
1110
1111 if (options & kIOServiceAsynchronous) {
1112 sync = false;
1113 }
1114
1115 needConfig = (0 == (__state[1] & (kIOServiceNeedConfigState | kIOServiceConfigRunning)))
1116 && (0 == (__state[0] & kIOServiceInactiveState));
1117
1118 __state[1] |= kIOServiceNeedConfigState;
1119
1120// __state[0] &= ~kIOServiceInactiveState;
1121
1122// if( sync) LOG("OSKernelStackRemaining = %08x @ %s\n",
1123// OSKernelStackRemaining(), getName());
1124
1125 if (needConfig) {
1126 needWake = (0 != (kIOServiceSyncPubState & __state[1]));
1127 }
1128
1129 if (sync) {
1130 __state[1] |= kIOServiceSynchronousState;
1131 } else {
1132 __state[1] &= ~kIOServiceSynchronousState;
1133 }
1134
1135 if (needConfig) {
1136 prevBusy = _adjustBusy( delta: 1 );
1137 }
1138
1139 unlockForArbitration();
1140
1141 if (needConfig) {
1142 if (needWake) {
1143 IOLockLock( gIOServiceBusyLock );
1144 thread_wakeup((event_t) this /*&__state[1]*/ );
1145 IOLockUnlock( gIOServiceBusyLock );
1146 } else if (!sync || (kIOServiceAsynchronous & options)) {
1147 // assertion will be released when matching job is complete
1148 releaseAssertion = false;
1149 _IOServiceJob::startJob( nub: this, type: kMatchNubJob, options );
1150 } else {
1151 do {
1152 if ((__state[1] & kIOServiceNeedConfigState)) {
1153 doServiceMatch( options );
1154 }
1155
1156 lockForArbitration();
1157 IOLockLock( gIOServiceBusyLock );
1158
1159 waitAgain = ((prevBusy < (__state[1] & kIOServiceBusyStateMask))
1160 && (0 == (__state[0] & kIOServiceInactiveState)));
1161
1162 if (waitAgain) {
1163 __state[1] |= kIOServiceSyncPubState | kIOServiceBusyWaiterState;
1164 } else {
1165 __state[1] &= ~kIOServiceSyncPubState;
1166 }
1167
1168 unlockForArbitration();
1169
1170 if (waitAgain) {
1171 assert_wait(event: (event_t) this /*&__state[1]*/, THREAD_UNINT);
1172 }
1173
1174 IOLockUnlock( gIOServiceBusyLock );
1175 if (waitAgain) {
1176 thread_block(THREAD_CONTINUE_NULL);
1177 }
1178 } while (waitAgain);
1179 }
1180 }
1181
1182 if (releaseAssertion) {
1183 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
1184 }
1185}
1186
1187
1188void
1189IOService::startDeferredMatches(void)
1190{
1191#if !NO_KEXTD
1192 OSArray * array;
1193
1194 IOLockLock(gJobsLock);
1195 array = gIOMatchDeferList;
1196 gIOMatchDeferList = NULL;
1197 IOLockUnlock(gJobsLock);
1198
1199 if (array) {
1200 IOLog(format: "deferred rematching count %d\n", array->getCount());
1201 array->iterateObjects(block: ^bool (OSObject * obj)
1202 {
1203 ((IOService *)obj)->startMatching(options: kIOServiceAsynchronous);
1204 return false;
1205 });
1206 array->release();
1207 }
1208#endif /* !NO_KEXTD */
1209}
1210
1211void
1212IOService::iokitDaemonLaunched(void)
1213{
1214#if !NO_KEXTD
1215 if (!OSKext::iokitDaemonAvailable()) {
1216 panic(kIOKitDaemonName " is unavailable in this environment, but it was launched");
1217 }
1218 IOServiceTrace(IOSERVICE_KEXTD_READY, 0, 0, 0, 0);
1219 startDeferredMatches();
1220 getServiceRoot()->adjustBusy(delta: -1);
1221 IOService::publishUserResource(key: gIOResourceIOKitKey);
1222#endif /* !NO_KEXTD */
1223}
1224
1225/*
1226 * Possibly called with IORWLock from IOCatalog held.
1227 * This means that no calls to OSKext that could take
1228 * sKextLock can be performed from this function.
1229 */
1230IOReturn
1231IOService::catalogNewDrivers( OSOrderedSet * newTables )
1232{
1233 OSDictionary * table;
1234 OSSet * set;
1235 OSSet * allSet = NULL;
1236 IOService * service;
1237#if IOMATCHDEBUG
1238 SInt32 count = 0;
1239#endif
1240
1241 newTables->retain();
1242
1243 while ((table = (OSDictionary *) newTables->getFirstObject())) {
1244 LOCKWRITENOTIFY();
1245 set = (OSSet *) copyExistingServices( matching: table,
1246 inState: kIOServiceRegisteredState,
1247 options: kIOServiceExistingSet);
1248 UNLOCKNOTIFY();
1249 if (set) {
1250#if IOMATCHDEBUG
1251 count += set->getCount();
1252#endif
1253 if (allSet) {
1254 allSet->merge(set: (const OSSet *) set);
1255 set->release();
1256 } else {
1257 allSet = set;
1258 }
1259 }
1260
1261#if IOMATCHDEBUG
1262 if (getDebugFlags( props: table ) & kIOLogMatch) {
1263 LOG(fmt: "Matching service count = %ld\n", (long)count);
1264 }
1265#endif
1266 newTables->removeObject(anObject: table);
1267 }
1268
1269 if (allSet) {
1270 while ((service = (IOService *) allSet->getAnyObject())) {
1271 service->startMatching(options: kIOServiceAsynchronous);
1272 allSet->removeObject(anObject: service);
1273 }
1274 allSet->release();
1275 }
1276
1277 newTables->release();
1278
1279 return kIOReturnSuccess;
1280}
1281
1282_IOServiceJob *
1283_IOServiceJob::startJob( IOService * nub, int type,
1284 IOOptionBits options )
1285{
1286 _IOServiceJob * job;
1287
1288 job = new _IOServiceJob;
1289 if (job && !job->init()) {
1290 job->release();
1291 job = NULL;
1292 }
1293
1294 if (job) {
1295 job->type = type;
1296 job->nub = nub;
1297 job->options = options;
1298 nub->retain(); // thread will release()
1299 pingConfig( job );
1300 }
1301
1302 return job;
1303}
1304
1305/*
1306 * Called on a registered service to see if it matches
1307 * a property table.
1308 */
1309
1310bool
1311IOService::matchPropertyTable( OSDictionary * table, SInt32 * score )
1312{
1313 return matchPropertyTable(table);
1314}
1315
1316bool
1317IOService::matchPropertyTable( OSDictionary * table )
1318{
1319 return true;
1320}
1321
1322/*
1323 * Called on a matched service to allocate resources
1324 * before first driver is attached.
1325 */
1326
1327IOReturn
1328IOService::getResources( void )
1329{
1330 return kIOReturnSuccess;
1331}
1332
1333/*
1334 * Client/provider accessors
1335 */
1336
1337IOService *
1338IOService::getProvider( void ) const
1339{
1340 IOService * self = (IOService *) this;
1341 IOService * parent;
1342 SInt32 generation;
1343
1344 generation = getRegistryEntryGenerationCount();
1345 if (__providerGeneration == generation) {
1346 return __provider;
1347 }
1348
1349 parent = (IOService *) getParentEntry( plane: gIOServicePlane);
1350 if (parent == IORegistryEntry::getRegistryRoot()) {
1351 /* root is not an IOService */
1352 parent = NULL;
1353 }
1354
1355 self->__provider = parent;
1356 OSMemoryBarrier();
1357 // save the count from before call to getParentEntry()
1358 self->__providerGeneration = generation;
1359
1360 return parent;
1361}
1362
1363IOWorkLoop *
1364IOService::getWorkLoop() const
1365{
1366 IOService *provider = getProvider();
1367
1368 if (provider) {
1369 return provider->getWorkLoop();
1370 } else {
1371 return NULL;
1372 }
1373}
1374
1375OSIterator *
1376IOService::getProviderIterator( void ) const
1377{
1378 return getParentIterator( plane: gIOServicePlane);
1379}
1380
1381IOService *
1382IOService::getClient( void ) const
1383{
1384 return (IOService *) getChildEntry( plane: gIOServicePlane);
1385}
1386
1387OSIterator *
1388IOService::getClientIterator( void ) const
1389{
1390 return getChildIterator( plane: gIOServicePlane);
1391}
1392
1393OSIterator *
1394_IOOpenServiceIterator::iterator( OSIterator * _iter,
1395 const IOService * client,
1396 const IOService * provider )
1397{
1398 _IOOpenServiceIterator * inst;
1399
1400 if (!_iter) {
1401 return NULL;
1402 }
1403
1404 inst = new _IOOpenServiceIterator;
1405
1406 if (inst && !inst->init()) {
1407 inst->release();
1408 inst = NULL;
1409 }
1410 if (inst) {
1411 inst->iter = _iter;
1412 inst->client = client;
1413 inst->provider = provider;
1414 } else {
1415 OSSafeReleaseNULL(_iter);
1416 }
1417
1418 return inst;
1419}
1420
1421void
1422_IOOpenServiceIterator::free()
1423{
1424 iter->release();
1425 if (last) {
1426 last->unlockForArbitration();
1427 }
1428 OSIterator::free();
1429}
1430
1431OSObject *
1432_IOOpenServiceIterator::getNextObject()
1433{
1434 IOService * next;
1435
1436 if (last) {
1437 last->unlockForArbitration();
1438 }
1439
1440 while ((next = (IOService *) iter->getNextObject())) {
1441 next->lockForArbitration();
1442 if ((client && (next->isOpen( forClient: client )))
1443 || (provider && (provider->isOpen( forClient: next )))) {
1444 break;
1445 }
1446 next->unlockForArbitration();
1447 }
1448
1449 last = next;
1450
1451 return next;
1452}
1453
1454bool
1455_IOOpenServiceIterator::isValid()
1456{
1457 return iter->isValid();
1458}
1459
1460void
1461_IOOpenServiceIterator::reset()
1462{
1463 if (last) {
1464 last->unlockForArbitration();
1465 last = NULL;
1466 }
1467 iter->reset();
1468}
1469
1470OSIterator *
1471IOService::getOpenProviderIterator( void ) const
1472{
1473 return _IOOpenServiceIterator::iterator( iter: getProviderIterator(), client: this, NULL );
1474}
1475
1476OSIterator *
1477IOService::getOpenClientIterator( void ) const
1478{
1479 return _IOOpenServiceIterator::iterator( iter: getClientIterator(), NULL, provider: this );
1480}
1481
1482
1483IOReturn
1484IOService::callPlatformFunction( const OSSymbol * functionName,
1485 bool waitForFunction,
1486 void *param1, void *param2,
1487 void *param3, void *param4 )
1488{
1489 IOReturn result = kIOReturnUnsupported;
1490 IOService *provider;
1491
1492 if (functionName == gIOPlatformQuiesceActionKey ||
1493 functionName == gIOPlatformActiveActionKey ||
1494 functionName == gIOPlatformPanicActionKey) {
1495 /*
1496 * Services which register for IOPlatformQuiesceAction / IOPlatformActiveAction / IOPlatformPanicAction
1497 * must consume that event themselves, without passing it up to super/IOService.
1498 */
1499 if (gEnforcePlatformActionSafety) {
1500 panic("Class %s passed the %s action to IOService",
1501 getMetaClass()->getClassName(), functionName->getCStringNoCopy());
1502 }
1503 }
1504
1505 if (gIOPlatformFunctionHandlerSet == functionName) {
1506 const OSSymbol * functionHandlerName = (const OSSymbol *) param1;
1507 IOService * target = (IOService *) param2;
1508 bool enable = (param3 != NULL);
1509
1510 if (sCPULatencyFunctionName[kCpuDelayBusStall] == functionHandlerName) {
1511 result = setLatencyHandler(delayType: kCpuDelayBusStall, target, enable);
1512 }
1513#if defined(__x86_64__)
1514 else if (sCPULatencyFunctionName[kCpuDelayInterrupt] == param1) {
1515 result = setLatencyHandler(kCpuDelayInterrupt, target, enable);
1516 }
1517#endif /* defined(__x86_64__) */
1518 }
1519
1520 if ((kIOReturnUnsupported == result) && (provider = getProvider())) {
1521 result = provider->callPlatformFunction(functionName, waitForFunction,
1522 param1, param2, param3, param4);
1523 }
1524
1525 return result;
1526}
1527
1528IOReturn
1529IOService::callPlatformFunction( const char * functionName,
1530 bool waitForFunction,
1531 void *param1, void *param2,
1532 void *param3, void *param4 )
1533{
1534 IOReturn result = kIOReturnNoMemory;
1535 const OSSymbol *functionSymbol = OSSymbol::withCString(cString: functionName);
1536
1537 if (functionSymbol != NULL) {
1538 result = callPlatformFunction(functionName: functionSymbol, waitForFunction,
1539 param1, param2, param3, param4);
1540 functionSymbol->release();
1541 }
1542
1543 return result;
1544}
1545
1546
1547/*
1548 * Accessors for global services
1549 */
1550
1551IOPlatformExpert *
1552IOService::getPlatform( void )
1553{
1554 return gIOPlatform;
1555}
1556
1557class IOPMrootDomain *
1558 IOService::getPMRootDomain( void )
1559{
1560 return gIOPMRootDomain;
1561}
1562
1563IOService *
1564IOService::getResourceService( void )
1565{
1566 return gIOResources;
1567}
1568
1569IOService * gIOSystemStateNotificationService;
1570
1571IOService *
1572IOService::getSystemStateNotificationService(void)
1573{
1574 return gIOSystemStateNotificationService;
1575}
1576
1577void
1578IOService::setPlatform( IOPlatformExpert * platform)
1579{
1580 gIOPlatform = platform;
1581 gIOResources->attachToParent( parent: gIOServiceRoot, plane: gIOServicePlane );
1582
1583 gIOUserResources->attachToParent( parent: gIOServiceRoot, plane: gIOServicePlane );
1584
1585#if DEVELOPMENT || DEBUG
1586 // Test object that will be terminated for dext to match
1587 {
1588 IOService * ios;
1589 ios = OSTypeAlloc(IOService);
1590 ios->init();
1591 ios->attach(gIOUserResources);
1592 ios->setProperty(gIOMatchCategoryKey->getCStringNoCopy(), "com.apple.iokit.test");
1593 ios->setProperty(gIOModuleIdentifierKey->getCStringNoCopy(), "com.apple.kpi.iokit");
1594 ios->setProperty(gIOMatchedAtBootKey, kOSBooleanTrue);
1595 ios->setProperty(gIOPrimaryDriverTerminateOptionsKey, kOSBooleanTrue);
1596 ios->release();
1597 }
1598#endif
1599
1600 gIOSystemStateNotificationService = IOSystemStateNotification::initialize();
1601 gIOSystemStateNotificationService->attachToParent(parent: platform, plane: gIOServicePlane);
1602 gIOSystemStateNotificationService->registerService();
1603
1604 static const char * keys[kCpuNumDelayTypes] = {
1605 kIOPlatformMaxBusDelay,
1606#if defined(__x86_64__)
1607 kIOPlatformMaxInterruptDelay
1608#endif /* defined(__x86_64__) */
1609 };
1610 const OSObject * objs[2];
1611 OSArray * array;
1612 uint32_t idx;
1613
1614 for (idx = 0; idx < kCpuNumDelayTypes; idx++) {
1615 objs[0] = sCPULatencySet[idx];
1616 objs[1] = sCPULatencyHolder[idx];
1617 array = OSArray::withObjects(objects: objs, count: 2);
1618 if (!array) {
1619 break;
1620 }
1621 platform->setProperty(aKey: keys[idx], anObject: array);
1622 array->release();
1623 }
1624}
1625
1626void
1627IOService::setPMRootDomain( class IOPMrootDomain * rootDomain)
1628{
1629 gIOPMRootDomain = rootDomain;
1630}
1631
1632void
1633IOService::publishPMRootDomain( void )
1634{
1635 assert(getPMRootDomain() != NULL);
1636 publishResource(key: gIOResourceIOKitKey);
1637#if NO_KEXTD
1638 // Publish IOUserResources now since there is no IOKit daemon.
1639 publishUserResource(gIOResourceIOKitKey);
1640#endif
1641 IOServicePH::init(root: getPMRootDomain());
1642}
1643
1644/*
1645 * Stacking change
1646 */
1647
1648bool
1649IOService::lockForArbitration( bool isSuccessRequired )
1650{
1651 bool found;
1652 bool success;
1653 ArbitrationLockQueueElement * element;
1654 ArbitrationLockQueueElement * owner;
1655 ArbitrationLockQueueElement * active;
1656 ArbitrationLockQueueElement * waiting;
1657
1658 enum { kPutOnFreeQueue, kPutOnActiveQueue, kPutOnWaitingQueue } action;
1659
1660 // lock global access
1661 IOTakeLock( lock: gArbitrationLockQueueLock );
1662
1663 // obtain an unused queue element
1664 if (!queue_empty( &gArbitrationLockQueueFree )) {
1665 queue_remove_first( &gArbitrationLockQueueFree,
1666 element,
1667 ArbitrationLockQueueElement *,
1668 link );
1669 } else {
1670 element = IOMallocType(ArbitrationLockQueueElement);
1671 assert( element );
1672 }
1673
1674 // prepare the queue element
1675 element->thread = IOThreadSelf();
1676 element->service = this;
1677 element->count = 1;
1678 element->required = isSuccessRequired;
1679 element->aborted = false;
1680
1681 // determine whether this object is already locked (ie. on active queue)
1682 found = false;
1683 queue_iterate( &gArbitrationLockQueueActive,
1684 owner,
1685 ArbitrationLockQueueElement *,
1686 link )
1687 {
1688 if (owner->service == element->service) {
1689 found = true;
1690 break;
1691 }
1692 }
1693
1694 if (found) { // this object is already locked
1695 active = owner;
1696 // determine whether it is the same or a different thread trying to lock
1697 if (active->thread != element->thread) { // it is a different thread
1698 ArbitrationLockQueueElement * victim = NULL;
1699
1700 // before placing this new thread on the waiting queue, we look for
1701 // a deadlock cycle...
1702
1703 while (1) {
1704 // determine whether the active thread holding the object we
1705 // want is waiting for another object to be unlocked
1706 found = false;
1707 queue_iterate( &gArbitrationLockQueueWaiting,
1708 waiting,
1709 ArbitrationLockQueueElement *,
1710 link )
1711 {
1712 if (waiting->thread == active->thread) {
1713 assert( false == waiting->aborted );
1714 found = true;
1715 break;
1716 }
1717 }
1718
1719 if (found) { // yes, active thread waiting for another object
1720 // this may be a candidate for rejection if the required
1721 // flag is not set, should we detect a deadlock later on
1722 if (false == waiting->required) {
1723 victim = waiting;
1724 }
1725
1726 // find the thread that is holding this other object, that
1727 // is blocking the active thread from proceeding (fun :-)
1728 found = false;
1729 queue_iterate( &gArbitrationLockQueueActive,
1730 active, // (reuse active queue element)
1731 ArbitrationLockQueueElement *,
1732 link )
1733 {
1734 if (active->service == waiting->service) {
1735 found = true;
1736 break;
1737 }
1738 }
1739
1740 // someone must be holding it or it wouldn't be waiting
1741 assert( found );
1742
1743 if (active->thread == element->thread) {
1744 // doh, it's waiting for the thread that originated
1745 // this whole lock (ie. current thread) -> deadlock
1746 if (false == element->required) { // willing to fail?
1747 // the originating thread doesn't have the required
1748 // flag, so it can fail
1749 success = false; // (fail originating lock request)
1750 break; // (out of while)
1751 } else { // originating thread is not willing to fail
1752 // see if we came across a waiting thread that did
1753 // not have the 'required' flag set: we'll fail it
1754 if (victim) {
1755 // we do have a willing victim, fail it's lock
1756 victim->aborted = true;
1757
1758 // take the victim off the waiting queue
1759 queue_remove( &gArbitrationLockQueueWaiting,
1760 victim,
1761 ArbitrationLockQueueElement *,
1762 link );
1763
1764 // wake the victim
1765 wakeup_thread_with_inheritor(event: &victim->service->__machPortHoldDestroy, // event
1766 THREAD_AWAKENED, action: LCK_WAKE_DEFAULT, thread_towake: victim->thread);
1767
1768 // allow this thread to proceed (ie. wait)
1769 success = true; // (put request on wait queue)
1770 break; // (out of while)
1771 } else {
1772 // all the waiting threads we came across in
1773 // finding this loop had the 'required' flag
1774 // set, so we've got a deadlock we can't avoid
1775 panic("I/O Kit: Unrecoverable deadlock.");
1776 }
1777 }
1778 } else {
1779 // repeat while loop, redefining active thread to be the
1780 // thread holding "this other object" (see above), and
1781 // looking for threads waiting on it; note the active
1782 // variable points to "this other object" already... so
1783 // there nothing to do in this else clause.
1784 }
1785 } else { // no, active thread is not waiting for another object
1786 success = true; // (put request on wait queue)
1787 break; // (out of while)
1788 }
1789 } // while forever
1790
1791 if (success) { // put the request on the waiting queue?
1792 kern_return_t wait_result;
1793
1794 // place this thread on the waiting queue and put it to sleep;
1795 // we place it at the tail of the queue...
1796 queue_enter( &gArbitrationLockQueueWaiting,
1797 element,
1798 ArbitrationLockQueueElement *,
1799 link );
1800
1801 // declare that this thread will wait for a given event
1802restart_sleep:
1803 // unlock global access
1804 // & put thread to sleep, waiting for our event to fire...
1805 wait_result = lck_mtx_sleep_with_inheritor(lock: gArbitrationLockQueueLock,
1806 lck_sleep_action: LCK_SLEEP_UNLOCK,
1807 event: &element->service->__machPortHoldDestroy, // event
1808 inheritor: owner->thread,
1809 interruptible: element->required ? THREAD_UNINT : THREAD_INTERRUPTIBLE, TIMEOUT_WAIT_FOREVER);
1810
1811 // ...and we've been woken up; we might be in one of two states:
1812 // (a) we've been aborted and our queue element is not on
1813 // any of the three queues, but is floating around
1814 // (b) we're allowed to proceed with the lock and we have
1815 // already been moved from the waiting queue to the
1816 // active queue.
1817 // ...plus a 3rd state, should the thread have been interrupted:
1818 // (c) we're still on the waiting queue
1819
1820 // determine whether we were interrupted out of our sleep
1821 if (THREAD_INTERRUPTED == wait_result) {
1822 // re-lock global access
1823 IOTakeLock( lock: gArbitrationLockQueueLock );
1824
1825 // determine whether we're still on the waiting queue
1826 found = false;
1827 queue_iterate( &gArbitrationLockQueueWaiting,
1828 waiting, // (reuse waiting queue element)
1829 ArbitrationLockQueueElement *,
1830 link )
1831 {
1832 if (waiting == element) {
1833 found = true;
1834 break;
1835 }
1836 }
1837
1838 if (found) { // yes, we're still on the waiting queue
1839 // determine whether we're willing to fail
1840 if (false == element->required) {
1841 // mark us as aborted
1842 element->aborted = true;
1843
1844 // take us off the waiting queue
1845 queue_remove( &gArbitrationLockQueueWaiting,
1846 element,
1847 ArbitrationLockQueueElement *,
1848 link );
1849 } else { // we are not willing to fail
1850 // ignore interruption, go back to sleep
1851 goto restart_sleep;
1852 }
1853 }
1854
1855 // unlock global access
1856 IOUnlock( lock: gArbitrationLockQueueLock );
1857
1858 // proceed as though this were a normal wake up
1859 wait_result = THREAD_AWAKENED;
1860 }
1861
1862 assert( THREAD_AWAKENED == wait_result );
1863
1864 // determine whether we've been aborted while we were asleep
1865 if (element->aborted) {
1866 assert( false == element->required );
1867 // re-lock global access
1868 IOTakeLock( lock: gArbitrationLockQueueLock );
1869
1870 action = kPutOnFreeQueue;
1871 success = false;
1872 } else { // we weren't aborted, so we must be ready to go :-)
1873 // we've already been moved from waiting to active queue
1874 return true;
1875 }
1876 } else { // the lock request is to be failed
1877 // return unused queue element to queue
1878 action = kPutOnFreeQueue;
1879 }
1880 } else { // it is the same thread, recursive access is allowed
1881 // add one level of recursion
1882 active->count++;
1883
1884 // return unused queue element to queue
1885 action = kPutOnFreeQueue;
1886 success = true;
1887 }
1888 } else { // this object is not already locked, so let this thread through
1889 action = kPutOnActiveQueue;
1890 success = true;
1891 }
1892
1893 // put the new element on a queue
1894 if (kPutOnActiveQueue == action) {
1895 queue_enter( &gArbitrationLockQueueActive,
1896 element,
1897 ArbitrationLockQueueElement *,
1898 link );
1899 } else if (kPutOnFreeQueue == action) {
1900 queue_enter( &gArbitrationLockQueueFree,
1901 element,
1902 ArbitrationLockQueueElement *,
1903 link );
1904 } else {
1905 assert( 0 ); // kPutOnWaitingQueue never occurs, handled specially above
1906 }
1907
1908 // unlock global access
1909 IOUnlock( lock: gArbitrationLockQueueLock );
1910
1911 return success;
1912}
1913
1914void
1915IOService::unlockForArbitration( void )
1916{
1917 bool found;
1918 ArbitrationLockQueueElement * element;
1919
1920 // lock global access
1921 IOTakeLock( lock: gArbitrationLockQueueLock );
1922
1923 // find the lock element for this object (ie. on active queue)
1924 found = false;
1925 queue_iterate( &gArbitrationLockQueueActive,
1926 element,
1927 ArbitrationLockQueueElement *,
1928 link )
1929 {
1930 if (element->service == this) {
1931 found = true;
1932 break;
1933 }
1934 }
1935
1936 assert( found );
1937
1938 // determine whether the lock has been taken recursively
1939 if (element->count > 1) {
1940 // undo one level of recursion
1941 element->count--;
1942 } else {
1943 // remove it from the active queue
1944 queue_remove( &gArbitrationLockQueueActive,
1945 element,
1946 ArbitrationLockQueueElement *,
1947 link );
1948
1949 // put it on the free queue
1950 queue_enter( &gArbitrationLockQueueFree,
1951 element,
1952 ArbitrationLockQueueElement *,
1953 link );
1954
1955 // determine whether a thread is waiting for object
1956 thread_t woken;
1957 kern_return_t kr =
1958 wakeup_one_with_inheritor(event: &__machPortHoldDestroy, // event
1959 THREAD_AWAKENED, action: LCK_WAKE_DEFAULT, thread_wokenup: &woken);
1960
1961 if (KERN_SUCCESS == kr) {
1962 found = false;
1963 queue_iterate( &gArbitrationLockQueueWaiting,
1964 element,
1965 ArbitrationLockQueueElement *,
1966 link )
1967 {
1968 if (element->thread == woken) {
1969 found = true;
1970 break;
1971 }
1972 }
1973 assert(found); // we found an interested thread on waiting queue
1974 // remove it from the waiting queue
1975 queue_remove( &gArbitrationLockQueueWaiting,
1976 element,
1977 ArbitrationLockQueueElement *,
1978 link );
1979
1980 // put it on the active queue
1981 queue_enter( &gArbitrationLockQueueActive,
1982 element,
1983 ArbitrationLockQueueElement *,
1984 link );
1985
1986 thread_deallocate(thread: woken);
1987 }
1988 }
1989
1990 // unlock global access
1991 IOUnlock( lock: gArbitrationLockQueueLock );
1992}
1993
1994uint32_t
1995IOService::isLockedForArbitration(IOService * service)
1996{
1997#if DEBUG_NOTIFIER_LOCKED
1998 uint32_t count;
1999 ArbitrationLockQueueElement * active;
2000
2001 // lock global access
2002 IOLockLock(gArbitrationLockQueueLock);
2003
2004 // determine whether this object is already locked (ie. on active queue)
2005 count = 0;
2006 queue_iterate(&gArbitrationLockQueueActive,
2007 active,
2008 ArbitrationLockQueueElement *,
2009 link)
2010 {
2011 if ((active->thread == IOThreadSelf())
2012 && (!service || (active->service == service))) {
2013 count += 0x10000;
2014 count += active->count;
2015 }
2016 }
2017
2018 IOLockUnlock(gArbitrationLockQueueLock);
2019
2020 return count;
2021
2022#else /* DEBUG_NOTIFIER_LOCKED */
2023
2024 return 0;
2025
2026#endif /* DEBUG_NOTIFIER_LOCKED */
2027}
2028
2029void
2030IOService::setMachPortHoldDestroy(bool holdDestroy)
2031{
2032 __machPortHoldDestroy = holdDestroy;
2033}
2034
2035bool
2036IOService::machPortHoldDestroy()
2037{
2038 return __machPortHoldDestroy;
2039}
2040
2041void
2042IOService::applyToProviders( IOServiceApplierFunction applier,
2043 void * context )
2044{
2045 applyToParents(applier: (IORegistryEntryApplierFunction) applier,
2046 context, plane: gIOServicePlane );
2047}
2048
2049void
2050IOService::applyToClients( IOServiceApplierFunction applier,
2051 void * context )
2052{
2053 applyToChildren(applier: (IORegistryEntryApplierFunction) applier,
2054 context, plane: gIOServicePlane );
2055}
2056
2057
2058static void
2059IOServiceApplierToBlock(IOService * next, void * context)
2060{
2061 IOServiceApplierBlock block = (IOServiceApplierBlock) context;
2062 block(next);
2063}
2064
2065void
2066IOService::applyToProviders(IOServiceApplierBlock applier)
2067{
2068 applyToProviders(applier: &IOServiceApplierToBlock, context: applier);
2069}
2070
2071void
2072IOService::applyToClients(IOServiceApplierBlock applier)
2073{
2074 applyToClients(applier: &IOServiceApplierToBlock, context: applier);
2075}
2076
2077/*
2078 * Client messages
2079 */
2080
2081
2082// send a message to a client or interested party of this service
2083IOReturn
2084IOService::messageClient( UInt32 type, OSObject * client,
2085 void * argument, vm_size_t argSize )
2086{
2087 IOReturn ret;
2088 IOService * service;
2089 _IOServiceInterestNotifier * notify;
2090
2091 if ((service = OSDynamicCast( IOService, client))) {
2092 ret = service->message( type, provider: this, argument );
2093 } else if ((notify = OSDynamicCast( _IOServiceInterestNotifier, client))) {
2094 _IOServiceNotifierInvocation invocation;
2095 bool willNotify;
2096
2097 invocation.thread = current_thread();
2098
2099 LOCKWRITENOTIFY();
2100 willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
2101
2102 if (willNotify) {
2103 queue_enter( &notify->handlerInvocations, &invocation,
2104 _IOServiceNotifierInvocation *, link );
2105 }
2106 UNLOCKNOTIFY();
2107
2108 if (willNotify) {
2109 ret = (*notify->handler)( notify->target, notify->ref,
2110 type, this, argument, argSize );
2111
2112 LOCKWRITENOTIFY();
2113 queue_remove( &notify->handlerInvocations, &invocation,
2114 _IOServiceNotifierInvocation *, link );
2115 if (kIOServiceNotifyWaiter & notify->state) {
2116 notify->state &= ~kIOServiceNotifyWaiter;
2117 WAKEUPNOTIFY( notify );
2118 }
2119 UNLOCKNOTIFY();
2120 } else {
2121 ret = kIOReturnSuccess;
2122 }
2123 } else {
2124 ret = kIOReturnBadArgument;
2125 }
2126
2127 return ret;
2128}
2129
2130static void
2131applyToInterestNotifiers(const IORegistryEntry *target,
2132 const OSSymbol * typeOfInterest,
2133 OSObjectApplierFunction applier,
2134 void * context )
2135{
2136 OSArray * copyArray = NULL;
2137 OSObject * prop;
2138
2139 LOCKREADNOTIFY();
2140
2141 prop = target->copyProperty(aKey: typeOfInterest);
2142 IOCommand *notifyList = OSDynamicCast(IOCommand, prop);
2143
2144 if (notifyList) {
2145 copyArray = OSArray::withCapacity(capacity: 1);
2146
2147 // iterate over queue, entry is set to each element in the list
2148 iterqueue(&notifyList->fCommandChain, entry) {
2149 _IOServiceInterestNotifier * notify;
2150
2151 queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
2152 copyArray->setObject(notify);
2153 }
2154 }
2155 UNLOCKNOTIFY();
2156
2157 if (copyArray) {
2158 unsigned int index;
2159 OSObject * next;
2160
2161 for (index = 0; (next = copyArray->getObject( index )); index++) {
2162 (*applier)(next, context);
2163 }
2164 copyArray->release();
2165 }
2166
2167 OSSafeReleaseNULL(prop);
2168}
2169
2170void
2171IOService::applyToInterested( const OSSymbol * typeOfInterest,
2172 OSObjectApplierFunction applier,
2173 void * context )
2174{
2175 if (gIOGeneralInterest == typeOfInterest) {
2176 applyToClients(applier: (IOServiceApplierFunction) applier, context );
2177 }
2178 applyToInterestNotifiers(target: this, typeOfInterest, applier, context);
2179}
2180
2181struct MessageClientsContext {
2182 IOService * service;
2183 UInt32 type;
2184 void * argument;
2185 vm_size_t argSize;
2186 IOReturn ret;
2187};
2188
2189static void
2190messageClientsApplier( OSObject * object, void * ctx )
2191{
2192 IOReturn ret;
2193 MessageClientsContext * context = (MessageClientsContext *) ctx;
2194
2195 ret = context->service->messageClient( type: context->type,
2196 client: object, argument: context->argument, argSize: context->argSize );
2197 if (kIOReturnSuccess != ret) {
2198 context->ret = ret;
2199 }
2200}
2201
2202// send a message to all clients
2203IOReturn
2204IOService::messageClients( UInt32 type,
2205 void * argument, vm_size_t argSize )
2206{
2207 MessageClientsContext context;
2208
2209 context.service = this;
2210 context.type = type;
2211 context.argument = argument;
2212 context.argSize = argSize;
2213 context.ret = kIOReturnSuccess;
2214
2215 applyToInterested( typeOfInterest: gIOGeneralInterest,
2216 applier: &messageClientsApplier, context: &context );
2217
2218 return context.ret;
2219}
2220
2221IOReturn
2222IOService::acknowledgeNotification( IONotificationRef notification,
2223 IOOptionBits response )
2224{
2225 return kIOReturnUnsupported;
2226}
2227
2228IONotifier *
2229IOService::registerInterest( const OSSymbol * typeOfInterest,
2230 IOServiceInterestHandler handler, void * target, void * ref )
2231{
2232 _IOServiceInterestNotifier * notify = NULL;
2233 IOReturn rc = kIOReturnError;
2234
2235 notify = new _IOServiceInterestNotifier;
2236 if (!notify) {
2237 return NULL;
2238 }
2239
2240 if (notify->init()) {
2241 rc = registerInterestForNotifier(notify, typeOfInterest,
2242 handler, target, ref);
2243 }
2244
2245 if (rc != kIOReturnSuccess) {
2246 notify->release();
2247 notify = NULL;
2248 }
2249
2250 return notify;
2251}
2252
2253
2254
2255static IOReturn
2256IOServiceInterestHandlerToBlock( void * target __unused, void * refCon,
2257 UInt32 messageType, IOService * provider,
2258 void * messageArgument, vm_size_t argSize )
2259{
2260 return ((IOServiceInterestHandlerBlock) refCon)(messageType, provider, messageArgument, argSize);
2261}
2262
2263IONotifier *
2264IOService::registerInterest(const OSSymbol * typeOfInterest,
2265 IOServiceInterestHandlerBlock handler)
2266{
2267 IONotifier * notify;
2268 void * block;
2269
2270 block = Block_copy(handler);
2271 if (!block) {
2272 return NULL;
2273 }
2274
2275 notify = registerInterest(typeOfInterest, handler: &IOServiceInterestHandlerToBlock, NULL, ref: block);
2276
2277 if (!notify) {
2278 Block_release(block);
2279 }
2280
2281 return notify;
2282}
2283
2284IOReturn
2285IOService::registerInterestForNotifier( IONotifier *svcNotify, const OSSymbol * typeOfInterest,
2286 IOServiceInterestHandler handler, void * target, void * ref )
2287{
2288 IOReturn rc = kIOReturnSuccess;
2289 _IOServiceInterestNotifier *notify = NULL;
2290
2291
2292 if (!svcNotify || !(notify = OSDynamicCast(_IOServiceInterestNotifier, svcNotify)) || !handler) {
2293 return kIOReturnBadArgument;
2294 }
2295
2296 notify->handler = handler;
2297 notify->target = target;
2298 notify->ref = ref;
2299
2300 if ((typeOfInterest != gIOGeneralInterest)
2301 && (typeOfInterest != gIOBusyInterest)
2302 && (typeOfInterest != gIOAppPowerStateInterest)
2303 && (typeOfInterest != gIOConsoleSecurityInterest)
2304 && (typeOfInterest != gIOPriorityPowerStateInterest)) {
2305 return kIOReturnBadArgument;
2306 }
2307
2308 lockForArbitration();
2309 if (0 == (__state[0] & kIOServiceInactiveState)) {
2310 notify->state = kIOServiceNotifyEnable;
2311
2312 ////// queue
2313
2314 LOCKWRITENOTIFY();
2315
2316 // Get the head of the notifier linked list
2317 IOCommand * notifyList;
2318 OSObject * obj = copyProperty( aKey: typeOfInterest );
2319 if (!(notifyList = OSDynamicCast(IOCommand, obj))) {
2320 notifyList = OSTypeAlloc(IOCommand);
2321 if (notifyList) {
2322 notifyList->init();
2323 bool ok = setProperty( aKey: typeOfInterest, anObject: notifyList);
2324 notifyList->release();
2325 if (!ok) {
2326 notifyList = NULL;
2327 }
2328 }
2329 }
2330 if (obj) {
2331 obj->release();
2332 }
2333
2334 if (notifyList) {
2335 enqueue(&notifyList->fCommandChain, &notify->chain);
2336 notify->retain(); // ref'ed while in list
2337 }
2338
2339 UNLOCKNOTIFY();
2340 } else {
2341 rc = kIOReturnNotReady;
2342 }
2343 unlockForArbitration();
2344
2345 return rc;
2346}
2347
2348static void
2349cleanInterestList( OSObject * head )
2350{
2351 IOCommand *notifyHead = OSDynamicCast(IOCommand, head);
2352 if (!notifyHead) {
2353 return;
2354 }
2355
2356 LOCKWRITENOTIFY();
2357 while (queue_entry_t entry = dequeue(&notifyHead->fCommandChain)) {
2358 queue_next(entry) = queue_prev(entry) = NULL;
2359
2360 _IOServiceInterestNotifier * notify;
2361
2362 queue_element(entry, notify, _IOServiceInterestNotifier *, chain);
2363 notify->release();
2364 }
2365 UNLOCKNOTIFY();
2366}
2367
2368void
2369IOService::unregisterAllInterest( void )
2370{
2371 OSObject * prop;
2372
2373 prop = copyProperty(aKey: gIOGeneralInterest);
2374 cleanInterestList(head: prop);
2375 OSSafeReleaseNULL(prop);
2376
2377 prop = copyProperty(aKey: gIOBusyInterest);
2378 cleanInterestList(head: prop);
2379 OSSafeReleaseNULL(prop);
2380
2381 prop = copyProperty(aKey: gIOAppPowerStateInterest);
2382 cleanInterestList(head: prop);
2383 OSSafeReleaseNULL(prop);
2384
2385 prop = copyProperty(aKey: gIOPriorityPowerStateInterest);
2386 cleanInterestList(head: prop);
2387 OSSafeReleaseNULL(prop);
2388
2389 prop = copyProperty(aKey: gIOConsoleSecurityInterest);
2390 cleanInterestList(head: prop);
2391 OSSafeReleaseNULL(prop);
2392}
2393
2394/*
2395 * _IOServiceInterestNotifier
2396 */
2397
2398// wait for all threads, other than the current one,
2399// to exit the handler
2400
2401void
2402_IOServiceInterestNotifier::wait()
2403{
2404 _IOServiceNotifierInvocation * next;
2405 bool doWait;
2406
2407 do {
2408 doWait = false;
2409 queue_iterate( &handlerInvocations, next,
2410 _IOServiceNotifierInvocation *, link) {
2411 if (next->thread != current_thread()) {
2412 doWait = true;
2413 break;
2414 }
2415 }
2416 if (doWait) {
2417 state |= kIOServiceNotifyWaiter;
2418 SLEEPNOTIFY(this);
2419 }
2420 } while (doWait);
2421}
2422
2423void
2424_IOServiceInterestNotifier::free()
2425{
2426 assert( queue_empty( &handlerInvocations ));
2427
2428 if (handler == &IOServiceInterestHandlerToBlock) {
2429 Block_release(ref);
2430 }
2431
2432 OSObject::free();
2433}
2434
2435void
2436_IOServiceInterestNotifier::remove()
2437{
2438 LOCKWRITENOTIFY();
2439
2440 if (queue_next( &chain )) {
2441 remqueue(elt: &chain);
2442 queue_next( &chain) = queue_prev( &chain) = NULL;
2443 release();
2444 }
2445
2446 state &= ~kIOServiceNotifyEnable;
2447
2448 wait();
2449
2450 UNLOCKNOTIFY();
2451
2452 release();
2453}
2454
2455bool
2456_IOServiceInterestNotifier::disable()
2457{
2458 bool ret;
2459
2460 LOCKWRITENOTIFY();
2461
2462 ret = (0 != (kIOServiceNotifyEnable & state));
2463 state &= ~kIOServiceNotifyEnable;
2464 if (ret) {
2465 wait();
2466 }
2467
2468 UNLOCKNOTIFY();
2469
2470 return ret;
2471}
2472
2473void
2474_IOServiceInterestNotifier::enable( bool was )
2475{
2476 LOCKWRITENOTIFY();
2477 if (was) {
2478 state |= kIOServiceNotifyEnable;
2479 } else {
2480 state &= ~kIOServiceNotifyEnable;
2481 }
2482 UNLOCKNOTIFY();
2483}
2484
2485bool
2486_IOServiceInterestNotifier::init()
2487{
2488 queue_init( &handlerInvocations );
2489 return OSObject::init();
2490}
2491/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2492
2493/*
2494 * Termination
2495 */
2496
2497#define tailQ(o) setObject(o)
2498#define headQ(o) setObject(0, o)
2499#define TLOG(fmt, args...) { if(kIOLogYield & gIOKitDebug) { IOLog("[%llx] ", thread_tid(current_thread())); IOLog(fmt, ## args); }}
2500
2501static void
2502_workLoopAction( IOWorkLoop::Action action,
2503 IOService * service,
2504 void * p0 = NULL, void * p1 = NULL,
2505 void * p2 = NULL, void * p3 = NULL )
2506{
2507 IOWorkLoop * wl;
2508
2509 if ((wl = service->getWorkLoop())) {
2510 wl->retain();
2511 wl->runAction( action, target: service, arg0: p0, arg1: p1, arg2: p2, arg3: p3 );
2512 wl->release();
2513 } else {
2514 (*action)( service, p0, p1, p2, p3 );
2515 }
2516}
2517
2518bool
2519IOService::requestTerminate( IOService * provider, IOOptionBits options )
2520{
2521 bool ok;
2522
2523 // if its our only provider
2524 ok = isParent( parent: provider, plane: gIOServicePlane, onlyParent: true);
2525
2526 // -- compat
2527 if (ok) {
2528 provider->terminateClient( client: this, options: options | kIOServiceRecursing );
2529 ok = (0 != (kIOServiceInactiveState & __state[0]));
2530 }
2531 // --
2532
2533 return ok;
2534}
2535
2536bool
2537IOService::terminatePhase1( IOOptionBits options )
2538{
2539 IOService * victim;
2540 IOService * client;
2541 IOService * rematchProvider;
2542 OSIterator * iter;
2543 OSArray * makeInactive;
2544 OSArray * waitingInactive;
2545 IOOptionBits callerOptions;
2546 int waitResult = THREAD_AWAKENED;
2547 bool wait;
2548 bool ok;
2549 bool didInactive;
2550 bool startPhase2 = false;
2551
2552 TLOG("%s[0x%qx]::terminatePhase1(%08llx)\n", getName(), getRegistryEntryID(), (long long)options);
2553
2554 callerOptions = options;
2555 rematchProvider = NULL;
2556 uint64_t regID = getRegistryEntryID();
2557 IOServiceTrace(
2558 IOSERVICE_TERMINATE_PHASE1,
2559 (uintptr_t) regID,
2560 (uintptr_t) (regID >> 32),
2561 (uintptr_t) this,
2562 (uintptr_t) options);
2563
2564 // -- compat
2565 if (options & kIOServiceRecursing) {
2566 lockForArbitration();
2567 if (0 == (kIOServiceInactiveState & __state[0])) {
2568 __state[0] |= kIOServiceInactiveState;
2569 __state[1] |= kIOServiceRecursing | kIOServiceTermPhase1State;
2570 }
2571 unlockForArbitration();
2572
2573 return true;
2574 }
2575 // --
2576
2577 makeInactive = OSArray::withCapacity( capacity: 16 );
2578 waitingInactive = OSArray::withCapacity( capacity: 16 );
2579 if (!makeInactive || !waitingInactive) {
2580 OSSafeReleaseNULL(makeInactive);
2581 OSSafeReleaseNULL(waitingInactive);
2582 return false;
2583 }
2584
2585 victim = this;
2586 victim->retain();
2587
2588 while (victim) {
2589 didInactive = victim->lockForArbitration( isSuccessRequired: true );
2590 if (didInactive) {
2591 uint64_t regID1 = victim->getRegistryEntryID();
2592 IOServiceTrace(IOSERVICE_TERM_SET_INACTIVE,
2593 (uintptr_t) regID1,
2594 (uintptr_t) (regID1 >> 32),
2595 (uintptr_t) victim->__state[1],
2596 (uintptr_t) 0);
2597
2598 enum { kRP1 = kIOServiceRecursing | kIOServiceTermPhase1State };
2599 didInactive = (kRP1 == (victim->__state[1] & kRP1))
2600 || (0 == (victim->__state[0] & kIOServiceInactiveState));
2601
2602 if (!didInactive) {
2603 // a multiply attached IOService can be visited twice
2604 if (-1U == waitingInactive->getNextIndexOfObject(anObject: victim, index: 0)) {
2605 do{
2606 IOLockLock(gIOServiceBusyLock);
2607 wait = (victim->__state[1] & kIOServiceTermPhase1State);
2608 if (wait) {
2609 TLOG("%s[0x%qx]::waitPhase1(%s[0x%qx])\n",
2610 getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2611 victim->__state[1] |= kIOServiceTerm1WaiterState;
2612 victim->unlockForArbitration();
2613 assert_wait(event: (event_t)&victim->__state[1], THREAD_UNINT);
2614 }
2615 IOLockUnlock(gIOServiceBusyLock);
2616 if (wait) {
2617 waitResult = thread_block(THREAD_CONTINUE_NULL);
2618 TLOG("%s[0x%qx]::did waitPhase1(%s[0x%qx])\n",
2619 getName(), getRegistryEntryID(), victim->getName(), victim->getRegistryEntryID());
2620 victim->lockForArbitration();
2621 }
2622 }while (wait && (waitResult != THREAD_TIMED_OUT));
2623 }
2624 } else {
2625 victim->__state[0] |= kIOServiceInactiveState;
2626 victim->__state[0] &= ~(kIOServiceRegisteredState | kIOServiceMatchedState
2627 | kIOServiceFirstPublishState | kIOServiceFirstMatchState);
2628 victim->__state[1] &= ~kIOServiceRecursing;
2629 victim->__state[1] |= kIOServiceTermPhase1State;
2630 waitingInactive->headQ(victim);
2631 if (victim == this) {
2632 if (kIOServiceTerminateNeedWillTerminate & options) {
2633 victim->__state[1] |= kIOServiceNeedWillTerminate;
2634 }
2635 }
2636 victim->_adjustBusy( delta: 1 );
2637
2638 if ((options & kIOServiceTerminateWithRematch) && (victim == this)) {
2639 if ((options & kIOServiceTerminateWithRematchCurrentDext)) {
2640 OSObject * obj;
2641 OSObject * rematchProps;
2642 OSNumber * num;
2643 uint32_t count;
2644
2645 rematchProvider = getProvider();
2646 if (rematchProvider) {
2647 obj = rematchProvider->copyProperty(aKey: gIORematchCountKey);
2648 num = OSDynamicCast(OSNumber, obj);
2649 count = 0;
2650 if (num) {
2651 count = num->unsigned32BitValue();
2652 count++;
2653 }
2654 num = OSNumber::withNumber(value: count, numberOfBits: 32);
2655 rematchProvider->setProperty(aKey: gIORematchCountKey, anObject: num);
2656 rematchProps = copyProperty(aKey: gIOMatchedPersonalityKey);
2657 rematchProvider->setProperty(aKey: gIORematchPersonalityKey, anObject: rematchProps);
2658 OSSafeReleaseNULL(num);
2659 OSSafeReleaseNULL(rematchProps);
2660 OSSafeReleaseNULL(obj);
2661 }
2662 }
2663 victim->__state[1] |= kIOServiceRematchOnDetach;
2664 }
2665 }
2666 victim->unlockForArbitration();
2667 }
2668 if (victim == this) {
2669 options &= ~(kIOServiceTerminateWithRematch | kIOServiceTerminateWithRematchCurrentDext);
2670 startPhase2 = didInactive;
2671 }
2672 if (didInactive) {
2673 OSArray * notifiers;
2674 notifiers = victim->copyNotifiers(type: gIOTerminatedNotification, orNewState: 0, andNewState: 0xffffffff);
2675 victim->invokeNotifiers(willSend: &notifiers);
2676
2677 IOUserClient::destroyUserReferences( obj: victim );
2678
2679 iter = victim->getClientIterator();
2680 if (iter) {
2681 while ((client = (IOService *) iter->getNextObject())) {
2682 TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], %08llx)\n",
2683 client->getName(), client->getRegistryEntryID(),
2684 victim->getName(), victim->getRegistryEntryID(), (long long)options);
2685 ok = client->requestTerminate( provider: victim, options );
2686 TLOG("%s[0x%qx]::requestTerminate(%s[0x%qx], ok = %d)\n",
2687 client->getName(), client->getRegistryEntryID(),
2688 victim->getName(), victim->getRegistryEntryID(), ok);
2689
2690 uint64_t regID1 = client->getRegistryEntryID();
2691 uint64_t regID2 = victim->getRegistryEntryID();
2692 IOServiceTrace(
2693 (ok ? IOSERVICE_TERMINATE_REQUEST_OK
2694 : IOSERVICE_TERMINATE_REQUEST_FAIL),
2695 (uintptr_t) regID1,
2696 (uintptr_t) (regID1 >> 32),
2697 (uintptr_t) regID2,
2698 (uintptr_t) (regID2 >> 32));
2699
2700 if (ok) {
2701 makeInactive->setObject( client );
2702 }
2703 }
2704 iter->release();
2705 }
2706 }
2707 victim->release();
2708 victim = (IOService *) makeInactive->getObject(index: 0);
2709 if (victim) {
2710 victim->retain();
2711 makeInactive->removeObject(index: 0);
2712 }
2713 }
2714
2715 makeInactive->release();
2716
2717 while ((victim = (IOService *) waitingInactive->getObject(index: 0))) {
2718 victim->retain();
2719 waitingInactive->removeObject(index: 0);
2720
2721 victim->lockForArbitration();
2722 victim->__state[1] &= ~kIOServiceTermPhase1State;
2723 if (kIOServiceTerm1WaiterState & victim->__state[1]) {
2724 victim->__state[1] &= ~kIOServiceTerm1WaiterState;
2725 TLOG("%s[0x%qx]::wakePhase1\n", victim->getName(), victim->getRegistryEntryID());
2726 IOLockLock( gIOServiceBusyLock );
2727 thread_wakeup((event_t) &victim->__state[1]);
2728 IOLockUnlock( gIOServiceBusyLock );
2729 }
2730 victim->unlockForArbitration();
2731 victim->release();
2732 }
2733
2734 waitingInactive->release();
2735
2736 if (startPhase2) {
2737 retain();
2738 lockForArbitration();
2739 scheduleTerminatePhase2(options);
2740 unlockForArbitration();
2741 release();
2742 }
2743
2744 if (rematchProvider) {
2745 DKLOG(DKS " rematching after dext crash\n", DKN(rematchProvider));
2746 }
2747
2748 return true;
2749}
2750
2751void
2752IOService::setTerminateDefer(IOService * provider, bool defer)
2753{
2754 lockForArbitration();
2755 if (defer) {
2756 __state[1] |= kIOServiceStartState;
2757 } else {
2758 __state[1] &= ~kIOServiceStartState;
2759 }
2760 unlockForArbitration();
2761
2762 if (provider && !defer) {
2763 provider->lockForArbitration();
2764 provider->scheduleTerminatePhase2();
2765 provider->unlockForArbitration();
2766 }
2767}
2768
2769// Must call this while holding gJobsLock
2770void
2771IOService::waitToBecomeTerminateThread(void)
2772{
2773 IOLockAssert(gJobsLock, kIOLockAssertOwned);
2774 bool wait;
2775 do {
2776 wait = (gIOTerminateThread != THREAD_NULL);
2777 if (wait) {
2778 IOLockSleep(lock: gJobsLock, event: &gIOTerminateThread, THREAD_UNINT);
2779 }
2780 } while (wait);
2781 gIOTerminateThread = current_thread();
2782}
2783
2784// call with lockForArbitration
2785void
2786IOService::scheduleTerminatePhase2( IOOptionBits options )
2787{
2788 AbsoluteTime deadline;
2789 uint64_t regID1;
2790 int waitResult = THREAD_AWAKENED;
2791 bool wait = false, haveDeadline = false;
2792
2793 if (!(__state[0] & kIOServiceInactiveState)) {
2794 return;
2795 }
2796
2797 regID1 = getRegistryEntryID();
2798 IOServiceTrace(
2799 IOSERVICE_TERM_SCHED_PHASE2,
2800 (uintptr_t) regID1,
2801 (uintptr_t) (regID1 >> 32),
2802 (uintptr_t) __state[1],
2803 (uintptr_t) options);
2804
2805 if (__state[1] & kIOServiceTermPhase1State) {
2806 return;
2807 }
2808
2809 retain();
2810 unlockForArbitration();
2811 options |= kIOServiceRequired;
2812 IOLockLock( gJobsLock );
2813
2814 if ((options & kIOServiceSynchronous)
2815 && (current_thread() != gIOTerminateThread)) {
2816 waitToBecomeTerminateThread();
2817 gIOTerminatePhase2List->setObject( this );
2818 gIOTerminateWork++;
2819
2820 do {
2821 while (gIOTerminateWork) {
2822 terminateWorker( options );
2823 }
2824 wait = (0 != (__state[1] & kIOServiceBusyStateMask));
2825 if (wait) {
2826 /* wait for the victim to go non-busy */
2827 if (!haveDeadline) {
2828 clock_interval_to_deadline( interval: 15, scale_factor: kSecondScale, result: &deadline );
2829 haveDeadline = true;
2830 }
2831 /* let others do work while we wait */
2832 gIOTerminateThread = NULL;
2833 IOLockWakeup( lock: gJobsLock, event: (event_t) &gIOTerminateThread, /* one-thread */ oneThread: false);
2834 waitResult = IOLockSleepDeadline( lock: gJobsLock, event: &gIOTerminateWork,
2835 deadline, THREAD_UNINT );
2836 if (__improbable(waitResult == THREAD_TIMED_OUT)) {
2837 IOLog(format: "%s[0x%qx]::terminate(kIOServiceSynchronous): THREAD_TIMED_OUT. "
2838 "Attempting to auto-resolve your deadlock. PLEASE FIX!\n", getName(), getRegistryEntryID());
2839 }
2840 waitToBecomeTerminateThread();
2841 }
2842 } while (gIOTerminateWork || (wait && (waitResult != THREAD_TIMED_OUT)));
2843
2844 gIOTerminateThread = NULL;
2845 IOLockWakeup( lock: gJobsLock, event: (event_t) &gIOTerminateThread, /* one-thread */ oneThread: false);
2846 } else {
2847 // ! kIOServiceSynchronous
2848
2849 gIOTerminatePhase2List->setObject( this );
2850 if (0 == gIOTerminateWork++) {
2851 assert(gIOTerminateWorkerThread);
2852 IOLockWakeup(lock: gJobsLock, event: (event_t)&gIOTerminateWork, /* one-thread */ oneThread: false );
2853 }
2854 }
2855
2856 IOLockUnlock( gJobsLock );
2857 lockForArbitration();
2858 release();
2859}
2860
2861__attribute__((__noreturn__))
2862void
2863IOService::terminateThread( void * arg, wait_result_t waitResult )
2864{
2865 // IOLockSleep re-acquires the lock on wakeup, so we only need to do this once
2866 IOLockLock(gJobsLock);
2867 while (true) {
2868 if (gIOTerminateThread != gIOTerminateWorkerThread) {
2869 waitToBecomeTerminateThread();
2870 }
2871
2872 while (gIOTerminateWork) {
2873 terminateWorker(options: (IOOptionBits)(uintptr_t)arg );
2874 }
2875
2876 gIOTerminateThread = NULL;
2877 IOLockWakeup( lock: gJobsLock, event: (event_t) &gIOTerminateThread, /* one-thread */ oneThread: false);
2878 IOLockSleep(lock: gJobsLock, event: &gIOTerminateWork, THREAD_UNINT);
2879 }
2880}
2881
2882void
2883IOService::scheduleStop( IOService * provider )
2884{
2885 uint64_t regID1 = getRegistryEntryID();
2886 uint64_t regID2 = provider->getRegistryEntryID();
2887
2888 TLOG("%s[0x%qx]::scheduleStop(%s[0x%qx])\n", getName(), regID1, provider->getName(), regID2);
2889 IOServiceTrace(
2890 IOSERVICE_TERMINATE_SCHEDULE_STOP,
2891 (uintptr_t) regID1,
2892 (uintptr_t) (regID1 >> 32),
2893 (uintptr_t) regID2,
2894 (uintptr_t) (regID2 >> 32));
2895
2896 IOLockLock( gJobsLock );
2897 gIOStopList->tailQ( this );
2898 gIOStopProviderList->tailQ( provider );
2899
2900 if (0 == gIOTerminateWork++) {
2901 assert(gIOTerminateWorkerThread);
2902 IOLockWakeup(lock: gJobsLock, event: (event_t)&gIOTerminateWork, /* one-thread */ oneThread: false );
2903 }
2904
2905 IOLockUnlock( gJobsLock );
2906}
2907
2908void
2909IOService::scheduleFinalize(bool now)
2910{
2911 uint64_t regID1 = getRegistryEntryID();
2912
2913 TLOG("%s[0x%qx]::scheduleFinalize\n", getName(), regID1);
2914 IOServiceTrace(
2915 IOSERVICE_TERMINATE_SCHEDULE_FINALIZE,
2916 (uintptr_t) regID1,
2917 (uintptr_t) (regID1 >> 32),
2918 0, 0);
2919
2920 if (now || IOUserClient::finalizeUserReferences(obj: this)) {
2921 IOLockLock( gJobsLock );
2922 gIOFinalizeList->tailQ(this);
2923 if (0 == gIOTerminateWork++) {
2924 assert(gIOTerminateWorkerThread);
2925 IOLockWakeup(lock: gJobsLock, event: (event_t)&gIOTerminateWork, /* one-thread */ oneThread: false );
2926 }
2927 IOLockUnlock( gJobsLock );
2928 }
2929}
2930
2931bool
2932IOService::willTerminate( IOService * provider, IOOptionBits options )
2933{
2934 if (reserved->uvars) {
2935 IOUserServer::serviceWillTerminate(client: this, provider, options);
2936 }
2937 return true;
2938}
2939
2940bool
2941IOService::didTerminate( IOService * provider, IOOptionBits options, bool * defer )
2942{
2943 if (reserved->uvars) {
2944 IOUserServer::serviceDidTerminate(client: this, provider, options, defer);
2945 }
2946
2947 if (false == *defer) {
2948 if (lockForArbitration( isSuccessRequired: true )) {
2949 if (false == provider->handleIsOpen( forClient: this )) {
2950 scheduleStop( provider );
2951 }
2952 // -- compat
2953 else {
2954 message( kIOMessageServiceIsRequestingClose, provider, argument: (void *)(uintptr_t) options );
2955 if (false == provider->handleIsOpen( forClient: this )) {
2956 scheduleStop( provider );
2957 }
2958 }
2959 // --
2960 unlockForArbitration();
2961 }
2962 }
2963
2964 return true;
2965}
2966
2967void
2968IOService::actionWillTerminate( IOService * victim, IOOptionBits options,
2969 OSArray * doPhase2List,
2970 bool user,
2971 void *unused3 __unused)
2972{
2973 OSIterator * iter;
2974 IOService * client;
2975 bool ok;
2976 uint64_t regID1, regID2 = victim->getRegistryEntryID();
2977
2978 iter = victim->getClientIterator();
2979 if (iter) {
2980 while ((client = (IOService *) iter->getNextObject())) {
2981 if (user != (NULL != client->reserved->uvars)) {
2982 continue;
2983 }
2984 regID1 = client->getRegistryEntryID();
2985 TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
2986 client->getName(), regID1,
2987 victim->getName(), regID2, (long long)options);
2988 IOServiceTrace(
2989 IOSERVICE_TERMINATE_WILL,
2990 (uintptr_t) regID1,
2991 (uintptr_t) (regID1 >> 32),
2992 (uintptr_t) regID2,
2993 (uintptr_t) (regID2 >> 32));
2994
2995 ok = client->willTerminate( provider: victim, options );
2996 doPhase2List->tailQ( client );
2997 }
2998 iter->release();
2999 }
3000}
3001
3002void
3003IOService::actionDidTerminate( IOService * victim, IOOptionBits options,
3004 void *unused1 __unused, void *unused2 __unused,
3005 void *unused3 __unused )
3006{
3007 OSIterator * iter;
3008 IOService * client;
3009 bool defer;
3010 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3011
3012 victim->messageClients( kIOMessageServiceIsTerminated, argument: (void *)(uintptr_t) options );
3013
3014 iter = victim->getClientIterator();
3015 if (iter) {
3016 while ((client = (IOService *) iter->getNextObject())) {
3017 regID1 = client->getRegistryEntryID();
3018 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
3019 client->getName(), regID1,
3020 victim->getName(), regID2, (long long)options);
3021 defer = false;
3022 client->didTerminate( provider: victim, options, defer: &defer );
3023
3024 IOServiceTrace(
3025 (defer ? IOSERVICE_TERMINATE_DID_DEFER
3026 : IOSERVICE_TERMINATE_DID),
3027 (uintptr_t) regID1,
3028 (uintptr_t) (regID1 >> 32),
3029 (uintptr_t) regID2,
3030 (uintptr_t) (regID2 >> 32));
3031
3032 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
3033 client->getName(), regID1,
3034 victim->getName(), regID2, defer);
3035 }
3036 iter->release();
3037 }
3038}
3039
3040
3041void
3042IOService::actionWillStop( IOService * victim, IOOptionBits options,
3043 void *unused1 __unused, void *unused2 __unused,
3044 void *unused3 __unused )
3045{
3046 OSIterator * iter;
3047 IOService * provider;
3048 bool ok;
3049 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3050
3051 iter = victim->getProviderIterator();
3052 if (iter) {
3053 while ((provider = (IOService *) iter->getNextObject())) {
3054 regID1 = provider->getRegistryEntryID();
3055 TLOG("%s[0x%qx]::willTerminate(%s[0x%qx], %08llx)\n",
3056 victim->getName(), regID2,
3057 provider->getName(), regID1, (long long)options);
3058 IOServiceTrace(
3059 IOSERVICE_TERMINATE_WILL,
3060 (uintptr_t) regID2,
3061 (uintptr_t) (regID2 >> 32),
3062 (uintptr_t) regID1,
3063 (uintptr_t) (regID1 >> 32));
3064
3065 ok = victim->willTerminate( provider, options );
3066 }
3067 iter->release();
3068 }
3069}
3070
3071void
3072IOService::actionDidStop( IOService * victim, IOOptionBits options,
3073 void *unused1 __unused, void *unused2 __unused,
3074 void *unused3 __unused )
3075{
3076 OSIterator * iter;
3077 IOService * provider;
3078 bool defer = false;
3079 uint64_t regID1, regID2 = victim->getRegistryEntryID();
3080
3081 iter = victim->getProviderIterator();
3082 if (iter) {
3083 while ((provider = (IOService *) iter->getNextObject())) {
3084 regID1 = provider->getRegistryEntryID();
3085 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], %08llx)\n",
3086 victim->getName(), regID2,
3087 provider->getName(), regID1, (long long)options);
3088 victim->didTerminate( provider, options, defer: &defer );
3089
3090 IOServiceTrace(
3091 (defer ? IOSERVICE_TERMINATE_DID_DEFER
3092 : IOSERVICE_TERMINATE_DID),
3093 (uintptr_t) regID2,
3094 (uintptr_t) (regID2 >> 32),
3095 (uintptr_t) regID1,
3096 (uintptr_t) (regID1 >> 32));
3097
3098 TLOG("%s[0x%qx]::didTerminate(%s[0x%qx], defer %d)\n",
3099 victim->getName(), regID2,
3100 provider->getName(), regID1, defer);
3101 }
3102 iter->release();
3103 }
3104}
3105
3106
3107void
3108IOService::actionFinalize( IOService * victim, IOOptionBits options,
3109 void *unused1 __unused, void *unused2 __unused,
3110 void *unused3 __unused )
3111{
3112 uint64_t regID1 = victim->getRegistryEntryID();
3113 TLOG("%s[0x%qx]::finalize(%08llx)\n", victim->getName(), regID1, (long long)options);
3114 IOServiceTrace(
3115 IOSERVICE_TERMINATE_FINALIZE,
3116 (uintptr_t) regID1,
3117 (uintptr_t) (regID1 >> 32),
3118 0, 0);
3119
3120 victim->finalize( options );
3121}
3122
3123void
3124IOService::actionStop( IOService * provider, IOService * client,
3125 void *unused1 __unused, void *unused2 __unused,
3126 void *unused3 __unused )
3127{
3128 uint64_t regID1 = provider->getRegistryEntryID();
3129 uint64_t regID2 = client->getRegistryEntryID();
3130
3131 TLOG("%s[0x%qx]::stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3132 IOServiceTrace(
3133 IOSERVICE_TERMINATE_STOP,
3134 (uintptr_t) regID1,
3135 (uintptr_t) (regID1 >> 32),
3136 (uintptr_t) regID2,
3137 (uintptr_t) (regID2 >> 32));
3138
3139 client->stop( provider );
3140 if (provider->isOpen( forClient: client )) {
3141 provider->close( forClient: client );
3142 }
3143
3144 TLOG("%s[0x%qx]::detach(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3145 client->detach( provider );
3146}
3147
3148#pragma clang diagnostic push
3149#pragma clang diagnostic ignored "-Wcast-function-type"
3150
3151void
3152IOService::terminateWorker( IOOptionBits options )
3153{
3154 OSArray * doPhase2List;
3155 OSArray * didPhase2List;
3156 OSSet * freeList;
3157 OSIterator * iter;
3158 UInt32 workDone;
3159 IOService * victim;
3160 IOService * client;
3161 IOService * provider;
3162 unsigned int idx;
3163 bool moreToDo;
3164 bool doPhase2;
3165 bool doPhase3;
3166
3167 options |= kIOServiceRequired;
3168
3169 doPhase2List = OSArray::withCapacity( capacity: 16 );
3170 didPhase2List = OSArray::withCapacity( capacity: 16 );
3171 freeList = OSSet::withCapacity( capacity: 16 );
3172 if ((NULL == doPhase2List) || (NULL == didPhase2List) || (NULL == freeList)) {
3173 OSSafeReleaseNULL(doPhase2List);
3174 OSSafeReleaseNULL(didPhase2List);
3175 OSSafeReleaseNULL(freeList);
3176 return;
3177 }
3178
3179 do {
3180 workDone = gIOTerminateWork;
3181
3182 while ((victim = (IOService *) gIOTerminatePhase2List->getObject(index: 0))) {
3183 victim->retain();
3184 gIOTerminatePhase2List->removeObject(index: 0);
3185 IOLockUnlock( gJobsLock );
3186
3187 uint64_t regID1 = victim->getRegistryEntryID();
3188 IOServiceTrace(
3189 IOSERVICE_TERM_START_PHASE2,
3190 (uintptr_t) regID1,
3191 (uintptr_t) (regID1 >> 32),
3192 (uintptr_t) 0,
3193 (uintptr_t) 0);
3194
3195 while (victim) {
3196 doPhase2 = victim->lockForArbitration( isSuccessRequired: true );
3197 if (doPhase2) {
3198 doPhase2 = (0 != (kIOServiceInactiveState & victim->__state[0]));
3199 if (doPhase2) {
3200 uint64_t regID1 = victim->getRegistryEntryID();
3201 IOServiceTrace(
3202 IOSERVICE_TERM_TRY_PHASE2,
3203 (uintptr_t) regID1,
3204 (uintptr_t) (regID1 >> 32),
3205 (uintptr_t) victim->__state[1],
3206 (uintptr_t) 0);
3207
3208 doPhase2 = (0 == (victim->__state[1] &
3209 (kIOServiceTermPhase1State
3210 | kIOServiceTermPhase2State
3211 | kIOServiceConfigState)));
3212
3213 if (doPhase2 && (iter = victim->getClientIterator())) {
3214 while (doPhase2 && (client = (IOService *) iter->getNextObject())) {
3215 doPhase2 = (0 == (client->__state[1] & kIOServiceStartState));
3216 if (!doPhase2) {
3217 uint64_t regID1 = client->getRegistryEntryID();
3218 IOServiceTrace(
3219 IOSERVICE_TERM_UC_DEFER,
3220 (uintptr_t) regID1,
3221 (uintptr_t) (regID1 >> 32),
3222 (uintptr_t) client->__state[1],
3223 (uintptr_t) 0);
3224 TLOG("%s[0x%qx]::defer phase2(%s[0x%qx])\n",
3225 victim->getName(), victim->getRegistryEntryID(),
3226 client->getName(), client->getRegistryEntryID());
3227 }
3228 }
3229 iter->release();
3230 }
3231 if (doPhase2) {
3232 victim->__state[1] |= kIOServiceTermPhase2State;
3233 }
3234 }
3235 victim->unlockForArbitration();
3236 }
3237 if (doPhase2) {
3238 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3239 if (NULL == victim->reserved->uvars) {
3240 _workLoopAction(action: (IOWorkLoop::Action) &actionWillStop,
3241 service: victim, p0: (void *)(uintptr_t) options);
3242 } else {
3243 actionWillStop(victim, options, NULL, NULL, NULL);
3244 }
3245 }
3246
3247 OSArray * notifiers;
3248 notifiers = victim->copyNotifiers(type: gIOWillTerminateNotification, orNewState: 0, andNewState: 0xffffffff);
3249 victim->invokeNotifiers(willSend: &notifiers);
3250
3251 _workLoopAction(action: (IOWorkLoop::Action) &actionWillTerminate,
3252 service: victim,
3253 p0: (void *)(uintptr_t) options,
3254 p1: (void *)(uintptr_t) doPhase2List,
3255 p2: (void *)(uintptr_t) false);
3256
3257 actionWillTerminate(
3258 victim, options, doPhase2List, user: true, NULL);
3259
3260 didPhase2List->headQ( victim );
3261 }
3262 victim->release();
3263 victim = (IOService *) doPhase2List->getObject(index: 0);
3264 if (victim) {
3265 victim->retain();
3266 doPhase2List->removeObject(index: 0);
3267 }
3268 }
3269
3270 while ((victim = (IOService *) didPhase2List->getObject(index: 0))) {
3271 bool scheduleFinalize = false;
3272 if (victim->lockForArbitration( isSuccessRequired: true )) {
3273 victim->__state[1] |= kIOServiceTermPhase3State;
3274 scheduleFinalize = (NULL == victim->getClient());
3275 victim->unlockForArbitration();
3276 }
3277 _workLoopAction(action: (IOWorkLoop::Action) &actionDidTerminate,
3278 service: victim, p0: (void *)(uintptr_t) options );
3279 if (kIOServiceNeedWillTerminate & victim->__state[1]) {
3280 _workLoopAction(action: (IOWorkLoop::Action) &actionDidStop,
3281 service: victim, p0: (void *)(uintptr_t) options, NULL );
3282 }
3283 // no clients - will go to finalize
3284 if (scheduleFinalize) {
3285 victim->scheduleFinalize(now: false);
3286 }
3287 didPhase2List->removeObject(index: 0);
3288 }
3289 IOLockLock( gJobsLock );
3290 }
3291
3292 // phase 3
3293 do {
3294 doPhase3 = false;
3295 // finalize leaves
3296 while ((victim = (IOService *) gIOFinalizeList->getObject(index: 0))) {
3297 bool sendFinal = false;
3298 IOLockUnlock( gJobsLock );
3299 if (victim->lockForArbitration(isSuccessRequired: true)) {
3300 sendFinal = (0 == (victim->__state[1] & kIOServiceFinalized));
3301 if (sendFinal) {
3302 victim->__state[1] |= kIOServiceFinalized;
3303 }
3304 victim->unlockForArbitration();
3305 }
3306 if (sendFinal) {
3307 _workLoopAction(action: (IOWorkLoop::Action) &actionFinalize,
3308 service: victim, p0: (void *)(uintptr_t) options );
3309 }
3310 IOLockLock( gJobsLock );
3311 // hold off free
3312 freeList->setObject( victim );
3313 // safe if finalize list is append only
3314 gIOFinalizeList->removeObject(index: 0);
3315 }
3316
3317 for (idx = 0;
3318 (!doPhase3) && (client = (IOService *) gIOStopList->getObject(index: idx));) {
3319 provider = (IOService *) gIOStopProviderList->getObject(index: idx);
3320 assert( provider );
3321
3322 uint64_t regID1 = provider->getRegistryEntryID();
3323 uint64_t regID2 = client->getRegistryEntryID();
3324
3325 if (!provider->isChild( child: client, plane: gIOServicePlane )) {
3326 // may be multiply queued - nop it
3327 TLOG("%s[0x%qx]::nop stop(%s[0x%qx])\n", client->getName(), regID2, provider->getName(), regID1);
3328 IOServiceTrace(
3329 IOSERVICE_TERMINATE_STOP_NOP,
3330 (uintptr_t) regID1,
3331 (uintptr_t) (regID1 >> 32),
3332 (uintptr_t) regID2,
3333 (uintptr_t) (regID2 >> 32));
3334 } else {
3335 // a terminated client is not ready for stop if it has clients, skip it
3336 bool deferStop = (0 != (kIOServiceInactiveState & client->__state[0]));
3337 IOLockUnlock( gJobsLock );
3338 if (deferStop && client->lockForArbitration(isSuccessRequired: true)) {
3339 deferStop = (0 == (client->__state[1] & kIOServiceFinalized));
3340 //deferStop = (!deferStop && (0 != client->getClient()));
3341 //deferStop = (0 != client->getClient());
3342 client->unlockForArbitration();
3343 if (deferStop) {
3344 TLOG("%s[0x%qx]::defer stop()\n", client->getName(), regID2);
3345 IOServiceTrace(IOSERVICE_TERMINATE_STOP_DEFER,
3346 (uintptr_t) regID1,
3347 (uintptr_t) (regID1 >> 32),
3348 (uintptr_t) regID2,
3349 (uintptr_t) (regID2 >> 32));
3350
3351 idx++;
3352 IOLockLock( gJobsLock );
3353 continue;
3354 }
3355 }
3356 _workLoopAction(action: (IOWorkLoop::Action) &actionStop,
3357 service: provider, p0: (void *) client );
3358 IOLockLock( gJobsLock );
3359 // check the finalize list now
3360 doPhase3 = true;
3361 }
3362 // hold off free
3363 freeList->setObject( client );
3364 freeList->setObject( provider );
3365
3366 // safe if stop list is append only
3367 gIOStopList->removeObject( index: idx );
3368 gIOStopProviderList->removeObject( index: idx );
3369 idx = 0;
3370 }
3371 } while (doPhase3);
3372
3373 gIOTerminateWork -= workDone;
3374 moreToDo = (gIOTerminateWork != 0);
3375
3376 if (!moreToDo) {
3377 TLOG("iokit terminate done, %d stops remain\n", gIOStopList->getCount());
3378 IOServiceTrace(
3379 IOSERVICE_TERMINATE_DONE,
3380 (uintptr_t) gIOStopList->getCount(), 0, 0, 0);
3381 }
3382 } while (moreToDo);
3383
3384 IOLockUnlock( gJobsLock );
3385
3386 freeList->release();
3387 doPhase2List->release();
3388 didPhase2List->release();
3389
3390 IOLockLock( gJobsLock );
3391}
3392
3393#pragma clang diagnostic pop
3394
3395bool
3396IOService::finalize( IOOptionBits options )
3397{
3398 OSIterator * iter;
3399 IOService * provider;
3400 uint64_t regID1, regID2 = getRegistryEntryID();
3401
3402 iter = getProviderIterator();
3403 assert( iter );
3404
3405 if (iter) {
3406 while ((provider = (IOService *) iter->getNextObject())) {
3407 // -- compat
3408 if (0 == (__state[1] & kIOServiceTermPhase3State)) {
3409 /* we come down here on programmatic terminate */
3410
3411 regID1 = provider->getRegistryEntryID();
3412 TLOG("%s[0x%qx]::stop1(%s[0x%qx])\n", getName(), regID2, provider->getName(), regID1);
3413 IOServiceTrace(
3414 IOSERVICE_TERMINATE_STOP,
3415 (uintptr_t) regID1,
3416 (uintptr_t) (regID1 >> 32),
3417 (uintptr_t) regID2,
3418 (uintptr_t) (regID2 >> 32));
3419
3420 stop( provider );
3421 if (provider->isOpen( forClient: this )) {
3422 provider->close( forClient: this );
3423 }
3424 detach( provider );
3425 } else {
3426 //--
3427 if (provider->lockForArbitration( isSuccessRequired: true )) {
3428 if (0 == (provider->__state[1] & kIOServiceTermPhase3State)) {
3429 scheduleStop( provider );
3430 }
3431 provider->unlockForArbitration();
3432 }
3433 }
3434 }
3435 iter->release();
3436 }
3437
3438 return true;
3439}
3440
3441#undef tailQ
3442#undef headQ
3443
3444/*
3445 * Terminate
3446 */
3447
3448void
3449IOService::doServiceTerminate( IOOptionBits options )
3450{
3451}
3452
3453// a method in case someone needs to override it
3454bool
3455IOService::terminateClient( IOService * client, IOOptionBits options )
3456{
3457 bool ok;
3458
3459 if (client->isParent( parent: this, plane: gIOServicePlane, onlyParent: true)) {
3460 // we are the clients only provider
3461 ok = client->terminate( options );
3462 } else {
3463 ok = true;
3464 }
3465
3466 return ok;
3467}
3468
3469bool
3470IOService::terminate( IOOptionBits options )
3471{
3472 options |= kIOServiceTerminate;
3473
3474 return terminatePhase1( options );
3475}
3476
3477/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3478
3479/*
3480 * Open & close
3481 */
3482
3483struct ServiceOpenMessageContext {
3484 IOService * service;
3485 UInt32 type;
3486 IOService * excludeClient;
3487 IOOptionBits options;
3488};
3489
3490static void
3491serviceOpenMessageApplier( OSObject * object, void * ctx )
3492{
3493 ServiceOpenMessageContext * context = (ServiceOpenMessageContext *) ctx;
3494
3495 if (object != context->excludeClient) {
3496 context->service->messageClient( type: context->type, client: object, argument: (void *)(uintptr_t) context->options );
3497 }
3498}
3499
3500bool
3501IOService::open( IOService * forClient,
3502 IOOptionBits options,
3503 void * arg )
3504{
3505 bool ok;
3506 kern_return_t ret = kIOReturnSuccess;
3507 ServiceOpenMessageContext context;
3508
3509 context.service = this;
3510 context.type = kIOMessageServiceIsAttemptingOpen;
3511 context.excludeClient = forClient;
3512 context.options = options;
3513
3514 applyToInterested( typeOfInterest: gIOGeneralInterest,
3515 applier: &serviceOpenMessageApplier, context: &context );
3516
3517 if (false == lockForArbitration(isSuccessRequired: false)) {
3518 return false;
3519 }
3520
3521 ok = (0 == (__state[0] & kIOServiceInactiveState));
3522
3523 if (ok && forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3524 ret = forClient->reserved->uvars->userServer->serviceOpen(provider: this, client: forClient);
3525 if (ret != kIOReturnSuccess) {
3526 ok = false;
3527 }
3528 }
3529
3530 if (ok) {
3531 ok = handleOpen( forClient, options, arg );
3532
3533 if (!ok && forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3534 forClient->reserved->uvars->userServer->serviceClose(provider: this, client: forClient);
3535 }
3536 }
3537
3538 unlockForArbitration();
3539
3540 return ok;
3541}
3542
3543void
3544IOService::close( IOService * forClient,
3545 IOOptionBits options )
3546{
3547 bool wasClosed;
3548 bool last = false;
3549
3550 lockForArbitration();
3551
3552 wasClosed = handleIsOpen( forClient );
3553 if (wasClosed) {
3554 handleClose( forClient, options );
3555 last = (__state[1] & kIOServiceTermPhase3State);
3556
3557 if (forClient && forClient->reserved->uvars && forClient->reserved->uvars->userServer) {
3558 forClient->reserved->uvars->userServer->serviceClose(provider: this, client: forClient);
3559 }
3560 }
3561
3562 unlockForArbitration();
3563
3564 if (last) {
3565 forClient->scheduleStop( provider: this );
3566 } else if (wasClosed) {
3567 ServiceOpenMessageContext context;
3568
3569 context.service = this;
3570 context.type = kIOMessageServiceWasClosed;
3571 context.excludeClient = forClient;
3572 context.options = options;
3573
3574 applyToInterested( typeOfInterest: gIOGeneralInterest,
3575 applier: &serviceOpenMessageApplier, context: &context );
3576 }
3577}
3578
3579bool
3580IOService::isOpen( const IOService * forClient ) const
3581{
3582 IOService * self = (IOService *) this;
3583 bool ok;
3584
3585 self->lockForArbitration();
3586
3587 ok = handleIsOpen( forClient );
3588
3589 self->unlockForArbitration();
3590
3591 return ok;
3592}
3593
3594bool
3595IOService::handleOpen( IOService * forClient,
3596 IOOptionBits options,
3597 void * arg )
3598{
3599 bool ok;
3600
3601 ok = (NULL == __owner);
3602 if (ok) {
3603 __owner = forClient;
3604 } else if (options & kIOServiceSeize) {
3605 ok = (kIOReturnSuccess == messageClient( kIOMessageServiceIsRequestingClose,
3606 client: __owner, argument: (void *)(uintptr_t) options ));
3607 if (ok && (NULL == __owner)) {
3608 __owner = forClient;
3609 } else {
3610 ok = false;
3611 }
3612 }
3613 return ok;
3614}
3615
3616void
3617IOService::handleClose( IOService * forClient,
3618 IOOptionBits options )
3619{
3620 if (__owner == forClient) {
3621 __owner = NULL;
3622 }
3623}
3624
3625bool
3626IOService::handleIsOpen( const IOService * forClient ) const
3627{
3628 if (forClient) {
3629 return __owner == forClient;
3630 } else {
3631 return __owner != forClient;
3632 }
3633}
3634
3635/*
3636 * Probing & starting
3637 */
3638static SInt32
3639IONotifyOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3640{
3641 const _IOServiceNotifier * obj1 = (const _IOServiceNotifier *) inObj1;
3642 const _IOServiceNotifier * obj2 = (const _IOServiceNotifier *) inObj2;
3643 SInt32 val1;
3644 SInt32 val2;
3645
3646 val1 = 0;
3647 val2 = 0;
3648 if (obj1) {
3649 val1 = obj1->priority;
3650 }
3651 if (obj2) {
3652 val2 = obj2->priority;
3653 }
3654 if (val1 > val2) {
3655 return 1;
3656 }
3657 if (val1 < val2) {
3658 return -1;
3659 }
3660 return 0;
3661}
3662
3663static SInt32
3664IOServiceObjectOrder( const OSObject * entry, void * ref)
3665{
3666 OSDictionary * dict;
3667 IOService * service;
3668 _IOServiceNotifier * notify;
3669 OSSymbol * key = (OSSymbol *) ref;
3670 OSNumber * offset;
3671 OSObject * prop;
3672 SInt32 result;
3673
3674 prop = NULL;
3675 result = kIODefaultProbeScore;
3676 if ((dict = OSDynamicCast( OSDictionary, entry))) {
3677 offset = OSDynamicCast(OSNumber, dict->getObject( key ));
3678 } else if ((notify = OSDynamicCast( _IOServiceNotifier, entry))) {
3679 return notify->priority;
3680 } else if ((service = OSDynamicCast( IOService, entry))) {
3681 prop = service->copyProperty(aKey: key);
3682 offset = OSDynamicCast(OSNumber, prop);
3683 } else {
3684 assert( false );
3685 offset = NULL;
3686 }
3687
3688 if (offset) {
3689 result = offset->unsigned32BitValue();
3690 }
3691
3692 OSSafeReleaseNULL(prop);
3693
3694 return result;
3695}
3696
3697__attribute__((no_sanitize("signed-integer-overflow"))) SInt32
3698IOServiceOrdering( const OSMetaClassBase * inObj1, const OSMetaClassBase * inObj2, void * ref )
3699{
3700 const OSObject * obj1 = (const OSObject *) inObj1;
3701 const OSObject * obj2 = (const OSObject *) inObj2;
3702 SInt32 val1;
3703 SInt32 val2;
3704
3705 val1 = 0;
3706 val2 = 0;
3707
3708 if (obj1) {
3709 val1 = IOServiceObjectOrder( entry: obj1, ref );
3710 }
3711
3712 if (obj2) {
3713 val2 = IOServiceObjectOrder( entry: obj2, ref );
3714 }
3715
3716 return val1 - val2;
3717}
3718
3719IOService *
3720IOService::copyClientWithCategory( const OSSymbol * category )
3721{
3722 IOService * service = NULL;
3723 OSIterator * iter;
3724 const OSSymbol * nextCat;
3725
3726 iter = getClientIterator();
3727 if (iter) {
3728 while ((service = (IOService *) iter->getNextObject())) {
3729 if (kIOServiceInactiveState & service->__state[0]) {
3730 if (!(kIOServiceRematchOnDetach & service->__state[1])) {
3731 continue;
3732 }
3733 }
3734 nextCat = (const OSSymbol *) OSDynamicCast( OSSymbol,
3735 service->getProperty( gIOMatchCategoryKey ));
3736 if (category == nextCat) {
3737 service->retain();
3738 break;
3739 }
3740 }
3741 iter->release();
3742 }
3743 return service;
3744}
3745
3746IOService *
3747IOService::getClientWithCategory( const OSSymbol * category )
3748{
3749 IOService *
3750 service = copyClientWithCategory(category);
3751 if (service) {
3752 service->release();
3753 }
3754 return service;
3755}
3756
3757bool
3758IOService::invokeNotifier( _IOServiceNotifier * notify )
3759{
3760 _IOServiceNotifierInvocation invocation;
3761 bool willNotify;
3762 bool ret = true;
3763 invocation.thread = current_thread();
3764
3765#if DEBUG_NOTIFIER_LOCKED
3766 uint32_t count;
3767 if ((count = isLockedForArbitration(0))) {
3768 IOLog("[%s, 0x%x]\n", notify->type->getCStringNoCopy(), count);
3769 panic("[%s, 0x%x]", notify->type->getCStringNoCopy(), count);
3770 }
3771#endif /* DEBUG_NOTIFIER_LOCKED */
3772
3773 LOCKWRITENOTIFY();
3774 willNotify = (0 != (kIOServiceNotifyEnable & notify->state));
3775
3776 if (willNotify) {
3777 queue_enter( &notify->handlerInvocations, &invocation,
3778 _IOServiceNotifierInvocation *, link );
3779 }
3780 UNLOCKNOTIFY();
3781
3782 if (willNotify) {
3783 ret = (*notify->handler)(notify->target, notify->ref, this, notify);
3784
3785 LOCKWRITENOTIFY();
3786 queue_remove( &notify->handlerInvocations, &invocation,
3787 _IOServiceNotifierInvocation *, link );
3788 if (kIOServiceNotifyWaiter & notify->state) {
3789 notify->state &= ~kIOServiceNotifyWaiter;
3790 WAKEUPNOTIFY( notify );
3791 }
3792 UNLOCKNOTIFY();
3793 }
3794
3795 return ret;
3796}
3797
3798bool
3799IOService::invokeNotifiers(OSArray * willSend[])
3800{
3801 OSArray * array;
3802 _IOServiceNotifier * notify;
3803 bool ret = true;
3804
3805 array = *willSend;
3806 if (!array) {
3807 return true;
3808 }
3809 *willSend = NULL;
3810
3811 for (unsigned int idx = 0;
3812 (notify = (_IOServiceNotifier *) array->getObject(index: idx));
3813 idx++) {
3814 ret &= invokeNotifier(notify);
3815 }
3816 array->release();
3817
3818 return ret;
3819}
3820
3821
3822TUNABLE(bool, iokit_print_verbose_match_logs, "iokit_print_verbose_match_logs", false);
3823
3824/*
3825 * Alloc and probe matching classes,
3826 * called on the provider instance
3827 */
3828
3829void
3830IOService::probeCandidates( OSOrderedSet * matches )
3831{
3832 OSDictionary * match = NULL;
3833 OSSymbol * symbol;
3834 IOService * inst;
3835 IOService * newInst;
3836 OSDictionary * props;
3837 SInt32 score;
3838 OSNumber * newPri;
3839 OSOrderedSet * familyMatches = NULL;
3840 OSOrderedSet * startList;
3841 OSSet * kexts = NULL;
3842 OSObject * kextRef;
3843
3844 OSDictionary * startDict = NULL;
3845 const OSSymbol * category;
3846 OSIterator * iter;
3847 _IOServiceNotifier * notify;
3848 OSObject * nextMatch = NULL;
3849 bool started;
3850 bool needReloc = false;
3851 bool matchDeferred = false;
3852#if IOMATCHDEBUG
3853 SInt64 debugFlags;
3854#endif
3855 IOService * client = NULL;
3856 OSObject * prop1;
3857 OSObject * rematchCountProp;
3858 OSDictionary * rematchPersonality;
3859 OSNumber * num;
3860 uint32_t count;
3861 uint32_t dextCount;
3862 bool isDext;
3863 bool categoryConsumed;
3864
3865 rematchCountProp = NULL;
3866 count = 0;
3867 prop1 = copyProperty(aKey: gIORematchPersonalityKey);
3868 rematchPersonality = OSDynamicCast(OSDictionary, prop1);
3869 if (rematchPersonality) {
3870 rematchCountProp = copyProperty(aKey: gIORematchCountKey);
3871 num = OSDynamicCast(OSNumber, rematchCountProp);
3872 if (num) {
3873 count = num->unsigned32BitValue();
3874 }
3875 removeProperty(aKey: gIORematchPersonalityKey);
3876 }
3877 dextCount = 0;
3878
3879 assert( matches );
3880 while (!needReloc
3881 && (nextMatch = matches->getFirstObject())) {
3882 nextMatch->retain();
3883 matches->removeObject(anObject: nextMatch);
3884
3885 if ((notify = OSDynamicCast( _IOServiceNotifier, nextMatch ))) {
3886 if (0 == (__state[0] & kIOServiceInactiveState)) {
3887 invokeNotifier( notify );
3888 }
3889 nextMatch->release();
3890 nextMatch = NULL;
3891 continue;
3892 } else if (!(match = OSDynamicCast( OSDictionary, nextMatch ))) {
3893 nextMatch->release();
3894 nextMatch = NULL;
3895 continue;
3896 }
3897
3898 props = NULL;
3899#if IOMATCHDEBUG
3900 debugFlags = getDebugFlags( props: match );
3901#endif
3902
3903 bool newIsBoot = false;
3904 bool existingIsBoot = false;
3905 bool isReplacementCandidate = false;
3906
3907 do {
3908 client = NULL;
3909 isDext = (NULL != match->getObject(aKey: gIOUserServerNameKey));
3910 if (isDext && !(kIODKEnable & gIODKDebug)) {
3911 continue;
3912 }
3913 if (isDext && !gIODextRelaunchMax && rematchCountProp) {
3914 continue;
3915 }
3916 newIsBoot = gIOCatalogue->personalityIsBoot(match);
3917
3918 category = OSDynamicCast( OSSymbol,
3919 match->getObject( gIOMatchCategoryKey ));
3920 if (NULL == category) {
3921 category = gIODefaultMatchCategoryKey;
3922 }
3923 client = copyClientWithCategory(category);
3924
3925 categoryConsumed = (client != NULL);
3926 if (categoryConsumed) {
3927#if IOMATCHDEBUG
3928 if ((debugFlags & kIOLogMatch) && (this != gIOResources)) {
3929 LOG(fmt: "%s: match category %s exists\n", getName(),
3930 category->getCStringNoCopy());
3931 }
3932#endif
3933 existingIsBoot = client->propertyExists(aKey: gIOMatchedAtBootKey);
3934 isReplacementCandidate = existingIsBoot && !newIsBoot;
3935 if (!isDext && !isReplacementCandidate) {
3936 break;
3937 }
3938 }
3939
3940 // create a copy now in case its modified during matching
3941 props = OSDictionary::withDictionary(dict: match, capacity: match->getCount());
3942 if (NULL == props) {
3943 break;
3944 }
3945 props->setCapacityIncrement(1);
3946
3947 // check the nub matches
3948 if (false == matchPassive(table: props, options: kIOServiceChangesOK | kIOServiceClassDone)) {
3949 break;
3950 }
3951 if (isReplacementCandidate) {
3952 if (canTerminateForReplacement(client)) {
3953 client->terminate(options: kIOServiceTerminateNeedWillTerminate | kIOServiceTerminateWithRematch);
3954 break;
3955 }
3956 }
3957
3958 if (isDext || isReplacementCandidate) {
3959 if (isDext) {
3960 dextCount++;
3961 }
3962 if (categoryConsumed) {
3963 break;
3964 }
3965 }
3966 if (rematchPersonality) {
3967 bool personalityMatch = match->isEqualTo(aDictionary: rematchPersonality);
3968 if (count > gIODextRelaunchMax) {
3969 personalityMatch = !personalityMatch;
3970 }
3971 if (!personalityMatch) {
3972 break;
3973 }
3974 }
3975
3976 // Check to see if driver reloc has been loaded.
3977 needReloc = (false == gIOCatalogue->isModuleLoaded( driver: match, kextRef: &kextRef ));
3978 if (needReloc) {
3979#if IOMATCHDEBUG
3980 if (debugFlags & kIOLogCatalogue) {
3981 LOG(fmt: "%s: stalling for module\n", getName());
3982 }
3983#endif
3984 // If reloc hasn't been loaded, exit;
3985 // reprobing will occur after reloc has been loaded.
3986 break;
3987 }
3988 if (kextRef) {
3989 if (NULL == kexts) {
3990 kexts = OSSet::withCapacity(capacity: 1);
3991 }
3992 if (kexts) {
3993 kexts->setObject(kextRef);
3994 kextRef->release();
3995 }
3996 }
3997 if (newIsBoot) {
3998 props->setObject(aKey: gIOMatchedAtBootKey, anObject: kOSBooleanTrue);
3999 }
4000 if (isDext) {
4001 // copy saved for rematchng
4002 props->setObject(aKey: gIOMatchedPersonalityKey, anObject: match);
4003 }
4004 // reorder on family matchPropertyTable score.
4005 if (NULL == familyMatches) {
4006 familyMatches = OSOrderedSet::withCapacity( capacity: 1,
4007 orderFunc: IOServiceOrdering, orderingContext: (void *) gIOProbeScoreKey );
4008 }
4009 if (familyMatches) {
4010 familyMatches->setObject( props );
4011 }
4012 } while (false);
4013
4014 OSSafeReleaseNULL(client);
4015 OSSafeReleaseNULL(nextMatch);
4016 OSSafeReleaseNULL(props);
4017 }
4018 OSSafeReleaseNULL(matches);
4019 OSSafeReleaseNULL(rematchCountProp);
4020
4021 if (familyMatches) {
4022 while (!needReloc
4023 && (props = (OSDictionary *) familyMatches->getFirstObject())) {
4024 props->retain();
4025 familyMatches->removeObject( anObject: props );
4026
4027 inst = NULL;
4028 newInst = NULL;
4029#if IOMATCHDEBUG
4030 debugFlags = getDebugFlags( props );
4031#endif
4032 do {
4033 symbol = OSDynamicCast( OSSymbol,
4034 props->getObject( gIOClassKey));
4035 if (!symbol) {
4036 continue;
4037 }
4038
4039 //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), IOSERVICE_OBFUSCATE(symbol), IOSERVICE_OBFUSCATE(props));
4040
4041 // alloc the driver instance
4042 inst = (IOService *) OSMetaClass::allocClassWithName( name: symbol);
4043
4044 if (!inst || !OSDynamicCast(IOService, inst)) {
4045 IOLog(format: "Couldn't alloc class \"%s\"\n",
4046 symbol->getCStringNoCopy());
4047 continue;
4048 }
4049
4050 // init driver instance
4051 if (!(inst->init( dictionary: props ))) {
4052#if IOMATCHDEBUG
4053 if (debugFlags & kIOLogStart) {
4054 IOLog(format: "%s::init fails\n", symbol->getCStringNoCopy());
4055 }
4056#endif
4057 continue;
4058 }
4059 if (__state[1] & kIOServiceSynchronousState) {
4060 inst->__state[1] |= kIOServiceSynchronousState;
4061 }
4062
4063 // give the driver the default match category if not specified
4064 category = OSDynamicCast( OSSymbol,
4065 props->getObject( gIOMatchCategoryKey ));
4066 if (NULL == category) {
4067 category = gIODefaultMatchCategoryKey;
4068 }
4069 inst->setProperty( aKey: gIOMatchCategoryKey, anObject: (OSObject *) category );
4070 // attach driver instance
4071 if (!(inst->attach( provider: this ))) {
4072 continue;
4073 }
4074
4075 // pass in score from property table
4076 score = familyMatches->orderObject( anObject: props );
4077
4078 // & probe the new driver instance
4079#if IOMATCHDEBUG
4080 if (debugFlags & kIOLogProbe) {
4081 LOG(fmt: "%s::probe(%s)\n",
4082 inst->getMetaClass()->getClassName(), getName());
4083 }
4084#endif
4085 newInst = inst->probe( provider: this, score: &score );
4086 inst->detach( provider: this );
4087 if (NULL == newInst) {
4088#if IOMATCHDEBUG
4089 if (debugFlags & kIOLogProbe) {
4090 IOLog(format: "%s::probe fails\n", symbol->getCStringNoCopy());
4091 }
4092#endif
4093 continue;
4094 }
4095
4096 // save the score
4097 newPri = OSNumber::withNumber( value: score, numberOfBits: 32 );
4098 if (newPri) {
4099 newInst->setProperty( aKey: gIOProbeScoreKey, anObject: newPri );
4100 newPri->release();
4101 }
4102
4103 // add to start list for the match category
4104 if (NULL == startDict) {
4105 startDict = OSDictionary::withCapacity( capacity: 1 );
4106 }
4107 assert( startDict );
4108 startList = (OSOrderedSet *)
4109 startDict->getObject( aKey: category );
4110 if (NULL == startList) {
4111 startList = OSOrderedSet::withCapacity( capacity: 1,
4112 orderFunc: IOServiceOrdering, orderingContext: (void *) gIOProbeScoreKey );
4113 if (startDict && startList) {
4114 startDict->setObject( aKey: category, anObject: startList );
4115 startList->release();
4116 }
4117 }
4118 assert( startList );
4119 if (startList) {
4120 startList->setObject( newInst );
4121 }
4122 } while (false);
4123
4124 props->release();
4125 if (inst) {
4126 inst->release();
4127 }
4128 }
4129 familyMatches->release();
4130 familyMatches = NULL;
4131 }
4132
4133 if ((debugFlags & kIOLogMatch) && iokit_print_verbose_match_logs && startDict != NULL) {
4134 IOLog(format: "%s(0x%qx): %u categories\n", getName(), getRegistryEntryID(), startList->getCount());
4135 startDict->iterateObjects(block: ^(const OSSymbol *key, OSObject *value) {
4136 OSOrderedSet *startList = OSDynamicCast(OSOrderedSet, value);
4137 if (startList) {
4138 IOLog(format: "%s(0x%qx): category %s, %u matches\n", getName(), getRegistryEntryID(), key->getCStringNoCopy(), startList->getCount());
4139 startList->iterateObjects(block: ^(OSObject *obj) {
4140 IOService *match = OSDynamicCast(IOService, obj);
4141 OSNumber *probeScore = OSDynamicCast(OSNumber, match->getProperty(gIOProbeScoreKey));
4142
4143 if (match && probeScore) {
4144 IOLog(format: "%s(0x%qx): category %s: matched %s, probe score %qd\n", getName(), getRegistryEntryID(), key->getCStringNoCopy(), match->getName(), probeScore->unsigned64BitValue());
4145 }
4146 return false;
4147 });
4148 }
4149 return false;
4150 });
4151 }
4152
4153 // start the best (until success) of each category
4154
4155 iter = OSCollectionIterator::withCollection( inColl: startDict );
4156 assert(startDict || !iter);
4157 if (iter) {
4158 while ((category = (const OSSymbol *) iter->getNextObject())) {
4159 startList = (OSOrderedSet *) startDict->getObject( aKey: category );
4160 assert( startList );
4161 if (!startList) {
4162 continue;
4163 }
4164 started = false;
4165 while (true // (!started)
4166 && !matchDeferred
4167 && (inst = (IOService *)startList->getFirstObject())) {
4168 inst->retain();
4169 startList->removeObject(anObject: inst);
4170#if IOMATCHDEBUG
4171 debugFlags = getDebugFlags( inst );
4172
4173 if (debugFlags & kIOLogStart) {
4174 if (started) {
4175 LOG( fmt: "match category exists, skipping " );
4176 }
4177 LOG( fmt: "%s::start(%s) <%d>\n", inst->getName(),
4178 getName(), inst->getRetainCount());
4179 }
4180#endif
4181 if (false == started) {
4182#if !NO_KEXTD
4183 IOLockLock(gJobsLock);
4184 matchDeferred = (gIOMatchDeferList
4185 && kOSBooleanTrue == inst->getProperty(aKey: gIOMatchDeferKey));
4186 if (matchDeferred && (-1U == gIOMatchDeferList->getNextIndexOfObject(anObject: this, index: 0))) {
4187 gIOMatchDeferList->setObject(this);
4188 }
4189 if (matchDeferred) {
4190 symbol = OSDynamicCast(OSSymbol, inst->getProperty(gIOClassKey));
4191 IOLog(format: "%s(0x%qx): matching deferred by %s%s\n",
4192 getName(), getRegistryEntryID(),
4193 symbol ? symbol->getCStringNoCopy() : "",
4194 gInUserspaceReboot ? " in userspace reboot" : "");
4195 // rematching will occur after the IOKit daemon loads all plists
4196 }
4197 IOLockUnlock(gJobsLock);
4198#endif
4199 if (!matchDeferred) {
4200 /* TODO
4201 * If a dext fails to start because an upgrade happened
4202 * concurrently, then the matching process has to restart
4203 */
4204 started = startCandidate( candidate: inst );
4205#if IOMATCHDEBUG
4206 if ((debugFlags & kIOLogStart) && (false == started)) {
4207 LOG( fmt: "%s::start(%s) <%d> failed\n", inst->getName(), getName(),
4208 inst->getRetainCount());
4209 }
4210#endif
4211 if (!started && inst->propertyExists(aKey: gIOServiceMatchDeferredKey)) {
4212 matchDeferred = true;
4213 }
4214 }
4215 }
4216 inst->release();
4217 }
4218 }
4219 iter->release();
4220 }
4221
4222 OSSafeReleaseNULL(prop1);
4223
4224 if (dextCount) {
4225 num = OSNumber::withNumber(value: dextCount, numberOfBits: 32);
4226 setProperty(aKey: gIODEXTMatchCountKey, anObject: num);
4227 OSSafeReleaseNULL(num);
4228 } else if (rematchPersonality) {
4229 removeProperty(aKey: gIODEXTMatchCountKey);
4230 }
4231
4232 // now that instances are created, drop the refs on any kexts allowing unload
4233 if (kexts) {
4234 OSKext::dropMatchingReferences(kexts);
4235 OSSafeReleaseNULL(kexts);
4236 }
4237
4238 // adjust the busy count by +1 if matching is stalled for a module,
4239 // or -1 if a previously stalled matching is complete.
4240 lockForArbitration();
4241 SInt32 adjBusy = 0;
4242 uint64_t regID = getRegistryEntryID();
4243
4244 if (needReloc) {
4245 adjBusy = (__state[1] & kIOServiceModuleStallState) ? 0 : 1;
4246 if (adjBusy) {
4247 IOServiceTrace(
4248 IOSERVICE_MODULESTALL,
4249 (uintptr_t) regID,
4250 (uintptr_t) (regID >> 32),
4251 (uintptr_t) this,
4252 0);
4253
4254 __state[1] |= kIOServiceModuleStallState;
4255 }
4256 } else if (__state[1] & kIOServiceModuleStallState) {
4257 IOServiceTrace(
4258 IOSERVICE_MODULEUNSTALL,
4259 (uintptr_t) regID,
4260 (uintptr_t) (regID >> 32),
4261 (uintptr_t) this,
4262 0);
4263
4264 __state[1] &= ~kIOServiceModuleStallState;
4265 adjBusy = -1;
4266 }
4267 if (adjBusy) {
4268 _adjustBusy( delta: adjBusy );
4269 }
4270 unlockForArbitration();
4271
4272 if (startDict) {
4273 startDict->release();
4274 }
4275}
4276
4277/*
4278 * Wait for a IOUserServer to check in
4279 */
4280
4281static
4282__attribute__((noinline, not_tail_called))
4283IOUserServer *
4284__WAITING_FOR_USER_SERVER__(IOUserServerCheckInToken * token)
4285{
4286 IOUserServer * result = NULL;
4287 IOService * server = NULL;
4288 const OSSymbol * serverName = token->copyServerName();
4289 OSNumber * serverTag = token->copyServerTag();
4290 OSDictionary * matching = IOService::serviceMatching(className: gIOUserServerClassKey);
4291
4292 if (!matching || !serverName || !serverTag) {
4293 goto finish;
4294 }
4295 IOService::propertyMatching(key: gIOUserServerNameKey, value: serverName, table: matching);
4296 if (!(kIODKDisableDextTag & gIODKDebug)) {
4297 IOService::propertyMatching(key: gIOUserServerTagKey, value: serverTag, table: matching);
4298 }
4299
4300 server = IOService::waitForMatchingServiceWithToken(matching, timeout: kIOUserServerCheckInTimeoutSecs * NSEC_PER_SEC, token);
4301 result = OSDynamicCast(IOUserServer, server);
4302 if (!result) {
4303 OSSafeReleaseNULL(server);
4304 token->cancel();
4305 }
4306
4307finish:
4308 OSSafeReleaseNULL(matching);
4309 OSSafeReleaseNULL(serverName);
4310 OSSafeReleaseNULL(serverTag);
4311
4312 return result;
4313}
4314
4315void
4316IOService::willShutdown()
4317{
4318 gIOKitWillTerminate = true;
4319#if !NO_KEXTD
4320 IOUserServerCheckInToken::cancelAll();
4321#endif
4322 OSKext::willShutdown();
4323}
4324
4325void
4326IOService::userSpaceWillReboot()
4327{
4328 IOLockLock(gJobsLock);
4329#if !NO_KEXTD
4330 IOService * provider;
4331 IOService * service;
4332 OSIterator * iter;
4333
4334 // Recreate the defer list if it does not exist
4335 if (!gIOMatchDeferList && OSKext::iokitDaemonAvailable()) {
4336 gIOMatchDeferList = OSArray::withCapacity( capacity: 16 );
4337 }
4338
4339 if (gIOMatchDeferList) {
4340 iter = IORegistryIterator::iterateOver(plane: gIOServicePlane, options: kIORegistryIterateRecursively);
4341 if (iter) {
4342 do {
4343 iter->reset();
4344 while ((service = (IOService *)iter->getNextObject())) {
4345 /* Rematch providers of services that will be terminated on userspace reboot, after the userspace reboot
4346 * is complete. This normally happens automatically as the IOKit daemon sends personalities to the kernel
4347 * which triggers rematching. But if this doesn't happen (for example, if a feature flag is turned off),
4348 * then these services will never get rematched.
4349 */
4350 if (service->propertyHasValue(aKey: gIOMatchDeferKey, value: kOSBooleanTrue) || service->hasUserServer()) {
4351 provider = service->getProvider();
4352 IOLog(format: "deferring %s-%llx (provider of %s-%llx) matching after userspace reboot\n",
4353 provider->getName(), provider->getRegistryEntryID(), service->getName(), service->getRegistryEntryID());
4354 gIOMatchDeferList->setObject(provider);
4355 }
4356 }
4357 } while (!service && !iter->isValid());
4358
4359 OSSafeReleaseNULL(iter);
4360 }
4361 }
4362#endif
4363 gInUserspaceReboot = true;
4364 IOLockUnlock(gJobsLock);
4365}
4366
4367void
4368IOService::userSpaceDidReboot()
4369{
4370 IOLockLock(gJobsLock);
4371 gInUserspaceReboot = false;
4372 IOLockUnlock(gJobsLock);
4373}
4374
4375/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4376
4377void
4378IOServicePH::init(IOPMrootDomain * root)
4379{
4380 fUserServers = OSArray::withCapacity(capacity: 4);
4381 fMatchingWork = OSArray::withCapacity(capacity: 4);
4382
4383 assert(fUserServers && fMatchingWork);
4384
4385 fRootNotifier = root->registerInterest(
4386 typeOfInterest: gIOPriorityPowerStateInterest, handler: &IOServicePH::systemPowerChange, NULL, NULL);
4387
4388 assert(fRootNotifier);
4389
4390 fUserServerAckTimer = thread_call_allocate(func: &IOServicePH::userServerAckTimerExpired, param0: (thread_call_param_t)NULL);
4391}
4392
4393void
4394IOServicePH::lock()
4395{
4396 IOLockLock(gJobsLock);
4397}
4398
4399void
4400IOServicePH::unlock()
4401{
4402 IOLockUnlock(gJobsLock);
4403}
4404
4405void
4406IOServicePH::serverAdd(IOUserServer * server)
4407{
4408 uint32_t idx;
4409
4410 lock();
4411 idx = fUserServers->getNextIndexOfObject(anObject: server, index: 0);
4412 if (idx == -1U) {
4413 fUserServers->setObject(server);
4414 }
4415 unlock();
4416}
4417
4418void
4419IOServicePH::serverRemove(IOUserServer * server)
4420{
4421 uint32_t idx;
4422
4423 lock();
4424 idx = fUserServers->getNextIndexOfObject(anObject: server, index: 0);
4425 if (idx != -1U) {
4426 fUserServers->removeObject(index: idx);
4427 }
4428
4429 if (fWaitingUserServers) {
4430 fWaitingUserServers = false;
4431 IOLockWakeup(lock: gJobsLock, event: &fWaitingUserServers, /* one-thread */ oneThread: false);
4432 }
4433
4434 unlock();
4435}
4436
4437void
4438IOServicePH::serverAck(IOUserServer * server)
4439{
4440 uint32_t idx;
4441 IOService * ackTo;
4442 uint32_t ackToRef;
4443
4444 ackTo = NULL;
4445 lock();
4446 if (server && fUserServersWait) {
4447 idx = fUserServersWait->getNextIndexOfObject(anObject: server, index: 0);
4448 if (idx != -1U) {
4449 fUserServersWait->removeObject(index: idx);
4450 if (0 == fUserServersWait->getCount()) {
4451 OSSafeReleaseNULL(fUserServersWait);
4452 }
4453 }
4454 }
4455 if (!fUserServersWait && !fMatchingWork->getCount()) {
4456 ackTo = fSystemPowerAckTo;
4457 ackToRef = fSystemPowerAckRef;
4458 fSystemPowerAckTo = NULL;
4459 if (ackTo) {
4460 thread_call_cancel(call: fUserServerAckTimer);
4461 }
4462 }
4463 if (fUserServersWait && fUserServersWait->getCount() > 0 && fMatchingWork && fMatchingWork->getCount() > 0) {
4464 DKLOG("Waiting for %u user servers, %u matching work\n", fUserServersWait->getCount(), fMatchingWork->getCount());
4465 }
4466 unlock();
4467
4468 if (ackTo) {
4469 DKLOG("allowPowerChange\n");
4470 ackTo->allowPowerChange(refcon: (uintptr_t) ackToRef);
4471 }
4472}
4473
4474bool
4475IOServicePH::matchingStart(IOService * service)
4476{
4477 uint32_t idx;
4478 bool assertionActive = gIOPMRootDomain->acquireDriverKitMatchingAssertion() == kIOReturnSuccess;
4479
4480 lock();
4481 bool matchNow = !fSystemOff && assertionActive;
4482 if (matchNow) {
4483 idx = fMatchingWork->getNextIndexOfObject(anObject: service, index: 0);
4484 if (idx == -1U) {
4485 fMatchingWork->setObject(service);
4486 }
4487 } else {
4488 // Delay matching if system is transitioning to sleep
4489 if (!fMatchingDelayed) {
4490 fMatchingDelayed = OSArray::withObjects(objects: (const OSObject **) &service, count: 1, capacity: 1);
4491 } else {
4492 idx = fMatchingDelayed->getNextIndexOfObject(anObject: service, index: 0);
4493 if (idx == -1U) {
4494 fMatchingDelayed->setObject(service);
4495 }
4496 }
4497 }
4498 unlock();
4499 if (!matchNow && assertionActive) {
4500 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
4501 }
4502
4503 return matchNow;
4504}
4505
4506void
4507IOServicePH::matchingEnd(IOService * service)
4508{
4509 uint32_t idx;
4510 OSArray * notifyServers;
4511 OSArray * deferredMatches;
4512
4513 notifyServers = NULL;
4514 deferredMatches = NULL;
4515
4516 if (service) {
4517 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
4518 }
4519
4520 lock();
4521
4522 if (service) {
4523 idx = fMatchingWork->getNextIndexOfObject(anObject: service, index: 0);
4524 if (idx != -1U) {
4525 fMatchingWork->removeObject(index: idx);
4526 }
4527 }
4528
4529
4530 if ((fUserServerOff != fSystemOff) && fUserServers->getCount()) {
4531 if (fSystemOff) {
4532 if (0 == fMatchingWork->getCount()) {
4533 fUserServersWait = OSArray::withArray(array: fUserServers);
4534 notifyServers = OSArray::withArray(array: fUserServers);
4535 fUserServerOff = fSystemOff;
4536 }
4537 } else {
4538 notifyServers = OSArray::withArray(array: fUserServers);
4539 fUserServerOff = fSystemOff;
4540 }
4541 }
4542
4543 if (!fSystemOff && fMatchingDelayed) {
4544 deferredMatches = fMatchingDelayed;
4545 fMatchingDelayed = NULL;
4546 }
4547
4548 unlock();
4549
4550 if (notifyServers) {
4551 notifyServers->iterateObjects(block: ^bool (OSObject * obj) {
4552 IOUserServer * us;
4553 us = (typeof(us))obj;
4554 us->systemPower(powerOff: fSystemOff);
4555 return false;
4556 });
4557 OSSafeReleaseNULL(notifyServers);
4558 }
4559
4560 if (deferredMatches) {
4561 DKLOG("sleep deferred rematching count %d\n", deferredMatches->getCount());
4562 deferredMatches->iterateObjects(block: ^bool (OSObject * obj)
4563 {
4564 ((IOService *)obj)->startMatching(options: kIOServiceAsynchronous);
4565 return false;
4566 });
4567 deferredMatches->release();
4568 }
4569
4570 serverAck(NULL);
4571}
4572
4573TUNABLE(uint32_t, dk_shutdown_timeout_ms, "dk_shutdown_timeout_ms", 5000);
4574TUNABLE(bool, dk_panic_on_shutdown_hang, "dk_panic_on_shutdown_hang", false);
4575TUNABLE(bool, dk_panic_on_setpowerstate_hang, "dk_panic_on_setpowerstate_hang", false);
4576
4577void
4578IOServicePH::userServerAckTimerExpired(void *, void *)
4579{
4580 OSArray * userServers = NULL;
4581 lock();
4582 if (fSystemPowerAckTo) {
4583 DKLOG("ack timer expired\n");
4584 if (dk_panic_on_setpowerstate_hang) {
4585 panic("DK ack timer expired after %u ms", dk_shutdown_timeout_ms);
4586 }
4587 userServers = fUserServersWait;
4588 fUserServersWait = NULL;
4589 }
4590 unlock();
4591
4592 if (userServers != NULL) {
4593 userServers->iterateObjects(block: ^bool (OSObject *obj) {
4594 IOUserServer * us = OSDynamicCast(IOUserServer, obj);
4595 if (us) {
4596 DKLOG(DKS " power state transition failed\n", DKN(us));
4597 us->kill(reason: "Power Management Failed");
4598 }
4599 return false;
4600 });
4601 OSSafeReleaseNULL(userServers);
4602 }
4603
4604 serverAck(NULL);
4605}
4606
4607void
4608IOServicePH::systemHalt(int howto)
4609{
4610 OSArray * notifyServers;
4611 uint64_t deadline;
4612
4613 lock();
4614 notifyServers = OSArray::withArray(array: fUserServers);
4615 unlock();
4616
4617 if (notifyServers) {
4618 notifyServers->iterateObjects(block: ^bool (OSObject * obj) {
4619 IOUserServer * us;
4620 us = (typeof(us))obj;
4621 us->systemHalt(howto);
4622 return false;
4623 });
4624 OSSafeReleaseNULL(notifyServers);
4625 }
4626
4627 lock();
4628 clock_interval_to_deadline(interval: dk_shutdown_timeout_ms, scale_factor: kMillisecondScale, result: &deadline);
4629 while (0 < fUserServers->getCount()) {
4630 fWaitingUserServers = true;
4631 __assert_only int waitResult =
4632 IOLockSleepDeadline(lock: gJobsLock, event: &fWaitingUserServers, deadline, THREAD_UNINT);
4633 assert((THREAD_AWAKENED == waitResult) || (THREAD_TIMED_OUT == waitResult));
4634 if (THREAD_TIMED_OUT == waitResult) {
4635 IOUserServer::beginLeakingObjects();
4636#if DEVELOPMENT || DEBUG
4637 if (dk_panic_on_shutdown_hang) {
4638 panic("Shutdown timed out waiting for DK drivers to stop");
4639 }
4640#endif /* DEVELOPMENT || DEBUG */
4641 break;
4642 }
4643 }
4644 unlock();
4645}
4646
4647bool
4648IOServicePH::serverSlept(void)
4649{
4650 bool ret;
4651
4652 lock();
4653 ret = (kIOMessageSystemWillSleep == sSystemPower)
4654 || (kIOMessageSystemWillPowerOff == sSystemPower)
4655 || (kIOMessageSystemWillRestart == sSystemPower);
4656 unlock();
4657
4658 return ret;
4659}
4660
4661TUNABLE(uint32_t, dk_power_state_timeout_ms, "dk_power_state_timeout_ms", 30000);
4662
4663IOReturn
4664IOServicePH::systemPowerChange(
4665 void * target,
4666 void * refCon,
4667 UInt32 messageType, IOService * service,
4668 void * messageArgument, vm_size_t argSize)
4669{
4670 IOReturn ret;
4671 IOUserServer * us;
4672 IOPMSystemCapabilityChangeParameters * params;
4673 AbsoluteTime deadline;
4674
4675 us = NULL;
4676
4677 switch (messageType) {
4678 case kIOMessageSystemCapabilityChange:
4679
4680 params = (typeof params)messageArgument;
4681
4682 if (kIODKLogPM & gIODKDebug) {
4683 IOLog(format: "IOServicePH::kIOMessageSystemCapabilityChange: %s%s 0x%x->0x%x\n",
4684 params->changeFlags & kIOPMSystemCapabilityWillChange ? "will" : "",
4685 params->changeFlags & kIOPMSystemCapabilityDidChange ? "did" : "",
4686 params->fromCapabilities,
4687 params->toCapabilities);
4688 }
4689
4690 if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4691 (params->fromCapabilities & kIOPMSystemCapabilityCPU) &&
4692 ((params->toCapabilities & kIOPMSystemCapabilityCPU) == 0)) {
4693 lock();
4694 DKLOG("arming ack timer, %u ms\n", dk_power_state_timeout_ms);
4695 clock_interval_to_deadline(interval: dk_power_state_timeout_ms, scale_factor: kMillisecondScale, result: &deadline);
4696 fSystemOff = true;
4697 fSystemPowerAckRef = params->notifyRef;
4698 fSystemPowerAckTo = service;
4699 thread_call_enter_delayed(call: fUserServerAckTimer, deadline);
4700 unlock();
4701
4702 matchingEnd(NULL);
4703
4704 params->maxWaitForReply = dk_power_state_timeout_ms * 2 * 1000;
4705 ret = kIOReturnSuccess;
4706 } else if ((params->changeFlags & kIOPMSystemCapabilityWillChange) &&
4707 ((params->fromCapabilities & kIOPMSystemCapabilityCPU) == 0) &&
4708 (params->toCapabilities & kIOPMSystemCapabilityCPU)) {
4709 lock();
4710 fSystemOff = false;
4711 unlock();
4712
4713 matchingEnd(NULL);
4714
4715 params->maxWaitForReply = 0;
4716 ret = kIOReturnSuccess;
4717 } else {
4718 params->maxWaitForReply = 0;
4719 ret = kIOReturnSuccess;
4720 }
4721 break;
4722
4723 default:
4724 ret = kIOReturnUnsupported;
4725 break;
4726 }
4727
4728 return ret;
4729}
4730
4731bool
4732IOServicePH::checkPMReady(void)
4733{
4734 bool __block ready = true;
4735
4736 lock();
4737 fUserServers->iterateObjects(block: ^bool (OSObject *obj) {
4738 IOUserServer * us = OSDynamicCast(IOUserServer, obj);
4739 if (us) {
4740 if (!us->checkPMReady()) {
4741 ready = false;
4742 return true;
4743 }
4744 }
4745 return false;
4746 });
4747 unlock();
4748
4749 return ready;
4750}
4751
4752/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4753
4754/*
4755 * Start a previously attached & probed instance,
4756 * called on exporting object instance
4757 */
4758
4759bool
4760IOService::startCandidate( IOService * service )
4761{
4762 bool ok;
4763 OSObject * obj;
4764 OSObject * prop;
4765 IOUserServer * userServer;
4766 bool ph;
4767
4768 userServer = NULL;
4769 obj = service->copyProperty(aKey: gIOUserServerNameKey);
4770
4771 if (obj && (this == gIOResources)) {
4772 ok = false;
4773 } else {
4774 ok = service->attach( provider: this );
4775 }
4776 if (!ok) {
4777 OSSafeReleaseNULL(obj);
4778 return false;
4779 }
4780
4781 if ((this != gIOResources) && (this != gIOUserResources)) {
4782 // stall for any nub resources
4783 checkResources();
4784 // stall for any driver resources
4785 service->checkResources();
4786 }
4787 ph = false;
4788 {
4789 OSString * bundleID;
4790 OSString * serverName;
4791 OSString * str;
4792 const OSSymbol * sym;
4793 OSNumber * serverTag;
4794 uint64_t entryID;
4795 IOUserServerCheckInToken * token;
4796 OSData * serverDUI;
4797
4798 if ((serverName = OSDynamicCast(OSString, obj))) {
4799 obj = service->copyProperty(aKey: gIOModuleIdentifierKey);
4800 bundleID = OSDynamicCast(OSString, obj);
4801 entryID = service->getRegistryEntryID();
4802 serverTag = OSNumber::withNumber(value: entryID, numberOfBits: 64);
4803 token = NULL;
4804
4805 if (kIODKDisableDextLaunch & gIODKDebug) {
4806 DKLOG(DKS " dext launches are disabled \n", DKN(service));
4807 service->detach(provider: this);
4808 OSSafeReleaseNULL(serverName);
4809 OSSafeReleaseNULL(obj);
4810 OSSafeReleaseNULL(serverTag);
4811 return false;
4812 }
4813
4814 if (gIOKitWillTerminate) {
4815 DKLOG("%s disabled in shutdown\n", serverName->getCStringNoCopy());
4816 service->detach(provider: this);
4817 OSSafeReleaseNULL(serverName);
4818 OSSafeReleaseNULL(obj);
4819 OSSafeReleaseNULL(serverTag);
4820 return false;
4821 }
4822
4823 ph = IOServicePH::matchingStart(service: this);
4824 if (!ph) {
4825 DKLOG("%s deferred in sleep\n", serverName->getCStringNoCopy());
4826 service->setProperty(aKey: gIOServiceMatchDeferredKey, anObject: kOSBooleanTrue);
4827 service->detach(provider: this);
4828 OSSafeReleaseNULL(serverName);
4829 OSSafeReleaseNULL(obj);
4830 OSSafeReleaseNULL(serverTag);
4831 return false;
4832 }
4833
4834 prop = service->copyProperty(aKey: gIOUserClassKey);
4835 str = OSDynamicCast(OSString, prop);
4836 if (str) {
4837 service->setName(name: str);
4838 }
4839 OSSafeReleaseNULL(prop);
4840
4841 sym = OSSymbol::withString(aString: serverName);
4842 bool reuse = service->propertyExists(aKey: gIOUserServerOneProcessKey);
4843 serverDUI = OSDynamicCast(OSData, service->getProperty(kOSBundleDextUniqueIdentifierKey));
4844 userServer = IOUserServer::launchUserServer(bundleID, serverName: sym, serverTag, reuseIfExists: reuse, token: &token, serverDUI);
4845 OSSafeReleaseNULL(sym);
4846 OSSafeReleaseNULL(serverTag);
4847 OSSafeReleaseNULL(serverName);
4848 if (userServer) {
4849 DKLOG(DKS " using existing server " DKS "\n", DKN(service), DKN(userServer));
4850 } else if (token != NULL) {
4851 const OSSymbol * tokenServerName = token->copyServerName();
4852 OSNumber * tokenServerTag = token->copyServerTag();
4853 assert(tokenServerName && tokenServerTag);
4854 DKLOG(DKS " waiting for server %s-%llx\n", DKN(service), tokenServerName->getCStringNoCopy(), tokenServerTag->unsigned64BitValue());
4855 userServer = __WAITING_FOR_USER_SERVER__(token);
4856 OSSafeReleaseNULL(tokenServerName);
4857 OSSafeReleaseNULL(tokenServerTag);
4858 } else {
4859 DKLOG(DKS " failed to launch server\n", DKN(service));
4860 }
4861
4862
4863 if (!userServer) {
4864 service->detach(provider: this);
4865 IOServicePH::matchingEnd(service: this);
4866 OSSafeReleaseNULL(obj);
4867
4868 if (token != NULL) {
4869 DKLOG(DKS " user server timeout\n", DKN(service));
4870#if DEVELOPMENT || DEBUG
4871 driverkit_checkin_timed_out = mach_absolute_time();
4872#endif
4873 }
4874
4875 OSSafeReleaseNULL(token);
4876 return false;
4877 }
4878
4879 if (token && !(kIODKDisableCheckInTokenVerification & gIODKDebug)) {
4880 if (!userServer->serviceMatchesCheckInToken(token)) {
4881 OSSafeReleaseNULL(token);
4882 service->detach(provider: this);
4883 IOServicePH::matchingEnd(service: this);
4884 OSSafeReleaseNULL(obj);
4885 userServer->exit(reason: "Check In Token verification failed");
4886 userServer->release();
4887 return false;
4888 }
4889 }
4890 OSSafeReleaseNULL(token);
4891 OSSafeReleaseNULL(obj);
4892
4893 if (!(kIODKDisableEntitlementChecking & gIODKDebug)) {
4894 if (!userServer->checkEntitlements(provider: this, dext: service)) {
4895 service->detach(provider: this);
4896 IOServicePH::matchingEnd(service: this);
4897 userServer->exit(reason: "Entitlements check failed");
4898 userServer->release();
4899 return false;
4900 }
4901 }
4902#if !XNU_TARGET_OS_OSX && !(DEVELOPMENT || DEBUG)
4903 // Prevent third party drivers from matching IOUserResources/IOResources, except when signed for development
4904 if (!userServer->isPlatformDriver() &&
4905 userServer->getCSValidationCategory() != CS_VALIDATION_CATEGORY_DEVELOPMENT
4906 && (this == gIOUserResources || this == gIOResources)) {
4907 service->detach(this);
4908 IOServicePH::matchingEnd(this);
4909 userServer->exit("Third party driver may only match real hardware");
4910 userServer->release();
4911 return false;
4912 }
4913#endif /* !XNU_TARGET_OS_OSX && !(DEVELOPMENT || DEBUG) */
4914
4915 userServer->serviceAttach(service, provider: this);
4916 } else {
4917 OSSafeReleaseNULL(obj);
4918 }
4919 }
4920
4921 AbsoluteTime startTime;
4922 AbsoluteTime endTime;
4923 UInt64 nano;
4924 bool recordTime = (kIOLogStart & gIOKitDebug) != 0;
4925
4926 if (recordTime) {
4927 clock_get_uptime(result: &startTime);
4928 }
4929
4930 ok = service->start(provider: this);
4931
4932 if (recordTime) {
4933 clock_get_uptime(result: &endTime);
4934
4935 if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0) {
4936 SUB_ABSOLUTETIME(&endTime, &startTime);
4937 absolutetime_to_nanoseconds(abstime: endTime, result: &nano);
4938 if (nano > 500000000ULL) {
4939 IOLog(format: "%s::start took %ld ms\n", service->getName(), (long)(UInt32)(nano / 1000000ULL));
4940 }
4941 }
4942 }
4943 if (userServer) {
4944 userServer->serviceStarted(service, provider: this, result: ok);
4945 userServer->release();
4946 }
4947
4948 if (ok) {
4949 IOInstallServiceSleepPlatformActions(service);
4950#if 00
4951 if (!strcmp("XHC1", getName())) {
4952 service->setProperty(gIOPrimaryDriverTerminateOptionsKey, kOSBooleanTrue);
4953 }
4954#endif
4955 }
4956
4957 if (!ok) {
4958 service->detach( provider: this );
4959 }
4960
4961 if (ph) {
4962 IOServicePH::matchingEnd(service: this);
4963 }
4964
4965 return ok;
4966}
4967
4968void
4969IOService::publishResource( const char * key, OSObject * value )
4970{
4971 const OSSymbol * sym;
4972
4973 if ((sym = OSSymbol::withCString( cString: key))) {
4974 publishResource( key: sym, value);
4975 sym->release();
4976 }
4977}
4978
4979void
4980IOService::publishResource( const OSSymbol * key, OSObject * value )
4981{
4982 if (NULL == value) {
4983 value = (OSObject *) gIOServiceKey;
4984 }
4985
4986 gIOResources->setProperty( aKey: key, anObject: value);
4987
4988 if (IORecursiveLockHaveLock( lock: gNotificationLock)) {
4989 return;
4990 }
4991
4992 gIOResourceGenerationCount++;
4993 gIOResources->registerService();
4994}
4995
4996void
4997IOService::publishUserResource( const OSSymbol * key, OSObject * value )
4998{
4999 if (NULL == value) {
5000 value = (OSObject *) gIOServiceKey;
5001 }
5002
5003 gIOUserResources->setProperty( aKey: key, anObject: value);
5004
5005 if (IORecursiveLockHaveLock( lock: gNotificationLock)) {
5006 return;
5007 }
5008
5009 gIOResourceGenerationCount++;
5010 gIOUserResources->registerService();
5011}
5012
5013bool
5014IOService::addNeededResource( const char * key )
5015{
5016 OSObject * resourcesProp;
5017 OSSet * set;
5018 OSString * newKey;
5019 bool ret;
5020
5021 resourcesProp = copyProperty( aKey: gIOResourceMatchKey );
5022 if (!resourcesProp) {
5023 return false;
5024 }
5025
5026 newKey = OSString::withCString( cString: key );
5027 if (!newKey) {
5028 resourcesProp->release();
5029 return false;
5030 }
5031
5032 set = OSDynamicCast( OSSet, resourcesProp );
5033 if (!set) {
5034 set = OSSet::withCapacity( capacity: 1 );
5035 set->setObject( resourcesProp );
5036 } else {
5037 set->retain();
5038 }
5039
5040 set->setObject( newKey );
5041 newKey->release();
5042 ret = setProperty( aKey: gIOResourceMatchKey, anObject: set );
5043 set->release();
5044 resourcesProp->release();
5045
5046 return ret;
5047}
5048
5049bool
5050IOService::checkResource( OSObject * matching )
5051{
5052 OSString * str;
5053 OSDictionary * table;
5054
5055 if ((str = OSDynamicCast( OSString, matching ))) {
5056 if (gIOResources->getProperty( aKey: str )) {
5057 return true;
5058 }
5059 }
5060
5061 if (str) {
5062 table = resourceMatching( name: str );
5063 } else if ((table = OSDynamicCast( OSDictionary, matching ))) {
5064 table->retain();
5065 } else {
5066 IOLog(format: "%s: Can't match using: %s\n", getName(),
5067 matching->getMetaClass()->getClassName());
5068 /* false would stall forever */
5069 return true;
5070 }
5071
5072 if (gIOKitDebug & kIOLogConfig) {
5073 LOG(fmt: "config(%p): stalling %s\n", IOSERVICE_OBFUSCATE(IOThreadSelf()), getName());
5074 }
5075
5076 waitForService( matching: table );
5077
5078 if (gIOKitDebug & kIOLogConfig) {
5079 LOG(fmt: "config(%p): waking\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
5080 }
5081
5082 return true;
5083}
5084
5085bool
5086IOService::checkResources( void )
5087{
5088 OSObject * resourcesProp;
5089 OSSet * set;
5090 OSObject * obj;
5091 OSIterator * iter;
5092 bool ok;
5093
5094 resourcesProp = copyProperty( aKey: gIOResourceMatchKey );
5095 if (NULL == resourcesProp) {
5096 return true;
5097 }
5098
5099 if ((set = OSDynamicCast( OSSet, resourcesProp ))) {
5100 iter = OSCollectionIterator::withCollection( inColl: set );
5101 ok = (NULL != iter);
5102 while (ok && (obj = iter->getNextObject())) {
5103 ok = checkResource( matching: obj );
5104 }
5105 if (iter) {
5106 iter->release();
5107 }
5108 } else {
5109 ok = checkResource( matching: resourcesProp );
5110 }
5111
5112 OSSafeReleaseNULL(resourcesProp);
5113
5114 return ok;
5115}
5116
5117
5118void
5119_IOConfigThread::configThread( const char * name )
5120{
5121 _IOConfigThread * inst;
5122
5123 do {
5124 if (!(inst = new _IOConfigThread)) {
5125 continue;
5126 }
5127 if (!inst->init()) {
5128 continue;
5129 }
5130 thread_t thread;
5131 if (KERN_SUCCESS != kernel_thread_start(continuation: &_IOConfigThread::main, parameter: inst, new_thread: &thread)) {
5132 continue;
5133 }
5134
5135 char threadName[MAXTHREADNAMESIZE];
5136 snprintf(threadName, count: sizeof(threadName), "IOConfigThread_'%s'", name);
5137 thread_set_thread_name(th: thread, name: threadName);
5138 thread_deallocate(thread);
5139
5140 return;
5141 } while (false);
5142
5143 if (inst) {
5144 inst->release();
5145 }
5146
5147 return;
5148}
5149
5150/*
5151 * To support driver replacement of boot matched drivers later in boot, drivers can
5152 * opt-in to be being terminated if a non-boot driver matches their provider, by
5153 * setting the gIOPrimaryDriverTerminateOptionsKey property. The driver providing the
5154 * root disk media may not be terminated.
5155 * IOMedia objects are hidden from user space until all drivers are available, but any
5156 * associated with the root disk must be published immediately.
5157 */
5158
5159struct FindRootMediaContext {
5160 OSArray * services;
5161 IOService * parent;
5162};
5163
5164bool
5165IOService::hasParent(IOService * parent)
5166{
5167 IOService * service;
5168
5169 for (service = this;
5170 service && (service != parent);
5171 service = service->getProvider()) {
5172 }
5173
5174 return service != NULL;
5175}
5176
5177bool
5178IOService::publishHiddenMediaApplier(const OSObject * entry, void * context)
5179{
5180 FindRootMediaContext * ctx = (typeof(ctx))context;
5181 IOService * service = (typeof(service))entry;
5182
5183 do {
5184 if (ctx->parent && !service->hasParent(parent: ctx->parent)) {
5185 break;
5186 }
5187 if (ctx->services) {
5188 ctx->services->setObject(service);
5189 } else {
5190 ctx->services = OSArray::withObjects(objects: (const OSObject **) &service, count: 1);
5191 assert(ctx->services);
5192 }
5193 } while (false);
5194
5195 return false;
5196}
5197
5198// publish to user space any hidden IOMedia under the 'parent' object, or all
5199// if 'parent' is NULL
5200
5201void
5202IOService::publishHiddenMedia(IOService * parent)
5203{
5204 const OSMetaClass * iomediaClass;
5205 bool wasHiding;
5206
5207 iomediaClass = OSMetaClass::getMetaClassWithName(name: gIOMediaKey);
5208 assert(iomediaClass);
5209
5210 LOCKWRITENOTIFY();
5211 wasHiding = gIOServiceHideIOMedia;
5212 if (wasHiding && !parent) {
5213 gIOServiceHideIOMedia = false;
5214 }
5215 UNLOCKNOTIFY();
5216
5217 FindRootMediaContext ctx = { .services = NULL, .parent = parent };
5218
5219 if (wasHiding) {
5220 iomediaClass->applyToInstances(applier: publishHiddenMediaApplier, context: &ctx);
5221 }
5222 if (ctx.services) {
5223 unsigned int idx, notiIdx;
5224 IOService * service;
5225 OSArray * notifiers[3] = {};
5226
5227 for (idx = 0; (service = (IOService *) ctx.services->getObject(index: idx)); idx++) {
5228 service->lockForArbitration(isSuccessRequired: true);
5229 if (!(kIOServiceUserInvisibleMatchState & service->__state[0])) {
5230 service->unlockForArbitration();
5231 continue;
5232 }
5233 service->__state[0] &= ~kIOServiceUserInvisibleMatchState;
5234 service->__state[1] |= kIOServiceUserUnhidden;
5235 notifiers[0] = service->copyNotifiers(type: gIOFirstPublishNotification, orNewState: 0, andNewState: 0xffffffff);
5236 if (kIOServiceMatchedState & service->__state[0]) {
5237 notifiers[1] = service->copyNotifiers(type: gIOMatchedNotification, orNewState: 0, andNewState: 0xffffffff);
5238 }
5239 if (kIOServiceFirstMatchState & service->__state[0]) {
5240 notifiers[2] = service->copyNotifiers(type: gIOFirstMatchNotification, orNewState: 0, andNewState: 0xffffffff);
5241 }
5242 service->unlockForArbitration();
5243 for (notiIdx = 0; notiIdx < 3; notiIdx++) {
5244 service->invokeNotifiers(willSend: &notifiers[notiIdx]);
5245 }
5246 }
5247 OSSafeReleaseNULL(ctx.services);
5248 }
5249}
5250
5251// Find the block storage driver providing the root disk, or NULL if not booting from
5252// a block device
5253
5254void
5255IOService::setRootMedia(IOService * root)
5256{
5257 const OSMetaClass * ioblockstoragedriverClass;
5258 bool unhide;
5259
5260 ioblockstoragedriverClass = OSMetaClass::getMetaClassWithName(name: gIOBlockStorageDriverKey);
5261 assert(ioblockstoragedriverClass);
5262
5263 while (root) {
5264 if (root->metaCast(toMeta: ioblockstoragedriverClass)) {
5265 break;
5266 }
5267 root = root->getProvider();
5268 }
5269
5270 LOCKWRITENOTIFY();
5271 unhide = (kIOServiceRootMediaParentInvalid == gIOServiceRootMediaParent);
5272 if (unhide) {
5273 gIOServiceRootMediaParent = root;
5274 }
5275 UNLOCKNOTIFY();
5276
5277 if (unhide) {
5278 publishHiddenMedia(parent: root);
5279 }
5280}
5281
5282// Check if the driver may be terminated when a later driver could be used instead
5283
5284bool
5285IOService::canTerminateForReplacement(IOService * client)
5286{
5287 IOService * parent;
5288
5289 assert(kIOServiceRootMediaParentInvalid != gIOServiceRootMediaParent);
5290
5291 if (!client->propertyExists(aKey: gIOPrimaryDriverTerminateOptionsKey)) {
5292 return false;
5293 }
5294 if (!gIOServiceRootMediaParent) {
5295 return false;
5296 }
5297 parent = gIOServiceRootMediaParent;
5298 while (parent && (parent != client)) {
5299 parent = parent->getProvider();
5300 }
5301 if (parent) {
5302 IOLog(format: "Can't replace primary matched driver on root media %s-0x%qx\n",
5303 client->getName(), client->getRegistryEntryID());
5304 return false;
5305 }
5306 return true;
5307}
5308
5309void
5310IOService::doServiceMatch( IOOptionBits options )
5311{
5312 _IOServiceNotifier * notify;
5313 OSIterator * iter;
5314 OSOrderedSet * matches;
5315 OSArray * resourceKeys = NULL;
5316 SInt32 catalogGeneration;
5317 bool keepGuessing = true;
5318 bool reRegistered = true;
5319 bool didRegister;
5320 OSArray * notifiers[2] = {NULL};
5321
5322// job->nub->deliverNotification( gIOPublishNotification,
5323// kIOServiceRegisteredState, 0xffffffff );
5324
5325 while (keepGuessing) {
5326 matches = gIOCatalogue->findDrivers( service: this, generationCount: &catalogGeneration );
5327 // the matches list should always be created by findDrivers()
5328 if (matches) {
5329 lockForArbitration();
5330 if (0 == (__state[0] & kIOServiceFirstPublishState)) {
5331 getMetaClass()->addInstance(instance: this);
5332 notifiers[0] = copyNotifiers(type: gIOFirstPublishNotification,
5333 orNewState: kIOServiceFirstPublishState, andNewState: 0xffffffff );
5334 }
5335 LOCKREADNOTIFY();
5336 __state[1] &= ~kIOServiceNeedConfigState;
5337 __state[1] |= kIOServiceConfigState | kIOServiceConfigRunning;
5338 didRegister = (0 == (kIOServiceRegisteredState & __state[0]));
5339 __state[0] |= kIOServiceRegisteredState;
5340
5341 if (gIOServiceHideIOMedia
5342 && metaCast(toMeta: gIOMediaKey)
5343 && !(kIOServiceUserUnhidden & __state[1])
5344 && gIOServiceRootMediaParent
5345 && !hasParent(parent: gIOServiceRootMediaParent)
5346 && propertyExists(aKey: gIOPrimaryDriverTerminateOptionsKey, plane: gIOServicePlane)) {
5347 __state[0] |= kIOServiceUserInvisibleMatchState;
5348 }
5349
5350 keepGuessing &= (0 == (__state[0] & kIOServiceInactiveState));
5351 if (reRegistered && keepGuessing) {
5352 iter = OSCollectionIterator::withCollection(inColl: (OSOrderedSet *)
5353 gNotifications->getObject( aKey: gIOPublishNotification ));
5354 if (iter) {
5355 while ((notify = (_IOServiceNotifier *)
5356 iter->getNextObject())) {
5357 if (matchPassive(table: notify->matching, options: 0)
5358 && (kIOServiceNotifyEnable & notify->state)) {
5359 matches->setObject( notify );
5360 }
5361 }
5362 iter->release();
5363 }
5364 }
5365
5366 UNLOCKNOTIFY();
5367 unlockForArbitration();
5368 invokeNotifiers(willSend: &notifiers[0]);
5369 if (keepGuessing && matches->getCount() && (kIOReturnSuccess == getResources())) {
5370 if ((this == gIOResources) || (this == gIOUserResources)) {
5371 if (resourceKeys) {
5372 resourceKeys->release();
5373 }
5374 resourceKeys = copyPropertyKeys();
5375 }
5376 probeCandidates( matches );
5377 } else {
5378 matches->release();
5379 }
5380 }
5381
5382 lockForArbitration();
5383 reRegistered = (0 != (__state[1] & kIOServiceNeedConfigState));
5384 keepGuessing =
5385 (reRegistered || (catalogGeneration !=
5386 gIOCatalogue->getGenerationCount()))
5387 && (0 == (__state[0] & kIOServiceInactiveState));
5388
5389 if (keepGuessing) {
5390 unlockForArbitration();
5391 }
5392 }
5393
5394 if ((0 == (__state[0] & kIOServiceInactiveState))
5395 && (0 == (__state[1] & kIOServiceModuleStallState))) {
5396 if (resourceKeys) {
5397 setProperty(aKey: gIOResourceMatchedKey, anObject: resourceKeys);
5398 }
5399
5400 notifiers[0] = copyNotifiers(type: gIOMatchedNotification,
5401 orNewState: kIOServiceMatchedState, andNewState: 0xffffffff);
5402 if (0 == (__state[0] & kIOServiceFirstMatchState)) {
5403 notifiers[1] = copyNotifiers(type: gIOFirstMatchNotification,
5404 orNewState: kIOServiceFirstMatchState, andNewState: 0xffffffff);
5405 }
5406 }
5407
5408 __state[1] &= ~kIOServiceConfigRunning;
5409 unlockForArbitration();
5410
5411 if (resourceKeys) {
5412 resourceKeys->release();
5413 }
5414
5415 invokeNotifiers(willSend: &notifiers[0]);
5416 invokeNotifiers(willSend: &notifiers[1]);
5417
5418 lockForArbitration();
5419 __state[1] &= ~kIOServiceConfigState;
5420 scheduleTerminatePhase2();
5421
5422 _adjustBusy( delta: -1 );
5423 unlockForArbitration();
5424}
5425
5426UInt32
5427IOService::_adjustBusy( SInt32 delta )
5428{
5429 IOService * next;
5430 UInt32 count;
5431 UInt32 result;
5432 bool wasQuiet, nowQuiet, needWake;
5433
5434 next = this;
5435 result = __state[1] & kIOServiceBusyStateMask;
5436
5437 if (delta) {
5438 do {
5439 if (next != this) {
5440 next->lockForArbitration();
5441 }
5442 count = next->__state[1] & kIOServiceBusyStateMask;
5443 wasQuiet = (0 == count);
5444 if (((delta < 0) && wasQuiet) || ((delta > 0) && (kIOServiceBusyMax == count))) {
5445 OSReportWithBacktrace(str: "%s: bad busy count (%d,%d)\n", next->getName(), (uint32_t)count, (int)delta);
5446 } else {
5447 count += delta;
5448 }
5449 next->__state[1] = (next->__state[1] & ~kIOServiceBusyStateMask) | count;
5450 nowQuiet = (0 == count);
5451 needWake = (0 != (kIOServiceBusyWaiterState & next->__state[1]));
5452
5453 if (needWake) {
5454 next->__state[1] &= ~kIOServiceBusyWaiterState;
5455 IOLockLock( gIOServiceBusyLock );
5456 thread_wakeup((event_t) next);
5457 IOLockUnlock( gIOServiceBusyLock );
5458 }
5459 if (next != this) {
5460 next->unlockForArbitration();
5461 }
5462
5463 if ((wasQuiet || nowQuiet)) {
5464 uint64_t regID = next->getRegistryEntryID();
5465 IOServiceTrace(
5466 ((wasQuiet /*nowBusy*/) ? IOSERVICE_BUSY : IOSERVICE_NONBUSY),
5467 (uintptr_t) regID,
5468 (uintptr_t) (regID >> 32),
5469 (uintptr_t) next,
5470 0);
5471
5472 if (wasQuiet) {
5473 next->__timeBusy = mach_absolute_time();
5474 } else {
5475 next->__accumBusy += mach_absolute_time() - next->__timeBusy;
5476 next->__timeBusy = 0;
5477 }
5478
5479 MessageClientsContext context;
5480
5481 context.service = next;
5482 context.type = kIOMessageServiceBusyStateChange;
5483 context.argument = (void *) wasQuiet; /*nowBusy*/
5484 context.argSize = 0;
5485
5486 applyToInterestNotifiers( target: next, typeOfInterest: gIOBusyInterest,
5487 applier: &messageClientsApplier, context: &context );
5488
5489#if !NO_KEXTD
5490 if (nowQuiet && (next == gIOServiceRoot)) {
5491 if (gIOServiceHideIOMedia) {
5492 publishHiddenMedia(NULL);
5493 }
5494
5495 OSKext::considerUnloads();
5496 IOServiceTrace(IOSERVICE_REGISTRY_QUIET, 0, 0, 0, 0);
5497 }
5498#endif
5499 }
5500
5501 delta = nowQuiet ? -1 : +1;
5502 } while ((wasQuiet || nowQuiet) && (next = next->getProvider()));
5503 }
5504
5505 return result;
5506}
5507
5508void
5509IOService::adjustBusy( SInt32 delta )
5510{
5511 lockForArbitration();
5512 _adjustBusy( delta );
5513 unlockForArbitration();
5514}
5515
5516uint64_t
5517IOService::getAccumulatedBusyTime( void )
5518{
5519 uint64_t accumBusy = __accumBusy;
5520 uint64_t timeBusy = __timeBusy;
5521 uint64_t nano;
5522
5523 do{
5524 accumBusy = __accumBusy;
5525 timeBusy = __timeBusy;
5526 if (timeBusy) {
5527 accumBusy += mach_absolute_time() - timeBusy;
5528 }
5529 }while (timeBusy != __timeBusy);
5530
5531 absolutetime_to_nanoseconds(abstime: *(AbsoluteTime *)&accumBusy, result: &nano);
5532
5533 return nano;
5534}
5535
5536UInt32
5537IOService::getBusyState( void )
5538{
5539 return __state[1] & kIOServiceBusyStateMask;
5540}
5541
5542IOReturn
5543IOService::waitForState( UInt32 mask, UInt32 value,
5544 mach_timespec_t * timeout )
5545{
5546 panic("waitForState");
5547 return kIOReturnUnsupported;
5548}
5549
5550IOReturn
5551IOService::waitForState( UInt32 mask, UInt32 value,
5552 uint64_t timeout )
5553{
5554 bool wait;
5555 int waitResult = THREAD_AWAKENED;
5556 bool computeDeadline = true;
5557 AbsoluteTime abstime;
5558
5559 do {
5560 lockForArbitration();
5561 IOLockLock( gIOServiceBusyLock );
5562 wait = (value != (__state[1] & mask));
5563 if (wait) {
5564 __state[1] |= kIOServiceBusyWaiterState;
5565 unlockForArbitration();
5566 if (timeout != UINT64_MAX) {
5567 if (computeDeadline) {
5568 AbsoluteTime nsinterval;
5569 nanoseconds_to_absolutetime(nanoseconds: timeout, result: &nsinterval );
5570 clock_absolutetime_interval_to_deadline(abstime: nsinterval, result: &abstime);
5571 computeDeadline = false;
5572 }
5573 assert_wait_deadline(event: (event_t)this, THREAD_UNINT, deadline: __OSAbsoluteTime(abstime));
5574 } else {
5575 assert_wait(event: (event_t)this, THREAD_UNINT );
5576 }
5577 } else {
5578 unlockForArbitration();
5579 }
5580 IOLockUnlock( gIOServiceBusyLock );
5581 if (wait) {
5582 waitResult = thread_block(THREAD_CONTINUE_NULL);
5583 }
5584 } while (wait && (waitResult != THREAD_TIMED_OUT));
5585
5586 if (waitResult == THREAD_TIMED_OUT) {
5587 return kIOReturnTimeout;
5588 } else {
5589 return kIOReturnSuccess;
5590 }
5591}
5592
5593IOReturn
5594IOService::waitQuietWithOptions( uint64_t timeout, IOOptionBits options )
5595{
5596 IOReturn ret;
5597 uint32_t loops, timeoutExtensions;
5598 char * busyEntriesString = NULL;
5599 char * panicString = NULL;
5600 size_t busyEntriesStringLen;
5601 size_t panicStringLen;
5602 uint64_t time;
5603 uint64_t nano;
5604 bool pendingRequests;
5605 bool registryRootBusy;
5606 bool multipleEntries;
5607 bool dopanic = false;
5608
5609 enum { kIOServiceBusyTimeoutExtensionsMax = 8 };
5610#if KASAN
5611 /*
5612 * On kasan kernels, everything takes longer, so double the number of
5613 * timeout extensions. This should help with issues like 41259215
5614 * where WindowServer was timing out waiting for kextd to get all the
5615 * kasan kexts loaded and started.
5616 *
5617 * On legacy/x86 systems give a bit more time since we may be
5618 * booting from a HDD.
5619 */
5620 enum { kTimeoutExtensions = 8 };
5621#define WITH_IOWAITQUIET_EXTENSIONS 1
5622#elif defined(__x86_64__)
5623 enum { kTimeoutExtensions = 4 };
5624#define WITH_IOWAITQUIET_EXTENSIONS 1
5625#elif defined(XNU_TARGET_OS_OSX)
5626 enum { kTimeoutExtensions = 1 };
5627#define WITH_IOWAITQUIET_EXTENSIONS 1
5628#else
5629 enum { kTimeoutExtensions = 1 };
5630#define WITH_IOWAITQUIET_EXTENSIONS 0
5631#endif
5632
5633 timeoutExtensions = kTimeoutExtensions;
5634 time = mach_absolute_time();
5635 pendingRequests = false;
5636 for (loops = 0; loops < timeoutExtensions; loops++) {
5637 ret = waitForState( mask: kIOServiceBusyStateMask, value: 0, timeout );
5638
5639 if (loops && (kIOReturnSuccess == ret)) {
5640 time = mach_absolute_time() - time;
5641 absolutetime_to_nanoseconds(abstime: *(AbsoluteTime *)&time, result: &nano);
5642 IOLog(format: "busy extended ok[%d], (%llds, %llds)\n",
5643 loops, timeout / 1000000000ULL, nano / 1000000000ULL);
5644 break;
5645 } else if (kIOReturnTimeout != ret) {
5646 break;
5647 } else if (timeout < (41ull * NSEC_PER_SEC)) {
5648 break;
5649 }
5650
5651 {
5652 IORegistryIterator * iter;
5653 OSOrderedSet * set;
5654 OSOrderedSet * leaves;
5655 IOService * next;
5656 IOService * nextParent;
5657 char * s;
5658 size_t l;
5659 size_t busyEntriesStringRemaining;
5660
5661 busyEntriesStringLen = 256;
5662 panicStringLen = 256;
5663 if (!busyEntriesString) {
5664 busyEntriesString = IONewZeroData(char, busyEntriesStringLen);
5665 assert(busyEntriesString != NULL);
5666 }
5667 if (!panicString) {
5668 panicString = IONewZeroData(char, panicStringLen);
5669 assert(panicString != NULL);
5670 }
5671
5672 set = NULL;
5673 pendingRequests = OSKext::pendingIOKitDaemonRequests();
5674 iter = IORegistryIterator::iterateOver(start: this, plane: gIOServicePlane, options: kIORegistryIterateRecursively);
5675 leaves = OSOrderedSet::withCapacity(capacity: 4);
5676 if (iter) {
5677 set = iter->iterateAll();
5678 }
5679 if (leaves && set) {
5680 busyEntriesString[0] = panicString[0] = 0;
5681 set->setObject(this);
5682 while ((next = (IOService *) set->getLastObject())) {
5683 if (next->getBusyState()) {
5684 if (kIOServiceModuleStallState & next->__state[1]) {
5685 pendingRequests = true;
5686 }
5687#if defined(XNU_TARGET_OS_OSX)
5688 OSObject * prop;
5689 if ((prop = next->copyProperty(kIOServiceBusyTimeoutExtensionsKey))) {
5690 OSNumber * num;
5691 uint32_t value;
5692 if ((num = OSDynamicCast(OSNumber, prop))) {
5693 value = num->unsigned32BitValue();
5694 if (value
5695 && (value <= kIOServiceBusyTimeoutExtensionsMax)
5696 && (value > timeoutExtensions)) {
5697 timeoutExtensions = value;
5698 }
5699 }
5700 OSSafeReleaseNULL(prop);
5701 }
5702#endif /* defined(XNU_TARGET_OS_OSX) */
5703 leaves->setObject(next);
5704 nextParent = next;
5705 while ((nextParent = nextParent->getProvider())) {
5706 set->removeObject(anObject: nextParent);
5707 leaves->removeObject(anObject: nextParent);
5708 }
5709 }
5710 set->removeObject(anObject: next);
5711 }
5712 registryRootBusy = leaves->getCount() == 1 && leaves->getObject(index: 0) == getServiceRoot();
5713 multipleEntries = leaves->getCount() > 1;
5714 s = busyEntriesString;
5715 busyEntriesStringRemaining = busyEntriesStringLen;
5716
5717 if (registryRootBusy) {
5718 snprintf(s, count: busyEntriesStringRemaining, "registry root held busy, " kIOKitDaemonName " %s checked in", OSKext::iokitDaemonActive() ? "has" : "has not");
5719 } else {
5720 while ((next = (IOService *) leaves->getLastObject())) {
5721 l = snprintf(s, count: busyEntriesStringRemaining, "%s'%s' (%x,%x)", ((s == busyEntriesString) ? "" : ", "), next->getName(), (uint32_t)next->__state[0], (uint32_t)next->__state[1]);
5722 if (l >= busyEntriesStringRemaining) {
5723 break;
5724 }
5725 s += l;
5726 busyEntriesStringRemaining -= l;
5727 leaves->removeObject(anObject: next);
5728 }
5729 }
5730 }
5731 OSSafeReleaseNULL(leaves);
5732 OSSafeReleaseNULL(set);
5733 OSSafeReleaseNULL(iter);
5734 }
5735
5736 dopanic = (kIOWaitQuietPanics & gIOKitDebug) && (options & kIOWaitQuietPanicOnFailure);
5737#if WITH_IOWAITQUIET_EXTENSIONS
5738 dopanic = (dopanic && (loops >= (timeoutExtensions - 1)));
5739#endif
5740 assert(panicString != NULL);
5741 if (multipleEntries) {
5742 snprintf(panicString, count: panicStringLen,
5743 "%s[%d], (%llds): multiple entries holding the registry busy, IOKit termination queue depth %u: %s",
5744 pendingRequests ? "IOKit Daemon (" kIOKitDaemonName ") stall" : "busy timeout",
5745 loops, timeout / 1000000000ULL,
5746 (uint32_t)gIOTerminateWork,
5747 busyEntriesString ? busyEntriesString : "");
5748 } else {
5749 snprintf(panicString, count: panicStringLen,
5750 "%s[%d], (%llds): %s",
5751 pendingRequests ? "IOKit Daemon (" kIOKitDaemonName ") stall" : "busy timeout",
5752 loops, timeout / 1000000000ULL,
5753 busyEntriesString ? busyEntriesString : "");
5754 }
5755
5756 IOLog(format: "%s\n", panicString);
5757 if (dopanic) {
5758 panic("%s", panicString);
5759 } else if (!loops) {
5760 getPMRootDomain()->startSpinDump(spindumpKind: 1);
5761 }
5762 }
5763
5764 if (busyEntriesString) {
5765 IODeleteData(busyEntriesString, char, busyEntriesStringLen);
5766 }
5767 if (panicString) {
5768 IODeleteData(panicString, char, panicStringLen);
5769 }
5770
5771 return ret;
5772}
5773
5774IOReturn
5775IOService::waitQuiet( uint64_t timeout )
5776{
5777 return waitQuietWithOptions(timeout);
5778}
5779
5780IOReturn
5781IOService::waitQuiet( mach_timespec_t * timeout )
5782{
5783 uint64_t timeoutNS;
5784
5785 if (timeout) {
5786 timeoutNS = timeout->tv_sec;
5787 timeoutNS *= kSecondScale;
5788 timeoutNS += timeout->tv_nsec;
5789 } else {
5790 timeoutNS = UINT64_MAX;
5791 }
5792
5793 return waitQuiet(timeout: timeoutNS);
5794}
5795
5796bool
5797IOService::serializeProperties( OSSerialize * s ) const
5798{
5799#if 0
5800 ((IOService *)this)->setProperty(((IOService *)this)->__state,
5801 sizeof(__state), "__state");
5802#endif
5803 return super::serializeProperties(serialize: s);
5804}
5805
5806void
5807IOService::resetRematchProperties()
5808{
5809 removeProperty(aKey: gIORematchCountKey);
5810 removeProperty(aKey: gIORematchPersonalityKey);
5811}
5812
5813
5814void
5815_IOConfigThread::main(void * arg, wait_result_t result)
5816{
5817 _IOConfigThread * self = (_IOConfigThread *) arg;
5818 _IOServiceJob * job;
5819 IOService * nub;
5820 bool alive = true;
5821 kern_return_t kr;
5822 thread_precedence_policy_data_t precedence = { .importance: -1 };
5823
5824 kr = thread_policy_set(thread: current_thread(),
5825 THREAD_PRECEDENCE_POLICY,
5826 policy_info: (thread_policy_t) &precedence,
5827 THREAD_PRECEDENCE_POLICY_COUNT);
5828 if (KERN_SUCCESS != kr) {
5829 IOLog(format: "thread_policy_set(%d)\n", kr);
5830 }
5831
5832 do {
5833// randomDelay();
5834
5835 semaphore_wait( semaphore: gJobsSemaphore );
5836
5837 IOTakeLock( lock: gJobsLock );
5838 job = (_IOServiceJob *) gJobs->getFirstObject();
5839 job->retain();
5840 gJobs->removeObject(anObject: job);
5841 if (job) {
5842 gOutstandingJobs--;
5843// gNumConfigThreads--; // we're out of service
5844 gNumWaitingThreads--; // we're out of service
5845 }
5846 IOUnlock( lock: gJobsLock );
5847
5848 if (job) {
5849 nub = job->nub;
5850
5851 if (gIOKitDebug & kIOLogConfig) {
5852 LOG(fmt: "config(%p): starting on %s, %d\n",
5853 IOSERVICE_OBFUSCATE(IOThreadSelf()), job->nub->getName(), job->type);
5854 }
5855
5856 switch (job->type) {
5857 case kMatchNubJob:
5858 nub->doServiceMatch( options: job->options );
5859 break;
5860
5861 default:
5862 LOG(fmt: "config(%p): strange type (%d)\n",
5863 IOSERVICE_OBFUSCATE(IOThreadSelf()), job->type );
5864 break;
5865 }
5866
5867 if (job->options & kIOServiceDextRequirePowerForMatching) {
5868 gIOPMRootDomain->releaseDriverKitMatchingAssertion();
5869 }
5870
5871 OSSafeReleaseNULL(nub);
5872 OSSafeReleaseNULL(job);
5873
5874 IOTakeLock( lock: gJobsLock );
5875 alive = (gOutstandingJobs > gNumWaitingThreads);
5876 if (alive) {
5877 gNumWaitingThreads++; // back in service
5878 }
5879// gNumConfigThreads++;
5880 else {
5881 if (0 == --gNumConfigThreads) {
5882// IOLog("MATCH IDLE\n");
5883 IOLockWakeup( lock: gJobsLock, event: (event_t) &gNumConfigThreads, /* one-thread */ oneThread: false );
5884 }
5885 }
5886 IOUnlock( lock: gJobsLock );
5887 }
5888 } while (alive);
5889
5890 if (gIOKitDebug & kIOLogConfig) {
5891 LOG(fmt: "config(%p): terminating\n", IOSERVICE_OBFUSCATE(IOThreadSelf()));
5892 }
5893
5894 self->release();
5895}
5896
5897IOReturn
5898IOService::waitMatchIdle( UInt32 msToWait )
5899{
5900 bool wait;
5901 int waitResult = THREAD_AWAKENED;
5902 bool computeDeadline = true;
5903 AbsoluteTime deadline;
5904
5905 IOLockLock( gJobsLock );
5906 do {
5907 wait = (0 != gNumConfigThreads);
5908 if (wait) {
5909 if (msToWait) {
5910 if (computeDeadline) {
5911 clock_interval_to_deadline(
5912 interval: msToWait, scale_factor: kMillisecondScale, result: &deadline );
5913 computeDeadline = false;
5914 }
5915 waitResult = IOLockSleepDeadline( lock: gJobsLock, event: &gNumConfigThreads,
5916 deadline, THREAD_UNINT );
5917 } else {
5918 waitResult = IOLockSleep( lock: gJobsLock, event: &gNumConfigThreads,
5919 THREAD_UNINT );
5920 }
5921 }
5922 } while (wait && (waitResult != THREAD_TIMED_OUT));
5923 IOLockUnlock( gJobsLock );
5924
5925 if (waitResult == THREAD_TIMED_OUT) {
5926 return kIOReturnTimeout;
5927 } else {
5928 return kIOReturnSuccess;
5929 }
5930}
5931
5932void
5933IOService::cpusRunning(void)
5934{
5935 gCPUsRunning = true;
5936}
5937
5938void
5939_IOServiceJob::pingConfig( _IOServiceJob * job )
5940{
5941 int count;
5942 bool create;
5943 IOService * nub;
5944
5945 assert( job );
5946 nub = job->nub;
5947
5948 IOTakeLock( lock: gJobsLock );
5949
5950 gOutstandingJobs++;
5951 if (nub == gIOResources) {
5952 gJobs->setFirstObject( job );
5953 } else {
5954 gJobs->setLastObject( job );
5955 }
5956
5957 count = gNumWaitingThreads;
5958// if( gNumConfigThreads) count++;// assume we're called from a config thread
5959
5960 create = ((gOutstandingJobs > count)
5961 && ((gNumConfigThreads < gMaxConfigThreads)
5962 || (nub == gIOResources)
5963 || !gCPUsRunning));
5964 if (create) {
5965 gNumConfigThreads++;
5966 gNumWaitingThreads++;
5967 if (gNumConfigThreads > gHighNumConfigThreads) {
5968 gHighNumConfigThreads = gNumConfigThreads;
5969 }
5970 }
5971
5972 IOUnlock( lock: gJobsLock );
5973
5974 job->release();
5975
5976 if (create) {
5977 if (gIOKitDebug & kIOLogConfig) {
5978 LOG(fmt: "config(%d): creating\n", gNumConfigThreads - 1);
5979 }
5980 _IOConfigThread::configThread(name: nub->getName());
5981 }
5982
5983 semaphore_signal( semaphore: gJobsSemaphore );
5984}
5985
5986struct IOServiceMatchContext {
5987 OSDictionary * table;
5988 OSObject * result;
5989 uint32_t options;
5990 uint32_t state;
5991 uint32_t count;
5992 uint32_t done;
5993};
5994
5995bool
5996IOService::instanceMatch(const OSObject * entry, void * context)
5997{
5998 IOServiceMatchContext * ctx = (typeof(ctx))context;
5999 IOService * service = (typeof(service))entry;
6000 OSDictionary * table = ctx->table;
6001 uint32_t options = ctx->options;
6002 uint32_t state = ctx->state;
6003 uint32_t done;
6004 bool match;
6005
6006 done = 0;
6007 do{
6008 match = ((state == (state & service->__state[0]))
6009 && (0 == (service->__state[0] & kIOServiceInactiveState)));
6010 if (!match) {
6011 break;
6012 }
6013
6014 match = service->matchInternal(table, options, did: &done);
6015 if (match) {
6016 ctx->count += table->getCount();
6017 ctx->done += done;
6018 }
6019 }while (false);
6020 if (!match) {
6021 return false;
6022 }
6023
6024 if ((kIONotifyOnce & options) && (ctx->done == ctx->count)) {
6025 service->retain();
6026 ctx->result = service;
6027 return true;
6028 } else if (!ctx->result) {
6029 ctx->result = OSSet::withObjects(objects: (const OSObject **) &service, count: 1, capacity: 1);
6030 } else {
6031 ((OSSet *)ctx->result)->setObject(service);
6032 }
6033 return false;
6034}
6035
6036// internal - call with gNotificationLock
6037OSObject *
6038IOService::copyExistingServices( OSDictionary * matching,
6039 IOOptionBits inState, IOOptionBits options )
6040{
6041 OSObject * current = NULL;
6042 OSIterator * iter;
6043 IOService * service;
6044 OSObject * obj;
6045 OSString * str;
6046
6047 if (!matching) {
6048 return NULL;
6049 }
6050
6051#if MATCH_DEBUG
6052 OSSerialize * s = OSSerialize::withCapacity(128);
6053 matching->serialize(s);
6054#endif
6055
6056 if ((obj = matching->getObject(aKey: gIOProviderClassKey))
6057 && gIOResourcesKey
6058 && gIOResourcesKey->isEqualTo(anObject: obj)
6059 && (service = gIOResources)) {
6060 if ((inState == (service->__state[0] & inState))
6061 && (0 == (service->__state[0] & kIOServiceInactiveState))
6062 && service->matchPassive(table: matching, options)) {
6063 if (options & kIONotifyOnce) {
6064 service->retain();
6065 current = service;
6066 } else {
6067 current = OSSet::withObjects(objects: (const OSObject **) &service, count: 1, capacity: 1 );
6068 }
6069 }
6070 } else {
6071 IOServiceMatchContext ctx;
6072
6073 options |= kIOServiceClassDone;
6074 ctx.table = matching;
6075 ctx.state = inState;
6076 ctx.count = 0;
6077 ctx.done = 0;
6078 ctx.options = options;
6079 ctx.result = NULL;
6080
6081 if ((str = OSDynamicCast(OSString, obj))) {
6082 const OSSymbol * sym = OSSymbol::withString(aString: str);
6083 OSMetaClass::applyToInstancesOfClassName(name: sym, applier: instanceMatch, context: &ctx);
6084 sym->release();
6085 } else {
6086 IOService::gMetaClass.applyToInstances(applier: instanceMatch, context: &ctx);
6087 }
6088
6089 if (((!(options & kIONotifyOnce) || !ctx.result))
6090 && matching->getObject(aKey: gIOCompatibilityMatchKey)) {
6091 IOServiceCompatibility::gMetaClass.applyToInstances(applier: instanceMatch, context: &ctx);
6092 }
6093
6094 current = ctx.result;
6095 options |= kIOServiceInternalDone;
6096 if (current && (ctx.done != ctx.count)) {
6097 OSSet * source = OSDynamicCast(OSSet, current);
6098 current = NULL;
6099 while ((service = (IOService *) source->getAnyObject())) {
6100 if (service->matchPassive(table: matching, options)) {
6101 if (options & kIONotifyOnce) {
6102 service->retain();
6103 current = service;
6104 break;
6105 }
6106 if (current) {
6107 ((OSSet *)current)->setObject( service );
6108 } else {
6109 current = OSSet::withObjects(
6110 objects: (const OSObject **) &service, count: 1, capacity: 1 );
6111 }
6112 }
6113 source->removeObject(anObject: service);
6114 }
6115 source->release();
6116 }
6117 }
6118
6119#if MATCH_DEBUG
6120 {
6121 OSObject * _current = 0;
6122
6123 iter = IORegistryIterator::iterateOver( gIOServicePlane,
6124 kIORegistryIterateRecursively );
6125 if (iter) {
6126 do {
6127 iter->reset();
6128 while ((service = (IOService *) iter->getNextObject())) {
6129 if ((inState == (service->__state[0] & inState))
6130 && (0 == (service->__state[0] & kIOServiceInactiveState))
6131 && service->matchPassive(matching, 0)) {
6132 if (options & kIONotifyOnce) {
6133 service->retain();
6134 _current = service;
6135 break;
6136 }
6137 if (_current) {
6138 ((OSSet *)_current)->setObject( service );
6139 } else {
6140 _current = OSSet::withObjects(
6141 (const OSObject **) &service, 1, 1 );
6142 }
6143 }
6144 }
6145 } while (!service && !iter->isValid());
6146 iter->release();
6147 }
6148
6149 if (((current != 0) != (_current != 0))
6150 || (current && _current && !current->isEqualTo(_current))) {
6151 OSSerialize * s1 = OSSerialize::withCapacity(128);
6152 OSSerialize * s2 = OSSerialize::withCapacity(128);
6153 current->serialize(s1);
6154 _current->serialize(s2);
6155 kprintf("**mismatch** %p %p\n%s\n%s\n%s\n", IOSERVICE_OBFUSCATE(current),
6156 IOSERVICE_OBFUSCATE(_current), s->text(), s1->text(), s2->text());
6157 s1->release();
6158 s2->release();
6159 }
6160
6161 if (_current) {
6162 _current->release();
6163 }
6164 }
6165
6166 s->release();
6167#endif
6168
6169 if (current && (0 == (options & (kIONotifyOnce | kIOServiceExistingSet)))) {
6170 iter = OSCollectionIterator::withCollection(inColl: (OSSet *)current );
6171 current->release();
6172 current = iter;
6173 }
6174
6175 return current;
6176}
6177
6178// public version
6179OSIterator *
6180IOService::getMatchingServices( OSDictionary * matching )
6181{
6182 OSIterator * iter;
6183
6184 // is a lock even needed?
6185 LOCKWRITENOTIFY();
6186
6187 iter = (OSIterator *) copyExistingServices( matching,
6188 inState: kIOServiceMatchedState );
6189
6190 UNLOCKNOTIFY();
6191
6192 return iter;
6193}
6194
6195IOService *
6196IOService::copyMatchingService( OSDictionary * matching )
6197{
6198 IOService * service;
6199
6200 // is a lock even needed?
6201 LOCKWRITENOTIFY();
6202
6203 service = (IOService *) copyExistingServices( matching,
6204 inState: kIOServiceMatchedState, options: kIONotifyOnce );
6205
6206 UNLOCKNOTIFY();
6207
6208 return service;
6209}
6210
6211struct _IOServiceMatchingNotificationHandlerRef {
6212 IOServiceNotificationHandler handler;
6213 void * ref;
6214};
6215
6216static bool
6217_IOServiceMatchingNotificationHandler( void * target, void * refCon,
6218 IOService * newService,
6219 IONotifier * notifier )
6220{
6221 return (*((_IOServiceNotifier *) notifier)->compatHandler)(target, refCon, newService);
6222}
6223
6224// internal - call with gNotificationLock
6225IONotifier *
6226IOService::setNotification(
6227 const OSSymbol * type, OSDictionary * matching,
6228 IOServiceMatchingNotificationHandler handler, void * target, void * ref,
6229 SInt32 priority )
6230{
6231 _IOServiceNotifier * notify = NULL;
6232 OSOrderedSet * set;
6233
6234 if (!matching) {
6235 return NULL;
6236 }
6237
6238 notify = new _IOServiceNotifier;
6239 if (notify && !notify->init()) {
6240 notify->release();
6241 notify = NULL;
6242 }
6243
6244 if (notify) {
6245 notify->handler = handler;
6246 notify->target = target;
6247 notify->type = type;
6248 notify->matching = matching;
6249 matching->retain();
6250 if (handler == &_IOServiceMatchingNotificationHandler) {
6251 notify->compatHandler = ((_IOServiceMatchingNotificationHandlerRef *)ref)->handler;
6252 notify->ref = ((_IOServiceMatchingNotificationHandlerRef *)ref)->ref;
6253 } else {
6254 notify->ref = ref;
6255 }
6256 notify->priority = priority;
6257 notify->state = kIOServiceNotifyEnable;
6258 queue_init( &notify->handlerInvocations );
6259
6260 ////// queue
6261
6262 if (NULL == (set = (OSOrderedSet *) gNotifications->getObject( aKey: type ))) {
6263 set = OSOrderedSet::withCapacity( capacity: 1,
6264 orderFunc: IONotifyOrdering, NULL );
6265 if (set) {
6266 gNotifications->setObject( aKey: type, anObject: set );
6267 set->release();
6268 }
6269 }
6270 notify->whence = set;
6271 if (set) {
6272 set->setObject( notify );
6273 }
6274 }
6275
6276 return notify;
6277}
6278
6279// internal - call with gNotificationLock
6280IONotifier *
6281IOService::doInstallNotification(
6282 const OSSymbol * type, OSDictionary * matching,
6283 IOServiceMatchingNotificationHandler handler,
6284 void * target, void * ref,
6285 SInt32 priority, OSIterator ** existing )
6286{
6287 OSIterator * exist;
6288 IONotifier * notify;
6289 IOOptionBits inState;
6290
6291 if (!matching) {
6292 return NULL;
6293 }
6294
6295 if (type == gIOPublishNotification) {
6296 inState = kIOServiceRegisteredState;
6297 } else if (type == gIOFirstPublishNotification) {
6298 inState = kIOServiceFirstPublishState;
6299 } else if (type == gIOMatchedNotification) {
6300 inState = kIOServiceMatchedState;
6301 } else if (type == gIOFirstMatchNotification) {
6302 inState = kIOServiceFirstMatchState;
6303 } else if ((type == gIOTerminatedNotification) || (type == gIOWillTerminateNotification)) {
6304 inState = 0;
6305 } else {
6306 return NULL;
6307 }
6308
6309 notify = setNotification( type, matching, handler, target, ref, priority );
6310
6311 if (inState) {
6312 // get the current set
6313 exist = (OSIterator *) copyExistingServices( matching, inState );
6314 } else {
6315 exist = NULL;
6316 }
6317
6318 *existing = exist;
6319
6320 return notify;
6321}
6322
6323#if !defined(__LP64__)
6324IONotifier *
6325IOService::installNotification(const OSSymbol * type, OSDictionary * matching,
6326 IOServiceNotificationHandler handler,
6327 void * target, void * refCon,
6328 SInt32 priority, OSIterator ** existing )
6329{
6330 IONotifier * result;
6331 _IOServiceMatchingNotificationHandlerRef ref;
6332 ref.handler = handler;
6333 ref.ref = refCon;
6334
6335 result = (_IOServiceNotifier *) installNotification( type, matching,
6336 &_IOServiceMatchingNotificationHandler,
6337 target, &ref, priority, existing );
6338 if (result) {
6339 matching->release();
6340 }
6341
6342 return result;
6343}
6344
6345#endif /* !defined(__LP64__) */
6346
6347
6348IONotifier *
6349IOService::installNotification(
6350 const OSSymbol * type, OSDictionary * matching,
6351 IOServiceMatchingNotificationHandler handler,
6352 void * target, void * ref,
6353 SInt32 priority, OSIterator ** existing )
6354{
6355 IONotifier * notify;
6356
6357 LOCKWRITENOTIFY();
6358
6359 notify = doInstallNotification( type, matching, handler, target, ref,
6360 priority, existing );
6361
6362 // in case handler remove()s
6363 if (notify) {
6364 notify->retain();
6365 }
6366
6367 UNLOCKNOTIFY();
6368
6369 return notify;
6370}
6371
6372IONotifier *
6373IOService::addNotification(
6374 const OSSymbol * type, OSDictionary * matching,
6375 IOServiceNotificationHandler handler,
6376 void * target, void * refCon,
6377 SInt32 priority )
6378{
6379 IONotifier * result;
6380 _IOServiceMatchingNotificationHandlerRef ref;
6381
6382 ref.handler = handler;
6383 ref.ref = refCon;
6384
6385 result = addMatchingNotification(type, matching, handler: &_IOServiceMatchingNotificationHandler,
6386 target, ref: &ref, priority);
6387
6388 if (result) {
6389 matching->release();
6390 }
6391
6392 return result;
6393}
6394
6395IONotifier *
6396IOService::addMatchingNotification(
6397 const OSSymbol * type, OSDictionary * matching,
6398 IOServiceMatchingNotificationHandler handler,
6399 void * target, void * ref,
6400 SInt32 priority )
6401{
6402 OSIterator * existing = NULL;
6403 IONotifier * ret;
6404 _IOServiceNotifier * notify;
6405 IOService * next;
6406
6407 ret = notify = (_IOServiceNotifier *) installNotification( type, matching,
6408 handler, target, ref, priority, existing: &existing );
6409 if (!ret) {
6410 OSSafeReleaseNULL(existing);
6411 return NULL;
6412 }
6413
6414 // send notifications for existing set
6415 if (existing) {
6416 while ((next = (IOService *) existing->getNextObject())) {
6417 if (0 == (next->__state[0] & kIOServiceInactiveState)) {
6418 next->invokeNotifier( notify );
6419 }
6420 }
6421 existing->release();
6422 }
6423
6424 LOCKWRITENOTIFY();
6425 bool removed = (NULL == notify->whence);
6426 notify->release();
6427 if (removed) {
6428 ret = gIOServiceNullNotifier;
6429 }
6430 UNLOCKNOTIFY();
6431
6432 return ret;
6433}
6434
6435static bool
6436IOServiceMatchingNotificationHandlerToBlock( void * target __unused, void * refCon,
6437 IOService * newService,
6438 IONotifier * notifier )
6439{
6440 return ((IOServiceMatchingNotificationHandlerBlock) refCon)(newService, notifier);
6441}
6442
6443IONotifier *
6444IOService::addMatchingNotification(
6445 const OSSymbol * type, OSDictionary * matching,
6446 SInt32 priority,
6447 IOServiceMatchingNotificationHandlerBlock handler)
6448{
6449 IONotifier * notify;
6450 void * block;
6451
6452 block = Block_copy(handler);
6453 if (!block) {
6454 return NULL;
6455 }
6456
6457 notify = addMatchingNotification(type, matching,
6458 handler: &IOServiceMatchingNotificationHandlerToBlock, NULL, ref: block, priority);
6459
6460 if (!notify) {
6461 Block_release(block);
6462 }
6463
6464 return notify;
6465}
6466
6467struct IOUserServerCancellationHandlerArgs {
6468 IOService ** ref;
6469 bool canceled;
6470};
6471
6472void
6473IOService::userServerCheckInTokenCancellationHandler(
6474 __unused IOUserServerCheckInToken *token,
6475 void *ref)
6476{
6477 IOUserServerCancellationHandlerArgs * args = (typeof(args))ref;
6478 LOCKWRITENOTIFY();
6479 WAKEUPNOTIFY(args->ref);
6480 args->canceled = true;
6481 UNLOCKNOTIFY();
6482}
6483
6484bool
6485IOService::syncNotificationHandler(
6486 void * /* target */, void * ref,
6487 IOService * newService,
6488 IONotifier * notifier )
6489{
6490 LOCKWRITENOTIFY();
6491 if (!*((IOService **) ref)) {
6492 newService->retain();
6493 (*(IOService **) ref) = newService;
6494 WAKEUPNOTIFY(ref);
6495 }
6496 UNLOCKNOTIFY();
6497
6498 return false;
6499}
6500
6501IOService *
6502IOService::waitForMatchingServiceWithToken( OSDictionary * matching,
6503 uint64_t timeout,
6504 IOUserServerCheckInToken * checkInToken)
6505{
6506 IONotifier * notify = NULL;
6507 // priority doesn't help us much since we need a thread wakeup
6508 SInt32 priority = 0;
6509 IOService * result;
6510 IOUserServerCancellationHandlerArgs cancelArgs;
6511 _IOUserServerCheckInCancellationHandler * cancellationHandler = NULL;
6512
6513 if (!matching) {
6514 return NULL;
6515 }
6516
6517 result = NULL;
6518 cancelArgs.ref = &result;
6519 cancelArgs.canceled = false;
6520
6521#if DEBUG || DEVELOPMENT
6522 char currentName[MAXTHREADNAMESIZE];
6523 char newName[MAXTHREADNAMESIZE];
6524 OSObject * obj;
6525 OSString * str;
6526 OSDictionary * dict;
6527
6528 currentName[0] = '\0';
6529 if (thread_has_thread_name(current_thread())) {
6530 dict = matching;
6531 obj = matching->getObject(gIOPropertyMatchKey);
6532 if ((dict = OSDynamicCast(OSDictionary, obj))) {
6533 OSObject * result __block = NULL;
6534 dict->iterateObjects(^bool (const OSSymbol * sym, OSObject * value) {
6535 result = __DECONST(OSObject *, sym);
6536 return true;
6537 });
6538 obj = result;
6539 }
6540 if (!obj) {
6541 obj = matching->getObject(gIOResourceMatchKey);
6542 }
6543 if (!obj) {
6544 obj = matching->getObject(gIONameMatchKey);
6545 }
6546 if (!obj) {
6547 obj = matching->getObject(gIOProviderClassKey);
6548 }
6549 if ((str = OSDynamicCast(OSString, obj))) {
6550 thread_get_thread_name(current_thread(), currentName);
6551 snprintf(newName, sizeof(newName), "Waiting_'%s'", str->getCStringNoCopy());
6552 thread_set_thread_name(current_thread(), newName);
6553 }
6554 }
6555#endif /* DEBUG || DEVELOPMENT */
6556
6557 if (checkInToken) {
6558 cancellationHandler = checkInToken->setCancellationHandler(handler: &IOService::userServerCheckInTokenCancellationHandler,
6559 handlerArgs: &cancelArgs);
6560 }
6561
6562 LOCKWRITENOTIFY();
6563 do{
6564 if (cancelArgs.canceled) {
6565 // token was already canceled, no need to wait or find services
6566 break;
6567 }
6568 result = (IOService *) copyExistingServices( matching,
6569 inState: kIOServiceMatchedState, options: kIONotifyOnce );
6570 if (result) {
6571 break;
6572 }
6573 notify = IOService::setNotification( type: gIOMatchedNotification, matching,
6574 handler: &IOService::syncNotificationHandler, target: (void *) NULL,
6575 ref: &result, priority );
6576 if (!notify) {
6577 break;
6578 }
6579 if (UINT64_MAX != timeout) {
6580 AbsoluteTime deadline;
6581 nanoseconds_to_absolutetime(nanoseconds: timeout, result: &deadline);
6582 clock_absolutetime_interval_to_deadline(abstime: deadline, result: &deadline);
6583 SLEEPNOTIFYTO(&result, deadline);
6584 } else {
6585 SLEEPNOTIFY(&result);
6586 }
6587 }while (false);
6588
6589 UNLOCKNOTIFY();
6590
6591 if (checkInToken && cancellationHandler) {
6592 checkInToken->removeCancellationHandler(handler: cancellationHandler);
6593 }
6594
6595#if DEBUG || DEVELOPMENT
6596 if (currentName[0]) {
6597 thread_set_thread_name(current_thread(), currentName);
6598 }
6599#endif /* DEBUG || DEVELOPMENT */
6600
6601 if (notify) {
6602 notify->remove(); // dequeues
6603 }
6604
6605 OSSafeReleaseNULL(cancellationHandler);
6606
6607 return result;
6608}
6609
6610IOService *
6611IOService::waitForMatchingService( OSDictionary * matching,
6612 uint64_t timeout)
6613{
6614 return IOService::waitForMatchingServiceWithToken(matching, timeout, NULL);
6615}
6616
6617IOService *
6618IOService::waitForService( OSDictionary * matching,
6619 mach_timespec_t * timeout )
6620{
6621 IOService * result;
6622 uint64_t timeoutNS;
6623
6624 if (timeout) {
6625 timeoutNS = timeout->tv_sec;
6626 timeoutNS *= kSecondScale;
6627 timeoutNS += timeout->tv_nsec;
6628 } else {
6629 timeoutNS = UINT64_MAX;
6630 }
6631
6632 result = waitForMatchingService(matching, timeout: timeoutNS);
6633
6634 matching->release();
6635 if (result) {
6636 result->release();
6637 }
6638
6639 return result;
6640}
6641
6642__dead2
6643void
6644IOService::deliverNotification( const OSSymbol * type,
6645 IOOptionBits orNewState, IOOptionBits andNewState )
6646{
6647 panic("deliverNotification");
6648}
6649
6650OSArray *
6651IOService::copyNotifiers(const OSSymbol * type,
6652 IOOptionBits orNewState, IOOptionBits andNewState )
6653{
6654 _IOServiceNotifier * notify;
6655 OSIterator * iter;
6656 OSArray * willSend = NULL;
6657
6658 lockForArbitration();
6659
6660 if ((0 == (__state[0] & kIOServiceInactiveState))
6661 || (type == gIOTerminatedNotification)
6662 || (type == gIOWillTerminateNotification)) {
6663 LOCKREADNOTIFY();
6664
6665 iter = OSCollectionIterator::withCollection(inColl: (OSOrderedSet *)
6666 gNotifications->getObject( aKey: type ));
6667
6668 if (iter) {
6669 while ((notify = (_IOServiceNotifier *) iter->getNextObject())) {
6670 if (matchPassive(table: notify->matching, options: 0)
6671 && (kIOServiceNotifyEnable & notify->state)) {
6672 if (NULL == willSend) {
6673 willSend = OSArray::withCapacity(capacity: 8);
6674 }
6675 if (willSend) {
6676 willSend->setObject( notify );
6677 }
6678 }
6679 }
6680 iter->release();
6681 }
6682 __state[0] = (__state[0] | orNewState) & andNewState;
6683 UNLOCKNOTIFY();
6684 }
6685
6686 unlockForArbitration();
6687
6688 return willSend;
6689}
6690
6691IOOptionBits
6692IOService::getState( void ) const
6693{
6694 return __state[0];
6695}
6696
6697/*
6698 * Helpers to make matching objects for simple cases
6699 */
6700
6701OSDictionary *
6702IOService::serviceMatching( const OSString * name,
6703 OSDictionary * table )
6704{
6705 const OSString * str;
6706
6707 str = OSSymbol::withString(aString: name);
6708 if (!str) {
6709 return NULL;
6710 }
6711
6712 if (!table) {
6713 table = OSDictionary::withCapacity( capacity: 2 );
6714 }
6715 if (table) {
6716 table->setObject(aKey: gIOProviderClassKey, anObject: (OSObject *)str );
6717 }
6718 str->release();
6719
6720 return table;
6721}
6722
6723
6724OSSharedPtr<OSDictionary>
6725IOService::serviceMatching( const OSString * name,
6726 OSSharedPtr<OSDictionary> table)
6727{
6728 OSDictionary * result = serviceMatching(name, table: table.get());
6729 if (table) {
6730 return OSSharedPtr<OSDictionary>(result, OSRetain);
6731 } else {
6732 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6733 }
6734}
6735
6736
6737OSDictionary *
6738IOService::serviceMatching( const char * name,
6739 OSDictionary * table )
6740{
6741 const OSString * str;
6742
6743 str = OSSymbol::withCString( cString: name );
6744 if (!str) {
6745 return NULL;
6746 }
6747
6748 table = serviceMatching( name: str, table );
6749 str->release();
6750 return table;
6751}
6752
6753
6754OSSharedPtr<OSDictionary>
6755IOService::serviceMatching( const char * className,
6756 OSSharedPtr<OSDictionary> table)
6757{
6758 OSDictionary * result = serviceMatching(name: className, table: table.get());
6759 if (table) {
6760 return OSSharedPtr<OSDictionary>(result, OSRetain);
6761 } else {
6762 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6763 }
6764}
6765
6766
6767OSDictionary *
6768IOService::nameMatching( const OSString * name,
6769 OSDictionary * table )
6770{
6771 if (!table) {
6772 table = OSDictionary::withCapacity( capacity: 2 );
6773 }
6774 if (table) {
6775 table->setObject( aKey: gIONameMatchKey, anObject: (OSObject *)name );
6776 }
6777
6778 return table;
6779}
6780
6781
6782OSSharedPtr<OSDictionary>
6783IOService::nameMatching( const OSString * name,
6784 OSSharedPtr<OSDictionary> table)
6785{
6786 OSDictionary * result = nameMatching(name, table: table.get());
6787 if (table) {
6788 return OSSharedPtr<OSDictionary>(result, OSRetain);
6789 } else {
6790 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6791 }
6792}
6793
6794
6795OSDictionary *
6796IOService::nameMatching( const char * name,
6797 OSDictionary * table )
6798{
6799 const OSString * str;
6800
6801 str = OSSymbol::withCString( cString: name );
6802 if (!str) {
6803 return NULL;
6804 }
6805
6806 table = nameMatching( name: str, table );
6807 str->release();
6808 return table;
6809}
6810
6811
6812OSSharedPtr<OSDictionary>
6813IOService::nameMatching( const char * name,
6814 OSSharedPtr<OSDictionary> table)
6815{
6816 OSDictionary * result = nameMatching(name, table: table.get());
6817 if (table) {
6818 return OSSharedPtr<OSDictionary>(result, OSRetain);
6819 } else {
6820 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6821 }
6822}
6823
6824
6825OSDictionary *
6826IOService::resourceMatching( const OSString * str,
6827 OSDictionary * table )
6828{
6829 table = serviceMatching( name: gIOResourcesKey, table );
6830 if (table) {
6831 table->setObject( aKey: gIOResourceMatchKey, anObject: (OSObject *) str );
6832 }
6833
6834 return table;
6835}
6836
6837
6838OSSharedPtr<OSDictionary>
6839IOService::resourceMatching( const OSString * str,
6840 OSSharedPtr<OSDictionary> table)
6841{
6842 OSDictionary * result = resourceMatching(str, table: table.get());
6843 if (table) {
6844 return OSSharedPtr<OSDictionary>(result, OSRetain);
6845 } else {
6846 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6847 }
6848}
6849
6850
6851OSDictionary *
6852IOService::resourceMatching( const char * name,
6853 OSDictionary * table )
6854{
6855 const OSSymbol * str;
6856
6857 str = OSSymbol::withCString( cString: name );
6858 if (!str) {
6859 return NULL;
6860 }
6861
6862 table = resourceMatching( str, table );
6863 str->release();
6864
6865 return table;
6866}
6867
6868
6869OSSharedPtr<OSDictionary>
6870IOService::resourceMatching( const char * name,
6871 OSSharedPtr<OSDictionary> table)
6872{
6873 OSDictionary * result = resourceMatching(name, table: table.get());
6874 if (table) {
6875 return OSSharedPtr<OSDictionary>(result, OSRetain);
6876 } else {
6877 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6878 }
6879}
6880
6881
6882OSDictionary *
6883IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
6884 OSDictionary * table )
6885{
6886 OSDictionary * properties;
6887
6888 properties = OSDictionary::withCapacity( capacity: 2 );
6889 if (!properties) {
6890 return NULL;
6891 }
6892 properties->setObject( aKey: key, anObject: value );
6893
6894 if (!table) {
6895 table = OSDictionary::withCapacity( capacity: 2 );
6896 }
6897 if (table) {
6898 table->setObject( aKey: gIOPropertyMatchKey, anObject: properties );
6899 }
6900
6901 properties->release();
6902
6903 return table;
6904}
6905
6906
6907OSSharedPtr<OSDictionary>
6908IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
6909 OSSharedPtr<OSDictionary> table)
6910{
6911 OSDictionary * result = propertyMatching(key, value, table: table.get());
6912 if (table) {
6913 return OSSharedPtr<OSDictionary>(result, OSRetain);
6914 } else {
6915 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6916 }
6917}
6918
6919
6920OSDictionary *
6921IOService::registryEntryIDMatching( uint64_t entryID,
6922 OSDictionary * table )
6923{
6924 OSNumber * num;
6925
6926 num = OSNumber::withNumber( value: entryID, numberOfBits: 64 );
6927 if (!num) {
6928 return NULL;
6929 }
6930
6931 if (!table) {
6932 table = OSDictionary::withCapacity( capacity: 2 );
6933 }
6934 if (table) {
6935 table->setObject( aKey: gIORegistryEntryIDKey, anObject: num );
6936 }
6937
6938 if (num) {
6939 num->release();
6940 }
6941
6942 return table;
6943}
6944
6945
6946OSSharedPtr<OSDictionary>
6947IOService::registryEntryIDMatching( uint64_t entryID,
6948 OSSharedPtr<OSDictionary> table)
6949{
6950 OSDictionary * result = registryEntryIDMatching(entryID, table: table.get());
6951 if (table) {
6952 return OSSharedPtr<OSDictionary>(result, OSRetain);
6953 } else {
6954 return OSSharedPtr<OSDictionary>(result, OSNoRetain);
6955 }
6956}
6957
6958
6959
6960/*
6961 * _IOServiceNotifier
6962 */
6963
6964// wait for all threads, other than the current one,
6965// to exit the handler
6966
6967void
6968_IOServiceNotifier::wait()
6969{
6970 _IOServiceNotifierInvocation * next;
6971 bool doWait;
6972
6973 do {
6974 doWait = false;
6975 queue_iterate( &handlerInvocations, next,
6976 _IOServiceNotifierInvocation *, link) {
6977 if (next->thread != current_thread()) {
6978 doWait = true;
6979 break;
6980 }
6981 }
6982 if (doWait) {
6983 state |= kIOServiceNotifyWaiter;
6984 SLEEPNOTIFY(this);
6985 }
6986 } while (doWait);
6987}
6988
6989void
6990_IOServiceNotifier::free()
6991{
6992 assert( queue_empty( &handlerInvocations ));
6993
6994 if (handler == &IOServiceMatchingNotificationHandlerToBlock) {
6995 Block_release(ref);
6996 }
6997
6998 OSObject::free();
6999}
7000
7001void
7002_IOServiceNotifier::remove()
7003{
7004 LOCKWRITENOTIFY();
7005
7006 if (whence) {
7007 whence->removeObject(anObject: (OSObject *) this );
7008 whence = NULL;
7009 }
7010 if (matching) {
7011 matching->release();
7012 matching = NULL;
7013 }
7014
7015 state &= ~kIOServiceNotifyEnable;
7016
7017 wait();
7018
7019 UNLOCKNOTIFY();
7020
7021 release();
7022}
7023
7024bool
7025_IOServiceNotifier::disable()
7026{
7027 bool ret;
7028
7029 LOCKWRITENOTIFY();
7030
7031 ret = (0 != (kIOServiceNotifyEnable & state));
7032 state &= ~kIOServiceNotifyEnable;
7033 if (ret) {
7034 wait();
7035 }
7036
7037 UNLOCKNOTIFY();
7038
7039 return ret;
7040}
7041
7042void
7043_IOServiceNotifier::enable( bool was )
7044{
7045 LOCKWRITENOTIFY();
7046 if (was) {
7047 state |= kIOServiceNotifyEnable;
7048 } else {
7049 state &= ~kIOServiceNotifyEnable;
7050 }
7051 UNLOCKNOTIFY();
7052}
7053
7054
7055/*
7056 * _IOServiceNullNotifier
7057 */
7058
7059void
7060_IOServiceNullNotifier::taggedRetain(const void *tag) const
7061{
7062}
7063void
7064_IOServiceNullNotifier::taggedRelease(const void *tag, const int when) const
7065{
7066}
7067void
7068_IOServiceNullNotifier::free()
7069{
7070}
7071void
7072_IOServiceNullNotifier::wait()
7073{
7074}
7075void
7076_IOServiceNullNotifier::remove()
7077{
7078}
7079void
7080_IOServiceNullNotifier::enable(bool was)
7081{
7082}
7083bool
7084_IOServiceNullNotifier::disable()
7085{
7086 return false;
7087}
7088
7089/*
7090 * IOResources
7091 */
7092
7093IOService *
7094IOResources::resources( void )
7095{
7096 IOResources * inst;
7097
7098 inst = new IOResources;
7099 if (inst && !inst->init()) {
7100 inst->release();
7101 inst = NULL;
7102 }
7103
7104 return inst;
7105}
7106
7107bool
7108IOResources::init( OSDictionary * dictionary )
7109{
7110 // Do super init first
7111 if (!IOService::init()) {
7112 return false;
7113 }
7114
7115 // Allow PAL layer to publish a value
7116 const char *property_name;
7117 int property_value;
7118
7119 pal_get_resource_property( property_name: &property_name, property_value: &property_value );
7120
7121 if (property_name) {
7122 OSNumber *num;
7123 const OSSymbol * sym;
7124
7125 if ((num = OSNumber::withNumber(value: property_value, numberOfBits: 32)) != NULL) {
7126 if ((sym = OSSymbol::withCString( cString: property_name)) != NULL) {
7127 this->setProperty( aKey: sym, anObject: num );
7128 sym->release();
7129 }
7130 num->release();
7131 }
7132 }
7133
7134 return true;
7135}
7136
7137IOReturn
7138IOResources::newUserClient(task_t owningTask, void * securityID,
7139 UInt32 type, OSDictionary * properties,
7140 IOUserClient ** handler)
7141{
7142 return kIOReturnUnsupported;
7143}
7144
7145IOWorkLoop *
7146IOResources::getWorkLoop() const
7147{
7148 // If we are the resource root
7149 // then use the platform's workloop
7150 if (this == (IOResources *) gIOResources) {
7151 return getPlatform()->getWorkLoop();
7152 } else {
7153 return IOService::getWorkLoop();
7154 }
7155}
7156
7157static bool
7158IOResourcesMatchPropertyTable(IOService * resources, OSDictionary * table)
7159{
7160 OSObject * prop;
7161 bool __block ok = true;
7162
7163 prop = table->getObject( aKey: gIOResourceMatchKey );
7164 if (prop) {
7165 prop->iterateObjects(block: ^bool (OSObject * obj)
7166 {
7167 OSString *
7168 str = OSDynamicCast(OSString, obj);
7169 ok = (NULL != resources->getProperty(aKey: str));
7170 return !ok;
7171 });
7172 } else if ((prop = table->getObject(aKey: gIOResourceMatchedKey))) {
7173 OSObject * obj;
7174 OSArray * keys;
7175
7176 obj = resources->copyProperty(aKey: gIOResourceMatchedKey);
7177 keys = OSDynamicCast(OSArray, obj);
7178 ok = false;
7179 if (keys) {
7180 // assuming OSSymbol
7181 ok = ((-1U) != keys->getNextIndexOfObject(anObject: prop, index: 0));
7182 }
7183 OSSafeReleaseNULL(obj);
7184 }
7185
7186 return ok;
7187}
7188
7189bool
7190IOResources::matchPropertyTable( OSDictionary * table )
7191{
7192 return IOResourcesMatchPropertyTable(resources: this, table);
7193}
7194
7195/*
7196 * IOUserResources
7197 */
7198
7199IOService *
7200IOUserResources::resources( void )
7201{
7202 IOUserResources * inst;
7203
7204 inst = OSTypeAlloc(IOUserResources);
7205 if (inst && !inst->init()) {
7206 inst->release();
7207 inst = NULL;
7208 }
7209
7210 return inst;
7211}
7212
7213bool
7214IOUserResources::init( OSDictionary * dictionary )
7215{
7216 // Do super init first
7217 if (!IOService::init()) {
7218 return false;
7219 }
7220 return true;
7221}
7222
7223IOReturn
7224IOUserResources::newUserClient(task_t owningTask, void * securityID,
7225 UInt32 type, OSDictionary * properties,
7226 IOUserClient ** handler)
7227{
7228 return kIOReturnUnsupported;
7229}
7230
7231IOWorkLoop *
7232IOUserResources::getWorkLoop() const
7233{
7234 return getPlatform()->getWorkLoop();
7235}
7236
7237bool
7238IOUserResources::matchPropertyTable( OSDictionary * table )
7239{
7240 return IOResourcesMatchPropertyTable(resources: this, table);
7241}
7242
7243// --
7244
7245void
7246IOService::consoleLockTimer(thread_call_param_t p0, thread_call_param_t p1)
7247{
7248 IOService::updateConsoleUsers(NULL, systemMessage: 0);
7249}
7250
7251void
7252IOService::updateConsoleUsers(OSArray * consoleUsers, IOMessage systemMessage, bool afterUserspaceReboot)
7253{
7254 IORegistryEntry * regEntry;
7255 OSObject * locked = kOSBooleanFalse;
7256 uint32_t idx;
7257 bool publish;
7258 OSDictionary * user;
7259 clock_sec_t now = 0;
7260 clock_usec_t microsecs;
7261
7262 regEntry = IORegistryEntry::getRegistryRoot();
7263
7264 if (!gIOChosenEntry) {
7265 gIOChosenEntry = IORegistryEntry::fromPath(path: "/chosen", plane: gIODTPlane);
7266 }
7267
7268 IOLockLock(gIOConsoleUsersLock);
7269
7270 if (systemMessage) {
7271 sSystemPower = systemMessage;
7272#if HIBERNATION
7273 if (kIOMessageSystemHasPoweredOn == systemMessage) {
7274 uint32_t lockState = IOHibernateWasScreenLocked();
7275 switch (lockState) {
7276 case 0:
7277 break;
7278 case kIOScreenLockLocked:
7279 case kIOScreenLockFileVaultDialog:
7280 gIOConsoleBooterLockState = kOSBooleanTrue;
7281 break;
7282 case kIOScreenLockNoLock:
7283 gIOConsoleBooterLockState = NULL;
7284 break;
7285 case kIOScreenLockUnlocked:
7286 default:
7287 gIOConsoleBooterLockState = kOSBooleanFalse;
7288 break;
7289 }
7290 }
7291#endif /* HIBERNATION */
7292 }
7293
7294 if (consoleUsers) {
7295 OSNumber * num = NULL;
7296 bool loginLocked = true;
7297
7298 gIOConsoleLoggedIn = false;
7299 for (idx = 0;
7300 (user = OSDynamicCast(OSDictionary, consoleUsers->getObject(idx)));
7301 idx++) {
7302 gIOConsoleLoggedIn |= ((kOSBooleanTrue == user->getObject(aKey: gIOConsoleSessionOnConsoleKey))
7303 && (kOSBooleanTrue == user->getObject(aKey: gIOConsoleSessionLoginDoneKey)));
7304
7305 loginLocked &= (kOSBooleanTrue == user->getObject(aKey: gIOConsoleSessionScreenIsLockedKey));
7306 if (!num) {
7307 num = OSDynamicCast(OSNumber, user->getObject(gIOConsoleSessionScreenLockedTimeKey));
7308 }
7309 }
7310#if HIBERNATION
7311 if (!loginLocked || afterUserspaceReboot) {
7312 gIOConsoleBooterLockState = NULL;
7313 }
7314 IOLog("IOConsoleUsers: time(%d) %ld->%d, lin %d, llk %d, \n",
7315 (num != NULL), gIOConsoleLockTime, (num ? num->unsigned32BitValue() : 0),
7316 gIOConsoleLoggedIn, loginLocked);
7317#endif /* HIBERNATION */
7318 gIOConsoleLockTime = num ? num->unsigned32BitValue() : 0;
7319 }
7320
7321 if (!gIOConsoleLoggedIn
7322 || (kIOMessageSystemWillSleep == sSystemPower)
7323 || (kIOMessageSystemPagingOff == sSystemPower)) {
7324 if (afterUserspaceReboot) {
7325 // set "locked" to false after a user space reboot
7326 // because the reboot happens directly after a user
7327 // logs into the machine via fvunlock mode.
7328 locked = kOSBooleanFalse;
7329 } else {
7330 locked = kOSBooleanTrue;
7331 }
7332 }
7333#if HIBERNATION
7334 else if (gIOConsoleBooterLockState) {
7335 locked = gIOConsoleBooterLockState;
7336 }
7337#endif /* HIBERNATION */
7338 else if (gIOConsoleLockTime) {
7339 clock_get_calendar_microtime(secs: &now, microsecs: &microsecs);
7340 if (gIOConsoleLockTime > now) {
7341 AbsoluteTime deadline;
7342 clock_sec_t interval;
7343 uint32_t interval32;
7344
7345 interval = (gIOConsoleLockTime - now);
7346 interval32 = (uint32_t) interval;
7347 if (interval32 != interval) {
7348 interval32 = UINT_MAX;
7349 }
7350 clock_interval_to_deadline(interval: interval32, scale_factor: kSecondScale, result: &deadline);
7351 thread_call_enter_delayed(call: gIOConsoleLockCallout, deadline);
7352 } else {
7353 locked = kOSBooleanTrue;
7354 }
7355 }
7356
7357 publish = (consoleUsers || (locked != regEntry->getProperty(aKey: gIOConsoleLockedKey)));
7358 if (publish) {
7359 regEntry->setProperty(aKey: gIOConsoleLockedKey, anObject: locked);
7360 if (consoleUsers) {
7361 regEntry->setProperty(aKey: gIOConsoleUsersKey, anObject: consoleUsers);
7362 }
7363 OSIncrementAtomic( &gIOConsoleUsersSeed );
7364 }
7365
7366#if HIBERNATION
7367 if (gIOChosenEntry) {
7368 if (locked == kOSBooleanTrue) {
7369 gIOScreenLockState = kIOScreenLockLocked;
7370 } else if (gIOConsoleLockTime) {
7371 gIOScreenLockState = kIOScreenLockUnlocked;
7372 } else {
7373 gIOScreenLockState = kIOScreenLockNoLock;
7374 }
7375 gIOChosenEntry->setProperty(kIOScreenLockStateKey, &gIOScreenLockState, sizeof(gIOScreenLockState));
7376
7377 IOLog("IOConsoleUsers: gIOScreenLockState %d, hs %d, bs %d, now %ld, sm 0x%x\n",
7378 gIOScreenLockState, gIOHibernateState, (gIOConsoleBooterLockState != NULL), now, systemMessage);
7379 }
7380#endif /* HIBERNATION */
7381
7382 IOLockUnlock(gIOConsoleUsersLock);
7383
7384 if (publish) {
7385 publishResource( key: gIOConsoleUsersSeedKey, value: gIOConsoleUsersSeedValue );
7386
7387 MessageClientsContext context;
7388
7389 context.service = getServiceRoot();
7390 context.type = kIOMessageConsoleSecurityChange;
7391 context.argument = (void *) regEntry;
7392 context.argSize = 0;
7393
7394 applyToInterestNotifiers(target: getServiceRoot(), typeOfInterest: gIOConsoleSecurityInterest,
7395 applier: &messageClientsApplier, context: &context );
7396 }
7397}
7398
7399IOReturn
7400IOResources::setProperties( OSObject * properties )
7401{
7402 IOReturn err;
7403 const OSSymbol * key;
7404 OSDictionary * dict;
7405 OSCollectionIterator * iter;
7406
7407 if (!IOCurrentTaskHasEntitlement(kIOResourcesSetPropertyKey)) {
7408 err = IOUserClient::clientHasPrivilege(securityToken: current_task(), kIOClientPrivilegeAdministrator);
7409 if (kIOReturnSuccess != err) {
7410 return err;
7411 }
7412 }
7413
7414 dict = OSDynamicCast(OSDictionary, properties);
7415 if (NULL == dict) {
7416 return kIOReturnBadArgument;
7417 }
7418
7419 iter = OSCollectionIterator::withCollection( inColl: dict);
7420 if (NULL == iter) {
7421 return kIOReturnBadArgument;
7422 }
7423
7424 while ((key = OSDynamicCast(OSSymbol, iter->getNextObject()))) {
7425 if (gIOConsoleUsersKey == key) {
7426 do{
7427 OSArray * consoleUsers;
7428 consoleUsers = OSDynamicCast(OSArray, dict->getObject(key));
7429 if (!consoleUsers) {
7430 continue;
7431 }
7432 IOService::updateConsoleUsers(consoleUsers, systemMessage: 0);
7433 }while (false);
7434 }
7435
7436 publishResource( key, value: dict->getObject(aKey: key));
7437 }
7438
7439 iter->release();
7440
7441 return kIOReturnSuccess;
7442}
7443
7444/*
7445 * Helpers for matching dictionaries.
7446 * Keys existing in matching are checked in properties.
7447 * Keys may be a string or OSCollection of IOStrings
7448 */
7449
7450bool
7451IOService::compareProperty( OSDictionary * matching,
7452 const char * key )
7453{
7454 OSObject * value;
7455 OSObject * prop;
7456 bool ok;
7457
7458 value = matching->getObject( aKey: key );
7459 if (value) {
7460 prop = copyProperty(aKey: key);
7461 ok = value->isEqualTo(anObject: prop);
7462 if (prop) {
7463 prop->release();
7464 }
7465 } else {
7466 ok = true;
7467 }
7468
7469 return ok;
7470}
7471
7472
7473bool
7474IOService::compareProperty( OSDictionary * matching,
7475 const OSString * key )
7476{
7477 OSObject * value;
7478 OSObject * prop;
7479 bool ok;
7480
7481 value = matching->getObject( aKey: key );
7482 if (value) {
7483 prop = copyProperty(aKey: key);
7484 ok = value->isEqualTo(anObject: prop);
7485 if (prop) {
7486 prop->release();
7487 }
7488 } else {
7489 ok = true;
7490 }
7491
7492 return ok;
7493}
7494
7495#ifndef __clang_analyzer__
7496// Implementation of this function is hidden from the static analyzer.
7497// The analyzer was worried about this function's confusing contract over
7498// the 'keys' parameter. The contract is to either release it or not release it
7499// depending on whether 'matching' is non-null. Such contracts are discouraged
7500// but changing it now would break compatibility.
7501bool
7502IOService::compareProperties( OSDictionary * matching,
7503 OSCollection * keys )
7504{
7505 OSCollectionIterator * iter;
7506 const OSString * key;
7507 bool ok = true;
7508
7509 if (!matching || !keys) {
7510 return false;
7511 }
7512
7513 iter = OSCollectionIterator::withCollection( inColl: keys );
7514
7515 if (iter) {
7516 while (ok && (key = OSDynamicCast( OSString, iter->getNextObject()))) {
7517 ok = compareProperty( matching, key );
7518 }
7519
7520 iter->release();
7521 }
7522 keys->release(); // !! consume a ref !!
7523
7524 return ok;
7525}
7526#endif // __clang_analyzer__
7527
7528/* Helper to add a location matching dict to the table */
7529
7530OSDictionary *
7531IOService::addLocation( OSDictionary * table )
7532{
7533 OSDictionary * dict;
7534
7535 if (!table) {
7536 return NULL;
7537 }
7538
7539 dict = OSDictionary::withCapacity( capacity: 1 );
7540 if (dict) {
7541 bool ok = table->setObject( aKey: gIOLocationMatchKey, anObject: dict );
7542 dict->release();
7543 if (!ok) {
7544 dict = NULL;
7545 }
7546 }
7547
7548 return dict;
7549}
7550
7551/*
7552 * Go looking for a provider to match a location dict.
7553 */
7554
7555IOService *
7556IOService::matchLocation( IOService * /* client */ )
7557{
7558 IOService * parent;
7559
7560 parent = getProvider();
7561
7562 if (parent) {
7563 parent = parent->matchLocation( this );
7564 }
7565
7566 return parent;
7567}
7568
7569bool
7570IOService::matchInternal(OSDictionary * table, uint32_t options, uint32_t * did)
7571{
7572 OSString * matched;
7573 OSObject * obj;
7574 OSString * str;
7575 OSDictionary * matchProps;
7576 IORegistryEntry * entry;
7577 OSNumber * num;
7578 bool match = true;
7579 bool changesOK = (0 != (kIOServiceChangesOK & options));
7580 uint32_t count;
7581 uint32_t done;
7582
7583 do{
7584 count = table->getCount();
7585 done = 0;
7586 matchProps = NULL;
7587 bool isUser;
7588
7589 isUser = (NULL != table->getObject(aKey: gIOServiceNotificationUserKey));
7590 if (isUser) {
7591 done++;
7592 match = (0 == (kIOServiceUserInvisibleMatchState & __state[0]));
7593 if ((!match) || (done == count)) {
7594 break;
7595 }
7596 }
7597
7598 if (propertyExists(aKey: gIOExclaveAssignedKey)) {
7599 if (!table->getObject(aKey: gIOExclaveProxyKey) && !isUser) {
7600 match = false;
7601 break;
7602 }
7603 } else if (table->getObject(aKey: gIOExclaveProxyKey)) {
7604 match = false;
7605 break;
7606 }
7607
7608 if (table->getObject(aKey: gIOCompatibilityMatchKey)) {
7609 done++;
7610 obj = copyProperty(aKey: gIOCompatibilityPropertiesKey);
7611 matchProps = OSDynamicCast(OSDictionary, obj);
7612 if (!matchProps) {
7613 OSSafeReleaseNULL(obj);
7614 }
7615 }
7616
7617 str = OSDynamicCast(OSString, table->getObject(gIOProviderClassKey));
7618 if (str) {
7619 done++;
7620 if (matchProps && (obj = matchProps->getObject(aKey: gIOClassKey))) {
7621 match = str->isEqualTo(anObject: obj);
7622 } else {
7623 match = ((kIOServiceClassDone & options) || (NULL != metaCast(toMeta: str)));
7624 }
7625
7626#if MATCH_DEBUG
7627 match = (0 != metaCast( str ));
7628 if ((kIOServiceClassDone & options) && !match) {
7629 panic("classDone");
7630 }
7631#endif
7632 if ((!match) || (done == count)) {
7633 break;
7634 }
7635 }
7636
7637 obj = table->getObject( aKey: gIONameMatchKey );
7638 if (obj) {
7639 done++;
7640 match = compareNames( name: obj, matched: changesOK ? &matched : NULL );
7641 if (!match) {
7642 break;
7643 }
7644 if (changesOK && matched) {
7645 // leave a hint as to which name matched
7646 table->setObject( aKey: gIONameMatchedKey, anObject: matched );
7647 matched->release();
7648 }
7649 if (done == count) {
7650 break;
7651 }
7652 }
7653
7654 str = OSDynamicCast( OSString, table->getObject( gIOLocationMatchKey ));
7655 if (str) {
7656 const OSSymbol * sym;
7657 done++;
7658 match = false;
7659 sym = copyLocation();
7660 if (sym) {
7661 match = sym->isEqualTo( anObject: str );
7662 sym->release();
7663 }
7664 if ((!match) || (done == count)) {
7665 break;
7666 }
7667 }
7668
7669 obj = table->getObject( aKey: gIOPropertyMatchKey );
7670 if (obj) {
7671 OSDictionary * nextDict;
7672 OSIterator * iter;
7673 done++;
7674 match = false;
7675 if (!matchProps) {
7676 matchProps = dictionaryWithProperties();
7677 }
7678 if (matchProps) {
7679 nextDict = OSDynamicCast( OSDictionary, obj);
7680 if (nextDict) {
7681 iter = NULL;
7682 } else {
7683 iter = OSCollectionIterator::withCollection(
7684 OSDynamicCast(OSCollection, obj));
7685 }
7686
7687 while (nextDict
7688 || (iter && (NULL != (nextDict = OSDynamicCast(OSDictionary,
7689 iter->getNextObject()))))) {
7690 match = matchProps->isEqualTo( aDictionary: nextDict, keys: nextDict);
7691 if (match) {
7692 break;
7693 }
7694 nextDict = NULL;
7695 }
7696 if (iter) {
7697 iter->release();
7698 }
7699 }
7700 if ((!match) || (done == count)) {
7701 break;
7702 }
7703 }
7704
7705 obj = table->getObject( aKey: gIOPropertyExistsMatchKey );
7706 if (obj) {
7707 OSString * nextKey;
7708 OSIterator * iter;
7709 done++;
7710 match = false;
7711 if (!matchProps) {
7712 matchProps = dictionaryWithProperties();
7713 }
7714 if (matchProps) {
7715 nextKey = OSDynamicCast( OSString, obj);
7716 if (nextKey) {
7717 iter = NULL;
7718 } else {
7719 iter = OSCollectionIterator::withCollection(
7720 OSDynamicCast(OSCollection, obj));
7721 }
7722
7723 while (nextKey
7724 || (iter && (NULL != (nextKey = OSDynamicCast(OSString,
7725 iter->getNextObject()))))) {
7726 match = (NULL != matchProps->getObject(aKey: nextKey));
7727 if (match) {
7728 break;
7729 }
7730 nextKey = NULL;
7731 }
7732 if (iter) {
7733 iter->release();
7734 }
7735 }
7736 if ((!match) || (done == count)) {
7737 break;
7738 }
7739 }
7740
7741 str = OSDynamicCast( OSString, table->getObject( gIOPathMatchKey ));
7742 if (str) {
7743 done++;
7744 entry = IORegistryEntry::fromPath( path: str->getCStringNoCopy());
7745 match = (this == entry);
7746 if (entry) {
7747 entry->release();
7748 }
7749 if (!match && matchProps && (obj = matchProps->getObject(aKey: gIOPathKey))) {
7750 match = str->isEqualTo(anObject: obj);
7751 }
7752 if ((!match) || (done == count)) {
7753 break;
7754 }
7755 }
7756
7757 num = OSDynamicCast( OSNumber, table->getObject( gIORegistryEntryIDKey ));
7758 if (num) {
7759 done++;
7760 match = (getRegistryEntryID() == num->unsigned64BitValue());
7761 if ((!match) || (done == count)) {
7762 break;
7763 }
7764 }
7765
7766 num = OSDynamicCast( OSNumber, table->getObject( gIOMatchedServiceCountKey ));
7767 if (num) {
7768 OSIterator * iter;
7769 IOService * service = NULL;
7770 UInt32 serviceCount = 0;
7771
7772 done++;
7773 iter = getClientIterator();
7774 if (iter) {
7775 while ((service = (IOService *) iter->getNextObject())) {
7776 if (kIOServiceInactiveState & service->__state[0]) {
7777 continue;
7778 }
7779 if (NULL == service->getProperty( aKey: gIOMatchCategoryKey )) {
7780 continue;
7781 }
7782 ++serviceCount;
7783 }
7784 iter->release();
7785 }
7786 match = (serviceCount == num->unsigned32BitValue());
7787 if ((!match) || (done == count)) {
7788 break;
7789 }
7790 }
7791
7792#define propMatch(key) \
7793 obj = table->getObject(key); \
7794 if (obj) \
7795 { \
7796 OSObject * prop; \
7797 done++; \
7798 prop = copyProperty(key); \
7799 match = obj->isEqualTo(prop); \
7800 if (prop) prop->release(); \
7801 if ((!match) || (done == count)) break; \
7802 }
7803 propMatch(gIOBSDNameKey)
7804 propMatch(gIOBSDMajorKey)
7805 propMatch(gIOBSDMinorKey)
7806 propMatch(gIOBSDUnitKey)
7807#undef propMatch
7808 }while (false);
7809
7810 OSSafeReleaseNULL(matchProps);
7811
7812 if (did) {
7813 *did = done;
7814 }
7815 return match;
7816}
7817
7818bool
7819IOService::passiveMatch( OSDictionary * table, bool changesOK )
7820{
7821 return matchPassive(table, options: changesOK ? kIOServiceChangesOK : 0);
7822}
7823
7824bool
7825IOService::matchPassive(OSDictionary * table, uint32_t options)
7826{
7827 IOService * where;
7828 OSDictionary * nextTable;
7829 SInt32 score;
7830 OSNumber * newPri;
7831 bool match = true;
7832 bool matchParent = false;
7833 uint32_t count;
7834 uint32_t done;
7835
7836 assert( table );
7837
7838#if defined(XNU_TARGET_OS_OSX)
7839 OSArray* aliasServiceRegIds = NULL;
7840 IOService* foundAlternateService = NULL;
7841#endif /* defined(XNU_TARGET_OS_OSX) */
7842
7843#if MATCH_DEBUG
7844 OSDictionary * root = table;
7845#endif
7846
7847 where = this;
7848 do{
7849 do{
7850 count = table->getCount();
7851 if (!(kIOServiceInternalDone & options)) {
7852 match = where->matchInternal(table, options, did: &done);
7853 // don't call family if we've done all the entries in the table
7854 if ((!match) || (done == count)) {
7855 break;
7856 }
7857 }
7858
7859 // pass in score from property table
7860 score = IOServiceObjectOrder( entry: table, ref: (void *) gIOProbeScoreKey);
7861
7862 // do family specific matching
7863 match = where->matchPropertyTable( table, score: &score );
7864
7865 if (!match) {
7866#if IOMATCHDEBUG
7867 if (kIOLogMatch & getDebugFlags( props: table )) {
7868 LOG(fmt: "%s: family specific matching fails\n", where->getName());
7869 }
7870#endif
7871 break;
7872 }
7873
7874 if (kIOServiceChangesOK & options) {
7875 // save the score
7876 newPri = OSNumber::withNumber( value: score, numberOfBits: 32 );
7877 if (newPri) {
7878 table->setObject( aKey: gIOProbeScoreKey, anObject: newPri );
7879 newPri->release();
7880 }
7881 }
7882
7883 options = 0;
7884 matchParent = false;
7885
7886 nextTable = OSDynamicCast(OSDictionary,
7887 table->getObject( gIOParentMatchKey ));
7888 if (nextTable) {
7889 // look for a matching entry anywhere up to root
7890 match = false;
7891 matchParent = true;
7892 table = nextTable;
7893 break;
7894 }
7895
7896 table = OSDynamicCast(OSDictionary,
7897 table->getObject( gIOLocationMatchKey ));
7898 if (table) {
7899 // look for a matching entry at matchLocation()
7900 match = false;
7901 where = where->getProvider();
7902 if (where && (where = where->matchLocation(where))) {
7903 continue;
7904 }
7905 }
7906 break;
7907 }while (true);
7908
7909 if (match == true) {
7910 break;
7911 }
7912
7913 if (matchParent == true) {
7914#if defined(XNU_TARGET_OS_OSX)
7915 // check if service has an alias to search its other "parents" if a parent match isn't found
7916 OSObject * prop = where->copyProperty(aKey: gIOServiceLegacyMatchingRegistryIDKey);
7917 OSNumber * alternateRegistryID = OSDynamicCast(OSNumber, prop);
7918 if (alternateRegistryID != NULL) {
7919 if (aliasServiceRegIds == NULL) {
7920 aliasServiceRegIds = OSArray::withCapacity(capacity: sizeof(alternateRegistryID));
7921 }
7922 aliasServiceRegIds->setObject(alternateRegistryID);
7923 }
7924 OSSafeReleaseNULL(prop);
7925#endif /* defined(XNU_TARGET_OS_OSX) */
7926 } else {
7927 break;
7928 }
7929
7930 where = where->getProvider();
7931#if defined(XNU_TARGET_OS_OSX)
7932 if (where == NULL) {
7933 // there were no matching parent services, check to see if there are aliased services that have a matching parent
7934 if (aliasServiceRegIds != NULL) {
7935 unsigned int numAliasedServices = aliasServiceRegIds->getCount();
7936 if (numAliasedServices != 0) {
7937 OSNumber* alternateRegistryID = OSDynamicCast(OSNumber, aliasServiceRegIds->getObject(numAliasedServices - 1));
7938 if (alternateRegistryID != NULL) {
7939 OSDictionary* alternateMatchingDict = IOService::registryEntryIDMatching(entryID: alternateRegistryID->unsigned64BitValue());
7940 aliasServiceRegIds->removeObject(index: numAliasedServices - 1);
7941 if (alternateMatchingDict != NULL) {
7942 OSSafeReleaseNULL(foundAlternateService);
7943 foundAlternateService = IOService::copyMatchingService(matching: alternateMatchingDict);
7944 alternateMatchingDict->release();
7945 if (foundAlternateService != NULL) {
7946 where = foundAlternateService;
7947 }
7948 }
7949 }
7950 }
7951 }
7952 }
7953#endif /* defined(XNU_TARGET_OS_OSX) */
7954 }while (where != NULL);
7955
7956#if defined(XNU_TARGET_OS_OSX)
7957 OSSafeReleaseNULL(foundAlternateService);
7958 OSSafeReleaseNULL(aliasServiceRegIds);
7959#endif /* defined(XNU_TARGET_OS_OSX) */
7960
7961#if MATCH_DEBUG
7962 if (where != this) {
7963 OSSerialize * s = OSSerialize::withCapacity(128);
7964 root->serialize(s);
7965 kprintf("parent match 0x%llx, %d,\n%s\n", getRegistryEntryID(), match, s->text());
7966 s->release();
7967 }
7968#endif
7969
7970 return match;
7971}
7972
7973
7974IOReturn
7975IOService::newUserClient( task_t owningTask, void * securityID,
7976 UInt32 type, OSDictionary * properties,
7977 IOUserClient ** handler )
7978{
7979 const OSSymbol *userClientClass = NULL;
7980 IOUserClient *client;
7981 OSObject *prop;
7982 OSObject *temp;
7983
7984 if (reserved && reserved->uvars && reserved->uvars->userServer) {
7985 return reserved->uvars->userServer->serviceNewUserClient(service: this, owningTask, securityID, type, properties, handler);
7986 }
7987
7988 if (kIOReturnSuccess == newUserClient( owningTask, securityID, type, handler )) {
7989 return kIOReturnSuccess;
7990 }
7991
7992 // First try my own properties for a user client class name
7993 prop = copyProperty(aKey: gIOUserClientClassKey);
7994 if (prop) {
7995 if (OSDynamicCast(OSSymbol, prop)) {
7996 userClientClass = (const OSSymbol *) prop;
7997 prop = NULL;
7998 } else if (OSDynamicCast(OSString, prop)) {
7999 userClientClass = OSSymbol::withString(aString: (OSString *) prop);
8000 OSSafeReleaseNULL(prop);
8001 if (userClientClass) {
8002 setProperty(aKey: gIOUserClientClassKey,
8003 anObject: (OSObject *) userClientClass);
8004 }
8005 } else {
8006 OSSafeReleaseNULL(prop);
8007 }
8008 }
8009
8010 // Didn't find one so lets just bomb out now without further ado.
8011 if (!userClientClass) {
8012 return kIOReturnUnsupported;
8013 }
8014
8015 // This reference is consumed by the IOServiceOpen call
8016 temp = OSMetaClass::allocClassWithName(name: userClientClass);
8017 OSSafeReleaseNULL(userClientClass);
8018 if (!temp) {
8019 return kIOReturnNoMemory;
8020 }
8021
8022 if (OSDynamicCast(IOUserClient, temp)) {
8023 client = (IOUserClient *) temp;
8024 } else {
8025 temp->release();
8026 return kIOReturnUnsupported;
8027 }
8028
8029 if (!client->initWithTask(owningTask, securityToken: securityID, type, properties)) {
8030 client->release();
8031 return kIOReturnBadArgument;
8032 }
8033
8034 if (!client->attach(provider: this)) {
8035 client->release();
8036 return kIOReturnUnsupported;
8037 }
8038
8039 if (!client->start(provider: this)) {
8040 client->detach(provider: this);
8041 client->release();
8042 return kIOReturnUnsupported;
8043 }
8044
8045 *handler = client;
8046 return kIOReturnSuccess;
8047}
8048
8049IOReturn
8050IOService::newUserClient( task_t owningTask, void * securityID,
8051 UInt32 type, OSDictionary * properties,
8052 OSSharedPtr<IOUserClient>& handler )
8053{
8054 IOUserClient* handlerRaw = NULL;
8055 IOReturn result = newUserClient(owningTask, securityID, type, properties, handler: &handlerRaw);
8056 handler.reset(p: handlerRaw, OSNoRetain);
8057 return result;
8058}
8059
8060IOReturn
8061IOService::newUserClient( task_t owningTask, void * securityID,
8062 UInt32 type, IOUserClient ** handler )
8063{
8064 return kIOReturnUnsupported;
8065}
8066
8067IOReturn
8068IOService::newUserClient( task_t owningTask, void * securityID,
8069 UInt32 type, OSSharedPtr<IOUserClient>& handler )
8070{
8071 IOUserClient* handlerRaw = nullptr;
8072 IOReturn result = IOService::newUserClient(owningTask, securityID, type, handler: &handlerRaw);
8073 handler.reset(p: handlerRaw, OSNoRetain);
8074 return result;
8075}
8076
8077
8078IOReturn
8079IOService::requestProbe( IOOptionBits options )
8080{
8081 return kIOReturnUnsupported;
8082}
8083
8084bool
8085IOService::hasUserServer() const
8086{
8087 return reserved && reserved->uvars && reserved->uvars->userServer;
8088}
8089
8090/*
8091 * Convert an IOReturn to text. Subclasses which add additional
8092 * IOReturn's should override this method and call
8093 * super::stringFromReturn if the desired value is not found.
8094 */
8095
8096const char *
8097IOService::stringFromReturn( IOReturn rtn )
8098{
8099 static const IONamedValue IOReturn_values[] = {
8100 {kIOReturnSuccess, .name: "success" },
8101 {kIOReturnError, .name: "general error" },
8102 {kIOReturnNoMemory, .name: "memory allocation error" },
8103 {kIOReturnNoResources, .name: "resource shortage" },
8104 {kIOReturnIPCError, .name: "Mach IPC failure" },
8105 {kIOReturnNoDevice, .name: "no such device" },
8106 {kIOReturnNotPrivileged, .name: "privilege violation" },
8107 {kIOReturnBadArgument, .name: "invalid argument" },
8108 {kIOReturnLockedRead, .name: "device is read locked" },
8109 {kIOReturnLockedWrite, .name: "device is write locked" },
8110 {kIOReturnExclusiveAccess, .name: "device is exclusive access" },
8111 {kIOReturnBadMessageID, .name: "bad IPC message ID" },
8112 {kIOReturnUnsupported, .name: "unsupported function" },
8113 {kIOReturnVMError, .name: "virtual memory error" },
8114 {kIOReturnInternalError, .name: "internal driver error" },
8115 {kIOReturnIOError, .name: "I/O error" },
8116 {kIOReturnCannotLock, .name: "cannot acquire lock" },
8117 {kIOReturnNotOpen, .name: "device is not open" },
8118 {kIOReturnNotReadable, .name: "device is not readable" },
8119 {kIOReturnNotWritable, .name: "device is not writeable" },
8120 {kIOReturnNotAligned, .name: "alignment error" },
8121 {kIOReturnBadMedia, .name: "media error" },
8122 {kIOReturnStillOpen, .name: "device is still open" },
8123 {kIOReturnRLDError, .name: "rld failure" },
8124 {kIOReturnDMAError, .name: "DMA failure" },
8125 {kIOReturnBusy, .name: "device is busy" },
8126 {kIOReturnTimeout, .name: "I/O timeout" },
8127 {kIOReturnOffline, .name: "device is offline" },
8128 {kIOReturnNotReady, .name: "device is not ready" },
8129 {kIOReturnNotAttached, .name: "device/channel is not attached" },
8130 {kIOReturnNoChannels, .name: "no DMA channels available" },
8131 {kIOReturnNoSpace, .name: "no space for data" },
8132 {kIOReturnPortExists, .name: "device port already exists" },
8133 {kIOReturnCannotWire, .name: "cannot wire physical memory" },
8134 {kIOReturnNoInterrupt, .name: "no interrupt attached" },
8135 {kIOReturnNoFrames, .name: "no DMA frames enqueued" },
8136 {kIOReturnMessageTooLarge, .name: "message is too large" },
8137 {kIOReturnNotPermitted, .name: "operation is not permitted" },
8138 {kIOReturnNoPower, .name: "device is without power" },
8139 {kIOReturnNoMedia, .name: "media is not present" },
8140 {kIOReturnUnformattedMedia, .name: "media is not formatted" },
8141 {kIOReturnUnsupportedMode, .name: "unsupported mode" },
8142 {kIOReturnUnderrun, .name: "data underrun" },
8143 {kIOReturnOverrun, .name: "data overrun" },
8144 {kIOReturnDeviceError, .name: "device error" },
8145 {kIOReturnNoCompletion, .name: "no completion routine" },
8146 {kIOReturnAborted, .name: "operation was aborted" },
8147 {kIOReturnNoBandwidth, .name: "bus bandwidth would be exceeded" },
8148 {kIOReturnNotResponding, .name: "device is not responding" },
8149 {kIOReturnInvalid, .name: "unanticipated driver error" },
8150 {.value: 0, NULL }
8151 };
8152
8153 return IOFindNameForValue(value: rtn, namedValueArray: IOReturn_values);
8154}
8155
8156/*
8157 * Convert an IOReturn to an errno.
8158 */
8159int
8160IOService::errnoFromReturn( IOReturn rtn )
8161{
8162 if (unix_err(err_get_code(rtn)) == rtn) {
8163 return err_get_code(rtn);
8164 }
8165
8166 switch (rtn) {
8167 // (obvious match)
8168 case kIOReturnSuccess:
8169 return 0;
8170 case kIOReturnNoMemory:
8171 return ENOMEM;
8172 case kIOReturnNoDevice:
8173 return ENXIO;
8174 case kIOReturnVMError:
8175 return EFAULT;
8176 case kIOReturnNotPermitted:
8177 return EPERM;
8178 case kIOReturnNotPrivileged:
8179 return EACCES;
8180 case kIOReturnIOError:
8181 return EIO;
8182 case kIOReturnNotWritable:
8183 return EROFS;
8184 case kIOReturnBadArgument:
8185 return EINVAL;
8186 case kIOReturnUnsupported:
8187 return ENOTSUP;
8188 case kIOReturnBusy:
8189 return EBUSY;
8190 case kIOReturnNoPower:
8191 return EPWROFF;
8192 case kIOReturnDeviceError:
8193 return EDEVERR;
8194 case kIOReturnTimeout:
8195 return ETIMEDOUT;
8196 case kIOReturnMessageTooLarge:
8197 return EMSGSIZE;
8198 case kIOReturnNoSpace:
8199 return ENOSPC;
8200 case kIOReturnCannotLock:
8201 return ENOLCK;
8202
8203 // (best match)
8204 case kIOReturnBadMessageID:
8205 case kIOReturnNoCompletion:
8206 case kIOReturnNotAligned:
8207 return EINVAL;
8208 case kIOReturnNotReady:
8209 return EBUSY;
8210 case kIOReturnRLDError:
8211 return EBADMACHO;
8212 case kIOReturnPortExists:
8213 case kIOReturnStillOpen:
8214 return EEXIST;
8215 case kIOReturnExclusiveAccess:
8216 case kIOReturnLockedRead:
8217 case kIOReturnLockedWrite:
8218 case kIOReturnNotOpen:
8219 case kIOReturnNotReadable:
8220 return EACCES;
8221 case kIOReturnCannotWire:
8222 case kIOReturnNoResources:
8223 return ENOMEM;
8224 case kIOReturnAborted:
8225 case kIOReturnOffline:
8226 case kIOReturnNotResponding:
8227 return EBUSY;
8228 case kIOReturnBadMedia:
8229 case kIOReturnNoMedia:
8230 case kIOReturnNotAttached:
8231 case kIOReturnUnformattedMedia:
8232 return ENXIO; // (media error)
8233 case kIOReturnDMAError:
8234 case kIOReturnOverrun:
8235 case kIOReturnUnderrun:
8236 return EIO; // (transfer error)
8237 case kIOReturnNoBandwidth:
8238 case kIOReturnNoChannels:
8239 case kIOReturnNoFrames:
8240 case kIOReturnNoInterrupt:
8241 return EIO; // (hardware error)
8242 case kIOReturnError:
8243 case kIOReturnInternalError:
8244 case kIOReturnInvalid:
8245 return EIO; // (generic error)
8246 case kIOReturnIPCError:
8247 return EIO; // (ipc error)
8248 default:
8249 return EIO; // (all other errors)
8250 }
8251}
8252
8253IOReturn
8254IOService::message( UInt32 type, IOService * provider,
8255 void * argument )
8256{
8257 /*
8258 * Generic entry point for calls from the provider. A return value of
8259 * kIOReturnSuccess indicates that the message was received, and where
8260 * applicable, that it was successful.
8261 */
8262
8263 return kIOReturnUnsupported;
8264}
8265
8266/*
8267 * Device memory
8268 */
8269
8270IOItemCount
8271IOService::getDeviceMemoryCount( void )
8272{
8273 OSArray * array;
8274 IOItemCount count;
8275
8276 array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8277 if (array) {
8278 count = array->getCount();
8279 } else {
8280 count = 0;
8281 }
8282
8283 return count;
8284}
8285
8286IODeviceMemory *
8287IOService::getDeviceMemoryWithIndex( unsigned int index )
8288{
8289 OSArray * array;
8290 IODeviceMemory * range;
8291
8292 array = OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8293 if (array) {
8294 range = (IODeviceMemory *) array->getObject( index );
8295 } else {
8296 range = NULL;
8297 }
8298
8299 return range;
8300}
8301
8302IOMemoryMap *
8303IOService::mapDeviceMemoryWithIndex( unsigned int index,
8304 IOOptionBits options )
8305{
8306 IODeviceMemory * range;
8307 IOMemoryMap * map;
8308
8309 range = getDeviceMemoryWithIndex( index );
8310 if (range) {
8311 map = range->map( options );
8312 } else {
8313 map = NULL;
8314 }
8315
8316 return map;
8317}
8318
8319OSArray *
8320IOService::getDeviceMemory( void )
8321{
8322 return OSDynamicCast( OSArray, getProperty( gIODeviceMemoryKey));
8323}
8324
8325
8326void
8327IOService::setDeviceMemory( OSArray * array )
8328{
8329 setProperty( aKey: gIODeviceMemoryKey, anObject: array);
8330}
8331
8332static void
8333requireMaxCpuDelay(IOService * service, UInt32 ns, UInt32 delayType)
8334{
8335 static const UInt kNoReplace = -1U; // Must be an illegal index
8336 UInt replace = kNoReplace;
8337 bool setCpuDelay = false;
8338
8339 IORecursiveLockLock(lock: sCpuDelayLock);
8340
8341 UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
8342 CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
8343 IOService * holder = NULL;
8344
8345 if (ns) {
8346 const CpuDelayEntry ne = {.fService: service, .fMaxDelay: ns, .fDelayType: delayType};
8347 holder = service;
8348 // Set maximum delay.
8349 for (UInt i = 0; i < count; i++) {
8350 IOService *thisService = entries[i].fService;
8351 bool sameType = (delayType == entries[i].fDelayType);
8352 if ((service == thisService) && sameType) {
8353 replace = i;
8354 } else if (!thisService) {
8355 if (kNoReplace == replace) {
8356 replace = i;
8357 }
8358 } else if (sameType) {
8359 const UInt32 thisMax = entries[i].fMaxDelay;
8360 if (thisMax < ns) {
8361 ns = thisMax;
8362 holder = thisService;
8363 }
8364 }
8365 }
8366
8367 setCpuDelay = true;
8368 if (kNoReplace == replace) {
8369 sCpuDelayData->appendBytes(bytes: &ne, numBytes: sizeof(ne));
8370 } else {
8371 entries[replace] = ne;
8372 }
8373 } else {
8374 ns = -1U; // Set to max unsigned, i.e. no restriction
8375
8376 for (UInt i = 0; i < count; i++) {
8377 // Clear a maximum delay.
8378 IOService *thisService = entries[i].fService;
8379 if (thisService && (delayType == entries[i].fDelayType)) {
8380 UInt32 thisMax = entries[i].fMaxDelay;
8381 if (service == thisService) {
8382 replace = i;
8383 } else if (thisMax < ns) {
8384 ns = thisMax;
8385 holder = thisService;
8386 }
8387 }
8388 }
8389
8390 // Check if entry found
8391 if (kNoReplace != replace) {
8392 entries[replace].fService = NULL; // Null the entry
8393 setCpuDelay = true;
8394 }
8395 }
8396
8397 if (setCpuDelay) {
8398 if (holder && debug_boot_arg) {
8399 strlcpy(dst: sCPULatencyHolderName[delayType], src: holder->getName(), n: sizeof(sCPULatencyHolderName[delayType]));
8400 }
8401
8402 // Must be safe to call from locked context
8403 if (delayType == kCpuDelayBusStall) {
8404#if defined(__x86_64__)
8405 ml_set_maxbusdelay(ns);
8406#endif /* defined(__x86_64__) */
8407 }
8408#if defined(__x86_64__)
8409 else if (delayType == kCpuDelayInterrupt) {
8410 ml_set_maxintdelay(ns);
8411 }
8412#endif /* defined(__x86_64__) */
8413 sCPULatencyHolder[delayType]->setValue(holder ? holder->getRegistryEntryID() : 0);
8414 sCPULatencySet[delayType]->setValue(ns);
8415
8416 OSArray * handlers = sCpuLatencyHandlers[delayType];
8417 IOService * target;
8418 if (handlers) {
8419 for (unsigned int idx = 0;
8420 (target = (IOService *) handlers->getObject(index: idx));
8421 idx++) {
8422 target->callPlatformFunction(functionName: sCPULatencyFunctionName[delayType], waitForFunction: false,
8423 param1: (void *) (uintptr_t) ns, param2: holder,
8424 NULL, NULL);
8425 }
8426 }
8427 }
8428
8429 IORecursiveLockUnlock(lock: sCpuDelayLock);
8430}
8431
8432static IOReturn
8433setLatencyHandler(UInt32 delayType, IOService * target, bool enable)
8434{
8435 IOReturn result = kIOReturnNotFound;
8436 OSArray * array;
8437 unsigned int idx;
8438
8439 IORecursiveLockLock(lock: sCpuDelayLock);
8440
8441 do{
8442 if (enable && !sCpuLatencyHandlers[delayType]) {
8443 sCpuLatencyHandlers[delayType] = OSArray::withCapacity(capacity: 4);
8444 }
8445 array = sCpuLatencyHandlers[delayType];
8446 if (!array) {
8447 break;
8448 }
8449 idx = array->getNextIndexOfObject(anObject: target, index: 0);
8450 if (!enable) {
8451 if (-1U != idx) {
8452 array->removeObject(index: idx);
8453 result = kIOReturnSuccess;
8454 }
8455 } else {
8456 if (-1U != idx) {
8457 result = kIOReturnExclusiveAccess;
8458 break;
8459 }
8460 array->setObject(target);
8461
8462 UInt count = sCpuDelayData->getLength() / sizeof(CpuDelayEntry);
8463 CpuDelayEntry *entries = (CpuDelayEntry *) sCpuDelayData->getBytesNoCopy();
8464 UInt32 ns = -1U; // Set to max unsigned, i.e. no restriction
8465 IOService * holder = NULL;
8466
8467 for (UInt i = 0; i < count; i++) {
8468 if (entries[i].fService
8469 && (delayType == entries[i].fDelayType)
8470 && (entries[i].fMaxDelay < ns)) {
8471 ns = entries[i].fMaxDelay;
8472 holder = entries[i].fService;
8473 }
8474 }
8475 target->callPlatformFunction(functionName: sCPULatencyFunctionName[delayType], waitForFunction: false,
8476 param1: (void *) (uintptr_t) ns, param2: holder,
8477 NULL, NULL);
8478 result = kIOReturnSuccess;
8479 }
8480 }while (false);
8481
8482 IORecursiveLockUnlock(lock: sCpuDelayLock);
8483
8484 return result;
8485}
8486
8487IOReturn
8488IOService::requireMaxBusStall(UInt32 ns)
8489{
8490#if !defined(__x86_64__)
8491 switch (ns) {
8492 case kIOMaxBusStall40usec:
8493 case kIOMaxBusStall30usec:
8494 case kIOMaxBusStall25usec:
8495 case kIOMaxBusStall20usec:
8496 case kIOMaxBusStall10usec:
8497 case kIOMaxBusStall5usec:
8498 case kIOMaxBusStallNone:
8499 break;
8500 default:
8501 return kIOReturnBadArgument;
8502 }
8503#endif /* !defined(__x86_64__) */
8504 requireMaxCpuDelay(service: this, ns, delayType: kCpuDelayBusStall);
8505 return kIOReturnSuccess;
8506}
8507
8508IOReturn
8509IOService::requireMaxInterruptDelay(uint32_t ns)
8510{
8511#if defined(__x86_64__)
8512 requireMaxCpuDelay(this, ns, kCpuDelayInterrupt);
8513 return kIOReturnSuccess;
8514#else /* defined(__x86_64__) */
8515 return kIOReturnUnsupported;
8516#endif /* defined(__x86_64__) */
8517}
8518
8519/*
8520 * Device interrupts
8521 */
8522
8523IOReturn
8524IOService::resolveInterrupt(IOService *nub, int source)
8525{
8526 IOInterruptController *interruptController;
8527 OSArray *array;
8528 OSData *data;
8529 OSSymbol *interruptControllerName;
8530 unsigned int numSources;
8531 IOInterruptSource *interruptSources;
8532 IOInterruptSourcePrivate *interruptSourcesPrivate;
8533
8534 // Get the parents list from the nub.
8535 array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptControllersKey));
8536 if (array == NULL) {
8537 return kIOReturnNoResources;
8538 }
8539
8540 // Allocate space for the IOInterruptSources if needed... then return early.
8541 if (nub->_interruptSources == NULL) {
8542 numSources = array->getCount();
8543 interruptSources = IONewZero(IOInterruptSource, numSources);
8544 interruptSourcesPrivate = IONewZero(IOInterruptSourcePrivate, numSources);
8545
8546 if (interruptSources == NULL || interruptSourcesPrivate == NULL) {
8547 IODelete(interruptSources, IOInterruptSource, numSources);
8548 IODelete(interruptSourcesPrivate, IOInterruptSourcePrivate, numSources);
8549 return kIOReturnNoMemory;
8550 }
8551
8552 nub->_numInterruptSources = numSources;
8553 nub->_interruptSources = interruptSources;
8554 nub->reserved->interruptSourcesPrivate = interruptSourcesPrivate;
8555 return kIOReturnSuccess;
8556 }
8557
8558 interruptControllerName = OSDynamicCast(OSSymbol, array->getObject(source));
8559 if (interruptControllerName == NULL) {
8560 return kIOReturnNoResources;
8561 }
8562
8563 interruptController = getPlatform()->lookUpInterruptController(name: interruptControllerName);
8564 if (interruptController == NULL) {
8565 return kIOReturnNoResources;
8566 }
8567
8568 // Get the interrupt numbers from the nub.
8569 array = OSDynamicCast(OSArray, nub->getProperty(gIOInterruptSpecifiersKey));
8570 if (array == NULL) {
8571 return kIOReturnNoResources;
8572 }
8573 data = OSDynamicCast(OSData, array->getObject(source));
8574 if (data == NULL) {
8575 return kIOReturnNoResources;
8576 }
8577
8578 // Set the interruptController and interruptSource in the nub's table.
8579 interruptSources = nub->_interruptSources;
8580 interruptSources[source].interruptController = interruptController;
8581 interruptSources[source].vectorData = data;
8582
8583 return kIOReturnSuccess;
8584}
8585
8586IOReturn
8587IOService::lookupInterrupt(int source, bool resolve, IOInterruptController **interruptController)
8588{
8589 IOReturn ret;
8590
8591 /* Make sure the _interruptSources are set */
8592 if (_interruptSources == NULL) {
8593 ret = resolveInterrupt(nub: this, source);
8594 if (ret != kIOReturnSuccess) {
8595 return ret;
8596 }
8597 }
8598
8599 /* Make sure the local source number is valid */
8600 if ((source < 0) || (source >= _numInterruptSources)) {
8601 return kIOReturnNoInterrupt;
8602 }
8603
8604 /* Look up the contoller for the local source */
8605 *interruptController = _interruptSources[source].interruptController;
8606
8607 if (*interruptController == NULL) {
8608 if (!resolve) {
8609 return kIOReturnNoInterrupt;
8610 }
8611
8612 /* Try to resolve the interrupt */
8613 ret = resolveInterrupt(nub: this, source);
8614 if (ret != kIOReturnSuccess) {
8615 return ret;
8616 }
8617
8618 *interruptController = _interruptSources[source].interruptController;
8619 }
8620
8621 return kIOReturnSuccess;
8622}
8623
8624IOReturn
8625IOService::registerInterrupt(int source, OSObject *target,
8626 IOInterruptAction handler,
8627 void *refCon)
8628{
8629 IOInterruptController *interruptController;
8630 IOReturn ret;
8631
8632 ret = lookupInterrupt(source, resolve: true, interruptController: &interruptController);
8633 if (ret != kIOReturnSuccess) {
8634 return ret;
8635 }
8636
8637 /* Register the source */
8638 return interruptController->registerInterrupt(nub: this, source, target,
8639 handler: (IOInterruptHandler)handler,
8640 refCon);
8641}
8642
8643static void
8644IOServiceInterruptActionToBlock( OSObject * target, void * refCon,
8645 IOService * nub, int source )
8646{
8647 ((IOInterruptActionBlock)(refCon))(nub, source);
8648}
8649
8650IOReturn
8651IOService::registerInterruptBlock(int source, OSObject *target,
8652 IOInterruptActionBlock handler)
8653{
8654 IOReturn ret;
8655 void * block;
8656
8657 block = Block_copy(handler);
8658 if (!block) {
8659 return kIOReturnNoMemory;
8660 }
8661
8662 ret = registerInterrupt(source, target, handler: &IOServiceInterruptActionToBlock, refCon: block);
8663 if (kIOReturnSuccess != ret) {
8664 Block_release(block);
8665 return ret;
8666 }
8667
8668 reserved->interruptSourcesPrivate[source].vectorBlock = block;
8669
8670 return ret;
8671}
8672
8673IOReturn
8674IOService::unregisterInterrupt(int source)
8675{
8676 IOReturn ret;
8677 IOInterruptController *interruptController;
8678 IOInterruptSourcePrivate *priv;
8679 void *block;
8680
8681 ret = lookupInterrupt(source, resolve: false, interruptController: &interruptController);
8682 if (ret != kIOReturnSuccess) {
8683 return ret;
8684 }
8685
8686 /* Unregister the source */
8687 priv = &reserved->interruptSourcesPrivate[source];
8688 block = priv->vectorBlock;
8689 ret = interruptController->unregisterInterrupt(nub: this, source);
8690 if ((kIOReturnSuccess == ret) && (block = priv->vectorBlock)) {
8691 priv->vectorBlock = NULL;
8692 Block_release(block);
8693 }
8694
8695 return ret;
8696}
8697
8698IOReturn
8699IOService::addInterruptStatistics(IOInterruptAccountingData * statistics, int source)
8700{
8701 IOReportLegend * legend = NULL;
8702 IOInterruptAccountingData * oldValue = NULL;
8703 IOInterruptAccountingReporter * newArray = NULL;
8704 char subgroupName[64];
8705 int newArraySize = 0;
8706 int i = 0;
8707
8708 if (source < 0) {
8709 return kIOReturnBadArgument;
8710 }
8711
8712 /*
8713 * We support statistics on a maximum of 256 interrupts per nub; if a nub
8714 * has more than 256 interrupt specifiers associated with it, and tries
8715 * to register a high interrupt index with interrupt accounting, panic.
8716 * Having more than 256 interrupts associated with a single nub is
8717 * probably a sign that something fishy is going on.
8718 */
8719 if (source > IA_INDEX_MAX) {
8720 panic("addInterruptStatistics called for an excessively large index (%d)", source);
8721 }
8722
8723 /*
8724 * TODO: This is ugly (wrapping a lock around an allocation). I'm only
8725 * leaving it as is because the likelihood of contention where we are
8726 * actually growing the array is minimal (we would realistically need
8727 * to be starting a driver for the first time, with an IOReporting
8728 * client already in place). Nonetheless, cleanup that can be done
8729 * to adhere to best practices; it'll make the code more complicated,
8730 * unfortunately.
8731 */
8732 IOLockLock(&reserved->interruptStatisticsLock);
8733
8734 /*
8735 * Lazily allocate the statistics array.
8736 */
8737 if (!reserved->interruptStatisticsArray) {
8738 reserved->interruptStatisticsArray = IONew(IOInterruptAccountingReporter, 1);
8739 assert(reserved->interruptStatisticsArray);
8740 reserved->interruptStatisticsArrayCount = 1;
8741 bzero(s: reserved->interruptStatisticsArray, n: sizeof(*reserved->interruptStatisticsArray));
8742 }
8743
8744 if (source >= reserved->interruptStatisticsArrayCount) {
8745 /*
8746 * We're still within the range of supported indices, but we are out
8747 * of space in the current array. Do a nasty realloc (because
8748 * IORealloc isn't a thing) here. We'll double the size with each
8749 * reallocation.
8750 *
8751 * Yes, the "next power of 2" could be more efficient; but this will
8752 * be invoked incredibly rarely. Who cares.
8753 */
8754 newArraySize = (reserved->interruptStatisticsArrayCount << 1);
8755
8756 while (newArraySize <= source) {
8757 newArraySize = (newArraySize << 1);
8758 }
8759 newArray = IONew(IOInterruptAccountingReporter, newArraySize);
8760
8761 assert(newArray);
8762
8763 /*
8764 * TODO: This even zeroes the memory it is about to overwrite.
8765 * Shameful; fix it. Not particularly high impact, however.
8766 */
8767 bzero(s: newArray, n: newArraySize * sizeof(*newArray));
8768 memcpy(dst: newArray, src: reserved->interruptStatisticsArray, n: reserved->interruptStatisticsArrayCount * sizeof(*newArray));
8769 IODelete(reserved->interruptStatisticsArray, IOInterruptAccountingReporter, reserved->interruptStatisticsArrayCount);
8770 reserved->interruptStatisticsArray = newArray;
8771 reserved->interruptStatisticsArrayCount = newArraySize;
8772 }
8773
8774 if (!reserved->interruptStatisticsArray[source].reporter) {
8775 /*
8776 * We don't have a reporter associated with this index yet, so we
8777 * need to create one.
8778 */
8779 /*
8780 * TODO: Some statistics do in fact have common units (time); should this be
8781 * split into separate reporters to communicate this?
8782 */
8783 reserved->interruptStatisticsArray[source].reporter = IOSimpleReporter::with(reportingService: this, kIOReportCategoryPower, kIOReportUnitNone);
8784
8785 /*
8786 * Each statistic is given an identifier based on the interrupt index (which
8787 * should be unique relative to any single nub) and the statistic involved.
8788 * We should now have a sane (small and positive) index, so start
8789 * constructing the channels for statistics.
8790 */
8791 for (i = 0; i < IA_NUM_INTERRUPT_ACCOUNTING_STATISTICS; i++) {
8792 /*
8793 * TODO: Currently, this does not add channels for disabled statistics.
8794 * Will this be confusing for clients? If so, we should just add the
8795 * channels; we can avoid updating the channels even if they exist.
8796 */
8797 if (IA_GET_STATISTIC_ENABLED(i)) {
8798 reserved->interruptStatisticsArray[source].reporter->addChannel(IA_GET_CHANNEL_ID(source, i), channelName: kInterruptAccountingStatisticNameArray[i]);
8799 }
8800 }
8801
8802 /*
8803 * We now need to add the legend for this reporter to the registry.
8804 */
8805 OSObject * prop = copyProperty(kIOReportLegendKey);
8806 legend = IOReportLegend::with(OSDynamicCast(OSArray, prop));
8807 OSSafeReleaseNULL(prop);
8808
8809 /*
8810 * Note that while we compose the subgroup name, we do not need to
8811 * manage its lifecycle (the reporter will handle this).
8812 */
8813 snprintf(subgroupName, count: sizeof(subgroupName), "%s %d", getName(), source);
8814 subgroupName[sizeof(subgroupName) - 1] = 0;
8815 legend->addReporterLegend(reporter: reserved->interruptStatisticsArray[source].reporter, kInterruptAccountingGroupName, subGroupName: subgroupName);
8816 setProperty(kIOReportLegendKey, anObject: legend->getLegend());
8817 legend->release();
8818
8819 /*
8820 * TODO: Is this a good idea? Probably not; my assumption is it opts
8821 * all entities who register interrupts into public disclosure of all
8822 * IOReporting channels. Unfortunately, this appears to be as fine
8823 * grain as it gets.
8824 */
8825 setProperty(kIOReportLegendPublicKey, aBoolean: true);
8826 }
8827
8828 /*
8829 * Don't stomp existing entries. If we are about to, panic; this
8830 * probably means we failed to tear down our old interrupt source
8831 * correctly.
8832 */
8833 oldValue = reserved->interruptStatisticsArray[source].statistics;
8834
8835 if (oldValue) {
8836 panic("addInterruptStatistics call for index %d would have clobbered existing statistics", source);
8837 }
8838
8839 reserved->interruptStatisticsArray[source].statistics = statistics;
8840
8841 /*
8842 * Inherit the reporter values for each statistic. The target may
8843 * be torn down as part of the runtime of the service (especially
8844 * for sleep/wake), so we inherit in order to avoid having values
8845 * reset for no apparent reason. Our statistics are ultimately
8846 * tied to the index and the sevice, not to an individual target,
8847 * so we should maintain them accordingly.
8848 */
8849 interruptAccountingDataInheritChannels(data: reserved->interruptStatisticsArray[source].statistics, reporter: reserved->interruptStatisticsArray[source].reporter);
8850
8851 IOLockUnlock(&reserved->interruptStatisticsLock);
8852
8853 return kIOReturnSuccess;
8854}
8855
8856IOReturn
8857IOService::removeInterruptStatistics(int source)
8858{
8859 IOInterruptAccountingData * value = NULL;
8860
8861 if (source < 0) {
8862 return kIOReturnBadArgument;
8863 }
8864
8865 IOLockLock(&reserved->interruptStatisticsLock);
8866
8867 /*
8868 * We dynamically grow the statistics array, so an excessively
8869 * large index value has NEVER been registered. This either
8870 * means our cap on the array size is too small (unlikely), or
8871 * that we have been passed a corrupt index (this must be passed
8872 * the plain index into the interrupt specifier list).
8873 */
8874 if (source >= reserved->interruptStatisticsArrayCount) {
8875 panic("removeInterruptStatistics called for index %d, which was never registered", source);
8876 }
8877
8878 assert(reserved->interruptStatisticsArray);
8879
8880 /*
8881 * If there is no existing entry, we are most likely trying to
8882 * free an interrupt owner twice, or we have corrupted the
8883 * index value.
8884 */
8885 value = reserved->interruptStatisticsArray[source].statistics;
8886
8887 if (!value) {
8888 panic("removeInterruptStatistics called for empty index %d", source);
8889 }
8890
8891 /*
8892 * We update the statistics, so that any delta with the reporter
8893 * state is not lost.
8894 */
8895 interruptAccountingDataUpdateChannels(data: reserved->interruptStatisticsArray[source].statistics, reporter: reserved->interruptStatisticsArray[source].reporter);
8896 reserved->interruptStatisticsArray[source].statistics = NULL;
8897 IOLockUnlock(&reserved->interruptStatisticsLock);
8898
8899 return kIOReturnSuccess;
8900}
8901
8902IOReturn
8903IOService::getInterruptType(int source, int *interruptType)
8904{
8905 IOInterruptController *interruptController;
8906 IOReturn ret;
8907
8908 ret = lookupInterrupt(source, resolve: true, interruptController: &interruptController);
8909 if (ret != kIOReturnSuccess) {
8910 return ret;
8911 }
8912
8913 /* Return the type */
8914 return interruptController->getInterruptType(nub: this, source, interruptType);
8915}
8916
8917IOReturn
8918IOService::enableInterrupt(int source)
8919{
8920 IOInterruptController *interruptController;
8921 IOReturn ret;
8922
8923 ret = lookupInterrupt(source, resolve: false, interruptController: &interruptController);
8924 if (ret != kIOReturnSuccess) {
8925 return ret;
8926 }
8927
8928 /* Enable the source */
8929 return interruptController->enableInterrupt(nub: this, source);
8930}
8931
8932IOReturn
8933IOService::disableInterrupt(int source)
8934{
8935 IOInterruptController *interruptController;
8936 IOReturn ret;
8937
8938 ret = lookupInterrupt(source, resolve: false, interruptController: &interruptController);
8939 if (ret != kIOReturnSuccess) {
8940 return ret;
8941 }
8942
8943 /* Disable the source */
8944 return interruptController->disableInterrupt(nub: this, source);
8945}
8946
8947IOReturn
8948IOService::causeInterrupt(int source)
8949{
8950 IOInterruptController *interruptController;
8951 IOReturn ret;
8952
8953 ret = lookupInterrupt(source, resolve: false, interruptController: &interruptController);
8954 if (ret != kIOReturnSuccess) {
8955 return ret;
8956 }
8957
8958 /* Cause an interrupt for the source */
8959 return interruptController->causeInterrupt(nub: this, source);
8960}
8961
8962IOReturn
8963IOService::configureReport(IOReportChannelList *channelList,
8964 IOReportConfigureAction action,
8965 void *result,
8966 void *destination)
8967{
8968 unsigned cnt;
8969
8970 for (cnt = 0; cnt < channelList->nchannels; cnt++) {
8971 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
8972 if (pwrMgt) {
8973 configurePowerStatesReport(action, result);
8974 } else {
8975 return kIOReturnUnsupported;
8976 }
8977 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
8978 if (pwrMgt) {
8979 configureSimplePowerReport(action, result);
8980 } else {
8981 return kIOReturnUnsupported;
8982 }
8983 }
8984 }
8985
8986 IOLockLock(&reserved->interruptStatisticsLock);
8987
8988 /* The array count is signed (because the interrupt indices are signed), hence the cast */
8989 for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
8990 if (reserved->interruptStatisticsArray[cnt].reporter) {
8991 /*
8992 * If the reporter is currently associated with the statistics
8993 * for an event source, we may need to update the reporter.
8994 */
8995 if (reserved->interruptStatisticsArray[cnt].statistics) {
8996 interruptAccountingDataUpdateChannels(data: reserved->interruptStatisticsArray[cnt].statistics, reporter: reserved->interruptStatisticsArray[cnt].reporter);
8997 }
8998
8999 reserved->interruptStatisticsArray[cnt].reporter->configureReport(channelList, action, result, destination);
9000 }
9001 }
9002
9003 IOLockUnlock(&reserved->interruptStatisticsLock);
9004
9005 if (hasUserServer()) {
9006 return _ConfigureReport(channels: channelList, action, result, destination);
9007 } else {
9008 return kIOReturnSuccess;
9009 }
9010}
9011
9012IOReturn
9013IOService::updateReport(IOReportChannelList *channelList,
9014 IOReportUpdateAction action,
9015 void *result,
9016 void *destination)
9017{
9018 unsigned cnt;
9019
9020 for (cnt = 0; cnt < channelList->nchannels; cnt++) {
9021 if (channelList->channels[cnt].channel_id == kPMPowerStatesChID) {
9022 if (pwrMgt) {
9023 updatePowerStatesReport(action, result, destination);
9024 } else {
9025 return kIOReturnUnsupported;
9026 }
9027 } else if (channelList->channels[cnt].channel_id == kPMCurrStateChID) {
9028 if (pwrMgt) {
9029 updateSimplePowerReport(action, result, destination);
9030 } else {
9031 return kIOReturnUnsupported;
9032 }
9033 }
9034 }
9035
9036 IOLockLock(&reserved->interruptStatisticsLock);
9037
9038 /* The array count is signed (because the interrupt indices are signed), hence the cast */
9039 for (cnt = 0; cnt < (unsigned) reserved->interruptStatisticsArrayCount; cnt++) {
9040 if (reserved->interruptStatisticsArray[cnt].reporter) {
9041 /*
9042 * If the reporter is currently associated with the statistics
9043 * for an event source, we need to update the reporter.
9044 */
9045 if (reserved->interruptStatisticsArray[cnt].statistics) {
9046 interruptAccountingDataUpdateChannels(data: reserved->interruptStatisticsArray[cnt].statistics, reporter: reserved->interruptStatisticsArray[cnt].reporter);
9047 }
9048
9049 reserved->interruptStatisticsArray[cnt].reporter->updateReport(channelList, action, result, destination);
9050 }
9051 }
9052
9053 IOLockUnlock(&reserved->interruptStatisticsLock);
9054
9055
9056 if (hasUserServer()) {
9057 return _UpdateReport(channels: channelList, action, result, destination);
9058 } else {
9059 return kIOReturnSuccess;
9060 }
9061}
9062
9063uint64_t
9064IOService::getAuthorizationID( void )
9065{
9066 return reserved->authorizationID;
9067}
9068
9069IOReturn
9070IOService::setAuthorizationID( uint64_t authorizationID )
9071{
9072 OSObject * entitlement;
9073 IOReturn status;
9074
9075 entitlement = IOUserClient::copyClientEntitlement( task: current_task(), entitlement: "com.apple.private.iokit.IOServiceSetAuthorizationID" );
9076
9077 if (entitlement) {
9078 if (entitlement == kOSBooleanTrue) {
9079 reserved->authorizationID = authorizationID;
9080
9081 status = kIOReturnSuccess;
9082 } else {
9083 status = kIOReturnNotPrivileged;
9084 }
9085
9086 entitlement->release();
9087 } else {
9088 status = kIOReturnNotPrivileged;
9089 }
9090
9091 return status;
9092}
9093
9094/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
9095
9096#if __LP64__
9097OSMetaClassDefineReservedUsedX86(IOService, 0);
9098OSMetaClassDefineReservedUsedX86(IOService, 1);
9099OSMetaClassDefineReservedUnused(IOService, 2);
9100OSMetaClassDefineReservedUnused(IOService, 3);
9101OSMetaClassDefineReservedUnused(IOService, 4);
9102OSMetaClassDefineReservedUnused(IOService, 5);
9103OSMetaClassDefineReservedUnused(IOService, 6);
9104OSMetaClassDefineReservedUnused(IOService, 7);
9105#else
9106OSMetaClassDefineReservedUsedX86(IOService, 0);
9107OSMetaClassDefineReservedUsedX86(IOService, 1);
9108OSMetaClassDefineReservedUsedX86(IOService, 2);
9109OSMetaClassDefineReservedUsedX86(IOService, 3);
9110OSMetaClassDefineReservedUsedX86(IOService, 4);
9111OSMetaClassDefineReservedUsedX86(IOService, 5);
9112OSMetaClassDefineReservedUsedX86(IOService, 6);
9113OSMetaClassDefineReservedUsedX86(IOService, 7);
9114#endif
9115OSMetaClassDefineReservedUnused(IOService, 8);
9116OSMetaClassDefineReservedUnused(IOService, 9);
9117OSMetaClassDefineReservedUnused(IOService, 10);
9118OSMetaClassDefineReservedUnused(IOService, 11);
9119OSMetaClassDefineReservedUnused(IOService, 12);
9120OSMetaClassDefineReservedUnused(IOService, 13);
9121OSMetaClassDefineReservedUnused(IOService, 14);
9122OSMetaClassDefineReservedUnused(IOService, 15);
9123OSMetaClassDefineReservedUnused(IOService, 16);
9124OSMetaClassDefineReservedUnused(IOService, 17);
9125OSMetaClassDefineReservedUnused(IOService, 18);
9126OSMetaClassDefineReservedUnused(IOService, 19);
9127OSMetaClassDefineReservedUnused(IOService, 20);
9128OSMetaClassDefineReservedUnused(IOService, 21);
9129OSMetaClassDefineReservedUnused(IOService, 22);
9130OSMetaClassDefineReservedUnused(IOService, 23);
9131OSMetaClassDefineReservedUnused(IOService, 24);
9132OSMetaClassDefineReservedUnused(IOService, 25);
9133OSMetaClassDefineReservedUnused(IOService, 26);
9134OSMetaClassDefineReservedUnused(IOService, 27);
9135OSMetaClassDefineReservedUnused(IOService, 28);
9136OSMetaClassDefineReservedUnused(IOService, 29);
9137OSMetaClassDefineReservedUnused(IOService, 30);
9138OSMetaClassDefineReservedUnused(IOService, 31);
9139OSMetaClassDefineReservedUnused(IOService, 32);
9140OSMetaClassDefineReservedUnused(IOService, 33);
9141OSMetaClassDefineReservedUnused(IOService, 34);
9142OSMetaClassDefineReservedUnused(IOService, 35);
9143OSMetaClassDefineReservedUnused(IOService, 36);
9144OSMetaClassDefineReservedUnused(IOService, 37);
9145OSMetaClassDefineReservedUnused(IOService, 38);
9146OSMetaClassDefineReservedUnused(IOService, 39);
9147OSMetaClassDefineReservedUnused(IOService, 40);
9148OSMetaClassDefineReservedUnused(IOService, 41);
9149OSMetaClassDefineReservedUnused(IOService, 42);
9150OSMetaClassDefineReservedUnused(IOService, 43);
9151OSMetaClassDefineReservedUnused(IOService, 44);
9152OSMetaClassDefineReservedUnused(IOService, 45);
9153OSMetaClassDefineReservedUnused(IOService, 46);
9154OSMetaClassDefineReservedUnused(IOService, 47);
9155