IOUserServer.cpp source code [xnu/iokit/Kernel/IOUserServer.cpp]

1	/*
2	* Copyright (c) 1998-2021 Apple Inc. All rights reserved.
3	*
4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5	*
6	* This file contains Original Code and/or Modifications of Original Code
7	* as defined in and that are subject to the Apple Public Source License
8	* Version 2.0 (the 'License'). You may not use this file except in
9	* compliance with the License. The rights granted to you under the License
10	* may not be used to create, or enable the creation or redistribution of,
11	* unlawful or unlicensed copies of an Apple operating system, or to
12	* circumvent, violate, or enable the circumvention or violation of, any
13	* terms of an Apple operating system software license agreement.
14	*
15	* Please obtain a copy of the License at
16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
17	*
18	* The Original Code and all software distributed under the License are
19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23	* Please see the License for the specific language governing rights and
24	* limitations under the License.
25	*
26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27	*/
28
29	#include <IOKit/IORPC.h>
30	#include <IOKit/IOKitServer.h>
31	#include <IOKit/IOKitKeysPrivate.h>
32	#include <IOKit/IOKernelReportStructs.h>
33	#include <IOKit/IOUserClient.h>
34	#include <IOKit/IOService.h>
35	#include <IOKit/IORegistryEntry.h>
36	#include <IOKit/IOCatalogue.h>
37	#include <IOKit/IOMemoryDescriptor.h>
38	#include <IOKit/IOBufferMemoryDescriptor.h>
39	#include <IOKit/IOSubMemoryDescriptor.h>
40	#include <IOKit/IOMultiMemoryDescriptor.h>
41	#include <IOKit/IOMapper.h>
42	#include <IOKit/IOLib.h>
43	#include <IOKit/IOHibernatePrivate.h>
44	#include <IOKit/IOBSD.h>
45	#include <IOKit/system.h>
46	#include <IOKit/IOUserServer.h>
47	#include <IOKit/IOInterruptEventSource.h>
48	#include <IOKit/IOTimerEventSource.h>
49	#include <IOKit/pwr_mgt/RootDomain.h>
50	#include <IOKit/pwr_mgt/IOPowerConnection.h>
51	#include <libkern/c++/OSAllocation.h>
52	#include <libkern/c++/OSKext.h>
53	#include <libkern/c++/OSSharedPtr.h>
54	#include <libkern/OSDebug.h>
55	#include <libkern/Block.h>
56	#include <kern/cs_blobs.h>
57	#include <kern/thread_call.h>
58	#include <os/atomic_private.h>
59	#include <sys/proc.h>
60	#include <sys/reboot.h>
61	#include <sys/codesign.h>
62	#include "IOKitKernelInternal.h"
63
64	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
65
66	#include <DriverKit/IODispatchQueue.h>
67	#include <DriverKit/OSObject.h>
68	#include <DriverKit/OSAction.h>
69	#include <DriverKit/IODispatchSource.h>
70	#include <DriverKit/IOInterruptDispatchSource.h>
71	#include <DriverKit/IOService.h>
72	#include <DriverKit/IOMemoryDescriptor.h>
73	#include <DriverKit/IOBufferMemoryDescriptor.h>
74	#include <DriverKit/IOMemoryMap.h>
75	#include <DriverKit/IODataQueueDispatchSource.h>
76	#include <DriverKit/IOServiceNotificationDispatchSource.h>
77	#include <DriverKit/IOServiceStateNotificationDispatchSource.h>
78	#include <DriverKit/IOEventLink.h>
79	#include <DriverKit/IOWorkGroup.h>
80	#include <DriverKit/IOUserServer.h>
81
82	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
83
84	#include <System/IODataQueueDispatchSourceShared.h>
85
86	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
87
88	TUNABLE(SInt64, gIODKDebug, "dk", kIODKEnable);
89
90	#if DEBUG \|\| DEVELOPMENT
91	TUNABLE(bool, disable_dext_crash_reboot, "disable_dext_crash_reboot", `0`);
92	#endif /* DEBUG \|\| DEVELOPMENT */
93
94	static OSString * gIOSystemStateSleepDescriptionKey;
95	static const OSSymbol * gIOSystemStateSleepDescriptionReasonKey;
96	static const OSSymbol * gIOSystemStateSleepDescriptionHibernateStateKey;
97
98	static OSString * gIOSystemStateWakeDescriptionKey;
99	static const OSSymbol * gIOSystemStateWakeDescriptionWakeReasonKey;
100
101	static OSString * gIOSystemStateHaltDescriptionKey;
102	static const OSSymbol * gIOSystemStateHaltDescriptionHaltStateKey;
103
104	static OSString * gIOSystemStatePowerSourceDescriptionKey;
105	static const OSSymbol * gIOSystemStatePowerSourceDescriptionACAttachedKey;
106
107	extern bool gInUserspaceReboot;
108
109	extern void iokit_clear_registered_ports(task_t task);
110
111	static IORPCMessage *
112	IORPCMessageFromMachReply(IORPCMessageMach * msg);
113
114	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
115
116	struct IOPStrings;
117
118	class OSUserMetaClass : public OSObject
119	{
120	OSDeclareDefaultStructors(OSUserMetaClass);
121	public:
122	const OSSymbol * name;
123	const OSMetaClass * meta;
124	OSUserMetaClass * superMeta;
125
126	queue_chain_t link;
127
128	OSClassDescription * description;
129	IOPStrings * queueNames;
130	uint32_t methodCount;
131	uint64_t * methods;
132
133	virtual void free() override;
134	virtual kern_return_t Dispatch(const IORPC rpc) APPLE_KEXT_OVERRIDE;
135	};
136	OSDefineMetaClassAndStructors(OSUserMetaClass, OSObject);
137
138	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
139
140	class IOUserService : public IOService
141	{
142	friend class IOService;
143
144	OSDeclareDefaultStructors(IOUserService)
145
146	virtual bool
147	start(IOService * provider) APPLE_KEXT_OVERRIDE;
148	};
149
150	OSDefineMetaClassAndStructors(IOUserService, IOService)
151
152	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
153
154	class IOUserUserClient : public IOUserClient
155	{
156	OSDeclareDefaultStructors(IOUserUserClient);
157	public:
158	task_t fTask;
159	OSDictionary * fWorkGroups;
160	OSDictionary * fEventLinks;
161	IOLock * fLock;
162
163	IOReturn setTask(task_t task);
164	IOReturn eventlinkConfigurationTrap(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6);
165	IOReturn workgroupConfigurationTrap(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6);
166
167	virtual bool init( OSDictionary * dictionary ) APPLE_KEXT_OVERRIDE;
168	virtual void free() APPLE_KEXT_OVERRIDE;
169	virtual void stop(IOService * provider) APPLE_KEXT_OVERRIDE;
170	virtual IOReturn clientClose(void) APPLE_KEXT_OVERRIDE;
171	virtual IOReturn setProperties(OSObject * properties) APPLE_KEXT_OVERRIDE;
172	virtual IOReturn externalMethod(uint32_t selector, IOExternalMethodArguments * args,
173	IOExternalMethodDispatch * dispatch, OSObject * target, void * reference) APPLE_KEXT_OVERRIDE;
174	virtual IOReturn clientMemoryForType(UInt32 type,
175	IOOptionBits * options,
176	IOMemoryDescriptor ** memory) APPLE_KEXT_OVERRIDE;
177	virtual IOExternalTrap * getTargetAndTrapForIndex( IOService **targetP, UInt32 index ) APPLE_KEXT_OVERRIDE;
178	};
179
180	OSDefineMetaClassAndStructors(IOUserServerCheckInToken, OSObject);
181	OSDefineMetaClassAndStructors(_IOUserServerCheckInCancellationHandler, OSObject);
182
183	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
184
185
186	bool
187	IOUserService::start(IOService * provider)
188	{
189	bool ok = true;
190	IOReturn ret;
191
192	ret = Start(provider);
193	if (kIOReturnSuccess != ret) {
194	return false;
195	}
196
197	return ok;
198	}
199
200	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
201
202	#undef super
203
204	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
205
206	struct IODispatchQueue_IVars {
207	IOUserServer * userServer;
208	IODispatchQueue * queue;
209	queue_chain_t link;
210	uint64_t tid;
211
212	mach_port_t serverPort;
213	};
214
215	struct OSAction_IVars {
216	OSObject * target;
217	uint64_t targetmsgid;
218	uint64_t msgid;
219	IOUserServer * userServer;
220	OSActionAbortedHandler abortedHandler;
221	OSString * typeName;
222	void * reference;
223	size_t referenceSize;
224	bool aborted;
225	};
226
227	struct IOWorkGroup_IVars {
228	IOUserServer * userServer;
229	OSString * name;
230	IOUserUserClient * userClient;
231	};
232
233	struct IOEventLink_IVars {
234	IOUserServer * userServer;
235	OSString * name;
236	IOUserUserClient * userClient;
237	};
238
239	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
240
241	kern_return_t
242	IOService::GetRegistryEntryID_Impl(
243	uint64_t * registryEntryID)
244	{
245	IOReturn ret = kIOReturnSuccess;
246
247	*registryEntryID = getRegistryEntryID();
248
249	return ret;
250	}
251
252	kern_return_t
253	IOService::SetName_Impl(
254	const char * name)
255	{
256	IOReturn ret = kIOReturnSuccess;
257
258	setName(name);
259
260	return ret;
261	}
262
263	kern_return_t
264	IOService::CopyName_Impl(
265	OSString ** name)
266	{
267	const OSString * str = copyName();
268	name = __DECONST(OSString , str);
269	return str ? kIOReturnSuccess : kIOReturnError;
270	}
271
272
273	kern_return_t
274	IOService::Start_Impl(
275	IOService * provider)
276	{
277	IOReturn ret = kIOReturnSuccess;
278	return ret;
279	}
280
281
282	IOReturn
283	IOService::UpdateReport_Impl(OSData *channels, uint32_t action,
284	uint32_t *outElementCount,
285	uint64_t offset, uint64_t capacity,
286	IOMemoryDescriptor *buffer)
287	{
288	return kIOReturnUnsupported;
289	}
290
291	IOReturn
292	IOService::ConfigureReport_Impl(OSData channels, uint32_t action, uint32_t outCount)
293	{
294	return kIOReturnUnsupported;
295	}
296
297	// adapt old signature of configureReport to the iig-friendly signature of ConfigureReport
298	IOReturn
299	IOService::_ConfigureReport(IOReportChannelList *channelList,
300	IOReportConfigureAction action,
301	void *result,
302	void *destination)
303	{
304	if (action != kIOReportEnable && action != kIOReportGetDimensions && action != kIOReportDisable) {
305	return kIOReturnUnsupported;
306	}
307	static_assert(sizeof(IOReportChannelList) == `8`);
308	static_assert(sizeof(IOReportChannel) == `16`);
309	unsigned int size_of_channels;
310	bool overflow = os_mul_and_add_overflow(channelList->nchannels, sizeof(IOReportChannel), sizeof(IOReportChannelList), &size_of_channels);
311	if (overflow) {
312	return kIOReturnOverrun;
313	}
314	OSSharedPtr<OSData> sp_channels(OSData::withBytesNoCopy(bytes: channelList, numBytes: size_of_channels), libkern::no_retain);
315	if (!sp_channels) {
316	return kIOReturnNoMemory;
317	}
318	int resultp = (int**) result;
319	uint32_t count = `0`;
320	IOReturn r = ConfigureReport(channels: sp_channels.get(), action, outCount: &count);
321	int new_result;
322	overflow = os_add_overflow(*resultp, count, &new_result);
323	if (overflow) {
324	return kIOReturnOverrun;
325	}
326	*resultp = new_result;
327	return r;
328	}
329
330	// adapt old signature of updateReport to the iig-friendly signature of UpdateReport
331	IOReturn
332	IOService::_UpdateReport(IOReportChannelList *channelList,
333	IOReportUpdateAction action,
334	void *result,
335	void *destination)
336	{
337	if (action != kIOReportCopyChannelData) {
338	return kIOReturnUnsupported;
339	}
340	unsigned int size_of_channels;
341	bool overflow = os_mul_and_add_overflow(channelList->nchannels, sizeof(IOReportChannel), sizeof(IOReportChannelList), &size_of_channels);
342	if (overflow) {
343	return kIOReturnOverrun;
344	}
345	OSSharedPtr<OSData> sp_channels(OSData::withBytesNoCopy(bytes: channelList, numBytes: size_of_channels), libkern::no_retain);
346	if (!sp_channels) {
347	return kIOReturnNoMemory;
348	}
349	int resultp = (int**) result;
350	uint32_t count = `0`;
351	auto buffer = (IOBufferMemoryDescriptor*) destination;
352	uint64_t length = buffer->getLength();
353	buffer->setLength(buffer->getCapacity());
354	IOReturn r = UpdateReport(channels: sp_channels.get(), action, outElementCount: &count, offset: length, capacity: buffer->getCapacity() - length, buffer);
355	int new_result;
356	overflow = os_add_overflow(*resultp, count, &new_result);
357	size_t new_length;
358	overflow = overflow \|\| os_mul_and_add_overflow(count, sizeof(IOReportElement), length, &new_length);
359	if (overflow \|\| new_length > buffer->getCapacity()) {
360	buffer->setLength(length);
361	return kIOReturnOverrun;
362	}
363	*resultp = new_result;
364	buffer->setLength(new_length);
365	return r;
366	}
367
368
369	IOReturn
370	IOService::SetLegend_Impl(OSArray legend, bool* is_public)
371	{
372	bool ok = setProperty(kIOReportLegendKey, anObject: legend);
373	ok = ok && setProperty(kIOReportLegendPublicKey, aBoolean: is_public);
374	return ok ? kIOReturnSuccess : kIOReturnError;
375	}
376
377
378	kern_return_t
379	IOService::RegisterService_Impl()
380	{
381	IOReturn ret = kIOReturnSuccess;
382	bool started;
383
384	IOUserServer us = (typeof*(us))thread_iokit_tls_get(index: `0`);
385	if (reserved != NULL && reserved->uvars != NULL && reserved->uvars->userServer == us) {
386	started = reserved->uvars->started;
387	} else {
388	// assume started
389	started = true;
390	}
391
392	if (OSDynamicCast(IOUserServer, this) != NULL \|\| started) {
393	registerService(options: kIOServiceAsynchronous);
394	} else {
395	assert(reserved != NULL && reserved->uvars != NULL);
396	reserved->uvars->deferredRegisterService = true;
397	}
398
399	return ret;
400	}
401
402	kern_return_t
403	IOService::CopyDispatchQueue_Impl(
404	const char * name,
405	IODispatchQueue ** queue)
406	{
407	IODispatchQueue * result;
408	IOService * service;
409	IOReturn ret;
410	uint32_t index;
411
412	if (!reserved->uvars) {
413	return kIOReturnError;
414	}
415
416	if (!reserved->uvars->queueArray) {
417	// CopyDispatchQueue should not be called after the service has stopped
418	return kIOReturnError;
419	}
420
421	ret = kIOReturnNotFound;
422	index = -`1U`;
423	if (!strcmp(s1: "Default", s2: name)) {
424	index = `0`;
425	} else if (reserved->uvars->userMeta
426	&& reserved->uvars->userMeta->queueNames) {
427	index = reserved->uvars->userServer->stringArrayIndex(array: reserved->uvars->userMeta->queueNames, look: name);
428	if (index != -`1U`) {
429	index++;
430	}
431	}
432	if (index == -`1U`) {
433	if ((service = getProvider())) {
434	ret = service->CopyDispatchQueue(name, queue);
435	}
436	} else {
437	result = reserved->uvars->queueArray [index];
438	if (result) {
439	result->retain();
440	*queue = result;
441	ret = kIOReturnSuccess;
442	}
443	}
444
445	return ret;
446	}
447
448	kern_return_t
449	IOService::CreateDefaultDispatchQueue_Impl(
450	IODispatchQueue ** queue)
451	{
452	return kIOReturnError;
453	}
454
455	kern_return_t
456	IOService::CoreAnalyticsSendEvent_Impl(
457	uint64_t options,
458	OSString * eventName,
459	OSDictionary * eventPayload)
460	{
461	kern_return_t ret;
462
463	if (NULL == gIOCoreAnalyticsSendEventProc) {
464	// perhaps save for later?
465	return kIOReturnNotReady;
466	}
467
468	ret = (*gIOCoreAnalyticsSendEventProc)(options, eventName, eventPayload);
469
470	return ret;
471	}
472
473	kern_return_t
474	IOService::SetDispatchQueue_Impl(
475	const char * name,
476	IODispatchQueue * queue)
477	{
478	IOReturn ret = kIOReturnSuccess;
479	uint32_t index;
480
481	if (!reserved->uvars) {
482	return kIOReturnError;
483	}
484
485	if (kIODKLogSetup & gIODKDebug) {
486	DKLOG(DKS "::SetDispatchQueue(%s)\n", DKN(this), name);
487	}
488	queue->ivars->userServer = reserved->uvars->userServer;
489	index = -`1U`;
490	if (!strcmp(s1: "Default", s2: name)) {
491	index = `0`;
492	} else if (reserved->uvars->userMeta
493	&& reserved->uvars->userMeta->queueNames) {
494	index = reserved->uvars->userServer->stringArrayIndex(array: reserved->uvars->userMeta->queueNames, look: name);
495	if (index != -`1U`) {
496	index++;
497	}
498	}
499	if (index == -`1U`) {
500	ret = kIOReturnBadArgument;
501	} else {
502	reserved->uvars->queueArray [index] = queue;
503	queue->retain();
504	}
505
506	return ret;
507	}
508
509	IOService *
510	IOService::GetProvider() const
511	{
512	return getProvider();
513	}
514
515	kern_return_t
516	IOService::SetProperties_Impl(
517	OSDictionary * properties)
518	{
519	IOUserServer * us;
520	OSDictionary * dict;
521	IOReturn ret;
522
523	us = (typeof(us))thread_iokit_tls_get(index: `0`);
524	dict = OSDynamicCast(OSDictionary, properties);
525	if (NULL == us) {
526	if (!dict) {
527	return kIOReturnBadArgument;
528	}
529	bool ok __block = true;
530	dict->iterateObjects(block: ^bool (const OSSymbol * key, OSObject * value) {
531	ok = setProperty(aKey: key, anObject: value);
532	return !ok;
533	});
534	ret = ok ? kIOReturnSuccess : kIOReturnNotWritable;
535	return ret;
536	}
537
538	ret = setProperties(properties);
539
540	if (kIOReturnUnsupported == ret) {
541	if (dict && reserved->uvars && (reserved->uvars->userServer == us)) {
542	ret = runPropertyActionBlock(block: ^IOReturn (void) {
543	OSDictionary * userProps;
544	IOReturn ret;
545
546	userProps = OSDynamicCast(OSDictionary, getProperty(gIOUserServicePropertiesKey));
547	if (userProps) {
548	userProps = (typeof(userProps))userProps->copyCollection();
549	} else {
550	userProps = OSDictionary::withCapacity(capacity: `4`);
551	}
552	if (!userProps) {
553	ret = kIOReturnNoMemory;
554	} else {
555	bool ok = userProps->merge(aDictionary: dict);
556	if (ok) {
557	ok = setProperty(aKey: gIOUserServicePropertiesKey, anObject: userProps);
558	}
559	OSSafeReleaseNULL(userProps);
560	ret = ok ? kIOReturnSuccess : kIOReturnNotWritable;
561	}
562	return ret;
563	});
564	}
565	}
566
567	return ret;
568	}
569
570	kern_return_t
571	IOService::RemoveProperty_Impl(OSString * propertyName)
572	{
573	IOUserServer * us = (IOUserServer *)thread_iokit_tls_get(index: `0`);
574	IOReturn ret = kIOReturnUnsupported;
575
576	if (NULL == propertyName) {
577	return kIOReturnUnsupported;
578	}
579	if (NULL == us) {
580	removeProperty(aKey: propertyName);
581	return kIOReturnSuccess;
582	}
583	if (reserved && reserved->uvars && reserved->uvars->userServer == us) {
584	ret = runPropertyActionBlock(block: ^IOReturn (void) {
585	OSDictionary * userProps;
586	userProps = OSDynamicCast(OSDictionary, getProperty(gIOUserServicePropertiesKey));
587	if (userProps) {
588	userProps = (OSDictionary *)userProps->copyCollection();
589	if (!userProps) {
590	return kIOReturnNoMemory;
591	}
592	userProps->removeObject(aKey: propertyName);
593	bool ok = setProperty(aKey: gIOUserServicePropertiesKey, anObject: userProps);
594	OSSafeReleaseNULL(userProps);
595	return ok ? kIOReturnSuccess : kIOReturnNotWritable;
596	} else {
597	return kIOReturnNotFound;
598	}
599	});
600	}
601	return ret;
602	}
603
604	kern_return_t
605	IOService::CopyProperties_Local(
606	OSDictionary ** properties)
607	{
608	OSDictionary * props;
609	OSDictionary * userProps;
610
611	props = dictionaryWithProperties();
612	userProps = OSDynamicCast(OSDictionary, props->getObject(gIOUserServicePropertiesKey));
613	if (userProps) {
614	props->merge(aDictionary: userProps);
615	props->removeObject(aKey: gIOUserServicePropertiesKey);
616	}
617
618	*properties = props;
619
620	return props ? kIOReturnSuccess : kIOReturnNoMemory;
621	}
622
623	kern_return_t
624	IOService::CopyProperties_Impl(
625	OSDictionary ** properties)
626	{
627	return CopyProperties_Local(properties);
628	}
629
630	kern_return_t
631	IOService::RequireMaxBusStall_Impl(
632	uint64_t u64ns)
633	{
634	IOReturn ret;
635	UInt32 ns;
636
637	if (os_convert_overflow(u64ns, &ns)) {
638	return kIOReturnBadArgument;
639	}
640	ret = requireMaxBusStall(ns);
641
642	return ret;
643	}
644
645	#if PRIVATE_WIFI_ONLY
646	kern_return_t
647	IOService::UserSetProperties_Impl(
648	OSContainer * properties)
649	{
650	return kIOReturnUnsupported;
651	}
652
653	kern_return_t
654	IOService::SendIOMessageServicePropertyChange_Impl(void)
655	{
656	return messageClients(kIOMessageServicePropertyChange);
657	}
658	#endif /* PRIVATE_WIFI_ONLY */
659
660	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
661
662	kern_return_t
663	IOMemoryDescriptor::_CopyState_Impl(
664	_IOMDPrivateState * state)
665	{
666	IOReturn ret;
667
668	state->length = _length;
669	state->options = _flags;
670
671	ret = kIOReturnSuccess;
672
673	return ret;
674	}
675
676	kern_return_t
677	IOMemoryDescriptor::GetLength(uint64_t * returnLength)
678	{
679	*returnLength = getLength();
680
681	return kIOReturnSuccess;
682	}
683
684	kern_return_t
685	IOMemoryDescriptor::CreateMapping_Impl(
686	uint64_t options,
687	uint64_t address,
688	uint64_t offset,
689	uint64_t length,
690	uint64_t alignment,
691	IOMemoryMap ** map)
692	{
693	IOReturn ret;
694	IOMemoryMap * resultMap;
695	IOOptionBits koptions;
696	mach_vm_address_t atAddress;
697
698	ret = kIOReturnSuccess;
699	koptions = `0`;
700	resultMap = NULL;
701
702	if (kIOMemoryMapFixedAddress & options) {
703	atAddress = address;
704	koptions = `0`;
705	} else {
706	switch (kIOMemoryMapGuardedMask & options) {
707	default:
708	case kIOMemoryMapGuardedDefault:
709	koptions \|= kIOMapGuardedSmall;
710	break;
711	case kIOMemoryMapGuardedNone:
712	break;
713	case kIOMemoryMapGuardedSmall:
714	koptions \|= kIOMapGuardedSmall;
715	break;
716	case kIOMemoryMapGuardedLarge:
717	koptions \|= kIOMapGuardedLarge;
718	break;
719	}
720	atAddress = `0`;
721	koptions \|= kIOMapAnywhere;
722	}
723
724	if ((kIOMemoryMapReadOnly & options) \|\| (kIODirectionOut == getDirection())) {
725	if (!reserved \|\| (current_task() != reserved->creator)) {
726	koptions \|= kIOMapReadOnly;
727	}
728	}
729
730	switch (`0xFF00` & options) {
731	case kIOMemoryMapCacheModeDefault:
732	koptions \|= kIOMapDefaultCache;
733	break;
734	case kIOMemoryMapCacheModeInhibit:
735	koptions \|= kIOMapInhibitCache;
736	break;
737	case kIOMemoryMapCacheModeCopyback:
738	koptions \|= kIOMapCopybackCache;
739	break;
740	case kIOMemoryMapCacheModeWriteThrough:
741	koptions \|= kIOMapWriteThruCache;
742	break;
743	default:
744	ret = kIOReturnBadArgument;
745	}
746
747	if (kIOReturnSuccess == ret) {
748	resultMap = createMappingInTask(intoTask: current_task(), atAddress, options: koptions, offset, length);
749	if (!resultMap) {
750	ret = kIOReturnError;
751	}
752	}
753
754	*map = resultMap;
755
756	return ret;
757	}
758
759	kern_return_t
760	IOMemoryDescriptor::CreateSubMemoryDescriptor_Impl(
761	uint64_t memoryDescriptorCreateOptions,
762	uint64_t offset,
763	uint64_t length,
764	IOMemoryDescriptor * ofDescriptor,
765	IOMemoryDescriptor ** memory)
766	{
767	IOReturn ret;
768	IOMemoryDescriptor * iomd;
769	IOByteCount mdOffset;
770	IOByteCount mdLength;
771	IOByteCount mdEnd;
772
773	if (!ofDescriptor) {
774	return kIOReturnBadArgument;
775	}
776	if (memoryDescriptorCreateOptions & ~kIOMemoryDirectionOutIn) {
777	return kIOReturnBadArgument;
778	}
779	if (os_convert_overflow(offset, &mdOffset)) {
780	return kIOReturnBadArgument;
781	}
782	if (os_convert_overflow(length, &mdLength)) {
783	return kIOReturnBadArgument;
784	}
785	if (os_add_overflow(mdOffset, mdLength, &mdEnd)) {
786	return kIOReturnBadArgument;
787	}
788	if (mdEnd > ofDescriptor->getLength()) {
789	return kIOReturnBadArgument;
790	}
791
792	iomd = IOSubMemoryDescriptor::withSubRange(
793	of: ofDescriptor, offset: mdOffset, length: mdLength, options: (IOOptionBits) memoryDescriptorCreateOptions);
794
795	if (iomd) {
796	ret = kIOReturnSuccess;
797	*memory = iomd;
798	} else {
799	ret = kIOReturnNoMemory;
800	*memory = NULL;
801	}
802
803	return ret;
804	}
805
806	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
807
808	kern_return_t
809	IOMemoryDescriptor::CreateWithMemoryDescriptors_Impl(
810	uint64_t memoryDescriptorCreateOptions,
811	uint32_t withDescriptorsCount,
812	IOMemoryDescriptor ** const withDescriptors,
813	IOMemoryDescriptor ** memory)
814	{
815	IOReturn ret;
816	IOMemoryDescriptor * iomd;
817
818	if (!withDescriptors) {
819	return kIOReturnBadArgument;
820	}
821	if (!withDescriptorsCount) {
822	return kIOReturnBadArgument;
823	}
824	if (memoryDescriptorCreateOptions & ~kIOMemoryDirectionOutIn) {
825	return kIOReturnBadArgument;
826	}
827
828	for (unsigned int idx = `0`; idx < withDescriptorsCount; idx++) {
829	if (NULL == withDescriptors[idx]) {
830	return kIOReturnBadArgument;
831	}
832	}
833
834	iomd = IOMultiMemoryDescriptor::withDescriptors(descriptors: withDescriptors, withCount: withDescriptorsCount,
835	withDirection: (IODirection) memoryDescriptorCreateOptions, asReference: false);
836
837	if (iomd) {
838	ret = kIOReturnSuccess;
839	*memory = iomd;
840	} else {
841	ret = kIOReturnNoMemory;
842	*memory = NULL;
843	}
844
845	return ret;
846	}
847
848	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
849
850	kern_return_t
851	IOUserClient::CreateMemoryDescriptorFromClient_Impl(
852	uint64_t memoryDescriptorCreateOptions,
853	uint32_t segmentsCount,
854	const IOAddressSegment segments[`32`],
855	IOMemoryDescriptor ** memory)
856	{
857	IOReturn ret;
858	IOMemoryDescriptor * iomd;
859	IOOptionBits mdOptions;
860	IOUserUserClient * me;
861	IOAddressRange * ranges;
862
863	me = OSDynamicCast(IOUserUserClient, this);
864	if (!me) {
865	return kIOReturnBadArgument;
866	}
867	if (!me->fTask) {
868	return kIOReturnNotReady;
869	}
870
871	mdOptions = kIOMemoryThreadSafe;
872	if (kIOMemoryDirectionOut & memoryDescriptorCreateOptions) {
873	mdOptions \|= kIODirectionOut;
874	}
875	if (kIOMemoryDirectionIn & memoryDescriptorCreateOptions) {
876	mdOptions \|= kIODirectionIn;
877	}
878	if (!(kIOMemoryDisableCopyOnWrite & memoryDescriptorCreateOptions)) {
879	mdOptions \|= kIOMemoryMapCopyOnWrite;
880	}
881
882	static_assert(sizeof(IOAddressRange) == sizeof(IOAddressSegment));
883	ranges = __DECONST(IOAddressRange *, &segments[`0`]);
884
885	iomd = IOMemoryDescriptor::withAddressRanges(
886	ranges, rangeCount: segmentsCount,
887	options: mdOptions, task: me->fTask);
888
889	if (iomd) {
890	ret = kIOReturnSuccess;
891	*memory = iomd;
892	} else {
893	ret = kIOReturnNoMemory;
894	*memory = NULL;
895	}
896
897	return ret;
898	}
899
900	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
901
902	kern_return_t
903	IOMemoryMap::_CopyState_Impl(
904	_IOMemoryMapPrivateState * state)
905	{
906	IOReturn ret;
907
908	state->offset = fOffset;
909	state->length = getLength();
910	state->address = getAddress();
911	state->options = getMapOptions();
912
913	ret = kIOReturnSuccess;
914
915	return ret;
916	}
917
918	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
919
920	kern_return_t
921	IOBufferMemoryDescriptor::Create_Impl(
922	uint64_t options,
923	uint64_t capacity,
924	uint64_t alignment,
925	IOBufferMemoryDescriptor ** memory)
926	{
927	IOReturn ret;
928	IOOptionBits bmdOptions;
929	IOBufferMemoryDescriptor * bmd;
930	IOMemoryDescriptorReserved * reserved;
931
932	if (options & ~((uint64_t) kIOMemoryDirectionOutIn)) {
933	// no other options currently defined
934	return kIOReturnBadArgument;
935	}
936	bmdOptions = (options & kIOMemoryDirectionOutIn) \| kIOMemoryKernelUserShared \| kIOMemoryThreadSafe;
937	bmd = IOBufferMemoryDescriptor::inTaskWithOptions(
938	inTask: kernel_task, options: bmdOptions, capacity, alignment);
939
940	*memory = bmd;
941
942	if (!bmd) {
943	return kIOReturnNoMemory;
944	}
945
946	reserved = bmd->getKernelReserved();
947	reserved->creator = current_task();
948	task_reference(reserved->creator);
949
950	ret = kIOReturnSuccess;
951
952	return ret;
953	}
954
955	kern_return_t
956	IOBufferMemoryDescriptor::SetLength_Impl(
957	uint64_t length)
958	{
959	setLength(length);
960	return kIOReturnSuccess;
961	}
962
963
964	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
965
966	kern_return_t
967	IODMACommand::Create_Impl(
968	IOService * device,
969	uint64_t options,
970	const IODMACommandSpecification * specification,
971	IODMACommand ** command)
972	{
973	IOReturn ret;
974	IODMACommand * dma;
975	IODMACommand::SegmentOptions segmentOptions;
976	IOMapper * mapper;
977
978	if (options & ~((uint64_t) kIODMACommandCreateNoOptions)) {
979	// no other options currently defined
980	return kIOReturnBadArgument;
981	}
982
983	if (os_convert_overflow(specification->maxAddressBits, &segmentOptions.fNumAddressBits)) {
984	return kIOReturnBadArgument;
985	}
986	segmentOptions.fMaxSegmentSize = `0`;
987	segmentOptions.fMaxTransferSize = `0`;
988	segmentOptions.fAlignment = `1`;
989	segmentOptions.fAlignmentLength = `1`;
990	segmentOptions.fAlignmentInternalSegments = `1`;
991	segmentOptions.fStructSize = sizeof(segmentOptions);
992
993	mapper = IOMapper::copyMapperForDevice(device);
994
995	dma = IODMACommand::withSpecification(
996	kIODMACommandOutputHost64,
997	segmentOptions: &segmentOptions,
998	mappingOptions: kIODMAMapOptionDextOwner \|
999	kIODMAMapOptionMapped,
1000	mapper,
1001	NULL);
1002
1003	OSSafeReleaseNULL(mapper);
1004	*command = dma;
1005
1006	if (!dma) {
1007	return kIOReturnNoMemory;
1008	}
1009	ret = kIOReturnSuccess;
1010
1011	return ret;
1012	}
1013
1014	#define fInternalState reserved
1015
1016	kern_return_t
1017	IODMACommand::PrepareForDMA_Impl(
1018	uint64_t options,
1019	IOMemoryDescriptor * memory,
1020	uint64_t offset,
1021	uint64_t length,
1022	uint64_t * flags,
1023	uint32_t * segmentsCount,
1024	IOAddressSegment * segments)
1025	{
1026	IOReturn ret;
1027	uint64_t lflags, mdFlags;
1028	UInt32 numSegments;
1029	UInt64 genOffset;
1030
1031	if (options & ~((uint64_t) kIODMACommandPrepareForDMANoOptions)) {
1032	// no other options currently defined
1033	return kIOReturnBadArgument;
1034	}
1035
1036	if (memory == NULL) {
1037	return kIOReturnBadArgument;
1038	}
1039
1040	assert(fInternalState->fDextLock);
1041	IOLockLock(fInternalState->fDextLock);
1042
1043	// uses IOMD direction
1044	ret = memory->prepare();
1045	if (kIOReturnSuccess != ret) {
1046	goto exit;
1047	}
1048
1049	ret = setMemoryDescriptor(mem: memory, autoPrepare: false);
1050	if (kIOReturnSuccess != ret) {
1051	memory->complete();
1052	goto exit;
1053	}
1054
1055	ret = prepare(offset, length);
1056	if (kIOReturnSuccess != ret) {
1057	clearMemoryDescriptor(autoComplete: false);
1058	memory->complete();
1059	goto exit;
1060	}
1061
1062	static_assert(sizeof(IODMACommand::Segment64) == sizeof(IOAddressSegment));
1063
1064	numSegments = *segmentsCount;
1065	genOffset = `0`;
1066	ret = genIOVMSegments(offset: &genOffset, segments, numSegments: &numSegments);
1067
1068	if (kIOReturnSuccess != ret) {
1069	clearMemoryDescriptor(autoComplete: true);
1070	memory->complete();
1071	goto exit;
1072	}
1073
1074	mdFlags = fMemory->getFlags();
1075	lflags = `0`;
1076	if (kIODirectionOut & mdFlags) {
1077	lflags \|= kIOMemoryDirectionOut;
1078	}
1079	if (kIODirectionIn & mdFlags) {
1080	lflags \|= kIOMemoryDirectionIn;
1081	}
1082	*flags = lflags;
1083	*segmentsCount = numSegments;
1084
1085	exit:
1086	IOLockUnlock(fInternalState->fDextLock);
1087
1088	return ret;
1089	}
1090
1091	kern_return_t
1092	IODMACommand::CompleteDMA_Impl(
1093	uint64_t options)
1094	{
1095	IOReturn ret, completeRet;
1096	IOMemoryDescriptor * md;
1097
1098	if (options & ~((uint64_t) kIODMACommandCompleteDMANoOptions)) {
1099	// no other options currently defined
1100	return kIOReturnBadArgument;
1101	}
1102
1103	assert(fInternalState->fDextLock);
1104	IOLockLock(fInternalState->fDextLock);
1105
1106	if (!fInternalState->fPrepared) {
1107	ret = kIOReturnNotReady;
1108	goto exit;
1109	}
1110
1111	md = __DECONST(IOMemoryDescriptor *, fMemory);
1112	if (md) {
1113	md->retain();
1114	}
1115
1116	ret = clearMemoryDescriptor(autoComplete: true);
1117
1118	if (md) {
1119	completeRet = md->complete();
1120	OSSafeReleaseNULL(md);
1121	if (kIOReturnSuccess == ret) {
1122	ret = completeRet;
1123	}
1124	}
1125	exit:
1126	IOLockUnlock(fInternalState->fDextLock);
1127
1128	return ret;
1129	}
1130
1131	kern_return_t
1132	IODMACommand::GetPreparation_Impl(
1133	uint64_t * offset,
1134	uint64_t * length,
1135	IOMemoryDescriptor ** memory)
1136	{
1137	IOReturn ret;
1138	IOMemoryDescriptor * md;
1139
1140	if (!fActive) {
1141	return kIOReturnNotReady;
1142	}
1143
1144	ret = getPreparedOffsetAndLength(offset, length);
1145	if (kIOReturnSuccess != ret) {
1146	return ret;
1147	}
1148
1149	if (memory) {
1150	md = __DECONST(IOMemoryDescriptor *, fMemory);
1151	*memory = md;
1152	if (!md) {
1153	ret = kIOReturnNotReady;
1154	} else {
1155	md->retain();
1156	}
1157	}
1158	return ret;
1159	}
1160
1161	kern_return_t
1162	IODMACommand::PerformOperation_Impl(
1163	uint64_t options,
1164	uint64_t dmaOffset,
1165	uint64_t length,
1166	uint64_t dataOffset,
1167	IOMemoryDescriptor * data)
1168	{
1169	IOReturn ret;
1170	OSDataAllocation<uint8_t> buffer;
1171	UInt64 copiedDMA;
1172	IOByteCount mdOffset, mdLength, copied;
1173
1174	if (options & ~((uint64_t)
1175	(kIODMACommandPerformOperationOptionRead
1176	\| kIODMACommandPerformOperationOptionWrite
1177	\| kIODMACommandPerformOperationOptionZero))) {
1178	// no other options currently defined
1179	return kIOReturnBadArgument;
1180	}
1181
1182	if (!fActive) {
1183	return kIOReturnNotReady;
1184	}
1185	if (os_convert_overflow(dataOffset, &mdOffset)) {
1186	return kIOReturnBadArgument;
1187	}
1188	if (os_convert_overflow(length, &mdLength)) {
1189	return kIOReturnBadArgument;
1190	}
1191	if (length > fMemory->getLength()) {
1192	return kIOReturnBadArgument;
1193	}
1194	buffer = OSDataAllocation<uint8_t>(length, OSAllocateMemory);
1195	if (!buffer) {
1196	return kIOReturnNoMemory;
1197	}
1198
1199	switch (options) {
1200	case kIODMACommandPerformOperationOptionZero:
1201	bzero(s: buffer.data(), n: length);
1202	copiedDMA = writeBytes(offset: dmaOffset, bytes: buffer.data(), length);
1203	if (copiedDMA != length) {
1204	ret = kIOReturnUnderrun;
1205	break;
1206	}
1207	ret = kIOReturnSuccess;
1208	break;
1209
1210	case kIODMACommandPerformOperationOptionRead:
1211	case kIODMACommandPerformOperationOptionWrite:
1212
1213	if (!data) {
1214	ret = kIOReturnBadArgument;
1215	break;
1216	}
1217	if (length > data->getLength()) {
1218	ret = kIOReturnBadArgument;
1219	break;
1220	}
1221	if (kIODMACommandPerformOperationOptionWrite == options) {
1222	copied = data->readBytes(offset: mdOffset, bytes: buffer.data(), withLength: mdLength);
1223	if (copied != mdLength) {
1224	ret = kIOReturnUnderrun;
1225	break;
1226	}
1227	copiedDMA = writeBytes(offset: dmaOffset, bytes: buffer.data(), length);
1228	if (copiedDMA != length) {
1229	ret = kIOReturnUnderrun;
1230	break;
1231	}
1232	} else { / kIODMACommandPerformOperationOptionRead /
1233	copiedDMA = readBytes(offset: dmaOffset, bytes: buffer.data(), length);
1234	if (copiedDMA != length) {
1235	ret = kIOReturnUnderrun;
1236	break;
1237	}
1238	copied = data->writeBytes(offset: mdOffset, bytes: buffer.data(), withLength: mdLength);
1239	if (copied != mdLength) {
1240	ret = kIOReturnUnderrun;
1241	break;
1242	}
1243	}
1244	ret = kIOReturnSuccess;
1245	break;
1246	default:
1247	ret = kIOReturnBadArgument;
1248	break;
1249	}
1250
1251	return ret;
1252	}
1253
1254
1255	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
1256
1257	static kern_return_t
1258	OSActionCreateWithTypeNameInternal(OSObject * target, uint64_t targetmsgid, uint64_t msgid, size_t referenceSize, OSString * typeName, bool fromKernel, OSAction ** action)
1259	{
1260	OSAction * inst = NULL;
1261	void * reference = NULL; // must release
1262	const OSSymbol sym = NULL; // must release*
1263	OSObject obj = NULL; // must release*
1264	const OSMetaClass actionMetaClass = NULL; // do not release*
1265	kern_return_t ret;
1266
1267	if (fromKernel && typeName) {
1268	/ The action is being constructed in the kernel with a type name /
1269	sym = OSSymbol::withString(aString: typeName);
1270	actionMetaClass = OSMetaClass::getMetaClassWithName(name: sym);
1271	if (actionMetaClass && actionMetaClass->getSuperClass() == OSTypeID(OSAction)) {
1272	obj = actionMetaClass->alloc();
1273	if (!obj) {
1274	ret = kIOReturnNoMemory;
1275	goto finish;
1276	}
1277	inst = OSDynamicCast(OSAction, obj);
1278	obj = NULL; // prevent release
1279	assert(inst); // obj is a subclass of OSAction so the dynamic cast should always work
1280	} else {
1281	DKLOG("Attempted to create action object with type \"%s\" which does not inherit from OSAction\n", typeName->getCStringNoCopy());
1282	ret = kIOReturnBadArgument;
1283	goto finish;
1284	}
1285	} else {
1286	inst = OSTypeAlloc(OSAction);
1287	if (!inst) {
1288	ret = kIOReturnNoMemory;
1289	goto finish;
1290	}
1291	}
1292
1293	if (referenceSize != `0`) {
1294	reference = IONewZeroData(uint8_t, referenceSize);
1295	if (reference == NULL) {
1296	ret = kIOReturnNoMemory;
1297	goto finish;
1298	}
1299	}
1300
1301	inst->ivars = IONewZero(OSAction_IVars, `1`);
1302	if (!inst->ivars) {
1303	ret = kIOReturnNoMemory;
1304	goto finish;
1305	}
1306	if (target) {
1307	target->retain();
1308	if (!fromKernel && !OSDynamicCast(IOService, target)) {
1309	IOUserServer * us;
1310	us = (typeof(us))thread_iokit_tls_get(index: `0`);
1311	inst->ivars->userServer = OSDynamicCast(IOUserServer, us);
1312	assert(inst->ivars->userServer);
1313	inst->ivars->userServer->retain();
1314	}
1315	}
1316	inst->ivars->target = target;
1317	inst->ivars->targetmsgid = targetmsgid;
1318	inst->ivars->msgid = msgid;
1319
1320	inst->ivars->reference = reference;
1321	inst->ivars->referenceSize = referenceSize;
1322	reference = NULL; // prevent release
1323
1324	if (typeName) {
1325	typeName->retain();
1326	}
1327	inst->ivars->typeName = typeName;
1328
1329	*action = inst;
1330	inst = NULL; // prevent release
1331	ret = kIOReturnSuccess;
1332
1333	finish:
1334	OSSafeReleaseNULL(obj);
1335	OSSafeReleaseNULL(sym);
1336	OSSafeReleaseNULL(inst);
1337	if (reference) {
1338	IODeleteData(reference, uint8_t, referenceSize);
1339	}
1340
1341	return ret;
1342	}
1343
1344	kern_return_t
1345	OSAction::Create(OSAction_Create_Args)
1346	{
1347	return OSAction::CreateWithTypeName(target, targetmsgid, msgid, referenceSize, NULL, action);
1348	}
1349
1350	kern_return_t
1351	OSAction::CreateWithTypeName(OSAction_CreateWithTypeName_Args)
1352	{
1353	return OSActionCreateWithTypeNameInternal(target, targetmsgid, msgid, referenceSize, typeName, fromKernel: true, action);
1354	}
1355
1356	kern_return_t
1357	OSAction::Create_Impl(
1358	OSObject * target,
1359	uint64_t targetmsgid,
1360	uint64_t msgid,
1361	size_t referenceSize,
1362	OSAction ** action)
1363	{
1364	return OSAction::CreateWithTypeName_Impl(target, targetmsgid, msgid, referenceSize, NULL, action);
1365	}
1366
1367	kern_return_t
1368	OSAction::CreateWithTypeName_Impl(
1369	OSObject * target,
1370	uint64_t targetmsgid,
1371	uint64_t msgid,
1372	size_t referenceSize,
1373	OSString * typeName,
1374	OSAction ** action)
1375	{
1376	return OSActionCreateWithTypeNameInternal(target, targetmsgid, msgid, referenceSize, typeName, fromKernel: false, action);
1377	}
1378
1379	void
1380	OSAction::free()
1381	{
1382	if (ivars) {
1383	if (ivars->abortedHandler) {
1384	Block_release(ivars->abortedHandler);
1385	ivars->abortedHandler = NULL;
1386	}
1387	OSSafeReleaseNULL(ivars->target);
1388	OSSafeReleaseNULL(ivars->typeName);
1389	OSSafeReleaseNULL(ivars->userServer);
1390	if (ivars->reference) {
1391	assert(ivars->referenceSize > `0`);
1392	IODeleteData(ivars->reference, uint8_t, ivars->referenceSize);
1393	}
1394	IOSafeDeleteNULL(ivars, OSAction_IVars, `1`);
1395	}
1396	return super::free();
1397	}
1398
1399	void *
1400	OSAction::GetReference()
1401	{
1402	assert(ivars && ivars->referenceSize && ivars->reference);
1403	return ivars->reference;
1404	}
1405
1406	kern_return_t
1407	OSAction::SetAbortedHandler(OSActionAbortedHandler handler)
1408	{
1409	ivars->abortedHandler = Block_copy(handler);
1410	return kIOReturnSuccess;
1411	}
1412
1413	void
1414	OSAction::Aborted_Impl(void)
1415	{
1416	if (!os_atomic_cmpxchg(&ivars->aborted, false, true, relaxed)) {
1417	// already aborted
1418	return;
1419	}
1420	if (ivars->abortedHandler) {
1421	ivars->abortedHandler();
1422	}
1423	}
1424
1425	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
1426
1427	struct IODispatchSource_IVars {
1428	queue_chain_t link;
1429	IODispatchSource * source;
1430	IOUserServer * server;
1431	IODispatchQueue_IVars * queue;
1432	bool enabled;
1433	};
1434
1435	bool
1436	IODispatchSource::init()
1437	{
1438	if (!super::init()) {
1439	return false;
1440	}
1441
1442	ivars = IOMallocType(IODispatchSource_IVars);
1443
1444	ivars->source = this;
1445
1446	return true;
1447	}
1448
1449	void
1450	IODispatchSource::free()
1451	{
1452	IOFreeType(ivars, IODispatchSource_IVars);
1453	super::free();
1454	}
1455
1456	kern_return_t
1457	IODispatchSource::SetEnable_Impl(
1458	bool enable)
1459	{
1460	return SetEnableWithCompletion(enable, NULL);
1461	}
1462
1463	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
1464
1465	struct IOInterruptDispatchSource_IVars {
1466	IOService * provider;
1467	uint32_t intIndex;
1468	uint32_t flags;
1469	int interruptType;
1470	IOSimpleLock * lock;
1471	thread_t waiter;
1472	uint64_t count;
1473	uint64_t time;
1474	OSAction * action;
1475	bool enable;
1476	bool canceled;
1477	};
1478
1479	static void
1480	IOInterruptDispatchSourceInterrupt(OSObject * target, void * refCon,
1481	IOService * nub, int source )
1482	{
1483	IOInterruptDispatchSource_IVars * ivars = (typeof(ivars))refCon;
1484	IOInterruptState is;
1485
1486	is = IOSimpleLockLockDisableInterrupt(lock: ivars->lock);
1487	ivars->count++;
1488	ivars->time = (kIOInterruptSourceContinuousTime & ivars->flags)
1489	? mach_continuous_time() : mach_absolute_time();
1490	if (ivars->waiter) {
1491	thread_wakeup_thread(event: (event_t) ivars, thread: ivars->waiter);
1492	ivars->waiter = NULL;
1493	}
1494	if (kIOInterruptTypeLevel & ivars->interruptType) {
1495	ivars->provider->disableInterrupt(source: ivars->intIndex);
1496	}
1497	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
1498	}
1499
1500	kern_return_t
1501	IOInterruptDispatchSource::Create_Impl(
1502	IOService * provider,
1503	uint32_t indexAndFlags,
1504	IODispatchQueue * queue,
1505	IOInterruptDispatchSource ** source)
1506	{
1507	IOReturn ret;
1508	IOInterruptDispatchSource * inst;
1509	uint32_t index;
1510	uint32_t flags;
1511
1512	index = indexAndFlags & kIOInterruptSourceIndexMask;
1513	flags = indexAndFlags & ~kIOInterruptSourceIndexMask;
1514
1515	inst = OSTypeAlloc(IOInterruptDispatchSource);
1516	if (!inst->init()) {
1517	inst->free();
1518	return kIOReturnNoMemory;
1519	}
1520
1521	inst->ivars->lock = IOSimpleLockAlloc();
1522
1523	ret = provider->getInterruptType(source: index, interruptType: &inst->ivars->interruptType);
1524	if (kIOReturnSuccess != ret) {
1525	OSSafeReleaseNULL(inst);
1526	return ret;
1527	}
1528	ret = provider->registerInterrupt(source: index, target: inst, handler: IOInterruptDispatchSourceInterrupt, refCon: inst->ivars);
1529	if (kIOReturnSuccess == ret) {
1530	inst->ivars->intIndex = index;
1531	inst->ivars->flags = flags;
1532	inst->ivars->provider = provider;
1533	inst->ivars->provider->retain();
1534	*source = inst;
1535	}
1536	return ret;
1537	}
1538
1539	kern_return_t
1540	IOInterruptDispatchSource::GetInterruptType_Impl(
1541	IOService * provider,
1542	uint32_t index,
1543	uint64_t * interruptType)
1544	{
1545	IOReturn ret;
1546	int type;
1547
1548	*interruptType = `0`;
1549	ret = provider->getInterruptType(source: index, interruptType: &type);
1550	if (kIOReturnSuccess == ret) {
1551	*interruptType = type;
1552	}
1553
1554	return ret;
1555	}
1556
1557	bool
1558	IOInterruptDispatchSource::init()
1559	{
1560	if (!super::init()) {
1561	return false;
1562	}
1563	ivars = IOMallocType(IOInterruptDispatchSource_IVars);
1564
1565	return true;
1566	}
1567
1568	void
1569	IOInterruptDispatchSource::free()
1570	{
1571	IOReturn ret;
1572
1573	if (ivars && ivars->provider) {
1574	ret = ivars->provider->unregisterInterrupt(source: ivars->intIndex);
1575	assert(kIOReturnSuccess == ret);
1576	ivars->provider->release();
1577	}
1578
1579	if (ivars && ivars->lock) {
1580	IOSimpleLockFree(lock: ivars->lock);
1581	}
1582
1583	IOFreeType(ivars, IOInterruptDispatchSource_IVars);
1584
1585	super::free();
1586	}
1587
1588	kern_return_t
1589	IOInterruptDispatchSource::SetHandler_Impl(
1590	OSAction * action)
1591	{
1592	IOReturn ret;
1593	OSAction * oldAction;
1594
1595	oldAction = (typeof(oldAction))ivars->action;
1596	if (oldAction && OSCompareAndSwapPtr(oldAction, NULL, &ivars->action)) {
1597	oldAction->release();
1598	}
1599	action->retain();
1600	ivars->action = action;
1601
1602	ret = kIOReturnSuccess;
1603
1604	return ret;
1605	}
1606
1607	kern_return_t
1608	IOInterruptDispatchSource::SetEnableWithCompletion_Impl(
1609	bool enable,
1610	IODispatchSourceCancelHandler handler)
1611	{
1612	IOReturn ret;
1613	IOInterruptState is;
1614
1615	if (enable == ivars->enable) {
1616	return kIOReturnSuccess;
1617	}
1618
1619	if (enable) {
1620	is = IOSimpleLockLockDisableInterrupt(lock: ivars->lock);
1621	ivars->enable = enable;
1622	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
1623	ret = ivars->provider->enableInterrupt(source: ivars->intIndex);
1624	} else {
1625	ret = ivars->provider->disableInterrupt(source: ivars->intIndex);
1626	is = IOSimpleLockLockDisableInterrupt(lock: ivars->lock);
1627	ivars->enable = enable;
1628	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
1629	}
1630
1631	return ret;
1632	}
1633
1634	kern_return_t
1635	IOInterruptDispatchSource::Cancel_Impl(
1636	IODispatchSourceCancelHandler handler)
1637	{
1638	IOInterruptState is;
1639
1640	is = IOSimpleLockLockDisableInterrupt(lock: ivars->lock);
1641	ivars->canceled = true;
1642	if (ivars->waiter) {
1643	thread_wakeup_thread(event: (event_t) ivars, thread: ivars->waiter);
1644	ivars->waiter = NULL;
1645	}
1646	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
1647
1648	return kIOReturnSuccess;
1649	}
1650
1651	kern_return_t
1652	IOInterruptDispatchSource::CheckForWork_Impl(
1653	const IORPC rpc,
1654	bool synchronous)
1655	{
1656	IOReturn ret = kIOReturnNotReady;
1657	IOInterruptState is;
1658	bool willWait;
1659	bool canceled;
1660	wait_result_t waitResult;
1661	uint64_t icount;
1662	uint64_t itime;
1663	thread_t self;
1664
1665	self = current_thread();
1666	icount = `0`;
1667	do {
1668	willWait = false;
1669	is = IOSimpleLockLockDisableInterrupt(lock: ivars->lock);
1670	canceled = ivars->canceled;
1671	if (!canceled) {
1672	if ((icount = ivars->count)) {
1673	itime = ivars->time;
1674	ivars->count = `0`;
1675	waitResult = THREAD_AWAKENED;
1676	} else if (synchronous) {
1677	assert(NULL == ivars->waiter);
1678	ivars->waiter = self;
1679	waitResult = assert_wait(event: (event_t) ivars, THREAD_INTERRUPTIBLE);
1680	}
1681	willWait = (synchronous && (waitResult == THREAD_WAITING));
1682	if (willWait && (kIOInterruptTypeLevel & ivars->interruptType) && ivars->enable) {
1683	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
1684	ivars->provider->enableInterrupt(source: ivars->intIndex);
1685	} else {
1686	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
1687	}
1688	} else {
1689	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
1690	}
1691	if (willWait) {
1692	waitResult = thread_block(THREAD_CONTINUE_NULL);
1693	if (THREAD_INTERRUPTED == waitResult) {
1694	is = IOSimpleLockLockDisableInterrupt(lock: ivars->lock);
1695	ivars->waiter = NULL;
1696	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
1697	canceled = true;
1698	break;
1699	}
1700	}
1701	} while (synchronous && !icount && !canceled);
1702
1703	if (icount && ivars->action) {
1704	ret = InterruptOccurred(rpc, action: ivars->action, count: icount, time: itime);
1705	}
1706
1707	return ret;
1708	}
1709
1710	void
1711	IOInterruptDispatchSource::InterruptOccurred_Impl(
1712	OSAction * action,
1713	uint64_t count,
1714	uint64_t time)
1715	{
1716	}
1717
1718	kern_return_t
1719	IOInterruptDispatchSource::GetLastInterrupt_Impl(
1720	uint64_t * pCount,
1721	uint64_t * pTime)
1722	{
1723	IOInterruptState is;
1724	uint64_t count, time;
1725
1726	is = IOSimpleLockLockDisableInterrupt(lock: ivars->lock);
1727	count = ivars->count;
1728	time = ivars->time;
1729	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
1730
1731	if (pCount) {
1732	*pCount = count;
1733	}
1734	if (pTime) {
1735	*pTime = time;
1736	}
1737	return kIOReturnSuccess;
1738	}
1739
1740	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
1741
1742	enum {
1743	kIOServiceNotificationTypeCount = kIOServiceNotificationTypeLast + `1`,
1744	};
1745
1746	struct IOServiceNotificationDispatchSource_IVars {
1747	OSObject * serverName;
1748	OSAction * action;
1749	IOLock * lock;
1750	IONotifier * notifier;
1751	OSDictionary * interestNotifiers;
1752	OSBoundedArray<OSArray *, kIOServiceNotificationTypeCount> pending;
1753	bool enable;
1754	};
1755
1756	kern_return_t
1757	IOServiceNotificationDispatchSource::Create_Impl(
1758	OSDictionary * matching,
1759	uint64_t options,
1760	IODispatchQueue * queue,
1761	IOServiceNotificationDispatchSource ** notification)
1762	{
1763	IOUserServer * us;
1764	IOReturn ret;
1765	IOServiceNotificationDispatchSource * inst;
1766
1767	inst = OSTypeAlloc(IOServiceNotificationDispatchSource);
1768	if (!inst->init()) {
1769	OSSafeReleaseNULL(inst);
1770	return kIOReturnNoMemory;
1771	}
1772
1773	us = (typeof(us))thread_iokit_tls_get(index: `0`);
1774	assert(OSDynamicCast(IOUserServer, us));
1775	if (!us) {
1776	OSSafeReleaseNULL(inst);
1777	return kIOReturnError;
1778	}
1779	inst->ivars->serverName = us->copyProperty(aKey: gIOUserServerNameKey);
1780	if (!inst->ivars->serverName) {
1781	OSSafeReleaseNULL(inst);
1782	return kIOReturnNoMemory;
1783	}
1784
1785	inst->ivars->lock = IOLockAlloc();
1786	if (!inst->ivars->lock) {
1787	OSSafeReleaseNULL(inst);
1788	return kIOReturnNoMemory;
1789	}
1790	for (uint32_t idx = `0`; idx < kIOServiceNotificationTypeCount; idx++) {
1791	inst->ivars->pending [idx] = OSArray::withCapacity(capacity: `4`);
1792	if (!inst->ivars->pending [idx]) {
1793	OSSafeReleaseNULL(inst);
1794	return kIOReturnNoMemory;
1795	}
1796	}
1797	inst->ivars->interestNotifiers = OSDictionary::withCapacity(capacity: `4`);
1798	if (!inst->ivars->interestNotifiers) {
1799	OSSafeReleaseNULL(inst);
1800	return kIOReturnNoMemory;
1801	}
1802
1803	inst->ivars->notifier = IOService::addMatchingNotification(type: gIOMatchedNotification, matching, priority: `0` /priority/,
1804	handler: ^bool (IOService * newService, IONotifier * notifier) {
1805	bool notifyReady = false;
1806	IONotifier * interest;
1807	OSObject * serverName;
1808	bool okToUse;
1809
1810	serverName = newService->copyProperty(aKey: gIOUserServerNameKey);
1811	okToUse = (serverName && inst->ivars->serverName->isEqualTo(anObject: serverName));
1812	OSSafeReleaseNULL(serverName);
1813	if (!okToUse) {
1814	OSObject * prop;
1815	OSObject * str;
1816
1817	if (!newService->reserved->uvars \|\| !newService->reserved->uvars->userServer) {
1818	return false;
1819	}
1820	str = OSString::withCStringNoCopy(kIODriverKitAllowsPublishEntitlementsKey);
1821	if (!str) {
1822	return false;
1823	}
1824	okToUse = newService->reserved->uvars->userServer->checkEntitlements(prop: str, NULL, NULL);
1825	if (!okToUse) {
1826	DKLOG(DKS ": publisher entitlements check failed\n", DKN(newService));
1827	return false;
1828	}
1829	prop = newService->copyProperty(kIODriverKitPublishEntitlementsKey);
1830	if (!prop) {
1831	return false;
1832	}
1833	okToUse = us->checkEntitlements(prop, NULL, NULL);
1834	if (!okToUse) {
1835	DKLOG(DKS ": subscriber entitlements check failed\n", DKN(newService));
1836	return false;
1837	}
1838	}
1839
1840	IOLockLock(inst->ivars->lock);
1841	notifyReady = (`0` == inst->ivars->pending [kIOServiceNotificationTypeMatched]->getCount());
1842	inst->ivars->pending [kIOServiceNotificationTypeMatched]->setObject(newService);
1843	IOLockUnlock(inst->ivars->lock);
1844
1845	interest = newService->registerInterest(typeOfInterest: gIOGeneralInterest,
1846	handler: ^IOReturn (uint32_t messageType, IOService * provider,
1847	void * messageArgument, size_t argSize) {
1848	IONotifier * interest;
1849	bool notifyReady = false;
1850
1851	switch (messageType) {
1852	case kIOMessageServiceIsTerminated:
1853	IOLockLock(inst->ivars->lock);
1854	notifyReady = (`0` == inst->ivars->pending [kIOServiceNotificationTypeTerminated]->getCount());
1855	inst->ivars->pending [kIOServiceNotificationTypeTerminated]->setObject(provider);
1856	if (inst->ivars->interestNotifiers != NULL) {
1857	interest = (typeof(interest))inst->ivars->interestNotifiers->getObject(aKey: (const OSSymbol *) newService);
1858	assert(interest);
1859	interest->remove();
1860	inst->ivars->interestNotifiers->removeObject(aKey: (const OSSymbol *) newService);
1861	}
1862	IOLockUnlock(inst->ivars->lock);
1863	break;
1864	default:
1865	break;
1866	}
1867	if (notifyReady && inst->ivars->action) {
1868	inst->ServiceNotificationReady(action: inst->ivars->action);
1869	}
1870	return kIOReturnSuccess;
1871	});
1872	if (interest) {
1873	IOLockLock(inst->ivars->lock);
1874	inst->ivars->interestNotifiers->setObject(aKey: (const OSSymbol *) newService, anObject: interest);
1875	IOLockUnlock(inst->ivars->lock);
1876	}
1877	if (notifyReady) {
1878	if (inst->ivars->action) {
1879	inst->ServiceNotificationReady(action: inst->ivars->action);
1880	}
1881	}
1882	return false;
1883	});
1884
1885	if (!inst->ivars->notifier) {
1886	OSSafeReleaseNULL(inst);
1887	ret = kIOReturnError;
1888	}
1889
1890	*notification = inst;
1891	ret = kIOReturnSuccess;
1892
1893	return ret;
1894	}
1895
1896	kern_return_t
1897	IOServiceNotificationDispatchSource::CopyNextNotification_Impl(
1898	uint64_t * type,
1899	IOService ** service,
1900	uint64_t * options)
1901	{
1902	IOService * next;
1903	uint32_t idx;
1904
1905	IOLockLock(ivars->lock);
1906	for (idx = `0`; idx < kIOServiceNotificationTypeCount; idx++) {
1907	next = (IOService *) ivars->pending [idx]->getObject(index: `0`);
1908	if (next) {
1909	next->retain();
1910	ivars->pending [idx]->removeObject(index: `0`);
1911	break;
1912	}
1913	}
1914	IOLockUnlock(ivars->lock);
1915
1916	if (idx == kIOServiceNotificationTypeCount) {
1917	idx = kIOServiceNotificationTypeNone;
1918	}
1919	*type = idx;
1920	*service = next;
1921	*options = `0`;
1922
1923	return kIOReturnSuccess;
1924	}
1925
1926	bool
1927	IOServiceNotificationDispatchSource::init()
1928	{
1929	if (!super::init()) {
1930	return false;
1931	}
1932	ivars = IOMallocType(IOServiceNotificationDispatchSource_IVars);
1933
1934	return true;
1935	}
1936
1937	void
1938	IOServiceNotificationDispatchSource::free()
1939	{
1940	if (ivars) {
1941	if (ivars->notifier) {
1942	ivars->notifier->remove();
1943	ivars->notifier = NULL;
1944	}
1945	if (ivars->interestNotifiers) {
1946	OSDictionary * savedInterestNotifiers = NULL;
1947
1948	// the lock is always initialized first, so it should exist
1949	assert(ivars->lock);
1950
1951	// Prevent additional changes to interestNotifiers
1952	IOLockLock(ivars->lock);
1953	savedInterestNotifiers = ivars->interestNotifiers;
1954	ivars->interestNotifiers = NULL;
1955	IOLockUnlock(ivars->lock);
1956
1957	// Remove all interest notifiers
1958	savedInterestNotifiers->iterateObjects(block: ^bool (const OSSymbol * key, OSObject * object) {
1959	IONotifier * interest = (typeof(interest))object;
1960	interest->remove();
1961	return false;
1962	});
1963	OSSafeReleaseNULL(savedInterestNotifiers);
1964	}
1965	for (uint32_t idx = `0`; idx < kIOServiceNotificationTypeCount; idx++) {
1966	OSSafeReleaseNULL(ivars->pending[idx]);
1967	}
1968	if (ivars->lock) {
1969	IOLockFree(lock: ivars->lock);
1970	ivars->lock = NULL;
1971	}
1972	OSSafeReleaseNULL(ivars->serverName);
1973	IOFreeType(ivars, IOServiceNotificationDispatchSource_IVars);
1974	}
1975
1976	super::free();
1977	}
1978
1979	kern_return_t
1980	IOServiceNotificationDispatchSource::SetHandler_Impl(
1981	OSAction * action)
1982	{
1983	IOReturn ret;
1984	bool notifyReady;
1985
1986	notifyReady = false;
1987
1988	IOLockLock(ivars->lock);
1989	OSSafeReleaseNULL(ivars->action);
1990	action->retain();
1991	ivars->action = action;
1992	if (action) {
1993	for (uint32_t idx = `0`; idx < kIOServiceNotificationTypeCount; idx++) {
1994	notifyReady = (ivars->pending [idx]->getCount());
1995	if (notifyReady) {
1996	break;
1997	}
1998	}
1999	}
2000	IOLockUnlock(ivars->lock);
2001
2002	if (notifyReady) {
2003	ServiceNotificationReady(action);
2004	}
2005	ret = kIOReturnSuccess;
2006
2007	return ret;
2008	}
2009
2010	kern_return_t
2011	IOServiceNotificationDispatchSource::SetEnableWithCompletion_Impl(
2012	bool enable,
2013	IODispatchSourceCancelHandler handler)
2014	{
2015	if (enable == ivars->enable) {
2016	return kIOReturnSuccess;
2017	}
2018
2019	IOLockLock(ivars->lock);
2020	ivars->enable = enable;
2021	IOLockUnlock(ivars->lock);
2022
2023	return kIOReturnSuccess;
2024	}
2025
2026	kern_return_t
2027	IOServiceNotificationDispatchSource::Cancel_Impl(
2028	IODispatchSourceCancelHandler handler)
2029	{
2030	return kIOReturnUnsupported;
2031	}
2032
2033	kern_return_t
2034	IOServiceNotificationDispatchSource::CheckForWork_Impl(
2035	const IORPC rpc,
2036	bool synchronous)
2037	{
2038	return kIOReturnNotReady;
2039	}
2040
2041	kern_return_t
2042	IOServiceNotificationDispatchSource::DeliverNotifications(IOServiceNotificationBlock block)
2043	{
2044	return kIOReturnUnsupported;
2045	}
2046
2047
2048	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
2049
2050	OSDictionary *
2051	IOService::CreatePropertyMatchingDictionary(const char * key, OSObjectPtr value, OSDictionary * matching)
2052	{
2053	OSDictionary * result;
2054	const OSSymbol * keySym;
2055
2056	keySym = OSSymbol::withCString(cString: key);
2057	result = propertyMatching(key: keySym, value: (const OSObject *) value, table: matching);
2058	OSSafeReleaseNULL(keySym);
2059
2060	return result;
2061	}
2062
2063	OSDictionary *
2064	IOService::CreatePropertyMatchingDictionary(const char * key, const char * stringValue, OSDictionary * matching)
2065	{
2066	OSDictionary * result;
2067	OSString * value;
2068
2069	value = OSString::withCString(cString: stringValue);
2070	result = CreatePropertyMatchingDictionary(key, value, matching);
2071	OSSafeReleaseNULL(value);
2072
2073	return result;
2074	}
2075
2076	OSDictionary *
2077	IOService::CreateKernelClassMatchingDictionary(OSString * className, OSDictionary * matching)
2078	{
2079	if (!className) {
2080	return NULL;
2081	}
2082	if (!matching) {
2083	matching = OSDictionary::withCapacity(capacity: `2`);
2084	if (!matching) {
2085	return NULL;
2086	}
2087	}
2088	matching->setObject(kIOProviderClassKey, anObject: className);
2089
2090	return matching;
2091	}
2092
2093	OSDictionary *
2094	IOService::CreateKernelClassMatchingDictionary(const char * className, OSDictionary * matching)
2095	{
2096	OSDictionary * result;
2097	OSString * string;
2098
2099	string = OSString::withCString(cString: className);
2100	result = CreateKernelClassMatchingDictionary(className: string, matching);
2101	OSSafeReleaseNULL(string);
2102
2103	return result;
2104	}
2105
2106	OSDictionary *
2107	IOService::CreateUserClassMatchingDictionary(OSString * className, OSDictionary * matching)
2108	{
2109	return CreatePropertyMatchingDictionary(kIOUserClassKey, value: className, matching);
2110	}
2111
2112	OSDictionary *
2113	IOService::CreateUserClassMatchingDictionary(const char * className, OSDictionary * matching)
2114	{
2115	return CreatePropertyMatchingDictionary(kIOUserClassKey, stringValue: className, matching);
2116	}
2117
2118	OSDictionary *
2119	IOService::CreateNameMatchingDictionary(OSString * serviceName, OSDictionary * matching)
2120	{
2121	if (!serviceName) {
2122	return NULL;
2123	}
2124	if (!matching) {
2125	matching = OSDictionary::withCapacity(capacity: `2`);
2126	if (!matching) {
2127	return NULL;
2128	}
2129	}
2130	matching->setObject(kIONameMatchKey, anObject: serviceName);
2131
2132	return matching;
2133	}
2134
2135	OSDictionary *
2136	IOService::CreateNameMatchingDictionary(const char * serviceName, OSDictionary * matching)
2137	{
2138	OSDictionary * result;
2139	OSString * string;
2140
2141	string = OSString::withCString(cString: serviceName);
2142	result = CreateNameMatchingDictionary(serviceName: string, matching);
2143	OSSafeReleaseNULL(string);
2144
2145	return result;
2146	}
2147
2148
2149
2150	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
2151
2152	kern_return_t
2153	IOUserServer::waitInterruptTrap(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6)
2154	{
2155	IOReturn ret = kIOReturnBadArgument;
2156	IOInterruptState is;
2157	IOInterruptDispatchSource * interrupt;
2158	IOInterruptDispatchSource_IVars * ivars;
2159	IOInterruptDispatchSourcePayload payload;
2160
2161	bool willWait;
2162	bool canceled;
2163	wait_result_t waitResult;
2164	thread_t self;
2165
2166	OSObject * object;
2167
2168	object = iokit_lookup_object_with_port_name(name: (mach_port_name_t)(uintptr_t)p1, type: IKOT_UEXT_OBJECT, task: current_task());
2169
2170	if (!object) {
2171	return kIOReturnBadArgument;
2172	}
2173	if (!(interrupt = OSDynamicCast(IOInterruptDispatchSource, object))) {
2174	ret = kIOReturnBadArgument;
2175	} else {
2176	self = current_thread();
2177	ivars = interrupt->ivars;
2178	payload.count = `0`;
2179	do {
2180	willWait = false;
2181	is = IOSimpleLockLockDisableInterrupt(lock: ivars->lock);
2182	canceled = ivars->canceled;
2183	if (!canceled) {
2184	if ((payload.count = ivars->count)) {
2185	payload.time = ivars->time;
2186	ivars->count = `0`;
2187	waitResult = THREAD_AWAKENED;
2188	} else {
2189	assert(NULL == ivars->waiter);
2190	ivars->waiter = self;
2191	waitResult = assert_wait(event: (event_t) ivars, THREAD_INTERRUPTIBLE);
2192	}
2193	willWait = (waitResult == THREAD_WAITING);
2194	if (willWait && (kIOInterruptTypeLevel & ivars->interruptType) && ivars->enable) {
2195	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
2196	ivars->provider->enableInterrupt(source: ivars->intIndex);
2197	} else {
2198	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
2199	}
2200	} else {
2201	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
2202	}
2203	if (willWait) {
2204	waitResult = thread_block(THREAD_CONTINUE_NULL);
2205	if (THREAD_INTERRUPTED == waitResult) {
2206	is = IOSimpleLockLockDisableInterrupt(lock: ivars->lock);
2207	ivars->waiter = NULL;
2208	IOSimpleLockUnlockEnableInterrupt(lock: ivars->lock, state: is);
2209	canceled = true;
2210	break;
2211	}
2212	}
2213	} while (!payload.count && !canceled);
2214	ret = (payload.count ? kIOReturnSuccess : kIOReturnAborted);
2215	}
2216
2217	if (kIOReturnSuccess == ret) {
2218	int copyerr = copyout(&payload, (user_addr_t) p2, sizeof(payload));
2219	if (copyerr) {
2220	ret = kIOReturnVMError;
2221	}
2222	}
2223
2224	object->release();
2225
2226	return ret;
2227	}
2228
2229	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
2230
2231	kern_return_t
2232	IOUserServer::Create_Impl(
2233	const char * name,
2234	uint64_t tag,
2235	uint64_t options,
2236	OSString * bundleID,
2237	IOUserServer ** server)
2238	{
2239	IOReturn ret;
2240	IOUserServer * us;
2241	const OSSymbol * sym;
2242	OSNumber * serverTag;
2243	io_name_t rname;
2244	OSKext * kext;
2245
2246	us = (typeof(us))thread_iokit_tls_get(index: `0`);
2247	assert(OSDynamicCast(IOUserServer, us));
2248	if (kIODKLogSetup & gIODKDebug) {
2249	DKLOG(DKS "::Create(" DKS ") %p\n", DKN(us), name, tag, us);
2250	}
2251	if (!us) {
2252	return kIOReturnError;
2253	}
2254
2255	if (bundleID) {
2256	kext = OSKext::lookupKextWithIdentifier(kextIdentifier: bundleID->getCStringNoCopy());
2257	if (kext) {
2258	us->setTaskLoadTag(kext);
2259	us->setDriverKitUUID(kext);
2260	us->setDriverKitStatistics(kext);
2261	OSKext::OSKextLogDriverKitInfoLoad(kext);
2262	OSSafeReleaseNULL(kext);
2263	} else {
2264	DKLOG(DKS "::Create(" DKS "): could not find OSKext for %s\n", DKN(us), name, tag, bundleID->getCStringNoCopy());
2265	}
2266
2267	us->fAllocationName = kern_allocation_name_allocate(name: bundleID->getCStringNoCopy(), suballocs: `0`);
2268	assert(us->fAllocationName);
2269	}
2270
2271	sym = OSSymbol::withCString(cString: name);
2272	serverTag = OSNumber::withNumber(value: tag, numberOfBits: `64`);
2273
2274	us->setProperty(aKey: gIOUserServerNameKey, anObject: (OSObject *) sym);
2275	us->setProperty(aKey: gIOUserServerTagKey, anObject: serverTag);
2276
2277	serverTag->release();
2278	OSSafeReleaseNULL(sym);
2279
2280	snprintf(rname, count: sizeof(rname), "IOUserServer(%s-0x%qx)", name, tag);
2281	us->setName(name: rname);
2282
2283	us->retain();
2284	*server = us;
2285	ret = kIOReturnSuccess;
2286
2287	return ret;
2288	}
2289
2290	kern_return_t
2291	IOUserServer::RegisterService_Impl()
2292	{
2293	kern_return_t ret = IOService::RegisterService_Impl();
2294
2295	return ret;
2296	}
2297
2298	kern_return_t
2299	IOUserServer::Exit_Impl(
2300	const char * reason)
2301	{
2302	return kIOReturnUnsupported;
2303	}
2304
2305	kern_return_t
2306	IOUserServer::LoadModule_Impl(
2307	const char * path)
2308	{
2309	return kIOReturnUnsupported;
2310	}
2311
2312
2313	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
2314
2315	kern_return_t
2316	IODispatchQueue::Create_Impl(
2317	const char * name,
2318	uint64_t options,
2319	uint64_t priority,
2320	IODispatchQueue ** queue)
2321	{
2322	IODispatchQueue * result;
2323	IOUserServer * us;
2324
2325	result = OSTypeAlloc(IODispatchQueue);
2326	if (!result) {
2327	return kIOReturnNoMemory;
2328	}
2329	if (!result->init()) {
2330	OSSafeReleaseNULL(result);
2331	return kIOReturnNoMemory;
2332	}
2333
2334	*queue = result;
2335
2336	if (!strcmp(s1: "Root", s2: name)) {
2337	us = (typeof(us))thread_iokit_tls_get(index: `0`);
2338	assert(OSDynamicCast(IOUserServer, us));
2339	us->setRootQueue(result);
2340	}
2341
2342	if (kIODKLogSetup & gIODKDebug) {
2343	DKLOG("IODispatchQueue::Create %s %p\n", name, result);
2344	}
2345
2346	return kIOReturnSuccess;
2347	}
2348
2349	kern_return_t
2350	IODispatchQueue::SetPort_Impl(
2351	mach_port_t port)
2352	{
2353	if (MACH_PORT_NULL != ivars->serverPort) {
2354	return kIOReturnNotReady;
2355	}
2356
2357	ivars->serverPort = port;
2358	return kIOReturnSuccess;
2359	}
2360
2361	bool
2362	IODispatchQueue::init()
2363	{
2364	ivars = IOMallocType(IODispatchQueue_IVars);
2365	ivars->queue = this;
2366
2367	return true;
2368	}
2369
2370	void
2371	IODispatchQueue::free()
2372	{
2373	if (ivars && ivars->serverPort) {
2374	ipc_port_release_send(port: ivars->serverPort);
2375	ivars->serverPort = MACH_PORT_NULL;
2376	}
2377	IOFreeType(ivars, IODispatchQueue_IVars);
2378	super::free();
2379	}
2380
2381	bool
2382	IODispatchQueue::OnQueue()
2383	{
2384	return false;
2385	}
2386
2387	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
2388
2389
2390	kern_return_t
2391	OSMetaClassBase::Dispatch(IORPC rpc)
2392	{
2393	return kIOReturnUnsupported;
2394	}
2395
2396	kern_return_t
2397	OSMetaClassBase::Invoke(IORPC rpc)
2398	{
2399	IOReturn ret = kIOReturnUnsupported;
2400	OSMetaClassBase * object;
2401	OSAction * action;
2402	IOService * service;
2403	IOUserServer * us;
2404	IORPCMessage * message;
2405
2406	assert(rpc.sendSize >= (sizeof(IORPCMessageMach) + sizeof(IORPCMessage)));
2407	message = IORPCMessageFromMach(msg: rpc.message, reply: false);
2408	if (!message) {
2409	return kIOReturnIPCError;
2410	}
2411	message->flags \|= kIORPCMessageKernel;
2412
2413	us = NULL;
2414	if (!(kIORPCMessageLocalHost & message->flags)) {
2415	us = OSDynamicCast(IOUserServer, this);
2416	if (!us) {
2417	IOEventLink * eventLink = NULL;
2418	IOWorkGroup * workgroup = NULL;
2419
2420	if ((action = OSDynamicCast(OSAction, this))) {
2421	object = IOUserServer::target(action, message);
2422	} else {
2423	object = this;
2424	}
2425	if ((service = OSDynamicCast(IOService, object))
2426	&& service->reserved->uvars) {
2427	// xxx other classes
2428	us = service->reserved->uvars->userServer;
2429	} else if (action) {
2430	us = action->ivars->userServer;
2431	} else if ((eventLink = OSDynamicCast(IOEventLink, object))) {
2432	us = eventLink->ivars->userServer;
2433	} else if ((workgroup = OSDynamicCast(IOWorkGroup, object))) {
2434	us = workgroup->ivars->userServer;
2435	}
2436	}
2437	}
2438	if (us) {
2439	message->flags \|= kIORPCMessageRemote;
2440	ret = us->rpc(rpc);
2441	if (kIOReturnSuccess != ret) {
2442	if (kIODKLogIPC & gIODKDebug) {
2443	DKLOG("OSMetaClassBase::Invoke user 0x%x\n", ret);
2444	}
2445	}
2446	} else {
2447	if (kIODKLogIPC & gIODKDebug) {
2448	DKLOG("OSMetaClassBase::Invoke kernel %s 0x%qx\n", getMetaClass()->getClassName(), message->msgid);
2449	}
2450	void * prior = thread_iokit_tls_get(index: `0`);
2451	thread_iokit_tls_set(index: `0`, NULL);
2452	ret = Dispatch(rpc);
2453	thread_iokit_tls_set(index: `0`, data: prior);
2454	}
2455
2456	return ret;
2457	}
2458
2459	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
2460
2461	struct IOPStrings {
2462	uint32_t dataSize;
2463	uint32_t count;
2464	const char strings[`0`];
2465	};
2466
2467	kern_return_t
2468	OSUserMetaClass::Dispatch(IORPC rpc)
2469	{
2470	if (meta) {
2471	return const_cast<OSMetaClass *>(meta)->Dispatch(rpc);
2472	} else {
2473	return kIOReturnUnsupported;
2474	}
2475	}
2476
2477	void
2478	OSUserMetaClass::free()
2479	{
2480	if (queueNames) {
2481	IOFreeData(address: queueNames, size: sizeof(IOPStrings) + queueNames->dataSize * sizeof(char));
2482	queueNames = NULL;
2483	}
2484	if (description) {
2485	IOFreeData(address: description, size: description->descriptionSize);
2486	description = NULL;
2487	}
2488	IODeleteData(methods, uint64_t, `2` * methodCount);
2489	if (meta) {
2490	meta->releaseMetaClass();
2491	}
2492	if (name) {
2493	name->release();
2494	}
2495	OSObject::free();
2496	}
2497
2498	/*
2499	* Sets the loadTag of the associated OSKext
2500	* in the dext task.
2501	* NOTE: different instances of the same OSKext
2502	* (so same BounleID but different tasks)
2503	* will have the same loadTag.
2504	*/
2505	void
2506	IOUserServer::setTaskLoadTag(OSKext *kext)
2507	{
2508	task_t owningTask;
2509	uint32_t loadTag, prev_taskloadTag;
2510
2511	owningTask = this->fOwningTask;
2512	if (!owningTask) {
2513	printf("%s: fOwningTask not found\n", __FUNCTION__);
2514	return;
2515	}
2516
2517	loadTag = kext->getLoadTag();
2518	prev_taskloadTag = set_task_loadTag(task: owningTask, loadTag);
2519	if (prev_taskloadTag) {
2520	printf("%s: found the task loadTag already set to %u (set to %u)\n",
2521	__FUNCTION__, prev_taskloadTag, loadTag);
2522	}
2523	}
2524
2525	/*
2526	* Sets the OSKext uuid as the uuid of the userspace
2527	* dext executable.
2528	*/
2529	void
2530	IOUserServer::setDriverKitUUID(OSKext *kext)
2531	{
2532	task_t task;
2533	proc_t p;
2534	uuid_t p_uuid, k_uuid;
2535	OSData *k_data_uuid;
2536	OSData *new_uuid;
2537	uuid_string_t uuid_string = "";
2538
2539	task = this->fOwningTask;
2540	if (!task) {
2541	printf("%s: fOwningTask not found\n", __FUNCTION__);
2542	return;
2543	}
2544
2545	p = (proc_t)(get_bsdtask_info(task));
2546	if (!p) {
2547	printf("%s: proc not found\n", __FUNCTION__);
2548	return;
2549	}
2550	proc_getexecutableuuid(p, p_uuid, sizeof(p_uuid));
2551
2552	k_data_uuid = kext->copyUUID();
2553	if (k_data_uuid) {
2554	memcpy(dst: &k_uuid, src: k_data_uuid->getBytesNoCopy(), n: sizeof(k_uuid));
2555	OSSafeReleaseNULL(k_data_uuid);
2556	if (uuid_compare(uu1: k_uuid, uu2: p_uuid) != `0`) {
2557	printf("%s: uuid not matching\n", __FUNCTION__);
2558	}
2559	return;
2560	}
2561
2562	uuid_unparse(uu: p_uuid, out: uuid_string);
2563	new_uuid = OSData::withValue(value: p_uuid);
2564	kext->setDriverKitUUID(new_uuid);
2565	}
2566
2567	void
2568	IOUserServer::setDriverKitStatistics(OSKext *kext)
2569	{
2570	OSDextStatistics * statistics = kext->copyDextStatistics();
2571	if (statistics == NULL) {
2572	panic("Kext %s was not a DriverKit OSKext", kext->getIdentifierCString());
2573	}
2574	fStatistics = statistics;
2575	}
2576
2577	void
2578	IOUserServer::setCheckInToken(IOUserServerCheckInToken *token)
2579	{
2580	if (token != NULL && fCheckInToken == NULL) {
2581	token->retain();
2582	fCheckInToken = token;
2583	iokit_clear_registered_ports(task: fOwningTask);
2584	} else {
2585	printf("%s: failed to set check in token. token=%p, fCheckInToken=%p\n", __FUNCTION__, token, fCheckInToken);
2586	}
2587	}
2588
2589	bool
2590	IOUserServer::serviceMatchesCheckInToken(IOUserServerCheckInToken *token)
2591	{
2592	if (token != NULL) {
2593	bool result = token == fCheckInToken;
2594	if (result) {
2595	fCheckInToken->complete();
2596	}
2597	return result;
2598	} else {
2599	printf("%s: null check in token\n", __FUNCTION__);
2600	return false;
2601	}
2602	}
2603
2604	// entitlements - dict of entitlements to check
2605	// prop - string - if present return true
2606	// - array of strings - if any present return true
2607	// - array of arrays of strings - in each leaf array all must be present
2608	// - if any top level array succeeds return true
2609	// consumes one reference of prop
2610	bool
2611	IOUserServer::checkEntitlements(
2612	OSDictionary * entitlements, OSObject * prop,
2613	IOService * provider, IOService * dext)
2614	{
2615	OSDictionary * matching;
2616
2617	if (!prop) {
2618	return true;
2619	}
2620	if (!entitlements) {
2621	OSSafeReleaseNULL(prop);
2622	return false;
2623	}
2624
2625	matching = NULL;
2626	if (dext) {
2627	matching = dext->dictionaryWithProperties();
2628	if (!matching) {
2629	OSSafeReleaseNULL(prop);
2630	return false;
2631	}
2632	}
2633
2634	bool allPresent __block = false;
2635	prop->iterateObjects(block: ^bool (OSObject * object) {
2636	allPresent = false;
2637	object->iterateObjects(block: ^bool (OSObject * object) {
2638	OSString * string;
2639	OSObject * value;
2640	string = OSDynamicCast(OSString, object);
2641	value = entitlements->getObject(aKey: string);
2642	if (matching && value) {
2643	matching->setObject(aKey: string, anObject: value);
2644	}
2645	allPresent = (NULL != value);
2646	// early terminate if not found
2647	return !allPresent;
2648	});
2649	// early terminate if found
2650	return allPresent;
2651	});
2652
2653	if (allPresent && matching && provider) {
2654	allPresent = provider->matchPropertyTable(table: matching);
2655	}
2656
2657	OSSafeReleaseNULL(matching);
2658	OSSafeReleaseNULL(prop);
2659
2660	return allPresent;
2661	}
2662
2663	bool
2664	IOUserServer::checkEntitlements(OSObject * prop, IOService * provider, IOService * dext)
2665	{
2666	return checkEntitlements(entitlements: fEntitlements, prop, provider, dext);
2667	}
2668
2669	bool
2670	IOUserServer::checkEntitlements(IOService * provider, IOService * dext)
2671	{
2672	OSObject * prop;
2673	bool ok;
2674
2675	if (!fOwningTask) {
2676	return false;
2677	}
2678
2679	prop = provider->copyProperty(aKey: gIOServiceDEXTEntitlementsKey);
2680	ok = checkEntitlements(entitlements: fEntitlements, prop, provider, dext);
2681	if (!ok) {
2682	DKLOG(DKS ": provider entitlements check failed\n", DKN(dext));
2683	}
2684	if (ok) {
2685	prop = dext->copyProperty(aKey: gIOServiceDEXTEntitlementsKey);
2686	ok = checkEntitlements(entitlements: fEntitlements, prop, NULL, NULL);
2687	if (!ok) {
2688	DKLOG(DKS ": family entitlements check failed\n", DKN(dext));
2689	}
2690	}
2691
2692	return ok;
2693	}
2694
2695	IOReturn
2696	IOUserServer::exit(const char * reason)
2697	{
2698	DKLOG("%s::exit(%s)\n", getName(), reason);
2699	Exit(reason);
2700	return kIOReturnSuccess;
2701	}
2702
2703	IOReturn
2704	IOUserServer::kill(const char * reason)
2705	{
2706	IOReturn ret = kIOReturnError;
2707	if (fOwningTask != NULL) {
2708	DKLOG("%s::kill(%s)\n", getName(), reason);
2709	task_bsdtask_kill(fOwningTask);
2710	ret = kIOReturnSuccess;
2711	}
2712	return ret;
2713	}
2714
2715	OSObjectUserVars *
2716	IOUserServer::varsForObject(OSObject * obj)
2717	{
2718	IOService * service;
2719
2720	if ((service = OSDynamicCast(IOService, obj))) {
2721	return service->reserved->uvars;
2722	}
2723
2724	return NULL;
2725	}
2726
2727	IOPStrings *
2728	IOUserServer::copyInStringArray(const char * string, uint32_t userSize)
2729	{
2730	IOPStrings * array;
2731	vm_size_t alloc;
2732	size_t len;
2733	const char * end;
2734	OSBoundedPtr<const char> cstr;
2735
2736	if (userSize <= `1`) {
2737	return NULL;
2738	}
2739
2740	if (os_add_overflow(sizeof(IOPStrings), userSize, &alloc)) {
2741	assert(false);
2742	return NULL;
2743	}
2744	if (alloc > `16384`) {
2745	assert(false);
2746	return NULL;
2747	}
2748	array = (typeof(array))IOMallocData(alloc);
2749	if (!array) {
2750	return NULL;
2751	}
2752	array->dataSize = userSize;
2753	bcopy(src: string, dst: (void *) &array->strings[`0`], n: userSize);
2754
2755	array->count = `0`;
2756	end = &array->strings[array->dataSize];
2757	cstr = OSBoundedPtr<const char>(&array->strings[`0`], &array->strings[`0`], end);
2758	while ((len = (unsigned char)cstr [`0`])) {
2759	cstr ++;
2760	if ((cstr + len) >= end) {
2761	break;
2762	}
2763	cstr += len;
2764	array->count++;
2765	}
2766	if (len) {
2767	IOFreeData(address: array, size: alloc);
2768	array = NULL;
2769	}
2770
2771	return array;
2772	}
2773
2774	uint32_t
2775	IOUserServer::stringArrayIndex(IOPStrings * array, const char * look)
2776	{
2777	uint32_t idx;
2778	size_t len, llen;
2779	OSBoundedPtr<const char> cstr;
2780	const char * end;
2781
2782	idx = `0`;
2783	end = &array->strings[array->dataSize];
2784	cstr = OSBoundedPtr<const char>(&array->strings[`0`], &array->strings[`0`], end);
2785
2786	llen = strlen(s: look);
2787	while ((len = (unsigned char)cstr [`0`])) {
2788	cstr ++;
2789	if ((cstr + len) >= end) {
2790	break;
2791	}
2792	if ((len == llen) && !strncmp(s1: cstr.discard_bounds(), s2: look, n: len)) {
2793	return idx;
2794	}
2795	cstr += len;
2796	idx++;
2797	}
2798
2799	return -`1U`;
2800	}
2801	#define kIODispatchQueueStopped ((IODispatchQueue *) -1L)
2802
2803	IODispatchQueue *
2804	IOUserServer::queueForObject(OSObject * obj, uint64_t msgid)
2805	{
2806	IODispatchQueue * queue;
2807	OSObjectUserVars * uvars;
2808	uint64_t option;
2809
2810	uvars = varsForObject(obj);
2811	if (!uvars) {
2812	return NULL;
2813	}
2814	if (!uvars->queueArray) {
2815	if (uvars->stopped) {
2816	return kIODispatchQueueStopped;
2817	}
2818	return NULL;
2819	}
2820	queue = uvars->queueArray [`0`];
2821
2822	if (uvars->userMeta
2823	&& uvars->userMeta->methods) {
2824	uint32_t idx, baseIdx;
2825	uint32_t lim;
2826	// bsearch
2827	for (baseIdx = `0`, lim = uvars->userMeta->methodCount; lim; lim >>= `1`) {
2828	idx = baseIdx + (lim >> `1`);
2829	if (msgid == uvars->userMeta->methods[idx]) {
2830	option = uvars->userMeta->methods[uvars->userMeta->methodCount + idx];
2831	option &= `0xFF`;
2832	if (option < uvars->userMeta->queueNames->count) {
2833	queue = uvars->queueArray [option + `1`];
2834	}
2835	break;
2836	} else if (msgid > uvars->userMeta->methods[idx]) {
2837	// move right
2838	baseIdx += (lim >> `1`) + `1`;
2839	lim--;
2840	}
2841	// else move left
2842	}
2843	}
2844	return queue;
2845	}
2846
2847	IOReturn
2848	IOUserServer::objectInstantiate(OSObject * obj, IORPC rpc, IORPCMessage * message)
2849	{
2850	IOReturn ret;
2851	OSString * str;
2852	OSObject * prop;
2853	IOService * service;
2854
2855	OSAction * action;
2856	OSObject * target;
2857	uint32_t queueCount, queueAlloc;
2858	const char * resultClassName;
2859	uint64_t resultFlags;
2860
2861	mach_msg_size_t replySize;
2862	uint32_t methodCount;
2863	const uint64_t * methods;
2864	IODispatchQueue * queue;
2865	OSUserMetaClass * userMeta;
2866	OSObjectUserVars * uvars;
2867	uint32_t idx;
2868	ipc_port_t sendPort;
2869
2870	OSObject_Instantiate_Rpl_Content * reply;
2871	IODispatchQueue ** unboundedQueueArray = NULL;
2872	queueCount = `0`;
2873	methodCount = `0`;
2874	methods = NULL;
2875	str = NULL;
2876	prop = NULL;
2877	userMeta = NULL;
2878	resultClassName = NULL;
2879	resultFlags = `0`;
2880	ret = kIOReturnUnsupportedMode;
2881
2882	service = OSDynamicCast(IOService, obj);
2883	action = OSDynamicCast(OSAction, obj);
2884	if (!service) {
2885	// xxx other classes hosted
2886	resultFlags \|= kOSObjectRPCKernel;
2887	resultFlags \|= kOSObjectRPCRemote;
2888	} else {
2889	if (service->isInactive()) {
2890	DKLOG(DKS "::instantiate inactive\n", DKN(service));
2891	return kIOReturnOffline;
2892	}
2893	prop = service->copyProperty(aKey: gIOUserClassKey);
2894	str = OSDynamicCast(OSString, prop);
2895	if (!service->reserved->uvars) {
2896	resultFlags \|= kOSObjectRPCRemote;
2897	resultFlags \|= kOSObjectRPCKernel;
2898	} else if (this != service->reserved->uvars->userServer) {
2899	// remote, use base class
2900	resultFlags \|= kOSObjectRPCRemote;
2901	}
2902	if (service->reserved->uvars && service->reserved->uvars->userServer) {
2903	if (!str) {
2904	DKLOG("no IOUserClass defined for " DKS "\n", DKN(service));
2905	OSSafeReleaseNULL(prop);
2906	return kIOReturnError;
2907	}
2908	IOLockLock(service->reserved->uvars->userServer->fLock);
2909	userMeta = (typeof(userMeta))service->reserved->uvars->userServer->fClasses->getObject(aKey: str);
2910	IOLockUnlock(service->reserved->uvars->userServer->fLock);
2911	}
2912	}
2913	if (!str && !userMeta) {
2914	const OSMetaClass * meta;
2915	meta = obj->getMetaClass();
2916	IOLockLock(fLock);
2917	if (action) {
2918	str = action->ivars->typeName;
2919	if (str) {
2920	userMeta = (typeof(userMeta))fClasses->getObject(aKey: str);
2921	}
2922	}
2923	while (meta && !userMeta) {
2924	str = (OSString *) meta->getClassNameSymbol();
2925	userMeta = (typeof(userMeta))fClasses->getObject(aKey: str);
2926	if (!userMeta) {
2927	meta = meta->getSuperClass();
2928	}
2929	}
2930	IOLockUnlock(fLock);
2931	}
2932	if (str) {
2933	if (!userMeta) {
2934	IOLockLock(fLock);
2935	userMeta = (typeof(userMeta))fClasses->getObject(aKey: str);
2936	IOLockUnlock(fLock);
2937	}
2938	if (kIODKLogSetup & gIODKDebug) {
2939	DKLOG("userMeta %s %p\n", str->getCStringNoCopy(), userMeta);
2940	}
2941	if (userMeta) {
2942	if (kOSObjectRPCRemote & resultFlags) {
2943	if (!action) {
2944	/ Special case: For OSAction subclasses, do not use the superclass /
2945	while (userMeta && !(kOSClassCanRemote & userMeta->description->flags)) {
2946	userMeta = userMeta->superMeta;
2947	}
2948	}
2949	if (userMeta) {
2950	resultClassName = userMeta->description->name;
2951	ret = kIOReturnSuccess;
2952	}
2953	} else {
2954	service->reserved->uvars->userMeta = userMeta;
2955	queueAlloc = `1`;
2956	if (userMeta->queueNames) {
2957	queueAlloc += userMeta->queueNames->count;
2958	}
2959	unboundedQueueArray = IONewZero(IODispatchQueue *, queueAlloc);
2960	service->reserved->uvars->queueArray =
2961	OSBoundedArrayRef<IODispatchQueue *>(unboundedQueueArray, queueAlloc);
2962	resultClassName = str->getCStringNoCopy();
2963	ret = kIOReturnSuccess;
2964	}
2965	} else if (kIODKLogSetup & gIODKDebug) {
2966	DKLOG("userMeta %s was not found in fClasses\n", str->getCStringNoCopy());
2967	IOLockLock(fLock);
2968	fClasses->iterateObjects(block: ^bool (const OSSymbol * key, OSObject * val) {
2969	DKLOG(" fClasses[\"%s\"] => %p\n", key->getCStringNoCopy(), val);
2970	return false;
2971	});
2972	IOLockUnlock(fLock);
2973	}
2974	}
2975	OSSafeReleaseNULL(prop);
2976
2977	IORPCMessageMach * machReply = rpc.reply;
2978	replySize = sizeof(OSObject_Instantiate_Rpl);
2979
2980	if ((kIOReturnSuccess == ret) && (kOSObjectRPCRemote & resultFlags)) {
2981	target = obj;
2982	if (action) {
2983	if (action->ivars->referenceSize) {
2984	resultFlags \|= kOSObjectRPCKernel;
2985	} else {
2986	resultFlags &= ~kOSObjectRPCKernel;
2987	if (action->ivars->target) {
2988	target = action->ivars->target;
2989	queueCount = `1`;
2990	queue = queueForObject(obj: target, msgid: action->ivars->targetmsgid);
2991	if (!queue && action->ivars->userServer) {
2992	queue = action->ivars->userServer->fRootQueue;
2993	}
2994	idx = `0`;
2995	sendPort = NULL;
2996	if (queue && (kIODispatchQueueStopped != queue)) {
2997	sendPort = ipc_port_copy_send_mqueue(queue->ivars->serverPort);
2998	}
2999	replySize = sizeof(OSObject_Instantiate_Rpl)
3000	+ queueCount * sizeof(machReply->objects[`0`])
3001	+ `2` * methodCount * sizeof(reply->methods[`0`]);
3002	if (replySize > rpc.replySize) {
3003	assert(false);
3004	return kIOReturnIPCError;
3005	}
3006	machReply->objects[idx].type = MACH_MSG_PORT_DESCRIPTOR;
3007	machReply->objects[idx].disposition = MACH_MSG_TYPE_MOVE_SEND;
3008	machReply->objects[idx].name = sendPort;
3009	machReply->objects[idx].pad2 = `0`;
3010	machReply->objects[idx].pad_end = `0`;
3011	}
3012	}
3013	} else {
3014	uvars = varsForObject(obj: target);
3015	if (uvars && uvars->userMeta) {
3016	queueCount = `1`;
3017	if (uvars->userMeta->queueNames) {
3018	queueCount += uvars->userMeta->queueNames->count;
3019	}
3020	methods = &uvars->userMeta->methods[`0`];
3021	methodCount = uvars->userMeta->methodCount;
3022	replySize = sizeof(OSObject_Instantiate_Rpl)
3023	+ queueCount * sizeof(machReply->objects[`0`])
3024	+ `2` * methodCount * sizeof(reply->methods[`0`]);
3025	if (replySize > rpc.replySize) {
3026	assert(false);
3027	return kIOReturnIPCError;
3028	}
3029	for (idx = `0`; idx < queueCount; idx++) {
3030	queue = uvars->queueArray [idx];
3031	sendPort = NULL;
3032	if (queue) {
3033	sendPort = ipc_port_copy_send_mqueue(queue->ivars->serverPort);
3034	}
3035	machReply->objects[idx].type = MACH_MSG_PORT_DESCRIPTOR;
3036	machReply->objects[idx].disposition = MACH_MSG_TYPE_MOVE_SEND;
3037	machReply->objects[idx].name = sendPort;
3038	machReply->objects[idx].pad2 = `0`;
3039	machReply->objects[idx].pad_end = `0`;
3040	}
3041	}
3042	}
3043	}
3044
3045	if (kIODKLogIPC & gIODKDebug) {
3046	DKLOG("instantiate object %s with user class %s\n", obj->getMetaClass()->getClassName(), str ? str->getCStringNoCopy() : "(null)");
3047	}
3048
3049	if (kIOReturnSuccess != ret) {
3050	DKLOG("%s: no user class found\n", str ? str->getCStringNoCopy() : obj->getMetaClass()->getClassName());
3051	resultClassName = "unknown";
3052	}
3053
3054	machReply->msgh.msgh_id = kIORPCVersionCurrentReply;
3055	machReply->msgh.msgh_size = replySize;
3056	machReply->msgh_body.msgh_descriptor_count = queueCount;
3057
3058	reply = (typeof(reply))IORPCMessageFromMachReply(msg: machReply);
3059	if (!reply) {
3060	return kIOReturnIPCError;
3061	}
3062	if (methodCount) {
3063	bcopy(src: methods, dst: &reply->methods[`0`], n: methodCount * `2` * sizeof(reply->methods[`0`]));
3064	}
3065	reply->__hdr.msgid = OSObject_Instantiate_ID;
3066	reply->__hdr.flags = kIORPCMessageOneway;
3067	reply->__hdr.objectRefs = `0`;
3068	reply->__pad = `0`;
3069	reply->flags = resultFlags;
3070	strlcpy(dst: reply->classname, src: resultClassName, n: sizeof(reply->classname));
3071	reply->__result = ret;
3072
3073	ret = kIOReturnSuccess;
3074
3075	return ret;
3076	}
3077
3078	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
3079
3080	IOReturn
3081	IOUserServer::kernelDispatch(OSObject * obj, IORPC rpc)
3082	{
3083	IOReturn ret;
3084	IORPCMessage * message;
3085
3086	message = rpc.kernelContent;
3087	if (!message) {
3088	return kIOReturnIPCError;
3089	}
3090
3091	if (OSObject_Instantiate_ID == message->msgid) {
3092	ret = objectInstantiate(obj, rpc, message);
3093	if (kIOReturnSuccess != ret) {
3094	DKLOG("%s: instantiate failed 0x%x\n", obj->getMetaClass()->getClassName(), ret);
3095	}
3096	} else {
3097	if (kIODKLogIPC & gIODKDebug) {
3098	DKLOG("%s::Dispatch kernel 0x%qx\n", obj->getMetaClass()->getClassName(), message->msgid);
3099	}
3100	ret = obj->Dispatch(rpc);
3101	if (kIODKLogIPC & gIODKDebug) {
3102	DKLOG("%s::Dispatch kernel 0x%qx result 0x%x\n", obj->getMetaClass()->getClassName(), message->msgid, ret);
3103	}
3104	}
3105
3106	return ret;
3107	}
3108
3109
3110	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
3111
3112	OSObject *
3113	IOUserServer::target(OSAction * action, IORPCMessage * message)
3114	{
3115	OSObject * object;
3116
3117	if (message->msgid != action->ivars->msgid) {
3118	return action;
3119	}
3120	object = action->ivars->target;
3121	if (!object) {
3122	return action;
3123	}
3124	message->msgid = action->ivars->targetmsgid;
3125	message->objects[`0`] = (OSObjectRef) object;
3126	if (kIORPCMessageRemote & message->flags) {
3127	object->retain();
3128	#ifndef __clang_analyzer__
3129	// Hide the release of 'action' from the clang static analyzer to suppress
3130	// an overrelease diagnostic. The analyzer doesn't have a way to express the
3131	// non-standard contract of this method, which is that it releases 'action' when
3132	// the message flags have kIORPCMessageRemote set.
3133	action->release();
3134	#endif
3135	}
3136	if (kIODKLogIPC & gIODKDebug) {
3137	DKLOG("TARGET %s msg 0x%qx from 0x%qx\n", object->getMetaClass()->getClassName(), message->msgid, action->ivars->msgid);
3138	}
3139
3140	return object;
3141	}
3142
3143	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
3144
3145	kern_return_t
3146	uext_server(ipc_port_t receiver, ipc_kmsg_t requestkmsg, ipc_kmsg_t * pReply)
3147	{
3148	kern_return_t ret;
3149	OSObject * object;
3150	IOUserServer * server;
3151
3152	object = IOUserServer::copyObjectForSendRight(port: receiver, type: IKOT_UEXT_OBJECT);
3153	server = OSDynamicCast(IOUserServer, object);
3154	if (!server) {
3155	OSSafeReleaseNULL(object);
3156	return KERN_INVALID_NAME;
3157	}
3158
3159	IORPCMessage * message = (typeof(message))ikm_udata_from_header(kmsg: requestkmsg);
3160
3161	ret = server->server(requestkmsg, message, preply: pReply);
3162	object->release();
3163
3164	return ret;
3165	}
3166
3167	/*
3168	* Chosen to hit kalloc zones (as opposed to the VM).
3169	* doesn't include the trailer size which ipc_kmsg_alloc() will add
3170	*/
3171	#define MAX_UEXT_REPLY_SIZE 0x17c0
3172	static_assert(MAX_UEXT_REPLY_SIZE + MAX_TRAILER_SIZE <= KALLOC_SAFE_ALLOC_SIZE);
3173
3174	kern_return_t
3175	IOUserServer::server(ipc_kmsg_t requestkmsg, IORPCMessage * message, ipc_kmsg_t * pReply)
3176	{
3177	kern_return_t ret;
3178	mach_msg_size_t replyAlloc;
3179	ipc_kmsg_t replykmsg;
3180	IORPCMessageMach * msgin;
3181	IORPCMessageMach * msgout;
3182	IORPCMessage * reply;
3183	uint32_t replySize;
3184	OSObject * object;
3185	OSAction * action;
3186	bool oneway;
3187	uint64_t msgid;
3188
3189	msgin = (typeof(msgin))ikm_header(requestkmsg);
3190	replyAlloc = `0`;
3191	msgout = NULL;
3192	replykmsg = NULL;
3193
3194	if (msgin->msgh.msgh_size < (sizeof(IORPCMessageMach) + sizeof(IORPCMessage))) {
3195	if (kIODKLogIPC & gIODKDebug) {
3196	DKLOG("UEXT notify %o\n", msgin->msgh.msgh_id);
3197	}
3198	return KERN_NOT_SUPPORTED;
3199	}
3200
3201	if (!(MACH_MSGH_BITS_COMPLEX & msgin->msgh.msgh_bits)) {
3202	msgin->msgh_body.msgh_descriptor_count = `0`;
3203	}
3204	if (!message) {
3205	return kIOReturnIPCError;
3206	}
3207	if (message->objectRefs == `0`) {
3208	return kIOReturnIPCError;
3209	}
3210	ret = copyInObjects(mach: msgin, message, size: msgin->msgh.msgh_size, copyObjects: true, consumePorts: false);
3211	if (kIOReturnSuccess != ret) {
3212	if (kIODKLogIPC & gIODKDebug) {
3213	DKLOG("UEXT copyin(0x%x) %x\n", ret, msgin->msgh.msgh_id);
3214	}
3215	return KERN_NOT_SUPPORTED;
3216	}
3217
3218	if (msgin->msgh_body.msgh_descriptor_count < `1`) {
3219	return KERN_NOT_SUPPORTED;
3220	}
3221	object = (OSObject *) message->objects[`0`];
3222	msgid = message->msgid;
3223	message->flags &= ~kIORPCMessageKernel;
3224	message->flags \|= kIORPCMessageRemote;
3225
3226	if ((action = OSDynamicCast(OSAction, object))) {
3227	object = target(action, message);
3228	msgid = message->msgid;
3229	}
3230
3231	oneway = (`0` != (kIORPCMessageOneway & message->flags));
3232	assert(oneway \|\| (MACH_PORT_NULL != msgin->msgh.msgh_local_port));
3233
3234	replyAlloc = oneway ? `0` : MAX_UEXT_REPLY_SIZE;
3235
3236
3237
3238
3239	if (replyAlloc) {
3240	/*
3241	* Same as:
3242	* ipc_kmsg_alloc(MAX_UEXT_REPLY_SIZE_MACH, MAX_UEXT_REPLY_SIZE_MESSAGE,
3243	* IPC_KMSG_ALLOC_KERNEL \| IPC_KMSG_ALLOC_ZERO \| IPC_KMSG_ALLOC_LINEAR \|
3244	* IPC_KMSG_ALLOC_NOFAIL);
3245	*/
3246	replykmsg = ipc_kmsg_alloc_uext_reply(MAX_UEXT_REPLY_SIZE);
3247	msgout = (typeof(msgout))ikm_header(replykmsg);
3248	}
3249
3250	IORPC rpc = { .message = msgin, .reply = msgout, .sendSize = msgin->msgh.msgh_size, .replySize = replyAlloc, .kernelContent = message };
3251
3252	if (object) {
3253	kern_allocation_name_t prior;
3254	bool setAllocationName;
3255
3256	setAllocationName = (NULL != fAllocationName);
3257	if (setAllocationName) {
3258	prior = thread_set_allocation_name(new_name: fAllocationName);
3259	}
3260	thread_iokit_tls_set(index: `0`, data: this);
3261	ret = kernelDispatch(obj: object, rpc);
3262	thread_iokit_tls_set(index: `0`, NULL);
3263	if (setAllocationName) {
3264	thread_set_allocation_name(new_name: prior);
3265	}
3266	} else {
3267	ret = kIOReturnBadArgument;
3268	}
3269
3270	// release objects
3271	consumeObjects(message, messageSize: msgin->msgh.msgh_size);
3272
3273	// release ports
3274	copyInObjects(mach: msgin, message, size: msgin->msgh.msgh_size, copyObjects: false, consumePorts: true);
3275
3276	if (!oneway) {
3277	if (kIOReturnSuccess == ret) {
3278	replySize = msgout->msgh.msgh_size;
3279	reply = IORPCMessageFromMachReply(msg: msgout);
3280	if (!reply) {
3281	ret = kIOReturnIPCError;
3282	} else {
3283	ret = copyOutObjects(mach: msgout, message: reply, size: replySize, consume: (kIORPCVersionCurrentReply == msgout->msgh.msgh_id) / =>!InvokeReply /);
3284	}
3285	}
3286	if (kIOReturnSuccess != ret) {
3287	IORPCMessageErrorReturnContent * errorMsg;
3288
3289	msgout->msgh_body.msgh_descriptor_count = `0`;
3290	msgout->msgh.msgh_id = kIORPCVersionCurrentReply;
3291	errorMsg = (typeof(errorMsg))IORPCMessageFromMachReply(msg: msgout);
3292	errorMsg->hdr.msgid = message->msgid;
3293	errorMsg->hdr.flags = kIORPCMessageOneway \| kIORPCMessageError;
3294	errorMsg->hdr.objectRefs = `0`;
3295	errorMsg->result = ret;
3296	errorMsg->pad = `0`;
3297	replySize = sizeof(IORPCMessageErrorReturn);
3298	}
3299
3300	msgout->msgh.msgh_bits = MACH_MSGH_BITS_COMPLEX \|
3301	MACH_MSGH_BITS_SET(MACH_MSGH_BITS_LOCAL(msgin->msgh.msgh_bits) /remote/, `0` /local/, `0`, `0`);
3302
3303	msgout->msgh.msgh_remote_port = msgin->msgh.msgh_local_port;
3304	msgout->msgh.msgh_local_port = MACH_PORT_NULL;
3305	msgout->msgh.msgh_voucher_port = (mach_port_name_t) `0`;
3306	msgout->msgh.msgh_reserved = `0`;
3307	msgout->msgh.msgh_size = replySize;
3308	}
3309
3310	*pReply = replykmsg;
3311	return KERN_SUCCESS;
3312	}
3313
3314	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
3315
3316	#define MAX_OBJECT_COUNT(mach, size, message) \
3317	((uint32_t)(((((size) + ((uintptr_t) (mach))) - ((uintptr_t) (&message->objects[0]))) / sizeof(OSObjectRef))))
3318
3319	#pragma pack(push, 4)
3320	struct UEXTTrapReply {
3321	uint64_t replySize;
3322	IORPCMessage replyMessage;
3323	};
3324	#pragma pack(pop)
3325
3326	kern_return_t
3327	IOUserServerUEXTTrap(OSObject * object, void * p1, void * p2, void * p3, void * p4, void * p5, void * p6)
3328	{
3329	const user_addr_t msg = (uintptr_t) p1;
3330	size_t inSize = (uintptr_t) p2;
3331	user_addr_t out = (uintptr_t) p3;
3332	size_t outSize = (uintptr_t) p4;
3333	mach_port_name_t objectName1 = (mach_port_name_t)(uintptr_t) p5;
3334	size_t totalSize;
3335	OSObject * objectArg1;
3336
3337	IORPCMessageMach * mach;
3338	mach_msg_port_descriptor_t * descs;
3339
3340	#pragma pack(4)
3341	struct {
3342	uint32_t pad;
3343	IORPCMessageMach mach;
3344	mach_msg_port_descriptor_t objects[`2`];
3345	IOTrapMessageBuffer buffer;
3346	} buffer;
3347	#pragma pack()
3348
3349	IOReturn ret;
3350	OSAction * action;
3351	int copyerr;
3352	IORPCMessage * message;
3353	IORPCMessage * reply;
3354	IORPC rpc;
3355	uint64_t refs;
3356	uint32_t maxObjectCount;
3357	size_t copySize;
3358	UEXTTrapReply * replyHdr;
3359	uintptr_t p;
3360
3361	bzero(s: &buffer, n: sizeof(buffer));
3362
3363	p = (typeof(p)) & buffer.buffer[`0`];
3364	if (os_add_overflow(inSize, outSize, &totalSize)) {
3365	return kIOReturnMessageTooLarge;
3366	}
3367	if (totalSize > sizeof(buffer.buffer)) {
3368	return kIOReturnMessageTooLarge;
3369	}
3370	if (inSize < sizeof(IORPCMessage)) {
3371	return kIOReturnIPCError;
3372	}
3373	copyerr = copyin(msg, &buffer.buffer[`0`], inSize);
3374	if (copyerr) {
3375	return kIOReturnVMError;
3376	}
3377
3378	message = (typeof(message))p;
3379	refs = message->objectRefs;
3380	if ((refs > `2`) \|\| !refs) {
3381	return kIOReturnUnsupported;
3382	}
3383	if (!(kIORPCMessageSimpleReply & message->flags)) {
3384	return kIOReturnUnsupported;
3385	}
3386	message->flags &= ~(kIORPCMessageKernel \| kIORPCMessageRemote);
3387
3388	descs = (typeof(descs))(p - refs * sizeof(*descs));
3389	mach = (typeof(mach))(p - refs * sizeof(descs) - sizeof(mach));
3390
3391	mach->msgh.msgh_id = kIORPCVersionCurrent;
3392	mach->msgh.msgh_size = (mach_msg_size_t) (sizeof(IORPCMessageMach) + refs * sizeof(descs) + inSize); // totalSize was checked*
3393	mach->msgh_body.msgh_descriptor_count = ((mach_msg_size_t) refs);
3394
3395	rpc.message = mach;
3396	rpc.sendSize = mach->msgh.msgh_size;
3397	rpc.reply = (IORPCMessageMach *) (p + inSize);
3398	rpc.replySize = ((uint32_t) (sizeof(buffer.buffer) - inSize)); // inSize was checked
3399	rpc.kernelContent = message;
3400
3401	message->objects[`0`] = `0`;
3402	if ((action = OSDynamicCast(OSAction, object))) {
3403	maxObjectCount = MAX_OBJECT_COUNT(rpc.message, rpc.sendSize, message);
3404	if (refs > maxObjectCount) {
3405	return kIOReturnBadArgument;
3406	}
3407	if (refs < `2`) {
3408	DKLOG("invalid refs count %qd in message id 0x%qx\n", refs, message->msgid);
3409	return kIOReturnBadArgument;
3410	}
3411	object = IOUserServer::target(action, message);
3412	message->objects[`1`] = (OSObjectRef) action;
3413	if (kIODKLogIPC & gIODKDebug) {
3414	DKLOG("%s::Dispatch(trap) kernel 0x%qx\n", object->getMetaClass()->getClassName(), message->msgid);
3415	}
3416	ret = object->Dispatch(rpc);
3417	} else {
3418	objectArg1 = NULL;
3419	if (refs > `1`) {
3420	if (objectName1) {
3421	objectArg1 = iokit_lookup_uext_ref_current_task(name: objectName1);
3422	if (!objectArg1) {
3423	return kIOReturnIPCError;
3424	}
3425	}
3426	message->objects[`1`] = (OSObjectRef) objectArg1;
3427	}
3428	if (kIODKLogIPC & gIODKDebug) {
3429	DKLOG("%s::Dispatch(trap) kernel 0x%qx\n", object->getMetaClass()->getClassName(), message->msgid);
3430	}
3431	ret = object->Dispatch(rpc);
3432	if (kIODKLogIPC & gIODKDebug) {
3433	DKLOG("%s::Dispatch(trap) kernel 0x%qx 0x%x\n", object->getMetaClass()->getClassName(), message->msgid, ret);
3434	}
3435	OSSafeReleaseNULL(objectArg1);
3436
3437	if (kIOReturnSuccess == ret) {
3438	if (rpc.reply->msgh_body.msgh_descriptor_count) {
3439	return kIOReturnIPCError;
3440	}
3441	reply = IORPCMessageFromMachReply(msg: rpc.reply);
3442	if (!reply) {
3443	return kIOReturnIPCError;
3444	}
3445	copySize = rpc.reply->msgh.msgh_size - (((uintptr_t) reply) - ((uintptr_t) rpc.reply)) + sizeof(uint64_t);
3446	if (copySize > outSize) {
3447	return kIOReturnIPCError;
3448	}
3449	replyHdr = (UEXTTrapReply ) ((uintptr_t)reply - sizeof*(uint64_t));
3450	replyHdr->replySize = copySize;
3451	copyerr = copyout(replyHdr, out, copySize);
3452	if (copyerr) {
3453	return kIOReturnVMError;
3454	}
3455	}
3456	}
3457
3458	return ret;
3459	}
3460
3461	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
3462
3463	IOReturn
3464	IOUserServer::rpc(IORPC rpc)
3465	{
3466	if (isInactive() && !fRootQueue) {
3467	return kIOReturnOffline;
3468	}
3469
3470	IOReturn ret;
3471	IORPCMessage * message;
3472	IORPCMessageMach * mach;
3473	mach_msg_id_t machid;
3474	uint32_t sendSize, replySize;
3475	bool oneway;
3476	uint64_t msgid;
3477	IODispatchQueue * queue;
3478	IOService * service;
3479	ipc_port_t port;
3480	ipc_port_t sendPort;
3481
3482	queue = NULL;
3483	port = NULL;
3484	sendPort = NULL;
3485
3486	mach = rpc.message;
3487	sendSize = rpc.sendSize;
3488	replySize = rpc.replySize;
3489
3490	assert(sendSize >= (sizeof(IORPCMessageMach) + sizeof(IORPCMessage)));
3491
3492	message = rpc.kernelContent;
3493	if (!message) {
3494	return kIOReturnIPCError;
3495	}
3496	msgid = message->msgid;
3497	machid = (msgid >> `32`);
3498
3499	if (mach->msgh_body.msgh_descriptor_count < `1`) {
3500	return kIOReturnNoMedia;
3501	}
3502
3503	IOLockLock(gIOUserServerLock);
3504	if ((service = OSDynamicCast(IOService, (OSObject *) message->objects[`0`]))) {
3505	queue = queueForObject(obj: service, msgid);
3506	}
3507	if (!queue) {
3508	queue = fRootQueue;
3509	}
3510	if (queue && (kIODispatchQueueStopped != queue)) {
3511	port = queue->ivars->serverPort;
3512	}
3513	if (port) {
3514	sendPort = ipc_port_copy_send_mqueue(port);
3515	}
3516	IOLockUnlock(gIOUserServerLock);
3517	if (!sendPort) {
3518	return kIOReturnNotReady;
3519	}
3520
3521	oneway = (`0` != (kIORPCMessageOneway & message->flags));
3522
3523	ret = copyOutObjects(mach, message, size: sendSize, consume: false);
3524
3525	mach->msgh.msgh_bits = MACH_MSGH_BITS_COMPLEX \|
3526	MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, (oneway ? `0` : MACH_MSG_TYPE_MAKE_SEND_ONCE));
3527	mach->msgh.msgh_remote_port = sendPort;
3528	mach->msgh.msgh_local_port = (oneway ? MACH_PORT_NULL : mig_get_reply_port());
3529	mach->msgh.msgh_id = kIORPCVersionCurrent;
3530	mach->msgh.msgh_reserved = `0`;
3531
3532	boolean_t message_moved;
3533
3534	if (oneway) {
3535	ret = kernel_mach_msg_send(msg: &mach->msgh, send_size: sendSize,
3536	MACH_SEND_MSG \| MACH_SEND_ALWAYS \| MACH_SEND_NOIMPORTANCE,
3537	timeout_val: `0`, message_moved: &message_moved);
3538	} else {
3539	assert(replySize >= (sizeof(IORPCMessageMach) + sizeof(IORPCMessage)));
3540	ret = kernel_mach_msg_rpc(msg: &mach->msgh, send_size: sendSize, rcv_size: replySize, FALSE, message_moved: &message_moved);
3541	}
3542
3543	ipc_port_release_send(port: sendPort);
3544
3545	if (MACH_MSG_SUCCESS != ret) {
3546	if (kIODKLogIPC & gIODKDebug) {
3547	DKLOG("mach_msg() failed 0x%x\n", ret);
3548	}
3549	if (!message_moved) {
3550	// release ports
3551	copyInObjects(mach, message, size: sendSize, copyObjects: false, consumePorts: true);
3552	}
3553	}
3554
3555	if ((KERN_SUCCESS == ret) && !oneway) {
3556	if (kIORPCVersionCurrentReply != mach->msgh.msgh_id) {
3557	ret = (MACH_NOTIFY_SEND_ONCE == mach->msgh.msgh_id) ? MIG_SERVER_DIED : MIG_REPLY_MISMATCH;
3558	} else if ((replySize = mach->msgh.msgh_size) < (sizeof(IORPCMessageMach) + sizeof(IORPCMessage))) {
3559	// printf("BAD REPLY SIZE\n");
3560	ret = MIG_BAD_ARGUMENTS;
3561	} else {
3562	if (!(MACH_MSGH_BITS_COMPLEX & mach->msgh.msgh_bits)) {
3563	mach->msgh_body.msgh_descriptor_count = `0`;
3564	}
3565	message = IORPCMessageFromMachReply(msg: mach);
3566	if (!message) {
3567	ret = kIOReturnIPCError;
3568	} else if (message->msgid != msgid) {
3569	// printf("BAD REPLY ID\n");
3570	ret = MIG_BAD_ARGUMENTS;
3571	} else {
3572	bool isError = (`0` != (kIORPCMessageError & message->flags));
3573	ret = copyInObjects(mach, message, size: replySize, copyObjects: !isError, consumePorts: true);
3574	if (kIOReturnSuccess != ret) {
3575	if (kIODKLogIPC & gIODKDebug) {
3576	DKLOG("rpc copyin(0x%x) %x\n", ret, mach->msgh.msgh_id);
3577	}
3578	return KERN_NOT_SUPPORTED;
3579	}
3580	if (isError) {
3581	IORPCMessageErrorReturnContent * errorMsg = (typeof(errorMsg))message;
3582	ret = errorMsg->result;
3583	}
3584	}
3585	}
3586	}
3587
3588	return ret;
3589	}
3590
3591	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
3592
3593	static IORPCMessage *
3594	IORPCMessageFromMachReply(IORPCMessageMach * msg)
3595	{
3596	mach_msg_size_t idx, count;
3597	mach_msg_port_descriptor_t * desc;
3598	mach_msg_port_descriptor_t * maxDesc;
3599	size_t size, msgsize;
3600	bool upgrade;
3601	bool reply = true;
3602
3603	msgsize = msg->msgh.msgh_size;
3604	count = msg->msgh_body.msgh_descriptor_count;
3605	desc = &msg->objects[`0`];
3606	maxDesc = (typeof(maxDesc))(((uintptr_t) msg) + msgsize);
3607	upgrade = (msg->msgh.msgh_id != (reply ? kIORPCVersionCurrentReply : kIORPCVersionCurrent));
3608
3609	if (upgrade) {
3610	OSReportWithBacktrace(str: "obsolete message");
3611	return NULL;
3612	}
3613
3614	for (idx = `0`; idx < count; idx++) {
3615	if (desc >= maxDesc) {
3616	return NULL;
3617	}
3618	switch (desc->type) {
3619	case MACH_MSG_PORT_DESCRIPTOR:
3620	size = sizeof(mach_msg_port_descriptor_t);
3621	break;
3622	case MACH_MSG_OOL_DESCRIPTOR:
3623	size = sizeof(mach_msg_ool_descriptor_t);
3624	break;
3625	default:
3626	return NULL;
3627	}
3628	desc = (typeof(desc))(((uintptr_t) desc) + size);
3629	}
3630	return (IORPCMessage *)(uintptr_t) desc;
3631	}
3632
3633	IORPCMessage *
3634	IORPCMessageFromMach(IORPCMessageMach * msg, bool reply)
3635	{
3636	mach_msg_size_t idx, count;
3637	mach_msg_port_descriptor_t * desc;
3638	mach_msg_port_descriptor_t * maxDesc;
3639	size_t size, msgsize;
3640	bool upgrade;
3641
3642	msgsize = msg->msgh.msgh_size;
3643	count = msg->msgh_body.msgh_descriptor_count;
3644	desc = &msg->objects[`0`];
3645	maxDesc = (typeof(maxDesc))(((uintptr_t) msg) + msgsize);
3646	upgrade = (msg->msgh.msgh_id != (reply ? kIORPCVersionCurrentReply : kIORPCVersionCurrent));
3647
3648	if (upgrade) {
3649	OSReportWithBacktrace(str: "obsolete message");
3650	return NULL;
3651	}
3652
3653	for (idx = `0`; idx < count; idx++) {
3654	if (desc >= maxDesc) {
3655	return NULL;
3656	}
3657	switch (desc->type) {
3658	case MACH_MSG_PORT_DESCRIPTOR:
3659	size = sizeof(mach_msg_port_descriptor_t);
3660	break;
3661	case MACH_MSG_OOL_DESCRIPTOR:
3662	size = sizeof(mach_msg_ool_descriptor_t);
3663	break;
3664	default:
3665	return NULL;
3666	}
3667	desc = (typeof(desc))(((uintptr_t) desc) + size);
3668	}
3669	return (IORPCMessage *)(uintptr_t) desc;
3670	}
3671
3672	ipc_port_t
3673	IOUserServer::copySendRightForObject(OSObject * object, ipc_kobject_type_t type)
3674	{
3675	ipc_port_t port;
3676	ipc_port_t sendPort = NULL;
3677	ipc_kobject_t kobj;
3678
3679	port = iokit_port_for_object(obj: object, type, kobj: &kobj);
3680	if (port) {
3681	sendPort = ipc_kobject_make_send(port, kobj, type);
3682	iokit_release_port(port);
3683	}
3684
3685	return sendPort;
3686	}
3687
3688	OSObject *
3689	IOUserServer::copyObjectForSendRight(ipc_port_t port, ipc_kobject_type_t type)
3690	{
3691	OSObject * object;
3692	object = iokit_lookup_io_object(port, type);
3693	return object;
3694	}
3695
3696	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
3697
3698	// Create a vm_map_copy_t or kalloc'ed data for memory
3699	// to be copied out. ipc will free after the copyout.
3700
3701	static kern_return_t
3702	copyoutkdata(const void * data, vm_size_t len, void ** buf)
3703	{
3704	kern_return_t err;
3705	vm_map_copy_t copy;
3706
3707	err = vm_map_copyin( src_map: kernel_map, CAST_USER_ADDR_T(data), len,
3708	src_destroy: false / src_destroy /, copy_result: &copy);
3709
3710	assert( err == KERN_SUCCESS );
3711	if (err == KERN_SUCCESS) {
3712	buf = (char* *) copy;
3713	}
3714
3715	return err;
3716	}
3717
3718	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
3719
3720	IOReturn
3721	IOUserServer::copyOutObjects(IORPCMessageMach * mach, IORPCMessage * message,
3722	size_t size, bool consume)
3723	{
3724	uint64_t refs;
3725	uint32_t idx, maxObjectCount;
3726	ipc_port_t port;
3727	OSObject * object;
3728	size_t descsize;
3729	mach_msg_port_descriptor_t * desc;
3730	mach_msg_ool_descriptor_t * ool;
3731	vm_map_copy_t copy;
3732	void * address;
3733	mach_msg_size_t length;
3734	kern_return_t kr;
3735	OSSerialize * s;
3736
3737	refs = message->objectRefs;
3738	maxObjectCount = MAX_OBJECT_COUNT(mach, size, message);
3739	// assert(refs <= mach->msgh_body.msgh_descriptor_count);
3740	// assert(refs <= maxObjectCount);
3741	if (refs > mach->msgh_body.msgh_descriptor_count) {
3742	return kIOReturnBadArgument;
3743	}
3744	if (refs > maxObjectCount) {
3745	return kIOReturnBadArgument;
3746	}
3747
3748	desc = &mach->objects[`0`];
3749	for (idx = `0`; idx < refs; idx++) {
3750	object = (OSObject *) message->objects[idx];
3751
3752	switch (desc->type) {
3753	case MACH_MSG_PORT_DESCRIPTOR:
3754	descsize = sizeof(mach_msg_port_descriptor_t);
3755	port = NULL;
3756	if (object) {
3757	#if DEVELOPMENT \|\| DEBUG
3758	if (kIODKLogIPC & gIODKDebug) {
3759	IOMemoryDescriptor * iomd = OSDynamicCast(IOMemoryDescriptor, object);
3760	if (iomd != NULL && (iomd->getFlags() & kIOMemoryThreadSafe) == `0`) {
3761	OSReportWithBacktrace("IOMemoryDescriptor %p was created without kIOMemoryThreadSafe flag", iomd);
3762	}
3763	}
3764	#endif /* DEVELOPMENT \|\| DEBUG */
3765
3766	port = copySendRightForObject(object, type: IKOT_UEXT_OBJECT);
3767	if (!port) {
3768	break;
3769	}
3770	if (consume) {
3771	object->release();
3772	}
3773	message->objects[idx] = `0`;
3774	}
3775	// desc->type = MACH_MSG_PORT_DESCRIPTOR;
3776	desc->disposition = MACH_MSG_TYPE_MOVE_SEND;
3777	desc->name = port;
3778	desc->pad2 = `0`;
3779	desc->pad_end = `0`;
3780	break;
3781
3782	case MACH_MSG_OOL_DESCRIPTOR:
3783	descsize = sizeof(mach_msg_ool_descriptor_t);
3784
3785	length = `0`;
3786	address = NULL;
3787	if (object) {
3788	s = OSSerialize::binaryWithCapacity(inCapacity: `4096`);
3789	assert(s);
3790	if (!s) {
3791	break;
3792	}
3793	s->setIndexed(true);
3794	if (!object->serialize(serializer: s)) {
3795	assert(false);
3796	descsize = -`1UL`;
3797	s->release();
3798	break;
3799	}
3800	length = s->getLength();
3801	kr = copyoutkdata(data: s->text(), len: length, buf: &address);
3802	s->release();
3803	if (KERN_SUCCESS != kr) {
3804	descsize = -`1UL`;
3805	address = NULL;
3806	length = `0`;
3807	}
3808	if (consume) {
3809	object->release();
3810	}
3811	message->objects[idx] = `0`;
3812	}
3813	ool = (typeof(ool))desc;
3814	// ool->type = MACH_MSG_OOL_DESCRIPTOR;
3815	ool->deallocate = false;
3816	ool->copy = MACH_MSG_PHYSICAL_COPY;
3817	ool->size = length;
3818	ool->address = address;
3819	break;
3820
3821	default:
3822	descsize = -`1UL`;
3823	break;
3824	}
3825	if (-`1UL` == descsize) {
3826	break;
3827	}
3828	desc = (typeof(desc))(((uintptr_t) desc) + descsize);
3829	}
3830
3831	if (idx >= refs) {
3832	return kIOReturnSuccess;
3833	}
3834
3835	desc = &mach->objects[`0`];
3836	while (idx--) {
3837	switch (desc->type) {
3838	case MACH_MSG_PORT_DESCRIPTOR:
3839	descsize = sizeof(mach_msg_port_descriptor_t);
3840	port = desc->name;
3841	if (port) {
3842	ipc_port_release_send(port);
3843	}
3844	break;
3845
3846	case MACH_MSG_OOL_DESCRIPTOR:
3847	descsize = sizeof(mach_msg_ool_descriptor_t);
3848	ool = (typeof(ool))desc;
3849	copy = (vm_map_copy_t) ool->address;
3850	if (copy) {
3851	vm_map_copy_discard(copy);
3852	}
3853	break;
3854
3855	default:
3856	descsize = -`1UL`;
3857	break;
3858	}
3859	if (-`1UL` == descsize) {
3860	break;
3861	}
3862	desc = (typeof(desc))(((uintptr_t) desc) + descsize);
3863	}
3864
3865	return kIOReturnBadArgument;
3866	}
3867
3868	IOReturn
3869	IOUserServer::copyInObjects(IORPCMessageMach * mach, IORPCMessage * message,
3870	size_t size, bool copyObjects, bool consumePorts)
3871	{
3872	uint64_t refs;
3873	uint32_t idx, maxObjectCount;
3874	ipc_port_t port;
3875	OSObject * object;
3876	size_t descsize;
3877	mach_msg_port_descriptor_t * desc;
3878	mach_msg_ool_descriptor_t * ool;
3879	vm_map_address_t copyoutdata;
3880	kern_return_t kr;
3881
3882	refs = message->objectRefs;
3883	maxObjectCount = MAX_OBJECT_COUNT(mach, size, message);
3884	// assert(refs <= mach->msgh_body.msgh_descriptor_count);
3885	// assert(refs <= maxObjectCount);
3886	if (refs > mach->msgh_body.msgh_descriptor_count) {
3887	return kIOReturnBadArgument;
3888	}
3889	if (refs > maxObjectCount) {
3890	return kIOReturnBadArgument;
3891	}
3892
3893	desc = &mach->objects[`0`];
3894	for (idx = `0`; idx < refs; idx++) {
3895	switch (desc->type) {
3896	case MACH_MSG_PORT_DESCRIPTOR:
3897	descsize = sizeof(mach_msg_port_descriptor_t);
3898
3899	object = NULL;
3900	port = desc->name;
3901	if (port) {
3902	if (copyObjects) {
3903	object = copyObjectForSendRight(port, type: IKOT_UEXT_OBJECT);
3904	if (!object) {
3905	descsize = -`1UL`;
3906	break;
3907	}
3908	}
3909	if (consumePorts) {
3910	ipc_port_release_send(port);
3911	}
3912	}
3913	break;
3914
3915	case MACH_MSG_OOL_DESCRIPTOR:
3916	descsize = sizeof(mach_msg_ool_descriptor_t);
3917	ool = (typeof(ool))desc;
3918
3919	object = NULL;
3920	if (copyObjects && ool->size && ool->address) {
3921	kr = vm_map_copyout(dst_map: kernel_map, dst_addr: &copyoutdata, copy: (vm_map_copy_t) ool->address);
3922	if (KERN_SUCCESS == kr) {
3923	object = OSUnserializeXML(buffer: (const char *) copyoutdata, bufferSize: ool->size);
3924	kr = vm_deallocate(target_task: kernel_map, address: copyoutdata, size: ool->size);
3925	assert(KERN_SUCCESS == kr);
3926	// vm_map_copyout() has consumed the vm_map_copy_t in the message
3927	ool->size = `0`;
3928	ool->address = NULL;
3929	}
3930	if (!object) {
3931	descsize = -`1UL`;
3932	break;
3933	}
3934	}
3935	break;
3936
3937	default:
3938	descsize = -`1UL`;
3939	break;
3940	}
3941	if (-`1UL` == descsize) {
3942	break;
3943	}
3944	if (copyObjects) {
3945	message->objects[idx] = (OSObjectRef) object;
3946	}
3947	desc = (typeof(desc))(((uintptr_t) desc) + descsize);
3948	}
3949
3950	if (idx >= refs) {
3951	return kIOReturnSuccess;
3952	}
3953
3954	while (idx--) {
3955	object = (OSObject *) message->objects[idx];
3956	OSSafeReleaseNULL(object);
3957	message->objects[idx] = `0`;
3958	}
3959
3960	return kIOReturnBadArgument;
3961	}
3962
3963	IOReturn
3964	IOUserServer::consumeObjects(IORPCMessage * message, size_t messageSize)
3965	{
3966	uint64_t refs, idx;
3967	OSObject * object;
3968
3969	refs = message->objectRefs;
3970	for (idx = `0`; idx < refs; idx++) {
3971	object = (OSObject *) message->objects[idx];
3972	if (object) {
3973	object->release();
3974	message->objects[idx] = `0`;
3975	}
3976	}
3977
3978	return kIOReturnSuccess;
3979	}
3980
3981	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
3982
3983	bool
3984	IOUserServer::finalize(IOOptionBits options)
3985	{
3986	OSArray * services;
3987
3988	if (kIODKLogSetup & gIODKDebug) {
3989	DKLOG("%s::finalize(%p)\n", getName(), this);
3990	}
3991
3992	IOLockLock(gIOUserServerLock);
3993	OSSafeReleaseNULL(fRootQueue);
3994	IOLockUnlock(gIOUserServerLock);
3995
3996	services = NULL;
3997	IOLockLock(fLock);
3998	if (fServices) {
3999	services = OSArray::withArray(array: fServices);
4000	}
4001	IOLockUnlock(fLock);
4002
4003	IOOptionBits terminateFlags = kIOServiceTerminateNeedWillTerminate \| kIOServiceTerminateWithRematch;
4004	if (fCheckInToken) {
4005	bool can_rematch = fCheckInToken->dextTerminate();
4006	if (can_rematch) {
4007	terminateFlags \|= kIOServiceTerminateWithRematchCurrentDext;
4008	} else {
4009	DKLOG("%s::finalize(%p) dext was replaced, do not rematch current dext\n", getName(), this);
4010	}
4011	} else {
4012	terminateFlags \|= kIOServiceTerminateWithRematchCurrentDext;
4013	DKLOG("%s::finalize(%p) could not find fCheckInToken\n", getName(), this);
4014	}
4015
4016	if (services) {
4017	services->iterateObjects(block: ^bool (OSObject * obj) {
4018	int service __unused; // hide outer defn
4019	IOService * nextService;
4020	IOService * provider;
4021	bool started = false;
4022
4023	nextService = (IOService *) obj;
4024	if (kIODKLogSetup & gIODKDebug) {
4025	DKLOG("%s::terminate(" DKS ")\n", getName(), DKN(nextService));
4026	}
4027	if (nextService->reserved->uvars) {
4028	IOUserClient * nextUserClient = OSDynamicCast(IOUserClient, nextService);
4029	provider = nextService->getProvider();
4030	if (nextUserClient) {
4031	nextUserClient->setTerminateDefer(provider, defer: false);
4032	}
4033	started = nextService->reserved->uvars->started;
4034	nextService->reserved->uvars->serverDied = true;
4035
4036	serviceDidStop(client: nextService, provider);
4037	if (provider != NULL && (terminateFlags & kIOServiceTerminateWithRematchCurrentDext) == `0`) {
4038	provider->resetRematchProperties();
4039	}
4040	if (started) {
4041	nextService->terminate(options: terminateFlags);
4042	}
4043	}
4044	if (!started) {
4045	DKLOG("%s::terminate(" DKS ") server exit before start()\n", getName(), DKN(nextService));
4046	serviceStop(service: nextService, NULL);
4047	}
4048	return false;
4049	});
4050	services->release();
4051	}
4052
4053	return IOUserClient::finalize(options);
4054	}
4055
4056	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
4057
4058	#undef super
4059	#define super IOUserClient2022
4060
4061	OSDefineMetaClassAndStructors(IOUserServer, IOUserClient2022)
4062
4063	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
4064
4065	IOUserClient * IOUserServer::withTask(task_t owningTask)
4066	{
4067	IOUserServer * inst;
4068
4069	assert(owningTask == current_task());
4070	if (!task_is_driver(task: owningTask)) {
4071	DKLOG("IOUserServer may only be created with driver tasks\n");
4072	return NULL;
4073	}
4074
4075	inst = new IOUserServer;
4076	if (inst && !inst->init()) {
4077	inst->release();
4078	inst = NULL;
4079	return inst;
4080	}
4081	OS_ANALYZER_SUPPRESS("82033761") inst->PMinit();
4082
4083	inst->fOwningTask = current_task();
4084	task_reference(inst->fOwningTask);
4085
4086	inst->fEntitlements = IOUserClient::copyClientEntitlements(task: inst->fOwningTask);
4087
4088	if (!(kIODKDisableEntitlementChecking & gIODKDebug)) {
4089	proc_t p;
4090	pid_t pid;
4091	const char * name;
4092	p = (proc_t)get_bsdtask_info(inst->fOwningTask);
4093	if (p) {
4094	name = proc_best_name(p);
4095	pid = proc_pid(p);
4096	} else {
4097	name = "unknown";
4098	pid = `0`;
4099	}
4100
4101	if (inst->fEntitlements == NULL) {
4102	#if DEVELOPMENT \|\| DEBUG
4103	panic("entitlements are missing for %s[%d]\n", name, pid);
4104	#else
4105	DKLOG("entitlements are missing for %s[%d]\n", name, pid);
4106	#endif /* DEVELOPMENT \|\| DEBUG */
4107	}
4108
4109
4110	const char * dextTeamID = csproc_get_teamid(p);
4111	if (dextTeamID != NULL) {
4112	inst->fTeamIdentifier = OSString::withCString(cString: dextTeamID);
4113	DKLOG("%s[%d] has team identifier %s\n", name, pid, dextTeamID);
4114	}
4115
4116	if (!IOCurrentTaskHasEntitlement(entitlement: gIODriverKitEntitlementKey->getCStringNoCopy())) {
4117	IOLog(kIODriverKitEntitlementKey " entitlement check failed for %s[%d]\n", name, pid);
4118	inst->release();
4119	inst = NULL;
4120	return inst;
4121	}
4122	}
4123
4124	/ Mark the current task's space as eligible for uext object ports /
4125	iokit_label_dext_task(task: inst->fOwningTask);
4126
4127	inst->fLock = IOLockAlloc();
4128	inst->fServices = OSArray::withCapacity(capacity: `4`);
4129	inst->fClasses = OSDictionary::withCapacity(capacity: `16`);
4130	inst->fClasses->setOptions(options: OSCollection::kSort, mask: OSCollection::kSort);
4131	inst->fPlatformDriver = task_get_platform_binary(task: inst->fOwningTask);
4132	if (csproc_get_validation_category(current_proc(), &inst->fCSValidationCategory) != KERN_SUCCESS) {
4133	inst->fCSValidationCategory = CS_VALIDATION_CATEGORY_INVALID;
4134	}
4135
4136	inst->setProperty(kIOUserClientDefaultLockingKey, anObject: kOSBooleanTrue);
4137	inst->setProperty(kIOUserClientDefaultLockingSetPropertiesKey, anObject: kOSBooleanTrue);
4138	inst->setProperty(kIOUserClientDefaultLockingSingleThreadExternalMethodKey, anObject: kOSBooleanTrue);
4139	//requirement for gIODriverKitEntitlementKey is enforced elsewhere conditionally
4140	inst->setProperty(kIOUserClientEntitlementsKey, anObject: kOSBooleanFalse);
4141
4142	return inst;
4143	}
4144
4145	static bool gIOUserServerLeakObjects = false;
4146
4147	bool
4148	IOUserServer::shouldLeakObjects()
4149	{
4150	return gIOUserServerLeakObjects;
4151	}
4152
4153	void
4154	IOUserServer::beginLeakingObjects()
4155	{
4156	gIOUserServerLeakObjects = true;
4157	}
4158
4159	bool
4160	IOUserServer::isPlatformDriver()
4161	{
4162	return fPlatformDriver;
4163	}
4164
4165	int
4166	IOUserServer::getCSValidationCategory()
4167	{
4168	return fCSValidationCategory;
4169	}
4170
4171
4172	struct IOUserServerRecordExitReasonContext {
4173	task_t task;
4174	os_reason_t reason;
4175	};
4176
4177	static bool
4178	IOUserServerRecordExitReasonMatch(const OSObject obj, void* * context)
4179	{
4180	IOUserServerRecordExitReasonContext * ctx = (IOUserServerRecordExitReasonContext *)context;
4181	IOUserServer * us = OSDynamicCast(IOUserServer, obj);
4182	if (us == NULL) {
4183	return false;
4184	}
4185
4186	if (us->fOwningTask == ctx->task) {
4187	assert(us->fTaskCrashReason == OS_REASON_NULL);
4188	assert(ctx->reason != OS_REASON_NULL);
4189	os_reason_ref(cur_reason: ctx->reason);
4190	us->fTaskCrashReason = ctx->reason;
4191	return true;
4192	}
4193
4194	return false;
4195	}
4196
4197	extern "C" void
4198	IOUserServerRecordExitReason(task_t task, os_reason_t reason)
4199	{
4200	IOUserServerRecordExitReasonContext ctx { .task: task, .reason: reason };
4201	IOUserServer::gMetaClass.applyToInstances(applier: IOUserServerRecordExitReasonMatch, context: &ctx);
4202	}
4203
4204	IOReturn
4205	IOUserServer::clientClose(void)
4206	{
4207	OSArray * services;
4208	bool __block unexpectedExit = false;
4209
4210	if (kIODKLogSetup & gIODKDebug) {
4211	DKLOG("%s::clientClose(%p)\n", getName(), this);
4212	}
4213	services = NULL;
4214	IOLockLock(fLock);
4215	if (fServices) {
4216	services = OSArray::withArray(array: fServices);
4217	}
4218	IOLockUnlock(fLock);
4219
4220	// if this was a an expected exit, termination and stop should have detached at this
4221	// point, so send any provider still attached and not owned by this user server
4222	// the ClientCrashed() notification
4223	if (services) {
4224	services->iterateObjects(block: ^bool (OSObject * obj) {
4225	int service __unused; // hide outer defn
4226	IOService * nextService;
4227	IOService * provider;
4228
4229	nextService = (IOService *) obj;
4230	if (nextService->isInactive()) {
4231	return false;
4232	}
4233	if (nextService->reserved && nextService->reserved->uvars && nextService->reserved->uvars->started) {
4234	unexpectedExit = true;
4235	}
4236	provider = nextService->getProvider();
4237	if (provider
4238	&& (!provider->reserved->uvars \|\| (provider->reserved->uvars->userServer != this))) {
4239	if (kIODKLogSetup & gIODKDebug) {
4240	DKLOG(DKS "::ClientCrashed(" DKS ")\n", DKN(provider), DKN(nextService));
4241	}
4242	provider->ClientCrashed(client: nextService, options: `0`);
4243	}
4244	return false;
4245	});
4246	services->release();
4247	}
4248
4249	if (unexpectedExit &&
4250	!gInUserspaceReboot &&
4251	(fTaskCrashReason != OS_REASON_NULL && fTaskCrashReason->osr_namespace != OS_REASON_JETSAM && fTaskCrashReason->osr_namespace != OS_REASON_RUNNINGBOARD) &&
4252	fStatistics != NULL) {
4253	OSDextCrashPolicy policy = fStatistics->recordCrash();
4254	bool allowPanic;
4255	#if DEVELOPMENT \|\| DEBUG
4256	allowPanic = fPlatformDriver && fEntitlements->getObject(gIODriverKitTestDriverEntitlementKey) != kOSBooleanTrue && !disable_dext_crash_reboot;
4257	#else
4258	allowPanic = fPlatformDriver;
4259	#endif /* DEVELOPMENT \|\| DEBUG */
4260
4261	if (policy == kOSDextCrashPolicyReboot && allowPanic) {
4262	panic("Driver %s has crashed too many times\n", getName());
4263	}
4264	}
4265
4266	terminate();
4267	return kIOReturnSuccess;
4268	}
4269
4270	IOReturn
4271	IOUserServer::setProperties(OSObject * properties)
4272	{
4273	IOReturn kr = kIOReturnUnsupported;
4274	return kr;
4275	}
4276
4277	void
4278	IOUserServer::stop(IOService * provider)
4279	{
4280	if (fOwningTask) {
4281	task_deallocate(fOwningTask);
4282	fOwningTask = TASK_NULL;
4283	}
4284
4285	PMstop();
4286
4287	IOServicePH::serverRemove(server: this);
4288
4289	OSSafeReleaseNULL(fRootQueue);
4290
4291	if (fInterruptLock) {
4292	IOSimpleLockFree(lock: fInterruptLock);
4293	}
4294	}
4295
4296	void
4297	IOUserServer::free()
4298	{
4299	OSSafeReleaseNULL(fEntitlements);
4300	OSSafeReleaseNULL(fClasses);
4301	if (fOwningTask) {
4302	task_deallocate(fOwningTask);
4303	fOwningTask = TASK_NULL;
4304	}
4305	if (fLock) {
4306	IOLockFree(lock: fLock);
4307	}
4308	OSSafeReleaseNULL(fServices);
4309	OSSafeReleaseNULL(fCheckInToken);
4310	OSSafeReleaseNULL(fStatistics);
4311	OSSafeReleaseNULL(fTeamIdentifier);
4312	if (fAllocationName) {
4313	kern_allocation_name_release(allocation: fAllocationName);
4314	fAllocationName = NULL;
4315	}
4316	if (fTaskCrashReason != OS_REASON_NULL) {
4317	os_reason_free(cur_reason: fTaskCrashReason);
4318	}
4319	IOUserClient::free();
4320	}
4321
4322	IOReturn
4323	IOUserServer::registerClass(OSClassDescription * desc, uint32_t size, OSUserMetaClass ** pCls)
4324	{
4325	OSUserMetaClass * cls;
4326	const OSSymbol * sym;
4327	uint64_t * methodOptions;
4328	const char * queueNames;
4329	uint32_t methodOptionsEnd, queueNamesEnd;
4330	IOReturn ret = kIOReturnSuccess;
4331
4332	if (size < sizeof(OSClassDescription)) {
4333	assert(false);
4334	return kIOReturnBadArgument;
4335	}
4336
4337	if (kIODKLogSetup & gIODKDebug) {
4338	DKLOG("%s::registerClass %s, %d, %d\n", getName(), desc->name, desc->queueNamesSize, desc->methodNamesSize);
4339	}
4340
4341	if (desc->descriptionSize != size) {
4342	assert(false);
4343	return kIOReturnBadArgument;
4344	}
4345	if (os_add_overflow(desc->queueNamesOffset, desc->queueNamesSize, &queueNamesEnd)) {
4346	assert(false);
4347	return kIOReturnBadArgument;
4348	}
4349	if (queueNamesEnd > size) {
4350	assert(false);
4351	return kIOReturnBadArgument;
4352	}
4353	if (os_add_overflow(desc->methodOptionsOffset, desc->methodOptionsSize, &methodOptionsEnd)) {
4354	assert(false);
4355	return kIOReturnBadArgument;
4356	}
4357	if (methodOptionsEnd > size) {
4358	assert(false);
4359	return kIOReturnBadArgument;
4360	}
4361	// overlaps?
4362	if ((desc->queueNamesOffset >= desc->methodOptionsOffset) && (desc->queueNamesOffset < methodOptionsEnd)) {
4363	assert(false);
4364	return kIOReturnBadArgument;
4365	}
4366	if ((queueNamesEnd >= desc->methodOptionsOffset) && (queueNamesEnd < methodOptionsEnd)) {
4367	assert(false);
4368	return kIOReturnBadArgument;
4369	}
4370
4371	if (desc->methodOptionsSize & ((`2` * sizeof(uint64_t)) - `1`)) {
4372	assert(false);
4373	return kIOReturnBadArgument;
4374	}
4375	if (sizeof(desc->name) == strnlen(s: desc->name, n: sizeof(desc->name))) {
4376	assert(false);
4377	return kIOReturnBadArgument;
4378	}
4379	if (sizeof(desc->superName) == strnlen(s: desc->superName, n: sizeof(desc->superName))) {
4380	assert(false);
4381	return kIOReturnBadArgument;
4382	}
4383
4384	cls = OSTypeAlloc(OSUserMetaClass);
4385	assert(cls);
4386	if (!cls) {
4387	return kIOReturnNoMemory;
4388	}
4389
4390	cls->description = (typeof(cls->description))IOMallocData(size);
4391	assert(cls->description);
4392	if (!cls->description) {
4393	assert(false);
4394	cls->release();
4395	return kIOReturnNoMemory;
4396	}
4397	bcopy(src: desc, dst: cls->description, n: size);
4398
4399	cls->methodCount = desc->methodOptionsSize / (`2` * sizeof(uint64_t));
4400	cls->methods = IONewData(uint64_t, `2` * cls->methodCount);
4401	if (!cls->methods) {
4402	assert(false);
4403	cls->release();
4404	return kIOReturnNoMemory;
4405	}
4406
4407	methodOptions = (typeof(methodOptions))(((uintptr_t) desc) + desc->methodOptionsOffset);
4408	bcopy(src: methodOptions, dst: cls->methods, n: `2` * cls->methodCount * sizeof(uint64_t));
4409
4410	queueNames = (typeof(queueNames))(((uintptr_t) desc) + desc->queueNamesOffset);
4411	cls->queueNames = copyInStringArray(string: queueNames, userSize: desc->queueNamesSize);
4412
4413	sym = OSSymbol::withCString(cString: desc->name);
4414	assert(sym);
4415	if (!sym) {
4416	assert(false);
4417	cls->release();
4418	return kIOReturnNoMemory;
4419	}
4420
4421	cls->name = sym;
4422	cls->meta = OSMetaClass::copyMetaClassWithName(name: sym);
4423	IOLockLock(fLock);
4424	cls->superMeta = OSDynamicCast(OSUserMetaClass, fClasses->getObject(desc->superName));
4425	if (fClasses->getObject(aKey: sym) != NULL) {
4426	/ class with this name exists /
4427	ret = kIOReturnBadArgument;
4428	} else {
4429	if (fClasses->setObject(aKey: sym, anObject: cls)) {
4430	*pCls = cls;
4431	} else {
4432	/ could not add class to fClasses /
4433	ret = kIOReturnNoMemory;
4434	}
4435	}
4436	IOLockUnlock(fLock);
4437	cls->release();
4438	return ret;
4439	}
4440
4441	IOReturn
4442	IOUserServer::registerClass(OSClassDescription * desc, uint32_t size, OSSharedPtr<OSUserMetaClass>& pCls)
4443	{
4444	OSUserMetaClass* pClsRaw = NULL;
4445	IOReturn result = registerClass(desc, size, pCls: &pClsRaw);
4446	if (result == kIOReturnSuccess) {
4447	pCls.reset(p: pClsRaw, OSRetain);
4448	}
4449	return result;
4450	}
4451
4452	IOReturn
4453	IOUserServer::setRootQueue(IODispatchQueue * queue)
4454	{
4455	assert(!fRootQueue);
4456	if (fRootQueue) {
4457	return kIOReturnStillOpen;
4458	}
4459	queue->retain();
4460	fRootQueue = queue;
4461
4462	return kIOReturnSuccess;
4463	}
4464
4465
4466	IOReturn
4467	IOUserServer::externalMethod(uint32_t selector, IOExternalMethodArgumentsOpaque * args)
4468	{
4469	static const IOExternalMethodDispatch2022 dispatchArray[] = {
4470	[kIOUserServerMethodRegisterClass] = {
4471	.function = &IOUserServer::externalMethodRegisterClass,
4472	.checkScalarInputCount = `0`,
4473	.checkStructureInputSize = kIOUCVariableStructureSize,
4474	.checkScalarOutputCount = `2`,
4475	.checkStructureOutputSize = `0`,
4476	.allowAsync = false,
4477	.checkEntitlement = NULL,
4478	},
4479	[kIOUserServerMethodStart] = {
4480	.function = &IOUserServer::externalMethodStart,
4481	.checkScalarInputCount = `1`,
4482	.checkStructureInputSize = `0`,
4483	.checkScalarOutputCount = `1`,
4484	.checkStructureOutputSize = `0`,
4485	.allowAsync = false,
4486	.checkEntitlement = NULL,
4487	},
4488	};
4489
4490	return dispatchExternalMethod(selector, arguments: args, dispatchArray, dispatchArrayCount: sizeof(dispatchArray) / sizeof(dispatchArray[`0`]), target: this, NULL);
4491	}
4492
4493	IOReturn
4494	IOUserServer::externalMethodRegisterClass(OSObject * target, void * reference, IOExternalMethodArguments * args)
4495	{
4496	IOReturn ret = kIOReturnBadArgument;
4497	mach_port_name_t portname;
4498
4499	IOUserServer * me = (typeof(me))target;
4500
4501	OSUserMetaClass * cls;
4502	if (!args->structureInputSize) {
4503	return kIOReturnBadArgument;
4504	}
4505
4506	ret = me->registerClass(desc: (OSClassDescription *) args->structureInput, size: args->structureInputSize, pCls: &cls);
4507	if (kIOReturnSuccess == ret) {
4508	portname = iokit_make_send_right(task: me->fOwningTask, obj: cls, type: IKOT_UEXT_OBJECT);
4509	assert(portname);
4510	args->scalarOutput[`0`] = portname;
4511	args->scalarOutput[`1`] = kOSObjectRPCRemote;
4512	}
4513
4514	return ret;
4515	}
4516
4517	IOReturn
4518	IOUserServer::externalMethodStart(OSObject * target, void * reference, IOExternalMethodArguments * args)
4519	{
4520	mach_port_name_t portname;
4521
4522	IOUserServer * me = (typeof(me))target;
4523
4524	if (!(kIODKDisableCheckInTokenVerification & gIODKDebug)) {
4525	mach_port_name_t checkInPortName = ((typeof(checkInPortName))args->scalarInput[`0`]);
4526	OSObject * obj = iokit_lookup_object_with_port_name(name: checkInPortName, type: IKOT_IOKIT_IDENT, task: me->fOwningTask);
4527	IOUserServerCheckInToken * retrievedToken = OSDynamicCast(IOUserServerCheckInToken, obj);
4528	if (retrievedToken != NULL) {
4529	me->setCheckInToken(retrievedToken);
4530	} else {
4531	OSSafeReleaseNULL(obj);
4532	return kIOReturnBadArgument;
4533	}
4534	OSSafeReleaseNULL(obj);
4535	}
4536	portname = iokit_make_send_right(task: me->fOwningTask, obj: me, type: IKOT_UEXT_OBJECT);
4537	assert(portname);
4538	args->scalarOutput[`0`] = portname;
4539	return kIOReturnSuccess;
4540	}
4541	IOExternalTrap *
4542	IOUserServer::getTargetAndTrapForIndex( IOService **targetP, UInt32 index )
4543	{
4544	static const OSBoundedArray<IOExternalTrap, `1`> trapTemplate = {.data_: {
4545	{ NULL, .func: (IOTrap) & IOUserServer::waitInterruptTrap},
4546	}};
4547	if (index >= trapTemplate.size()) {
4548	return NULL;
4549	}
4550	targetP = this*;
4551	return (IOExternalTrap *)&trapTemplate [index];
4552	}
4553
4554	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
4555
4556	IOReturn
4557	IOUserServer::serviceAttach(IOService * service, IOService * provider)
4558	{
4559	IOReturn ret;
4560	OSObjectUserVars * vars;
4561	OSObject * prop;
4562	OSString * str;
4563	OSSymbol const* bundleID;
4564	char execPath[`1024`];
4565
4566	vars = IOMallocType(OSObjectUserVars);
4567	service->reserved->uvars = vars;
4568
4569	vars->userServer = this;
4570	vars->userServer->retain();
4571	vars->uvarsLock = IOLockAlloc();
4572	IOLockLock(fLock);
4573	if (-`1U` == fServices->getNextIndexOfObject(anObject: service, index: `0`)) {
4574	fServices->setObject(service);
4575
4576	// Add to IOAssociatedServices
4577	OSObject * serviceArrayObj = copyProperty(aKey: gIOAssociatedServicesKey);
4578	OSArray * serviceArray = OSDynamicCast(OSArray, serviceArrayObj);
4579	if (!serviceArray) {
4580	serviceArray = OSArray::withCapacity(capacity: `0`);
4581	} else {
4582	serviceArray = OSDynamicCast(OSArray, serviceArray->copyCollection());
4583	assert(serviceArray != NULL);
4584	}
4585
4586	OSNumber * registryEntryNumber = OSNumber::withNumber(value: service->getRegistryEntryID(), numberOfBits: `64`);
4587	serviceArray->setObject(registryEntryNumber);
4588	setProperty(aKey: gIOAssociatedServicesKey, anObject: serviceArray);
4589	OSSafeReleaseNULL(registryEntryNumber);
4590	OSSafeReleaseNULL(serviceArray);
4591	OSSafeReleaseNULL(serviceArrayObj);
4592
4593	// populate kIOUserClassesKey
4594
4595	OSUserMetaClass * userMeta;
4596	OSArray * classesArray;
4597	const OSString * str2;
4598
4599	classesArray = OSArray::withCapacity(capacity: `4`);
4600	prop = service->copyProperty(aKey: gIOUserClassKey);
4601	str2 = OSDynamicCast(OSString, prop);
4602	userMeta = (typeof(userMeta))service->reserved->uvars->userServer->fClasses->getObject(aKey: str2);
4603	while (str2 && userMeta) {
4604	classesArray->setObject(str2);
4605	userMeta = userMeta->superMeta;
4606	if (userMeta) {
4607	str2 = userMeta->name;
4608	}
4609	}
4610	service->setProperty(aKey: gIOUserClassesKey, anObject: classesArray);
4611	OSSafeReleaseNULL(classesArray);
4612	OSSafeReleaseNULL(prop);
4613	}
4614	IOLockUnlock(fLock);
4615
4616	prop = service->copyProperty(aKey: gIOUserClassKey);
4617	str = OSDynamicCast(OSString, prop);
4618	if (str) {
4619	service->setName(name: str);
4620	}
4621	OSSafeReleaseNULL(prop);
4622
4623	prop = service->copyProperty(aKey: gIOModuleIdentifierKey);
4624	bundleID = OSDynamicCast(OSSymbol, prop);
4625	if (bundleID) {
4626	execPath[`0`] = `0`;
4627	bool ok = OSKext::copyUserExecutablePath(bundleID, pathResult: execPath, pathSize: sizeof(execPath));
4628	if (ok) {
4629	ret = LoadModule(path: execPath);
4630	if (kIODKLogSetup & gIODKDebug) {
4631	DKLOG("%s::LoadModule 0x%x %s\n", getName(), ret, execPath);
4632	}
4633	}
4634	}
4635	OSSafeReleaseNULL(prop);
4636
4637	ret = kIOReturnSuccess;
4638
4639	return ret;
4640	}
4641
4642	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
4643
4644	IOReturn
4645	IOUserServer::serviceNewUserClient(IOService * service, task_t owningTask, void * securityID,
4646	uint32_t type, OSDictionary * properties, IOUserClient ** handler)
4647	{
4648	IOReturn ret;
4649	IOUserClient * uc;
4650	IOUserUserClient * userUC;
4651	OSDictionary * entitlements;
4652	OSObject * prop;
4653	OSObject * bundleID;
4654	bool ok = false;
4655
4656	entitlements = IOUserClient::copyClientEntitlements(task: owningTask);
4657	if (!entitlements) {
4658	entitlements = OSDictionary::withCapacity(capacity: `8`);
4659	}
4660	if (entitlements) {
4661	if (kIOReturnSuccess == clientHasPrivilege(securityToken: (void *) owningTask, kIOClientPrivilegeAdministrator)) {
4662	entitlements->setObject(kIODriverKitUserClientEntitlementAdministratorKey, anObject: kOSBooleanTrue);
4663	}
4664	OSString * creatorName = IOCopyLogNameForPID(pid: proc_selfpid());
4665	if (creatorName) {
4666	entitlements->setObject(kIOUserClientCreatorKey, anObject: creatorName);
4667	OSSafeReleaseNULL(creatorName);
4668	}
4669	}
4670
4671	*handler = NULL;
4672	ret = service->_NewUserClient(type, entitlements, userClient: &uc);
4673	if (kIOReturnSuccess != ret) {
4674	OSSafeReleaseNULL(entitlements);
4675	return ret;
4676	}
4677	userUC = OSDynamicCast(IOUserUserClient, uc);
4678	if (!userUC) {
4679	uc->terminate(options: kIOServiceTerminateNeedWillTerminate);
4680	uc->setTerminateDefer(provider: service, defer: false);
4681	OSSafeReleaseNULL(uc);
4682	OSSafeReleaseNULL(entitlements);
4683	return kIOReturnUnsupported;
4684	}
4685	userUC->setTask(owningTask);
4686
4687	if (!(kIODKDisableEntitlementChecking & gIODKDebug)) {
4688	do {
4689	bool checkiOS3pEntitlements;
4690
4691	// check if client has com.apple.private.driverkit.driver-access and the required entitlements match the driver's entitlements
4692	if (entitlements && (prop = entitlements->getObject(aKey: gIODriverKitRequiredEntitlementsKey))) {
4693	prop->retain();
4694	ok = checkEntitlements(entitlements: fEntitlements, prop, NULL, NULL);
4695	if (ok) {
4696	break;
4697	} else {
4698	DKLOG(DKS ":UC failed required entitlement check\n", DKN(userUC));
4699	}
4700	}
4701
4702	#if XNU_TARGET_OS_IOS
4703	checkiOS3pEntitlements = !fPlatformDriver;
4704	if (checkiOS3pEntitlements && fTeamIdentifier == NULL) {
4705	DKLOG("warning: " DKS " does not have a team identifier\n", DKN(this));
4706	}
4707	#else
4708	checkiOS3pEntitlements = false;
4709	#endif
4710	if (checkiOS3pEntitlements) {
4711	// App must have com.apple.developer.driverkit.communicates-with-drivers
4712	ok = entitlements && entitlements->getObject(aKey: gIODriverKitUserClientEntitlementCommunicatesWithDriversKey) == kOSBooleanTrue;
4713	if (ok) {
4714	// check team ID
4715	const char * clientTeamID = csproc_get_teamid(current_proc());
4716	bool sameTeam = fTeamIdentifier != NULL && clientTeamID != NULL && strncmp(s1: fTeamIdentifier->getCStringNoCopy(), s2: clientTeamID, CS_MAX_TEAMID_LEN) == `0`;
4717
4718	if (sameTeam) {
4719	ok = true;
4720	} else {
4721	// different team IDs, dext must have com.apple.developer.driverkit.allow-third-party-userclients
4722	ok = fEntitlements && fEntitlements->getObject(aKey: gIODriverKitUserClientEntitlementAllowThirdPartyUserClientsKey) == kOSBooleanTrue;
4723	}
4724	if (!ok) {
4725	DKLOG(DKS ":UC failed team ID check. client team=%s, driver team=%s\n", DKN(userUC), clientTeamID ? clientTeamID : "(null)", fTeamIdentifier ? fTeamIdentifier->getCStringNoCopy() : "(null)");
4726	}
4727	} else {
4728	DKLOG(DKS ":UC entitlement check failed, app does not have %s entitlement\n", DKN(userUC), gIODriverKitUserClientEntitlementCommunicatesWithDriversKey->getCStringNoCopy());
4729	}
4730
4731	// When checking iOS 3rd party entitlements, do not fall through to other entitlement checks
4732	break;
4733	}
4734
4735	// first party dexts and third party macOS dexts
4736
4737	// check if driver has com.apple.developer.driverkit.allow-any-userclient-access
4738	if (fEntitlements && fEntitlements->getObject(aKey: gIODriverKitUserClientEntitlementAllowAnyKey)) {
4739	ok = true;
4740	break;
4741	}
4742
4743	// check if client has com.apple.developer.driverkit.userclient-access and its value matches the bundle ID of the service
4744	bundleID = service->copyProperty(aKey: gIOModuleIdentifierKey);
4745	ok = (entitlements
4746	&& bundleID
4747	&& (prop = entitlements->getObject(aKey: gIODriverKitUserClientEntitlementsKey)));
4748	if (ok) {
4749	bool found __block = false;
4750	ok = prop->iterateObjects(block: ^bool (OSObject * object) {
4751	found = object->isEqualTo(anObject: bundleID);
4752	return found;
4753	});
4754	ok = found;
4755	} else {
4756	OSString * bundleIDStr = OSDynamicCast(OSString, bundleID);
4757	DKLOG(DKS ":UC failed userclient-access check, needed bundle ID %s\n", DKN(userUC), bundleIDStr ? bundleIDStr->getCStringNoCopy() : "(null)");
4758	}
4759	OSSafeReleaseNULL(bundleID);
4760	} while (false);
4761
4762	if (ok) {
4763	prop = userUC->copyProperty(aKey: gIOServiceDEXTEntitlementsKey);
4764	ok = checkEntitlements(entitlements, prop, NULL, NULL);
4765	}
4766
4767	if (!ok) {
4768	DKLOG(DKS ":UC entitlements check failed\n", DKN(userUC));
4769	uc->terminate(options: kIOServiceTerminateNeedWillTerminate);
4770	uc->setTerminateDefer(provider: service, defer: false);
4771	OSSafeReleaseNULL(uc);
4772	OSSafeReleaseNULL(entitlements);
4773	return kIOReturnNotPermitted;
4774	}
4775	}
4776
4777	OSSafeReleaseNULL(entitlements);
4778	*handler = userUC;
4779
4780	return ret;
4781	}
4782
4783	IOReturn
4784	IOUserServer::serviceNewUserClient(IOService * service, task_t owningTask, void * securityID,
4785	uint32_t type, OSDictionary * properties, OSSharedPtr<IOUserClient>& handler)
4786	{
4787	IOUserClient* handlerRaw = NULL;
4788	IOReturn result = serviceNewUserClient(service, owningTask, securityID, type, properties, handler: &handlerRaw);
4789	handler.reset(p: handlerRaw, OSNoRetain);
4790	return result;
4791	}
4792
4793	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
4794
4795	static IOPMPowerState
4796	sPowerStates[] = {
4797	{ .version = kIOPMPowerStateVersion1,
4798	.capabilityFlags = `0`,
4799	.outputPowerCharacter = `0`,
4800	.inputPowerRequirement = `0`},
4801	{ .version = kIOPMPowerStateVersion1,
4802	.capabilityFlags = kIOPMLowPower,
4803	.outputPowerCharacter = kIOPMLowPower,
4804	.inputPowerRequirement = kIOPMLowPower},
4805	{ .version = kIOPMPowerStateVersion1,
4806	.capabilityFlags = kIOPMPowerOn,
4807	.outputPowerCharacter = kIOPMPowerOn,
4808	.inputPowerRequirement = kIOPMPowerOn},
4809	};
4810
4811	enum {
4812	kUserServerMaxPowerState = `2`
4813	};
4814
4815	IOReturn
4816	IOUserServer::serviceJoinPMTree(IOService * service)
4817	{
4818	IOReturn ret;
4819	IOService * pmProvider;
4820	bool joinTree;
4821
4822	if (service->reserved->uvars->userServerPM) {
4823	return kIOReturnSuccess;
4824	}
4825
4826	if (!fRootNotifier) {
4827	ret = registerPowerDriver(controllingDriver: this, powerStates: sPowerStates, numberOfStates: sizeof(sPowerStates) / sizeof(sPowerStates[`0`]));
4828	assert(kIOReturnSuccess == ret);
4829	IOServicePH::serverAdd(server: this);
4830	fRootNotifier = true;
4831	}
4832
4833	joinTree = false;
4834	if (!(kIODKDisablePM & gIODKDebug) && !service->pm_vars) {
4835	kern_return_t kr;
4836	OSDictionary * props;
4837	kr = service->CopyProperties_Local(properties: &props);
4838	if (kIOReturnSuccess == kr) {
4839	if (props->getObject(kIOPMResetPowerStateOnWakeKey) == kOSBooleanTrue) {
4840	service->setProperty(kIOPMResetPowerStateOnWakeKey, anObject: kOSBooleanTrue);
4841	}
4842	OSSafeReleaseNULL(props);
4843	}
4844	service->PMinit();
4845	ret = service->registerPowerDriver(controllingDriver: this, powerStates: sPowerStates, numberOfStates: sizeof(sPowerStates) / sizeof(sPowerStates[`0`]));
4846	assert(kIOReturnSuccess == ret);
4847	joinTree = true;
4848	}
4849
4850	pmProvider = service;
4851	while (pmProvider && !pmProvider->inPlane(plane: gIOPowerPlane)) {
4852	pmProvider = pmProvider->getProvider();
4853	}
4854	if (!pmProvider) {
4855	pmProvider = getPMRootDomain();
4856	}
4857	if (pmProvider) {
4858	IOService * entry;
4859	OSObject * prop;
4860	OSObject * nextProp;
4861	OSString * str;
4862
4863	entry = pmProvider;
4864	prop = NULL;
4865	do {
4866	nextProp = entry->copyProperty(aKey: "non-removable");
4867	if (nextProp) {
4868	OSSafeReleaseNULL(prop);
4869	prop = nextProp;
4870	}
4871	entry = entry->getProvider();
4872	} while (entry);
4873	if (prop) {
4874	str = OSDynamicCast(OSString, prop);
4875	if (str && str->isEqualTo(cString: "yes")) {
4876	pmProvider = NULL;
4877	}
4878	prop->release();
4879	}
4880	}
4881
4882	if (!(kIODKDisablePM & gIODKDebug) && pmProvider) {
4883	IOLockLock(fLock);
4884	service->reserved->uvars->powerState = true;
4885	IOLockUnlock(fLock);
4886
4887	if (joinTree) {
4888	pmProvider->joinPMtree(driver: service);
4889	service->reserved->uvars->userServerPM = true;
4890	service->reserved->uvars->resetPowerOnWake = service->propertyExists(kIOPMResetPowerStateOnWakeKey);
4891	}
4892	}
4893
4894	service->registerInterestedDriver(theDriver: this);
4895	return kIOReturnSuccess;
4896	}
4897
4898	IOReturn
4899	IOUserServer::setPowerState(unsigned long state, IOService * service)
4900	{
4901	if (kIODKLogPM & gIODKDebug) {
4902	DKLOG(DKS "::setPowerState(%ld) %d\n", DKN(service), state, fSystemPowerAck);
4903	}
4904	return kIOPMAckImplied;
4905	}
4906
4907
4908	IOReturn
4909	IOUserServer::serviceSetPowerState(IOService * controllingDriver, IOService * service, IOPMPowerFlags flags, unsigned long state)
4910	{
4911	IOReturn ret;
4912	bool sendIt = false;
4913
4914	IOLockLock(fLock);
4915	if (service->reserved->uvars) {
4916	if (!fSystemOff && !(kIODKDisablePM & gIODKDebug)) {
4917	OSDictionary * wakeDescription;
4918	OSObject * prop;
4919	char wakeReasonString[`128`];
4920
4921	wakeDescription = OSDictionary::withCapacity(capacity: `4`);
4922	if (wakeDescription) {
4923	wakeReasonString[`0`] = `0`;
4924	getPMRootDomain()->copyWakeReasonString(outBuf: wakeReasonString, bufSize: sizeof(wakeReasonString));
4925
4926	if (wakeReasonString[`0`]) {
4927	prop = OSString::withCString(cString: &wakeReasonString[`0`]);
4928	wakeDescription->setObject(aKey: gIOSystemStateWakeDescriptionWakeReasonKey, anObject: prop);
4929	OSSafeReleaseNULL(prop);
4930	}
4931	getSystemStateNotificationService()->StateNotificationItemSet(itemName: gIOSystemStateWakeDescriptionKey, value: wakeDescription);
4932	OSSafeReleaseNULL(wakeDescription);
4933	}
4934
4935	service->reserved->uvars->willPower = true;
4936	service->reserved->uvars->willPowerState = state;
4937	service->reserved->uvars->controllingDriver = controllingDriver;
4938	sendIt = true;
4939	} else {
4940	service->reserved->uvars->willPower = false;
4941	}
4942	}
4943	IOLockUnlock(fLock);
4944
4945	if (sendIt) {
4946	if (kIODKLogPM & gIODKDebug) {
4947	DKLOG(DKS "::serviceSetPowerState(%ld) %d\n", DKN(service), state, fSystemPowerAck);
4948	}
4949	ret = service->SetPowerState(powerFlags: (uint32_t) flags);
4950	if (kIOReturnSuccess == ret) {
4951	return `20` * `1000` * `1000`;
4952	} else {
4953	IOLockLock(fLock);
4954	service->reserved->uvars->willPower = false;
4955	IOLockUnlock(fLock);
4956	}
4957	}
4958
4959	return kIOPMAckImplied;
4960	}
4961
4962	IOReturn
4963	IOUserServer::powerStateWillChangeTo(IOPMPowerFlags flags, unsigned long state, IOService * service)
4964	{
4965	return kIOPMAckImplied;
4966	}
4967
4968	IOReturn
4969	IOUserServer::powerStateDidChangeTo(IOPMPowerFlags flags, unsigned long state, IOService * service)
4970	{
4971	unsigned int idx;
4972	bool pmAck;
4973
4974	pmAck = false;
4975	IOLockLock(fLock);
4976	idx = fServices->getNextIndexOfObject(anObject: service, index: `0`);
4977	if (-`1U` == idx) {
4978	IOLockUnlock(fLock);
4979	return kIOPMAckImplied;
4980	}
4981
4982	service->reserved->uvars->powerState = (`0` != state);
4983	bool allPowerStates __block = service->reserved->uvars->powerState;
4984	if (!allPowerStates) {
4985	// any service on?
4986	fServices->iterateObjects(block: ^bool (OSObject * obj) {
4987	int service __unused; // hide outer defn
4988	IOService * nextService;
4989	nextService = (IOService *) obj;
4990	allPowerStates = nextService->reserved->uvars->powerState;
4991	// early terminate if true
4992	return allPowerStates;
4993	});
4994	}
4995	if (kIODKLogPM & gIODKDebug) {
4996	DKLOG(DKS "::powerStateDidChangeTo(%ld) %d, %d\n", DKN(service), state, allPowerStates, fSystemPowerAck);
4997	}
4998	if (!allPowerStates && (pmAck = fSystemPowerAck)) {
4999	fSystemPowerAck = false;
5000	fSystemOff = true;
5001	}
5002	IOLockUnlock(fLock);
5003
5004	if (pmAck) {
5005	IOServicePH::serverAck(server: this);
5006	}
5007
5008	return kIOPMAckImplied;
5009	}
5010
5011	bool
5012	IOUserServer::checkPMReady()
5013	{
5014	bool __block ready = true;
5015
5016	IOLockLock(fLock);
5017	// Check if any services have not completely joined the PM tree (i.e.
5018	// addPowerChild has not compeleted).
5019	fServices->iterateObjects(block: ^bool (OSObject * obj) {
5020	IOPowerConnection *conn;
5021	IOService service = (IOService ) obj;
5022	IORegistryEntry *parent = service->getParentEntry(plane: gIOPowerPlane);
5023	if ((conn = OSDynamicCast(IOPowerConnection, parent))) {
5024	if (!conn->getReadyFlag()) {
5025	ready = false;
5026	return true;
5027	}
5028	}
5029	return false;
5030	});
5031	IOLockUnlock(fLock);
5032
5033	return ready;
5034	}
5035
5036	kern_return_t
5037	IOService::JoinPMTree_Impl(void)
5038	{
5039	if (!reserved->uvars \|\| !reserved->uvars->userServer) {
5040	return kIOReturnNotReady;
5041	}
5042	return reserved->uvars->userServer->serviceJoinPMTree(service: this);
5043	}
5044
5045	kern_return_t
5046	IOService::SetPowerState_Impl(
5047	uint32_t powerFlags)
5048	{
5049	if (kIODKLogPM & gIODKDebug) {
5050	DKLOG(DKS "::SetPowerState(%d), %d\n", DKN(this), powerFlags, reserved->uvars->willPower);
5051	}
5052	if (reserved->uvars
5053	&& reserved->uvars->userServer
5054	&& reserved->uvars->willPower) {
5055	IOReturn ret;
5056	reserved->uvars->willPower = false;
5057	ret = reserved->uvars->controllingDriver->setPowerState(powerStateOrdinal: reserved->uvars->willPowerState, whatDevice: this);
5058	if (kIOPMAckImplied == ret) {
5059	acknowledgeSetPowerState();
5060	}
5061	return kIOReturnSuccess;
5062	}
5063	return kIOReturnNotReady;
5064	}
5065
5066	kern_return_t
5067	IOService::ChangePowerState_Impl(
5068	uint32_t powerFlags)
5069	{
5070	switch (powerFlags) {
5071	case kIOServicePowerCapabilityOff:
5072	changePowerStateToPriv(ordinal: `0`);
5073	break;
5074	case kIOServicePowerCapabilityLow:
5075	changePowerStateToPriv(ordinal: `1`);
5076	break;
5077	case kIOServicePowerCapabilityOn:
5078	changePowerStateToPriv(ordinal: `2`);
5079	break;
5080	default:
5081	return kIOReturnBadArgument;
5082	}
5083
5084	return kIOReturnSuccess;
5085	}
5086
5087	kern_return_t
5088	IOService::_ClaimSystemWakeEvent_Impl(
5089	IOService * device,
5090	uint64_t flags,
5091	const char * reason,
5092	OSContainer * details)
5093	{
5094	IOPMrootDomain * rootDomain;
5095	IOOptionBits pmFlags;
5096
5097	rootDomain = getPMRootDomain();
5098	if (!rootDomain) {
5099	return kIOReturnNotReady;
5100	}
5101	if (os_convert_overflow(flags, &pmFlags)) {
5102	return kIOReturnBadArgument;
5103	}
5104	rootDomain->claimSystemWakeEvent(device, flags: pmFlags, reason, details);
5105
5106	return kIOReturnSuccess;
5107	}
5108
5109	kern_return_t
5110	IOService::Create_Impl(
5111	IOService * provider,
5112	const char * propertiesKey,
5113	IOService ** result)
5114	{
5115	OSObject * inst;
5116	IOService * service;
5117	OSString * str;
5118	const OSSymbol * sym;
5119	OSObject * prop = NULL;
5120	OSObject * moduleIdentifier = NULL;
5121	OSObject * userServerName = NULL;
5122	OSDictionary * properties = NULL;
5123	OSDictionary * copyProperties = NULL;
5124	kern_return_t ret;
5125
5126	if (provider != this) {
5127	return kIOReturnUnsupported;
5128	}
5129
5130	ret = kIOReturnUnsupported;
5131	inst = NULL;
5132	service = NULL;
5133
5134	prop = copyProperty(aKey: propertiesKey);
5135	properties = OSDynamicCast(OSDictionary, prop);
5136	if (!properties) {
5137	ret = kIOReturnBadArgument;
5138	goto finish;
5139	}
5140	copyProperties = OSDynamicCast(OSDictionary, properties->copyCollection());
5141	if (!copyProperties) {
5142	ret = kIOReturnNoMemory;
5143	goto finish;
5144	}
5145	moduleIdentifier = copyProperty(aKey: gIOModuleIdentifierKey);
5146	if (moduleIdentifier) {
5147	copyProperties->setObject(aKey: gIOModuleIdentifierKey, anObject: moduleIdentifier);
5148	}
5149	userServerName = reserved->uvars->userServer->copyProperty(aKey: gIOUserServerNameKey);
5150	if (userServerName) {
5151	copyProperties->setObject(aKey: gIOUserServerNameKey, anObject: userServerName);
5152	}
5153
5154	str = OSDynamicCast(OSString, copyProperties->getObject(gIOClassKey));
5155	if (!str) {
5156	ret = kIOReturnBadArgument;
5157	goto finish;
5158	}
5159	sym = OSSymbol::withString(aString: str);
5160	if (sym) {
5161	inst = OSMetaClass::allocClassWithName(name: sym);
5162	service = OSDynamicCast(IOService, inst);
5163	if (service && service->init(dictionary: copyProperties) && service->attach(provider: this)) {
5164	reserved->uvars->userServer->serviceAttach(service, provider: this);
5165	service->reserved->uvars->started = true;
5166	ret = kIOReturnSuccess;
5167	*result = service;
5168	}
5169	OSSafeReleaseNULL(sym);
5170	}
5171
5172	finish:
5173	OSSafeReleaseNULL(prop);
5174	OSSafeReleaseNULL(copyProperties);
5175	OSSafeReleaseNULL(moduleIdentifier);
5176	OSSafeReleaseNULL(userServerName);
5177	if (kIOReturnSuccess != ret) {
5178	OSSafeReleaseNULL(inst);
5179	}
5180
5181	return ret;
5182	}
5183
5184	kern_return_t
5185	IOService::Terminate_Impl(
5186	uint64_t options)
5187	{
5188	IOUserServer * us;
5189
5190	if (options) {
5191	return kIOReturnUnsupported;
5192	}
5193
5194	us = (typeof(us))thread_iokit_tls_get(index: `0`);
5195	if (us && (!reserved->uvars
5196	\|\| (reserved->uvars->userServer != us))) {
5197	return kIOReturnNotPermitted;
5198	}
5199	terminate(options: kIOServiceTerminateNeedWillTerminate);
5200
5201	return kIOReturnSuccess;
5202	}
5203
5204	kern_return_t
5205	IOService::NewUserClient_Impl(
5206	uint32_t type,
5207	IOUserClient ** userClient)
5208	{
5209	return kIOReturnError;
5210	}
5211
5212	kern_return_t
5213	IOService::_NewUserClient_Impl(
5214	uint32_t type,
5215	OSDictionary * entitlements,
5216	IOUserClient ** userClient)
5217	{
5218	return kIOReturnError;
5219	}
5220
5221	kern_return_t
5222	IOService::SearchProperty_Impl(
5223	const char * name,
5224	const char * plane,
5225	uint64_t options,
5226	OSContainer ** property)
5227	{
5228	OSObject * object __block;
5229	IOService * provider;
5230	IOOptionBits regOptions;
5231
5232	if (kIOServiceSearchPropertyParents & options) {
5233	regOptions = kIORegistryIterateParents \| kIORegistryIterateRecursively;
5234	} else {
5235	regOptions = `0`;
5236	}
5237
5238	object = copyProperty(aKey: name, plane: IORegistryEntry::getPlane(name: plane), options: regOptions);
5239
5240	if (NULL == object) {
5241	for (provider = this; provider; provider = provider->getProvider()) {
5242	provider->runPropertyActionBlock(block: ^IOReturn (void) {
5243	OSDictionary * userProps;
5244	object = provider->getProperty(aKey: name);
5245	if (!object
5246	&& (userProps = OSDynamicCast(OSDictionary, provider->getProperty(gIOUserServicePropertiesKey)))) {
5247	object = userProps->getObject(aKey: name);
5248	}
5249	if (object) {
5250	object->retain();
5251	}
5252	return kIOReturnSuccess;
5253	});
5254	if (object \|\| !(kIORegistryIterateParents & options)) {
5255	break;
5256	}
5257	}
5258	}
5259
5260	*property = object;
5261
5262	return object ? kIOReturnSuccess : kIOReturnNotFound;
5263	}
5264
5265	kern_return_t
5266	IOService::StringFromReturn_Impl(
5267	IOReturn retval,
5268	OSString ** str)
5269	{
5270	OSString *obj = OSString::withCString(cString: stringFromReturn(rtn: retval));
5271	*str = obj;
5272	return obj ? kIOReturnSuccess : kIOReturnError;
5273	}
5274
5275	#if PRIVATE_WIFI_ONLY
5276	const char *
5277	IOService::StringFromReturn(
5278	IOReturn retval)
5279	{
5280	return stringFromReturn(rtn: retval);
5281	}
5282	#endif /* PRIVATE_WIFI_ONLY */
5283
5284	kern_return_t
5285	IOService::CopyProviderProperties_Impl(
5286	OSArray * propertyKeys,
5287	OSArray ** properties)
5288	{
5289	IOReturn ret;
5290	OSArray * result;
5291	IOService * provider;
5292
5293	result = OSArray::withCapacity(capacity: `8`);
5294	if (!result) {
5295	return kIOReturnNoMemory;
5296	}
5297
5298	ret = kIOReturnSuccess;
5299	for (provider = this; provider; provider = provider->getProvider()) {
5300	OSObject * obj;
5301	OSDictionary * props;
5302
5303	obj = provider->copyProperty(aKey: gIOSupportedPropertiesKey);
5304	props = OSDynamicCast(OSDictionary, obj);
5305	if (!props) {
5306	OSSafeReleaseNULL(obj);
5307	props = provider->dictionaryWithProperties();
5308	}
5309	if (!props) {
5310	ret = kIOReturnNoMemory;
5311	break;
5312	}
5313
5314	bool __block addClass = true;
5315	if (propertyKeys) {
5316	OSDictionary * retProps;
5317	retProps = OSDictionary::withCapacity(capacity: `4`);
5318	addClass = false;
5319	if (!retProps) {
5320	ret = kIOReturnNoMemory;
5321	OSSafeReleaseNULL(props);
5322	break;
5323	}
5324	propertyKeys->iterateObjects(block: ^bool (OSObject * _key) {
5325	OSString * key = OSDynamicCast(OSString, _key);
5326	if (gIOClassKey->isEqualTo(anObject: key)) {
5327	addClass = true;
5328	return false;
5329	}
5330	retProps->setObject(aKey: key, anObject: props->getObject(aKey: key));
5331	return false;
5332	});
5333	OSSafeReleaseNULL(props);
5334	props = retProps;
5335	}
5336	if (addClass) {
5337	OSArray * classes = OSArray::withCapacity(capacity: `8`);
5338	if (!classes) {
5339	OSSafeReleaseNULL(props);
5340	ret = kIOReturnNoMemory;
5341	break;
5342	}
5343	for (const OSMetaClass * meta = provider->getMetaClass(); meta; meta = meta->getSuperClass()) {
5344	classes->setObject(meta->getClassNameSymbol());
5345	}
5346	props->setObject(aKey: gIOClassKey, anObject: classes);
5347	OSSafeReleaseNULL(classes);
5348	}
5349	bool ok = result->setObject(props);
5350	props->release();
5351	if (!ok) {
5352	ret = kIOReturnNoMemory;
5353	break;
5354	}
5355	}
5356	if (kIOReturnSuccess != ret) {
5357	OSSafeReleaseNULL(result);
5358	}
5359	*properties = result;
5360	return ret;
5361	}
5362
5363	IOReturn
5364	IOService::AdjustBusy_Impl(int32_t delta)
5365	{
5366	adjustBusy(delta);
5367	return kIOReturnSuccess;
5368	}
5369
5370	IOReturn
5371	IOService::GetBusyState_Impl(uint32_t *busyState)
5372	{
5373	*busyState = getBusyState();
5374	return kIOReturnSuccess;
5375	}
5376
5377	void
5378	IOUserServer::systemPower(bool powerOff)
5379	{
5380	OSArray * services;
5381	{
5382	OSDictionary * sleepDescription;
5383	OSObject * prop;
5384
5385	sleepDescription = OSDictionary::withCapacity(capacity: `4`);
5386	if (sleepDescription) {
5387	prop = getPMRootDomain()->copyProperty(kRootDomainSleepReasonKey);
5388	if (prop) {
5389	sleepDescription->setObject(aKey: gIOSystemStateSleepDescriptionReasonKey, anObject: prop);
5390	OSSafeReleaseNULL(prop);
5391	}
5392	prop = getPMRootDomain()->copyProperty(kIOHibernateStateKey);
5393	if (prop) {
5394	sleepDescription->setObject(aKey: gIOSystemStateSleepDescriptionHibernateStateKey, anObject: prop);
5395	OSSafeReleaseNULL(prop);
5396	}
5397	getSystemStateNotificationService()->StateNotificationItemSet(itemName: gIOSystemStateSleepDescriptionKey, value: sleepDescription);
5398	OSSafeReleaseNULL(sleepDescription);
5399	}
5400	}
5401
5402	IOLockLock(fLock);
5403
5404	services = OSArray::withArray(array: fServices);
5405
5406	bool allPowerStates __block = `0`;
5407	// any service on?
5408	fServices->iterateObjects(block: ^bool (OSObject * obj) {
5409	int service __unused; // hide outer defn
5410	IOService * nextService;
5411	nextService = (IOService *) obj;
5412	allPowerStates = nextService->reserved->uvars->powerState;
5413	// early terminate if true
5414	return allPowerStates;
5415	});
5416
5417	if (kIODKLogPM & gIODKDebug) {
5418	DKLOG("%s::powerOff(%d) %d\n", getName(), powerOff, allPowerStates);
5419	}
5420
5421	if (powerOff) {
5422	fSystemPowerAck = allPowerStates;
5423	if (!fSystemPowerAck) {
5424	fSystemOff = true;
5425	}
5426	IOLockUnlock(fLock);
5427
5428	if (!fSystemPowerAck) {
5429	IOServicePH::serverAck(server: this);
5430	} else {
5431	if (services) {
5432	services->iterateObjects(block: ^bool (OSObject * obj) {
5433	int service __unused; // hide outer defn
5434	IOService * nextService;
5435	nextService = (IOService *) obj;
5436	if (kIODKLogPM & gIODKDebug) {
5437	DKLOG("changePowerStateWithOverrideTo(" DKS ", %d)\n", DKN(nextService), `0`);
5438	}
5439	nextService->reserved->uvars->powerOverride = nextService->reserved->uvars->userServerPM ? kUserServerMaxPowerState : nextService->getPowerState();
5440	nextService->changePowerStateWithOverrideTo(ordinal: `0`, tag: `0`);
5441	return false;
5442	});
5443	}
5444	}
5445	} else {
5446	fSystemOff = false;
5447	IOLockUnlock(fLock);
5448	if (services) {
5449	services->iterateObjects(block: ^bool (OSObject * obj) {
5450	int service __unused; // hide outer defn
5451	IOService * nextService;
5452	nextService = (IOService *) obj;
5453	if (-`1U` != nextService->reserved->uvars->powerOverride) {
5454	if (kIODKLogPM & gIODKDebug) {
5455	DKLOG("%schangePowerStateWithOverrideTo(" DKS ", %d)\n", nextService->reserved->uvars->resetPowerOnWake ? "!" : "", DKN(nextService), nextService->reserved->uvars->powerOverride);
5456	}
5457	if (!nextService->reserved->uvars->resetPowerOnWake) {
5458	nextService->changePowerStateWithOverrideTo(ordinal: nextService->reserved->uvars->powerOverride, tag: `0`);
5459	}
5460	nextService->reserved->uvars->powerOverride = -`1U`;
5461	}
5462	return false;
5463	});
5464	}
5465	}
5466	OSSafeReleaseNULL(services);
5467	}
5468
5469
5470	void
5471	IOUserServer::systemHalt(int howto)
5472	{
5473	OSArray * services;
5474
5475	if (true \|\| (kIODKLogPM & gIODKDebug)) {
5476	DKLOG("%s::systemHalt()\n", getName());
5477	}
5478
5479	{
5480	OSDictionary * haltDescription;
5481	OSNumber * state;
5482	uint64_t haltStateFlags;
5483
5484	haltDescription = OSDictionary::withCapacity(capacity: `4`);
5485	if (haltDescription) {
5486	haltStateFlags = `0`;
5487	if (RB_HALT & howto) {
5488	haltStateFlags \|= kIOServiceHaltStatePowerOff;
5489	} else {
5490	haltStateFlags \|= kIOServiceHaltStateRestart;
5491	}
5492	state = OSNumber::withNumber(value: haltStateFlags, numberOfBits: `64`);
5493	haltDescription->setObject(aKey: gIOSystemStateHaltDescriptionHaltStateKey, anObject: state);
5494	getSystemStateNotificationService()->StateNotificationItemSet(itemName: gIOSystemStateHaltDescriptionKey, value: haltDescription);
5495
5496	OSSafeReleaseNULL(state);
5497	OSSafeReleaseNULL(haltDescription);
5498	}
5499	}
5500
5501	IOLockLock(fLock);
5502	services = OSArray::withArray(array: fServices);
5503	IOLockUnlock(fLock);
5504
5505	if (services) {
5506	services->iterateObjects(block: ^bool (OSObject * obj) {
5507	int service __unused; // hide outer defn
5508	IOService * nextService;
5509	IOService * provider;
5510	IOOptionBits terminateOptions;
5511	bool root;
5512
5513	nextService = (IOService *) obj;
5514	provider = nextService->getProvider();
5515	if (!provider) {
5516	DKLOG("stale service " DKS " found, skipping termination\n", DKN(nextService));
5517	return false;
5518	}
5519	root = (NULL == provider->getProperty(aKey: gIOUserServerNameKey, plane: gIOServicePlane));
5520	if (true \|\| (kIODKLogPM & gIODKDebug)) {
5521	DKLOG("%d: terminate(" DKS ")\n", root, DKN(nextService));
5522	}
5523	if (!root) {
5524	return false;
5525	}
5526	terminateOptions = kIOServiceRequired \| kIOServiceTerminateNeedWillTerminate;
5527	if (!nextService->terminate(options: terminateOptions)) {
5528	IOLog(format: "failed to terminate service %s-0x%llx\n", nextService->getName(), nextService->getRegistryEntryID());
5529	}
5530	return false;
5531	});
5532	}
5533	OSSafeReleaseNULL(services);
5534	}
5535
5536	void
5537	IOUserServer::powerSourceChanged(bool acAttached)
5538	{
5539	OSDictionary * powerSourceDescription;
5540
5541	powerSourceDescription = OSDictionary::withCapacity(capacity: `4`);
5542	if (!powerSourceDescription) {
5543	return;
5544	}
5545	powerSourceDescription->setObject(aKey: gIOSystemStatePowerSourceDescriptionACAttachedKey, anObject: acAttached ? kOSBooleanTrue : kOSBooleanFalse);
5546	getSystemStateNotificationService()->StateNotificationItemSet(itemName: gIOSystemStatePowerSourceDescriptionKey, value: powerSourceDescription);
5547
5548	OSSafeReleaseNULL(powerSourceDescription);
5549	}
5550
5551	IOReturn
5552	IOUserServer::serviceStarted(IOService * service, IOService * provider, bool result)
5553	{
5554	IOReturn ret;
5555
5556	DKLOG(DKS "::start(" DKS ") %s\n", DKN(service), DKN(provider), result ? "ok" : "fail");
5557
5558	if (!result) {
5559	ret = kIOReturnSuccess;
5560	return ret;
5561	}
5562
5563	ret = serviceJoinPMTree(service);
5564
5565	service->reserved->uvars->started = true;
5566
5567	if (service->reserved->uvars->deferredRegisterService) {
5568	service->registerService(options: kIOServiceAsynchronous \| kIOServiceDextRequirePowerForMatching);
5569	service->reserved->uvars->deferredRegisterService = false;
5570	}
5571
5572	return kIOReturnSuccess;
5573	}
5574
5575
5576	IOReturn
5577	IOUserServer::serviceOpen(IOService * provider, IOService * client)
5578	{
5579	OSObjectUserVars * uvars;
5580	IOReturn ret;
5581
5582	IOLockLock(client->reserved->uvars->uvarsLock);
5583	uvars = client->reserved->uvars;
5584	if (uvars->willTerminate \|\| uvars->stopped) {
5585	DKLOG(DKS "- " DKS " blocked attempt to open " DKS "\n", DKN(this), DKN(client), DKN(provider));
5586	ret = kIOReturnBadArgument;
5587	} else {
5588	if (!uvars->openProviders) {
5589	uvars->openProviders = OSArray::withObjects(objects: (const OSObject **) &provider, count: `1`);
5590	} else if (-`1U` == uvars->openProviders->getNextIndexOfObject(anObject: provider, index: `0`)) {
5591	uvars->openProviders->setObject(provider);
5592	}
5593	ret = kIOReturnSuccess;
5594	}
5595
5596	IOLockUnlock(client->reserved->uvars->uvarsLock);
5597
5598	return ret;
5599	}
5600
5601	IOReturn
5602	IOUserServer::serviceClose(IOService * provider, IOService * client)
5603	{
5604	OSObjectUserVars * uvars;
5605	unsigned int idx;
5606	IOReturn ret;
5607
5608	IOLockLock(client->reserved->uvars->uvarsLock);
5609	uvars = client->reserved->uvars;
5610	if (!uvars->openProviders) {
5611	ret = kIOReturnNotOpen;
5612	goto finish;
5613	}
5614	idx = uvars->openProviders->getNextIndexOfObject(anObject: provider, index: `0`);
5615	if (-`1U` == idx) {
5616	ret = kIOReturnNotOpen;
5617	goto finish;
5618	}
5619	uvars->openProviders->removeObject(index: idx);
5620	if (!uvars->openProviders->getCount()) {
5621	OSSafeReleaseNULL(uvars->openProviders);
5622	}
5623
5624	ret = kIOReturnSuccess;
5625
5626	finish:
5627	IOLockUnlock(client->reserved->uvars->uvarsLock);
5628
5629	return ret;
5630	}
5631
5632
5633	IOReturn
5634	IOUserServer::serviceStop(IOService * service, IOService *)
5635	{
5636	IOReturn ret;
5637	uint32_t idx, queueAlloc;
5638	bool pmAck;
5639	OSObjectUserVars * uvars;
5640	IODispatchQueue ** unboundedQueueArray = NULL;
5641	pmAck = false;
5642	IOLockLock(fLock);
5643	idx = fServices->getNextIndexOfObject(anObject: service, index: `0`);
5644	if (-`1U` != idx) {
5645	fServices->removeObject(index: idx);
5646
5647	// Remove the service from IOAssociatedServices
5648	OSObject * serviceArrayObj = copyProperty(aKey: gIOAssociatedServicesKey);
5649	OSArray * serviceArray = OSDynamicCast(OSArray, serviceArrayObj);
5650	assert(serviceArray != NULL);
5651
5652	serviceArray = OSDynamicCast(OSArray, serviceArray->copyCollection());
5653	assert(serviceArray != NULL);
5654
5655	// Index should be the same as it was in fServices
5656	OSNumber * __assert_only registryEntryID = OSDynamicCast(OSNumber, serviceArray->getObject(idx));
5657	assert(registryEntryID);
5658
5659	// ensure it is the right service
5660	assert(registryEntryID->unsigned64BitValue() == service->getRegistryEntryID());
5661	serviceArray->removeObject(index: idx);
5662
5663	setProperty(aKey: gIOAssociatedServicesKey, anObject: serviceArray);
5664	OSSafeReleaseNULL(serviceArray);
5665	OSSafeReleaseNULL(serviceArrayObj);
5666
5667	uvars = service->reserved->uvars;
5668	uvars->stopped = true;
5669	uvars->powerState = `0`;
5670
5671	bool allPowerStates __block = `0`;
5672	// any service on?
5673	fServices->iterateObjects(block: ^bool (OSObject * obj) {
5674	int service __unused; // hide outer defn
5675	IOService * nextService;
5676	nextService = (IOService *) obj;
5677	allPowerStates = nextService->reserved->uvars->powerState;
5678	// early terminate if true
5679	return allPowerStates;
5680	});
5681
5682	if (!allPowerStates && (pmAck = fSystemPowerAck)) {
5683	fSystemPowerAck = false;
5684	fSystemOff = true;
5685	}
5686	}
5687	IOLockUnlock(fLock);
5688	if (pmAck) {
5689	IOServicePH::serverAck(server: this);
5690	}
5691
5692	if (-`1U` == idx) {
5693	return kIOReturnSuccess;
5694	}
5695
5696	if (uvars->queueArray && uvars->userMeta) {
5697	queueAlloc = `1`;
5698	if (uvars->userMeta->queueNames) {
5699	queueAlloc += uvars->userMeta->queueNames->count;
5700	}
5701	for (idx = `0`; idx < queueAlloc; idx++) {
5702	OSSafeReleaseNULL(uvars->queueArray[idx]);
5703	}
5704	unboundedQueueArray = uvars->queueArray.data();
5705	IOSafeDeleteNULL(unboundedQueueArray, IODispatchQueue *, queueAlloc);
5706	uvars->queueArray = OSBoundedArrayRef<IODispatchQueue *>();
5707	}
5708
5709	(void) service->deRegisterInterestedDriver(theDriver: this);
5710	if (uvars->userServerPM) {
5711	service->PMstop();
5712	}
5713
5714	ret = kIOReturnSuccess;
5715	return ret;
5716	}
5717
5718	void
5719	IOUserServer::serviceFree(IOService * service)
5720	{
5721	OSObjectUserVars * uvars;
5722
5723	uvars = service->reserved->uvars;
5724	if (!uvars) {
5725	return;
5726	}
5727	OSSafeReleaseNULL(uvars->userServer);
5728	IOLockFree(lock: uvars->uvarsLock);
5729	IOFreeType(service->reserved->uvars, OSObjectUserVars);
5730	}
5731
5732	void
5733	IOUserServer::serviceWillTerminate(IOService * client, IOService * provider, IOOptionBits options)
5734	{
5735	IOReturn ret;
5736	bool willTerminate;
5737
5738	willTerminate = false;
5739	IOLockLock(client->reserved->uvars->uvarsLock);
5740	if (!client->reserved->uvars->serverDied
5741	&& !client->reserved->uvars->willTerminate) {
5742	client->reserved->uvars->willTerminate = true;
5743	willTerminate = true;
5744	}
5745	IOLockUnlock(client->reserved->uvars->uvarsLock);
5746
5747	if (willTerminate) {
5748	if (provider->isInactive() \|\| IOServicePH::serverSlept()) {
5749	client->Stop_async(provider);
5750	ret = kIOReturnOffline;
5751	} else {
5752	ret = client->Stop(provider);
5753	}
5754	if (kIOReturnSuccess != ret) {
5755	IOUserServer::serviceDidStop(client, provider);
5756	ret = kIOReturnSuccess;
5757	}
5758	}
5759	}
5760
5761	void
5762	IOUserServer::serviceDidTerminate(IOService * client, IOService * provider, IOOptionBits options, bool * defer)
5763	{
5764	IOLockLock(client->reserved->uvars->uvarsLock);
5765	client->reserved->uvars->didTerminate = true;
5766	if (!client->reserved->uvars->serverDied
5767	&& !client->reserved->uvars->stopped) {
5768	defer = true*;
5769	}
5770	IOLockUnlock(client->reserved->uvars->uvarsLock);
5771	}
5772
5773	void
5774	IOUserServer::serviceDidStop(IOService * client, IOService * provider)
5775	{
5776	bool complete;
5777	OSArray * closeArray;
5778
5779	complete = false;
5780	closeArray = NULL;
5781
5782	IOLockLock(client->reserved->uvars->uvarsLock);
5783	if (client->reserved->uvars
5784	&& client->reserved->uvars->willTerminate
5785	&& !client->reserved->uvars->stopped) {
5786	client->reserved->uvars->stopped = true;
5787	complete = client->reserved->uvars->didTerminate;
5788	}
5789
5790	if (client->reserved->uvars) {
5791	closeArray = client->reserved->uvars->openProviders;
5792	client->reserved->uvars->openProviders = NULL;
5793	}
5794	IOLockUnlock(client->reserved->uvars->uvarsLock);
5795
5796	if (closeArray) {
5797	closeArray->iterateObjects(block: ^bool (OSObject * obj) {
5798	IOService * toClose;
5799	toClose = OSDynamicCast(IOService, obj);
5800	if (toClose) {
5801	DKLOG(DKS ":force close (" DKS ")\n", DKN(client), DKN(toClose));
5802	toClose->close(forClient: client);
5803	}
5804	return false;
5805	});
5806	closeArray->release();
5807	}
5808
5809	if (complete) {
5810	bool defer = false;
5811	client->didTerminate(provider, options: `0`, defer: &defer);
5812	}
5813	}
5814
5815	kern_return_t
5816	IOService::ClientCrashed_Impl(
5817	IOService * client,
5818	uint64_t options)
5819	{
5820	return kIOReturnUnsupported;
5821	}
5822
5823	kern_return_t
5824	IOService::Stop_Impl(
5825	IOService * provider)
5826	{
5827	IOUserServer::serviceDidStop(client: this, provider);
5828
5829	return kIOReturnSuccess;
5830	}
5831
5832	void
5833	IOService::Stop_async_Impl(
5834	IOService * provider)
5835	{
5836	}
5837
5838	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
5839
5840	#undef super
5841	#define super IOUserClient
5842
5843	OSDefineMetaClassAndStructors(IOUserUserClient, IOUserClient)
5844
5845	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
5846
5847	bool
5848	IOUserUserClient::init(OSDictionary * properties)
5849	{
5850	if (!super::init(dictionary: properties)) {
5851	return false;
5852	}
5853
5854	fWorkGroups = OSDictionary::withCapacity(capacity: `0`);
5855	if (fWorkGroups == NULL) {
5856	return false;
5857	}
5858
5859	fEventLinks = OSDictionary::withCapacity(capacity: `0`);
5860	if (fEventLinks == NULL) {
5861	return false;
5862	}
5863
5864	fLock = IOLockAlloc();
5865
5866	return true;
5867	}
5868
5869	void
5870	IOUserUserClient::free()
5871	{
5872	OSSafeReleaseNULL(fWorkGroups);
5873	OSSafeReleaseNULL(fEventLinks);
5874	if (fLock) {
5875	IOLockFree(lock: fLock);
5876	}
5877
5878	super::free();
5879	}
5880
5881	IOReturn
5882	IOUserUserClient::setTask(task_t task)
5883	{
5884	task_reference(task);
5885	fTask = task;
5886
5887	return kIOReturnSuccess;
5888	}
5889
5890	void
5891	IOUserUserClient::stop(IOService * provider)
5892	{
5893	if (fTask) {
5894	task_deallocate(fTask);
5895	fTask = NULL;
5896	}
5897	super::stop(provider);
5898	}
5899
5900	IOReturn
5901	IOUserUserClient::clientClose(void)
5902	{
5903	terminate(options: kIOServiceTerminateNeedWillTerminate);
5904	return kIOReturnSuccess;
5905	}
5906
5907	IOReturn
5908	IOUserUserClient::setProperties(OSObject * properties)
5909	{
5910	IOReturn ret = kIOReturnUnsupported;
5911	return ret;
5912	}
5913
5914	// p1 - name of object
5915	// p2 - length of object name
5916	// p3 - mach port name
5917
5918	kern_return_t
5919	IOUserUserClient::eventlinkConfigurationTrap(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6)
5920	{
5921	user_addr_t userObjectName = (user_addr_t)p1;
5922	mach_port_name_t portName = (mach_port_name_t)(uintptr_t)p3;
5923	mach_port_t port = MACH_PORT_NULL;
5924	ipc_kobject_type_t portType;
5925	char eventlinkName[kIOEventLinkMaxNameLength + `1`] = {`0`};
5926	size_t eventLinkNameLen;
5927	OSString * eventlinkNameStr = NULL; // must release
5928	IOEventLink * eventLink = NULL; // do not release
5929	kern_return_t ret;
5930
5931	ret = copyinstr(uaddr: userObjectName, kaddr: &eventlinkName[`0`], len: sizeof(eventlinkName), done: &eventLinkNameLen);
5932	if (ret != kIOReturnSuccess) {
5933	goto finish;
5934	}
5935
5936	// ensure string length matches trap argument
5937	if (eventLinkNameLen != (size_t)p2 + `1`) {
5938	ret = kIOReturnBadArgument;
5939	goto finish;
5940	}
5941
5942	eventlinkNameStr = OSString::withCStringNoCopy(cString: eventlinkName);
5943	if (eventlinkNameStr == NULL) {
5944	ret = kIOReturnNoMemory;
5945	goto finish;
5946	}
5947
5948	IOLockLock(fLock);
5949	eventLink = OSDynamicCast(IOEventLink, fEventLinks->getObject(eventlinkNameStr));
5950	if (eventLink) {
5951	eventLink->retain();
5952	}
5953	IOLockUnlock(fLock);
5954
5955	if (eventLink == NULL) {
5956	ret = kIOReturnNotFound;
5957	goto finish;
5958	}
5959
5960	port = iokit_lookup_raw_current_task(name: portName, type: &portType);
5961
5962	if (port == NULL) {
5963	ret = kIOReturnNotFound;
5964	goto finish;
5965	}
5966
5967	if (portType != IKOT_EVENTLINK) {
5968	ret = kIOReturnBadArgument;
5969	goto finish;
5970	}
5971
5972	ret = eventLink->SetEventlinkPort(port);
5973	if (ret != kIOReturnSuccess) {
5974	if (kIODKLogSetup & gIODKDebug) {
5975	DKLOG(DKS " %s SetEventlinkPort() returned %x\n", DKN(this), eventlinkNameStr->getCStringNoCopy(), ret);
5976	}
5977	goto finish;
5978	}
5979
5980	finish:
5981	if (port != NULL) {
5982	iokit_release_port_send(port);
5983	}
5984
5985	OSSafeReleaseNULL(eventlinkNameStr);
5986	OSSafeReleaseNULL(eventLink);
5987
5988	return ret;
5989	}
5990
5991	kern_return_t
5992	IOUserUserClient::workgroupConfigurationTrap(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6)
5993	{
5994	user_addr_t userObjectName = (user_addr_t)p1;
5995	mach_port_name_t portName = (mach_port_name_t)(uintptr_t)p3;
5996	mach_port_t port = MACH_PORT_NULL;
5997	ipc_kobject_type_t portType;
5998	char workgroupName[kIOWorkGroupMaxNameLength + `1`] = {`0`};
5999	size_t workgroupNameLen;
6000	OSString * workgroupNameStr = NULL; // must release
6001	IOWorkGroup * workgroup = NULL; // do not release
6002	kern_return_t ret;
6003
6004	ret = copyinstr(uaddr: userObjectName, kaddr: &workgroupName[`0`], len: sizeof(workgroupName), done: &workgroupNameLen);
6005	if (ret != kIOReturnSuccess) {
6006	goto finish;
6007	}
6008
6009	// ensure string length matches trap argument
6010	if (workgroupNameLen != (size_t)p2 + `1`) {
6011	ret = kIOReturnBadArgument;
6012	goto finish;
6013	}
6014
6015	workgroupNameStr = OSString::withCStringNoCopy(cString: workgroupName);
6016	if (workgroupNameStr == NULL) {
6017	ret = kIOReturnNoMemory;
6018	goto finish;
6019	}
6020
6021	IOLockLock(fLock);
6022	workgroup = OSDynamicCast(IOWorkGroup, fWorkGroups->getObject(workgroupNameStr));
6023	if (workgroup) {
6024	workgroup->retain();
6025	}
6026	IOLockUnlock(fLock);
6027
6028	if (workgroup == NULL) {
6029	ret = kIOReturnNotFound;
6030	goto finish;
6031	}
6032
6033	port = iokit_lookup_raw_current_task(name: portName, type: &portType);
6034
6035	if (port == NULL) {
6036	ret = kIOReturnNotFound;
6037	goto finish;
6038	}
6039
6040	if (portType != IKOT_WORK_INTERVAL) {
6041	ret = kIOReturnBadArgument;
6042	goto finish;
6043	}
6044
6045	ret = workgroup->SetWorkGroupPort(port);
6046	if (ret != kIOReturnSuccess) {
6047	if (kIODKLogSetup & gIODKDebug) {
6048	DKLOG(DKS " %s SetWorkGroupPort() returned %x\n", DKN(this), workgroupNameStr->getCStringNoCopy(), ret);
6049	}
6050	goto finish;
6051	}
6052
6053	finish:
6054
6055	if (port != NULL) {
6056	iokit_release_port_send(port);
6057	}
6058
6059	OSSafeReleaseNULL(workgroupNameStr);
6060	OSSafeReleaseNULL(workgroup);
6061
6062	return ret;
6063	}
6064
6065	IOExternalTrap *
6066	IOUserUserClient::getTargetAndTrapForIndex( IOService **targetP, UInt32 index )
6067	{
6068	static const OSBoundedArray<IOExternalTrap, `2`> trapTemplate = {.data_: {
6069	{ NULL, .func: (IOTrap) & IOUserUserClient::eventlinkConfigurationTrap},
6070	{ NULL, .func: (IOTrap) & IOUserUserClient::workgroupConfigurationTrap},
6071	}};
6072	if (index >= trapTemplate.size()) {
6073	return NULL;
6074	}
6075	targetP = this*;
6076	return (IOExternalTrap *)&trapTemplate [index];
6077	}
6078
6079	kern_return_t
6080	IOUserClient::CopyClientEntitlements_Impl(OSDictionary ** entitlements)
6081	{
6082	return kIOReturnUnsupported;
6083	};
6084
6085	struct IOUserUserClientActionRef {
6086	OSAsyncReference64 asyncRef;
6087	};
6088
6089	void
6090	IOUserClient::KernelCompletion_Impl(
6091	OSAction * action,
6092	IOReturn status,
6093	const unsigned long long * asyncData,
6094	uint32_t asyncDataCount)
6095	{
6096	IOUserUserClientActionRef * ref;
6097
6098	ref = (typeof(ref))action->GetReference();
6099
6100	IOUserClient::sendAsyncResult64(reference: ref->asyncRef, result: status, args: (io_user_reference_t *) asyncData, numArgs: asyncDataCount);
6101	}
6102
6103	kern_return_t
6104	IOUserClient::_ExternalMethod_Impl(
6105	uint64_t selector,
6106	const unsigned long long * scalarInput,
6107	uint32_t scalarInputCount,
6108	OSData * structureInput,
6109	IOMemoryDescriptor * structureInputDescriptor,
6110	unsigned long long * scalarOutput,
6111	uint32_t * scalarOutputCount,
6112	uint64_t structureOutputMaximumSize,
6113	OSData ** structureOutput,
6114	IOMemoryDescriptor * structureOutputDescriptor,
6115	OSAction * completion)
6116	{
6117	return kIOReturnUnsupported;
6118	}
6119
6120	IOReturn
6121	IOUserUserClient::clientMemoryForType(UInt32 type,
6122	IOOptionBits * koptions,
6123	IOMemoryDescriptor ** kmemory)
6124	{
6125	IOReturn kr;
6126	uint64_t options;
6127	IOMemoryDescriptor * memory;
6128
6129	kr = CopyClientMemoryForType(type, options: &options, memory: &memory);
6130
6131	*koptions = `0`;
6132	*kmemory = NULL;
6133	if (kIOReturnSuccess != kr) {
6134	return kr;
6135	}
6136
6137	if (kIOUserClientMemoryReadOnly & options) {
6138	*koptions \|= kIOMapReadOnly;
6139	}
6140	*kmemory = memory;
6141
6142	return kr;
6143	}
6144
6145	IOReturn
6146	IOUserUserClient::externalMethod(uint32_t selector, IOExternalMethodArguments * args,
6147	IOExternalMethodDispatch * dispatch, OSObject * target, void * reference)
6148	{
6149	IOReturn kr;
6150	OSData * structureInput;
6151	OSData * structureOutput;
6152	size_t copylen;
6153	uint64_t structureOutputSize;
6154	OSAction * action;
6155	IOUserUserClientActionRef * ref;
6156	mach_port_t wake_port = MACH_PORT_NULL;
6157
6158	kr = kIOReturnUnsupported;
6159	structureInput = NULL;
6160	action = NULL;
6161	ref = NULL;
6162
6163	if (args->structureInputSize) {
6164	structureInput = OSData::withBytesNoCopy(bytes: (void *) args->structureInput, numBytes: args->structureInputSize);
6165	}
6166
6167	if (MACH_PORT_NULL != args->asyncWakePort) {
6168	// this retain is for the OSAction to release
6169	wake_port = ipc_port_make_send_mqueue(args->asyncWakePort);
6170	kr = CreateActionKernelCompletion(referenceSize: sizeof(IOUserUserClientActionRef), action: &action);
6171	assert(KERN_SUCCESS == kr);
6172	ref = (typeof(ref))action->GetReference();
6173	bcopy(src: args->asyncReference, dst: &ref->asyncRef[`0`], n: args->asyncReferenceCount * sizeof(ref->asyncRef[`0`]));
6174	kr = action->SetAbortedHandler(^(void) {
6175	IOUserUserClientActionRef * ref;
6176	IOReturn ret;
6177
6178	ref = (typeof(ref))action->GetReference();
6179	ret = releaseAsyncReference64(reference: ref->asyncRef);
6180	assert(kIOReturnSuccess == ret);
6181	bzero(s: &ref->asyncRef[`0`], n: sizeof(ref->asyncRef));
6182	});
6183	assert(KERN_SUCCESS == kr);
6184	}
6185
6186	if (args->structureVariableOutputData) {
6187	structureOutputSize = kIOUserClientVariableStructureSize;
6188	} else if (args->structureOutputDescriptor) {
6189	structureOutputSize = args->structureOutputDescriptor->getLength();
6190	} else {
6191	structureOutputSize = args->structureOutputSize;
6192	}
6193
6194	kr = _ExternalMethod(selector, scalarInput: &args->scalarInput[`0`], scalarInputCount: args->scalarInputCount,
6195	structureInput, structureInputDescriptor: args->structureInputDescriptor,
6196	scalarOutput: args->scalarOutput, scalarOutputCount: &args->scalarOutputCount,
6197	structureOutputMaximumSize: structureOutputSize, structureOutput: &structureOutput, structureOutputDescriptor: args->structureOutputDescriptor,
6198	completion: action);
6199
6200	OSSafeReleaseNULL(structureInput);
6201	OSSafeReleaseNULL(action);
6202
6203	if (kr == kIOReturnSuccess && structureOutput) {
6204	if (args->structureVariableOutputData) {
6205	*args->structureVariableOutputData = structureOutput;
6206	} else {
6207	copylen = structureOutput->getLength();
6208	if (copylen > args->structureOutputSize) {
6209	kr = kIOReturnBadArgument;
6210	} else {
6211	bcopy(src: (const void *) structureOutput->getBytesNoCopy(), dst: args->structureOutput, n: copylen);
6212	args->structureOutputSize = (uint32_t) copylen;
6213	}
6214	OSSafeReleaseNULL(structureOutput);
6215	}
6216	}
6217
6218	if (kIOReturnSuccess != kr) {
6219	// mig will destroy any async port
6220	return kr;
6221	}
6222
6223	// We must never return error after this point in order to preserve MIG ownership semantics
6224	assert(kr == kIOReturnSuccess);
6225	if (MACH_PORT_NULL != wake_port) {
6226	// this release is for the mig created send right
6227	iokit_release_port_send(port: wake_port);
6228	}
6229
6230	return kr;
6231	}
6232
6233	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
6234
6235	extern IORecursiveLock * gDriverKitLaunchLock;
6236	extern OSSet * gDriverKitLaunches;
6237
6238	_IOUserServerCheckInCancellationHandler *
6239	IOUserServerCheckInToken::setCancellationHandler(IOUserServerCheckInCancellationHandler handler,
6240	void* handlerArgs)
6241	{
6242	_IOUserServerCheckInCancellationHandler * handlerObj = _IOUserServerCheckInCancellationHandler::withHandler(handler, args: handlerArgs);
6243	if (!handlerObj) {
6244	goto finish;
6245	}
6246
6247	IORecursiveLockLock(lock: gDriverKitLaunchLock);
6248
6249	assert(fState != kIOUserServerCheckInComplete);
6250
6251	if (fState == kIOUserServerCheckInCanceled) {
6252	// Send cancel notification if we set the handler after this was canceled
6253	handlerObj->call(token: this);
6254	} else if (fState == kIOUserServerCheckInPending) {
6255	fHandlers->setObject(handlerObj);
6256	}
6257
6258	IORecursiveLockUnlock(lock: gDriverKitLaunchLock);
6259
6260	finish:
6261	return handlerObj;
6262	}
6263
6264	void
6265	IOUserServerCheckInToken::removeCancellationHandler(_IOUserServerCheckInCancellationHandler * handler)
6266	{
6267	IORecursiveLockLock(lock: gDriverKitLaunchLock);
6268
6269	fHandlers->removeObject(anObject: handler);
6270
6271	IORecursiveLockUnlock(lock: gDriverKitLaunchLock);
6272	}
6273
6274	void
6275	IOUserServerCheckInToken::cancel()
6276	{
6277	IORecursiveLockLock(lock: gDriverKitLaunchLock);
6278
6279	if (fState == kIOUserServerCheckInPending) {
6280	fState = kIOUserServerCheckInCanceled;
6281	if (gDriverKitLaunches != NULL) {
6282	// Remove pending launch from list, if we have not shut down yet.
6283	gDriverKitLaunches->removeObject(anObject: this);
6284	}
6285
6286	fHandlers->iterateObjects(block: ^bool (OSObject * obj){
6287	_IOUserServerCheckInCancellationHandler * handlerObj = OSDynamicCast(_IOUserServerCheckInCancellationHandler, obj);
6288	if (handlerObj) {
6289	handlerObj->call(token: this);
6290	}
6291	return false;
6292	});
6293	fHandlers->flushCollection();
6294	}
6295
6296	IORecursiveLockUnlock(lock: gDriverKitLaunchLock);
6297	}
6298
6299	void
6300	IOUserServerCheckInToken::complete()
6301	{
6302	IORecursiveLockLock(lock: gDriverKitLaunchLock);
6303
6304	if (fState == kIOUserServerCheckInPending && --fPendingCount == `0`) {
6305	fState = kIOUserServerCheckInComplete;
6306	if (gDriverKitLaunches != NULL) {
6307	// Remove pending launch from list, if we have not shut down yet.
6308	gDriverKitLaunches->removeObject(anObject: this);
6309	}
6310
6311	// No need to hold on to the cancellation handlers
6312	fHandlers->flushCollection();
6313	}
6314
6315	IORecursiveLockUnlock(lock: gDriverKitLaunchLock);
6316	}
6317
6318	bool
6319	IOUserServerCheckInToken::init(const OSSymbol * serverName, OSNumber * serverTag, OSKext driverKext, OSData serverDUI)
6320	{
6321	if (!OSObject::init()) {
6322	return false;
6323	}
6324
6325	if (!serverName) {
6326	return false;
6327	}
6328	fServerName = serverName;
6329	fServerName->retain();
6330
6331	if (!serverTag) {
6332	return false;
6333	}
6334	fServerTag = serverTag;
6335	fServerTag->retain();
6336
6337	fHandlers = OSSet::withCapacity(capacity: `0`);
6338	if (!fHandlers) {
6339	return false;
6340	}
6341
6342	fState = kIOUserServerCheckInPending;
6343	fPendingCount = `1`;
6344
6345	fKextBundleID = NULL;
6346	fNeedDextDec = false;
6347
6348	fExecutableName = NULL;
6349
6350	if (driverKext) {
6351	fExecutableName = OSDynamicCast(OSSymbol, driverKext->getBundleExecutable());
6352
6353	if (fExecutableName) {
6354	fExecutableName->retain();
6355	}
6356
6357	/*
6358	* We need to keep track of how many dexts we have started.
6359	* For every new dext we are going to create a new token, and
6360	* we consider the token creation as the initial step to
6361	* create a dext as it is the data structure that will back up
6362	* the userspace dance to start a dext.
6363	* We later have to decrement only once per token.
6364	* If no error occurs we consider the finalize() call on IOUserServer
6365	* as the moment in which we do not consider the dext "alive" anymore;
6366	* however in case of errors we will still need to decrement the count
6367	* otherwise upgrades of the dext will never make progress.
6368	*/
6369	if (OSKext::incrementDextLaunchCount(dext: driverKext, dextUniqueIDToMatch: serverDUI)) {
6370	/*
6371	* If fKext holds a pointer,
6372	* it is the indication that a decrements needs
6373	* to be called.
6374	*/
6375	fNeedDextDec = true;
6376	fKextBundleID = OSDynamicCast(OSString, driverKext->getIdentifier());
6377	fKextBundleID->retain();
6378	} else {
6379	return false;
6380	}
6381	}
6382
6383	return true;
6384	}
6385
6386	/*
6387	* Returns if the dext can be re-used
6388	* for matching.
6389	*/
6390	bool
6391	IOUserServerCheckInToken::dextTerminate(void)
6392	{
6393	bool ret = true;
6394
6395	if (fNeedDextDec == true) {
6396	/*
6397	* We can decrement DextLaunchCount only
6398	* once per token.
6399	*/
6400	ret = !(OSKext::decrementDextLaunchCount(bundleID: fKextBundleID));
6401	fNeedDextDec = false;
6402	}
6403
6404	return ret;
6405	}
6406
6407	void
6408	IOUserServerCheckInToken::free()
6409	{
6410	OSSafeReleaseNULL(fServerName);
6411	OSSafeReleaseNULL(fServerTag);
6412	OSSafeReleaseNULL(fExecutableName);
6413	OSSafeReleaseNULL(fHandlers);
6414	if (fKextBundleID != NULL) {
6415	dextTerminate();
6416	OSSafeReleaseNULL(fKextBundleID);
6417	}
6418
6419	OSObject::free();
6420	}
6421
6422	const OSSymbol *
6423	IOUserServerCheckInToken::copyServerName() const
6424	{
6425	fServerName->retain();
6426	return fServerName;
6427	}
6428
6429	OSNumber *
6430	IOUserServerCheckInToken::copyServerTag() const
6431	{
6432	fServerTag->retain();
6433	return fServerTag;
6434	}
6435
6436	IOUserServer *
6437	IOUserServer::launchUserServer(OSString * bundleID, const OSSymbol * serverName, OSNumber * serverTag, bool reuseIfExists, IOUserServerCheckInToken ** resultToken, OSData *serverDUI)
6438	{
6439	IOUserServer *me = NULL;
6440	IOUserServerCheckInToken * token = NULL;
6441	OSDictionary * matching = NULL; // must release
6442	OSKext * driverKext = NULL; // must release
6443	OSDextStatistics * driverStatistics = NULL; // must release
6444	bool reslide = false;
6445
6446	/ TODO: Check we are looking for same dextID*
6447	* and if it is not the same
6448	* restart the matching process.
6449	*/
6450	driverKext = OSKext::lookupDextWithIdentifier(dextIdentifier: bundleID, dextUniqueIdentifier: serverDUI);
6451	if (driverKext != NULL) {
6452	driverStatistics = driverKext->copyDextStatistics();
6453	if (driverStatistics == NULL) {
6454	panic("Kext %s was not a DriverKit OSKext", bundleID->getCStringNoCopy());
6455	}
6456	IOLog(format: "Driver %s has crashed %zu time(s)\n", bundleID->getCStringNoCopy(), driverStatistics->getCrashCount());
6457	reslide = driverStatistics->getCrashCount() > `0`;
6458	} else {
6459	DKLOG("Could not find OSKext for %s\n", bundleID->getCStringNoCopy());
6460	*resultToken = NULL;
6461	return NULL;
6462	}
6463
6464	IORecursiveLockLock(lock: gDriverKitLaunchLock);
6465
6466	if (gDriverKitLaunches == NULL) {
6467	// About to shut down, don't launch anything
6468	goto finish;
6469	}
6470
6471	if (reuseIfExists) {
6472	const char * serverNameCStr;
6473	const char * bundleIDCStr;
6474	const char * endOrgCStr;
6475
6476	serverNameCStr = serverName->getCStringNoCopy();
6477	bundleIDCStr = bundleID->getCStringNoCopy();
6478	(endOrgCStr = strchr(s: bundleIDCStr, c: `'.'`)) && (endOrgCStr = strchr(s: endOrgCStr + `1`, c: `'.'`));
6479	reuseIfExists = endOrgCStr && (`0` == strncmp(s1: bundleIDCStr, s2: serverNameCStr, n: endOrgCStr + `1` - bundleIDCStr));
6480	if (!reuseIfExists) {
6481	IOLog(kIOUserServerNameKey " \"%s\" not correct organization for bundleID \"%s\"\n", serverNameCStr, bundleIDCStr);
6482	}
6483	}
6484
6485	// Find existing server
6486	if (reuseIfExists) {
6487	token = IOUserServerCheckInToken::findExistingToken(serverName);
6488	if (token) {
6489	// Launch in progress, return token
6490	goto finish;
6491	} else {
6492	// Check if launch completed
6493	matching = IOService::serviceMatching(className: gIOUserServerClassKey);
6494	if (!matching) {
6495	goto finish;
6496	}
6497	IOService::propertyMatching(key: gIOUserServerNameKey, value: serverName, table: matching);
6498	IOService * service = IOService::copyMatchingService(matching);
6499	IOUserServer * userServer = OSDynamicCast(IOUserServer, service);
6500	if (userServer) {
6501	// found existing user server
6502	me = userServer;
6503	goto finish;
6504	} else {
6505	OSSafeReleaseNULL(service);
6506	}
6507	}
6508	}
6509
6510	// No existing server, request launch
6511	token = new IOUserServerCheckInToken;
6512	if (!token) {
6513	goto finish;
6514	}
6515
6516	/*
6517	* TODO: If the init fails because the personalities are not up to date
6518	* restart the whole matching process.
6519	*/
6520	if (token && !token->init(serverName, serverTag, driverKext, serverDUI)) {
6521	IOLog(format: "Could not initialize token\n");
6522	OSSafeReleaseNULL(token);
6523	goto finish;
6524	}
6525
6526	/*
6527	* If the launch fails at any point terminate() will
6528	* be called on this IOUserServer.
6529	*/
6530	gDriverKitLaunches->setObject(token);
6531	OSKext::requestDaemonLaunch(kextIdentifier: bundleID, serverName: (OSString *)serverName, serverTag, reslide: reslide ? kOSBooleanTrue : kOSBooleanFalse, checkInToken: token, serverDUI);
6532
6533	finish:
6534	IORecursiveLockUnlock(lock: gDriverKitLaunchLock);
6535	OSSafeReleaseNULL(matching);
6536	OSSafeReleaseNULL(driverStatistics);
6537	OSSafeReleaseNULL(driverKext);
6538
6539	if (resultToken) {
6540	*resultToken = token;
6541	} else {
6542	OSSafeReleaseNULL(token);
6543	}
6544
6545	return me;
6546	}
6547
6548	/*
6549	* IOUserServerCheckInTokens are used to track dext launches. They have three possible states:
6550	*
6551	* - Pending: A dext launch is pending
6552	* - Canceled: Dext launch failed
6553	* - Complete: Dext launch is complete
6554	*
6555	* A token can be shared among multiple IOServices that are waiting for dexts if the IOUserServerName
6556	* is the same. This allows dexts to be reused and host multiple services. All pending tokens are stored
6557	* in gDriverKitLaunches and we check here before creating a new token when launching a dext.
6558	*
6559	* A token starts in the pending state with a pending count of 1. When we reuse a token, we increase the
6560	* pending count of the token.
6561	*
6562	* The token is sent to userspace as a mach port through kernelmanagerd/driverkitd to the dext. The dext then
6563	* uses that token to check in to the kernel. If any part of the dext launch failed (dext crashed, kmd crashed, etc.)
6564	* we get a no-senders notification for the token in the kernel and the token goes into the Canceled state.
6565	*
6566	* Once the dext checks in to the kernel, we decrement the pending count for the token. When the pending count reaches
6567	* 0, the token goes into the Complete state. So if the token is in the Complete state, there are no kernel matching threads
6568	* waiting on the dext to check in.
6569	*/
6570
6571	IOUserServerCheckInToken *
6572	IOUserServerCheckInToken::findExistingToken(const OSSymbol * serverName)
6573	{
6574	IOUserServerCheckInToken * __block result = NULL;
6575
6576	IORecursiveLockLock(lock: gDriverKitLaunchLock);
6577	if (gDriverKitLaunches == NULL) {
6578	goto finish;
6579	}
6580
6581	gDriverKitLaunches->iterateObjects(block: ^(OSObject * obj) {
6582	IOUserServerCheckInToken * token = OSDynamicCast(IOUserServerCheckInToken, obj);
6583	if (token) {
6584	// Check if server name matches
6585	const OSSymbol * tokenServerName = token->fServerName;
6586	if (tokenServerName->isEqualTo(aSymbol: serverName)) {
6587	assert(token->fState == kIOUserServerCheckInPending);
6588	token->fPendingCount++;
6589	result = token;
6590	result->retain();
6591	}
6592	}
6593	return result != NULL;
6594	});
6595
6596	finish:
6597	IORecursiveLockUnlock(lock: gDriverKitLaunchLock);
6598	return result;
6599	}
6600
6601	void
6602	IOUserServerCheckInToken::cancelAll()
6603	{
6604	OSSet * tokensToCancel;
6605
6606	IORecursiveLockLock(lock: gDriverKitLaunchLock);
6607	tokensToCancel = gDriverKitLaunches;
6608	gDriverKitLaunches = NULL;
6609
6610
6611	tokensToCancel->iterateObjects(block: ^(OSObject *obj) {
6612	IOUserServerCheckInToken * token = OSDynamicCast(IOUserServerCheckInToken, obj);
6613	if (token) {
6614	token->cancel();
6615	}
6616	return false;
6617	});
6618
6619	IORecursiveLockUnlock(lock: gDriverKitLaunchLock);
6620
6621	OSSafeReleaseNULL(tokensToCancel);
6622	}
6623
6624	void
6625	_IOUserServerCheckInCancellationHandler::call(IOUserServerCheckInToken * token)
6626	{
6627	fHandler(token, fHandlerArgs);
6628	}
6629
6630	_IOUserServerCheckInCancellationHandler *
6631	_IOUserServerCheckInCancellationHandler::withHandler(IOUserServerCheckInCancellationHandler handler, void * args)
6632	{
6633	_IOUserServerCheckInCancellationHandler * handlerObj = NULL;
6634	if (!handler) {
6635	goto finish;
6636	}
6637
6638	handlerObj = new _IOUserServerCheckInCancellationHandler;
6639	if (!handlerObj) {
6640	goto finish;
6641	}
6642
6643	handlerObj->fHandler = handler;
6644	handlerObj->fHandlerArgs = args;
6645
6646	finish:
6647	return handlerObj;
6648	}
6649
6650	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
6651
6652	struct IOServiceStateNotificationDispatchSource_IVars {
6653	IOLock * fLock;
6654	IOService * fStateNotification;
6655	IOStateNotificationListenerRef fListener;
6656	OSAction * fAction;
6657	bool fEnable;
6658	bool fArmed;
6659	};
6660
6661	kern_return_t
6662	IOServiceStateNotificationDispatchSource::Create_Impl(IOService * service, OSArray * items,
6663	IODispatchQueue * queue, IOServiceStateNotificationDispatchSource ** outSource)
6664	{
6665	kern_return_t kr;
6666	IOServiceStateNotificationDispatchSource * source;
6667
6668	source = OSTypeAlloc(IOServiceStateNotificationDispatchSource);
6669	source->init();
6670
6671	source->ivars->fStateNotification = service;
6672	kr = service->stateNotificationListenerAdd(items, outRef: &source->ivars->fListener, handler: ^kern_return_t () {
6673	OSAction * action;
6674
6675	action = NULL;
6676	IOLockLock(source->ivars->fLock);
6677	if (source->ivars->fArmed && source->ivars->fAction) {
6678	source->ivars->fArmed = false;
6679	action = source->ivars->fAction;
6680	action->retain();
6681	}
6682	IOLockUnlock(source->ivars->fLock);
6683	if (action) {
6684	source->StateNotificationReady(action);
6685	OSSafeReleaseNULL(action);
6686	}
6687	return kIOReturnSuccess;
6688	});
6689
6690	if (kIOReturnSuccess != kr) {
6691	OSSafeReleaseNULL(source);
6692	}
6693	*outSource = source;
6694
6695	return kr;
6696	}
6697
6698
6699	bool
6700	IOServiceStateNotificationDispatchSource::init()
6701	{
6702	if (!IODispatchSource::init()) {
6703	return false;
6704	}
6705	ivars = IOMallocType(IOServiceStateNotificationDispatchSource_IVars);
6706	if (!ivars) {
6707	return false;
6708	}
6709	ivars->fLock = IOLockAlloc();
6710	if (!ivars->fLock) {
6711	return false;
6712	}
6713	ivars->fArmed = true;
6714
6715	return true;
6716	}
6717
6718	void
6719	IOServiceStateNotificationDispatchSource::free()
6720	{
6721	if (ivars) {
6722	if (ivars->fListener) {
6723	ivars->fStateNotification->stateNotificationListenerRemove(ref: ivars->fListener);
6724	}
6725	if (ivars->fLock) {
6726	IOLockFree(lock: ivars->fLock);
6727	}
6728	IOFreeType(ivars, IOServiceStateNotificationDispatchSource_IVars);
6729	}
6730	IODispatchSource::free();
6731	}
6732
6733	kern_return_t
6734	IOServiceStateNotificationDispatchSource::SetHandler_Impl(OSAction * action)
6735	{
6736	IOReturn ret;
6737	bool notifyReady;
6738
6739	notifyReady = false;
6740
6741	IOLockLock(ivars->fLock);
6742	action->retain();
6743	OSSafeReleaseNULL(ivars->fAction);
6744	ivars->fAction = action;
6745	if (action) {
6746	notifyReady = true;
6747	}
6748	IOLockUnlock(ivars->fLock);
6749
6750	if (notifyReady) {
6751	StateNotificationReady(action);
6752	}
6753	ret = kIOReturnSuccess;
6754
6755	return ret;
6756	}
6757
6758	kern_return_t
6759	IOServiceStateNotificationDispatchSource::SetEnableWithCompletion_Impl(
6760	bool enable,
6761	IODispatchSourceCancelHandler handler)
6762	{
6763	if (enable == ivars->fEnable) {
6764	return kIOReturnSuccess;
6765	}
6766
6767	IOLockLock(ivars->fLock);
6768	ivars->fEnable = enable;
6769	IOLockUnlock(ivars->fLock);
6770
6771	return kIOReturnSuccess;
6772	}
6773
6774	kern_return_t
6775	IOServiceStateNotificationDispatchSource::Cancel_Impl(
6776	IODispatchSourceCancelHandler handler)
6777	{
6778	return kIOReturnUnsupported;
6779	}
6780
6781	kern_return_t
6782	IOServiceStateNotificationDispatchSource::StateNotificationBegin_Impl(void)
6783	{
6784	IOLockLock(ivars->fLock);
6785	ivars->fArmed = true;
6786	IOLockUnlock(ivars->fLock);
6787
6788	return kIOReturnSuccess;
6789	}
6790
6791	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
6792
6793	#include <IOKit/IOServiceStateNotificationEventSource.h>
6794
6795	OSDefineMetaClassAndStructors(IOServiceStateNotificationEventSource, IOEventSource)
6796	OSMetaClassDefineReservedUnused(IOServiceStateNotificationEventSource, `0`);
6797	OSMetaClassDefineReservedUnused(IOServiceStateNotificationEventSource, `1`);
6798	OSMetaClassDefineReservedUnused(IOServiceStateNotificationEventSource, `2`);
6799	OSMetaClassDefineReservedUnused(IOServiceStateNotificationEventSource, `3`);
6800	OSMetaClassDefineReservedUnused(IOServiceStateNotificationEventSource, `4`);
6801	OSMetaClassDefineReservedUnused(IOServiceStateNotificationEventSource, `5`);
6802	OSMetaClassDefineReservedUnused(IOServiceStateNotificationEventSource, `6`);
6803	OSMetaClassDefineReservedUnused(IOServiceStateNotificationEventSource, `7`);
6804
6805	OSPtr<IOServiceStateNotificationEventSource>
6806	IOServiceStateNotificationEventSource::serviceStateNotificationEventSource(IOService *service,
6807	OSArray * items,
6808	ActionBlock inAction)
6809	{
6810	kern_return_t kr;
6811	IOServiceStateNotificationEventSource * source;
6812
6813	source = OSTypeAlloc(IOServiceStateNotificationEventSource);
6814	if (source && !source->init(owner: service, NULL)) {
6815	OSSafeReleaseNULL(source);
6816	}
6817
6818	if (!source) {
6819	return nullptr;
6820	}
6821
6822	source->fStateNotification = service;
6823	kr = service->stateNotificationListenerAdd(items, outRef: &source->fListener, handler: ^kern_return_t () {
6824	if (!source->workLoop) {
6825	return kIOReturnSuccess;
6826	}
6827	source->workLoop->runActionBlock(action: ^IOReturn (void) {
6828	source->fArmed = true;
6829	return kIOReturnSuccess;
6830	});
6831	source->signalWorkAvailable();
6832	return kIOReturnSuccess;
6833	});
6834
6835	if (kIOReturnSuccess != kr) {
6836	OSSafeReleaseNULL(source);
6837	}
6838
6839	if (source) {
6840	source->setActionBlock((IOEventSource::ActionBlock) inAction);
6841	}
6842
6843	return source;
6844	}
6845
6846	void
6847	IOServiceStateNotificationEventSource::free()
6848	{
6849	if (fListener) {
6850	fStateNotification->stateNotificationListenerRemove(ref: fListener);
6851	}
6852	IOEventSource::free();
6853	}
6854
6855	void
6856	IOServiceStateNotificationEventSource::enable()
6857	{
6858	fEnable = true;
6859	}
6860
6861	void
6862	IOServiceStateNotificationEventSource::disable()
6863	{
6864	fEnable = false;
6865	}
6866
6867	void
6868	IOServiceStateNotificationEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
6869	{
6870	IOEventSource::setWorkLoop(inWorkLoop);
6871	}
6872
6873	bool
6874	IOServiceStateNotificationEventSource::checkForWork()
6875	{
6876	ActionBlock intActionBlock = (ActionBlock) actionBlock;
6877
6878	if (fArmed) {
6879	fArmed = false;
6880	(intActionBlock)();
6881	}
6882
6883	return false;
6884	}
6885
6886	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
6887
6888	OSDefineMetaClassAndStructors(IOSystemStateNotification, IOService);
6889
6890	class IOStateNotificationItem : public OSObject
6891	{
6892	OSDeclareDefaultStructors(IOStateNotificationItem);
6893
6894	public:
6895	virtual bool init() override;
6896
6897	OSDictionary * fSchema;
6898	OSDictionary * fValue;
6899	OSSet * fListeners;
6900	};
6901	OSDefineMetaClassAndStructors(IOStateNotificationItem, OSObject);
6902
6903
6904	class IOStateNotificationListener : public OSObject
6905	{
6906	OSDeclareDefaultStructors(IOStateNotificationListener);
6907
6908	public:
6909	virtual bool init() override;
6910	virtual void free() override;
6911
6912	IOStateNotificationHandler fHandler;
6913	};
6914	OSDefineMetaClassAndStructors(IOStateNotificationListener, OSObject);
6915
6916
6917	bool
6918	IOStateNotificationItem::init()
6919	{
6920	return OSObject::init();
6921	}
6922
6923	bool
6924	IOStateNotificationListener::init()
6925	{
6926	return OSObject::init();
6927	}
6928
6929	void
6930	IOStateNotificationListener::free()
6931	{
6932	if (fHandler) {
6933	Block_release(fHandler);
6934	}
6935	OSObject::free();
6936	}
6937
6938
6939	struct IOServiceStateChangeVars {
6940	IOLock * fLock;
6941	OSDictionary * fItems;
6942	};
6943
6944	IOService *
6945	IOSystemStateNotification::initialize(void)
6946	{
6947	IOSystemStateNotification * me;
6948	IOServiceStateChangeVars * vars;
6949
6950	me = OSTypeAlloc(IOSystemStateNotification);
6951	me->init();
6952	vars = IOMallocType(IOServiceStateChangeVars);
6953	me->reserved->svars = vars;
6954	vars->fLock = IOLockAlloc();
6955	vars->fItems = OSDictionary::withCapacity(capacity: `16`);
6956	{
6957	kern_return_t ret;
6958
6959	gIOSystemStateSleepDescriptionKey = (OSString *)OSSymbol::withCStringNoCopy(kIOSystemStateSleepDescriptionKey);
6960	gIOSystemStateSleepDescriptionHibernateStateKey = OSSymbol::withCStringNoCopy(kIOSystemStateSleepDescriptionHibernateStateKey);
6961	gIOSystemStateSleepDescriptionReasonKey = OSSymbol::withCStringNoCopy(kIOSystemStateSleepDescriptionReasonKey);
6962
6963	ret = me->StateNotificationItemCreate(itemName: gIOSystemStateSleepDescriptionKey, NULL);
6964	assert(kIOReturnSuccess == ret);
6965
6966	gIOSystemStateWakeDescriptionKey = (OSString *)OSSymbol::withCStringNoCopy(kIOSystemStateWakeDescriptionKey);
6967	gIOSystemStateWakeDescriptionWakeReasonKey = OSSymbol::withCStringNoCopy(kIOSystemStateWakeDescriptionWakeReasonKey);
6968
6969	ret = me->StateNotificationItemCreate(itemName: gIOSystemStateWakeDescriptionKey, NULL);
6970	assert(kIOReturnSuccess == ret);
6971
6972	gIOSystemStateHaltDescriptionKey = (OSString *)OSSymbol::withCStringNoCopy(kIOSystemStateHaltDescriptionKey);
6973	gIOSystemStateHaltDescriptionHaltStateKey = OSSymbol::withCStringNoCopy(kIOSystemStateHaltDescriptionHaltStateKey);
6974
6975	ret = me->StateNotificationItemCreate(itemName: gIOSystemStateHaltDescriptionKey, NULL);
6976	assert(kIOReturnSuccess == ret);
6977
6978	gIOSystemStatePowerSourceDescriptionKey = (OSString *)OSSymbol::withCStringNoCopy(kIOSystemStatePowerSourceDescriptionKey);
6979	gIOSystemStatePowerSourceDescriptionACAttachedKey = OSSymbol::withCStringNoCopy(kIOSystemStatePowerSourceDescriptionACAttachedKey);
6980
6981	ret = me->StateNotificationItemCreate(itemName: gIOSystemStatePowerSourceDescriptionKey, NULL);
6982	assert(kIOReturnSuccess == ret);
6983	}
6984
6985	return me;
6986	}
6987
6988	bool
6989	IOSystemStateNotification::serializeProperties(OSSerialize * s) const
6990	{
6991	IOServiceStateChangeVars * ivars = reserved->svars;
6992
6993	bool ok;
6994	OSDictionary * result;
6995
6996	result = OSDictionary::withCapacity(capacity: `16`);
6997
6998	IOLockLock(ivars->fLock);
6999	ivars->fItems->iterateObjects(block: ^bool (const OSSymbol * key, OSObject * object) {
7000	IOStateNotificationItem * item;
7001
7002	item = (typeof(item))object;
7003	if (!item->fValue) {
7004	return false;
7005	}
7006	result->setObject(aKey: key, anObject: item->fValue);
7007	return false;
7008	});
7009	IOLockUnlock(ivars->fLock);
7010
7011	ok = result->serialize(serializer: s);
7012	OSSafeReleaseNULL(result);
7013
7014	return ok;
7015	}
7016
7017	kern_return_t
7018	IOSystemStateNotification::setProperties(OSObject * properties)
7019	{
7020	kern_return_t kr;
7021	OSDictionary * dict;
7022	OSDictionary * schema;
7023	OSDictionary * value;
7024	OSString * itemName;
7025
7026	dict = OSDynamicCast(OSDictionary, properties);
7027	if (!dict) {
7028	return kIOReturnBadArgument;
7029	}
7030
7031	if (!IOCurrentTaskHasEntitlement(kIOSystemStateEntitlement)) {
7032	return kIOReturnNotPermitted;
7033	}
7034
7035	if ((schema = OSDynamicCast(OSDictionary, dict->getObject(kIOStateNotificationItemCreateKey)))) {
7036	itemName = OSDynamicCast(OSString, schema->getObject(kIOStateNotificationNameKey));
7037	kr = StateNotificationItemCreate(itemName, schema);
7038	} else if ((value = OSDynamicCast(OSDictionary, dict->getObject(kIOStateNotificationItemSetKey)))) {
7039	itemName = OSDynamicCast(OSString, value->getObject(kIOStateNotificationNameKey));
7040	itemName->retain();
7041	value->removeObject(kIOStateNotificationNameKey);
7042	kr = StateNotificationItemSet(itemName, value);
7043	itemName->release();
7044	} else {
7045	kr = kIOReturnError;
7046	}
7047
7048	return kr;
7049	}
7050
7051	kern_return_t
7052	IOService::CopySystemStateNotificationService_Impl(IOService ** outService)
7053	{
7054	IOService * service;
7055
7056	service = getSystemStateNotificationService();
7057	service->retain();
7058	*outService = service;
7059
7060	return kIOReturnSuccess;
7061	}
7062
7063	IOStateNotificationItem *
7064	IOService::stateNotificationItemCopy(OSString * itemName, OSDictionary * schema)
7065	{
7066	IOServiceStateChangeVars * ivars = reserved->svars;
7067
7068	const OSSymbol * name;
7069	IOStateNotificationItem * item;
7070
7071	name = OSSymbol::withString(aString: itemName);
7072
7073	IOLockLock(ivars->fLock);
7074	if ((item = (typeof(item))ivars->fItems->getObject(aKey: name))) {
7075	item->retain();
7076	} else {
7077	item = OSTypeAlloc(IOStateNotificationItem);
7078	item->init();
7079	item->fListeners = OSSet::withCapacity(capacity: `16`);
7080
7081	if (schema) {
7082	schema->retain();
7083	} else {
7084	schema = OSDictionary::withCapacity(capacity: `8`);
7085	}
7086	schema->setObject(kIOStateNotificationNameKey, anObject: name);
7087	item->fSchema = schema;
7088	ivars->fItems->setObject(aKey: name, anObject: item);
7089	}
7090	IOLockUnlock(ivars->fLock);
7091
7092	OSSafeReleaseNULL(name);
7093
7094	return item;
7095	}
7096
7097	kern_return_t
7098	IOService::StateNotificationItemCreate_Impl(OSString * itemName, OSDictionary * schema)
7099	{
7100	IOStateNotificationItem * item;
7101
7102	item = stateNotificationItemCopy(itemName, schema);
7103	if (!item) {
7104	return kIOReturnNoMemory;
7105	}
7106	item->release();
7107
7108	return kIOReturnSuccess;
7109	}
7110
7111	kern_return_t
7112	IOService::StateNotificationItemSet_Impl(OSString * itemName, OSDictionary * value)
7113	{
7114	IOServiceStateChangeVars * ivars = reserved->svars;
7115
7116	OSSet * listeners;
7117	IOStateNotificationItem * item;
7118
7119	value->retain();
7120	IOLockLock(ivars->fLock);
7121	item = (typeof(item))ivars->fItems->getObject(aKey: itemName);
7122	OSSafeReleaseNULL(item->fValue);
7123	item->fValue = value;
7124	listeners = NULL;
7125	if (item->fListeners->getCount()) {
7126	listeners = OSSet::withSet(set: item->fListeners);
7127	}
7128	IOLockUnlock(ivars->fLock);
7129
7130	if (listeners) {
7131	listeners->iterateObjects(block: ^bool (OSObject * object) {
7132	IOStateNotificationListener * listener;
7133
7134	listener = (typeof(listener))object;
7135	listener->fHandler();
7136	return false;
7137	});
7138	OSSafeReleaseNULL(listeners);
7139	}
7140
7141	return kIOReturnSuccess;
7142	}
7143
7144	kern_return_t
7145	IOService::StateNotificationItemCopy_Impl(OSString * itemName, OSDictionary ** outValue)
7146	{
7147	IOServiceStateChangeVars * ivars = reserved->svars;
7148
7149	kern_return_t ret;
7150	IOStateNotificationItem * item;
7151	OSDictionary * value;
7152
7153	IOLockLock(ivars->fLock);
7154	item = (typeof(item))ivars->fItems->getObject(aKey: itemName);
7155	if (item) {
7156	value = item->fValue;
7157	} else {
7158	value = NULL;
7159	}
7160	if (!value) {
7161	ret = kIOReturnNotFound;
7162	} else {
7163	value->retain();
7164	ret = kIOReturnSuccess;
7165	}
7166	IOLockUnlock(ivars->fLock);
7167
7168	*outValue = value;
7169
7170	return ret;
7171	}
7172
7173	kern_return_t
7174	IOService::stateNotificationListenerAdd(OSArray * items,
7175	IOStateNotificationListenerRef * outRef,
7176	IOStateNotificationHandler handler)
7177	{
7178	IOServiceStateChangeVars * ivars = reserved->svars;
7179
7180	kern_return_t kr __block;
7181	IOStateNotificationListener * listener;
7182
7183	listener = OSTypeAlloc(IOStateNotificationListener);
7184	listener->init();
7185	listener->fHandler = Block_copy(handler);
7186
7187	kr = kIOReturnSuccess;
7188	items->iterateObjects(block: ^bool (OSObject * object) {
7189	OSString * itemName;
7190	IOStateNotificationItem * item;
7191
7192	itemName = OSDynamicCast(OSString, object);
7193	if (!itemName) {
7194	kr = kIOReturnBadArgument;
7195	return true;
7196	}
7197	item = stateNotificationItemCopy(itemName, NULL);
7198	if (!item) {
7199	kr = kIOReturnNoMemory;
7200	return true;
7201	}
7202	IOLockLock(ivars->fLock);
7203	item->fListeners->setObject(listener);
7204	IOLockUnlock(ivars->fLock);
7205	item->release();
7206	return false;
7207	});
7208
7209	if (kIOReturnSuccess != kr) {
7210	stateNotificationListenerRemove(ref: listener);
7211	OSSafeReleaseNULL(listener);
7212	}
7213	*outRef = listener;
7214
7215	return kr;
7216	}
7217
7218
7219	kern_return_t
7220	IOService::stateNotificationListenerRemove(IOStateNotificationListenerRef ref)
7221	{
7222	IOServiceStateChangeVars * ivars = reserved->svars;
7223
7224	IOStateNotificationListener * listener;
7225	kern_return_t kr;
7226
7227	kr = kIOReturnSuccess;
7228	listener = (typeof(listener))ref;
7229
7230	IOLockLock(ivars->fLock);
7231	ivars->fItems->iterateObjects(block: ^bool (const OSSymbol * key, OSObject * object) {
7232	IOStateNotificationItem * item;
7233
7234	item = (typeof(item))object;
7235	item->fListeners->removeObject(anObject: listener);
7236	return false;
7237	});
7238	IOLockUnlock(ivars->fLock);
7239
7240	return kr;
7241	}
7242
7243
7244
7245	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
7246
7247	kern_return_t
7248	IOWorkGroup::Create_Impl(OSString * name, IOUserClient * userClient, IOWorkGroup ** workgroup)
7249	{
7250	IOWorkGroup * inst = NULL;
7251	IOUserUserClient * uc = NULL;
7252	kern_return_t ret = kIOReturnError;
7253	IOUserServer * us;
7254
7255	if (name == NULL) {
7256	ret = kIOReturnBadArgument;
7257	goto finish;
7258	}
7259
7260	if (name->getLength() > kIOWorkGroupMaxNameLength) {
7261	ret = kIOReturnBadArgument;
7262	goto finish;
7263	}
7264
7265	uc = OSDynamicCast(IOUserUserClient, userClient);
7266	if (uc == NULL) {
7267	ret = kIOReturnBadArgument;
7268	goto finish;
7269	}
7270
7271	inst = OSTypeAlloc(IOWorkGroup);
7272	if (!inst->init()) {
7273	inst->free();
7274	inst = NULL;
7275	ret = kIOReturnNoMemory;
7276	goto finish;
7277	}
7278
7279	us = (typeof(us))thread_iokit_tls_get(index: `0`);
7280	inst->ivars->userServer = OSDynamicCast(IOUserServer, us);
7281
7282	if (inst->ivars->userServer == NULL) {
7283	ret = kIOReturnBadArgument;
7284	goto finish;
7285	}
7286	inst->ivars->userServer->retain();
7287
7288	inst->ivars->name = name;
7289	inst->ivars->name->retain();
7290
7291	inst->ivars->userClient = uc; // no retain
7292
7293	IOLockLock(uc->fLock);
7294	uc->fWorkGroups->setObject(aKey: name, anObject: inst);
7295	IOLockUnlock(uc->fLock);
7296	ret = kIOReturnSuccess;
7297
7298	finish:
7299	if (ret != kIOReturnSuccess) {
7300	OSSafeReleaseNULL(inst);
7301	} else {
7302	*workgroup = inst;
7303	}
7304
7305	return ret;
7306	}
7307
7308	kern_return_t
7309	IOWorkGroup::InvalidateKernel_Impl(IOUserClient * client)
7310	{
7311	IOUserUserClient * uc = OSDynamicCast(IOUserUserClient, client);
7312
7313	if (uc == NULL) {
7314	return kIOReturnBadArgument;
7315	}
7316
7317	if (uc != ivars->userClient) {
7318	return kIOReturnBadArgument;
7319	}
7320
7321	IOLockLock(uc->fLock);
7322	uc->fWorkGroups->removeObject(aKey: ivars->name);
7323	IOLockUnlock(uc->fLock);
7324
7325	return kIOReturnSuccess;
7326	}
7327
7328	kern_return_t
7329	IOWorkGroup::SetWorkGroupPort_Impl(mach_port_t port)
7330	{
7331	return kIOReturnUnsupported;
7332	}
7333
7334	bool
7335	IOWorkGroup::init()
7336	{
7337	if (!OSObject::init()) {
7338	return false;
7339	}
7340	ivars = IOMallocType(IOWorkGroup_IVars);
7341
7342	return true;
7343	}
7344
7345	void
7346	IOWorkGroup::free()
7347	{
7348	if (ivars) {
7349	OSSafeReleaseNULL(ivars->userServer);
7350	OSSafeReleaseNULL(ivars->name);
7351	IOFreeType(ivars, IOWorkGroup_IVars);
7352	}
7353
7354	OSObject::free();
7355	}
7356
7357	/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
7358
7359	kern_return_t
7360	IOEventLink::Create_Impl(OSString * name, IOUserClient * userClient, IOEventLink ** eventlink)
7361	{
7362	IOEventLink * inst = NULL;
7363	IOUserUserClient * uc = NULL;
7364	IOUserServer * us;
7365	kern_return_t ret = kIOReturnError;
7366
7367	if (name == NULL) {
7368	ret = kIOReturnBadArgument;
7369	goto finish;
7370	}
7371
7372	if (name->getLength() > kIOEventLinkMaxNameLength) {
7373	ret = kIOReturnBadArgument;
7374	goto finish;
7375	}
7376
7377	uc = OSDynamicCast(IOUserUserClient, userClient);
7378	if (uc == NULL) {
7379	ret = kIOReturnBadArgument;
7380	goto finish;
7381	}
7382
7383	inst = OSTypeAlloc(IOEventLink);
7384	if (!inst->init()) {
7385	inst->free();
7386	inst = NULL;
7387	ret = kIOReturnNoMemory;
7388	goto finish;
7389	}
7390
7391	us = (typeof(us))thread_iokit_tls_get(index: `0`);
7392	inst->ivars->userServer = OSDynamicCast(IOUserServer, us);
7393
7394	if (inst->ivars->userServer == NULL) {
7395	ret = kIOReturnBadArgument;
7396	goto finish;
7397	}
7398	inst->ivars->userServer->retain();
7399
7400	inst->ivars->name = name;
7401	inst->ivars->name->retain();
7402
7403	inst->ivars->userClient = uc; // no retain
7404
7405	IOLockLock(uc->fLock);
7406	uc->fEventLinks->setObject(aKey: name, anObject: inst);
7407	IOLockUnlock(uc->fLock);
7408
7409	ret = kIOReturnSuccess;
7410
7411	finish:
7412	if (ret != kIOReturnSuccess) {
7413	OSSafeReleaseNULL(inst);
7414	} else {
7415	*eventlink = inst;
7416	}
7417
7418	return ret;
7419	}
7420
7421	kern_return_t
7422	IOEventLink::InvalidateKernel_Impl(IOUserClient * client)
7423	{
7424	IOUserUserClient * uc = OSDynamicCast(IOUserUserClient, client);
7425
7426	if (uc == NULL) {
7427	return kIOReturnBadArgument;
7428	}
7429
7430	if (uc != ivars->userClient) {
7431	return kIOReturnBadArgument;
7432	}
7433
7434	IOLockLock(uc->fLock);
7435	uc->fEventLinks->removeObject(aKey: ivars->name);
7436	IOLockUnlock(uc->fLock);
7437
7438	return kIOReturnSuccess;
7439	}
7440
7441	bool
7442	IOEventLink::init()
7443	{
7444	if (!OSObject::init()) {
7445	return false;
7446	}
7447	ivars = IOMallocType(IOEventLink_IVars);
7448
7449	return true;
7450	}
7451
7452	void
7453	IOEventLink::free()
7454	{
7455	if (ivars) {
7456	OSSafeReleaseNULL(ivars->userServer);
7457	OSSafeReleaseNULL(ivars->name);
7458	IOFreeType(ivars, IOEventLink_IVars);
7459	}
7460
7461	OSObject::free();
7462	}
7463
7464	kern_return_t
7465	IOEventLink::SetEventlinkPort_Impl(mach_port_t port __unused)
7466	{
7467	return kIOReturnUnsupported;
7468	}
7469

Browse the source code of xnu/iokit/Kernel/IOUserServer.cpp