1/*
2 * Copyright (c) 1998-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <IOKit/IOService.h>
30#include <IOKit/IOInterruptEventSource.h>
31#include <IOKit/IOTimerEventSource.h>
32#include <IOKit/IOMapper.h>
33#include "../Kernel/IOServicePrivate.h"
34
35#include <Exclaves/Exclaves.h>
36
37#if CONFIG_EXCLAVES
38#include <mach/exclaves.h>
39#include <Exclaves/IOService.tightbeam.h>
40
41#define EXLOG(x...) do { \
42 if (kIOLogExclaves & gIOKitDebug) \
43 IOLog(x); \
44} while (false)
45
46/* Global IOExclaveProxyState lookup table */
47
48OSDictionary *gExclaveProxyStates;
49IORecursiveLock *gExclaveProxyStateLock;
50const OSSymbol *gDARTMapperFunctionSetActive;
51
52/* IOExclaveProxyState */
53
54class IOExclaveWorkLoopAperture {
55public:
56 IOWorkLoop *workLoop;
57 void
58 closeGate()
59 {
60 workLoop->closeGate();
61 }
62 void
63 openGate()
64 {
65 workLoop->openGate();
66 }
67};
68
69#endif /* CONFIG_EXCLAVES */
70
71struct IOService::IOExclaveProxyState {
72 IOService *service;
73 uint64_t mach_endpoint;
74#if CONFIG_EXCLAVES
75 tb_endpoint_t tb_endpoint;
76 ioservice_ioserviceconcrete client;
77 // ExclaveDriverKit related state
78 bool edk_endpoint_exists;
79 uint64_t edk_mach_endpoint;
80 tb_endpoint_t edk_tb_endpoint;
81 ioservice_ioserviceprivate edk_client;
82 OSDictionary *exclave_interrupts;
83 OSDictionary *exclave_timers;
84 uint32_t nextExclaveTimerId;
85
86 // TODO: implement properly once ExclaveAperture removed
87 IOExclaveWorkLoopAperture *ewla;
88
89 IOLock * exclaveAsyncNotificationEventSourcesLock;
90 OSArray *exclaveAsyncNotificationEventSources;
91
92 // ANE specific upcalls
93 ANEUpcallSetPowerStateHandler aneSetPowerStateUpcallHandler;
94 ANEUpcallWorkHandler aneWorkSubmitUpcallHandler;
95 ANEUpcallWorkHandler aneWorkBeginUpcallHandler;
96 ANEUpcallWorkHandler aneWorkEndUpcallHandler;
97#endif /* CONFIG_EXCLAVES */
98};
99
100#if CONFIG_EXCLAVES
101class IOExclaveProxyStateWrapper : public OSObject {
102 OSDeclareFinalStructors(IOExclaveProxyStateWrapper);
103public:
104 IOService::IOExclaveProxyState *proxyState;
105};
106OSDefineMetaClassAndFinalStructors(IOExclaveProxyStateWrapper, OSObject);
107#endif /* CONFIG_EXCLAVES */
108
109bool
110IOService::exclaveStart(IOService * provider, IOExclaveProxyState ** pRef)
111{
112 IOExclaveProxyState * ref;
113
114 ref = NULL;
115#if CONFIG_EXCLAVES
116 uint64_t serviceID;
117 char key[16];
118 do {
119 OSObject * prop;
120 OSData * data;
121 bool result;
122 uint64_t mach_endpoint = 0;
123 tb_error_t tberr;
124 tb_endpoint_t tb_endpoint;
125 ioservice_ioserviceconcrete client;
126 bool edk_endpoint_exists = false;
127 uint64_t edk_mach_endpoint = 0;
128 tb_endpoint_t edk_tb_endpoint;
129 ioservice_ioserviceprivate edk_client;
130 IOWorkLoop *wl;
131
132 // exit early if Exclaves are not available
133 if (exclaves_get_status() == EXCLAVES_STATUS_NOT_SUPPORTED) {
134 break;
135 }
136
137 prop = provider->copyProperty("exclave-endpoint");
138 if ((data = OSDynamicCast(OSData, prop))) {
139 mach_endpoint = ((uint32_t *)data->getBytesNoCopy())[0];
140 }
141 OSSafeReleaseNULL(prop);
142
143 prop = provider->copyProperty("exclave-edk-endpoint");
144 if ((data = OSDynamicCast(OSData, prop))) {
145 edk_mach_endpoint = ((uint32_t *)data->getBytesNoCopy())[0];
146 edk_endpoint_exists = true;
147 }
148 OSSafeReleaseNULL(prop);
149
150 // Initialize IOServiceConcrete endpoint
151 tb_endpoint = tb_endpoint_create_with_value(TB_TRANSPORT_TYPE_XNU, mach_endpoint, TB_ENDPOINT_OPTIONS_NONE);
152 assert(NULL != tb_endpoint);
153 if (NULL == tb_endpoint) {
154 break;
155 }
156 tberr = ioservice_ioserviceconcrete_init(&client, tb_endpoint);
157 assert(TB_ERROR_SUCCESS == tberr);
158 if (TB_ERROR_SUCCESS != tberr) {
159 break;
160 }
161
162 if (edk_endpoint_exists) {
163 // Initialize IOServicePrivate endpoint
164 edk_tb_endpoint = tb_endpoint_create_with_value(TB_TRANSPORT_TYPE_XNU, edk_mach_endpoint, TB_ENDPOINT_OPTIONS_NONE);
165 assert(NULL != edk_tb_endpoint);
166 if (NULL == edk_tb_endpoint) {
167 printf("%s: ERROR: Failed to create endpoint\n", __func__);
168 break;
169 }
170 tberr = ioservice_ioserviceprivate_init(&edk_client, edk_tb_endpoint);
171 assert(TB_ERROR_SUCCESS == tberr);
172 if (TB_ERROR_SUCCESS != tberr) {
173 printf("%s: ERROR: Failed to init IOServicePrivate\n", __func__);
174 break;
175 }
176 }
177
178 ref = IONewZero(IOExclaveProxyState, 1);
179 if (!ref) {
180 break;
181 }
182 ref->service = this;
183 ref->mach_endpoint = mach_endpoint;
184 ref->tb_endpoint = tb_endpoint;
185 ref->client = client;
186 ref->edk_endpoint_exists = edk_endpoint_exists;
187 if (edk_endpoint_exists) {
188 ref->edk_mach_endpoint = edk_mach_endpoint;
189 ref->edk_tb_endpoint = edk_tb_endpoint;
190 ref->edk_client = edk_client;
191 }
192 ref->exclave_interrupts = OSDictionary::withCapacity(1);
193 ref->exclave_timers = OSDictionary::withCapacity(1);
194 ref->exclaveAsyncNotificationEventSourcesLock = IOLockAlloc();
195
196 // TODO: remove once workloop aperture workaround removed
197 wl = getWorkLoop();
198 if (!wl) {
199 printf("%s ERROR: getWorkLoop failed\n", __func__);
200 break;
201 }
202 ref->ewla = IONew(IOExclaveWorkLoopAperture, 1);
203 if (!ref->ewla) {
204 printf("%s ERROR: exclaveWorkLoopAperture init failed\n", __func__);
205 break;
206 }
207 ref->ewla->workLoop = wl;
208
209 // Add proxy state to global lookup table
210 serviceID = getRegistryEntryID();
211 snprintf(key, sizeof(key), "%llu", serviceID);
212 IOExclaveProxyStateWrapper *wrapper = OSTypeAlloc(IOExclaveProxyStateWrapper);
213 wrapper->proxyState = ref;
214 IORecursiveLockLock(gExclaveProxyStateLock);
215 gExclaveProxyStates->setObject(key, wrapper);
216 IORecursiveLockUnlock(gExclaveProxyStateLock);
217
218 if (ref->edk_endpoint_exists) {
219 // Start() called after lookup table registration in case upcalls are made during exclave start().
220 // Use registry ID as exclave's upcall identifer
221 tberr = ioservice_ioserviceprivate_startprivate(&edk_client, serviceID, &result);
222 if (TB_ERROR_SUCCESS != tberr || !result) {
223 printf("%s ERROR: Failed StartPrivate\n", __func__);
224 // Deregister from lookup table if start() fails
225 IORecursiveLockLock(gExclaveProxyStateLock);
226 gExclaveProxyStates->removeObject(key);
227 IORecursiveLockUnlock(gExclaveProxyStateLock);
228 wrapper->release();
229 IODelete(ref, IOExclaveProxyState, 1);
230 ref = NULL;
231 break;
232 }
233 }
234 } while (false);
235#endif /* CONFIG_EXCLAVES */
236
237 if (!ref) {
238 return false;
239 }
240
241 *pRef = ref;
242 return true;
243}
244
245uint64_t
246IOService::exclaveEndpoint(IOExclaveProxyState * pRef)
247{
248 return pRef->mach_endpoint;
249}
250
251bool
252IOExclaveProxy::start(IOService * provider)
253{
254 bool ok;
255
256 ok = exclaveStart(provider, pRef: &exclaveState);
257
258 return ok;
259}
260
261/* Exclave upcall handlers */
262
263#if CONFIG_EXCLAVES
264
265static IOService::IOExclaveProxyState *
266getProxyStateFromRegistryID(uint64_t id)
267{
268 OSObject *obj = NULL;
269 IOExclaveProxyStateWrapper *wrapper = NULL;
270 char key[15];
271
272 snprintf(key, sizeof(key), "%llu", id);
273 IORecursiveLockLock(gExclaveProxyStateLock);
274 obj = gExclaveProxyStates->getObject(key);
275 IORecursiveLockUnlock(gExclaveProxyStateLock);
276 if (!obj) {
277 printf("%s ERROR: failed to find proxy state\n", __func__);
278 return NULL;
279 }
280
281 wrapper = OSDynamicCast(IOExclaveProxyStateWrapper, obj);
282 if (!wrapper) {
283 printf("%s ERROR: failed to cast IOExclaveProxyStateWrapper\n", __func__);
284 return NULL;
285 }
286
287 if (!wrapper->proxyState) {
288 printf("%s ERROR: IOExclaveProxyStateWrapper contains NULL proxy state\n", __func__);
289 return NULL;
290 }
291
292 return wrapper->proxyState;
293}
294
295bool
296IOExclaveInterruptUpcallHandler(uint64_t id, IOExclaveInterruptUpcallArgs *args)
297{
298 assert(args);
299 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
300 if (!ref || !args) {
301 return false;
302 }
303 ref->service->retain();
304
305 bool res;
306 switch (args->type) {
307 case kIOExclaveInterruptUpcallTypeRegister:
308 // Register interrupt
309 res = ref->service->exclaveRegisterInterrupt(ref, args->index, args->data.register_args.test_irq);
310 break;
311 case kIOExclaveInterruptUpcallTypeRemove:
312 // Remove interrupt
313 res = ref->service->exclaveRemoveInterrupt(ref, args->index);
314 break;
315 case kIOExclaveInterruptUpcallTypeEnable:
316 // Enable/disable interrupt
317 res = ref->service->exclaveEnableInterrupt(ref, args->index, args->data.enable_args.enable);
318 break;
319 default:
320 res = false;
321 printf("%s ERROR: invalid upcall type\n", __func__);
322 }
323
324 if (!res) {
325 printf("%s ERROR: upcall handler type %d failed\n", __func__, args->type);
326 ref->service->release();
327 return false;
328 }
329
330 ref->service->release();
331 return true;
332}
333
334bool
335IOExclaveTimerUpcallHandler(uint64_t id, IOExclaveTimerUpcallArgs *args)
336{
337 assert(args);
338 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
339 if (!ref || !args) {
340 return false;
341 }
342 ref->service->retain();
343
344 bool res;
345 uint32_t timer_id = args->timer_id;
346 switch (args->type) {
347 case kIOExclaveTimerUpcallTypeRegister:
348 // Register timer
349 res = ref->service->exclaveRegisterTimer(ref, &args->timer_id);
350 break;
351 case kIOExclaveTimerUpcallTypeRemove:
352 // Remove timer
353 res = ref->service->exclaveRemoveTimer(ref, timer_id);
354 break;
355 case kIOExclaveTimerUpcallTypeEnable:
356 {
357 // Enable/disable timer
358 bool enable = args->data.enable_args.enable;
359 res = ref->service->exclaveEnableTimer(ref, timer_id, enable);
360 break;
361 }
362 case kIOExclaveTimerUpcallTypeSetTimeout:
363 {
364 // Set timeout
365 uint32_t options = args->data.set_timeout_args.clock_continuous ? kIOTimeOptionsContinuous : 0;
366 AbsoluteTime duration = args->data.set_timeout_args.duration;
367 kern_return_t *kr = &args->data.set_timeout_args.kr;
368 res = ref->service->exclaveTimerSetTimeout(ref, timer_id, options, duration, 0, kr);
369 break;
370 }
371 case kIOExclaveTimerUpcallTypeCancelTimeout:
372 // Cancel timeout
373 res = ref->service->exclaveTimerCancelTimeout(ref, timer_id);
374 break;
375 default:
376 res = false;
377 printf("%s ERROR: invalid upcall type\n", __func__);
378 }
379
380 if (!res) {
381 printf("%s ERROR: upcall handler type %d failed\n", __func__, args->type);
382 ref->service->release();
383 return false;
384 }
385
386 ref->service->release();
387 return true;
388}
389
390bool
391IOExclaveLockWorkloop(uint64_t id, bool lock)
392{
393 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
394 if (!ref) {
395 return false;
396 }
397
398 // Lock or unlock workloop
399 if (lock) {
400 ref->ewla->closeGate();
401 EXLOG("%s locked workloop\n", __func__);
402 } else {
403 ref->ewla->openGate();
404 EXLOG("%s unlocked workloop\n", __func__);
405 }
406 return true;
407}
408
409static void
410getExclaveInterruptKey(int index, char *key, size_t size)
411{
412 snprintf(key, size, "%d", index);
413}
414
415static IOInterruptEventSource *
416getExclaveInterruptEventSource(IOService::IOExclaveProxyState * pRef, int index)
417{
418 OSObject *obj;
419 IOInterruptEventSource *ies;
420 char irqKey[5];
421
422 if (!pRef) {
423 return NULL;
424 }
425
426 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
427 obj = pRef->exclave_interrupts->getObject(irqKey);
428 if (!obj) {
429 return NULL;
430 }
431
432 ies = OSDynamicCast(IOInterruptEventSource, obj);
433 return ies;
434}
435
436// TODO: Remove after testing
437void
438IOExclaveTestSignalInterrupt(thread_call_param_t arg0, __unused thread_call_param_t arg1)
439{
440 EXLOG("%s called\n", __func__);
441
442 // Unpackage params
443 struct IOExclaveTestSignalInterruptParam *params = (struct IOExclaveTestSignalInterruptParam *) arg0;
444 if (params->id == -1 || params->index == -1) {
445 printf("%s: ERROR: id and irq index not initialized\n", __func__);
446 return;
447 }
448
449 uint64_t id = params->id;
450 int index = (int) params->index;
451
452 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
453 if (!ref) {
454 return;
455 }
456 ref->service->retain();
457
458 // Get interrupt
459 char irqKey[5];
460 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
461 OSObject *obj2 = ref->exclave_interrupts->getObject(irqKey);
462 if (!obj2) {
463 printf("%s: ERROR: failed to get ies\n", __func__);
464 ref->service->release();
465 return;
466 }
467
468 IOInterruptEventSource *ies = OSDynamicCast(IOInterruptEventSource, obj2);
469 if (!ies) {
470 printf("%s: ERROR: failed to cast ies\n", __func__);
471 ref->service->release();
472 return;
473 }
474
475 // Signal interrupt
476 ies->interruptOccurred(NULL, NULL, 1);
477
478 ref->service->release();
479}
480
481bool
482IOExclaveAsyncNotificationUpcallHandler(uint64_t id, struct IOExclaveAsyncNotificationUpcallArgs *args)
483{
484 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
485 bool ret = false;
486 if (!ref) {
487 return false;
488 }
489
490 switch (args->type) {
491 case AsyncNotificationUpcallTypeSignal:
492 ret = ref->service->exclaveAsyncNotificationSignal(ref, args->notificationID) == kIOReturnSuccess;
493 break;
494 default:
495 ret = false;
496 break;
497 }
498 return ret;
499}
500
501bool
502IOExclaveMapperOperationUpcallHandler(uint64_t id, IOExclaveMapperOperationUpcallArgs *args)
503{
504 assert(args);
505 IOService *provider = NULL;
506 bool res = false;
507 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
508 if (!ref) {
509 return false;
510 }
511 provider = ref->service->getProvider();
512
513 IOMapper *mapper = IOMapper::copyMapperForDeviceWithIndex(provider, (unsigned int)(args->mapperIndex));
514 if (!mapper) {
515 goto finish;
516 }
517
518 switch (args->type) {
519 case MapperActivate:
520 res = kIOReturnSuccess == mapper->callPlatformFunction(gDARTMapperFunctionSetActive, false, (void *)(true), (void *)(false), NULL, NULL);
521 break;
522 case MapperDeactivate:
523 res = kIOReturnSuccess == mapper->callPlatformFunction(gDARTMapperFunctionSetActive, false, (void *)(false), (void *)(false), NULL, NULL);
524 break;
525 default:
526 break;
527 }
528
529finish:
530 return res;
531}
532
533bool
534IOExclaveANEUpcallHandler(uint64_t id, struct IOExclaveANEUpcallArgs *args, bool *result)
535{
536 IOService::IOExclaveProxyState *ref = getProxyStateFromRegistryID(id);
537 bool ret = false;
538 bool _result = false;
539 if (!ref || !args) {
540 return false;
541 }
542
543 switch (args->type) {
544 case kIOExclaveANEUpcallTypeSetPowerState:
545 if (ref->aneSetPowerStateUpcallHandler) {
546 _result = (ref->aneSetPowerStateUpcallHandler)(
547 args->setpowerstate_args.desired_state
548 );
549 ret = true;
550 } else {
551 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
552 }
553 break;
554 case kIOExclaveANEUpcallTypeWorkSubmit:
555 if (ref->aneWorkSubmitUpcallHandler) {
556 _result = (ref->aneWorkSubmitUpcallHandler)(
557 args->work_args.arg0,
558 args->work_args.arg1,
559 args->work_args.arg2
560 );
561 ret = true;
562 } else {
563 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
564 }
565 break;
566 case kIOExclaveANEUpcallTypeWorkBegin:
567 if (ref->aneWorkBeginUpcallHandler) {
568 _result = (ref->aneWorkBeginUpcallHandler)(
569 args->work_args.arg0,
570 args->work_args.arg1,
571 args->work_args.arg2
572 );
573 ret = true;
574 } else {
575 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
576 }
577 break;
578 case kIOExclaveANEUpcallTypeWorkEnd:
579 if (ref->aneWorkEndUpcallHandler) {
580 _result = (ref->aneWorkEndUpcallHandler)(
581 args->work_args.arg0,
582 args->work_args.arg1,
583 args->work_args.arg2
584 );
585 ret = true;
586 } else {
587 printf("%s: no handler for upcall %d registered\n", __func__, (int)args->type);
588 }
589 break;
590 default:
591 ret = false;
592 break;
593 }
594
595 if (result) {
596 *result = _result;
597 }
598
599 return ret;
600}
601
602/* IOService exclave methods */
603
604#endif /* CONFIG_EXCLAVES */
605
606bool
607IOService::exclaveRegisterInterrupt(IOExclaveProxyState * pRef, int index, bool noProvider = false)
608{
609#if CONFIG_EXCLAVES
610 IOInterruptEventSource *ies = NULL;
611 IOInterruptEventSource::Action action;
612 IOWorkLoop *wl;
613 char irqKey[5];
614
615 assert(getWorkLoop() && getWorkLoop()->inGate());
616
617 if (!pRef) {
618 return false;
619 }
620
621 action = OSMemberFunctionCast(IOInterruptEventSource::Action,
622 this, &IOService::exclaveInterruptOccurred);
623 ies = IOInterruptEventSource::interruptEventSource(this, action, noProvider ? nullptr : getProvider(), index);
624 if (!ies) {
625 return false;
626 }
627
628 wl = getWorkLoop();
629 if (!wl) {
630 ies->release();
631 return false;
632 }
633 if (wl->addEventSource(ies) != kIOReturnSuccess) {
634 ies->release();
635 return false;
636 }
637
638 // Register IOIES in exclave proxy state
639 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
640 pRef->exclave_interrupts->setObject(irqKey, ies);
641
642 EXLOG("%s: IRQ %d register success!\n", __func__, index);
643 return true;
644#else /* CONFIG_EXCLAVES */
645 return false;
646#endif /* CONFIG_EXCLAVES */
647}
648
649bool
650IOService::exclaveRemoveInterrupt(IOExclaveProxyState * pRef, int index)
651{
652#if CONFIG_EXCLAVES
653 IOInterruptEventSource *ies;
654 IOWorkLoop *wl;
655 char irqKey[5];
656
657 assert(getWorkLoop() && getWorkLoop()->inGate());
658
659 if (!pRef) {
660 return false;
661 }
662
663 ies = getExclaveInterruptEventSource(pRef, index);
664 if (!ies) {
665 return false;
666 }
667
668 wl = getWorkLoop();
669 if (!wl) {
670 return false;
671 }
672
673 getExclaveInterruptKey(index, irqKey, sizeof(irqKey));
674 wl->removeEventSource(ies);
675 pRef->exclave_interrupts->removeObject(irqKey);
676 OSSafeReleaseNULL(ies);
677
678 EXLOG("%s: IRQ %d removed successfully\n", __func__, index);
679 return true;
680#else /* CONFIG_EXCLAVES */
681 return false;
682#endif /* CONFIG_EXCLAVES */
683}
684
685bool
686IOService::exclaveEnableInterrupt(IOExclaveProxyState * pRef, int index, bool enable)
687{
688#if CONFIG_EXCLAVES
689 IOInterruptEventSource *ies;
690
691 assert(getWorkLoop() && getWorkLoop()->inGate());
692
693 if (!pRef) {
694 return false;
695 }
696
697 ies = getExclaveInterruptEventSource(pRef, index);
698 if (!ies) {
699 return false;
700 }
701
702 if (enable) {
703 ies->enable();
704 } else {
705 ies->disable();
706 }
707
708 EXLOG("%s: IRQ %s success!\n", __func__, enable ? "enable" : "disable");
709 return true;
710#else /* CONFIG_EXCLAVES */
711 return false;
712#endif /* CONFIG_EXCLAVES */
713}
714
715
716void
717IOService::exclaveInterruptOccurred(IOInterruptEventSource *eventSource, int count)
718{
719#if CONFIG_EXCLAVES
720 tb_error_t tberr;
721 IOService::IOExclaveProxyState *ref;
722
723 if (!eventSource) {
724 printf("%s ERROR: IOInterruptEventSource is null\n", __func__);
725 return;
726 }
727
728 EXLOG("%s id 0x%llx (irq index %d)\n", __func__, getRegistryEntryID(), eventSource->getIntIndex());
729
730 ref = getProxyStateFromRegistryID(getRegistryEntryID());
731 if (!ref) {
732 printf("%s ERROR: failed to get IOExclaveProxyState\n", __func__);
733 return;
734 }
735
736 assert(ref->edk_endpoint_exists);
737 tberr = ioservice_ioserviceprivate_interruptoccurredprivate(&ref->edk_client, eventSource->getIntIndex(), count);
738 assert(TB_ERROR_SUCCESS == tberr);
739 if (TB_ERROR_SUCCESS != tberr) {
740 printf("%s ERROR: tightbeam call failed\n", __func__);
741 return;
742 }
743#endif /* CONFIG_EXCLAVES */
744}
745
746#if CONFIG_EXCLAVES
747static void
748getExclaveTimerKey(uint32_t timer_id, char *key, size_t size)
749{
750 snprintf(key, size, "%d", timer_id);
751}
752
753static IOTimerEventSource *
754getExclaveTimerEventSource(IOService::IOExclaveProxyState * pRef, uint32_t timer_id)
755{
756 OSObject *obj;
757 IOTimerEventSource *tes;
758 char timerKey[5];
759
760 if (!pRef) {
761 return NULL;
762 }
763
764 getExclaveTimerKey(timer_id, timerKey, sizeof(timerKey));
765 obj = pRef->exclave_timers->getObject(timerKey);
766 if (!obj) {
767 return NULL;
768 }
769
770 tes = OSDynamicCast(IOTimerEventSource, obj);
771 return tes;
772}
773#endif /* CONFIG_EXCLAVES */
774
775bool
776IOService::exclaveRegisterTimer(IOExclaveProxyState * pRef, uint32_t *timer_id)
777{
778#if CONFIG_EXCLAVES
779 IOTimerEventSource *tes = NULL;
780 IOTimerEventSource::Action action;
781 IOWorkLoop *wl;
782 char timerKey[5];
783
784 assert(getWorkLoop() && getWorkLoop()->inGate());
785
786 if (!pRef || !timer_id) {
787 return false;
788 }
789
790 action = OSMemberFunctionCast(IOTimerEventSource::Action,
791 this, &IOService::exclaveTimerFired);
792 tes = IOTimerEventSource::timerEventSource(this, action);
793 if (!tes) {
794 return false;
795 }
796
797 wl = getWorkLoop();
798 if (!wl) {
799 tes->release();
800 return false;
801 }
802 if (wl->addEventSource(tes) != kIOReturnSuccess) {
803 tes->release();
804 return false;
805 }
806
807 // Register IOTES in exclave proxy state
808 *timer_id = pRef->nextExclaveTimerId++;
809 getExclaveTimerKey(*timer_id, timerKey, sizeof(timerKey));
810 pRef->exclave_timers->setObject(timerKey, tes);
811
812 EXLOG("%s: timer %u register success!\n", __func__, *timer_id);
813 return true;
814#else /* CONFIG_EXCLAVES */
815 return false;
816#endif /* CONFIG_EXCLAVES */
817}
818
819bool
820IOService::exclaveRemoveTimer(IOExclaveProxyState * pRef, uint32_t timer_id)
821{
822#if CONFIG_EXCLAVES
823 IOTimerEventSource *tes;
824 IOWorkLoop *wl;
825 char timerKey[5];
826
827 assert(getWorkLoop() && getWorkLoop()->inGate());
828
829 if (!pRef) {
830 return false;
831 }
832
833 tes = getExclaveTimerEventSource(pRef, timer_id);
834 if (!tes) {
835 return false;
836 }
837
838 wl = getWorkLoop();
839 if (!wl) {
840 return false;
841 }
842
843 wl->removeEventSource(tes);
844 getExclaveTimerKey(timer_id, timerKey, sizeof(timerKey));
845 pRef->exclave_timers->removeObject(timerKey);
846 OSSafeReleaseNULL(tes);
847
848 EXLOG("%s: timer %u removed successfully\n", __func__, timer_id);
849 return true;
850#else /* CONFIG_EXCLAVES */
851 return false;
852#endif /* CONFIG_EXCLAVES */
853}
854
855bool
856IOService::exclaveEnableTimer(IOExclaveProxyState * pRef, uint32_t timer_id, bool enable)
857{
858#if CONFIG_EXCLAVES
859 IOTimerEventSource *tes;
860
861 assert(getWorkLoop() && getWorkLoop()->inGate());
862
863 if (!pRef) {
864 return false;
865 }
866
867 tes = getExclaveTimerEventSource(pRef, timer_id);
868 if (!tes) {
869 return false;
870 }
871
872 if (enable) {
873 tes->enable();
874 } else {
875 tes->disable();
876 }
877
878 EXLOG("%s: timer %u %s success\n", __func__, timer_id, enable ? "enable" : "disable");
879 return true;
880#else /* CONFIG_EXCLAVES */
881 return false;
882#endif /* CONFIG_EXCLAVES */
883}
884
885bool
886IOService::exclaveTimerSetTimeout(IOExclaveProxyState * pRef, uint32_t timer_id, uint32_t options, AbsoluteTime interval, AbsoluteTime leeway, kern_return_t *kr)
887{
888#if CONFIG_EXCLAVES
889 IOTimerEventSource *tes;
890
891 assert(getWorkLoop() && getWorkLoop()->inGate());
892
893 if (!pRef || !kr) {
894 return false;
895 }
896
897 tes = getExclaveTimerEventSource(pRef, timer_id);
898 if (!tes) {
899 return false;
900 }
901
902 *kr = tes->setTimeout(options, interval, leeway);
903
904 EXLOG("%s: timer %u setTimeout completed (kr %d)\n", __func__, timer_id, *kr);
905 return true;
906#else /* CONFIG_EXCLAVES */
907 return false;
908#endif /* CONFIG_EXCLAVES */
909}
910
911bool
912IOService::exclaveTimerCancelTimeout(IOExclaveProxyState * pRef, uint32_t timer_id)
913{
914#if CONFIG_EXCLAVES
915 IOTimerEventSource *tes;
916
917 assert(getWorkLoop() && getWorkLoop()->inGate());
918
919 if (!pRef) {
920 return false;
921 }
922
923 tes = getExclaveTimerEventSource(pRef, timer_id);
924 if (!tes) {
925 return false;
926 }
927
928 tes->cancelTimeout();
929 EXLOG("%s: timer %u setTimeout success\n", __func__, timer_id);
930 return true;
931#else /* CONFIG_EXCLAVES */
932 return false;
933#endif /* CONFIG_EXCLAVES */
934}
935
936void
937IOService::exclaveTimerFired(IOTimerEventSource *eventSource)
938{
939#if CONFIG_EXCLAVES
940 tb_error_t tberr;
941 IOService::IOExclaveProxyState *ref;
942 __block bool found = false;
943 __block uint32_t timer_id;
944
945 if (!eventSource) {
946 printf("%s ERROR: IOTimerEventSource is null\n", __func__);
947 return;
948 }
949
950 ref = getProxyStateFromRegistryID(getRegistryEntryID());
951 if (!ref) {
952 printf("%s ERROR: failed to get IOExclaveProxyState\n", __func__);
953 return;
954 }
955
956 // Find timer ID
957 ref->exclave_timers->iterateObjects(^bool (const OSSymbol * id, OSObject * obj)
958 {
959 if (obj == eventSource) {
960 found = true;
961 const char *key = id->getCStringNoCopy();
962 timer_id = (uint32_t) strtol(key, NULL, 0);
963 return true;
964 }
965 return false;
966 });
967
968 if (!found) {
969 printf("%s ERROR: Could not find timer ID\n", __func__);
970 return;
971 }
972
973 EXLOG("%s id 0x%llx (timer_id %u)\n", __func__, getRegistryEntryID(), timer_id);
974 assert(ref->edk_endpoint_exists);
975 tberr = ioservice_ioserviceprivate_timerfiredprivate(&ref->edk_client, timer_id);
976 assert(TB_ERROR_SUCCESS == tberr);
977 if (TB_ERROR_SUCCESS != tberr) {
978 printf("%s ERROR: tightbeam call failed\n", __func__);
979 return;
980 }
981#endif /* CONFIG_EXCLAVES */
982}
983
984
985kern_return_t
986IOService::exclaveAsyncNotificationRegister(IOExclaveProxyState * pRef, IOInterruptEventSource *notification, uint32_t *notificationID)
987{
988#if CONFIG_EXCLAVES
989 kern_return_t ret;
990 if (notification == NULL) {
991 return kIOReturnBadArgument;
992 }
993
994 IOLockLock(pRef->exclaveAsyncNotificationEventSourcesLock);
995 if (!pRef->exclaveAsyncNotificationEventSources) {
996 pRef->exclaveAsyncNotificationEventSources = OSArray::withCapacity(1);
997 }
998 if (pRef->exclaveAsyncNotificationEventSources) {
999 *notificationID = (uint32_t) pRef->exclaveAsyncNotificationEventSources->getCount();
1000 pRef->exclaveAsyncNotificationEventSources->setObject(notification);
1001 ret = kIOReturnSuccess;
1002 } else {
1003 ret = kIOReturnNoMemory;
1004 }
1005 IOLockUnlock(pRef->exclaveAsyncNotificationEventSourcesLock);
1006 return ret;
1007#else
1008#pragma unused(pRef, notification, notificationID)
1009 return kIOReturnUnsupported;
1010#endif /* CONFIG_EXCLAVES*/
1011}
1012
1013kern_return_t
1014IOService::exclaveAsyncNotificationSignal(IOExclaveProxyState * pRef, uint32_t notificationID)
1015{
1016#if CONFIG_EXCLAVES
1017 kern_return_t ret;
1018 IOInterruptEventSource *event;
1019
1020 IOLockLock(pRef->exclaveAsyncNotificationEventSourcesLock);
1021 if (pRef->exclaveAsyncNotificationEventSources && (event = OSDynamicCast(IOInterruptEventSource, pRef->exclaveAsyncNotificationEventSources->getObject((unsigned int)notificationID)))) {
1022 event->interruptOccurred(NULL, NULL, 0);
1023 ret = kIOReturnSuccess;
1024 } else {
1025 ret = kIOReturnError;
1026 }
1027 IOLockUnlock(pRef->exclaveAsyncNotificationEventSourcesLock);
1028 return ret;
1029#else
1030#pragma unused(pRef, notificationID)
1031 return kIOReturnUnsupported;
1032#endif /* CONFIG_EXCLAVES */
1033}
1034
1035#if CONFIG_EXCLAVES
1036
1037void
1038exclaves_wait_for_cpu_init()
1039{
1040 OSDictionary *match_dict = IOService::resourceMatching(gIOAllCPUInitializedKey);
1041 IOService *match = IOService::waitForMatchingService(match_dict);
1042 match_dict->release();
1043 match->release();
1044}
1045
1046/* ANE Upcalls */
1047
1048static kern_return_t
1049exclaveRegisterANEUpcallHelper(IOService::IOExclaveProxyState * pRef, IOExclaveANEUpcallType type, IOWorkLoop *wl, void *block)
1050{
1051 void __block * _block = block;
1052
1053 if (!_block) {
1054 return kIOReturnBadArgument;
1055 }
1056
1057 if (!pRef) {
1058 Block_release(_block);
1059 return kIOReturnBadArgument;
1060 }
1061
1062 if (wl != NULL) {
1063 return wl->runActionBlock(^{
1064 switch (type) {
1065 case kIOExclaveANEUpcallTypeSetPowerState:
1066 if (pRef->aneSetPowerStateUpcallHandler) {
1067 Block_release(pRef->aneSetPowerStateUpcallHandler);
1068 }
1069 pRef->aneSetPowerStateUpcallHandler = (ANEUpcallSetPowerStateHandler) _block;
1070 break;
1071 case kIOExclaveANEUpcallTypeWorkSubmit:
1072 if (pRef->aneWorkSubmitUpcallHandler) {
1073 Block_release(pRef->aneWorkSubmitUpcallHandler);
1074 }
1075 pRef->aneWorkSubmitUpcallHandler = (ANEUpcallWorkHandler) _block;
1076 break;
1077 case kIOExclaveANEUpcallTypeWorkBegin:
1078 if (pRef->aneWorkBeginUpcallHandler) {
1079 Block_release(pRef->aneWorkBeginUpcallHandler);
1080 }
1081 pRef->aneWorkBeginUpcallHandler = (ANEUpcallWorkHandler) _block;
1082 break;
1083 case kIOExclaveANEUpcallTypeWorkEnd:
1084 if (pRef->aneWorkEndUpcallHandler) {
1085 Block_release(pRef->aneWorkEndUpcallHandler);
1086 }
1087 pRef->aneWorkEndUpcallHandler = (ANEUpcallWorkHandler) _block;
1088 break;
1089 default:
1090 Block_release(_block);
1091 return kIOReturnBadArgument;
1092 }
1093 return kIOReturnSuccess;
1094 });
1095 } else {
1096 Block_release(_block);
1097 return kIOReturnError;
1098 }
1099}
1100
1101#endif /* CONFIG_EXCLAVES */
1102
1103kern_return_t
1104IOService::exclaveRegisterANEUpcallSetPowerState(IOExclaveProxyState * pRef, ANEUpcallSetPowerStateHandler handler)
1105{
1106#if CONFIG_EXCLAVES
1107 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeSetPowerState, getWorkLoop(), Block_copy(handler));
1108#else
1109#pragma unused(pRef, handler)
1110 return kIOReturnUnsupported;
1111#endif /* CONFIG_EXCLAVES*/
1112}
1113
1114kern_return_t
1115IOService::exclaveRegisterANEUpcallWorkSubmit(IOExclaveProxyState * pRef, ANEUpcallWorkHandler handler)
1116{
1117#if CONFIG_EXCLAVES
1118 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeWorkSubmit, getWorkLoop(), Block_copy(handler));
1119#else
1120#pragma unused(pRef, handler)
1121 return kIOReturnUnsupported;
1122#endif /* CONFIG_EXCLAVES*/
1123}
1124
1125kern_return_t
1126IOService::exclaveRegisterANEUpcallWorkBegin(IOExclaveProxyState * pRef, ANEUpcallWorkHandler handler)
1127{
1128#if CONFIG_EXCLAVES
1129 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeWorkBegin, getWorkLoop(), Block_copy(handler));
1130#else
1131#pragma unused(pRef, handler)
1132 return kIOReturnUnsupported;
1133#endif /* CONFIG_EXCLAVES*/
1134}
1135
1136kern_return_t
1137IOService::exclaveRegisterANEUpcallWorkEnd(IOExclaveProxyState * pRef, ANEUpcallWorkHandler handler)
1138{
1139#if CONFIG_EXCLAVES
1140 return exclaveRegisterANEUpcallHelper(pRef, kIOExclaveANEUpcallTypeWorkEnd, getWorkLoop(), Block_copy(handler));
1141#else
1142#pragma unused(pRef, handler)
1143 return kIOReturnUnsupported;
1144#endif /* CONFIG_EXCLAVES*/
1145}
1146