1/*
2 * Copyright (c) 2006-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <sys/uio.h>
30#include <sys/conf.h>
31
32#include <IOKit/IOLib.h>
33#include <IOKit/IOBSD.h>
34#include <IOKit/IOService.h>
35#include <IOKit/IOPlatformExpert.h>
36#include <IOKit/IOPolledInterface.h>
37#include <IOKit/IOHibernatePrivate.h>
38#include <IOKit/IOBufferMemoryDescriptor.h>
39#include <IOKit/AppleKeyStoreInterface.h>
40#include <libkern/c++/OSSharedPtr.h>
41#include "IOKitKernelInternal.h"
42
43#if defined(__arm64__)
44#include <pexpert/arm64/board_config.h>
45#if XNU_MONITOR_PPL_HIB
46#include <IOKit/SEPHibernator.h>
47#endif /* XNU_MONITOR_PPL_HIB */
48#endif /* defined(__arm64__) */
49
50/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
51
52OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
53
54OSMetaClassDefineReservedUsedX86(IOPolledInterface, 0);
55OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
56OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
57OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
58OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
59OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
60OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
61OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
62OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
63OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
64OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
65OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
66OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
67OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
68OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
69OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
70
71/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
72
73#ifndef kIOMediaPreferredBlockSizeKey
74#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
75#endif
76
77enum { kDefaultIOSize = 128 * 1024 };
78
79/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
80
81class IOPolledFilePollers : public OSObject
82{
83 OSDeclareDefaultStructors(IOPolledFilePollers);
84
85public:
86 IOService * media;
87 OSArray * pollers;
88 IOBufferMemoryDescriptor * ioBuffer;
89 bool abortable;
90 bool io;
91 IOReturn ioStatus;
92 uint32_t openCount;
93
94 static IOPolledFilePollers * copyPollers(IOService * media);
95};
96
97OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
98
99/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
100
101IOPolledFilePollers *
102IOPolledFilePollers::copyPollers(IOService * media)
103{
104 IOPolledFilePollers * vars;
105 IOReturn err;
106 IOService * service;
107 OSObject * obj;
108 IORegistryEntry * next;
109 IORegistryEntry * child;
110
111 if ((obj = media->copyProperty(kIOPolledInterfaceStackKey))) {
112 IOPolledFilePollers * ioPFPObj = OSDynamicCast(IOPolledFilePollers, obj);
113 if (!ioPFPObj) {
114 OSSafeReleaseNULL(obj);
115 }
116 return ioPFPObj;
117 }
118
119 do{
120 vars = OSTypeAlloc(IOPolledFilePollers);
121 vars->init();
122
123 vars->pollers = OSArray::withCapacity(capacity: 4);
124 if (!vars->pollers) {
125 err = kIOReturnNoMemory;
126 break;
127 }
128
129 next = vars->media = media;
130 do{
131 IOPolledInterface * poller;
132 OSObject * obj;
133
134 obj = next->getProperty(kIOPolledInterfaceSupportKey);
135 if (kOSBooleanFalse == obj) {
136 vars->pollers->flushCollection();
137 break;
138 } else if ((poller = OSDynamicCast(IOPolledInterface, obj))) {
139 vars->pollers->setObject(poller);
140 }
141
142 if ((service = OSDynamicCast(IOService, next))
143 && service->getDeviceMemory()
144 && !vars->pollers->getCount()) {
145 break;
146 }
147
148 child = next;
149 }while ((next = child->getParentEntry(plane: gIOServicePlane))
150 && child->isParent(parent: next, plane: gIOServicePlane, onlyParent: true));
151
152 if (!vars->pollers->getCount()) {
153 err = kIOReturnUnsupported;
154 break;
155 }
156 }while (false);
157
158 media->setProperty(kIOPolledInterfaceStackKey, anObject: vars);
159
160 return vars;
161}
162
163/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
164
165static IOReturn
166IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
167
168/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
169
170static IOReturn
171IOPolledFilePollersProbe(IOPolledFilePollers * vars)
172{
173 IOReturn err = kIOReturnError;
174 int32_t idx;
175 IOPolledInterface * poller;
176
177 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
178 poller = (IOPolledInterface *) vars->pollers->getObject(index: idx);
179 err = poller->probe(target: vars->media);
180 if (err) {
181 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
182 break;
183 }
184 }
185
186 return err;
187}
188
189/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
190
191IOReturn
192IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
193{
194 IOPolledFilePollers * vars = filevars->pollers;
195 IOBufferMemoryDescriptor * ioBuffer;
196 IOPolledInterface * poller;
197 IOService * next;
198 IOReturn err = kIOReturnError;
199 int32_t idx;
200
201 vars->abortable = abortable;
202 ioBuffer = NULL;
203
204 if (kIOPolledAfterSleepState == state) {
205 vars->ioStatus = 0;
206 vars->io = false;
207 }
208 (void) IOPolledFilePollersIODone(vars, abortable: false);
209
210 if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state)) {
211 ioBuffer = vars->ioBuffer;
212 if (!ioBuffer) {
213 vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(options: kIODirectionInOut,
214 capacity: 2 * kDefaultIOSize, alignment: page_size);
215 if (!ioBuffer) {
216 return kIOReturnNoMemory;
217 }
218 }
219 }
220
221 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
222 poller = (IOPolledInterface *) vars->pollers->getObject(index: idx);
223 err = poller->open(state, buffer: ioBuffer);
224 if (kIOReturnSuccess != err) {
225 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
226 break;
227 }
228 }
229 if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state)) {
230 next = vars->media;
231 while (next) {
232 next->setProperty(kIOPolledInterfaceActiveKey, anObject: kOSBooleanTrue);
233 next = next->getProvider();
234 }
235 }
236
237 return err;
238}
239
240/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
241
242IOReturn
243IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
244{
245 IOPolledFilePollers * vars = filevars->pollers;
246 IOPolledInterface * poller;
247 IORegistryEntry * next;
248 IOReturn err;
249 int32_t idx;
250
251 (void) IOPolledFilePollersIODone(vars, abortable: false);
252
253 if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
254 vars->openCount--;
255 }
256
257 for (idx = 0, err = kIOReturnSuccess;
258 (poller = (IOPolledInterface *) vars->pollers->getObject(index: idx));
259 idx++) {
260 err = poller->close(state);
261 if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state)) {
262 err = poller->close(state: kIOPolledBeforeSleepState);
263 }
264 if (err) {
265 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
266 }
267 }
268
269 if (kIOPolledPostflightState == state) {
270 next = vars->media;
271 while (next) {
272 next->removeProperty(kIOPolledInterfaceActiveKey);
273 next = next->getParentEntry(plane: gIOServicePlane);
274 }
275 }
276
277 if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
278 do{
279 if (vars->openCount) {
280 break;
281 }
282 if (vars->ioBuffer) {
283 vars->ioBuffer->release();
284 vars->ioBuffer = NULL;
285 }
286 }while (false);
287 }
288
289 return err;
290}
291
292/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
293
294IOReturn
295IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize)
296{
297 return kIOReturnUnsupported;
298}
299
300/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
301
302IOReturn
303IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,
304 const uint8_t * key, size_t keySize)
305{
306 IOReturn ret = kIOReturnUnsupported;
307 IOReturn err;
308 int32_t idx;
309 IOPolledFilePollers * vars = filevars->pollers;
310 IOPolledInterface * poller;
311
312 for (idx = 0;
313 (poller = (IOPolledInterface *) vars->pollers->getObject(index: idx));
314 idx++) {
315 poller = (IOPolledInterface *) vars->pollers->getObject(index: idx);
316 err = poller->setEncryptionKey(key, keySize);
317 if (kIOReturnSuccess == err) {
318 ret = err;
319 }
320 }
321
322 return ret;
323}
324
325/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
326
327IOMemoryDescriptor *
328IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
329{
330 return vars->pollers->ioBuffer;
331}
332
333/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
334
335static void
336IOPolledIOComplete(void * target,
337 void * parameter,
338 IOReturn status,
339 UInt64 actualByteCount)
340{
341 IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
342
343 vars->ioStatus = status;
344}
345
346/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
347
348static IOReturn
349IOStartPolledIO(IOPolledFilePollers * vars,
350 uint32_t operation, uint32_t bufferOffset,
351 uint64_t deviceOffset, uint64_t length)
352{
353 IOReturn err;
354 IOPolledInterface * poller;
355 IOPolledCompletion completion;
356
357 err = vars->ioStatus;
358 if (kIOReturnSuccess != err) {
359 return err;
360 }
361
362 completion.target = NULL;
363 completion.action = &IOPolledIOComplete;
364 completion.parameter = vars;
365
366 vars->ioStatus = -1;
367
368 poller = (IOPolledInterface *) vars->pollers->getObject(index: 0);
369 err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
370 if (err) {
371 if (kernel_debugger_entry_count) {
372 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
373 } else {
374 HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
375 vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
376 }
377 }
378 return err;
379}
380
381/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
382
383static IOReturn
384IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
385{
386 IOReturn err = kIOReturnSuccess;
387 int32_t idx = 0;
388 IOPolledInterface * poller;
389 AbsoluteTime deadline;
390
391 if (!vars->io) {
392 return kIOReturnSuccess;
393 }
394
395 abortable &= vars->abortable;
396
397 clock_interval_to_deadline(interval: 2000, scale_factor: kMillisecondScale, result: &deadline);
398
399 while (-1 == vars->ioStatus) {
400 for (idx = 0;
401 (poller = (IOPolledInterface *) vars->pollers->getObject(index: idx));
402 idx++) {
403 IOReturn newErr;
404 newErr = poller->checkForWork();
405 if ((newErr == kIOReturnAborted) && !abortable) {
406 newErr = kIOReturnSuccess;
407 }
408 if (kIOReturnSuccess == err) {
409 err = newErr;
410 }
411 }
412 if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline))) {
413 HIBLOG("IOPolledInterface::forced timeout\n");
414 vars->ioStatus = kIOReturnTimeout;
415 }
416 }
417 vars->io = false;
418
419#if HIBERNATION
420 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) {
421 err = kIOReturnAborted;
422 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
423 }
424#endif
425
426 if (err) {
427 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
428 } else {
429 err = vars->ioStatus;
430 if (kIOReturnSuccess != err) {
431 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
432 }
433 }
434
435 return err;
436}
437
438/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
439/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
440
441struct _OpenFileContext {
442 OSData * extents;
443 uint64_t size;
444};
445
446static void
447file_extent_callback(void * ref, uint64_t start, uint64_t length)
448{
449 _OpenFileContext * ctx = (_OpenFileContext *) ref;
450 IOPolledFileExtent extent;
451
452 extent.start = start;
453 extent.length = length;
454 ctx->extents->appendValue(value: extent);
455 ctx->size += length;
456}
457
458/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
459
460static IOService *
461IOCopyMediaForDev(dev_t device)
462{
463 OSDictionary * matching;
464 OSNumber * num;
465 OSIterator * iter;
466 IOService * result = NULL;
467
468 matching = IOService::serviceMatching(className: "IOMedia");
469 if (!matching) {
470 return NULL;
471 }
472 do{
473 num = OSNumber::withNumber(value: major(x: device), numberOfBits: 32);
474 if (!num) {
475 break;
476 }
477 matching->setObject(kIOBSDMajorKey, anObject: num);
478 num->release();
479 num = OSNumber::withNumber(value: minor(x: device), numberOfBits: 32);
480 if (!num) {
481 break;
482 }
483 matching->setObject(kIOBSDMinorKey, anObject: num);
484 num->release();
485 if (!num) {
486 break;
487 }
488 iter = IOService::getMatchingServices(matching);
489 if (iter) {
490 result = (IOService *) iter->getNextObject();
491 result->retain();
492 iter->release();
493 }
494 }while (false);
495 matching->release();
496
497 return result;
498}
499
500/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
501
502#if defined(__i386__) || defined(__x86_64__)
503#define APFSMEDIA_GETHIBERKEY "getHiberKey"
504
505static IOReturn
506IOGetVolumeCryptKey(dev_t block_dev,
507 LIBKERN_RETURNS_RETAINED OSString ** pKeyUUID,
508 uint8_t * volumeCryptKey,
509 size_t * keySize)
510{
511 IOReturn err;
512 IOService * part;
513 OSString * keyUUID = NULL;
514 OSString * keyStoreUUID = NULL;
515 uuid_t volumeKeyUUID;
516 aks_volume_key_t vek;
517 size_t callerKeySize;
518
519 static IOService * sKeyStore;
520
521 part = IOCopyMediaForDev(block_dev);
522 if (!part) {
523 return kIOReturnNotFound;
524 }
525
526 callerKeySize = *keySize;
527 // Try APFS first
528 {
529 uuid_t volUuid = {0};
530 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
531 if (kIOReturnBadArgument == err) {
532 // apfs fails on buffer size >32
533 *keySize = 32;
534 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
535 }
536 if (err != kIOReturnSuccess) {
537 *keySize = 0;
538 } else {
539 // No need to create uuid string if it's not requested
540 if (pKeyUUID) {
541 uuid_string_t volUuidStr;
542 uuid_unparse(volUuid, volUuidStr);
543 *pKeyUUID = OSString::withCString(volUuidStr);
544 }
545
546 part->release();
547 return kIOReturnSuccess;
548 }
549 }
550
551 // Then old CS path
552 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
553 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
554 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) {
555// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
556
557 if (!sKeyStore) {
558 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
559 }
560 if (sKeyStore) {
561 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
562 } else {
563 err = kIOReturnNoResources;
564 }
565 if (kIOReturnSuccess == err) {
566 err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
567 }
568 if (kIOReturnSuccess != err) {
569 IOLog("volume key err 0x%x\n", err);
570 } else {
571 if (vek.key.keybytecount <= callerKeySize) {
572 *keySize = vek.key.keybytecount;
573 }
574 bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize);
575 }
576 bzero(&vek, sizeof(vek));
577
578 if (pKeyUUID) {
579 // Create a copy because the caller would release it
580 *pKeyUUID = OSString::withString(keyUUID);
581 }
582 }
583
584 part->release();
585 return err;
586}
587#endif /* defined(__i386__) || defined(__x86_64__) */
588
589/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
590
591#if defined(__arm64__)
592static IOReturn
593IOGetHibernationCryptKey(uint8_t * hibernationKey,
594 size_t * keySize,
595 uint32_t *swSeed
596 )
597{
598 return kIOReturnNotFound;
599}
600#endif /* defined(__arm64__) */
601
602/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
603
604IOReturn
605IOPolledFileOpen(const char * filename,
606 uint32_t flags,
607 uint64_t setFileSize, uint64_t fsFreeSize,
608 void * write_file_addr, size_t write_file_len,
609 IOPolledFileIOVars ** fileVars,
610 OSData ** imagePath,
611 uint8_t * volumeCryptKey, size_t * keySize)
612{
613 IOReturn err = kIOReturnSuccess;
614 IOPolledFileIOVars * vars;
615 _OpenFileContext ctx;
616 OSData * extentsData = NULL;
617 OSNumber * num;
618 IOService * part = NULL;
619 dev_t block_dev;
620 dev_t image_dev;
621 AbsoluteTime startTime, endTime;
622 uint64_t nsec;
623
624 vars = IOMallocType(IOPolledFileIOVars);
625 vars->allocated = true;
626
627 do{
628 extentsData = OSData::withCapacity(capacity: 32);
629 ctx.extents = extentsData;
630 ctx.size = 0;
631 clock_get_uptime(result: &startTime);
632
633 vars->fileRef = kern_open_file_for_direct_io(name: filename,
634 flags,
635 callback: &file_extent_callback, callback_ref: &ctx,
636 set_file_size: setFileSize,
637 fs_free_size: fsFreeSize,
638 // write file:
639 write_file_offset: 0, write_file_addr, write_file_len,
640 // results
641 partition_device_result: &block_dev,
642 image_device_result: &image_dev,
643 partitionbase_result: &vars->block0,
644 maxiocount_result: &vars->maxiobytes,
645 oflags: &vars->flags);
646#if 0
647 uint32_t msDelay = (131071 & random());
648 HIBLOG("sleep %d\n", msDelay);
649 IOSleep(msDelay);
650#endif
651 clock_get_uptime(result: &endTime);
652 SUB_ABSOLUTETIME(&endTime, &startTime);
653 absolutetime_to_nanoseconds(abstime: endTime, result: &nsec);
654
655 if (!vars->fileRef) {
656 err = kIOReturnNoSpace;
657 }
658
659 HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
660 if (kIOReturnSuccess != err) {
661 break;
662 }
663
664 HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
665 (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
666 vars->maxiobytes, kIOPolledFileSSD & vars->flags);
667 assert(!vars->block0);
668 if (extentsData->getLength() < sizeof(IOPolledFileExtent)) {
669 err = kIOReturnNoSpace;
670 break;
671 }
672
673 vars->fileSize = ctx.size;
674 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
675
676 part = IOCopyMediaForDev(device: image_dev);
677 if (!part) {
678 err = kIOReturnNotFound;
679 break;
680 }
681
682 if (!(vars->pollers = IOPolledFilePollers::copyPollers(media: part))) {
683 break;
684 }
685
686 if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) {
687 vars->blockSize = num->unsigned32BitValue();
688 }
689 if (vars->blockSize < 4096) {
690 vars->blockSize = 4096;
691 }
692
693 HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
694 major(image_dev), minor(image_dev), (long)vars->blockSize,
695 vars->pollers->pollers->getCount());
696
697 OSString * keyUUID = NULL;
698#if defined(__i386__) || defined(__x86_64__)
699 if (volumeCryptKey) {
700 err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
701 }
702#elif defined(__arm64__)
703 uint32_t swSeed = 0;
704 if (volumeCryptKey) {
705 if (flags & kIOPolledFileHibernate) {
706 err = IOGetHibernationCryptKey(hibernationKey: volumeCryptKey, keySize, swSeed: &swSeed);
707 if (kIOReturnSuccess != err) {
708 HIBLOG("error 0x%x from IOGetHibernationCryptKey\n", err);
709 break;
710 }
711 } else {
712 *keySize = 0;
713 }
714 }
715#else
716 if (volumeCryptKey) {
717 HIBLOG("IOPolledFileOpen: unable to get volumeCryptKey\n");
718 err = kIOReturnNotFound;
719 break;
720 }
721#endif
722
723 *fileVars = vars;
724 vars->fileExtents = extentsData;
725
726 // make imagePath
727 OSData * data = NULL;
728 if (imagePath) {
729#if defined(__i386__) || defined(__x86_64__)
730 char str2[24 + sizeof(uuid_string_t) + 2];
731
732 if (keyUUID) {
733 snprintf(str2, sizeof(str2), "%qx:%s",
734 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
735 } else {
736 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
737 }
738
739 err = IOService::getPlatform()->callPlatformFunction(
740 gIOCreateEFIDevicePathSymbol, false,
741 (void *) part, (void *) str2,
742 (void *) (uintptr_t) true, (void *) &data);
743#elif defined(__arm64__)
744 char str2[26];
745 snprintf(str2, count: sizeof(str2), "%qx:%x", vars->extentMap[0].start, swSeed);
746 data = OSData::withBytes(bytes: str2, numBytes: (unsigned int) strlen(s: str2));
747 err = kIOReturnSuccess;
748#else
749 err = kIOReturnNotFound;
750#endif
751 if (kIOReturnSuccess != err) {
752 HIBLOG("error 0x%x getting path\n", err);
753 OSSafeReleaseNULL(keyUUID);
754 break;
755 }
756 *imagePath = data;
757 }
758
759 // Release key UUID if we have one
760 OSSafeReleaseNULL(keyUUID);
761 }while (false);
762
763 if (kIOReturnSuccess != err) {
764 HIBLOG("error 0x%x opening polled file\n", err);
765 IOPolledFileClose(pVars: &vars, write_offset: 0, NULL, write_length: 0, discard_offset: 0, discard_end: 0, unlink: false);
766 if (extentsData) {
767 extentsData->release();
768 }
769 }
770
771 if (part) {
772 part->release();
773 }
774
775 return err;
776}
777
778IOReturn
779IOPolledFileOpen(const char * filename,
780 uint32_t flags,
781 uint64_t setFileSize, uint64_t fsFreeSize,
782 void * write_file_addr, size_t write_file_len,
783 IOPolledFileIOVars ** fileVars,
784 OSSharedPtr<OSData>& imagePath,
785 uint8_t * volumeCryptKey, size_t * keySize)
786{
787 OSData* imagePathRaw = NULL;
788 IOReturn result = IOPolledFileOpen(filename, flags, setFileSize, fsFreeSize, write_file_addr, write_file_len,
789 fileVars, imagePath: &imagePathRaw, volumeCryptKey, keySize);
790 imagePath.reset(p: imagePathRaw, OSNoRetain);
791 return result;
792}
793
794/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
795
796IOReturn
797IOPolledFileClose(IOPolledFileIOVars ** pVars,
798 off_t write_offset, void * addr, size_t write_length,
799 off_t discard_offset, off_t discard_end, bool unlink)
800{
801 IOPolledFileIOVars * vars;
802
803 vars = *pVars;
804 if (!vars) {
805 return kIOReturnSuccess;
806 }
807
808 if (vars->fileRef) {
809 kern_close_file_for_direct_io(ref: vars->fileRef, write_offset, addr, write_length,
810 discard_offset, discard_end, unlink);
811 vars->fileRef = NULL;
812 }
813 if (vars->fileExtents) {
814 vars->fileExtents->release();
815 vars->fileExtents = NULL;
816 }
817 if (vars->pollers) {
818 vars->pollers->release();
819 vars->pollers = NULL;
820 }
821
822 if (vars->allocated) {
823 IOFreeType(vars, IOPolledFileIOVars);
824 } else {
825 bzero(s: vars, n: sizeof(IOPolledFileIOVars));
826 }
827 *pVars = NULL;
828
829 return kIOReturnSuccess;
830}
831
832/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
833
834IOReturn
835IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
836 uint32_t openState)
837{
838 IOReturn err;
839
840 err = kIOReturnSuccess;
841 do{
842 if (!vars->pollers->openCount) {
843 err = IOPolledFilePollersProbe(vars: vars->pollers);
844 if (kIOReturnSuccess != err) {
845 break;
846 }
847 }
848 err = IOPolledFilePollersOpen(filevars: vars, state: openState, abortable: false);
849 if (kIOReturnSuccess != err) {
850 break;
851 }
852 if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState)) {
853 vars->pollers->openCount++;
854 }
855 vars->pollers->io = false;
856 vars->buffer = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
857 vars->bufferHalf = 0;
858 vars->bufferOffset = 0;
859 assert(vars->pollers->ioBuffer->getLength() <= UINT_MAX);
860 vars->bufferSize = (typeof(vars->bufferSize))(vars->pollers->ioBuffer->getLength() >> 1);
861
862 if (vars->maxiobytes < vars->bufferSize) {
863 vars->bufferSize = (typeof(vars->bufferSize))vars->maxiobytes;
864 }
865 }while (false);
866
867 if (kIOReturnSuccess != err) {
868 HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
869 }
870
871 return err;
872}
873
874
875/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
876
877IOReturn
878IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
879{
880 IOPolledFileExtent * extentMap;
881
882 extentMap = vars->extentMap;
883
884 vars->position = position;
885
886 if (position > vars->fileSize) {
887 HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize);
888 return kIOReturnNoSpace;
889 }
890
891 while (position >= extentMap->length) {
892 position -= extentMap->length;
893 extentMap++;
894 }
895
896 vars->currentExtent = extentMap;
897 vars->extentRemaining = extentMap->length - position;
898 vars->extentPosition = vars->position - position;
899
900 if (vars->bufferSize <= vars->extentRemaining) {
901 vars->bufferLimit = vars->bufferSize;
902 } else {
903 vars->bufferLimit = ((uint32_t) vars->extentRemaining);
904 }
905
906 return kIOReturnSuccess;
907}
908
909/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
910
911IOReturn
912IOPolledFileWrite(IOPolledFileIOVars * vars,
913 const uint8_t * bytes, IOByteCount size,
914 IOPolledFileCryptVars * cryptvars)
915{
916 IOReturn err = kIOReturnSuccess;
917 IOByteCount copy, original_size = size;
918 bool flush = false;
919
920 do{
921 if (!bytes && !size) {
922 // seek to end of block & flush
923 size = vars->position & (vars->blockSize - 1);
924 if (size) {
925 size = vars->blockSize - size;
926 }
927 flush = true;
928 }
929
930 copy = vars->bufferLimit - vars->bufferOffset;
931 if (copy > size) {
932 copy = size;
933 } else {
934 flush = true;
935 }
936
937 if (bytes) {
938#if KASAN
939 /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
940 * avoid triggering global redzone sanitizer violations when accessing
941 * interstices between 'C' structures
942 */
943 __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
944#else
945 bcopy(src: bytes, dst: vars->buffer + vars->bufferHalf + vars->bufferOffset, n: copy);
946#endif
947 bytes += copy;
948 } else {
949 bzero(s: vars->buffer + vars->bufferHalf + vars->bufferOffset, n: copy);
950 }
951
952 size -= copy;
953 vars->bufferOffset += copy;
954 vars->position += copy;
955
956 if (flush && vars->bufferOffset) {
957 uint64_t offset = (vars->position - vars->bufferOffset
958 - vars->extentPosition + vars->currentExtent->start);
959 uint32_t length = (vars->bufferOffset);
960
961#if CRYPTO
962 if (cryptvars && vars->encryptStart
963 && (vars->position > vars->encryptStart)
964 && ((vars->position - length) < vars->encryptEnd)) {
965 AbsoluteTime startTime, endTime;
966
967 uint64_t encryptLen, encryptStart;
968 encryptLen = vars->position - vars->encryptStart;
969 if (encryptLen > length) {
970 encryptLen = length;
971 }
972 encryptStart = length - encryptLen;
973 if (vars->position > vars->encryptEnd) {
974 encryptLen -= (vars->position - vars->encryptEnd);
975 }
976
977 clock_get_uptime(result: &startTime);
978
979 assert(encryptLen <= UINT_MAX);
980 // encrypt the buffer
981 aes_encrypt_cbc(in_blk: vars->buffer + vars->bufferHalf + encryptStart,
982 in_iv: &cryptvars->aes_iv[0],
983 num_blk: (unsigned int) (encryptLen / AES_BLOCK_SIZE),
984 out_blk: vars->buffer + vars->bufferHalf + encryptStart,
985 cx: &cryptvars->ctx.encrypt);
986
987 clock_get_uptime(result: &endTime);
988 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
989 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
990 vars->cryptBytes += encryptLen;
991
992 // save initial vector for following encrypts
993 bcopy(src: vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
994 dst: &cryptvars->aes_iv[0],
995 AES_BLOCK_SIZE);
996 }
997#endif /* CRYPTO */
998
999 err = IOPolledFilePollersIODone(vars: vars->pollers, abortable: true);
1000 if (kIOReturnSuccess != err) {
1001 break;
1002 }
1003
1004 if (vars->position & (vars->blockSize - 1)) {
1005 HIBLOG("misaligned file pos %qx\n", vars->position);
1006 }
1007//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1008
1009 err = IOStartPolledIO(vars: vars->pollers, operation: kIOPolledWrite, bufferOffset: vars->bufferHalf, deviceOffset: offset, length);
1010 if (kIOReturnSuccess != err) {
1011 HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
1012 vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
1013 break;
1014 }
1015 vars->pollers->io = true;
1016
1017 vars->extentRemaining -= vars->bufferOffset;
1018 if (!vars->extentRemaining) {
1019 vars->currentExtent++;
1020 vars->extentRemaining = vars->currentExtent->length;
1021 vars->extentPosition = vars->position;
1022 }
1023
1024 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1025 vars->bufferOffset = 0;
1026 if (vars->bufferSize <= vars->extentRemaining) {
1027 vars->bufferLimit = vars->bufferSize;
1028 } else {
1029 vars->bufferLimit = ((uint32_t) vars->extentRemaining);
1030 }
1031
1032 if (!vars->extentRemaining) {
1033 err = kIOReturnOverrun;
1034 break;
1035 }
1036
1037 flush = false;
1038 }
1039 }while (size);
1040
1041 return err;
1042}
1043
1044/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1045
1046IOReturn
1047IOPolledFileFlush(IOPolledFileIOVars * vars)
1048{
1049 // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
1050 IOReturn err = kIOReturnSuccess;
1051
1052 err = IOPolledFilePollersIODone(vars: vars->pollers, abortable: true);
1053 if (kIOReturnSuccess != err) {
1054 return err;
1055 }
1056
1057 err = IOStartPolledIO(vars: vars->pollers, operation: kIOPolledFlush, bufferOffset: 0, deviceOffset: 0, length: 0);
1058 if (kIOReturnSuccess != err) {
1059 HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
1060 vars, vars->pollers, err);
1061 return err;
1062 }
1063 vars->pollers->io = true;
1064
1065 return err;
1066}
1067
1068/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1069
1070IOReturn
1071IOPolledFileRead(IOPolledFileIOVars * vars,
1072 uint8_t * bytes, IOByteCount size,
1073 IOPolledFileCryptVars * cryptvars)
1074{
1075 IOReturn err = kIOReturnSuccess;
1076 IOByteCount copy;
1077
1078// bytesWritten += size;
1079
1080 do{
1081 copy = vars->bufferLimit - vars->bufferOffset;
1082 if (copy > size) {
1083 copy = size;
1084 }
1085
1086 if (bytes) {
1087#if KASAN
1088 __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1089#else
1090 bcopy(src: vars->buffer + vars->bufferHalf + vars->bufferOffset, dst: bytes, n: copy);
1091#endif
1092 bytes += copy;
1093 }
1094 size -= copy;
1095 vars->bufferOffset += copy;
1096// vars->position += copy;
1097
1098 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) {
1099 if (!vars->pollers->io) {
1100 cryptvars = NULL;
1101 }
1102 err = IOPolledFilePollersIODone(vars: vars->pollers, abortable: true);
1103 if (kIOReturnSuccess != err) {
1104 break;
1105 }
1106
1107 if (vars->position & (vars->blockSize - 1)) {
1108 HIBLOG("misaligned file pos %qx\n", vars->position);
1109 }
1110
1111 vars->position += vars->lastRead;
1112 vars->extentRemaining -= vars->lastRead;
1113 vars->bufferLimit = vars->lastRead;
1114
1115 if (!vars->extentRemaining) {
1116 vars->currentExtent++;
1117 vars->extentRemaining = vars->currentExtent->length;
1118 vars->extentPosition = vars->position;
1119 if (!vars->extentRemaining) {
1120 err = kIOReturnOverrun;
1121 break;
1122 }
1123 }
1124
1125 uint32_t length;
1126 uint32_t lastReadLength = vars->lastRead;
1127 uint64_t offset = (vars->position
1128 - vars->extentPosition + vars->currentExtent->start);
1129 if (vars->extentRemaining <= vars->bufferSize) {
1130 length = ((uint32_t) vars->extentRemaining);
1131 } else {
1132 length = vars->bufferSize;
1133 }
1134 if ((length + vars->position) > vars->readEnd) {
1135 length = ((uint32_t) (vars->readEnd - vars->position));
1136 }
1137
1138 vars->lastRead = length;
1139 if (length) {
1140//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1141 err = IOStartPolledIO(vars: vars->pollers, operation: kIOPolledRead, bufferOffset: vars->bufferHalf, deviceOffset: offset, length);
1142 if (kIOReturnSuccess != err) {
1143 break;
1144 }
1145 vars->pollers->io = true;
1146 }
1147
1148 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1149 vars->bufferOffset = 0;
1150
1151#if CRYPTO
1152 if (cryptvars) {
1153 uint8_t thisVector[AES_BLOCK_SIZE];
1154 AbsoluteTime startTime, endTime;
1155
1156 // save initial vector for following decrypts
1157 bcopy(src: &cryptvars->aes_iv[0], dst: &thisVector[0], AES_BLOCK_SIZE);
1158 bcopy(src: vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1159 dst: &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1160
1161 // decrypt the buffer
1162 clock_get_uptime(result: &startTime);
1163
1164 assert(lastReadLength <= UINT_MAX);
1165 aes_decrypt_cbc(in_blk: vars->buffer + vars->bufferHalf,
1166 in_iv: &thisVector[0],
1167 num_blk: (unsigned int) (lastReadLength / AES_BLOCK_SIZE),
1168 out_blk: vars->buffer + vars->bufferHalf,
1169 cx: &cryptvars->ctx.decrypt);
1170
1171 clock_get_uptime(result: &endTime);
1172 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1173 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1174 vars->cryptBytes += lastReadLength;
1175 }
1176#endif /* CRYPTO */
1177 }
1178 }while (size);
1179
1180 return err;
1181}
1182
1183/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1184