1 | /* |
2 | * Copyright (c) 2006-2009 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <sys/uio.h> |
30 | #include <sys/conf.h> |
31 | |
32 | #include <IOKit/IOLib.h> |
33 | #include <IOKit/IOBSD.h> |
34 | #include <IOKit/IOService.h> |
35 | #include <IOKit/IOPlatformExpert.h> |
36 | #include <IOKit/IOPolledInterface.h> |
37 | #include <IOKit/IOHibernatePrivate.h> |
38 | #include <IOKit/IOBufferMemoryDescriptor.h> |
39 | #include <IOKit/AppleKeyStoreInterface.h> |
40 | #include "IOKitKernelInternal.h" |
41 | |
42 | |
43 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
44 | |
45 | OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject); |
46 | |
47 | OSMetaClassDefineReservedUsed(IOPolledInterface, 0); |
48 | OSMetaClassDefineReservedUnused(IOPolledInterface, 1); |
49 | OSMetaClassDefineReservedUnused(IOPolledInterface, 2); |
50 | OSMetaClassDefineReservedUnused(IOPolledInterface, 3); |
51 | OSMetaClassDefineReservedUnused(IOPolledInterface, 4); |
52 | OSMetaClassDefineReservedUnused(IOPolledInterface, 5); |
53 | OSMetaClassDefineReservedUnused(IOPolledInterface, 6); |
54 | OSMetaClassDefineReservedUnused(IOPolledInterface, 7); |
55 | OSMetaClassDefineReservedUnused(IOPolledInterface, 8); |
56 | OSMetaClassDefineReservedUnused(IOPolledInterface, 9); |
57 | OSMetaClassDefineReservedUnused(IOPolledInterface, 10); |
58 | OSMetaClassDefineReservedUnused(IOPolledInterface, 11); |
59 | OSMetaClassDefineReservedUnused(IOPolledInterface, 12); |
60 | OSMetaClassDefineReservedUnused(IOPolledInterface, 13); |
61 | OSMetaClassDefineReservedUnused(IOPolledInterface, 14); |
62 | OSMetaClassDefineReservedUnused(IOPolledInterface, 15); |
63 | |
64 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
65 | |
66 | #ifndef kIOMediaPreferredBlockSizeKey |
67 | #define kIOMediaPreferredBlockSizeKey "Preferred Block Size" |
68 | #endif |
69 | |
70 | enum { kDefaultIOSize = 128*1024 }; |
71 | |
72 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
73 | |
74 | class IOPolledFilePollers : public OSObject |
75 | { |
76 | OSDeclareDefaultStructors(IOPolledFilePollers) |
77 | |
78 | public: |
79 | IOService * media; |
80 | OSArray * pollers; |
81 | IOBufferMemoryDescriptor * ioBuffer; |
82 | bool abortable; |
83 | bool io; |
84 | IOReturn ioStatus; |
85 | uint32_t openCount; |
86 | |
87 | static IOPolledFilePollers * copyPollers(IOService * media); |
88 | }; |
89 | |
90 | OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject) |
91 | |
92 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
93 | |
94 | IOPolledFilePollers * |
95 | IOPolledFilePollers::copyPollers(IOService * media) |
96 | { |
97 | IOPolledFilePollers * vars; |
98 | IOReturn err; |
99 | IOService * service; |
100 | OSObject * obj; |
101 | IORegistryEntry * next; |
102 | IORegistryEntry * child; |
103 | |
104 | if ((obj = media->copyProperty(kIOPolledInterfaceStackKey))) |
105 | { |
106 | return (OSDynamicCast(IOPolledFilePollers, obj)); |
107 | } |
108 | |
109 | do |
110 | { |
111 | vars = OSTypeAlloc(IOPolledFilePollers); |
112 | vars->init(); |
113 | |
114 | vars->pollers = OSArray::withCapacity(4); |
115 | if (!vars->pollers) |
116 | { |
117 | err = kIOReturnNoMemory; |
118 | break; |
119 | } |
120 | |
121 | next = vars->media = media; |
122 | do |
123 | { |
124 | IOPolledInterface * poller; |
125 | OSObject * obj; |
126 | |
127 | obj = next->getProperty(kIOPolledInterfaceSupportKey); |
128 | if (kOSBooleanFalse == obj) |
129 | { |
130 | vars->pollers->flushCollection(); |
131 | break; |
132 | } |
133 | else if ((poller = OSDynamicCast(IOPolledInterface, obj))) |
134 | vars->pollers->setObject(poller); |
135 | |
136 | if ((service = OSDynamicCast(IOService, next)) |
137 | && service->getDeviceMemory() |
138 | && !vars->pollers->getCount()) break; |
139 | |
140 | child = next; |
141 | } |
142 | while ((next = child->getParentEntry(gIOServicePlane)) |
143 | && child->isParent(next, gIOServicePlane, true)); |
144 | |
145 | if (!vars->pollers->getCount()) |
146 | { |
147 | err = kIOReturnUnsupported; |
148 | break; |
149 | } |
150 | } |
151 | while (false); |
152 | |
153 | media->setProperty(kIOPolledInterfaceStackKey, vars); |
154 | |
155 | return (vars); |
156 | } |
157 | |
158 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
159 | |
160 | static IOReturn |
161 | IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable); |
162 | |
163 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
164 | |
165 | static IOReturn |
166 | IOPolledFilePollersProbe(IOPolledFilePollers * vars) |
167 | { |
168 | IOReturn err = kIOReturnError; |
169 | int32_t idx; |
170 | IOPolledInterface * poller; |
171 | |
172 | for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) |
173 | { |
174 | poller = (IOPolledInterface *) vars->pollers->getObject(idx); |
175 | err = poller->probe(vars->media); |
176 | if (err) |
177 | { |
178 | HIBLOG("IOPolledInterface::probe[%d] 0x%x\n" , idx, err); |
179 | break; |
180 | } |
181 | } |
182 | |
183 | return (err); |
184 | } |
185 | |
186 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
187 | |
188 | IOReturn |
189 | IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable) |
190 | { |
191 | |
192 | IOPolledFilePollers * vars = filevars->pollers; |
193 | IOBufferMemoryDescriptor * ioBuffer; |
194 | IOPolledInterface * poller; |
195 | IOService * next; |
196 | IOReturn err = kIOReturnError; |
197 | int32_t idx; |
198 | |
199 | vars->abortable = abortable; |
200 | ioBuffer = 0; |
201 | |
202 | if (kIOPolledAfterSleepState == state) |
203 | { |
204 | vars->ioStatus = 0; |
205 | vars->io = false; |
206 | } |
207 | (void) IOPolledFilePollersIODone(vars, false); |
208 | |
209 | if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state)) |
210 | { |
211 | ioBuffer = vars->ioBuffer; |
212 | if (!ioBuffer) |
213 | { |
214 | vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut, |
215 | 2 * kDefaultIOSize, page_size); |
216 | if (!ioBuffer) return (kIOReturnNoMemory); |
217 | } |
218 | } |
219 | |
220 | for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) |
221 | { |
222 | poller = (IOPolledInterface *) vars->pollers->getObject(idx); |
223 | err = poller->open(state, ioBuffer); |
224 | if (kIOReturnSuccess != err) |
225 | { |
226 | HIBLOG("IOPolledInterface::open[%d] 0x%x\n" , idx, err); |
227 | break; |
228 | } |
229 | } |
230 | if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state)) |
231 | { |
232 | next = vars->media; |
233 | while (next) |
234 | { |
235 | next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue); |
236 | next = next->getProvider(); |
237 | } |
238 | } |
239 | |
240 | return (err); |
241 | } |
242 | |
243 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
244 | |
245 | IOReturn |
246 | IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state) |
247 | { |
248 | IOPolledFilePollers * vars = filevars->pollers; |
249 | IOPolledInterface * poller; |
250 | IORegistryEntry * next; |
251 | IOReturn err; |
252 | int32_t idx; |
253 | |
254 | (void) IOPolledFilePollersIODone(vars, false); |
255 | |
256 | if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) |
257 | { |
258 | vars->openCount--; |
259 | } |
260 | |
261 | for (idx = 0, err = kIOReturnSuccess; |
262 | (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); |
263 | idx++) |
264 | { |
265 | err = poller->close(state); |
266 | if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state)) |
267 | { |
268 | err = poller->close(kIOPolledBeforeSleepState); |
269 | } |
270 | if (err) HIBLOG("IOPolledInterface::close[%d] 0x%x\n" , idx, err); |
271 | } |
272 | |
273 | if (kIOPolledPostflightState == state) |
274 | { |
275 | next = vars->media; |
276 | while (next) |
277 | { |
278 | next->removeProperty(kIOPolledInterfaceActiveKey); |
279 | next = next->getParentEntry(gIOServicePlane); |
280 | } |
281 | } |
282 | |
283 | if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) do |
284 | { |
285 | if (vars->openCount) break; |
286 | if (vars->ioBuffer) |
287 | { |
288 | vars->ioBuffer->release(); |
289 | vars->ioBuffer = 0; |
290 | } |
291 | } |
292 | while (false); |
293 | |
294 | return (err); |
295 | } |
296 | |
297 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
298 | |
299 | IOReturn IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize) |
300 | { |
301 | return (kIOReturnUnsupported); |
302 | } |
303 | |
304 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
305 | |
306 | IOReturn |
307 | (IOPolledFileIOVars * filevars, |
308 | const uint8_t * key, size_t keySize) |
309 | { |
310 | IOReturn ret = kIOReturnUnsupported; |
311 | IOReturn err; |
312 | int32_t idx; |
313 | IOPolledFilePollers * vars = filevars->pollers; |
314 | IOPolledInterface * poller; |
315 | |
316 | for (idx = 0; |
317 | (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); |
318 | idx++) |
319 | { |
320 | poller = (IOPolledInterface *) vars->pollers->getObject(idx); |
321 | err = poller->setEncryptionKey(key, keySize); |
322 | if (kIOReturnSuccess == err) ret = err; |
323 | } |
324 | |
325 | return (ret); |
326 | } |
327 | |
328 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
329 | |
330 | IOMemoryDescriptor * |
331 | IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars) |
332 | { |
333 | return (vars->pollers->ioBuffer); |
334 | } |
335 | |
336 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
337 | |
338 | static void |
339 | IOPolledIOComplete(void * target, |
340 | void * parameter, |
341 | IOReturn status, |
342 | UInt64 actualByteCount) |
343 | { |
344 | IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter; |
345 | |
346 | vars->ioStatus = status; |
347 | } |
348 | |
349 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
350 | |
351 | static IOReturn |
352 | IOStartPolledIO(IOPolledFilePollers * vars, |
353 | uint32_t operation, uint32_t bufferOffset, |
354 | uint64_t deviceOffset, uint64_t length) |
355 | { |
356 | IOReturn err; |
357 | IOPolledInterface * poller; |
358 | IOPolledCompletion completion; |
359 | |
360 | err = vars->ioStatus; |
361 | if (kIOReturnSuccess != err) return (err); |
362 | |
363 | completion.target = 0; |
364 | completion.action = &IOPolledIOComplete; |
365 | completion.parameter = vars; |
366 | |
367 | vars->ioStatus = -1; |
368 | |
369 | poller = (IOPolledInterface *) vars->pollers->getObject(0); |
370 | err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion); |
371 | if (err) { |
372 | if (kernel_debugger_entry_count) { |
373 | HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n" , 0, err); |
374 | } else { |
375 | HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x" , |
376 | vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err); |
377 | } |
378 | } |
379 | return (err); |
380 | } |
381 | |
382 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
383 | |
384 | static IOReturn |
385 | IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable) |
386 | { |
387 | IOReturn err = kIOReturnSuccess; |
388 | int32_t idx = 0; |
389 | IOPolledInterface * poller; |
390 | AbsoluteTime deadline; |
391 | |
392 | if (!vars->io) return (kIOReturnSuccess); |
393 | |
394 | abortable &= vars->abortable; |
395 | |
396 | clock_interval_to_deadline(2000, kMillisecondScale, &deadline); |
397 | |
398 | while (-1 == vars->ioStatus) |
399 | { |
400 | for (idx = 0; |
401 | (poller = (IOPolledInterface *) vars->pollers->getObject(idx)); |
402 | idx++) |
403 | { |
404 | IOReturn newErr; |
405 | newErr = poller->checkForWork(); |
406 | if ((newErr == kIOReturnAborted) && !abortable) |
407 | newErr = kIOReturnSuccess; |
408 | if (kIOReturnSuccess == err) |
409 | err = newErr; |
410 | } |
411 | if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline))) |
412 | { |
413 | HIBLOG("IOPolledInterface::forced timeout\n" ); |
414 | vars->ioStatus = kIOReturnTimeout; |
415 | } |
416 | } |
417 | vars->io = false; |
418 | |
419 | #if HIBERNATION |
420 | if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) |
421 | { |
422 | err = kIOReturnAborted; |
423 | HIBLOG("IOPolledInterface::checkForWork sw abort\n" ); |
424 | } |
425 | #endif |
426 | |
427 | if (err) |
428 | { |
429 | HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n" , idx, err); |
430 | } |
431 | else |
432 | { |
433 | err = vars->ioStatus; |
434 | if (kIOReturnSuccess != err) HIBLOG("IOPolledInterface::ioStatus 0x%x\n" , err); |
435 | } |
436 | |
437 | return (err); |
438 | } |
439 | |
440 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
441 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
442 | |
443 | struct _OpenFileContext |
444 | { |
445 | OSData * extents; |
446 | uint64_t size; |
447 | }; |
448 | |
449 | static void |
450 | file_extent_callback(void * ref, uint64_t start, uint64_t length) |
451 | { |
452 | _OpenFileContext * ctx = (_OpenFileContext *) ref; |
453 | IOPolledFileExtent extent; |
454 | |
455 | extent.start = start; |
456 | extent.length = length; |
457 | ctx->extents->appendBytes(&extent, sizeof(extent)); |
458 | ctx->size += length; |
459 | } |
460 | |
461 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
462 | |
463 | static IOService * |
464 | IOCopyMediaForDev(dev_t device) |
465 | { |
466 | OSDictionary * matching; |
467 | OSNumber * num; |
468 | OSIterator * iter; |
469 | IOService * result = 0; |
470 | |
471 | matching = IOService::serviceMatching("IOMedia" ); |
472 | if (!matching) |
473 | return (0); |
474 | do |
475 | { |
476 | num = OSNumber::withNumber(major(device), 32); |
477 | if (!num) |
478 | break; |
479 | matching->setObject(kIOBSDMajorKey, num); |
480 | num->release(); |
481 | num = OSNumber::withNumber(minor(device), 32); |
482 | if (!num) |
483 | break; |
484 | matching->setObject(kIOBSDMinorKey, num); |
485 | num->release(); |
486 | if (!num) |
487 | break; |
488 | iter = IOService::getMatchingServices(matching); |
489 | if (iter) |
490 | { |
491 | result = (IOService *) iter->getNextObject(); |
492 | result->retain(); |
493 | iter->release(); |
494 | } |
495 | } |
496 | while (false); |
497 | matching->release(); |
498 | |
499 | return (result); |
500 | } |
501 | |
502 | #define APFSMEDIA_GETHIBERKEY "getHiberKey" |
503 | |
504 | static IOReturn |
505 | IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID, |
506 | uint8_t * volumeCryptKey, size_t * keySize) |
507 | { |
508 | IOReturn err; |
509 | IOService * part; |
510 | OSString * keyUUID = 0; |
511 | OSString * keyStoreUUID = 0; |
512 | uuid_t volumeKeyUUID; |
513 | aks_volume_key_t vek; |
514 | size_t callerKeySize; |
515 | |
516 | static IOService * sKeyStore; |
517 | |
518 | part = IOCopyMediaForDev(block_dev); |
519 | if (!part) return (kIOReturnNotFound); |
520 | |
521 | callerKeySize = *keySize; |
522 | // Try APFS first |
523 | { |
524 | uuid_t volUuid = {0}; |
525 | err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize); |
526 | if (kIOReturnBadArgument == err) |
527 | { |
528 | // apfs fails on buffer size >32 |
529 | *keySize = 32; |
530 | err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize); |
531 | } |
532 | if (err != kIOReturnSuccess) *keySize = 0; |
533 | else |
534 | { |
535 | // No need to create uuid string if it's not requested |
536 | if (pKeyUUID) |
537 | { |
538 | uuid_string_t volUuidStr; |
539 | uuid_unparse(volUuid, volUuidStr); |
540 | *pKeyUUID = OSString::withCString(volUuidStr); |
541 | } |
542 | |
543 | part->release(); |
544 | return kIOReturnSuccess; |
545 | } |
546 | } |
547 | |
548 | // Then old CS path |
549 | err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, |
550 | (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL); |
551 | if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) |
552 | { |
553 | // IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy()); |
554 | |
555 | if (!sKeyStore) |
556 | sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane); |
557 | if (sKeyStore) |
558 | err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID); |
559 | else |
560 | err = kIOReturnNoResources; |
561 | if (kIOReturnSuccess == err) |
562 | err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL); |
563 | if (kIOReturnSuccess != err) |
564 | IOLog("volume key err 0x%x\n" , err); |
565 | else |
566 | { |
567 | if (vek.key.keybytecount <= callerKeySize) *keySize = vek.key.keybytecount; |
568 | bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize); |
569 | } |
570 | bzero(&vek, sizeof(vek)); |
571 | |
572 | if (pKeyUUID) |
573 | { |
574 | // Create a copy because the caller would release it |
575 | *pKeyUUID = OSString::withString(keyUUID); |
576 | } |
577 | } |
578 | |
579 | part->release(); |
580 | return (err); |
581 | } |
582 | |
583 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
584 | |
585 | IOReturn |
586 | IOPolledFileOpen(const char * filename, |
587 | uint32_t flags, |
588 | uint64_t setFileSize, uint64_t fsFreeSize, |
589 | void * write_file_addr, size_t write_file_len, |
590 | IOPolledFileIOVars ** fileVars, |
591 | OSData ** imagePath, |
592 | uint8_t * volumeCryptKey, size_t * keySize) |
593 | { |
594 | IOReturn err = kIOReturnSuccess; |
595 | IOPolledFileIOVars * vars; |
596 | _OpenFileContext ctx; |
597 | OSData * extentsData = NULL; |
598 | OSNumber * num; |
599 | IOService * part = 0; |
600 | dev_t block_dev; |
601 | dev_t image_dev; |
602 | AbsoluteTime startTime, endTime; |
603 | uint64_t nsec; |
604 | |
605 | vars = IONew(IOPolledFileIOVars, 1); |
606 | if (!vars) return (kIOReturnNoMemory); |
607 | bzero(vars, sizeof(*vars)); |
608 | vars->allocated = true; |
609 | |
610 | do |
611 | { |
612 | extentsData = OSData::withCapacity(32); |
613 | ctx.extents = extentsData; |
614 | ctx.size = 0; |
615 | clock_get_uptime(&startTime); |
616 | |
617 | vars->fileRef = kern_open_file_for_direct_io(filename, |
618 | flags, |
619 | &file_extent_callback, &ctx, |
620 | setFileSize, |
621 | fsFreeSize, |
622 | // write file: |
623 | 0, write_file_addr, write_file_len, |
624 | // results |
625 | &block_dev, |
626 | &image_dev, |
627 | &vars->block0, |
628 | &vars->maxiobytes, |
629 | &vars->flags); |
630 | #if 0 |
631 | uint32_t msDelay = (131071 & random()); |
632 | HIBLOG("sleep %d\n" , msDelay); |
633 | IOSleep(msDelay); |
634 | #endif |
635 | clock_get_uptime(&endTime); |
636 | SUB_ABSOLUTETIME(&endTime, &startTime); |
637 | absolutetime_to_nanoseconds(endTime, &nsec); |
638 | |
639 | if (!vars->fileRef) err = kIOReturnNoSpace; |
640 | |
641 | HIBLOG("kern_open_file_for_direct_io took %qd ms\n" , nsec / 1000000ULL); |
642 | if (kIOReturnSuccess != err) break; |
643 | |
644 | HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n" , filename, ctx.size, |
645 | (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1, |
646 | vars->maxiobytes, kIOPolledFileSSD & vars->flags); |
647 | assert(!vars->block0); |
648 | if (extentsData->getLength() < sizeof(IOPolledFileExtent)) |
649 | { |
650 | err = kIOReturnNoSpace; |
651 | break; |
652 | } |
653 | |
654 | vars->fileSize = ctx.size; |
655 | vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy(); |
656 | |
657 | part = IOCopyMediaForDev(image_dev); |
658 | if (!part) |
659 | { |
660 | err = kIOReturnNotFound; |
661 | break; |
662 | } |
663 | |
664 | if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break; |
665 | |
666 | if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) |
667 | vars->blockSize = num->unsigned32BitValue(); |
668 | if (vars->blockSize < 4096) vars->blockSize = 4096; |
669 | |
670 | HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n" , |
671 | major(image_dev), minor(image_dev), (long)vars->blockSize, |
672 | vars->pollers->pollers->getCount()); |
673 | |
674 | OSString * keyUUID = NULL; |
675 | if (volumeCryptKey) |
676 | { |
677 | err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize); |
678 | } |
679 | |
680 | *fileVars = vars; |
681 | vars->fileExtents = extentsData; |
682 | |
683 | // make imagePath |
684 | OSData * data; |
685 | if (imagePath) |
686 | { |
687 | #if defined(__i386__) || defined(__x86_64__) |
688 | char str2[24 + sizeof(uuid_string_t) + 2]; |
689 | |
690 | if (keyUUID) |
691 | snprintf(str2, sizeof(str2), "%qx:%s" , |
692 | vars->extentMap[0].start, keyUUID->getCStringNoCopy()); |
693 | else |
694 | snprintf(str2, sizeof(str2), "%qx" , vars->extentMap[0].start); |
695 | |
696 | err = IOService::getPlatform()->callPlatformFunction( |
697 | gIOCreateEFIDevicePathSymbol, false, |
698 | (void *) part, (void *) str2, |
699 | (void *) (uintptr_t) true, (void *) &data); |
700 | #else |
701 | data = 0; |
702 | err = kIOReturnSuccess; |
703 | #endif |
704 | if (kIOReturnSuccess != err) |
705 | { |
706 | HIBLOG("error 0x%x getting path\n" , err); |
707 | break; |
708 | } |
709 | *imagePath = data; |
710 | } |
711 | |
712 | // Release key UUID if we have one |
713 | if (keyUUID) |
714 | { |
715 | keyUUID->release(); |
716 | keyUUID = NULL; // Just in case |
717 | } |
718 | } |
719 | while (false); |
720 | |
721 | if (kIOReturnSuccess != err) |
722 | { |
723 | HIBLOG("error 0x%x opening polled file\n" , err); |
724 | IOPolledFileClose(&vars, 0, 0, 0, 0, 0); |
725 | if (extentsData) extentsData->release(); |
726 | } |
727 | |
728 | if (part) part->release(); |
729 | |
730 | return (err); |
731 | } |
732 | |
733 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
734 | |
735 | IOReturn |
736 | IOPolledFileClose(IOPolledFileIOVars ** pVars, |
737 | off_t write_offset, void * addr, size_t write_length, |
738 | off_t discard_offset, off_t discard_end) |
739 | { |
740 | IOPolledFileIOVars * vars; |
741 | |
742 | vars = *pVars; |
743 | if (!vars) return(kIOReturnSuccess); |
744 | |
745 | if (vars->fileRef) |
746 | { |
747 | kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length, |
748 | discard_offset, discard_end); |
749 | vars->fileRef = NULL; |
750 | } |
751 | if (vars->fileExtents) |
752 | { |
753 | vars->fileExtents->release(); |
754 | vars->fileExtents = 0; |
755 | } |
756 | if (vars->pollers) |
757 | { |
758 | vars->pollers->release(); |
759 | vars->pollers = 0; |
760 | } |
761 | |
762 | if (vars->allocated) IODelete(vars, IOPolledFileIOVars, 1); |
763 | else bzero(vars, sizeof(IOPolledFileIOVars)); |
764 | *pVars = NULL; |
765 | |
766 | return (kIOReturnSuccess); |
767 | } |
768 | |
769 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
770 | |
771 | IOReturn |
772 | (IOPolledFileIOVars * vars, |
773 | uint32_t openState) |
774 | { |
775 | IOReturn err; |
776 | |
777 | err = kIOReturnSuccess; |
778 | do |
779 | { |
780 | if (!vars->pollers->openCount) |
781 | { |
782 | err = IOPolledFilePollersProbe(vars->pollers); |
783 | if (kIOReturnSuccess != err) break; |
784 | } |
785 | err = IOPolledFilePollersOpen(vars, openState, false); |
786 | if (kIOReturnSuccess != err) break; |
787 | if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState)) |
788 | { |
789 | vars->pollers->openCount++; |
790 | } |
791 | vars->pollers->io = false; |
792 | vars->buffer = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy(); |
793 | vars->bufferHalf = 0; |
794 | vars->bufferOffset = 0; |
795 | vars->bufferSize = (vars->pollers->ioBuffer->getLength() >> 1); |
796 | |
797 | if (vars->maxiobytes < vars->bufferSize) vars->bufferSize = vars->maxiobytes; |
798 | } |
799 | while (false); |
800 | |
801 | if (kIOReturnSuccess != err) HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n" , openState, err); |
802 | |
803 | return (err); |
804 | } |
805 | |
806 | |
807 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
808 | |
809 | IOReturn |
810 | IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position) |
811 | { |
812 | IOPolledFileExtent * extentMap; |
813 | |
814 | extentMap = vars->extentMap; |
815 | |
816 | vars->position = position; |
817 | |
818 | if (position > vars->fileSize) { |
819 | HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n" , vars->position, vars->fileSize); |
820 | return kIOReturnNoSpace; |
821 | } |
822 | |
823 | while (position >= extentMap->length) |
824 | { |
825 | position -= extentMap->length; |
826 | extentMap++; |
827 | } |
828 | |
829 | vars->currentExtent = extentMap; |
830 | vars->extentRemaining = extentMap->length - position; |
831 | vars->extentPosition = vars->position - position; |
832 | |
833 | if (vars->bufferSize <= vars->extentRemaining) |
834 | vars->bufferLimit = vars->bufferSize; |
835 | else |
836 | vars->bufferLimit = vars->extentRemaining; |
837 | |
838 | return (kIOReturnSuccess); |
839 | } |
840 | |
841 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
842 | |
843 | IOReturn |
844 | IOPolledFileWrite(IOPolledFileIOVars * vars, |
845 | const uint8_t * bytes, IOByteCount size, |
846 | IOPolledFileCryptVars * cryptvars) |
847 | { |
848 | IOReturn err = kIOReturnSuccess; |
849 | IOByteCount copy, original_size = size; |
850 | bool flush = false; |
851 | |
852 | do |
853 | { |
854 | if (!bytes && !size) |
855 | { |
856 | // seek to end of block & flush |
857 | size = vars->position & (vars->blockSize - 1); |
858 | if (size) |
859 | size = vars->blockSize - size; |
860 | flush = true; |
861 | // use some garbage for the fill |
862 | bytes = vars->buffer + vars->bufferOffset; |
863 | } |
864 | |
865 | copy = vars->bufferLimit - vars->bufferOffset; |
866 | if (copy > size) |
867 | copy = size; |
868 | else |
869 | flush = true; |
870 | |
871 | if (bytes) |
872 | { |
873 | #if KASAN |
874 | /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to |
875 | * avoid triggering global redzone sanitizer violations when accessing |
876 | * interstices between 'C' structures |
877 | */ |
878 | __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy); |
879 | #else |
880 | bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy); |
881 | #endif |
882 | bytes += copy; |
883 | } |
884 | else |
885 | bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy); |
886 | |
887 | size -= copy; |
888 | vars->bufferOffset += copy; |
889 | vars->position += copy; |
890 | |
891 | if (flush && vars->bufferOffset) |
892 | { |
893 | uint64_t offset = (vars->position - vars->bufferOffset |
894 | - vars->extentPosition + vars->currentExtent->start); |
895 | uint32_t length = (vars->bufferOffset); |
896 | |
897 | #if CRYPTO |
898 | if (cryptvars && vars->encryptStart |
899 | && (vars->position > vars->encryptStart) |
900 | && ((vars->position - length) < vars->encryptEnd)) |
901 | { |
902 | AbsoluteTime startTime, endTime; |
903 | |
904 | uint64_t encryptLen, encryptStart; |
905 | encryptLen = vars->position - vars->encryptStart; |
906 | if (encryptLen > length) |
907 | encryptLen = length; |
908 | encryptStart = length - encryptLen; |
909 | if (vars->position > vars->encryptEnd) |
910 | encryptLen -= (vars->position - vars->encryptEnd); |
911 | |
912 | clock_get_uptime(&startTime); |
913 | |
914 | // encrypt the buffer |
915 | aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart, |
916 | &cryptvars->aes_iv[0], |
917 | encryptLen / AES_BLOCK_SIZE, |
918 | vars->buffer + vars->bufferHalf + encryptStart, |
919 | &cryptvars->ctx.encrypt); |
920 | |
921 | clock_get_uptime(&endTime); |
922 | ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); |
923 | SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); |
924 | vars->cryptBytes += encryptLen; |
925 | |
926 | // save initial vector for following encrypts |
927 | bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE, |
928 | &cryptvars->aes_iv[0], |
929 | AES_BLOCK_SIZE); |
930 | } |
931 | #endif /* CRYPTO */ |
932 | |
933 | err = IOPolledFilePollersIODone(vars->pollers, true); |
934 | if (kIOReturnSuccess != err) |
935 | break; |
936 | |
937 | if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n" , vars->position); |
938 | //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length); |
939 | |
940 | err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length); |
941 | if (kIOReturnSuccess != err) { |
942 | HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n" , |
943 | vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err); |
944 | break; |
945 | } |
946 | vars->pollers->io = true; |
947 | |
948 | vars->extentRemaining -= vars->bufferOffset; |
949 | if (!vars->extentRemaining) |
950 | { |
951 | vars->currentExtent++; |
952 | vars->extentRemaining = vars->currentExtent->length; |
953 | vars->extentPosition = vars->position; |
954 | } |
955 | |
956 | vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; |
957 | vars->bufferOffset = 0; |
958 | if (vars->bufferSize <= vars->extentRemaining) |
959 | vars->bufferLimit = vars->bufferSize; |
960 | else |
961 | vars->bufferLimit = vars->extentRemaining; |
962 | |
963 | if (!vars->extentRemaining) |
964 | { |
965 | err = kIOReturnOverrun; |
966 | break; |
967 | } |
968 | |
969 | flush = false; |
970 | } |
971 | } |
972 | while (size); |
973 | |
974 | return (err); |
975 | } |
976 | |
977 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
978 | |
979 | IOReturn |
980 | IOPolledFileFlush(IOPolledFileIOVars * vars) |
981 | { |
982 | // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms) |
983 | IOReturn err = kIOReturnSuccess; |
984 | |
985 | err = IOPolledFilePollersIODone(vars->pollers, true); |
986 | if (kIOReturnSuccess != err) |
987 | return err; |
988 | |
989 | err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0); |
990 | if (kIOReturnSuccess != err) { |
991 | HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n" , |
992 | vars, vars->pollers, err); |
993 | return err; |
994 | } |
995 | vars->pollers->io = true; |
996 | |
997 | return err; |
998 | } |
999 | |
1000 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1001 | |
1002 | IOReturn |
1003 | IOPolledFileRead(IOPolledFileIOVars * vars, |
1004 | uint8_t * bytes, IOByteCount size, |
1005 | IOPolledFileCryptVars * cryptvars) |
1006 | { |
1007 | IOReturn err = kIOReturnSuccess; |
1008 | IOByteCount copy; |
1009 | |
1010 | // bytesWritten += size; |
1011 | |
1012 | do |
1013 | { |
1014 | copy = vars->bufferLimit - vars->bufferOffset; |
1015 | if (copy > size) |
1016 | copy = size; |
1017 | |
1018 | if (bytes) |
1019 | { |
1020 | #if KASAN |
1021 | __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy); |
1022 | #else |
1023 | bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy); |
1024 | #endif |
1025 | bytes += copy; |
1026 | } |
1027 | size -= copy; |
1028 | vars->bufferOffset += copy; |
1029 | // vars->position += copy; |
1030 | |
1031 | if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) |
1032 | { |
1033 | if (!vars->pollers->io) cryptvars = 0; |
1034 | err = IOPolledFilePollersIODone(vars->pollers, true); |
1035 | if (kIOReturnSuccess != err) |
1036 | break; |
1037 | |
1038 | if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n" , vars->position); |
1039 | |
1040 | vars->position += vars->lastRead; |
1041 | vars->extentRemaining -= vars->lastRead; |
1042 | vars->bufferLimit = vars->lastRead; |
1043 | |
1044 | if (!vars->extentRemaining) |
1045 | { |
1046 | vars->currentExtent++; |
1047 | vars->extentRemaining = vars->currentExtent->length; |
1048 | vars->extentPosition = vars->position; |
1049 | if (!vars->extentRemaining) |
1050 | { |
1051 | err = kIOReturnOverrun; |
1052 | break; |
1053 | } |
1054 | } |
1055 | |
1056 | uint64_t length; |
1057 | uint64_t lastReadLength = vars->lastRead; |
1058 | uint64_t offset = (vars->position |
1059 | - vars->extentPosition + vars->currentExtent->start); |
1060 | if (vars->extentRemaining <= vars->bufferSize) |
1061 | length = vars->extentRemaining; |
1062 | else |
1063 | length = vars->bufferSize; |
1064 | if ((length + vars->position) > vars->readEnd) |
1065 | length = vars->readEnd - vars->position; |
1066 | |
1067 | vars->lastRead = length; |
1068 | if (length) |
1069 | { |
1070 | //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length); |
1071 | err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length); |
1072 | if (kIOReturnSuccess != err) |
1073 | break; |
1074 | vars->pollers->io = true; |
1075 | } |
1076 | |
1077 | vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; |
1078 | vars->bufferOffset = 0; |
1079 | |
1080 | #if CRYPTO |
1081 | if (cryptvars) |
1082 | { |
1083 | uint8_t thisVector[AES_BLOCK_SIZE]; |
1084 | AbsoluteTime startTime, endTime; |
1085 | |
1086 | // save initial vector for following decrypts |
1087 | bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE); |
1088 | bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, |
1089 | &cryptvars->aes_iv[0], AES_BLOCK_SIZE); |
1090 | |
1091 | // decrypt the buffer |
1092 | clock_get_uptime(&startTime); |
1093 | |
1094 | aes_decrypt_cbc(vars->buffer + vars->bufferHalf, |
1095 | &thisVector[0], |
1096 | lastReadLength / AES_BLOCK_SIZE, |
1097 | vars->buffer + vars->bufferHalf, |
1098 | &cryptvars->ctx.decrypt); |
1099 | |
1100 | clock_get_uptime(&endTime); |
1101 | ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); |
1102 | SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); |
1103 | vars->cryptBytes += lastReadLength; |
1104 | } |
1105 | #endif /* CRYPTO */ |
1106 | } |
1107 | } |
1108 | while (size); |
1109 | |
1110 | return (err); |
1111 | } |
1112 | |
1113 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1114 | |
1115 | |