1 | /* |
2 | * Copyright (c) 2006-2019 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <sys/uio.h> |
30 | #include <sys/conf.h> |
31 | |
32 | #include <IOKit/IOLib.h> |
33 | #include <IOKit/IOBSD.h> |
34 | #include <IOKit/IOService.h> |
35 | #include <IOKit/IOPlatformExpert.h> |
36 | #include <IOKit/IOPolledInterface.h> |
37 | #include <IOKit/IOHibernatePrivate.h> |
38 | #include <IOKit/IOBufferMemoryDescriptor.h> |
39 | #include <IOKit/AppleKeyStoreInterface.h> |
40 | #include <libkern/c++/OSSharedPtr.h> |
41 | #include "IOKitKernelInternal.h" |
42 | |
43 | #if defined(__arm64__) |
44 | #include <pexpert/arm64/board_config.h> |
45 | #if XNU_MONITOR_PPL_HIB |
46 | #include <IOKit/SEPHibernator.h> |
47 | #endif /* XNU_MONITOR_PPL_HIB */ |
48 | #endif /* defined(__arm64__) */ |
49 | |
50 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
51 | |
52 | OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject); |
53 | |
54 | OSMetaClassDefineReservedUsedX86(IOPolledInterface, 0); |
55 | OSMetaClassDefineReservedUnused(IOPolledInterface, 1); |
56 | OSMetaClassDefineReservedUnused(IOPolledInterface, 2); |
57 | OSMetaClassDefineReservedUnused(IOPolledInterface, 3); |
58 | OSMetaClassDefineReservedUnused(IOPolledInterface, 4); |
59 | OSMetaClassDefineReservedUnused(IOPolledInterface, 5); |
60 | OSMetaClassDefineReservedUnused(IOPolledInterface, 6); |
61 | OSMetaClassDefineReservedUnused(IOPolledInterface, 7); |
62 | OSMetaClassDefineReservedUnused(IOPolledInterface, 8); |
63 | OSMetaClassDefineReservedUnused(IOPolledInterface, 9); |
64 | OSMetaClassDefineReservedUnused(IOPolledInterface, 10); |
65 | OSMetaClassDefineReservedUnused(IOPolledInterface, 11); |
66 | OSMetaClassDefineReservedUnused(IOPolledInterface, 12); |
67 | OSMetaClassDefineReservedUnused(IOPolledInterface, 13); |
68 | OSMetaClassDefineReservedUnused(IOPolledInterface, 14); |
69 | OSMetaClassDefineReservedUnused(IOPolledInterface, 15); |
70 | |
71 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
72 | |
73 | #ifndef kIOMediaPreferredBlockSizeKey |
74 | #define kIOMediaPreferredBlockSizeKey "Preferred Block Size" |
75 | #endif |
76 | |
77 | enum { kDefaultIOSize = 128 * 1024 }; |
78 | |
79 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
80 | |
81 | class IOPolledFilePollers : public OSObject |
82 | { |
83 | OSDeclareDefaultStructors(IOPolledFilePollers); |
84 | |
85 | public: |
86 | IOService * media; |
87 | OSArray * pollers; |
88 | IOBufferMemoryDescriptor * ioBuffer; |
89 | bool abortable; |
90 | bool io; |
91 | IOReturn ioStatus; |
92 | uint32_t openCount; |
93 | |
94 | static IOPolledFilePollers * copyPollers(IOService * media); |
95 | }; |
96 | |
97 | OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject) |
98 | |
99 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
100 | |
101 | IOPolledFilePollers * |
102 | IOPolledFilePollers::copyPollers(IOService * media) |
103 | { |
104 | IOPolledFilePollers * vars; |
105 | IOReturn err; |
106 | IOService * service; |
107 | OSObject * obj; |
108 | IORegistryEntry * next; |
109 | IORegistryEntry * child; |
110 | |
111 | if ((obj = media->copyProperty(kIOPolledInterfaceStackKey))) { |
112 | IOPolledFilePollers * ioPFPObj = OSDynamicCast(IOPolledFilePollers, obj); |
113 | if (!ioPFPObj) { |
114 | OSSafeReleaseNULL(obj); |
115 | } |
116 | return ioPFPObj; |
117 | } |
118 | |
119 | do{ |
120 | vars = OSTypeAlloc(IOPolledFilePollers); |
121 | vars->init(); |
122 | |
123 | vars->pollers = OSArray::withCapacity(capacity: 4); |
124 | if (!vars->pollers) { |
125 | err = kIOReturnNoMemory; |
126 | break; |
127 | } |
128 | |
129 | next = vars->media = media; |
130 | do{ |
131 | IOPolledInterface * poller; |
132 | OSObject * obj; |
133 | |
134 | obj = next->getProperty(kIOPolledInterfaceSupportKey); |
135 | if (kOSBooleanFalse == obj) { |
136 | vars->pollers->flushCollection(); |
137 | break; |
138 | } else if ((poller = OSDynamicCast(IOPolledInterface, obj))) { |
139 | vars->pollers->setObject(poller); |
140 | } |
141 | |
142 | if ((service = OSDynamicCast(IOService, next)) |
143 | && service->getDeviceMemory() |
144 | && !vars->pollers->getCount()) { |
145 | break; |
146 | } |
147 | |
148 | child = next; |
149 | }while ((next = child->getParentEntry(plane: gIOServicePlane)) |
150 | && child->isParent(parent: next, plane: gIOServicePlane, onlyParent: true)); |
151 | |
152 | if (!vars->pollers->getCount()) { |
153 | err = kIOReturnUnsupported; |
154 | break; |
155 | } |
156 | }while (false); |
157 | |
158 | media->setProperty(kIOPolledInterfaceStackKey, anObject: vars); |
159 | |
160 | return vars; |
161 | } |
162 | |
163 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
164 | |
165 | static IOReturn |
166 | IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable); |
167 | |
168 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
169 | |
170 | static IOReturn |
171 | IOPolledFilePollersProbe(IOPolledFilePollers * vars) |
172 | { |
173 | IOReturn err = kIOReturnError; |
174 | int32_t idx; |
175 | IOPolledInterface * poller; |
176 | |
177 | for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) { |
178 | poller = (IOPolledInterface *) vars->pollers->getObject(index: idx); |
179 | err = poller->probe(target: vars->media); |
180 | if (err) { |
181 | HIBLOG("IOPolledInterface::probe[%d] 0x%x\n" , idx, err); |
182 | break; |
183 | } |
184 | } |
185 | |
186 | return err; |
187 | } |
188 | |
189 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
190 | |
191 | IOReturn |
192 | IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable) |
193 | { |
194 | IOPolledFilePollers * vars = filevars->pollers; |
195 | IOBufferMemoryDescriptor * ioBuffer; |
196 | IOPolledInterface * poller; |
197 | IOService * next; |
198 | IOReturn err = kIOReturnError; |
199 | int32_t idx; |
200 | |
201 | vars->abortable = abortable; |
202 | ioBuffer = NULL; |
203 | |
204 | if (kIOPolledAfterSleepState == state) { |
205 | vars->ioStatus = 0; |
206 | vars->io = false; |
207 | } |
208 | (void) IOPolledFilePollersIODone(vars, abortable: false); |
209 | |
210 | if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state)) { |
211 | ioBuffer = vars->ioBuffer; |
212 | if (!ioBuffer) { |
213 | vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(options: kIODirectionInOut, |
214 | capacity: 2 * kDefaultIOSize, alignment: page_size); |
215 | if (!ioBuffer) { |
216 | return kIOReturnNoMemory; |
217 | } |
218 | } |
219 | } |
220 | |
221 | for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) { |
222 | poller = (IOPolledInterface *) vars->pollers->getObject(index: idx); |
223 | err = poller->open(state, buffer: ioBuffer); |
224 | if (kIOReturnSuccess != err) { |
225 | HIBLOG("IOPolledInterface::open[%d] 0x%x\n" , idx, err); |
226 | break; |
227 | } |
228 | } |
229 | if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state)) { |
230 | next = vars->media; |
231 | while (next) { |
232 | next->setProperty(kIOPolledInterfaceActiveKey, anObject: kOSBooleanTrue); |
233 | next = next->getProvider(); |
234 | } |
235 | } |
236 | |
237 | return err; |
238 | } |
239 | |
240 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
241 | |
242 | IOReturn |
243 | IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state) |
244 | { |
245 | IOPolledFilePollers * vars = filevars->pollers; |
246 | IOPolledInterface * poller; |
247 | IORegistryEntry * next; |
248 | IOReturn err; |
249 | int32_t idx; |
250 | |
251 | (void) IOPolledFilePollersIODone(vars, abortable: false); |
252 | |
253 | if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) { |
254 | vars->openCount--; |
255 | } |
256 | |
257 | for (idx = 0, err = kIOReturnSuccess; |
258 | (poller = (IOPolledInterface *) vars->pollers->getObject(index: idx)); |
259 | idx++) { |
260 | err = poller->close(state); |
261 | if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state)) { |
262 | err = poller->close(state: kIOPolledBeforeSleepState); |
263 | } |
264 | if (err) { |
265 | HIBLOG("IOPolledInterface::close[%d] 0x%x\n" , idx, err); |
266 | } |
267 | } |
268 | |
269 | if (kIOPolledPostflightState == state) { |
270 | next = vars->media; |
271 | while (next) { |
272 | next->removeProperty(kIOPolledInterfaceActiveKey); |
273 | next = next->getParentEntry(plane: gIOServicePlane); |
274 | } |
275 | } |
276 | |
277 | if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) { |
278 | do{ |
279 | if (vars->openCount) { |
280 | break; |
281 | } |
282 | if (vars->ioBuffer) { |
283 | vars->ioBuffer->release(); |
284 | vars->ioBuffer = NULL; |
285 | } |
286 | }while (false); |
287 | } |
288 | |
289 | return err; |
290 | } |
291 | |
292 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
293 | |
294 | IOReturn |
295 | IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize) |
296 | { |
297 | return kIOReturnUnsupported; |
298 | } |
299 | |
300 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
301 | |
302 | IOReturn |
303 | (IOPolledFileIOVars * filevars, |
304 | const uint8_t * key, size_t keySize) |
305 | { |
306 | IOReturn ret = kIOReturnUnsupported; |
307 | IOReturn err; |
308 | int32_t idx; |
309 | IOPolledFilePollers * vars = filevars->pollers; |
310 | IOPolledInterface * poller; |
311 | |
312 | for (idx = 0; |
313 | (poller = (IOPolledInterface *) vars->pollers->getObject(index: idx)); |
314 | idx++) { |
315 | poller = (IOPolledInterface *) vars->pollers->getObject(index: idx); |
316 | err = poller->setEncryptionKey(key, keySize); |
317 | if (kIOReturnSuccess == err) { |
318 | ret = err; |
319 | } |
320 | } |
321 | |
322 | return ret; |
323 | } |
324 | |
325 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
326 | |
327 | IOMemoryDescriptor * |
328 | IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars) |
329 | { |
330 | return vars->pollers->ioBuffer; |
331 | } |
332 | |
333 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
334 | |
335 | static void |
336 | IOPolledIOComplete(void * target, |
337 | void * parameter, |
338 | IOReturn status, |
339 | UInt64 actualByteCount) |
340 | { |
341 | IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter; |
342 | |
343 | vars->ioStatus = status; |
344 | } |
345 | |
346 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
347 | |
348 | static IOReturn |
349 | IOStartPolledIO(IOPolledFilePollers * vars, |
350 | uint32_t operation, uint32_t bufferOffset, |
351 | uint64_t deviceOffset, uint64_t length) |
352 | { |
353 | IOReturn err; |
354 | IOPolledInterface * poller; |
355 | IOPolledCompletion completion; |
356 | |
357 | err = vars->ioStatus; |
358 | if (kIOReturnSuccess != err) { |
359 | return err; |
360 | } |
361 | |
362 | completion.target = NULL; |
363 | completion.action = &IOPolledIOComplete; |
364 | completion.parameter = vars; |
365 | |
366 | vars->ioStatus = -1; |
367 | |
368 | poller = (IOPolledInterface *) vars->pollers->getObject(index: 0); |
369 | err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion); |
370 | if (err) { |
371 | if (kernel_debugger_entry_count) { |
372 | HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n" , 0, err); |
373 | } else { |
374 | HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x" , |
375 | vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err); |
376 | } |
377 | } |
378 | return err; |
379 | } |
380 | |
381 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
382 | |
383 | static IOReturn |
384 | IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable) |
385 | { |
386 | IOReturn err = kIOReturnSuccess; |
387 | int32_t idx = 0; |
388 | IOPolledInterface * poller; |
389 | AbsoluteTime deadline; |
390 | |
391 | if (!vars->io) { |
392 | return kIOReturnSuccess; |
393 | } |
394 | |
395 | abortable &= vars->abortable; |
396 | |
397 | clock_interval_to_deadline(interval: 2000, scale_factor: kMillisecondScale, result: &deadline); |
398 | |
399 | while (-1 == vars->ioStatus) { |
400 | for (idx = 0; |
401 | (poller = (IOPolledInterface *) vars->pollers->getObject(index: idx)); |
402 | idx++) { |
403 | IOReturn newErr; |
404 | newErr = poller->checkForWork(); |
405 | if ((newErr == kIOReturnAborted) && !abortable) { |
406 | newErr = kIOReturnSuccess; |
407 | } |
408 | if (kIOReturnSuccess == err) { |
409 | err = newErr; |
410 | } |
411 | } |
412 | if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline))) { |
413 | HIBLOG("IOPolledInterface::forced timeout\n" ); |
414 | vars->ioStatus = kIOReturnTimeout; |
415 | } |
416 | } |
417 | vars->io = false; |
418 | |
419 | #if HIBERNATION |
420 | if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) { |
421 | err = kIOReturnAborted; |
422 | HIBLOG("IOPolledInterface::checkForWork sw abort\n" ); |
423 | } |
424 | #endif |
425 | |
426 | if (err) { |
427 | HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n" , idx, err); |
428 | } else { |
429 | err = vars->ioStatus; |
430 | if (kIOReturnSuccess != err) { |
431 | HIBLOG("IOPolledInterface::ioStatus 0x%x\n" , err); |
432 | } |
433 | } |
434 | |
435 | return err; |
436 | } |
437 | |
438 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
439 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
440 | |
441 | struct _OpenFileContext { |
442 | OSData * extents; |
443 | uint64_t size; |
444 | }; |
445 | |
446 | static void |
447 | file_extent_callback(void * ref, uint64_t start, uint64_t length) |
448 | { |
449 | _OpenFileContext * ctx = (_OpenFileContext *) ref; |
450 | IOPolledFileExtent extent; |
451 | |
452 | extent.start = start; |
453 | extent.length = length; |
454 | ctx->extents->appendValue(value: extent); |
455 | ctx->size += length; |
456 | } |
457 | |
458 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
459 | |
460 | static IOService * |
461 | IOCopyMediaForDev(dev_t device) |
462 | { |
463 | OSDictionary * matching; |
464 | OSNumber * num; |
465 | OSIterator * iter; |
466 | IOService * result = NULL; |
467 | |
468 | matching = IOService::serviceMatching(className: "IOMedia" ); |
469 | if (!matching) { |
470 | return NULL; |
471 | } |
472 | do{ |
473 | num = OSNumber::withNumber(value: major(x: device), numberOfBits: 32); |
474 | if (!num) { |
475 | break; |
476 | } |
477 | matching->setObject(kIOBSDMajorKey, anObject: num); |
478 | num->release(); |
479 | num = OSNumber::withNumber(value: minor(x: device), numberOfBits: 32); |
480 | if (!num) { |
481 | break; |
482 | } |
483 | matching->setObject(kIOBSDMinorKey, anObject: num); |
484 | num->release(); |
485 | if (!num) { |
486 | break; |
487 | } |
488 | iter = IOService::getMatchingServices(matching); |
489 | if (iter) { |
490 | result = (IOService *) iter->getNextObject(); |
491 | result->retain(); |
492 | iter->release(); |
493 | } |
494 | }while (false); |
495 | matching->release(); |
496 | |
497 | return result; |
498 | } |
499 | |
500 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
501 | |
502 | #if defined(__i386__) || defined(__x86_64__) |
503 | #define APFSMEDIA_GETHIBERKEY "getHiberKey" |
504 | |
505 | static IOReturn |
506 | IOGetVolumeCryptKey(dev_t block_dev, |
507 | LIBKERN_RETURNS_RETAINED OSString ** pKeyUUID, |
508 | uint8_t * volumeCryptKey, |
509 | size_t * keySize) |
510 | { |
511 | IOReturn err; |
512 | IOService * part; |
513 | OSString * keyUUID = NULL; |
514 | OSString * keyStoreUUID = NULL; |
515 | uuid_t volumeKeyUUID; |
516 | aks_volume_key_t vek; |
517 | size_t callerKeySize; |
518 | |
519 | static IOService * sKeyStore; |
520 | |
521 | part = IOCopyMediaForDev(block_dev); |
522 | if (!part) { |
523 | return kIOReturnNotFound; |
524 | } |
525 | |
526 | callerKeySize = *keySize; |
527 | // Try APFS first |
528 | { |
529 | uuid_t volUuid = {0}; |
530 | err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize); |
531 | if (kIOReturnBadArgument == err) { |
532 | // apfs fails on buffer size >32 |
533 | *keySize = 32; |
534 | err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize); |
535 | } |
536 | if (err != kIOReturnSuccess) { |
537 | *keySize = 0; |
538 | } else { |
539 | // No need to create uuid string if it's not requested |
540 | if (pKeyUUID) { |
541 | uuid_string_t volUuidStr; |
542 | uuid_unparse(volUuid, volUuidStr); |
543 | *pKeyUUID = OSString::withCString(volUuidStr); |
544 | } |
545 | |
546 | part->release(); |
547 | return kIOReturnSuccess; |
548 | } |
549 | } |
550 | |
551 | // Then old CS path |
552 | err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, |
553 | (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL); |
554 | if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) { |
555 | // IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy()); |
556 | |
557 | if (!sKeyStore) { |
558 | sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane); |
559 | } |
560 | if (sKeyStore) { |
561 | err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID); |
562 | } else { |
563 | err = kIOReturnNoResources; |
564 | } |
565 | if (kIOReturnSuccess == err) { |
566 | err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL); |
567 | } |
568 | if (kIOReturnSuccess != err) { |
569 | IOLog("volume key err 0x%x\n" , err); |
570 | } else { |
571 | if (vek.key.keybytecount <= callerKeySize) { |
572 | *keySize = vek.key.keybytecount; |
573 | } |
574 | bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize); |
575 | } |
576 | bzero(&vek, sizeof(vek)); |
577 | |
578 | if (pKeyUUID) { |
579 | // Create a copy because the caller would release it |
580 | *pKeyUUID = OSString::withString(keyUUID); |
581 | } |
582 | } |
583 | |
584 | part->release(); |
585 | return err; |
586 | } |
587 | #endif /* defined(__i386__) || defined(__x86_64__) */ |
588 | |
589 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
590 | |
591 | #if defined(__arm64__) |
592 | static IOReturn |
593 | IOGetHibernationCryptKey(uint8_t * hibernationKey, |
594 | size_t * keySize, |
595 | uint32_t *swSeed |
596 | ) |
597 | { |
598 | return kIOReturnNotFound; |
599 | } |
600 | #endif /* defined(__arm64__) */ |
601 | |
602 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
603 | |
604 | IOReturn |
605 | IOPolledFileOpen(const char * filename, |
606 | uint32_t flags, |
607 | uint64_t setFileSize, uint64_t fsFreeSize, |
608 | void * write_file_addr, size_t write_file_len, |
609 | IOPolledFileIOVars ** fileVars, |
610 | OSData ** imagePath, |
611 | uint8_t * volumeCryptKey, size_t * keySize) |
612 | { |
613 | IOReturn err = kIOReturnSuccess; |
614 | IOPolledFileIOVars * vars; |
615 | _OpenFileContext ctx; |
616 | OSData * extentsData = NULL; |
617 | OSNumber * num; |
618 | IOService * part = NULL; |
619 | dev_t block_dev; |
620 | dev_t image_dev; |
621 | AbsoluteTime startTime, endTime; |
622 | uint64_t nsec; |
623 | |
624 | vars = IOMallocType(IOPolledFileIOVars); |
625 | vars->allocated = true; |
626 | |
627 | do{ |
628 | extentsData = OSData::withCapacity(capacity: 32); |
629 | ctx.extents = extentsData; |
630 | ctx.size = 0; |
631 | clock_get_uptime(result: &startTime); |
632 | |
633 | vars->fileRef = kern_open_file_for_direct_io(name: filename, |
634 | flags, |
635 | callback: &file_extent_callback, callback_ref: &ctx, |
636 | set_file_size: setFileSize, |
637 | fs_free_size: fsFreeSize, |
638 | // write file: |
639 | write_file_offset: 0, write_file_addr, write_file_len, |
640 | // results |
641 | partition_device_result: &block_dev, |
642 | image_device_result: &image_dev, |
643 | partitionbase_result: &vars->block0, |
644 | maxiocount_result: &vars->maxiobytes, |
645 | oflags: &vars->flags); |
646 | #if 0 |
647 | uint32_t msDelay = (131071 & random()); |
648 | HIBLOG("sleep %d\n" , msDelay); |
649 | IOSleep(msDelay); |
650 | #endif |
651 | clock_get_uptime(result: &endTime); |
652 | SUB_ABSOLUTETIME(&endTime, &startTime); |
653 | absolutetime_to_nanoseconds(abstime: endTime, result: &nsec); |
654 | |
655 | if (!vars->fileRef) { |
656 | err = kIOReturnNoSpace; |
657 | } |
658 | |
659 | HIBLOG("kern_open_file_for_direct_io took %qd ms\n" , nsec / 1000000ULL); |
660 | if (kIOReturnSuccess != err) { |
661 | break; |
662 | } |
663 | |
664 | HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n" , filename, ctx.size, |
665 | (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1, |
666 | vars->maxiobytes, kIOPolledFileSSD & vars->flags); |
667 | assert(!vars->block0); |
668 | if (extentsData->getLength() < sizeof(IOPolledFileExtent)) { |
669 | err = kIOReturnNoSpace; |
670 | break; |
671 | } |
672 | |
673 | vars->fileSize = ctx.size; |
674 | vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy(); |
675 | |
676 | part = IOCopyMediaForDev(device: image_dev); |
677 | if (!part) { |
678 | err = kIOReturnNotFound; |
679 | break; |
680 | } |
681 | |
682 | if (!(vars->pollers = IOPolledFilePollers::copyPollers(media: part))) { |
683 | break; |
684 | } |
685 | |
686 | if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) { |
687 | vars->blockSize = num->unsigned32BitValue(); |
688 | } |
689 | if (vars->blockSize < 4096) { |
690 | vars->blockSize = 4096; |
691 | } |
692 | |
693 | HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n" , |
694 | major(image_dev), minor(image_dev), (long)vars->blockSize, |
695 | vars->pollers->pollers->getCount()); |
696 | |
697 | OSString * keyUUID = NULL; |
698 | #if defined(__i386__) || defined(__x86_64__) |
699 | if (volumeCryptKey) { |
700 | err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize); |
701 | } |
702 | #elif defined(__arm64__) |
703 | uint32_t swSeed = 0; |
704 | if (volumeCryptKey) { |
705 | if (flags & kIOPolledFileHibernate) { |
706 | err = IOGetHibernationCryptKey(hibernationKey: volumeCryptKey, keySize, swSeed: &swSeed); |
707 | if (kIOReturnSuccess != err) { |
708 | HIBLOG("error 0x%x from IOGetHibernationCryptKey\n" , err); |
709 | break; |
710 | } |
711 | } else { |
712 | *keySize = 0; |
713 | } |
714 | } |
715 | #else |
716 | if (volumeCryptKey) { |
717 | HIBLOG("IOPolledFileOpen: unable to get volumeCryptKey\n" ); |
718 | err = kIOReturnNotFound; |
719 | break; |
720 | } |
721 | #endif |
722 | |
723 | *fileVars = vars; |
724 | vars->fileExtents = extentsData; |
725 | |
726 | // make imagePath |
727 | OSData * data = NULL; |
728 | if (imagePath) { |
729 | #if defined(__i386__) || defined(__x86_64__) |
730 | char str2[24 + sizeof(uuid_string_t) + 2]; |
731 | |
732 | if (keyUUID) { |
733 | snprintf(str2, sizeof(str2), "%qx:%s" , |
734 | vars->extentMap[0].start, keyUUID->getCStringNoCopy()); |
735 | } else { |
736 | snprintf(str2, sizeof(str2), "%qx" , vars->extentMap[0].start); |
737 | } |
738 | |
739 | err = IOService::getPlatform()->callPlatformFunction( |
740 | gIOCreateEFIDevicePathSymbol, false, |
741 | (void *) part, (void *) str2, |
742 | (void *) (uintptr_t) true, (void *) &data); |
743 | #elif defined(__arm64__) |
744 | char str2[26]; |
745 | snprintf(str2, count: sizeof(str2), "%qx:%x" , vars->extentMap[0].start, swSeed); |
746 | data = OSData::withBytes(bytes: str2, numBytes: (unsigned int) strlen(s: str2)); |
747 | err = kIOReturnSuccess; |
748 | #else |
749 | err = kIOReturnNotFound; |
750 | #endif |
751 | if (kIOReturnSuccess != err) { |
752 | HIBLOG("error 0x%x getting path\n" , err); |
753 | OSSafeReleaseNULL(keyUUID); |
754 | break; |
755 | } |
756 | *imagePath = data; |
757 | } |
758 | |
759 | // Release key UUID if we have one |
760 | OSSafeReleaseNULL(keyUUID); |
761 | }while (false); |
762 | |
763 | if (kIOReturnSuccess != err) { |
764 | HIBLOG("error 0x%x opening polled file\n" , err); |
765 | IOPolledFileClose(pVars: &vars, write_offset: 0, NULL, write_length: 0, discard_offset: 0, discard_end: 0, unlink: false); |
766 | if (extentsData) { |
767 | extentsData->release(); |
768 | } |
769 | } |
770 | |
771 | if (part) { |
772 | part->release(); |
773 | } |
774 | |
775 | return err; |
776 | } |
777 | |
778 | IOReturn |
779 | IOPolledFileOpen(const char * filename, |
780 | uint32_t flags, |
781 | uint64_t setFileSize, uint64_t fsFreeSize, |
782 | void * write_file_addr, size_t write_file_len, |
783 | IOPolledFileIOVars ** fileVars, |
784 | OSSharedPtr<OSData>& imagePath, |
785 | uint8_t * volumeCryptKey, size_t * keySize) |
786 | { |
787 | OSData* imagePathRaw = NULL; |
788 | IOReturn result = IOPolledFileOpen(filename, flags, setFileSize, fsFreeSize, write_file_addr, write_file_len, |
789 | fileVars, imagePath: &imagePathRaw, volumeCryptKey, keySize); |
790 | imagePath.reset(p: imagePathRaw, OSNoRetain); |
791 | return result; |
792 | } |
793 | |
794 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
795 | |
796 | IOReturn |
797 | IOPolledFileClose(IOPolledFileIOVars ** pVars, |
798 | off_t write_offset, void * addr, size_t write_length, |
799 | off_t discard_offset, off_t discard_end, bool unlink) |
800 | { |
801 | IOPolledFileIOVars * vars; |
802 | |
803 | vars = *pVars; |
804 | if (!vars) { |
805 | return kIOReturnSuccess; |
806 | } |
807 | |
808 | if (vars->fileRef) { |
809 | kern_close_file_for_direct_io(ref: vars->fileRef, write_offset, addr, write_length, |
810 | discard_offset, discard_end, unlink); |
811 | vars->fileRef = NULL; |
812 | } |
813 | if (vars->fileExtents) { |
814 | vars->fileExtents->release(); |
815 | vars->fileExtents = NULL; |
816 | } |
817 | if (vars->pollers) { |
818 | vars->pollers->release(); |
819 | vars->pollers = NULL; |
820 | } |
821 | |
822 | if (vars->allocated) { |
823 | IOFreeType(vars, IOPolledFileIOVars); |
824 | } else { |
825 | bzero(s: vars, n: sizeof(IOPolledFileIOVars)); |
826 | } |
827 | *pVars = NULL; |
828 | |
829 | return kIOReturnSuccess; |
830 | } |
831 | |
832 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
833 | |
834 | IOReturn |
835 | (IOPolledFileIOVars * vars, |
836 | uint32_t openState) |
837 | { |
838 | IOReturn err; |
839 | |
840 | err = kIOReturnSuccess; |
841 | do{ |
842 | if (!vars->pollers->openCount) { |
843 | err = IOPolledFilePollersProbe(vars: vars->pollers); |
844 | if (kIOReturnSuccess != err) { |
845 | break; |
846 | } |
847 | } |
848 | err = IOPolledFilePollersOpen(filevars: vars, state: openState, abortable: false); |
849 | if (kIOReturnSuccess != err) { |
850 | break; |
851 | } |
852 | if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState)) { |
853 | vars->pollers->openCount++; |
854 | } |
855 | vars->pollers->io = false; |
856 | vars->buffer = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy(); |
857 | vars->bufferHalf = 0; |
858 | vars->bufferOffset = 0; |
859 | assert(vars->pollers->ioBuffer->getLength() <= UINT_MAX); |
860 | vars->bufferSize = (typeof(vars->bufferSize))(vars->pollers->ioBuffer->getLength() >> 1); |
861 | |
862 | if (vars->maxiobytes < vars->bufferSize) { |
863 | vars->bufferSize = (typeof(vars->bufferSize))vars->maxiobytes; |
864 | } |
865 | }while (false); |
866 | |
867 | if (kIOReturnSuccess != err) { |
868 | HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n" , openState, err); |
869 | } |
870 | |
871 | return err; |
872 | } |
873 | |
874 | |
875 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
876 | |
877 | IOReturn |
878 | IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position) |
879 | { |
880 | IOPolledFileExtent * extentMap; |
881 | |
882 | extentMap = vars->extentMap; |
883 | |
884 | vars->position = position; |
885 | |
886 | if (position > vars->fileSize) { |
887 | HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n" , vars->position, vars->fileSize); |
888 | return kIOReturnNoSpace; |
889 | } |
890 | |
891 | while (position >= extentMap->length) { |
892 | position -= extentMap->length; |
893 | extentMap++; |
894 | } |
895 | |
896 | vars->currentExtent = extentMap; |
897 | vars->extentRemaining = extentMap->length - position; |
898 | vars->extentPosition = vars->position - position; |
899 | |
900 | if (vars->bufferSize <= vars->extentRemaining) { |
901 | vars->bufferLimit = vars->bufferSize; |
902 | } else { |
903 | vars->bufferLimit = ((uint32_t) vars->extentRemaining); |
904 | } |
905 | |
906 | return kIOReturnSuccess; |
907 | } |
908 | |
909 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
910 | |
911 | IOReturn |
912 | IOPolledFileWrite(IOPolledFileIOVars * vars, |
913 | const uint8_t * bytes, IOByteCount size, |
914 | IOPolledFileCryptVars * cryptvars) |
915 | { |
916 | IOReturn err = kIOReturnSuccess; |
917 | IOByteCount copy, original_size = size; |
918 | bool flush = false; |
919 | |
920 | do{ |
921 | if (!bytes && !size) { |
922 | // seek to end of block & flush |
923 | size = vars->position & (vars->blockSize - 1); |
924 | if (size) { |
925 | size = vars->blockSize - size; |
926 | } |
927 | flush = true; |
928 | } |
929 | |
930 | copy = vars->bufferLimit - vars->bufferOffset; |
931 | if (copy > size) { |
932 | copy = size; |
933 | } else { |
934 | flush = true; |
935 | } |
936 | |
937 | if (bytes) { |
938 | #if KASAN |
939 | /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to |
940 | * avoid triggering global redzone sanitizer violations when accessing |
941 | * interstices between 'C' structures |
942 | */ |
943 | __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy); |
944 | #else |
945 | bcopy(src: bytes, dst: vars->buffer + vars->bufferHalf + vars->bufferOffset, n: copy); |
946 | #endif |
947 | bytes += copy; |
948 | } else { |
949 | bzero(s: vars->buffer + vars->bufferHalf + vars->bufferOffset, n: copy); |
950 | } |
951 | |
952 | size -= copy; |
953 | vars->bufferOffset += copy; |
954 | vars->position += copy; |
955 | |
956 | if (flush && vars->bufferOffset) { |
957 | uint64_t offset = (vars->position - vars->bufferOffset |
958 | - vars->extentPosition + vars->currentExtent->start); |
959 | uint32_t length = (vars->bufferOffset); |
960 | |
961 | #if CRYPTO |
962 | if (cryptvars && vars->encryptStart |
963 | && (vars->position > vars->encryptStart) |
964 | && ((vars->position - length) < vars->encryptEnd)) { |
965 | AbsoluteTime startTime, endTime; |
966 | |
967 | uint64_t encryptLen, encryptStart; |
968 | encryptLen = vars->position - vars->encryptStart; |
969 | if (encryptLen > length) { |
970 | encryptLen = length; |
971 | } |
972 | encryptStart = length - encryptLen; |
973 | if (vars->position > vars->encryptEnd) { |
974 | encryptLen -= (vars->position - vars->encryptEnd); |
975 | } |
976 | |
977 | clock_get_uptime(result: &startTime); |
978 | |
979 | assert(encryptLen <= UINT_MAX); |
980 | // encrypt the buffer |
981 | aes_encrypt_cbc(in_blk: vars->buffer + vars->bufferHalf + encryptStart, |
982 | in_iv: &cryptvars->aes_iv[0], |
983 | num_blk: (unsigned int) (encryptLen / AES_BLOCK_SIZE), |
984 | out_blk: vars->buffer + vars->bufferHalf + encryptStart, |
985 | cx: &cryptvars->ctx.encrypt); |
986 | |
987 | clock_get_uptime(result: &endTime); |
988 | ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); |
989 | SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); |
990 | vars->cryptBytes += encryptLen; |
991 | |
992 | // save initial vector for following encrypts |
993 | bcopy(src: vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE, |
994 | dst: &cryptvars->aes_iv[0], |
995 | AES_BLOCK_SIZE); |
996 | } |
997 | #endif /* CRYPTO */ |
998 | |
999 | err = IOPolledFilePollersIODone(vars: vars->pollers, abortable: true); |
1000 | if (kIOReturnSuccess != err) { |
1001 | break; |
1002 | } |
1003 | |
1004 | if (vars->position & (vars->blockSize - 1)) { |
1005 | HIBLOG("misaligned file pos %qx\n" , vars->position); |
1006 | } |
1007 | //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length); |
1008 | |
1009 | err = IOStartPolledIO(vars: vars->pollers, operation: kIOPolledWrite, bufferOffset: vars->bufferHalf, deviceOffset: offset, length); |
1010 | if (kIOReturnSuccess != err) { |
1011 | HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n" , |
1012 | vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err); |
1013 | break; |
1014 | } |
1015 | vars->pollers->io = true; |
1016 | |
1017 | vars->extentRemaining -= vars->bufferOffset; |
1018 | if (!vars->extentRemaining) { |
1019 | vars->currentExtent++; |
1020 | vars->extentRemaining = vars->currentExtent->length; |
1021 | vars->extentPosition = vars->position; |
1022 | } |
1023 | |
1024 | vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; |
1025 | vars->bufferOffset = 0; |
1026 | if (vars->bufferSize <= vars->extentRemaining) { |
1027 | vars->bufferLimit = vars->bufferSize; |
1028 | } else { |
1029 | vars->bufferLimit = ((uint32_t) vars->extentRemaining); |
1030 | } |
1031 | |
1032 | if (!vars->extentRemaining) { |
1033 | err = kIOReturnOverrun; |
1034 | break; |
1035 | } |
1036 | |
1037 | flush = false; |
1038 | } |
1039 | }while (size); |
1040 | |
1041 | return err; |
1042 | } |
1043 | |
1044 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1045 | |
1046 | IOReturn |
1047 | IOPolledFileFlush(IOPolledFileIOVars * vars) |
1048 | { |
1049 | // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms) |
1050 | IOReturn err = kIOReturnSuccess; |
1051 | |
1052 | err = IOPolledFilePollersIODone(vars: vars->pollers, abortable: true); |
1053 | if (kIOReturnSuccess != err) { |
1054 | return err; |
1055 | } |
1056 | |
1057 | err = IOStartPolledIO(vars: vars->pollers, operation: kIOPolledFlush, bufferOffset: 0, deviceOffset: 0, length: 0); |
1058 | if (kIOReturnSuccess != err) { |
1059 | HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n" , |
1060 | vars, vars->pollers, err); |
1061 | return err; |
1062 | } |
1063 | vars->pollers->io = true; |
1064 | |
1065 | return err; |
1066 | } |
1067 | |
1068 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1069 | |
1070 | IOReturn |
1071 | IOPolledFileRead(IOPolledFileIOVars * vars, |
1072 | uint8_t * bytes, IOByteCount size, |
1073 | IOPolledFileCryptVars * cryptvars) |
1074 | { |
1075 | IOReturn err = kIOReturnSuccess; |
1076 | IOByteCount copy; |
1077 | |
1078 | // bytesWritten += size; |
1079 | |
1080 | do{ |
1081 | copy = vars->bufferLimit - vars->bufferOffset; |
1082 | if (copy > size) { |
1083 | copy = size; |
1084 | } |
1085 | |
1086 | if (bytes) { |
1087 | #if KASAN |
1088 | __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy); |
1089 | #else |
1090 | bcopy(src: vars->buffer + vars->bufferHalf + vars->bufferOffset, dst: bytes, n: copy); |
1091 | #endif |
1092 | bytes += copy; |
1093 | } |
1094 | size -= copy; |
1095 | vars->bufferOffset += copy; |
1096 | // vars->position += copy; |
1097 | |
1098 | if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) { |
1099 | if (!vars->pollers->io) { |
1100 | cryptvars = NULL; |
1101 | } |
1102 | err = IOPolledFilePollersIODone(vars: vars->pollers, abortable: true); |
1103 | if (kIOReturnSuccess != err) { |
1104 | break; |
1105 | } |
1106 | |
1107 | if (vars->position & (vars->blockSize - 1)) { |
1108 | HIBLOG("misaligned file pos %qx\n" , vars->position); |
1109 | } |
1110 | |
1111 | vars->position += vars->lastRead; |
1112 | vars->extentRemaining -= vars->lastRead; |
1113 | vars->bufferLimit = vars->lastRead; |
1114 | |
1115 | if (!vars->extentRemaining) { |
1116 | vars->currentExtent++; |
1117 | vars->extentRemaining = vars->currentExtent->length; |
1118 | vars->extentPosition = vars->position; |
1119 | if (!vars->extentRemaining) { |
1120 | err = kIOReturnOverrun; |
1121 | break; |
1122 | } |
1123 | } |
1124 | |
1125 | uint32_t length; |
1126 | uint32_t lastReadLength = vars->lastRead; |
1127 | uint64_t offset = (vars->position |
1128 | - vars->extentPosition + vars->currentExtent->start); |
1129 | if (vars->extentRemaining <= vars->bufferSize) { |
1130 | length = ((uint32_t) vars->extentRemaining); |
1131 | } else { |
1132 | length = vars->bufferSize; |
1133 | } |
1134 | if ((length + vars->position) > vars->readEnd) { |
1135 | length = ((uint32_t) (vars->readEnd - vars->position)); |
1136 | } |
1137 | |
1138 | vars->lastRead = length; |
1139 | if (length) { |
1140 | //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length); |
1141 | err = IOStartPolledIO(vars: vars->pollers, operation: kIOPolledRead, bufferOffset: vars->bufferHalf, deviceOffset: offset, length); |
1142 | if (kIOReturnSuccess != err) { |
1143 | break; |
1144 | } |
1145 | vars->pollers->io = true; |
1146 | } |
1147 | |
1148 | vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize; |
1149 | vars->bufferOffset = 0; |
1150 | |
1151 | #if CRYPTO |
1152 | if (cryptvars) { |
1153 | uint8_t thisVector[AES_BLOCK_SIZE]; |
1154 | AbsoluteTime startTime, endTime; |
1155 | |
1156 | // save initial vector for following decrypts |
1157 | bcopy(src: &cryptvars->aes_iv[0], dst: &thisVector[0], AES_BLOCK_SIZE); |
1158 | bcopy(src: vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, |
1159 | dst: &cryptvars->aes_iv[0], AES_BLOCK_SIZE); |
1160 | |
1161 | // decrypt the buffer |
1162 | clock_get_uptime(result: &startTime); |
1163 | |
1164 | assert(lastReadLength <= UINT_MAX); |
1165 | aes_decrypt_cbc(in_blk: vars->buffer + vars->bufferHalf, |
1166 | in_iv: &thisVector[0], |
1167 | num_blk: (unsigned int) (lastReadLength / AES_BLOCK_SIZE), |
1168 | out_blk: vars->buffer + vars->bufferHalf, |
1169 | cx: &cryptvars->ctx.decrypt); |
1170 | |
1171 | clock_get_uptime(result: &endTime); |
1172 | ADD_ABSOLUTETIME(&vars->cryptTime, &endTime); |
1173 | SUB_ABSOLUTETIME(&vars->cryptTime, &startTime); |
1174 | vars->cryptBytes += lastReadLength; |
1175 | } |
1176 | #endif /* CRYPTO */ |
1177 | } |
1178 | }while (size); |
1179 | |
1180 | return err; |
1181 | } |
1182 | |
1183 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1184 | |