1 | /* |
2 | * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #include <IOKit/assert.h> |
30 | |
31 | #include <libkern/OSTypes.h> |
32 | #include <libkern/OSByteOrder.h> |
33 | #include <libkern/OSDebug.h> |
34 | |
35 | #include <IOKit/IOReturn.h> |
36 | #include <IOKit/IOLib.h> |
37 | #include <IOKit/IODMACommand.h> |
38 | #include <IOKit/IOMapper.h> |
39 | #include <IOKit/IOMemoryDescriptor.h> |
40 | #include <IOKit/IOBufferMemoryDescriptor.h> |
41 | |
42 | #include "IOKitKernelInternal.h" |
43 | |
44 | #define MAPTYPE(type) ((UInt) (type) & kTypeMask) |
45 | #define IS_NONCOHERENT(type) (MAPTYPE(type) == kNonCoherent) |
46 | |
47 | enum |
48 | { |
49 | kWalkSyncIn = 0x01, // bounce -> md |
50 | kWalkSyncOut = 0x02, // bounce <- md |
51 | kWalkSyncAlways = 0x04, |
52 | kWalkPreflight = 0x08, |
53 | kWalkDoubleBuffer = 0x10, |
54 | kWalkPrepare = 0x20, |
55 | kWalkComplete = 0x40, |
56 | kWalkClient = 0x80 |
57 | }; |
58 | |
59 | |
60 | #define fInternalState reserved |
61 | #define fState reserved->fState |
62 | #define fMDSummary reserved->fMDSummary |
63 | |
64 | |
65 | #if 1 |
66 | // no direction => OutIn |
67 | #define SHOULD_COPY_DIR(op, direction) \ |
68 | ((kIODirectionNone == (direction)) \ |
69 | || (kWalkSyncAlways & (op)) \ |
70 | || (((kWalkSyncIn & (op)) ? kIODirectionIn : kIODirectionOut) \ |
71 | & (direction))) |
72 | |
73 | #else |
74 | #define SHOULD_COPY_DIR(state, direction) (true) |
75 | #endif |
76 | |
77 | #if 0 |
78 | #define DEBG(fmt, args...) { IOLog(fmt, ## args); kprintf(fmt, ## args); } |
79 | #else |
80 | #define DEBG(fmt, args...) {} |
81 | #endif |
82 | |
83 | /**************************** class IODMACommand ***************************/ |
84 | |
85 | #undef super |
86 | #define super IOCommand |
87 | OSDefineMetaClassAndStructors(IODMACommand, IOCommand); |
88 | |
89 | OSMetaClassDefineReservedUsed(IODMACommand, 0); |
90 | OSMetaClassDefineReservedUsed(IODMACommand, 1); |
91 | OSMetaClassDefineReservedUsed(IODMACommand, 2); |
92 | OSMetaClassDefineReservedUsed(IODMACommand, 3); |
93 | OSMetaClassDefineReservedUsed(IODMACommand, 4); |
94 | OSMetaClassDefineReservedUsed(IODMACommand, 5); |
95 | OSMetaClassDefineReservedUsed(IODMACommand, 6); |
96 | OSMetaClassDefineReservedUnused(IODMACommand, 7); |
97 | OSMetaClassDefineReservedUnused(IODMACommand, 8); |
98 | OSMetaClassDefineReservedUnused(IODMACommand, 9); |
99 | OSMetaClassDefineReservedUnused(IODMACommand, 10); |
100 | OSMetaClassDefineReservedUnused(IODMACommand, 11); |
101 | OSMetaClassDefineReservedUnused(IODMACommand, 12); |
102 | OSMetaClassDefineReservedUnused(IODMACommand, 13); |
103 | OSMetaClassDefineReservedUnused(IODMACommand, 14); |
104 | OSMetaClassDefineReservedUnused(IODMACommand, 15); |
105 | |
106 | IODMACommand * |
107 | IODMACommand::withRefCon(void * refCon) |
108 | { |
109 | IODMACommand * me = new IODMACommand; |
110 | |
111 | if (me && !me->initWithRefCon(refCon)) |
112 | { |
113 | me->release(); |
114 | return 0; |
115 | } |
116 | |
117 | return me; |
118 | } |
119 | |
120 | IODMACommand * |
121 | IODMACommand::withSpecification(SegmentFunction outSegFunc, |
122 | const SegmentOptions * segmentOptions, |
123 | uint32_t mappingOptions, |
124 | IOMapper * mapper, |
125 | void * refCon) |
126 | { |
127 | IODMACommand * me = new IODMACommand; |
128 | |
129 | if (me && !me->initWithSpecification(outSegFunc, segmentOptions, mappingOptions, |
130 | mapper, refCon)) |
131 | { |
132 | me->release(); |
133 | return 0; |
134 | } |
135 | |
136 | return me; |
137 | } |
138 | |
139 | IODMACommand * |
140 | IODMACommand::withSpecification(SegmentFunction outSegFunc, |
141 | UInt8 numAddressBits, |
142 | UInt64 maxSegmentSize, |
143 | MappingOptions mappingOptions, |
144 | UInt64 maxTransferSize, |
145 | UInt32 alignment, |
146 | IOMapper *mapper, |
147 | void *refCon) |
148 | { |
149 | IODMACommand * me = new IODMACommand; |
150 | |
151 | if (me && !me->initWithSpecification(outSegFunc, |
152 | numAddressBits, maxSegmentSize, |
153 | mappingOptions, maxTransferSize, |
154 | alignment, mapper, refCon)) |
155 | { |
156 | me->release(); |
157 | return 0; |
158 | } |
159 | |
160 | return me; |
161 | } |
162 | |
163 | IODMACommand * |
164 | IODMACommand::cloneCommand(void *refCon) |
165 | { |
166 | SegmentOptions segmentOptions = |
167 | { |
168 | .fStructSize = sizeof(segmentOptions), |
169 | .fNumAddressBits = (uint8_t)fNumAddressBits, |
170 | .fMaxSegmentSize = fMaxSegmentSize, |
171 | .fMaxTransferSize = fMaxTransferSize, |
172 | .fAlignment = fAlignMask + 1, |
173 | .fAlignmentLength = fAlignMaskInternalSegments + 1, |
174 | .fAlignmentInternalSegments = fAlignMaskLength + 1 |
175 | }; |
176 | |
177 | return (IODMACommand::withSpecification(fOutSeg, &segmentOptions, |
178 | fMappingOptions, fMapper, refCon)); |
179 | } |
180 | |
181 | #define kLastOutputFunction ((SegmentFunction) kLastOutputFunction) |
182 | |
183 | bool |
184 | IODMACommand::initWithRefCon(void * refCon) |
185 | { |
186 | if (!super::init()) return (false); |
187 | |
188 | if (!reserved) |
189 | { |
190 | reserved = IONew(IODMACommandInternal, 1); |
191 | if (!reserved) return false; |
192 | } |
193 | bzero(reserved, sizeof(IODMACommandInternal)); |
194 | fRefCon = refCon; |
195 | |
196 | return (true); |
197 | } |
198 | |
199 | bool |
200 | IODMACommand::initWithSpecification(SegmentFunction outSegFunc, |
201 | const SegmentOptions * segmentOptions, |
202 | uint32_t mappingOptions, |
203 | IOMapper * mapper, |
204 | void * refCon) |
205 | { |
206 | if (!initWithRefCon(refCon)) return false; |
207 | |
208 | if (kIOReturnSuccess != setSpecification(outSegFunc, segmentOptions, |
209 | mappingOptions, mapper)) return false; |
210 | |
211 | return (true); |
212 | } |
213 | |
214 | bool |
215 | IODMACommand::initWithSpecification(SegmentFunction outSegFunc, |
216 | UInt8 numAddressBits, |
217 | UInt64 maxSegmentSize, |
218 | MappingOptions mappingOptions, |
219 | UInt64 maxTransferSize, |
220 | UInt32 alignment, |
221 | IOMapper *mapper, |
222 | void *refCon) |
223 | { |
224 | SegmentOptions segmentOptions = |
225 | { |
226 | .fStructSize = sizeof(segmentOptions), |
227 | .fNumAddressBits = numAddressBits, |
228 | .fMaxSegmentSize = maxSegmentSize, |
229 | .fMaxTransferSize = maxTransferSize, |
230 | .fAlignment = alignment, |
231 | .fAlignmentLength = 1, |
232 | .fAlignmentInternalSegments = alignment |
233 | }; |
234 | |
235 | return (initWithSpecification(outSegFunc, &segmentOptions, mappingOptions, mapper, refCon)); |
236 | } |
237 | |
238 | IOReturn |
239 | IODMACommand::setSpecification(SegmentFunction outSegFunc, |
240 | const SegmentOptions * segmentOptions, |
241 | uint32_t mappingOptions, |
242 | IOMapper * mapper) |
243 | { |
244 | IOService * device = 0; |
245 | UInt8 numAddressBits; |
246 | UInt64 maxSegmentSize; |
247 | UInt64 maxTransferSize; |
248 | UInt32 alignment; |
249 | |
250 | bool is32Bit; |
251 | |
252 | if (!outSegFunc || !segmentOptions) return (kIOReturnBadArgument); |
253 | |
254 | is32Bit = ((OutputHost32 == outSegFunc) |
255 | || (OutputBig32 == outSegFunc) |
256 | || (OutputLittle32 == outSegFunc)); |
257 | |
258 | numAddressBits = segmentOptions->fNumAddressBits; |
259 | maxSegmentSize = segmentOptions->fMaxSegmentSize; |
260 | maxTransferSize = segmentOptions->fMaxTransferSize; |
261 | alignment = segmentOptions->fAlignment; |
262 | if (is32Bit) |
263 | { |
264 | if (!numAddressBits) |
265 | numAddressBits = 32; |
266 | else if (numAddressBits > 32) |
267 | return (kIOReturnBadArgument); // Wrong output function for bits |
268 | } |
269 | |
270 | if (numAddressBits && (numAddressBits < PAGE_SHIFT)) return (kIOReturnBadArgument); |
271 | |
272 | if (!maxSegmentSize) maxSegmentSize--; // Set Max segment to -1 |
273 | if (!maxTransferSize) maxTransferSize--; // Set Max transfer to -1 |
274 | |
275 | if (mapper && !OSDynamicCast(IOMapper, mapper)) |
276 | { |
277 | device = mapper; |
278 | mapper = 0; |
279 | } |
280 | if (!mapper && (kUnmapped != MAPTYPE(mappingOptions))) |
281 | { |
282 | IOMapper::checkForSystemMapper(); |
283 | mapper = IOMapper::gSystem; |
284 | } |
285 | |
286 | fNumSegments = 0; |
287 | fOutSeg = outSegFunc; |
288 | fNumAddressBits = numAddressBits; |
289 | fMaxSegmentSize = maxSegmentSize; |
290 | fMappingOptions = mappingOptions; |
291 | fMaxTransferSize = maxTransferSize; |
292 | if (!alignment) alignment = 1; |
293 | fAlignMask = alignment - 1; |
294 | |
295 | alignment = segmentOptions->fAlignmentLength; |
296 | if (!alignment) alignment = 1; |
297 | fAlignMaskLength = alignment - 1; |
298 | |
299 | alignment = segmentOptions->fAlignmentInternalSegments; |
300 | if (!alignment) alignment = (fAlignMask + 1); |
301 | fAlignMaskInternalSegments = alignment - 1; |
302 | |
303 | switch (MAPTYPE(mappingOptions)) |
304 | { |
305 | case kMapped: break; |
306 | case kUnmapped: break; |
307 | case kNonCoherent: break; |
308 | |
309 | case kBypassed: |
310 | if (!mapper) break; |
311 | return (kIOReturnBadArgument); |
312 | |
313 | default: |
314 | return (kIOReturnBadArgument); |
315 | }; |
316 | |
317 | if (mapper != fMapper) |
318 | { |
319 | if (mapper) mapper->retain(); |
320 | if (fMapper) fMapper->release(); |
321 | fMapper = mapper; |
322 | } |
323 | |
324 | fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions)); |
325 | fInternalState->fDevice = device; |
326 | |
327 | return (kIOReturnSuccess); |
328 | } |
329 | |
330 | void |
331 | IODMACommand::free() |
332 | { |
333 | if (reserved) IODelete(reserved, IODMACommandInternal, 1); |
334 | |
335 | if (fMapper) fMapper->release(); |
336 | |
337 | super::free(); |
338 | } |
339 | |
340 | IOReturn |
341 | IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare) |
342 | { |
343 | IOReturn err = kIOReturnSuccess; |
344 | |
345 | if (mem == fMemory) |
346 | { |
347 | if (!autoPrepare) |
348 | { |
349 | while (fActive) |
350 | complete(); |
351 | } |
352 | return kIOReturnSuccess; |
353 | } |
354 | |
355 | if (fMemory) { |
356 | // As we are almost certainly being called from a work loop thread |
357 | // if fActive is true it is probably not a good time to potentially |
358 | // block. Just test for it and return an error |
359 | if (fActive) |
360 | return kIOReturnBusy; |
361 | clearMemoryDescriptor(); |
362 | } |
363 | |
364 | if (mem) { |
365 | bzero(&fMDSummary, sizeof(fMDSummary)); |
366 | err = mem->dmaCommandOperation(kIOMDGetCharacteristics | (kMapped == MAPTYPE(fMappingOptions)), |
367 | &fMDSummary, sizeof(fMDSummary)); |
368 | if (err) |
369 | return err; |
370 | |
371 | ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage; |
372 | |
373 | if ((kMapped == MAPTYPE(fMappingOptions)) |
374 | && fMapper) |
375 | fInternalState->fCheckAddressing = false; |
376 | else |
377 | fInternalState->fCheckAddressing = (fNumAddressBits && (highPage >= (1UL << (fNumAddressBits - PAGE_SHIFT)))); |
378 | |
379 | fInternalState->fNewMD = true; |
380 | mem->retain(); |
381 | fMemory = mem; |
382 | fInternalState->fSetActiveNoMapper = (!fMapper); |
383 | if (fInternalState->fSetActiveNoMapper) mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0); |
384 | if (autoPrepare) { |
385 | err = prepare(); |
386 | if (err) { |
387 | clearMemoryDescriptor(); |
388 | } |
389 | } |
390 | } |
391 | |
392 | return err; |
393 | } |
394 | |
395 | IOReturn |
396 | IODMACommand::clearMemoryDescriptor(bool autoComplete) |
397 | { |
398 | if (fActive && !autoComplete) return (kIOReturnNotReady); |
399 | |
400 | if (fMemory) |
401 | { |
402 | while (fActive) complete(); |
403 | if (fInternalState->fSetActiveNoMapper) fMemory->dmaCommandOperation(kIOMDSetDMAInactive, this, 0); |
404 | fMemory->release(); |
405 | fMemory = 0; |
406 | } |
407 | |
408 | return (kIOReturnSuccess); |
409 | } |
410 | |
411 | const IOMemoryDescriptor * |
412 | IODMACommand::getMemoryDescriptor() const |
413 | { |
414 | return fMemory; |
415 | } |
416 | |
417 | IOMemoryDescriptor * |
418 | IODMACommand::getIOMemoryDescriptor() const |
419 | { |
420 | IOMemoryDescriptor * mem; |
421 | |
422 | mem = reserved->fCopyMD; |
423 | if (!mem) mem = __IODEQUALIFY(IOMemoryDescriptor *, fMemory); |
424 | |
425 | return (mem); |
426 | } |
427 | |
428 | IOReturn |
429 | IODMACommand::segmentOp( |
430 | void *reference, |
431 | IODMACommand *target, |
432 | Segment64 segment, |
433 | void *segments, |
434 | UInt32 segmentIndex) |
435 | { |
436 | IOOptionBits op = (uintptr_t) reference; |
437 | addr64_t maxPhys, address; |
438 | uint64_t length; |
439 | uint32_t numPages; |
440 | uint32_t mask; |
441 | |
442 | IODMACommandInternal * state = target->reserved; |
443 | |
444 | if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperAllocValid || !target->fMapper)) |
445 | maxPhys = (1ULL << target->fNumAddressBits); |
446 | else |
447 | maxPhys = 0; |
448 | maxPhys--; |
449 | |
450 | address = segment.fIOVMAddr; |
451 | length = segment.fLength; |
452 | |
453 | assert(length); |
454 | |
455 | if (!state->fMisaligned) |
456 | { |
457 | mask = (segmentIndex ? target->fAlignMaskInternalSegments : state->fSourceAlignMask); |
458 | state->fMisaligned |= (0 != (mask & address)); |
459 | if (state->fMisaligned) DEBG("misaligned address %qx:%qx, %x\n" , address, length, mask); |
460 | } |
461 | if (!state->fMisaligned) |
462 | { |
463 | mask = target->fAlignMaskLength; |
464 | state->fMisaligned |= (0 != (mask & length)); |
465 | if (state->fMisaligned) DEBG("misaligned length %qx:%qx, %x\n" , address, length, mask); |
466 | } |
467 | |
468 | if (state->fMisaligned && (kWalkPreflight & op)) |
469 | return (kIOReturnNotAligned); |
470 | |
471 | if (!state->fDoubleBuffer) |
472 | { |
473 | if ((address + length - 1) <= maxPhys) |
474 | { |
475 | length = 0; |
476 | } |
477 | else if (address <= maxPhys) |
478 | { |
479 | DEBG("tail %qx, %qx" , address, length); |
480 | length = (address + length - maxPhys - 1); |
481 | address = maxPhys + 1; |
482 | DEBG("-> %qx, %qx\n" , address, length); |
483 | } |
484 | } |
485 | |
486 | if (!length) |
487 | return (kIOReturnSuccess); |
488 | |
489 | numPages = atop_64(round_page_64((address & PAGE_MASK) + length)); |
490 | |
491 | if (kWalkPreflight & op) |
492 | { |
493 | state->fCopyPageCount += numPages; |
494 | } |
495 | else |
496 | { |
497 | vm_page_t lastPage; |
498 | lastPage = NULL; |
499 | if (kWalkPrepare & op) |
500 | { |
501 | lastPage = state->fCopyNext; |
502 | for (IOItemCount idx = 0; idx < numPages; idx++) |
503 | { |
504 | vm_page_set_offset(lastPage, atop_64(address) + idx); |
505 | lastPage = vm_page_get_next(lastPage); |
506 | } |
507 | } |
508 | |
509 | if (!lastPage || SHOULD_COPY_DIR(op, target->fMDSummary.fDirection)) |
510 | { |
511 | lastPage = state->fCopyNext; |
512 | for (IOItemCount idx = 0; idx < numPages; idx++) |
513 | { |
514 | if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection)) |
515 | { |
516 | addr64_t cpuAddr = address; |
517 | addr64_t remapAddr; |
518 | uint64_t chunk; |
519 | |
520 | if ((kMapped == MAPTYPE(target->fMappingOptions)) |
521 | && target->fMapper) |
522 | { |
523 | cpuAddr = target->fMapper->mapToPhysicalAddress(address); |
524 | } |
525 | |
526 | remapAddr = ptoa_64(vm_page_get_phys_page(lastPage)); |
527 | if (!state->fDoubleBuffer) |
528 | { |
529 | remapAddr += (address & PAGE_MASK); |
530 | } |
531 | chunk = PAGE_SIZE - (address & PAGE_MASK); |
532 | if (chunk > length) |
533 | chunk = length; |
534 | |
535 | DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n" , remapAddr, |
536 | (kWalkSyncIn & op) ? "->" : "<-" , |
537 | address, chunk, op); |
538 | |
539 | if (kWalkSyncIn & op) |
540 | { // cppvNoModSnk |
541 | copypv(remapAddr, cpuAddr, chunk, |
542 | cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc ); |
543 | } |
544 | else |
545 | { |
546 | copypv(cpuAddr, remapAddr, chunk, |
547 | cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc ); |
548 | } |
549 | address += chunk; |
550 | length -= chunk; |
551 | } |
552 | lastPage = vm_page_get_next(lastPage); |
553 | } |
554 | } |
555 | state->fCopyNext = lastPage; |
556 | } |
557 | |
558 | return kIOReturnSuccess; |
559 | } |
560 | |
561 | IOBufferMemoryDescriptor * |
562 | IODMACommand::createCopyBuffer(IODirection direction, UInt64 length) |
563 | { |
564 | mach_vm_address_t mask = 0xFFFFF000; //state->fSourceAlignMask |
565 | return (IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task, |
566 | direction, length, mask)); |
567 | } |
568 | |
569 | IOReturn |
570 | IODMACommand::walkAll(UInt8 op) |
571 | { |
572 | IODMACommandInternal * state = fInternalState; |
573 | |
574 | IOReturn ret = kIOReturnSuccess; |
575 | UInt32 numSegments; |
576 | UInt64 offset; |
577 | |
578 | if (kWalkPreflight & op) |
579 | { |
580 | state->fMisaligned = false; |
581 | state->fDoubleBuffer = false; |
582 | state->fPrepared = false; |
583 | state->fCopyNext = NULL; |
584 | state->fCopyPageAlloc = 0; |
585 | state->fCopyPageCount = 0; |
586 | state->fNextRemapPage = NULL; |
587 | state->fCopyMD = 0; |
588 | |
589 | if (!(kWalkDoubleBuffer & op)) |
590 | { |
591 | offset = 0; |
592 | numSegments = 0-1; |
593 | ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); |
594 | } |
595 | |
596 | op &= ~kWalkPreflight; |
597 | |
598 | state->fDoubleBuffer = (state->fMisaligned || state->fForceDoubleBuffer); |
599 | state->fForceDoubleBuffer = false; |
600 | if (state->fDoubleBuffer) |
601 | state->fCopyPageCount = atop_64(round_page(state->fPreparedLength)); |
602 | |
603 | if (state->fCopyPageCount) |
604 | { |
605 | vm_page_t mapBase = NULL; |
606 | |
607 | DEBG("preflight fCopyPageCount %d\n" , state->fCopyPageCount); |
608 | |
609 | if (!fMapper && !state->fDoubleBuffer) |
610 | { |
611 | kern_return_t kr; |
612 | |
613 | if (fMapper) panic("fMapper copying" ); |
614 | |
615 | kr = vm_page_alloc_list(state->fCopyPageCount, |
616 | KMA_LOMEM | KMA_NOPAGEWAIT, &mapBase); |
617 | if (KERN_SUCCESS != kr) |
618 | { |
619 | DEBG("vm_page_alloc_list(%d) failed (%d)\n" , state->fCopyPageCount, kr); |
620 | mapBase = NULL; |
621 | } |
622 | } |
623 | |
624 | if (mapBase) |
625 | { |
626 | state->fCopyPageAlloc = mapBase; |
627 | state->fCopyNext = state->fCopyPageAlloc; |
628 | offset = 0; |
629 | numSegments = 0-1; |
630 | ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); |
631 | state->fPrepared = true; |
632 | op &= ~(kWalkSyncIn | kWalkSyncOut); |
633 | } |
634 | else |
635 | { |
636 | DEBG("alloc IOBMD\n" ); |
637 | state->fCopyMD = createCopyBuffer(fMDSummary.fDirection, state->fPreparedLength); |
638 | |
639 | if (state->fCopyMD) |
640 | { |
641 | ret = kIOReturnSuccess; |
642 | state->fPrepared = true; |
643 | } |
644 | else |
645 | { |
646 | DEBG("IODMACommand !alloc IOBMD" ); |
647 | return (kIOReturnNoResources); |
648 | } |
649 | } |
650 | } |
651 | } |
652 | |
653 | if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op)) |
654 | { |
655 | if (state->fCopyPageCount) |
656 | { |
657 | DEBG("sync fCopyPageCount %d\n" , state->fCopyPageCount); |
658 | |
659 | if (state->fCopyPageAlloc) |
660 | { |
661 | state->fCopyNext = state->fCopyPageAlloc; |
662 | offset = 0; |
663 | numSegments = 0-1; |
664 | ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); |
665 | } |
666 | else if (state->fCopyMD) |
667 | { |
668 | DEBG("sync IOBMD\n" ); |
669 | |
670 | if (SHOULD_COPY_DIR(op, fMDSummary.fDirection)) |
671 | { |
672 | IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory); |
673 | |
674 | IOByteCount bytes; |
675 | |
676 | if (kWalkSyncIn & op) |
677 | bytes = poMD->writeBytes(state->fPreparedOffset, |
678 | state->fCopyMD->getBytesNoCopy(), |
679 | state->fPreparedLength); |
680 | else |
681 | bytes = poMD->readBytes(state->fPreparedOffset, |
682 | state->fCopyMD->getBytesNoCopy(), |
683 | state->fPreparedLength); |
684 | DEBG("fCopyMD %s %lx bytes\n" , (kWalkSyncIn & op) ? "wrote" : "read" , bytes); |
685 | ret = (bytes == state->fPreparedLength) ? kIOReturnSuccess : kIOReturnUnderrun; |
686 | } |
687 | else |
688 | ret = kIOReturnSuccess; |
689 | } |
690 | } |
691 | } |
692 | |
693 | if (kWalkComplete & op) |
694 | { |
695 | if (state->fCopyPageAlloc) |
696 | { |
697 | vm_page_free_list(state->fCopyPageAlloc, FALSE); |
698 | state->fCopyPageAlloc = 0; |
699 | state->fCopyPageCount = 0; |
700 | } |
701 | if (state->fCopyMD) |
702 | { |
703 | state->fCopyMD->release(); |
704 | state->fCopyMD = 0; |
705 | } |
706 | |
707 | state->fPrepared = false; |
708 | } |
709 | return (ret); |
710 | } |
711 | |
712 | UInt8 |
713 | IODMACommand::getNumAddressBits(void) |
714 | { |
715 | return (fNumAddressBits); |
716 | } |
717 | |
718 | UInt32 |
719 | IODMACommand::getAlignment(void) |
720 | { |
721 | return (fAlignMask + 1); |
722 | } |
723 | |
724 | uint32_t |
725 | IODMACommand::getAlignmentLength(void) |
726 | { |
727 | return (fAlignMaskLength + 1); |
728 | } |
729 | |
730 | uint32_t |
731 | IODMACommand::getAlignmentInternalSegments(void) |
732 | { |
733 | return (fAlignMaskInternalSegments + 1); |
734 | } |
735 | |
736 | IOReturn |
737 | IODMACommand::prepareWithSpecification(SegmentFunction outSegFunc, |
738 | const SegmentOptions * segmentOptions, |
739 | uint32_t mappingOptions, |
740 | IOMapper * mapper, |
741 | UInt64 offset, |
742 | UInt64 length, |
743 | bool flushCache, |
744 | bool synchronize) |
745 | { |
746 | IOReturn ret; |
747 | |
748 | if (fActive) return kIOReturnNotPermitted; |
749 | |
750 | ret = setSpecification(outSegFunc, segmentOptions, mappingOptions, mapper); |
751 | if (kIOReturnSuccess != ret) return (ret); |
752 | |
753 | ret = prepare(offset, length, flushCache, synchronize); |
754 | |
755 | return (ret); |
756 | } |
757 | |
758 | IOReturn |
759 | IODMACommand::prepareWithSpecification(SegmentFunction outSegFunc, |
760 | UInt8 numAddressBits, |
761 | UInt64 maxSegmentSize, |
762 | MappingOptions mappingOptions, |
763 | UInt64 maxTransferSize, |
764 | UInt32 alignment, |
765 | IOMapper *mapper, |
766 | UInt64 offset, |
767 | UInt64 length, |
768 | bool flushCache, |
769 | bool synchronize) |
770 | { |
771 | SegmentOptions segmentOptions = |
772 | { |
773 | .fStructSize = sizeof(segmentOptions), |
774 | .fNumAddressBits = numAddressBits, |
775 | .fMaxSegmentSize = maxSegmentSize, |
776 | .fMaxTransferSize = maxTransferSize, |
777 | .fAlignment = alignment, |
778 | .fAlignmentLength = 1, |
779 | .fAlignmentInternalSegments = alignment |
780 | }; |
781 | |
782 | return (prepareWithSpecification(outSegFunc, &segmentOptions, mappingOptions, mapper, |
783 | offset, length, flushCache, synchronize)); |
784 | } |
785 | |
786 | |
787 | IOReturn |
788 | IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchronize) |
789 | { |
790 | IODMACommandInternal * state = fInternalState; |
791 | IOReturn ret = kIOReturnSuccess; |
792 | uint32_t mappingOptions = fMappingOptions; |
793 | |
794 | // check specification has been set |
795 | if (!fOutSeg) return (kIOReturnNotReady); |
796 | |
797 | if (!length) length = fMDSummary.fLength; |
798 | |
799 | if (length > fMaxTransferSize) return kIOReturnNoSpace; |
800 | |
801 | if (fActive++) |
802 | { |
803 | if ((state->fPreparedOffset != offset) |
804 | || (state->fPreparedLength != length)) |
805 | ret = kIOReturnNotReady; |
806 | } |
807 | else |
808 | { |
809 | if (fAlignMaskLength & length) return (kIOReturnNotAligned); |
810 | |
811 | state->fPreparedOffset = offset; |
812 | state->fPreparedLength = length; |
813 | |
814 | state->fMapContig = false; |
815 | state->fMisaligned = false; |
816 | state->fDoubleBuffer = false; |
817 | state->fPrepared = false; |
818 | state->fCopyNext = NULL; |
819 | state->fCopyPageAlloc = 0; |
820 | state->fCopyPageCount = 0; |
821 | state->fNextRemapPage = NULL; |
822 | state->fCopyMD = 0; |
823 | state->fLocalMapperAlloc = 0; |
824 | state->fLocalMapperAllocValid = false; |
825 | state->fLocalMapperAllocLength = 0; |
826 | |
827 | state->fSourceAlignMask = fAlignMask; |
828 | if (fMapper) |
829 | state->fSourceAlignMask &= page_mask; |
830 | |
831 | state->fCursor = state->fIterateOnly |
832 | || (!state->fCheckAddressing |
833 | && (!state->fSourceAlignMask |
834 | || ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask))))); |
835 | |
836 | if (!state->fCursor) |
837 | { |
838 | IOOptionBits op = kWalkPrepare | kWalkPreflight; |
839 | if (synchronize) |
840 | op |= kWalkSyncOut; |
841 | ret = walkAll(op); |
842 | } |
843 | |
844 | if (IS_NONCOHERENT(mappingOptions) && flushCache) |
845 | { |
846 | if (state->fCopyMD) |
847 | { |
848 | state->fCopyMD->performOperation(kIOMemoryIncoherentIOStore, 0, length); |
849 | } |
850 | else |
851 | { |
852 | IOMemoryDescriptor * md = const_cast<IOMemoryDescriptor *>(fMemory); |
853 | md->performOperation(kIOMemoryIncoherentIOStore, offset, length); |
854 | } |
855 | } |
856 | |
857 | if (fMapper) |
858 | { |
859 | IOMDDMAMapArgs mapArgs; |
860 | bzero(&mapArgs, sizeof(mapArgs)); |
861 | mapArgs.fMapper = fMapper; |
862 | mapArgs.fCommand = this; |
863 | mapArgs.fMapSpec.device = state->fDevice; |
864 | mapArgs.fMapSpec.alignment = fAlignMask + 1; |
865 | mapArgs.fMapSpec.numAddressBits = fNumAddressBits ? fNumAddressBits : 64; |
866 | mapArgs.fLength = state->fPreparedLength; |
867 | const IOMemoryDescriptor * md = state->fCopyMD; |
868 | if (md) { mapArgs.fOffset = 0; } else |
869 | { |
870 | md = fMemory; |
871 | mapArgs.fOffset = state->fPreparedOffset; |
872 | } |
873 | ret = md->dmaCommandOperation(kIOMDDMAMap | state->fIterateOnly, &mapArgs, sizeof(mapArgs)); |
874 | //IOLog("dma %p 0x%x 0x%qx-0x%qx 0x%qx-0x%qx\n", this, ret, state->fPreparedOffset, state->fPreparedLength, mapArgs.fAlloc, mapArgs.fAllocLength); |
875 | |
876 | if (kIOReturnSuccess == ret) |
877 | { |
878 | state->fLocalMapperAlloc = mapArgs.fAlloc; |
879 | state->fLocalMapperAllocValid = true; |
880 | state->fLocalMapperAllocLength = mapArgs.fAllocLength; |
881 | state->fMapContig = mapArgs.fMapContig; |
882 | } |
883 | if (NULL != IOMapper::gSystem) ret = kIOReturnSuccess; |
884 | } |
885 | if (kIOReturnSuccess == ret) state->fPrepared = true; |
886 | } |
887 | return ret; |
888 | } |
889 | |
890 | IOReturn |
891 | IODMACommand::complete(bool invalidateCache, bool synchronize) |
892 | { |
893 | IODMACommandInternal * state = fInternalState; |
894 | IOReturn ret = kIOReturnSuccess; |
895 | IOMemoryDescriptor * copyMD; |
896 | |
897 | if (fActive < 1) |
898 | return kIOReturnNotReady; |
899 | |
900 | if (!--fActive) |
901 | { |
902 | copyMD = state->fCopyMD; |
903 | if (copyMD) copyMD->retain(); |
904 | |
905 | if (IS_NONCOHERENT(fMappingOptions) && invalidateCache) |
906 | { |
907 | if (copyMD) |
908 | { |
909 | copyMD->performOperation(kIOMemoryIncoherentIOFlush, 0, state->fPreparedLength); |
910 | } |
911 | else |
912 | { |
913 | IOMemoryDescriptor * md = const_cast<IOMemoryDescriptor *>(fMemory); |
914 | md->performOperation(kIOMemoryIncoherentIOFlush, state->fPreparedOffset, state->fPreparedLength); |
915 | } |
916 | } |
917 | |
918 | if (!state->fCursor) |
919 | { |
920 | IOOptionBits op = kWalkComplete; |
921 | if (synchronize) |
922 | op |= kWalkSyncIn; |
923 | ret = walkAll(op); |
924 | } |
925 | |
926 | if (state->fLocalMapperAllocValid) |
927 | { |
928 | IOMDDMAMapArgs mapArgs; |
929 | bzero(&mapArgs, sizeof(mapArgs)); |
930 | mapArgs.fMapper = fMapper; |
931 | mapArgs.fCommand = this; |
932 | mapArgs.fAlloc = state->fLocalMapperAlloc; |
933 | mapArgs.fAllocLength = state->fLocalMapperAllocLength; |
934 | const IOMemoryDescriptor * md = copyMD; |
935 | if (md) { mapArgs.fOffset = 0; } |
936 | else |
937 | { |
938 | md = fMemory; |
939 | mapArgs.fOffset = state->fPreparedOffset; |
940 | } |
941 | |
942 | ret = md->dmaCommandOperation(kIOMDDMAUnmap, &mapArgs, sizeof(mapArgs)); |
943 | |
944 | state->fLocalMapperAlloc = 0; |
945 | state->fLocalMapperAllocValid = false; |
946 | state->fLocalMapperAllocLength = 0; |
947 | } |
948 | if (copyMD) copyMD->release(); |
949 | state->fPrepared = false; |
950 | } |
951 | |
952 | return ret; |
953 | } |
954 | |
955 | IOReturn |
956 | IODMACommand::getPreparedOffsetAndLength(UInt64 * offset, UInt64 * length) |
957 | { |
958 | IODMACommandInternal * state = fInternalState; |
959 | if (fActive < 1) |
960 | return (kIOReturnNotReady); |
961 | |
962 | if (offset) |
963 | *offset = state->fPreparedOffset; |
964 | if (length) |
965 | *length = state->fPreparedLength; |
966 | |
967 | return (kIOReturnSuccess); |
968 | } |
969 | |
970 | IOReturn |
971 | IODMACommand::synchronize(IOOptionBits options) |
972 | { |
973 | IODMACommandInternal * state = fInternalState; |
974 | IOReturn ret = kIOReturnSuccess; |
975 | IOOptionBits op; |
976 | |
977 | if (kIODirectionOutIn == (kIODirectionOutIn & options)) |
978 | return kIOReturnBadArgument; |
979 | |
980 | if (fActive < 1) |
981 | return kIOReturnNotReady; |
982 | |
983 | op = 0; |
984 | if (kForceDoubleBuffer & options) |
985 | { |
986 | if (state->fDoubleBuffer) return kIOReturnSuccess; |
987 | ret = complete(false /* invalidateCache */, true /* synchronize */); |
988 | state->fCursor = false; |
989 | state->fForceDoubleBuffer = true; |
990 | ret = prepare(state->fPreparedOffset, state->fPreparedLength, false /* flushCache */, true /* synchronize */); |
991 | |
992 | return (ret); |
993 | } |
994 | else if (state->fCursor) |
995 | return kIOReturnSuccess; |
996 | |
997 | if (kIODirectionIn & options) |
998 | op |= kWalkSyncIn | kWalkSyncAlways; |
999 | else if (kIODirectionOut & options) |
1000 | op |= kWalkSyncOut | kWalkSyncAlways; |
1001 | |
1002 | ret = walkAll(op); |
1003 | |
1004 | return ret; |
1005 | } |
1006 | |
1007 | struct IODMACommandTransferContext |
1008 | { |
1009 | void * buffer; |
1010 | UInt64 bufferOffset; |
1011 | UInt64 remaining; |
1012 | UInt32 op; |
1013 | }; |
1014 | enum |
1015 | { |
1016 | kIODMACommandTransferOpReadBytes = 1, |
1017 | kIODMACommandTransferOpWriteBytes = 2 |
1018 | }; |
1019 | |
1020 | IOReturn |
1021 | IODMACommand::transferSegment(void *reference, |
1022 | IODMACommand *target, |
1023 | Segment64 segment, |
1024 | void *segments, |
1025 | UInt32 segmentIndex) |
1026 | { |
1027 | IODMACommandTransferContext * context = (IODMACommandTransferContext *) reference; |
1028 | UInt64 length = min(segment.fLength, context->remaining); |
1029 | addr64_t ioAddr = segment.fIOVMAddr; |
1030 | addr64_t cpuAddr = ioAddr; |
1031 | |
1032 | context->remaining -= length; |
1033 | |
1034 | while (length) |
1035 | { |
1036 | UInt64 copyLen = length; |
1037 | if ((kMapped == MAPTYPE(target->fMappingOptions)) |
1038 | && target->fMapper) |
1039 | { |
1040 | cpuAddr = target->fMapper->mapToPhysicalAddress(ioAddr); |
1041 | copyLen = min(copyLen, page_size - (ioAddr & (page_size - 1))); |
1042 | ioAddr += copyLen; |
1043 | } |
1044 | |
1045 | switch (context->op) |
1046 | { |
1047 | case kIODMACommandTransferOpReadBytes: |
1048 | copypv(cpuAddr, context->bufferOffset + (addr64_t) context->buffer, copyLen, |
1049 | cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap); |
1050 | break; |
1051 | case kIODMACommandTransferOpWriteBytes: |
1052 | copypv(context->bufferOffset + (addr64_t) context->buffer, cpuAddr, copyLen, |
1053 | cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap); |
1054 | break; |
1055 | } |
1056 | length -= copyLen; |
1057 | context->bufferOffset += copyLen; |
1058 | } |
1059 | |
1060 | return (context->remaining ? kIOReturnSuccess : kIOReturnOverrun); |
1061 | } |
1062 | |
1063 | UInt64 |
1064 | IODMACommand::transfer(IOOptionBits transferOp, UInt64 offset, void * buffer, UInt64 length) |
1065 | { |
1066 | IODMACommandInternal * state = fInternalState; |
1067 | IODMACommandTransferContext context; |
1068 | Segment64 segments[1]; |
1069 | UInt32 numSegments = 0-1; |
1070 | |
1071 | if (fActive < 1) |
1072 | return (0); |
1073 | |
1074 | if (offset >= state->fPreparedLength) |
1075 | return (0); |
1076 | length = min(length, state->fPreparedLength - offset); |
1077 | |
1078 | context.buffer = buffer; |
1079 | context.bufferOffset = 0; |
1080 | context.remaining = length; |
1081 | context.op = transferOp; |
1082 | (void) genIOVMSegments(kWalkClient, transferSegment, &context, &offset, &segments[0], &numSegments); |
1083 | |
1084 | return (length - context.remaining); |
1085 | } |
1086 | |
1087 | UInt64 |
1088 | IODMACommand::readBytes(UInt64 offset, void *bytes, UInt64 length) |
1089 | { |
1090 | return (transfer(kIODMACommandTransferOpReadBytes, offset, bytes, length)); |
1091 | } |
1092 | |
1093 | UInt64 |
1094 | IODMACommand::writeBytes(UInt64 offset, const void *bytes, UInt64 length) |
1095 | { |
1096 | return (transfer(kIODMACommandTransferOpWriteBytes, offset, const_cast<void *>(bytes), length)); |
1097 | } |
1098 | |
1099 | IOReturn |
1100 | IODMACommand::genIOVMSegments(UInt64 *offsetP, |
1101 | void *segmentsP, |
1102 | UInt32 *numSegmentsP) |
1103 | { |
1104 | return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) fOutSeg, |
1105 | offsetP, segmentsP, numSegmentsP)); |
1106 | } |
1107 | |
1108 | IOReturn |
1109 | IODMACommand::genIOVMSegments(uint32_t op, |
1110 | InternalSegmentFunction outSegFunc, |
1111 | void *reference, |
1112 | UInt64 *offsetP, |
1113 | void *segmentsP, |
1114 | UInt32 *numSegmentsP) |
1115 | { |
1116 | IODMACommandInternal * internalState = fInternalState; |
1117 | IOOptionBits mdOp = kIOMDWalkSegments; |
1118 | IOReturn ret = kIOReturnSuccess; |
1119 | |
1120 | if (!(kWalkComplete & op) && !fActive) |
1121 | return kIOReturnNotReady; |
1122 | |
1123 | if (!offsetP || !segmentsP || !numSegmentsP || !*numSegmentsP) |
1124 | return kIOReturnBadArgument; |
1125 | |
1126 | IOMDDMAWalkSegmentArgs *state = |
1127 | (IOMDDMAWalkSegmentArgs *)(void *) fState; |
1128 | |
1129 | UInt64 offset = *offsetP + internalState->fPreparedOffset; |
1130 | UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength; |
1131 | |
1132 | if (offset >= memLength) |
1133 | return kIOReturnOverrun; |
1134 | |
1135 | if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) { |
1136 | state->fOffset = 0; |
1137 | internalState->fIOVMAddrValid = state->fIOVMAddr = 0; |
1138 | internalState->fNextRemapPage = NULL; |
1139 | internalState->fNewMD = false; |
1140 | mdOp = kIOMDFirstSegment; |
1141 | if (fMapper) |
1142 | { |
1143 | if (internalState->fLocalMapperAllocValid) |
1144 | { |
1145 | state->fMapped = kIOMDDMAWalkMappedLocal; |
1146 | state->fMappedBase = internalState->fLocalMapperAlloc; |
1147 | } |
1148 | else state->fMapped = true; |
1149 | } |
1150 | }; |
1151 | |
1152 | UInt32 segIndex = 0; |
1153 | UInt32 numSegments = *numSegmentsP; |
1154 | Segment64 curSeg = { 0, 0 }; |
1155 | bool curSegValid = false; |
1156 | addr64_t maxPhys; |
1157 | |
1158 | if (fNumAddressBits && (fNumAddressBits < 64)) |
1159 | maxPhys = (1ULL << fNumAddressBits); |
1160 | else |
1161 | maxPhys = 0; |
1162 | maxPhys--; |
1163 | |
1164 | while (internalState->fIOVMAddrValid || (state->fOffset < memLength)) |
1165 | { |
1166 | // state = next seg |
1167 | if (!internalState->fIOVMAddrValid) { |
1168 | |
1169 | IOReturn rtn; |
1170 | |
1171 | state->fOffset = offset; |
1172 | state->fLength = memLength - offset; |
1173 | |
1174 | if (internalState->fMapContig && internalState->fLocalMapperAllocValid) |
1175 | { |
1176 | state->fIOVMAddr = internalState->fLocalMapperAlloc + offset - internalState->fPreparedOffset; |
1177 | rtn = kIOReturnSuccess; |
1178 | #if 0 |
1179 | { |
1180 | uint64_t checkOffset; |
1181 | IOPhysicalLength segLen; |
1182 | for (checkOffset = 0; checkOffset < state->fLength; ) |
1183 | { |
1184 | addr64_t phys = const_cast<IOMemoryDescriptor *>(fMemory)->getPhysicalSegment(checkOffset + offset, &segLen, kIOMemoryMapperNone); |
1185 | if (fMapper->mapAddr(state->fIOVMAddr + checkOffset) != phys) |
1186 | { |
1187 | panic("%llx != %llx:%llx, %llx phys: %llx %llx\n" , offset, |
1188 | state->fIOVMAddr + checkOffset, fMapper->mapAddr(state->fIOVMAddr + checkOffset), state->fLength, |
1189 | phys, checkOffset); |
1190 | } |
1191 | checkOffset += page_size - (phys & page_mask); |
1192 | } |
1193 | } |
1194 | #endif |
1195 | } |
1196 | else |
1197 | { |
1198 | const IOMemoryDescriptor * memory = |
1199 | internalState->fCopyMD ? internalState->fCopyMD : fMemory; |
1200 | rtn = memory->dmaCommandOperation(mdOp, fState, sizeof(fState)); |
1201 | mdOp = kIOMDWalkSegments; |
1202 | } |
1203 | |
1204 | if (rtn == kIOReturnSuccess) |
1205 | { |
1206 | internalState->fIOVMAddrValid = true; |
1207 | assert(state->fLength); |
1208 | if (curSegValid && ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr)) { |
1209 | UInt64 length = state->fLength; |
1210 | offset += length; |
1211 | curSeg.fLength += length; |
1212 | internalState->fIOVMAddrValid = state->fIOVMAddr = 0; |
1213 | } |
1214 | } |
1215 | else if (rtn == kIOReturnOverrun) |
1216 | internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end |
1217 | else |
1218 | return rtn; |
1219 | } |
1220 | |
1221 | // seg = state, offset = end of seg |
1222 | if (!curSegValid) |
1223 | { |
1224 | UInt64 length = state->fLength; |
1225 | offset += length; |
1226 | curSeg.fIOVMAddr = state->fIOVMAddr; |
1227 | curSeg.fLength = length; |
1228 | curSegValid = true; |
1229 | internalState->fIOVMAddrValid = state->fIOVMAddr = 0; |
1230 | } |
1231 | |
1232 | if (!internalState->fIOVMAddrValid) |
1233 | { |
1234 | // maxPhys |
1235 | if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys) |
1236 | { |
1237 | if (internalState->fCursor) |
1238 | { |
1239 | curSegValid = curSeg.fIOVMAddr = 0; |
1240 | ret = kIOReturnMessageTooLarge; |
1241 | break; |
1242 | } |
1243 | else if (curSeg.fIOVMAddr <= maxPhys) |
1244 | { |
1245 | UInt64 remain, newLength; |
1246 | |
1247 | newLength = (maxPhys + 1 - curSeg.fIOVMAddr); |
1248 | DEBG("trunc %qx, %qx-> %qx\n" , curSeg.fIOVMAddr, curSeg.fLength, newLength); |
1249 | remain = curSeg.fLength - newLength; |
1250 | state->fIOVMAddr = newLength + curSeg.fIOVMAddr; |
1251 | internalState->fIOVMAddrValid = true; |
1252 | curSeg.fLength = newLength; |
1253 | state->fLength = remain; |
1254 | offset -= remain; |
1255 | } |
1256 | else |
1257 | { |
1258 | UInt64 addr = curSeg.fIOVMAddr; |
1259 | ppnum_t addrPage = atop_64(addr); |
1260 | vm_page_t remap = NULL; |
1261 | UInt64 remain, newLength; |
1262 | |
1263 | DEBG("sparse switch %qx, %qx " , addr, curSeg.fLength); |
1264 | |
1265 | remap = internalState->fNextRemapPage; |
1266 | if (remap && (addrPage == vm_page_get_offset(remap))) |
1267 | { |
1268 | } |
1269 | else for (remap = internalState->fCopyPageAlloc; |
1270 | remap && (addrPage != vm_page_get_offset(remap)); |
1271 | remap = vm_page_get_next(remap)) |
1272 | { |
1273 | } |
1274 | |
1275 | if (!remap) panic("no remap page found" ); |
1276 | |
1277 | curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap)) |
1278 | + (addr & PAGE_MASK); |
1279 | curSegValid = true; |
1280 | internalState->fNextRemapPage = vm_page_get_next(remap); |
1281 | |
1282 | newLength = PAGE_SIZE - (addr & PAGE_MASK); |
1283 | if (newLength < curSeg.fLength) |
1284 | { |
1285 | remain = curSeg.fLength - newLength; |
1286 | state->fIOVMAddr = addr + newLength; |
1287 | internalState->fIOVMAddrValid = true; |
1288 | curSeg.fLength = newLength; |
1289 | state->fLength = remain; |
1290 | offset -= remain; |
1291 | } |
1292 | DEBG("-> %qx, %qx offset %qx\n" , curSeg.fIOVMAddr, curSeg.fLength, offset); |
1293 | } |
1294 | } |
1295 | |
1296 | // reduce size of output segment |
1297 | uint64_t reduce, leftover = 0; |
1298 | |
1299 | // fMaxSegmentSize |
1300 | if (curSeg.fLength > fMaxSegmentSize) |
1301 | { |
1302 | leftover += curSeg.fLength - fMaxSegmentSize; |
1303 | curSeg.fLength = fMaxSegmentSize; |
1304 | state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; |
1305 | internalState->fIOVMAddrValid = true; |
1306 | } |
1307 | |
1308 | // alignment current length |
1309 | |
1310 | reduce = (curSeg.fLength & fAlignMaskLength); |
1311 | if (reduce && (curSeg.fLength > reduce)) |
1312 | { |
1313 | leftover += reduce; |
1314 | curSeg.fLength -= reduce; |
1315 | state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; |
1316 | internalState->fIOVMAddrValid = true; |
1317 | } |
1318 | |
1319 | // alignment next address |
1320 | |
1321 | reduce = (state->fIOVMAddr & fAlignMaskInternalSegments); |
1322 | if (reduce && (curSeg.fLength > reduce)) |
1323 | { |
1324 | leftover += reduce; |
1325 | curSeg.fLength -= reduce; |
1326 | state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr; |
1327 | internalState->fIOVMAddrValid = true; |
1328 | } |
1329 | |
1330 | if (leftover) |
1331 | { |
1332 | DEBG("reduce seg by 0x%llx @ 0x%llx [0x%llx, 0x%llx]\n" , |
1333 | leftover, offset, |
1334 | curSeg.fIOVMAddr, curSeg.fLength); |
1335 | state->fLength = leftover; |
1336 | offset -= leftover; |
1337 | } |
1338 | |
1339 | // |
1340 | |
1341 | if (internalState->fCursor) |
1342 | { |
1343 | bool misaligned; |
1344 | uint32_t mask; |
1345 | |
1346 | mask = (segIndex ? fAlignMaskInternalSegments : internalState->fSourceAlignMask); |
1347 | misaligned = (0 != (mask & curSeg.fIOVMAddr)); |
1348 | if (!misaligned) |
1349 | { |
1350 | mask = fAlignMaskLength; |
1351 | misaligned |= (0 != (mask & curSeg.fLength)); |
1352 | } |
1353 | if (misaligned) |
1354 | { |
1355 | if (misaligned) DEBG("cursor misaligned %qx:%qx\n" , curSeg.fIOVMAddr, curSeg.fLength); |
1356 | curSegValid = curSeg.fIOVMAddr = 0; |
1357 | ret = kIOReturnNotAligned; |
1358 | break; |
1359 | } |
1360 | } |
1361 | |
1362 | if (offset >= memLength) |
1363 | { |
1364 | curSeg.fLength -= (offset - memLength); |
1365 | offset = memLength; |
1366 | internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end |
1367 | break; |
1368 | } |
1369 | } |
1370 | |
1371 | if (internalState->fIOVMAddrValid) { |
1372 | if ((segIndex + 1 == numSegments)) |
1373 | break; |
1374 | |
1375 | ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++); |
1376 | curSegValid = curSeg.fIOVMAddr = 0; |
1377 | if (kIOReturnSuccess != ret) |
1378 | break; |
1379 | } |
1380 | } |
1381 | |
1382 | if (curSegValid) { |
1383 | ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++); |
1384 | } |
1385 | |
1386 | if (kIOReturnSuccess == ret) |
1387 | { |
1388 | state->fOffset = offset; |
1389 | *offsetP = offset - internalState->fPreparedOffset; |
1390 | *numSegmentsP = segIndex; |
1391 | } |
1392 | return ret; |
1393 | } |
1394 | |
1395 | IOReturn |
1396 | IODMACommand::clientOutputSegment( |
1397 | void *reference, IODMACommand *target, |
1398 | Segment64 segment, void *vSegList, UInt32 outSegIndex) |
1399 | { |
1400 | SegmentFunction segmentFunction = (SegmentFunction) reference; |
1401 | IOReturn ret = kIOReturnSuccess; |
1402 | |
1403 | if (target->fNumAddressBits && (target->fNumAddressBits < 64) |
1404 | && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits) |
1405 | && (target->reserved->fLocalMapperAllocValid || !target->fMapper)) |
1406 | { |
1407 | DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n" , segment.fIOVMAddr, segment.fLength); |
1408 | ret = kIOReturnMessageTooLarge; |
1409 | } |
1410 | |
1411 | if (!(*segmentFunction)(target, segment, vSegList, outSegIndex)) |
1412 | { |
1413 | DEBG("kIOReturnMessageTooLarge(fOutSeg) %qx, %qx\n" , segment.fIOVMAddr, segment.fLength); |
1414 | ret = kIOReturnMessageTooLarge; |
1415 | } |
1416 | |
1417 | return (ret); |
1418 | } |
1419 | |
1420 | IOReturn |
1421 | IODMACommand::genIOVMSegments(SegmentFunction segmentFunction, |
1422 | UInt64 *offsetP, |
1423 | void *segmentsP, |
1424 | UInt32 *numSegmentsP) |
1425 | { |
1426 | return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) segmentFunction, |
1427 | offsetP, segmentsP, numSegmentsP)); |
1428 | } |
1429 | |
1430 | bool |
1431 | IODMACommand::OutputHost32(IODMACommand *, |
1432 | Segment64 segment, void *vSegList, UInt32 outSegIndex) |
1433 | { |
1434 | Segment32 *base = (Segment32 *) vSegList; |
1435 | base[outSegIndex].fIOVMAddr = (UInt32) segment.fIOVMAddr; |
1436 | base[outSegIndex].fLength = (UInt32) segment.fLength; |
1437 | return true; |
1438 | } |
1439 | |
1440 | bool |
1441 | IODMACommand::OutputBig32(IODMACommand *, |
1442 | Segment64 segment, void *vSegList, UInt32 outSegIndex) |
1443 | { |
1444 | const UInt offAddr = outSegIndex * sizeof(Segment32); |
1445 | const UInt offLen = offAddr + sizeof(UInt32); |
1446 | OSWriteBigInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr); |
1447 | OSWriteBigInt32(vSegList, offLen, (UInt32) segment.fLength); |
1448 | return true; |
1449 | } |
1450 | |
1451 | bool |
1452 | IODMACommand::OutputLittle32(IODMACommand *, |
1453 | Segment64 segment, void *vSegList, UInt32 outSegIndex) |
1454 | { |
1455 | const UInt offAddr = outSegIndex * sizeof(Segment32); |
1456 | const UInt offLen = offAddr + sizeof(UInt32); |
1457 | OSWriteLittleInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr); |
1458 | OSWriteLittleInt32(vSegList, offLen, (UInt32) segment.fLength); |
1459 | return true; |
1460 | } |
1461 | |
1462 | bool |
1463 | IODMACommand::OutputHost64(IODMACommand *, |
1464 | Segment64 segment, void *vSegList, UInt32 outSegIndex) |
1465 | { |
1466 | Segment64 *base = (Segment64 *) vSegList; |
1467 | base[outSegIndex] = segment; |
1468 | return true; |
1469 | } |
1470 | |
1471 | bool |
1472 | IODMACommand::OutputBig64(IODMACommand *, |
1473 | Segment64 segment, void *vSegList, UInt32 outSegIndex) |
1474 | { |
1475 | const UInt offAddr = outSegIndex * sizeof(Segment64); |
1476 | const UInt offLen = offAddr + sizeof(UInt64); |
1477 | OSWriteBigInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr); |
1478 | OSWriteBigInt64(vSegList, offLen, (UInt64) segment.fLength); |
1479 | return true; |
1480 | } |
1481 | |
1482 | bool |
1483 | IODMACommand::OutputLittle64(IODMACommand *, |
1484 | Segment64 segment, void *vSegList, UInt32 outSegIndex) |
1485 | { |
1486 | const UInt offAddr = outSegIndex * sizeof(Segment64); |
1487 | const UInt offLen = offAddr + sizeof(UInt64); |
1488 | OSWriteLittleInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr); |
1489 | OSWriteLittleInt64(vSegList, offLen, (UInt64) segment.fLength); |
1490 | return true; |
1491 | } |
1492 | |
1493 | |
1494 | |