1/*
2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#define IOKIT_ENABLE_SHARED_PTR
29
30#define _IOMEMORYDESCRIPTOR_INTERNAL_
31
32#include <IOKit/assert.h>
33#include <IOKit/system.h>
34
35#include <IOKit/IOLib.h>
36#include <IOKit/IOMapper.h>
37#include <IOKit/IOBufferMemoryDescriptor.h>
38#include <libkern/OSDebug.h>
39#include <mach/mach_vm.h>
40
41#include "IOKitKernelInternal.h"
42
43#ifdef IOALLOCDEBUG
44#include <libkern/c++/OSCPPDebug.h>
45#endif
46#include <IOKit/IOStatisticsPrivate.h>
47
48#if IOKITSTATS
49#define IOStatisticsAlloc(type, size) \
50do { \
51 IOStatistics::countAlloc(type, size); \
52} while (0)
53#else
54#define IOStatisticsAlloc(type, size)
55#endif /* IOKITSTATS */
56
57
58__BEGIN_DECLS
59void ipc_port_release_send(ipc_port_t port);
60#include <vm/pmap.h>
61
62KALLOC_HEAP_DEFINE(KHEAP_IOBMD_CONTROL, "IOBMD_control", KHEAP_ID_KT_VAR);
63__END_DECLS
64
65/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
66
67enum{
68 kInternalFlagPhysical = 0x00000001,
69 kInternalFlagPageSized = 0x00000002,
70 kInternalFlagPageAllocated = 0x00000004,
71 kInternalFlagInit = 0x00000008,
72 kInternalFlagHasPointers = 0x00000010,
73 kInternalFlagGuardPages = 0x00000020,
74};
75
76/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
77
78#define super IOGeneralMemoryDescriptor
79OSDefineMetaClassAndStructorsWithZone(IOBufferMemoryDescriptor,
80 IOGeneralMemoryDescriptor, ZC_ZFREE_CLEARMEM);
81
82/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
83
84#if defined(__x86_64__)
85static uintptr_t
86IOBMDPageProc(kalloc_heap_t kheap, iopa_t * a)
87{
88 kern_return_t kr;
89 vm_address_t vmaddr = 0;
90 kma_flags_t kma_flags = KMA_ZERO;
91
92 if (kheap == KHEAP_DATA_BUFFERS) {
93 kma_flags = (kma_flags_t) (kma_flags | KMA_DATA);
94 }
95 kr = kmem_alloc(kernel_map, &vmaddr, page_size,
96 kma_flags, VM_KERN_MEMORY_IOKIT);
97
98 if (KERN_SUCCESS != kr) {
99 vmaddr = 0;
100 }
101
102 return (uintptr_t) vmaddr;
103}
104#endif /* defined(__x86_64__) */
105
106/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
107
108#ifndef __LP64__
109bool
110IOBufferMemoryDescriptor::initWithOptions(
111 IOOptionBits options,
112 vm_size_t capacity,
113 vm_offset_t alignment,
114 task_t inTask)
115{
116 mach_vm_address_t physicalMask = 0;
117 return initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask);
118}
119#endif /* !__LP64__ */
120
121OSSharedPtr<IOBufferMemoryDescriptor>
122IOBufferMemoryDescriptor::withCopy(
123 task_t inTask,
124 IOOptionBits options,
125 vm_map_t sourceMap,
126 mach_vm_address_t source,
127 mach_vm_size_t size)
128{
129 OSSharedPtr<IOBufferMemoryDescriptor> inst;
130 kern_return_t err;
131 vm_map_copy_t copy;
132 vm_map_address_t address;
133
134 copy = NULL;
135 do {
136 err = kIOReturnNoMemory;
137 inst = OSMakeShared<IOBufferMemoryDescriptor>();
138 if (!inst) {
139 break;
140 }
141 inst->_ranges.v64 = IOMallocType(IOAddressRange);
142
143 err = vm_map_copyin(src_map: sourceMap, src_addr: source, len: size,
144 src_destroy: false /* src_destroy */, copy_result: &copy);
145 if (KERN_SUCCESS != err) {
146 break;
147 }
148
149 err = vm_map_copyout(dst_map: get_task_map(inTask), dst_addr: &address, copy);
150 if (KERN_SUCCESS != err) {
151 break;
152 }
153 copy = NULL;
154
155 inst->_ranges.v64->address = address;
156 inst->_ranges.v64->length = size;
157
158 if (!inst->initWithPhysicalMask(inTask, options, capacity: size, alignment: page_size, physicalMask: 0)) {
159 err = kIOReturnError;
160 }
161 } while (false);
162
163 if (KERN_SUCCESS == err) {
164 return inst;
165 }
166
167 if (copy) {
168 vm_map_copy_discard(copy);
169 }
170
171 return nullptr;
172}
173
174
175bool
176IOBufferMemoryDescriptor::initWithPhysicalMask(
177 task_t inTask,
178 IOOptionBits options,
179 mach_vm_size_t capacity,
180 mach_vm_address_t alignment,
181 mach_vm_address_t physicalMask)
182{
183 task_t mapTask = NULL;
184 kalloc_heap_t kheap = KHEAP_DATA_BUFFERS;
185 mach_vm_address_t highestMask = 0;
186 IOOptionBits iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference;
187 IODMAMapSpecification mapSpec;
188 bool mapped = false;
189 bool withCopy = false;
190 bool mappedOrShared = false;
191
192 if (!capacity) {
193 return false;
194 }
195
196 /*
197 * The IOKit constructor requests the allocator for zeroed memory
198 * so the members of the class do not need to be explicitly zeroed.
199 */
200 _options = options;
201 _capacity = capacity;
202
203 if (!_ranges.v64) {
204 _ranges.v64 = IOMallocType(IOAddressRange);
205 _ranges.v64->address = 0;
206 _ranges.v64->length = 0;
207 } else {
208 if (!_ranges.v64->address) {
209 return false;
210 }
211 if (!(kIOMemoryPageable & options)) {
212 return false;
213 }
214 if (!inTask) {
215 return false;
216 }
217 _buffer = (void *) _ranges.v64->address;
218 withCopy = true;
219 }
220
221 /*
222 * Set kalloc_heap to KHEAP_IOBMD_CONTROL if allocation contains pointers
223 */
224 if (kInternalFlagHasPointers & _internalFlags) {
225 kheap = KHEAP_IOBMD_CONTROL;
226 }
227
228 // make sure super::free doesn't dealloc _ranges before super::init
229 _flags = kIOMemoryAsReference;
230
231 // Grab IOMD bits from the Buffer MD options
232 iomdOptions |= (options & kIOBufferDescriptorMemoryFlags);
233
234 if (!(kIOMemoryMapperNone & options)) {
235 IOMapper::checkForSystemMapper();
236 mapped = (NULL != IOMapper::gSystem);
237 }
238
239 if (physicalMask && (alignment <= 1)) {
240 alignment = ((physicalMask ^ (-1ULL)) & (physicalMask - 1));
241 highestMask = (physicalMask | alignment);
242 alignment++;
243 if (alignment < page_size) {
244 alignment = page_size;
245 }
246 }
247
248 if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size)) {
249 alignment = page_size;
250 }
251
252 if (alignment >= page_size) {
253 if (round_page_overflow(capacity, &capacity)) {
254 return false;
255 }
256 }
257
258 if (alignment > page_size) {
259 options |= kIOMemoryPhysicallyContiguous;
260 }
261
262 _alignment = alignment;
263
264 if ((capacity + alignment) < _capacity) {
265 return false;
266 }
267
268 if ((inTask != kernel_task) && !(options & kIOMemoryPageable)) {
269 return false;
270 }
271
272 bzero(s: &mapSpec, n: sizeof(mapSpec));
273 mapSpec.alignment = _alignment;
274 mapSpec.numAddressBits = 64;
275 if (highestMask && mapped) {
276 if (highestMask <= 0xFFFFFFFF) {
277 mapSpec.numAddressBits = (uint8_t)(32 - __builtin_clz((unsigned int) highestMask));
278 } else {
279 mapSpec.numAddressBits = (uint8_t)(64 - __builtin_clz((unsigned int) (highestMask >> 32)));
280 }
281 highestMask = 0;
282 }
283
284 // set memory entry cache mode, pageable, purgeable
285 iomdOptions |= ((options & kIOMapCacheMask) >> kIOMapCacheShift) << kIOMemoryBufferCacheShift;
286 if (options & kIOMemoryPageable) {
287 if (_internalFlags & kInternalFlagGuardPages) {
288 printf("IOBMD: Unsupported use of guard pages with pageable memory.\n");
289 return false;
290 }
291 iomdOptions |= kIOMemoryBufferPageable;
292 if (options & kIOMemoryPurgeable) {
293 iomdOptions |= kIOMemoryBufferPurgeable;
294 }
295 } else {
296 // Buffer shouldn't auto prepare they should be prepared explicitly
297 // But it never was enforced so what are you going to do?
298 iomdOptions |= kIOMemoryAutoPrepare;
299
300 /* Allocate a wired-down buffer inside kernel space. */
301
302 bool contig = (0 != (options & kIOMemoryHostPhysicallyContiguous));
303
304 if (!contig && (0 != (options & kIOMemoryPhysicallyContiguous))) {
305 contig |= (!mapped);
306 contig |= (0 != (kIOMemoryMapperNone & options));
307#if 0
308 // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now
309 contig |= true;
310#endif
311 }
312
313 mappedOrShared = (mapped || (0 != (kIOMemorySharingTypeMask & options)));
314 if (contig || highestMask || (alignment > page_size)) {
315 if (_internalFlags & kInternalFlagGuardPages) {
316 printf("IOBMD: Unsupported use of guard pages with physical mask or contiguous memory.\n");
317 return false;
318 }
319 _internalFlags |= kInternalFlagPhysical;
320 if (highestMask) {
321 _internalFlags |= kInternalFlagPageSized;
322 if (round_page_overflow(capacity, &capacity)) {
323 return false;
324 }
325 }
326 _buffer = (void *) IOKernelAllocateWithPhysicalRestrict(kheap,
327 size: capacity, maxPhys: highestMask, alignment, contiguous: contig);
328 } else if (_internalFlags & kInternalFlagGuardPages) {
329 vm_offset_t address = 0;
330 kern_return_t kr;
331 uintptr_t alignMask;
332 kma_flags_t kma_flags = (kma_flags_t) (KMA_GUARD_FIRST |
333 KMA_GUARD_LAST | KMA_ZERO);
334
335 if (((uint32_t) alignment) != alignment) {
336 return false;
337 }
338 if (kheap == KHEAP_DATA_BUFFERS) {
339 kma_flags = (kma_flags_t) (kma_flags | KMA_DATA);
340 }
341
342 alignMask = (1UL << log2up(size: (uint32_t) alignment)) - 1;
343 kr = kernel_memory_allocate(map: kernel_map, addrp: &address,
344 size: capacity + page_size * 2, mask: alignMask, flags: kma_flags,
345 tag: IOMemoryTag(map: kernel_map));
346 if (kr != KERN_SUCCESS || address == 0) {
347 return false;
348 }
349#if IOALLOCDEBUG
350 OSAddAtomicLong(capacity, &debug_iomalloc_size);
351#endif
352 IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity);
353 _buffer = (void *)(address + page_size);
354#if defined(__x86_64__)
355 } else if (mappedOrShared
356 && (capacity + alignment) <= (page_size - gIOPageAllocChunkBytes)) {
357 _internalFlags |= kInternalFlagPageAllocated;
358 _buffer = (void *) iopa_alloc(&gIOBMDPageAllocator,
359 &IOBMDPageProc, kheap, capacity, alignment);
360 if (_buffer) {
361 bzero(_buffer, capacity);
362 IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity);
363#if IOALLOCDEBUG
364 OSAddAtomicLong(capacity, &debug_iomalloc_size);
365#endif
366 }
367#endif /* defined(__x86_64__) */
368 } else if (alignment > 1) {
369 /* BEGIN IGNORE CODESTYLE */
370 __typed_allocators_ignore_push
371 _buffer = IOMallocAligned_internal(kalloc_heap_cfg: kheap, size: capacity, alignment,
372 flags: Z_ZERO_VM_TAG_BT_BIT);
373 } else {
374 _buffer = IOMalloc_internal(kalloc_heap_cfg: kheap, size: capacity, flags: Z_ZERO_VM_TAG_BT_BIT);
375 __typed_allocators_ignore_pop
376 /* END IGNORE CODESTYLE */
377 }
378 if (!_buffer) {
379 return false;
380 }
381 }
382
383 if ((options & (kIOMemoryPageable | kIOMapCacheMask))) {
384 vm_size_t size = round_page(x: capacity);
385
386 // initWithOptions will create memory entry
387 if (!withCopy) {
388 iomdOptions |= kIOMemoryPersistent;
389 }
390
391 if (options & kIOMemoryPageable) {
392#if IOALLOCDEBUG
393 OSAddAtomicLong(size, &debug_iomallocpageable_size);
394#endif
395 if (!withCopy) {
396 mapTask = inTask;
397 }
398 if (NULL == inTask) {
399 inTask = kernel_task;
400 }
401 } else if (options & kIOMapCacheMask) {
402 // Prefetch each page to put entries into the pmap
403 volatile UInt8 * startAddr = (UInt8 *)_buffer;
404 volatile UInt8 * endAddr = (UInt8 *)_buffer + capacity;
405
406 while (startAddr < endAddr) {
407 UInt8 dummyVar = *startAddr;
408 (void) dummyVar;
409 startAddr += page_size;
410 }
411 }
412 }
413
414 _ranges.v64->address = (mach_vm_address_t) pgz_decode(_buffer, _capacity);
415 _ranges.v64->length = _capacity;
416
417 if (!super::initWithOptions(buffers: _ranges.v64, count: 1, offset: 0,
418 task: inTask, options: iomdOptions, /* System mapper */ NULL)) {
419 return false;
420 }
421
422 _internalFlags |= kInternalFlagInit;
423#if IOTRACKING
424 if (!(options & kIOMemoryPageable)) {
425 trackingAccumSize(capacity);
426 }
427#endif /* IOTRACKING */
428
429 // give any system mapper the allocation params
430 if (kIOReturnSuccess != dmaCommandOperation(op: kIOMDAddDMAMapSpec,
431 vData: &mapSpec, dataSize: sizeof(mapSpec))) {
432 return false;
433 }
434
435 if (mapTask) {
436 if (!reserved) {
437 reserved = IOMallocType(ExpansionData);
438 if (!reserved) {
439 return false;
440 }
441 }
442 reserved->map = createMappingInTask(intoTask: mapTask, atAddress: 0,
443 options: kIOMapAnywhere | (options & kIOMapPrefault) | (options & kIOMapCacheMask), offset: 0, length: 0).detach();
444 if (!reserved->map) {
445 _buffer = NULL;
446 return false;
447 }
448 release(); // map took a retain on this
449 reserved->map->retain();
450 removeMapping(mapping: reserved->map);
451 mach_vm_address_t buffer = reserved->map->getAddress();
452 _buffer = (void *) buffer;
453 if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions)) {
454 _ranges.v64->address = buffer;
455 }
456 }
457
458 setLength(_capacity);
459
460 return true;
461}
462
463bool
464IOBufferMemoryDescriptor::initControlWithPhysicalMask(
465 task_t inTask,
466 IOOptionBits options,
467 mach_vm_size_t capacity,
468 mach_vm_address_t alignment,
469 mach_vm_address_t physicalMask)
470{
471 _internalFlags = kInternalFlagHasPointers;
472 return initWithPhysicalMask(inTask, options, capacity, alignment,
473 physicalMask);
474}
475
476bool
477IOBufferMemoryDescriptor::initWithGuardPages(
478 task_t inTask,
479 IOOptionBits options,
480 mach_vm_size_t capacity)
481{
482 mach_vm_size_t roundedCapacity;
483
484 _internalFlags = kInternalFlagGuardPages;
485
486 if (round_page_overflow(capacity, &roundedCapacity)) {
487 return false;
488 }
489
490 return initWithPhysicalMask(inTask, options, capacity: roundedCapacity, alignment: page_size,
491 physicalMask: (mach_vm_address_t)0);
492}
493
494OSSharedPtr<IOBufferMemoryDescriptor>
495IOBufferMemoryDescriptor::inTaskWithOptions(
496 task_t inTask,
497 IOOptionBits options,
498 vm_size_t capacity,
499 vm_offset_t alignment)
500{
501 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
502
503 if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask: 0)) {
504 me.reset();
505 }
506 return me;
507}
508
509OSSharedPtr<IOBufferMemoryDescriptor>
510IOBufferMemoryDescriptor::inTaskWithOptions(
511 task_t inTask,
512 IOOptionBits options,
513 vm_size_t capacity,
514 vm_offset_t alignment,
515 uint32_t kernTag,
516 uint32_t userTag)
517{
518 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
519
520 if (me) {
521 me->setVMTags(kernelTag: kernTag, userTag);
522
523 if (!me->initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask: 0)) {
524 me.reset();
525 }
526 }
527 return me;
528}
529
530OSSharedPtr<IOBufferMemoryDescriptor>
531IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
532 task_t inTask,
533 IOOptionBits options,
534 mach_vm_size_t capacity,
535 mach_vm_address_t physicalMask)
536{
537 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
538
539 if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment: 1, physicalMask)) {
540 me.reset();
541 }
542 return me;
543}
544
545OSSharedPtr<IOBufferMemoryDescriptor>
546IOBufferMemoryDescriptor::inTaskWithGuardPages(
547 task_t inTask,
548 IOOptionBits options,
549 mach_vm_size_t capacity)
550{
551 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
552
553 if (me && !me->initWithGuardPages(inTask, options, capacity)) {
554 me.reset();
555 }
556 return me;
557}
558
559#ifndef __LP64__
560bool
561IOBufferMemoryDescriptor::initWithOptions(
562 IOOptionBits options,
563 vm_size_t capacity,
564 vm_offset_t alignment)
565{
566 return initWithPhysicalMask(kernel_task, options, capacity, alignment, (mach_vm_address_t)0);
567}
568#endif /* !__LP64__ */
569
570OSSharedPtr<IOBufferMemoryDescriptor>
571IOBufferMemoryDescriptor::withOptions(
572 IOOptionBits options,
573 vm_size_t capacity,
574 vm_offset_t alignment)
575{
576 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
577
578 if (me && !me->initWithPhysicalMask(inTask: kernel_task, options, capacity, alignment, physicalMask: 0)) {
579 me.reset();
580 }
581 return me;
582}
583
584
585/*
586 * withCapacity:
587 *
588 * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
589 * hold capacity bytes. The descriptor's length is initially set to the capacity.
590 */
591OSSharedPtr<IOBufferMemoryDescriptor>
592IOBufferMemoryDescriptor::withCapacity(vm_size_t inCapacity,
593 IODirection inDirection,
594 bool inContiguous)
595{
596 return IOBufferMemoryDescriptor::withOptions(
597 options: inDirection | kIOMemoryUnshared
598 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
599 capacity: inCapacity, alignment: inContiguous ? inCapacity : 1 );
600}
601
602#ifndef __LP64__
603/*
604 * initWithBytes:
605 *
606 * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
607 * The descriptor's length and capacity are set to the input buffer's size.
608 */
609bool
610IOBufferMemoryDescriptor::initWithBytes(const void * inBytes,
611 vm_size_t inLength,
612 IODirection inDirection,
613 bool inContiguous)
614{
615 if (!initWithPhysicalMask(kernel_task, inDirection | kIOMemoryUnshared
616 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
617 inLength, inLength, (mach_vm_address_t)0)) {
618 return false;
619 }
620
621 // start out with no data
622 setLength(0);
623
624 if (!appendBytes(inBytes, inLength)) {
625 return false;
626 }
627
628 return true;
629}
630#endif /* !__LP64__ */
631
632/*
633 * withBytes:
634 *
635 * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
636 * The descriptor's length and capacity are set to the input buffer's size.
637 */
638OSSharedPtr<IOBufferMemoryDescriptor>
639IOBufferMemoryDescriptor::withBytes(const void * inBytes,
640 vm_size_t inLength,
641 IODirection inDirection,
642 bool inContiguous)
643{
644 OSSharedPtr<IOBufferMemoryDescriptor> me = OSMakeShared<IOBufferMemoryDescriptor>();
645 mach_vm_address_t alignment;
646
647 alignment = (inLength <= page_size) ? inLength : page_size;
648 if (me && !me->initWithPhysicalMask(
649 inTask: kernel_task, options: inDirection | kIOMemoryUnshared
650 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
651 capacity: inLength, alignment, physicalMask: 0 )) {
652 me.reset();
653 }
654
655 if (me) {
656 // start out with no data
657 me->setLength(0);
658
659 if (!me->appendBytes(bytes: inBytes, withLength: inLength)) {
660 me.reset();
661 }
662 }
663 return me;
664}
665
666/*
667 * free:
668 *
669 * Free resources
670 */
671void
672IOBufferMemoryDescriptor::free()
673{
674 // Cache all of the relevant information on the stack for use
675 // after we call super::free()!
676 IOOptionBits flags = _flags;
677 IOOptionBits internalFlags = _internalFlags;
678 IOOptionBits options = _options;
679 vm_size_t size = _capacity;
680 void * buffer = _buffer;
681 IOMemoryMap * map = NULL;
682 IOAddressRange * range = _ranges.v64;
683 vm_offset_t alignment = _alignment;
684 kalloc_heap_t kheap = KHEAP_DATA_BUFFERS;
685 vm_size_t rsize;
686
687 if (alignment >= page_size) {
688 if (!round_page_overflow(size, &rsize)) {
689 size = rsize;
690 }
691 }
692
693 if (reserved) {
694 map = reserved->map;
695 IOFreeType(reserved, ExpansionData);
696 if (map) {
697 map->release();
698 }
699 }
700
701 if ((options & kIOMemoryPageable)
702 || (kInternalFlagPageSized & internalFlags)) {
703 if (!round_page_overflow(size, &rsize)) {
704 size = rsize;
705 }
706 }
707
708 if (internalFlags & kInternalFlagHasPointers) {
709 kheap = KHEAP_IOBMD_CONTROL;
710 }
711
712#if IOTRACKING
713 if (!(options & kIOMemoryPageable)
714 && buffer
715 && (kInternalFlagInit & _internalFlags)) {
716 trackingAccumSize(-size);
717 }
718#endif /* IOTRACKING */
719
720 /* super::free may unwire - deallocate buffer afterwards */
721 super::free();
722
723 if (options & kIOMemoryPageable) {
724#if IOALLOCDEBUG
725 OSAddAtomicLong(-size, &debug_iomallocpageable_size);
726#endif
727 } else if (buffer) {
728 if (kInternalFlagPhysical & internalFlags) {
729 IOKernelFreePhysical(kheap, address: (mach_vm_address_t) buffer, size);
730 } else if (kInternalFlagPageAllocated & internalFlags) {
731#if defined(__x86_64__)
732 uintptr_t page;
733 page = iopa_free(&gIOBMDPageAllocator, (uintptr_t) buffer, size);
734 if (page) {
735 kmem_free(kernel_map, page, page_size);
736 }
737#if IOALLOCDEBUG
738 OSAddAtomicLong(-size, &debug_iomalloc_size);
739#endif
740 IOStatisticsAlloc(kIOStatisticsFreeAligned, size);
741#else /* !defined(__x86_64__) */
742 /* should be unreachable */
743 panic("Attempting to free IOBMD with page allocated flag");
744#endif /* defined(__x86_64__) */
745 } else if (kInternalFlagGuardPages & internalFlags) {
746 vm_offset_t allocation = (vm_offset_t)buffer - page_size;
747 kmem_free(map: kernel_map, addr: allocation, size: size + page_size * 2);
748#if IOALLOCDEBUG
749 OSAddAtomicLong(-size, &debug_iomalloc_size);
750#endif
751 IOStatisticsAlloc(kIOStatisticsFreeAligned, size);
752 } else if (alignment > 1) {
753 /* BEGIN IGNORE CODESTYLE */
754 __typed_allocators_ignore_push
755 IOFreeAligned_internal(kalloc_heap_cfg: kheap, address: buffer, size);
756 } else {
757 IOFree_internal(kalloc_heap_cfg: kheap, inAddress: buffer, size);
758 __typed_allocators_ignore_pop
759 /* END IGNORE CODESTYLE */
760 }
761 }
762 if (range && (kIOMemoryAsReference & flags)) {
763 IOFreeType(range, IOAddressRange);
764 }
765}
766
767/*
768 * getCapacity:
769 *
770 * Get the buffer capacity
771 */
772vm_size_t
773IOBufferMemoryDescriptor::getCapacity() const
774{
775 return _capacity;
776}
777
778/*
779 * setLength:
780 *
781 * Change the buffer length of the memory descriptor. When a new buffer
782 * is created, the initial length of the buffer is set to be the same as
783 * the capacity. The length can be adjusted via setLength for a shorter
784 * transfer (there is no need to create more buffer descriptors when you
785 * can reuse an existing one, even for different transfer sizes). Note
786 * that the specified length must not exceed the capacity of the buffer.
787 */
788void
789IOBufferMemoryDescriptor::setLength(vm_size_t length)
790{
791 assert(length <= _capacity);
792 if (length > _capacity) {
793 return;
794 }
795
796 _length = length;
797 _ranges.v64->length = length;
798}
799
800/*
801 * setDirection:
802 *
803 * Change the direction of the transfer. This method allows one to redirect
804 * the descriptor's transfer direction. This eliminates the need to destroy
805 * and create new buffers when different transfer directions are needed.
806 */
807void
808IOBufferMemoryDescriptor::setDirection(IODirection direction)
809{
810 _flags = (_flags & ~kIOMemoryDirectionMask) | direction;
811#ifndef __LP64__
812 _direction = (IODirection) (_flags & kIOMemoryDirectionMask);
813#endif /* !__LP64__ */
814}
815
816/*
817 * appendBytes:
818 *
819 * Add some data to the end of the buffer. This method automatically
820 * maintains the memory descriptor buffer length. Note that appendBytes
821 * will not copy past the end of the memory descriptor's current capacity.
822 */
823bool
824IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
825{
826 vm_size_t actualBytesToCopy = min(withLength, _capacity - _length);
827 IOByteCount offset;
828
829 assert(_length <= _capacity);
830
831 offset = _length;
832 _length += actualBytesToCopy;
833 _ranges.v64->length += actualBytesToCopy;
834
835 if (_task == kernel_task) {
836 bcopy(/* from */ src: bytes, dst: (void *)(_ranges.v64->address + offset),
837 n: actualBytesToCopy);
838 } else {
839 writeBytes(offset, bytes, withLength: actualBytesToCopy);
840 }
841
842 return true;
843}
844
845/*
846 * getBytesNoCopy:
847 *
848 * Return the virtual address of the beginning of the buffer
849 */
850void *
851IOBufferMemoryDescriptor::getBytesNoCopy()
852{
853 if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask)) {
854 return _buffer;
855 } else {
856 return (void *)_ranges.v64->address;
857 }
858}
859
860
861/*
862 * getBytesNoCopy:
863 *
864 * Return the virtual address of an offset from the beginning of the buffer
865 */
866void *
867IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
868{
869 IOVirtualAddress address;
870
871 if ((start + withLength) < start) {
872 return NULL;
873 }
874
875 if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask)) {
876 address = (IOVirtualAddress) _buffer;
877 } else {
878 address = _ranges.v64->address;
879 }
880
881 if (start < _length && (start + withLength) <= _length) {
882 return (void *)(address + start);
883 }
884 return NULL;
885}
886
887#ifndef __LP64__
888void *
889IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset,
890 IOByteCount * lengthOfSegment)
891{
892 void * bytes = getBytesNoCopy(offset, 0);
893
894 if (bytes && lengthOfSegment) {
895 *lengthOfSegment = _length - offset;
896 }
897
898 return bytes;
899}
900#endif /* !__LP64__ */
901
902#ifdef __LP64__
903OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 0);
904OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1);
905#else /* !__LP64__ */
906OSMetaClassDefineReservedUsedX86(IOBufferMemoryDescriptor, 0);
907OSMetaClassDefineReservedUsedX86(IOBufferMemoryDescriptor, 1);
908#endif /* !__LP64__ */
909OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2);
910OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3);
911OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4);
912OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 5);
913OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 6);
914OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 7);
915OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 8);
916OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 9);
917OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 10);
918OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 11);
919OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 12);
920OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 13);
921OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 14);
922OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 15);
923