1 | /* |
2 | * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * HISTORY |
30 | * |
31 | * 17-Apr-91 Portions from libIO.m, Doug Mitchell at NeXT. |
32 | * 17-Nov-98 cpp |
33 | * |
34 | */ |
35 | |
36 | #include <IOKit/system.h> |
37 | #include <mach/sync_policy.h> |
38 | #include <machine/machine_routines.h> |
39 | #include <vm/vm_kern.h> |
40 | #include <libkern/c++/OSCPPDebug.h> |
41 | |
42 | #include <IOKit/assert.h> |
43 | |
44 | #include <IOKit/IOReturn.h> |
45 | #include <IOKit/IOLib.h> |
46 | #include <IOKit/IOLocks.h> |
47 | #include <IOKit/IOMapper.h> |
48 | #include <IOKit/IOBufferMemoryDescriptor.h> |
49 | #include <IOKit/IOKitDebug.h> |
50 | |
51 | #include "IOKitKernelInternal.h" |
52 | |
53 | #ifdef IOALLOCDEBUG |
54 | #include <libkern/OSDebug.h> |
55 | #include <sys/sysctl.h> |
56 | #endif |
57 | |
58 | #include "libkern/OSAtomic.h" |
59 | #include <libkern/c++/OSKext.h> |
60 | #include <IOKit/IOStatisticsPrivate.h> |
61 | #include <os/log_private.h> |
62 | #include <sys/msgbuf.h> |
63 | #include <console/serial_protos.h> |
64 | |
65 | #if IOKITSTATS |
66 | |
67 | #define IOStatisticsAlloc(type, size) \ |
68 | do { \ |
69 | IOStatistics::countAlloc(type, size); \ |
70 | } while (0) |
71 | |
72 | #else |
73 | |
74 | #define IOStatisticsAlloc(type, size) |
75 | |
76 | #endif /* IOKITSTATS */ |
77 | |
78 | |
79 | #define TRACK_ALLOC (IOTRACKING && (kIOTracking & gIOKitDebug)) |
80 | |
81 | |
82 | extern "C" |
83 | { |
84 | mach_timespec_t IOZeroTvalspec = { .tv_sec: 0, .tv_nsec: 0 }; |
85 | |
86 | extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); |
87 | |
88 | extern int |
89 | __doprnt( |
90 | const char *fmt, |
91 | va_list argp, |
92 | void (*putc)(int, void *), |
93 | void *arg, |
94 | int radix, |
95 | int is_log); |
96 | |
97 | extern bool bsd_log_lock(bool); |
98 | extern void bsd_log_unlock(void); |
99 | |
100 | |
101 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
102 | |
103 | lck_grp_t *IOLockGroup; |
104 | |
105 | /* |
106 | * Global variables for use by iLogger |
107 | * These symbols are for use only by Apple diagnostic code. |
108 | * Binary compatibility is not guaranteed for kexts that reference these symbols. |
109 | */ |
110 | |
111 | void *_giDebugLogInternal = NULL; |
112 | void *_giDebugLogDataInternal = NULL; |
113 | void *_giDebugReserved1 = NULL; |
114 | void *_giDebugReserved2 = NULL; |
115 | |
116 | #if defined(__x86_64__) |
117 | iopa_t gIOBMDPageAllocator; |
118 | #endif /* defined(__x86_64__) */ |
119 | |
120 | /* |
121 | * Static variables for this module. |
122 | */ |
123 | |
124 | static queue_head_t gIOMallocContiguousEntries; |
125 | static lck_mtx_t * gIOMallocContiguousEntriesLock; |
126 | |
127 | #if __x86_64__ |
128 | enum { kIOMaxPageableMaps = 8 }; |
129 | enum { kIOMaxFixedRanges = 4 }; |
130 | enum { kIOPageableMapSize = 512 * 1024 * 1024 }; |
131 | enum { kIOPageableMaxMapSize = 512 * 1024 * 1024 }; |
132 | #else |
133 | enum { kIOMaxPageableMaps = 16 }; |
134 | enum { kIOMaxFixedRanges = 4 }; |
135 | enum { kIOPageableMapSize = 96 * 1024 * 1024 }; |
136 | enum { kIOPageableMaxMapSize = 96 * 1024 * 1024 }; |
137 | #endif |
138 | |
139 | typedef struct { |
140 | vm_map_t map; |
141 | vm_offset_t address; |
142 | vm_offset_t end; |
143 | } IOMapData; |
144 | |
145 | static SECURITY_READ_ONLY_LATE(struct mach_vm_range) |
146 | gIOKitPageableFixedRanges[kIOMaxFixedRanges]; |
147 | |
148 | static struct { |
149 | UInt32 count; |
150 | UInt32 hint; |
151 | IOMapData maps[kIOMaxPageableMaps]; |
152 | lck_mtx_t * lock; |
153 | } gIOKitPageableSpace; |
154 | |
155 | #if defined(__x86_64__) |
156 | static iopa_t gIOPageablePageAllocator; |
157 | |
158 | uint32_t gIOPageAllocChunkBytes; |
159 | #endif /* defined(__x86_64__) */ |
160 | |
161 | #if IOTRACKING |
162 | IOTrackingQueue * gIOMallocTracking; |
163 | IOTrackingQueue * gIOWireTracking; |
164 | IOTrackingQueue * gIOMapTracking; |
165 | #endif /* IOTRACKING */ |
166 | |
167 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
168 | |
169 | KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed0, |
170 | &gIOKitPageableFixedRanges[0], kIOPageableMapSize); |
171 | KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed1, |
172 | &gIOKitPageableFixedRanges[1], kIOPageableMapSize); |
173 | KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed2, |
174 | &gIOKitPageableFixedRanges[2], kIOPageableMapSize); |
175 | KMEM_RANGE_REGISTER_STATIC(gIOKitPageableFixed3, |
176 | &gIOKitPageableFixedRanges[3], kIOPageableMapSize); |
177 | void |
178 | IOLibInit(void) |
179 | { |
180 | static bool libInitialized; |
181 | |
182 | if (libInitialized) { |
183 | return; |
184 | } |
185 | |
186 | IOLockGroup = lck_grp_alloc_init(grp_name: "IOKit" , LCK_GRP_ATTR_NULL); |
187 | |
188 | #if IOTRACKING |
189 | IOTrackingInit(); |
190 | gIOMallocTracking = IOTrackingQueueAlloc(kIOMallocTrackingName, 0, 0, 0, |
191 | kIOTrackingQueueTypeAlloc, |
192 | 37); |
193 | gIOWireTracking = IOTrackingQueueAlloc(kIOWireTrackingName, 0, 0, page_size, 0, 0); |
194 | |
195 | size_t mapCaptureSize = (kIOTracking & gIOKitDebug) ? page_size : (1024 * 1024); |
196 | gIOMapTracking = IOTrackingQueueAlloc(kIOMapTrackingName, 0, 0, mapCaptureSize, |
197 | kIOTrackingQueueTypeDefaultOn |
198 | | kIOTrackingQueueTypeMap |
199 | | kIOTrackingQueueTypeUser, |
200 | 0); |
201 | #endif |
202 | |
203 | gIOKitPageableSpace.maps[0].map = kmem_suballoc(parent: kernel_map, |
204 | addr: &gIOKitPageableFixedRanges[0].min_address, |
205 | size: kIOPageableMapSize, |
206 | vmc_options: VM_MAP_CREATE_PAGEABLE, |
207 | VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, |
208 | flags: (kms_flags_t)(KMS_PERMANENT | KMS_DATA | KMS_NOFAIL), |
209 | VM_KERN_MEMORY_IOKIT).kmr_submap; |
210 | |
211 | gIOKitPageableSpace.maps[0].address = gIOKitPageableFixedRanges[0].min_address; |
212 | gIOKitPageableSpace.maps[0].end = gIOKitPageableFixedRanges[0].max_address; |
213 | gIOKitPageableSpace.lock = lck_mtx_alloc_init(grp: IOLockGroup, LCK_ATTR_NULL); |
214 | gIOKitPageableSpace.hint = 0; |
215 | gIOKitPageableSpace.count = 1; |
216 | |
217 | gIOMallocContiguousEntriesLock = lck_mtx_alloc_init(grp: IOLockGroup, LCK_ATTR_NULL); |
218 | queue_init( &gIOMallocContiguousEntries ); |
219 | |
220 | #if defined(__x86_64__) |
221 | gIOPageAllocChunkBytes = PAGE_SIZE / 64; |
222 | |
223 | assert(sizeof(iopa_page_t) <= gIOPageAllocChunkBytes); |
224 | iopa_init(&gIOBMDPageAllocator); |
225 | iopa_init(&gIOPageablePageAllocator); |
226 | #endif /* defined(__x86_64__) */ |
227 | |
228 | |
229 | libInitialized = true; |
230 | } |
231 | |
232 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
233 | |
234 | vm_size_t |
235 | log2up(vm_size_t size) |
236 | { |
237 | if (size <= 1) { |
238 | size = 0; |
239 | } else { |
240 | #if __LP64__ |
241 | size = 64 - __builtin_clzl(size - 1); |
242 | #else |
243 | size = 32 - __builtin_clzl(size - 1); |
244 | #endif |
245 | } |
246 | return size; |
247 | } |
248 | |
249 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
250 | |
251 | IOThread |
252 | IOCreateThread(IOThreadFunc fcn, void *arg) |
253 | { |
254 | kern_return_t result; |
255 | thread_t thread; |
256 | |
257 | result = kernel_thread_start(continuation: (thread_continue_t)(void (*)(void))fcn, parameter: arg, new_thread: &thread); |
258 | if (result != KERN_SUCCESS) { |
259 | return NULL; |
260 | } |
261 | |
262 | thread_deallocate(thread); |
263 | |
264 | return thread; |
265 | } |
266 | |
267 | |
268 | void |
269 | IOExitThread(void) |
270 | { |
271 | (void) thread_terminate(target_act: current_thread()); |
272 | } |
273 | |
274 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
275 | |
276 | #if IOTRACKING |
277 | struct IOLibMallocHeader { |
278 | IOTrackingAddress tracking; |
279 | }; |
280 | #endif |
281 | |
282 | #if IOTRACKING |
283 | #define sizeofIOLibMallocHeader (sizeof(IOLibMallocHeader) - (TRACK_ALLOC ? 0 : sizeof(IOTrackingAddress))) |
284 | #else |
285 | #define (0) |
286 | #endif |
287 | |
288 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
289 | |
290 | __typed_allocators_ignore_push // allocator implementation |
291 | |
292 | void * |
293 | (IOMalloc_internal)(struct kalloc_heap *kheap, vm_size_t size, |
294 | zalloc_flags_t flags) |
295 | { |
296 | void * address; |
297 | vm_size_t allocSize; |
298 | |
299 | allocSize = size + sizeofIOLibMallocHeader; |
300 | #if IOTRACKING |
301 | if (sizeofIOLibMallocHeader && (allocSize <= size)) { |
302 | return NULL; // overflow |
303 | } |
304 | #endif |
305 | address = kheap_alloc(kheap, allocSize, |
306 | Z_VM_TAG(Z_WAITOK | flags, VM_KERN_MEMORY_IOKIT)); |
307 | |
308 | if (address) { |
309 | #if IOTRACKING |
310 | if (TRACK_ALLOC) { |
311 | IOLibMallocHeader * hdr; |
312 | hdr = (typeof(hdr))address; |
313 | bzero(&hdr->tracking, sizeof(hdr->tracking)); |
314 | hdr->tracking.address = ~(((uintptr_t) address) + sizeofIOLibMallocHeader); |
315 | hdr->tracking.size = size; |
316 | IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); |
317 | } |
318 | #endif |
319 | address = (typeof(address))(((uintptr_t) address) + sizeofIOLibMallocHeader); |
320 | |
321 | #if IOALLOCDEBUG |
322 | OSAddAtomicLong(size, &debug_iomalloc_size); |
323 | #endif |
324 | IOStatisticsAlloc(kIOStatisticsMalloc, size); |
325 | } |
326 | |
327 | return address; |
328 | } |
329 | |
330 | void |
331 | IOFree_internal(struct kalloc_heap *kheap, void * inAddress, vm_size_t size) |
332 | { |
333 | void * address; |
334 | |
335 | if ((address = inAddress)) { |
336 | address = (typeof(address))(((uintptr_t) address) - sizeofIOLibMallocHeader); |
337 | |
338 | #if IOTRACKING |
339 | if (TRACK_ALLOC) { |
340 | IOLibMallocHeader * hdr; |
341 | struct ptr_reference { void * ptr; }; |
342 | volatile struct ptr_reference ptr; |
343 | |
344 | // we're about to block in IOTrackingRemove(), make sure the original pointer |
345 | // exists in memory or a register for leak scanning to find |
346 | ptr.ptr = inAddress; |
347 | |
348 | hdr = (typeof(hdr))address; |
349 | if (size != hdr->tracking.size) { |
350 | OSReportWithBacktrace("bad IOFree size 0x%zx should be 0x%zx" , |
351 | (size_t)size, (size_t)hdr->tracking.size); |
352 | size = hdr->tracking.size; |
353 | } |
354 | IOTrackingRemoveAddress(gIOMallocTracking, &hdr->tracking, size); |
355 | ptr.ptr = NULL; |
356 | } |
357 | #endif |
358 | |
359 | kheap_free(kheap, address, size + sizeofIOLibMallocHeader); |
360 | #if IOALLOCDEBUG |
361 | OSAddAtomicLong(-size, &debug_iomalloc_size); |
362 | #endif |
363 | IOStatisticsAlloc(kIOStatisticsFree, size); |
364 | } |
365 | } |
366 | |
367 | void * |
368 | IOMalloc_external( |
369 | vm_size_t size); |
370 | void * |
371 | IOMalloc_external( |
372 | vm_size_t size) |
373 | { |
374 | return IOMalloc_internal(kheap: KHEAP_DEFAULT, size, flags: Z_VM_TAG_BT_BIT); |
375 | } |
376 | |
377 | void |
378 | IOFree(void * inAddress, vm_size_t size) |
379 | { |
380 | IOFree_internal(kheap: KHEAP_DEFAULT, inAddress, size); |
381 | } |
382 | |
383 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
384 | |
385 | void * |
386 | IOMallocZero_external( |
387 | vm_size_t size); |
388 | void * |
389 | IOMallocZero_external( |
390 | vm_size_t size) |
391 | { |
392 | return IOMalloc_internal(kheap: KHEAP_DEFAULT, size, flags: Z_ZERO_VM_TAG_BT_BIT); |
393 | } |
394 | |
395 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
396 | |
397 | vm_tag_t |
398 | IOMemoryTag(vm_map_t map) |
399 | { |
400 | vm_tag_t tag; |
401 | |
402 | if (!vm_kernel_map_is_kernel(map)) { |
403 | return VM_MEMORY_IOKIT; |
404 | } |
405 | |
406 | tag = vm_tag_bt(); |
407 | if (tag == VM_KERN_MEMORY_NONE) { |
408 | tag = VM_KERN_MEMORY_IOKIT; |
409 | } |
410 | |
411 | return tag; |
412 | } |
413 | |
414 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
415 | |
416 | struct { |
417 | mach_vm_size_t ; |
418 | mach_vm_offset_t ; |
419 | #if IOTRACKING |
420 | IOTrackingAddress tracking; |
421 | #endif |
422 | }; |
423 | |
424 | #if IOTRACKING |
425 | #define sizeofIOLibPageMallocHeader (sizeof(IOLibPageMallocHeader) - (TRACK_ALLOC ? 0 : sizeof(IOTrackingAddress))) |
426 | #else |
427 | #define (sizeof(IOLibPageMallocHeader)) |
428 | #endif |
429 | |
430 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
431 | |
432 | static __header_always_inline void |
433 | ( |
434 | IOLibPageMallocHeader *hdr, |
435 | mach_vm_size_t alignMask, |
436 | mach_vm_address_t allocationStart, |
437 | mach_vm_address_t alignedStart) |
438 | { |
439 | mach_vm_offset_t offset = alignedStart - allocationStart; |
440 | #if __has_feature(ptrauth_calls) |
441 | offset = (mach_vm_offset_t) ptrauth_sign_unauthenticated((void *)offset, |
442 | ptrauth_key_process_independent_data, |
443 | ptrauth_blend_discriminator((void *)(alignedStart | alignMask), |
444 | OS_PTRAUTH_DISCRIMINATOR("IOLibPageMallocHeader.allocationOffset" ))); |
445 | #endif /* __has_feature(ptrauth_calls) */ |
446 | hdr->allocationOffset = offset; |
447 | hdr->alignMask = alignMask; |
448 | } |
449 | |
450 | __abortlike |
451 | static void |
452 | IOMallocAlignedHdrCorruptionPanic( |
453 | mach_vm_offset_t offset, |
454 | mach_vm_size_t alignMask, |
455 | mach_vm_address_t alignedStart, |
456 | vm_size_t size) |
457 | { |
458 | mach_vm_address_t address = 0; |
459 | mach_vm_address_t recalAlignedStart = 0; |
460 | |
461 | if (os_sub_overflow(alignedStart, offset, &address)) { |
462 | panic("Invalid offset %p for aligned addr %p" , (void *)offset, |
463 | (void *)alignedStart); |
464 | } |
465 | if (os_add3_overflow(address, sizeofIOLibPageMallocHeader, alignMask, |
466 | &recalAlignedStart)) { |
467 | panic("alignMask 0x%llx overflows recalAlignedStart %p for provided addr " |
468 | "%p" , alignMask, (void *)recalAlignedStart, (void *)alignedStart); |
469 | } |
470 | if (((recalAlignedStart &= ~alignMask) != alignedStart) && |
471 | (round_page(x: recalAlignedStart) != alignedStart)) { |
472 | panic("Recalculated aligned addr %p doesn't match provided addr %p" , |
473 | (void *)recalAlignedStart, (void *)alignedStart); |
474 | } |
475 | if (offset < sizeofIOLibPageMallocHeader) { |
476 | panic("Offset %zd doesn't accomodate IOLibPageMallocHeader for aligned " |
477 | "addr %p" , (size_t)offset, (void *)alignedStart); |
478 | } |
479 | panic("alignMask 0x%llx overflows adjusted size %zd for aligned addr %p" , |
480 | alignMask, (size_t)size, (void *)alignedStart); |
481 | } |
482 | |
483 | static __header_always_inline mach_vm_address_t |
484 | ( |
485 | IOLibPageMallocHeader *hdr, |
486 | mach_vm_address_t alignedStart, |
487 | vm_size_t *size) |
488 | { |
489 | mach_vm_address_t address = 0; |
490 | mach_vm_address_t recalAlignedStart = 0; |
491 | mach_vm_offset_t offset = hdr->allocationOffset; |
492 | mach_vm_size_t alignMask = hdr->alignMask; |
493 | #if __has_feature(ptrauth_calls) |
494 | offset = (mach_vm_offset_t) ptrauth_auth_data((void *)offset, |
495 | ptrauth_key_process_independent_data, |
496 | ptrauth_blend_discriminator((void *)(alignedStart | alignMask), |
497 | OS_PTRAUTH_DISCRIMINATOR("IOLibPageMallocHeader.allocationOffset" ))); |
498 | #endif /* __has_feature(ptrauth_calls) */ |
499 | if (os_sub_overflow(alignedStart, offset, &address) || |
500 | os_add3_overflow(address, sizeofIOLibPageMallocHeader, alignMask, |
501 | &recalAlignedStart) || |
502 | (((recalAlignedStart &= ~alignMask) != alignedStart) && |
503 | (round_page(x: recalAlignedStart) != alignedStart)) || |
504 | (offset < sizeofIOLibPageMallocHeader) || |
505 | os_add_overflow(*size, alignMask, size)) { |
506 | IOMallocAlignedHdrCorruptionPanic(offset, alignMask, alignedStart, size: *size); |
507 | } |
508 | return address; |
509 | } |
510 | |
511 | void * |
512 | (IOMallocAligned_internal)(struct kalloc_heap *kheap, vm_size_t size, |
513 | vm_size_t alignment, zalloc_flags_t flags) |
514 | { |
515 | kern_return_t kr; |
516 | vm_offset_t address; |
517 | vm_offset_t allocationAddress; |
518 | vm_size_t adjustedSize; |
519 | uintptr_t alignMask; |
520 | IOLibPageMallocHeader * hdr; |
521 | kma_flags_t kma_flags = KMA_NONE; |
522 | |
523 | if (size == 0) { |
524 | return NULL; |
525 | } |
526 | if (((uint32_t) alignment) != alignment) { |
527 | return NULL; |
528 | } |
529 | |
530 | if (flags & Z_ZERO) { |
531 | kma_flags = KMA_ZERO; |
532 | } |
533 | |
534 | if (kheap == KHEAP_DATA_BUFFERS) { |
535 | kma_flags = (kma_flags_t) (kma_flags | KMA_DATA); |
536 | } |
537 | |
538 | alignment = (1UL << log2up(size: (uint32_t) alignment)); |
539 | alignMask = alignment - 1; |
540 | adjustedSize = size + sizeofIOLibPageMallocHeader; |
541 | |
542 | if (size > adjustedSize) { |
543 | address = 0; /* overflow detected */ |
544 | } else if (adjustedSize >= page_size) { |
545 | kr = kernel_memory_allocate(map: kernel_map, addrp: &address, |
546 | size, mask: alignMask, flags: kma_flags, tag: IOMemoryTag(map: kernel_map)); |
547 | if (KERN_SUCCESS != kr) { |
548 | address = 0; |
549 | } |
550 | #if IOTRACKING |
551 | else if (TRACK_ALLOC) { |
552 | IOTrackingAlloc(gIOMallocTracking, address, size); |
553 | } |
554 | #endif |
555 | } else { |
556 | adjustedSize += alignMask; |
557 | |
558 | if (adjustedSize >= page_size) { |
559 | kr = kmem_alloc(map: kernel_map, addrp: &allocationAddress, |
560 | size: adjustedSize, flags: kma_flags, tag: IOMemoryTag(map: kernel_map)); |
561 | if (KERN_SUCCESS != kr) { |
562 | allocationAddress = 0; |
563 | } |
564 | } else { |
565 | allocationAddress = (vm_address_t) kheap_alloc(kheap, |
566 | adjustedSize, Z_VM_TAG(Z_WAITOK | flags, VM_KERN_MEMORY_IOKIT)); |
567 | } |
568 | |
569 | if (allocationAddress) { |
570 | address = (allocationAddress + alignMask + sizeofIOLibPageMallocHeader) |
571 | & (~alignMask); |
572 | |
573 | hdr = (typeof(hdr))(address - sizeofIOLibPageMallocHeader); |
574 | IOMallocAlignedSetHdr(hdr, alignMask, allocationStart: allocationAddress, alignedStart: address); |
575 | #if IOTRACKING |
576 | if (TRACK_ALLOC) { |
577 | bzero(&hdr->tracking, sizeof(hdr->tracking)); |
578 | hdr->tracking.address = ~address; |
579 | hdr->tracking.size = size; |
580 | IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); |
581 | } |
582 | #endif |
583 | } else { |
584 | address = 0; |
585 | } |
586 | } |
587 | |
588 | assert(0 == (address & alignMask)); |
589 | |
590 | if (address) { |
591 | #if IOALLOCDEBUG |
592 | OSAddAtomicLong(size, &debug_iomalloc_size); |
593 | #endif |
594 | IOStatisticsAlloc(kIOStatisticsMallocAligned, size); |
595 | } |
596 | |
597 | return (void *) address; |
598 | } |
599 | |
600 | void |
601 | IOFreeAligned_internal(kalloc_heap_t kheap, void * address, vm_size_t size) |
602 | { |
603 | vm_address_t allocationAddress; |
604 | vm_size_t adjustedSize; |
605 | IOLibPageMallocHeader * hdr; |
606 | |
607 | if (!address) { |
608 | return; |
609 | } |
610 | |
611 | assert(size); |
612 | |
613 | adjustedSize = size + sizeofIOLibPageMallocHeader; |
614 | if (adjustedSize >= page_size) { |
615 | #if IOTRACKING |
616 | if (TRACK_ALLOC) { |
617 | IOTrackingFree(gIOMallocTracking, (uintptr_t) address, size); |
618 | } |
619 | #endif |
620 | kmem_free(map: kernel_map, addr: (vm_offset_t) address, size); |
621 | } else { |
622 | hdr = (typeof(hdr))(((uintptr_t)address) - sizeofIOLibPageMallocHeader); |
623 | allocationAddress = IOMallocAlignedGetAddress(hdr, |
624 | alignedStart: (mach_vm_address_t)address, size: &adjustedSize); |
625 | |
626 | #if IOTRACKING |
627 | if (TRACK_ALLOC) { |
628 | if (size != hdr->tracking.size) { |
629 | OSReportWithBacktrace("bad IOFreeAligned size 0x%zx should be 0x%zx" , |
630 | (size_t)size, (size_t)hdr->tracking.size); |
631 | size = hdr->tracking.size; |
632 | } |
633 | IOTrackingRemoveAddress(gIOMallocTracking, &hdr->tracking, size); |
634 | } |
635 | #endif |
636 | if (adjustedSize >= page_size) { |
637 | kmem_free(map: kernel_map, addr: allocationAddress, size: adjustedSize); |
638 | } else { |
639 | kheap_free(kheap, allocationAddress, adjustedSize); |
640 | } |
641 | } |
642 | |
643 | #if IOALLOCDEBUG |
644 | OSAddAtomicLong(-size, &debug_iomalloc_size); |
645 | #endif |
646 | |
647 | IOStatisticsAlloc(kIOStatisticsFreeAligned, size); |
648 | } |
649 | |
650 | void * |
651 | IOMallocAligned_external( |
652 | vm_size_t size, vm_size_t alignment); |
653 | void * |
654 | IOMallocAligned_external( |
655 | vm_size_t size, vm_size_t alignment) |
656 | { |
657 | return IOMallocAligned_internal(kheap: KHEAP_DATA_BUFFERS, size, alignment, |
658 | flags: Z_VM_TAG_BT_BIT); |
659 | } |
660 | |
661 | void |
662 | IOFreeAligned( |
663 | void * address, |
664 | vm_size_t size) |
665 | { |
666 | IOFreeAligned_internal(kheap: KHEAP_DATA_BUFFERS, address, size); |
667 | } |
668 | |
669 | __typed_allocators_ignore_pop |
670 | |
671 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
672 | |
673 | void |
674 | IOKernelFreePhysical( |
675 | kalloc_heap_t kheap, |
676 | mach_vm_address_t address, |
677 | mach_vm_size_t size) |
678 | { |
679 | vm_address_t allocationAddress; |
680 | vm_size_t adjustedSize; |
681 | IOLibPageMallocHeader * hdr; |
682 | |
683 | if (!address) { |
684 | return; |
685 | } |
686 | |
687 | assert(size); |
688 | |
689 | adjustedSize = (2 * size) + sizeofIOLibPageMallocHeader; |
690 | if (adjustedSize >= page_size) { |
691 | #if IOTRACKING |
692 | if (TRACK_ALLOC) { |
693 | IOTrackingFree(gIOMallocTracking, address, size); |
694 | } |
695 | #endif |
696 | kmem_free(map: kernel_map, addr: (vm_offset_t) address, size); |
697 | } else { |
698 | hdr = (typeof(hdr))(((uintptr_t)address) - sizeofIOLibPageMallocHeader); |
699 | allocationAddress = IOMallocAlignedGetAddress(hdr, alignedStart: address, size: &adjustedSize); |
700 | #if IOTRACKING |
701 | if (TRACK_ALLOC) { |
702 | IOTrackingRemoveAddress(gIOMallocTracking, &hdr->tracking, size); |
703 | } |
704 | #endif |
705 | __typed_allocators_ignore(kheap_free(kheap, allocationAddress, adjustedSize)); |
706 | } |
707 | |
708 | IOStatisticsAlloc(kIOStatisticsFreeContiguous, size); |
709 | #if IOALLOCDEBUG |
710 | OSAddAtomicLong(-size, &debug_iomalloc_size); |
711 | #endif |
712 | } |
713 | |
714 | #if __arm64__ |
715 | extern unsigned long gPhysBase, gPhysSize; |
716 | #endif |
717 | |
718 | mach_vm_address_t |
719 | IOKernelAllocateWithPhysicalRestrict( |
720 | kalloc_heap_t kheap, |
721 | mach_vm_size_t size, |
722 | mach_vm_address_t maxPhys, |
723 | mach_vm_size_t alignment, |
724 | bool contiguous) |
725 | { |
726 | kern_return_t kr; |
727 | mach_vm_address_t address; |
728 | mach_vm_address_t allocationAddress; |
729 | mach_vm_size_t adjustedSize; |
730 | mach_vm_address_t alignMask; |
731 | IOLibPageMallocHeader * hdr; |
732 | |
733 | if (size == 0) { |
734 | return 0; |
735 | } |
736 | if (alignment == 0) { |
737 | alignment = 1; |
738 | } |
739 | |
740 | alignMask = alignment - 1; |
741 | |
742 | if (os_mul_and_add_overflow(2, size, sizeofIOLibPageMallocHeader, &adjustedSize)) { |
743 | return 0; |
744 | } |
745 | |
746 | contiguous = (contiguous && (adjustedSize > page_size)) |
747 | || (alignment > page_size); |
748 | |
749 | if (contiguous || maxPhys) { |
750 | kma_flags_t options = KMA_ZERO; |
751 | vm_offset_t virt; |
752 | |
753 | if (kheap == KHEAP_DATA_BUFFERS) { |
754 | options = (kma_flags_t) (options | KMA_DATA); |
755 | } |
756 | |
757 | adjustedSize = size; |
758 | contiguous = (contiguous && (adjustedSize > page_size)) |
759 | || (alignment > page_size); |
760 | |
761 | if (!contiguous) { |
762 | #if __arm64__ |
763 | if (maxPhys >= (mach_vm_address_t)(gPhysBase + gPhysSize)) { |
764 | maxPhys = 0; |
765 | } else |
766 | #endif |
767 | if (maxPhys <= 0xFFFFFFFF) { |
768 | maxPhys = 0; |
769 | options = (kma_flags_t)(options | KMA_LOMEM); |
770 | } else if (gIOLastPage && (atop_64(maxPhys) > gIOLastPage)) { |
771 | maxPhys = 0; |
772 | } |
773 | } |
774 | if (contiguous || maxPhys) { |
775 | kr = kmem_alloc_contig(map: kernel_map, addrp: &virt, size, |
776 | mask: alignMask, max_pnum: (ppnum_t) atop(maxPhys), pnum_mask: (ppnum_t) atop(alignMask), |
777 | flags: options, tag: IOMemoryTag(map: kernel_map)); |
778 | } else { |
779 | kr = kernel_memory_allocate(map: kernel_map, addrp: &virt, |
780 | size, mask: alignMask, flags: options, tag: IOMemoryTag(map: kernel_map)); |
781 | } |
782 | if (KERN_SUCCESS == kr) { |
783 | address = virt; |
784 | #if IOTRACKING |
785 | if (TRACK_ALLOC) { |
786 | IOTrackingAlloc(gIOMallocTracking, address, size); |
787 | } |
788 | #endif |
789 | } else { |
790 | address = 0; |
791 | } |
792 | } else { |
793 | adjustedSize += alignMask; |
794 | if (adjustedSize < size) { |
795 | return 0; |
796 | } |
797 | /* BEGIN IGNORE CODESTYLE */ |
798 | __typed_allocators_ignore_push // allocator implementation |
799 | allocationAddress = (mach_vm_address_t) kheap_alloc(kheap, |
800 | adjustedSize, Z_VM_TAG_BT(Z_WAITOK, VM_KERN_MEMORY_IOKIT)); |
801 | __typed_allocators_ignore_pop |
802 | /* END IGNORE CODESTYLE */ |
803 | |
804 | if (allocationAddress) { |
805 | address = (allocationAddress + alignMask + sizeofIOLibPageMallocHeader) |
806 | & (~alignMask); |
807 | |
808 | if (atop_32(address) != atop_32(address + size - 1)) { |
809 | address = round_page(x: address); |
810 | } |
811 | |
812 | hdr = (typeof(hdr))(address - sizeofIOLibPageMallocHeader); |
813 | IOMallocAlignedSetHdr(hdr, alignMask, allocationStart: allocationAddress, alignedStart: address); |
814 | #if IOTRACKING |
815 | if (TRACK_ALLOC) { |
816 | bzero(&hdr->tracking, sizeof(hdr->tracking)); |
817 | hdr->tracking.address = ~address; |
818 | hdr->tracking.size = size; |
819 | IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); |
820 | } |
821 | #endif |
822 | } else { |
823 | address = 0; |
824 | } |
825 | } |
826 | |
827 | if (address) { |
828 | IOStatisticsAlloc(kIOStatisticsMallocContiguous, size); |
829 | #if IOALLOCDEBUG |
830 | OSAddAtomicLong(size, &debug_iomalloc_size); |
831 | #endif |
832 | } |
833 | |
834 | return address; |
835 | } |
836 | |
837 | |
838 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
839 | |
840 | struct _IOMallocContiguousEntry { |
841 | mach_vm_address_t virtualAddr; |
842 | IOBufferMemoryDescriptor * md; |
843 | queue_chain_t link; |
844 | }; |
845 | typedef struct _IOMallocContiguousEntry _IOMallocContiguousEntry; |
846 | |
847 | void * |
848 | IOMallocContiguous(vm_size_t size, vm_size_t alignment, |
849 | IOPhysicalAddress * physicalAddress) |
850 | { |
851 | mach_vm_address_t address = 0; |
852 | |
853 | if (size == 0) { |
854 | return NULL; |
855 | } |
856 | if (alignment == 0) { |
857 | alignment = 1; |
858 | } |
859 | |
860 | /* Do we want a physical address? */ |
861 | if (!physicalAddress) { |
862 | address = IOKernelAllocateWithPhysicalRestrict(kheap: KHEAP_DEFAULT, |
863 | size, maxPhys: 0 /*maxPhys*/, alignment, contiguous: true); |
864 | } else { |
865 | do { |
866 | IOBufferMemoryDescriptor * bmd; |
867 | mach_vm_address_t physicalMask; |
868 | vm_offset_t alignMask; |
869 | |
870 | alignMask = alignment - 1; |
871 | physicalMask = (0xFFFFFFFF ^ alignMask); |
872 | |
873 | bmd = IOBufferMemoryDescriptor::inTaskWithPhysicalMask( |
874 | inTask: kernel_task, options: kIOMemoryPhysicallyContiguous, capacity: size, physicalMask); |
875 | if (!bmd) { |
876 | break; |
877 | } |
878 | |
879 | _IOMallocContiguousEntry * |
880 | entry = IOMallocType(_IOMallocContiguousEntry); |
881 | if (!entry) { |
882 | bmd->release(); |
883 | break; |
884 | } |
885 | entry->virtualAddr = (mach_vm_address_t) bmd->getBytesNoCopy(); |
886 | entry->md = bmd; |
887 | lck_mtx_lock(lck: gIOMallocContiguousEntriesLock); |
888 | queue_enter( &gIOMallocContiguousEntries, entry, |
889 | _IOMallocContiguousEntry *, link ); |
890 | lck_mtx_unlock(lck: gIOMallocContiguousEntriesLock); |
891 | |
892 | address = (mach_vm_address_t) entry->virtualAddr; |
893 | *physicalAddress = bmd->getPhysicalAddress(); |
894 | }while (false); |
895 | } |
896 | |
897 | return (void *) address; |
898 | } |
899 | |
900 | void |
901 | IOFreeContiguous(void * _address, vm_size_t size) |
902 | { |
903 | _IOMallocContiguousEntry * entry; |
904 | IOMemoryDescriptor * md = NULL; |
905 | |
906 | mach_vm_address_t address = (mach_vm_address_t) _address; |
907 | |
908 | if (!address) { |
909 | return; |
910 | } |
911 | |
912 | assert(size); |
913 | |
914 | lck_mtx_lock(lck: gIOMallocContiguousEntriesLock); |
915 | queue_iterate( &gIOMallocContiguousEntries, entry, |
916 | _IOMallocContiguousEntry *, link ) |
917 | { |
918 | if (entry->virtualAddr == address) { |
919 | md = entry->md; |
920 | queue_remove( &gIOMallocContiguousEntries, entry, |
921 | _IOMallocContiguousEntry *, link ); |
922 | break; |
923 | } |
924 | } |
925 | lck_mtx_unlock(lck: gIOMallocContiguousEntriesLock); |
926 | |
927 | if (md) { |
928 | md->release(); |
929 | IOFreeType(entry, _IOMallocContiguousEntry); |
930 | } else { |
931 | IOKernelFreePhysical(kheap: KHEAP_DEFAULT, address: (mach_vm_address_t) address, size); |
932 | } |
933 | } |
934 | |
935 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
936 | |
937 | kern_return_t |
938 | IOIteratePageableMaps(vm_size_t size, |
939 | IOIteratePageableMapsCallback callback, void * ref) |
940 | { |
941 | kern_return_t kr = kIOReturnNotReady; |
942 | kmem_return_t kmr; |
943 | vm_size_t segSize; |
944 | UInt32 attempts; |
945 | UInt32 index; |
946 | mach_vm_offset_t min; |
947 | int flags; |
948 | |
949 | if (size > kIOPageableMaxMapSize) { |
950 | return kIOReturnBadArgument; |
951 | } |
952 | |
953 | do { |
954 | index = gIOKitPageableSpace.hint; |
955 | attempts = gIOKitPageableSpace.count; |
956 | while (attempts--) { |
957 | kr = (*callback)(gIOKitPageableSpace.maps[index].map, ref); |
958 | if (KERN_SUCCESS == kr) { |
959 | gIOKitPageableSpace.hint = index; |
960 | break; |
961 | } |
962 | if (index) { |
963 | index--; |
964 | } else { |
965 | index = gIOKitPageableSpace.count - 1; |
966 | } |
967 | } |
968 | if (KERN_NO_SPACE != kr) { |
969 | break; |
970 | } |
971 | |
972 | lck_mtx_lock( lck: gIOKitPageableSpace.lock ); |
973 | |
974 | index = gIOKitPageableSpace.count; |
975 | if (index >= (kIOMaxPageableMaps - 1)) { |
976 | lck_mtx_unlock( lck: gIOKitPageableSpace.lock ); |
977 | break; |
978 | } |
979 | |
980 | if (size < kIOPageableMapSize) { |
981 | segSize = kIOPageableMapSize; |
982 | } else { |
983 | segSize = size; |
984 | } |
985 | |
986 | /* |
987 | * Use the predefine ranges if available, else default to data |
988 | */ |
989 | if (index < kIOMaxFixedRanges) { |
990 | min = gIOKitPageableFixedRanges[index].min_address; |
991 | flags = VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE; |
992 | } else { |
993 | min = 0; |
994 | flags = VM_FLAGS_ANYWHERE; |
995 | } |
996 | kmr = kmem_suballoc(parent: kernel_map, |
997 | addr: &min, |
998 | size: segSize, |
999 | vmc_options: VM_MAP_CREATE_PAGEABLE, |
1000 | vm_flags: flags, |
1001 | flags: (kms_flags_t)(KMS_PERMANENT | KMS_DATA), |
1002 | VM_KERN_MEMORY_IOKIT); |
1003 | if (kmr.kmr_return != KERN_SUCCESS) { |
1004 | kr = kmr.kmr_return; |
1005 | lck_mtx_unlock( lck: gIOKitPageableSpace.lock ); |
1006 | break; |
1007 | } |
1008 | |
1009 | gIOKitPageableSpace.maps[index].map = kmr.kmr_submap; |
1010 | gIOKitPageableSpace.maps[index].address = min; |
1011 | gIOKitPageableSpace.maps[index].end = min + segSize; |
1012 | gIOKitPageableSpace.hint = index; |
1013 | gIOKitPageableSpace.count = index + 1; |
1014 | |
1015 | lck_mtx_unlock( lck: gIOKitPageableSpace.lock ); |
1016 | } while (true); |
1017 | |
1018 | return kr; |
1019 | } |
1020 | |
1021 | struct IOMallocPageableRef { |
1022 | vm_offset_t address; |
1023 | vm_size_t size; |
1024 | vm_tag_t tag; |
1025 | }; |
1026 | |
1027 | static kern_return_t |
1028 | IOMallocPageableCallback(vm_map_t map, void * _ref) |
1029 | { |
1030 | struct IOMallocPageableRef * ref = (struct IOMallocPageableRef *) _ref; |
1031 | kma_flags_t flags = (kma_flags_t)(KMA_PAGEABLE | KMA_DATA); |
1032 | |
1033 | return kmem_alloc( map, addrp: &ref->address, size: ref->size, flags, tag: ref->tag ); |
1034 | } |
1035 | |
1036 | static void * |
1037 | IOMallocPageablePages(vm_size_t size, vm_size_t alignment, vm_tag_t tag) |
1038 | { |
1039 | kern_return_t kr = kIOReturnNotReady; |
1040 | struct IOMallocPageableRef ref; |
1041 | |
1042 | if (alignment > page_size) { |
1043 | return NULL; |
1044 | } |
1045 | if (size > kIOPageableMaxMapSize) { |
1046 | return NULL; |
1047 | } |
1048 | |
1049 | ref.size = size; |
1050 | ref.tag = tag; |
1051 | kr = IOIteratePageableMaps( size, callback: &IOMallocPageableCallback, ref: &ref ); |
1052 | if (kIOReturnSuccess != kr) { |
1053 | ref.address = 0; |
1054 | } |
1055 | |
1056 | return (void *) ref.address; |
1057 | } |
1058 | |
1059 | vm_map_t |
1060 | IOPageableMapForAddress( uintptr_t address ) |
1061 | { |
1062 | vm_map_t map = NULL; |
1063 | UInt32 index; |
1064 | |
1065 | for (index = 0; index < gIOKitPageableSpace.count; index++) { |
1066 | if ((address >= gIOKitPageableSpace.maps[index].address) |
1067 | && (address < gIOKitPageableSpace.maps[index].end)) { |
1068 | map = gIOKitPageableSpace.maps[index].map; |
1069 | break; |
1070 | } |
1071 | } |
1072 | if (!map) { |
1073 | panic("IOPageableMapForAddress: null" ); |
1074 | } |
1075 | |
1076 | return map; |
1077 | } |
1078 | |
1079 | static void |
1080 | IOFreePageablePages(void * address, vm_size_t size) |
1081 | { |
1082 | vm_map_t map; |
1083 | |
1084 | map = IOPageableMapForAddress(address: (vm_address_t) address); |
1085 | if (map) { |
1086 | kmem_free( map, addr: (vm_offset_t) address, size); |
1087 | } |
1088 | } |
1089 | |
1090 | #if defined(__x86_64__) |
1091 | static uintptr_t |
1092 | IOMallocOnePageablePage(kalloc_heap_t kheap __unused, iopa_t * a) |
1093 | { |
1094 | return (uintptr_t) IOMallocPageablePages(page_size, page_size, VM_KERN_MEMORY_IOKIT); |
1095 | } |
1096 | #endif /* defined(__x86_64__) */ |
1097 | |
1098 | static void * |
1099 | IOMallocPageableInternal(vm_size_t size, vm_size_t alignment, bool zeroed) |
1100 | { |
1101 | void * addr; |
1102 | |
1103 | if (((uint32_t) alignment) != alignment) { |
1104 | return NULL; |
1105 | } |
1106 | #if defined(__x86_64__) |
1107 | if (size >= (page_size - 4 * gIOPageAllocChunkBytes) || |
1108 | alignment > page_size) { |
1109 | addr = IOMallocPageablePages(size, alignment, IOMemoryTag(kernel_map)); |
1110 | /* Memory allocated this way will already be zeroed. */ |
1111 | } else { |
1112 | addr = ((void *) iopa_alloc(&gIOPageablePageAllocator, |
1113 | &IOMallocOnePageablePage, KHEAP_DEFAULT, size, (uint32_t) alignment)); |
1114 | if (addr && zeroed) { |
1115 | bzero(addr, size); |
1116 | } |
1117 | } |
1118 | #else /* !defined(__x86_64__) */ |
1119 | vm_size_t allocSize = size; |
1120 | if (allocSize == 0) { |
1121 | allocSize = 1; |
1122 | } |
1123 | addr = IOMallocPageablePages(size: allocSize, alignment, tag: IOMemoryTag(map: kernel_map)); |
1124 | /* already zeroed */ |
1125 | #endif /* defined(__x86_64__) */ |
1126 | |
1127 | if (addr) { |
1128 | #if IOALLOCDEBUG |
1129 | OSAddAtomicLong(size, &debug_iomallocpageable_size); |
1130 | #endif |
1131 | IOStatisticsAlloc(kIOStatisticsMallocPageable, size); |
1132 | } |
1133 | |
1134 | return addr; |
1135 | } |
1136 | |
1137 | void * |
1138 | IOMallocPageable(vm_size_t size, vm_size_t alignment) |
1139 | { |
1140 | return IOMallocPageableInternal(size, alignment, /*zeroed*/ false); |
1141 | } |
1142 | |
1143 | void * |
1144 | IOMallocPageableZero(vm_size_t size, vm_size_t alignment) |
1145 | { |
1146 | return IOMallocPageableInternal(size, alignment, /*zeroed*/ true); |
1147 | } |
1148 | |
1149 | void |
1150 | IOFreePageable(void * address, vm_size_t size) |
1151 | { |
1152 | #if IOALLOCDEBUG |
1153 | OSAddAtomicLong(-size, &debug_iomallocpageable_size); |
1154 | #endif |
1155 | IOStatisticsAlloc(kIOStatisticsFreePageable, size); |
1156 | |
1157 | #if defined(__x86_64__) |
1158 | if (size < (page_size - 4 * gIOPageAllocChunkBytes)) { |
1159 | address = (void *) iopa_free(&gIOPageablePageAllocator, (uintptr_t) address, size); |
1160 | size = page_size; |
1161 | } |
1162 | if (address) { |
1163 | IOFreePageablePages(address, size); |
1164 | } |
1165 | #else /* !defined(__x86_64__) */ |
1166 | if (size == 0) { |
1167 | size = 1; |
1168 | } |
1169 | if (address) { |
1170 | IOFreePageablePages(address, size); |
1171 | } |
1172 | #endif /* defined(__x86_64__) */ |
1173 | } |
1174 | |
1175 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1176 | |
1177 | void * |
1178 | IOMallocData_external( |
1179 | vm_size_t size); |
1180 | void * |
1181 | IOMallocData_external(vm_size_t size) |
1182 | { |
1183 | return IOMalloc_internal(kheap: KHEAP_DATA_BUFFERS, size, flags: Z_VM_TAG_BT_BIT); |
1184 | } |
1185 | |
1186 | void * |
1187 | IOMallocZeroData_external( |
1188 | vm_size_t size); |
1189 | void * |
1190 | IOMallocZeroData_external(vm_size_t size) |
1191 | { |
1192 | return IOMalloc_internal(kheap: KHEAP_DATA_BUFFERS, size, flags: Z_ZERO_VM_TAG_BT_BIT); |
1193 | } |
1194 | |
1195 | void |
1196 | IOFreeData(void * address, vm_size_t size) |
1197 | { |
1198 | return IOFree_internal(kheap: KHEAP_DATA_BUFFERS, inAddress: address, size); |
1199 | } |
1200 | |
1201 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1202 | |
1203 | __typed_allocators_ignore_push // allocator implementation |
1204 | |
1205 | void * |
1206 | IOMallocTypeImpl(kalloc_type_view_t kt_view) |
1207 | { |
1208 | #if IOTRACKING |
1209 | /* |
1210 | * When leak detection is on default to using IOMalloc as kalloc |
1211 | * type infrastructure isn't aware of needing additional space for |
1212 | * the header. |
1213 | */ |
1214 | if (TRACK_ALLOC) { |
1215 | uint32_t kt_size = kalloc_type_get_size(kt_view->kt_size); |
1216 | void *mem = IOMalloc_internal(KHEAP_DEFAULT, kt_size, Z_ZERO); |
1217 | if (!IOMallocType_from_vm(kt_view)) { |
1218 | assert(mem); |
1219 | } |
1220 | return mem; |
1221 | } |
1222 | #endif |
1223 | zalloc_flags_t kt_flags = (zalloc_flags_t) (Z_WAITOK | Z_ZERO); |
1224 | if (!IOMallocType_from_vm(ktv: kt_view)) { |
1225 | kt_flags = (zalloc_flags_t) (kt_flags | Z_NOFAIL); |
1226 | } |
1227 | /* |
1228 | * Use external symbol for kalloc_type_impl as |
1229 | * kalloc_type_views generated at some external callsites |
1230 | * many not have been processed during boot. |
1231 | */ |
1232 | return kalloc_type_impl_external(kt_view, flags: kt_flags); |
1233 | } |
1234 | |
1235 | void |
1236 | IOFreeTypeImpl(kalloc_type_view_t kt_view, void * address) |
1237 | { |
1238 | #if IOTRACKING |
1239 | if (TRACK_ALLOC) { |
1240 | return IOFree_internal(KHEAP_DEFAULT, address, |
1241 | kalloc_type_get_size(kt_view->kt_size)); |
1242 | } |
1243 | #endif |
1244 | /* |
1245 | * Use external symbol for kalloc_type_impl as |
1246 | * kalloc_type_views generated at some external callsites |
1247 | * many not have been processed during boot. |
1248 | */ |
1249 | return kfree_type_impl_external(kt_view, ptr: address); |
1250 | } |
1251 | |
1252 | void * |
1253 | IOMallocTypeVarImpl(kalloc_type_var_view_t kt_view, vm_size_t size) |
1254 | { |
1255 | #if IOTRACKING |
1256 | /* |
1257 | * When leak detection is on default to using IOMalloc as kalloc |
1258 | * type infrastructure isn't aware of needing additional space for |
1259 | * the header. |
1260 | */ |
1261 | if (TRACK_ALLOC) { |
1262 | return IOMalloc_internal(KHEAP_DEFAULT, size, Z_ZERO); |
1263 | } |
1264 | #endif |
1265 | zalloc_flags_t kt_flags = (zalloc_flags_t) (Z_WAITOK | Z_ZERO); |
1266 | |
1267 | kt_flags = Z_VM_TAG_BT(kt_flags, VM_KERN_MEMORY_KALLOC_TYPE); |
1268 | return kalloc_type_var_impl(kt_view, size, kt_flags, NULL); |
1269 | } |
1270 | |
1271 | void |
1272 | IOFreeTypeVarImpl(kalloc_type_var_view_t kt_view, void * address, |
1273 | vm_size_t size) |
1274 | { |
1275 | #if IOTRACKING |
1276 | if (TRACK_ALLOC) { |
1277 | return IOFree_internal(KHEAP_DEFAULT, address, size); |
1278 | } |
1279 | #endif |
1280 | |
1281 | return kfree_type_var_impl(kt_view, ptr: address, size); |
1282 | } |
1283 | |
1284 | __typed_allocators_ignore_pop |
1285 | |
1286 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1287 | |
1288 | #if defined(__x86_64__) |
1289 | |
1290 | |
1291 | extern "C" void |
1292 | iopa_init(iopa_t * a) |
1293 | { |
1294 | bzero(a, sizeof(*a)); |
1295 | a->lock = IOLockAlloc(); |
1296 | queue_init(&a->list); |
1297 | } |
1298 | |
1299 | static uintptr_t |
1300 | iopa_allocinpage(iopa_page_t * pa, uint32_t count, uint64_t align) |
1301 | { |
1302 | uint32_t n, s; |
1303 | uint64_t avail = pa->avail; |
1304 | |
1305 | assert(avail); |
1306 | |
1307 | // find strings of count 1 bits in avail |
1308 | for (n = count; n > 1; n -= s) { |
1309 | s = n >> 1; |
1310 | avail = avail & (avail << s); |
1311 | } |
1312 | // and aligned |
1313 | avail &= align; |
1314 | |
1315 | if (avail) { |
1316 | n = __builtin_clzll(avail); |
1317 | pa->avail &= ~((-1ULL << (64 - count)) >> n); |
1318 | if (!pa->avail && pa->link.next) { |
1319 | remque(&pa->link); |
1320 | pa->link.next = NULL; |
1321 | } |
1322 | return n * gIOPageAllocChunkBytes + trunc_page((uintptr_t) pa); |
1323 | } |
1324 | |
1325 | return 0; |
1326 | } |
1327 | |
1328 | uintptr_t |
1329 | iopa_alloc( |
1330 | iopa_t * a, |
1331 | iopa_proc_t alloc, |
1332 | kalloc_heap_t kheap, |
1333 | vm_size_t bytes, |
1334 | vm_size_t balign) |
1335 | { |
1336 | static const uint64_t align_masks[] = { |
1337 | 0xFFFFFFFFFFFFFFFF, |
1338 | 0xAAAAAAAAAAAAAAAA, |
1339 | 0x8888888888888888, |
1340 | 0x8080808080808080, |
1341 | 0x8000800080008000, |
1342 | 0x8000000080000000, |
1343 | 0x8000000000000000, |
1344 | }; |
1345 | iopa_page_t * pa; |
1346 | uintptr_t addr = 0; |
1347 | uint32_t count; |
1348 | uint64_t align; |
1349 | vm_size_t align_masks_idx; |
1350 | |
1351 | if (((uint32_t) bytes) != bytes) { |
1352 | return 0; |
1353 | } |
1354 | if (!bytes) { |
1355 | bytes = 1; |
1356 | } |
1357 | count = (((uint32_t) bytes) + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes; |
1358 | |
1359 | align_masks_idx = log2up((balign + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes); |
1360 | assert(align_masks_idx < sizeof(align_masks) / sizeof(*align_masks)); |
1361 | align = align_masks[align_masks_idx]; |
1362 | |
1363 | IOLockLock(a->lock); |
1364 | __IGNORE_WCASTALIGN(pa = (typeof(pa))queue_first(&a->list)); |
1365 | while (!queue_end(&a->list, &pa->link)) { |
1366 | addr = iopa_allocinpage(pa, count, align); |
1367 | if (addr) { |
1368 | a->bytecount += bytes; |
1369 | break; |
1370 | } |
1371 | __IGNORE_WCASTALIGN(pa = (typeof(pa))queue_next(&pa->link)); |
1372 | } |
1373 | IOLockUnlock(a->lock); |
1374 | |
1375 | if (!addr) { |
1376 | addr = alloc(kheap, a); |
1377 | if (addr) { |
1378 | pa = (typeof(pa))(addr + page_size - gIOPageAllocChunkBytes); |
1379 | pa->signature = kIOPageAllocSignature; |
1380 | pa->avail = -2ULL; |
1381 | |
1382 | addr = iopa_allocinpage(pa, count, align); |
1383 | IOLockLock(a->lock); |
1384 | if (pa->avail) { |
1385 | enqueue_head(&a->list, &pa->link); |
1386 | } |
1387 | a->pagecount++; |
1388 | if (addr) { |
1389 | a->bytecount += bytes; |
1390 | } |
1391 | IOLockUnlock(a->lock); |
1392 | } |
1393 | } |
1394 | |
1395 | assert((addr & ((1 << log2up(balign)) - 1)) == 0); |
1396 | return addr; |
1397 | } |
1398 | |
1399 | uintptr_t |
1400 | iopa_free(iopa_t * a, uintptr_t addr, vm_size_t bytes) |
1401 | { |
1402 | iopa_page_t * pa; |
1403 | uint32_t count; |
1404 | uintptr_t chunk; |
1405 | |
1406 | if (((uint32_t) bytes) != bytes) { |
1407 | return 0; |
1408 | } |
1409 | if (!bytes) { |
1410 | bytes = 1; |
1411 | } |
1412 | |
1413 | chunk = (addr & page_mask); |
1414 | assert(0 == (chunk & (gIOPageAllocChunkBytes - 1))); |
1415 | |
1416 | pa = (typeof(pa))(addr | (page_size - gIOPageAllocChunkBytes)); |
1417 | assert(kIOPageAllocSignature == pa->signature); |
1418 | |
1419 | count = (((uint32_t) bytes) + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes; |
1420 | chunk /= gIOPageAllocChunkBytes; |
1421 | |
1422 | IOLockLock(a->lock); |
1423 | if (!pa->avail) { |
1424 | assert(!pa->link.next); |
1425 | enqueue_tail(&a->list, &pa->link); |
1426 | } |
1427 | pa->avail |= ((-1ULL << (64 - count)) >> chunk); |
1428 | if (pa->avail != -2ULL) { |
1429 | pa = NULL; |
1430 | } else { |
1431 | remque(&pa->link); |
1432 | pa->link.next = NULL; |
1433 | pa->signature = 0; |
1434 | a->pagecount--; |
1435 | // page to free |
1436 | pa = (typeof(pa))trunc_page(pa); |
1437 | } |
1438 | a->bytecount -= bytes; |
1439 | IOLockUnlock(a->lock); |
1440 | |
1441 | return (uintptr_t) pa; |
1442 | } |
1443 | |
1444 | #endif /* defined(__x86_64__) */ |
1445 | |
1446 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1447 | |
1448 | IOReturn |
1449 | IOSetProcessorCacheMode( task_t task, IOVirtualAddress address, |
1450 | IOByteCount length, IOOptionBits cacheMode ) |
1451 | { |
1452 | IOReturn ret = kIOReturnSuccess; |
1453 | ppnum_t pagenum; |
1454 | |
1455 | if (task != kernel_task) { |
1456 | return kIOReturnUnsupported; |
1457 | } |
1458 | if ((address | length) & PAGE_MASK) { |
1459 | // OSReportWithBacktrace("IOSetProcessorCacheMode(0x%x, 0x%x, 0x%x) fails\n", address, length, cacheMode); |
1460 | return kIOReturnUnsupported; |
1461 | } |
1462 | length = round_page(x: address + length) - trunc_page( address ); |
1463 | address = trunc_page( address ); |
1464 | |
1465 | // make map mode |
1466 | cacheMode = (cacheMode << kIOMapCacheShift) & kIOMapCacheMask; |
1467 | |
1468 | while ((kIOReturnSuccess == ret) && (length > 0)) { |
1469 | // Get the physical page number |
1470 | pagenum = pmap_find_phys(pmap: kernel_pmap, va: (addr64_t)address); |
1471 | if (pagenum) { |
1472 | ret = IOUnmapPages( map: get_task_map(task), va: address, length: page_size ); |
1473 | ret = IOMapPages( map: get_task_map(task), va: address, ptoa_64(pagenum), length: page_size, mapFlags: cacheMode ); |
1474 | } else { |
1475 | ret = kIOReturnVMError; |
1476 | } |
1477 | |
1478 | address += page_size; |
1479 | length -= page_size; |
1480 | } |
1481 | |
1482 | return ret; |
1483 | } |
1484 | |
1485 | |
1486 | IOReturn |
1487 | IOFlushProcessorCache( task_t task, IOVirtualAddress address, |
1488 | IOByteCount length ) |
1489 | { |
1490 | if (task != kernel_task) { |
1491 | return kIOReturnUnsupported; |
1492 | } |
1493 | |
1494 | flush_dcache64((addr64_t) address, (unsigned) length, false ); |
1495 | |
1496 | return kIOReturnSuccess; |
1497 | } |
1498 | |
1499 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1500 | |
1501 | vm_offset_t |
1502 | OSKernelStackRemaining( void ) |
1503 | { |
1504 | return ml_stack_remaining(); |
1505 | } |
1506 | |
1507 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1508 | |
1509 | /* |
1510 | * Spin for indicated number of milliseconds. |
1511 | */ |
1512 | void |
1513 | IOSleep(unsigned milliseconds) |
1514 | { |
1515 | delay_for_interval(interval: milliseconds, scale_factor: kMillisecondScale); |
1516 | } |
1517 | |
1518 | /* |
1519 | * Spin for indicated number of milliseconds, and potentially an |
1520 | * additional number of milliseconds up to the leeway values. |
1521 | */ |
1522 | void |
1523 | IOSleepWithLeeway(unsigned intervalMilliseconds, unsigned leewayMilliseconds) |
1524 | { |
1525 | delay_for_interval_with_leeway(interval: intervalMilliseconds, leeway: leewayMilliseconds, scale_factor: kMillisecondScale); |
1526 | } |
1527 | |
1528 | /* |
1529 | * Spin for indicated number of microseconds. |
1530 | */ |
1531 | void |
1532 | IODelay(unsigned microseconds) |
1533 | { |
1534 | delay_for_interval(interval: microseconds, scale_factor: kMicrosecondScale); |
1535 | } |
1536 | |
1537 | /* |
1538 | * Spin for indicated number of nanoseconds. |
1539 | */ |
1540 | void |
1541 | IOPause(unsigned nanoseconds) |
1542 | { |
1543 | delay_for_interval(interval: nanoseconds, scale_factor: kNanosecondScale); |
1544 | } |
1545 | |
1546 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1547 | |
1548 | static void _IOLogv(const char *format, va_list ap, void *caller) __printflike(1, 0); |
1549 | |
1550 | __attribute__((noinline, not_tail_called)) |
1551 | void |
1552 | IOLog(const char *format, ...) |
1553 | { |
1554 | void *caller = __builtin_return_address(0); |
1555 | va_list ap; |
1556 | |
1557 | va_start(ap, format); |
1558 | _IOLogv(format, ap, caller); |
1559 | va_end(ap); |
1560 | } |
1561 | |
1562 | __attribute__((noinline, not_tail_called)) |
1563 | void |
1564 | IOLogv(const char *format, va_list ap) |
1565 | { |
1566 | void *caller = __builtin_return_address(0); |
1567 | _IOLogv(format, ap, caller); |
1568 | } |
1569 | |
1570 | void |
1571 | _IOLogv(const char *format, va_list ap, void *caller) |
1572 | { |
1573 | va_list ap2; |
1574 | struct console_printbuf_state info_data; |
1575 | console_printbuf_state_init(data: &info_data, TRUE, TRUE); |
1576 | |
1577 | va_copy(ap2, ap); |
1578 | |
1579 | #pragma clang diagnostic push |
1580 | #pragma clang diagnostic ignored "-Wformat-nonliteral" |
1581 | os_log_with_args(OS_LOG_DEFAULT, type: OS_LOG_TYPE_DEFAULT, format, args: ap, ret_addr: caller); |
1582 | #pragma clang diagnostic pop |
1583 | |
1584 | if (!disable_iolog_serial_output) { |
1585 | __doprnt(fmt: format, argp: ap2, putc: console_printbuf_putc, arg: &info_data, radix: 16, TRUE); |
1586 | console_printbuf_clear(info: &info_data); |
1587 | } |
1588 | va_end(ap2); |
1589 | |
1590 | assertf(ml_get_interrupts_enabled() || ml_is_quiescing() || |
1591 | debug_mode_active() || !gCPUsRunning, |
1592 | "IOLog called with interrupts disabled" ); |
1593 | } |
1594 | |
1595 | #if !__LP64__ |
1596 | void |
1597 | IOPanic(const char *reason) |
1598 | { |
1599 | panic("%s" , reason); |
1600 | } |
1601 | #endif |
1602 | |
1603 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1604 | |
1605 | void |
1606 | IOKitKernelLogBuffer(const char * title, const void * buffer, size_t size, |
1607 | void (*output)(const char *format, ...)) |
1608 | { |
1609 | size_t idx, linestart; |
1610 | enum { bytelen = (sizeof("0xZZ, " ) - 1) }; |
1611 | char hex[(bytelen * 16) + 1]; |
1612 | uint8_t c, chars[17]; |
1613 | |
1614 | output("%s(0x%lx):\n" , title, size); |
1615 | output(" 0 1 2 3 4 5 6 7 8 9 A B C D E F\n" ); |
1616 | if (size > 4096) { |
1617 | size = 4096; |
1618 | } |
1619 | chars[16] = 0; |
1620 | for (idx = 0, linestart = 0; idx < size;) { |
1621 | c = ((char *)buffer)[idx]; |
1622 | snprintf(&hex[bytelen * (idx & 15)], count: bytelen + 1, "0x%02x, " , c); |
1623 | chars[idx & 15] = ((c >= 0x20) && (c <= 0x7f)) ? c : ' '; |
1624 | idx++; |
1625 | if ((idx == size) || !(idx & 15)) { |
1626 | if (idx & 15) { |
1627 | chars[idx & 15] = 0; |
1628 | } |
1629 | output("/* %04lx: */ %-96s /* |%-16s| */\n" , linestart, hex, chars); |
1630 | linestart += 16; |
1631 | } |
1632 | } |
1633 | } |
1634 | |
1635 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1636 | |
1637 | /* |
1638 | * Convert a integer constant (typically a #define or enum) to a string. |
1639 | */ |
1640 | static char noValue[80]; // that's pretty |
1641 | |
1642 | const char * |
1643 | IOFindNameForValue(int value, const IONamedValue *regValueArray) |
1644 | { |
1645 | for (; regValueArray->name; regValueArray++) { |
1646 | if (regValueArray->value == value) { |
1647 | return regValueArray->name; |
1648 | } |
1649 | } |
1650 | snprintf(noValue, count: sizeof(noValue), "0x%x (UNDEFINED)" , value); |
1651 | return (const char *)noValue; |
1652 | } |
1653 | |
1654 | IOReturn |
1655 | IOFindValueForName(const char *string, |
1656 | const IONamedValue *regValueArray, |
1657 | int *value) |
1658 | { |
1659 | for (; regValueArray->name; regValueArray++) { |
1660 | if (!strcmp(s1: regValueArray->name, s2: string)) { |
1661 | *value = regValueArray->value; |
1662 | return kIOReturnSuccess; |
1663 | } |
1664 | } |
1665 | return kIOReturnBadArgument; |
1666 | } |
1667 | |
1668 | OSString * |
1669 | IOCopyLogNameForPID(int pid) |
1670 | { |
1671 | char buf[128]; |
1672 | size_t len; |
1673 | snprintf(buf, count: sizeof(buf), "pid %d, " , pid); |
1674 | len = strlen(s: buf); |
1675 | proc_name(pid, buf: buf + len, size: (int) (sizeof(buf) - len)); |
1676 | return OSString::withCString(cString: buf); |
1677 | } |
1678 | |
1679 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1680 | |
1681 | IOAlignment |
1682 | IOSizeToAlignment(unsigned int size) |
1683 | { |
1684 | int shift; |
1685 | const int intsize = sizeof(unsigned int) * 8; |
1686 | |
1687 | for (shift = 1; shift < intsize; shift++) { |
1688 | if (size & 0x80000000) { |
1689 | return (IOAlignment)(intsize - shift); |
1690 | } |
1691 | size <<= 1; |
1692 | } |
1693 | return 0; |
1694 | } |
1695 | |
1696 | unsigned int |
1697 | IOAlignmentToSize(IOAlignment align) |
1698 | { |
1699 | unsigned int size; |
1700 | |
1701 | for (size = 1; align; align--) { |
1702 | size <<= 1; |
1703 | } |
1704 | return size; |
1705 | } |
1706 | } /* extern "C" */ |
1707 | |