1 | /* |
2 | * Copyright (c) 2000-2007 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | /* |
57 | */ |
58 | /* |
59 | * File: vm/vm_kern.c |
60 | * Author: Avadis Tevanian, Jr., Michael Wayne Young |
61 | * Date: 1985 |
62 | * |
63 | * Kernel memory management. |
64 | */ |
65 | |
66 | #include <mach/kern_return.h> |
67 | #include <mach/vm_param.h> |
68 | #include <kern/assert.h> |
69 | #include <kern/thread.h> |
70 | #include <vm/vm_kern.h> |
71 | #include <vm/vm_map.h> |
72 | #include <vm/vm_object.h> |
73 | #include <vm/vm_page.h> |
74 | #include <vm/vm_compressor.h> |
75 | #include <vm/vm_pageout.h> |
76 | #include <kern/misc_protos.h> |
77 | #include <vm/cpm.h> |
78 | |
79 | #include <string.h> |
80 | |
81 | #include <libkern/OSDebug.h> |
82 | #include <libkern/crypto/sha2.h> |
83 | #include <sys/kdebug.h> |
84 | |
85 | #include <san/kasan.h> |
86 | |
87 | /* |
88 | * Variables exported by this module. |
89 | */ |
90 | |
91 | vm_map_t kernel_map; |
92 | vm_map_t kernel_pageable_map; |
93 | |
94 | extern boolean_t vm_kernel_ready; |
95 | |
96 | /* |
97 | * Forward declarations for internal functions. |
98 | */ |
99 | extern kern_return_t kmem_alloc_pages( |
100 | vm_object_t object, |
101 | vm_object_offset_t offset, |
102 | vm_object_size_t size); |
103 | |
104 | kern_return_t |
105 | kmem_alloc_contig( |
106 | vm_map_t map, |
107 | vm_offset_t *addrp, |
108 | vm_size_t size, |
109 | vm_offset_t mask, |
110 | ppnum_t max_pnum, |
111 | ppnum_t pnum_mask, |
112 | int flags, |
113 | vm_tag_t tag) |
114 | { |
115 | vm_object_t object; |
116 | vm_object_offset_t offset; |
117 | vm_map_offset_t map_addr; |
118 | vm_map_offset_t map_mask; |
119 | vm_map_size_t map_size, i; |
120 | vm_map_entry_t entry; |
121 | vm_page_t m, pages; |
122 | kern_return_t kr; |
123 | |
124 | assert(VM_KERN_MEMORY_NONE != tag); |
125 | |
126 | if (map == VM_MAP_NULL || (flags & ~(KMA_KOBJECT | KMA_LOMEM | KMA_NOPAGEWAIT))) |
127 | return KERN_INVALID_ARGUMENT; |
128 | |
129 | map_size = vm_map_round_page(size, |
130 | VM_MAP_PAGE_MASK(map)); |
131 | map_mask = (vm_map_offset_t)mask; |
132 | |
133 | /* Check for zero allocation size (either directly or via overflow) */ |
134 | if (map_size == 0) { |
135 | *addrp = 0; |
136 | return KERN_INVALID_ARGUMENT; |
137 | } |
138 | |
139 | /* |
140 | * Allocate a new object (if necessary) and the reference we |
141 | * will be donating to the map entry. We must do this before |
142 | * locking the map, or risk deadlock with the default pager. |
143 | */ |
144 | if ((flags & KMA_KOBJECT) != 0) { |
145 | object = kernel_object; |
146 | vm_object_reference(object); |
147 | } else { |
148 | object = vm_object_allocate(map_size); |
149 | } |
150 | |
151 | kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, |
152 | VM_MAP_KERNEL_FLAGS_NONE, tag, &entry); |
153 | if (KERN_SUCCESS != kr) { |
154 | vm_object_deallocate(object); |
155 | return kr; |
156 | } |
157 | |
158 | if (object == kernel_object) { |
159 | offset = map_addr; |
160 | } else { |
161 | offset = 0; |
162 | } |
163 | VME_OBJECT_SET(entry, object); |
164 | VME_OFFSET_SET(entry, offset); |
165 | |
166 | /* Take an extra object ref in case the map entry gets deleted */ |
167 | vm_object_reference(object); |
168 | vm_map_unlock(map); |
169 | |
170 | kr = cpm_allocate(CAST_DOWN(vm_size_t, map_size), &pages, max_pnum, pnum_mask, FALSE, flags); |
171 | |
172 | if (kr != KERN_SUCCESS) { |
173 | vm_map_remove(map, |
174 | vm_map_trunc_page(map_addr, |
175 | VM_MAP_PAGE_MASK(map)), |
176 | vm_map_round_page(map_addr + map_size, |
177 | VM_MAP_PAGE_MASK(map)), |
178 | VM_MAP_REMOVE_NO_FLAGS); |
179 | vm_object_deallocate(object); |
180 | *addrp = 0; |
181 | return kr; |
182 | } |
183 | |
184 | vm_object_lock(object); |
185 | for (i = 0; i < map_size; i += PAGE_SIZE) { |
186 | m = pages; |
187 | pages = NEXT_PAGE(m); |
188 | *(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL; |
189 | m->vmp_busy = FALSE; |
190 | vm_page_insert(m, object, offset + i); |
191 | } |
192 | vm_object_unlock(object); |
193 | |
194 | kr = vm_map_wire_kernel(map, |
195 | vm_map_trunc_page(map_addr, |
196 | VM_MAP_PAGE_MASK(map)), |
197 | vm_map_round_page(map_addr + map_size, |
198 | VM_MAP_PAGE_MASK(map)), |
199 | VM_PROT_DEFAULT, tag, |
200 | FALSE); |
201 | |
202 | if (kr != KERN_SUCCESS) { |
203 | if (object == kernel_object) { |
204 | vm_object_lock(object); |
205 | vm_object_page_remove(object, offset, offset + map_size); |
206 | vm_object_unlock(object); |
207 | } |
208 | vm_map_remove(map, |
209 | vm_map_trunc_page(map_addr, |
210 | VM_MAP_PAGE_MASK(map)), |
211 | vm_map_round_page(map_addr + map_size, |
212 | VM_MAP_PAGE_MASK(map)), |
213 | VM_MAP_REMOVE_NO_FLAGS); |
214 | vm_object_deallocate(object); |
215 | return kr; |
216 | } |
217 | vm_object_deallocate(object); |
218 | |
219 | if (object == kernel_object) { |
220 | vm_map_simplify(map, map_addr); |
221 | vm_tag_update_size(tag, map_size); |
222 | } |
223 | *addrp = (vm_offset_t) map_addr; |
224 | assert((vm_map_offset_t) *addrp == map_addr); |
225 | |
226 | return KERN_SUCCESS; |
227 | } |
228 | |
229 | /* |
230 | * Master entry point for allocating kernel memory. |
231 | * NOTE: this routine is _never_ interrupt safe. |
232 | * |
233 | * map : map to allocate into |
234 | * addrp : pointer to start address of new memory |
235 | * size : size of memory requested |
236 | * flags : options |
237 | * KMA_HERE *addrp is base address, else "anywhere" |
238 | * KMA_NOPAGEWAIT don't wait for pages if unavailable |
239 | * KMA_KOBJECT use kernel_object |
240 | * KMA_LOMEM support for 32 bit devices in a 64 bit world |
241 | * if set and a lomemory pool is available |
242 | * grab pages from it... this also implies |
243 | * KMA_NOPAGEWAIT |
244 | */ |
245 | |
246 | kern_return_t |
247 | kernel_memory_allocate( |
248 | vm_map_t map, |
249 | vm_offset_t *addrp, |
250 | vm_size_t size, |
251 | vm_offset_t mask, |
252 | int flags, |
253 | vm_tag_t tag) |
254 | { |
255 | vm_object_t object; |
256 | vm_object_offset_t offset; |
257 | vm_object_offset_t pg_offset; |
258 | vm_map_entry_t entry = NULL; |
259 | vm_map_offset_t map_addr, fill_start; |
260 | vm_map_offset_t map_mask; |
261 | vm_map_size_t map_size, fill_size; |
262 | kern_return_t kr, pe_result; |
263 | vm_page_t mem; |
264 | vm_page_t guard_page_list = NULL; |
265 | vm_page_t wired_page_list = NULL; |
266 | int guard_page_count = 0; |
267 | int wired_page_count = 0; |
268 | int page_grab_count = 0; |
269 | int i; |
270 | int vm_alloc_flags; |
271 | vm_map_kernel_flags_t vmk_flags; |
272 | vm_prot_t kma_prot; |
273 | |
274 | if (! vm_kernel_ready) { |
275 | panic("kernel_memory_allocate: VM is not ready" ); |
276 | } |
277 | |
278 | map_size = vm_map_round_page(size, |
279 | VM_MAP_PAGE_MASK(map)); |
280 | map_mask = (vm_map_offset_t) mask; |
281 | |
282 | vm_alloc_flags = 0; //VM_MAKE_TAG(tag); |
283 | vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; |
284 | |
285 | /* Check for zero allocation size (either directly or via overflow) */ |
286 | if (map_size == 0) { |
287 | *addrp = 0; |
288 | return KERN_INVALID_ARGUMENT; |
289 | } |
290 | |
291 | /* |
292 | * limit the size of a single extent of wired memory |
293 | * to try and limit the damage to the system if |
294 | * too many pages get wired down |
295 | * limit raised to 2GB with 128GB max physical limit, |
296 | * but scaled by installed memory above this |
297 | */ |
298 | if (!(flags & (KMA_VAONLY | KMA_PAGEABLE)) && |
299 | map_size > MAX(1ULL<<31, sane_size/64)) { |
300 | return KERN_RESOURCE_SHORTAGE; |
301 | } |
302 | |
303 | /* |
304 | * Guard pages: |
305 | * |
306 | * Guard pages are implemented as ficticious pages. By placing guard pages |
307 | * on either end of a stack, they can help detect cases where a thread walks |
308 | * off either end of its stack. They are allocated and set up here and attempts |
309 | * to access those pages are trapped in vm_fault_page(). |
310 | * |
311 | * The map_size we were passed may include extra space for |
312 | * guard pages. If those were requested, then back it out of fill_size |
313 | * since vm_map_find_space() takes just the actual size not including |
314 | * guard pages. Similarly, fill_start indicates where the actual pages |
315 | * will begin in the range. |
316 | */ |
317 | |
318 | fill_start = 0; |
319 | fill_size = map_size; |
320 | |
321 | if (flags & KMA_GUARD_FIRST) { |
322 | vmk_flags.vmkf_guard_before = TRUE; |
323 | fill_start += PAGE_SIZE_64; |
324 | fill_size -= PAGE_SIZE_64; |
325 | if (map_size < fill_start + fill_size) { |
326 | /* no space for a guard page */ |
327 | *addrp = 0; |
328 | return KERN_INVALID_ARGUMENT; |
329 | } |
330 | guard_page_count++; |
331 | } |
332 | if (flags & KMA_GUARD_LAST) { |
333 | vmk_flags.vmkf_guard_after = TRUE; |
334 | fill_size -= PAGE_SIZE_64; |
335 | if (map_size <= fill_start + fill_size) { |
336 | /* no space for a guard page */ |
337 | *addrp = 0; |
338 | return KERN_INVALID_ARGUMENT; |
339 | } |
340 | guard_page_count++; |
341 | } |
342 | wired_page_count = (int) (fill_size / PAGE_SIZE_64); |
343 | assert(wired_page_count * PAGE_SIZE_64 == fill_size); |
344 | |
345 | #if DEBUG || DEVELOPMENT |
346 | VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_START, size, 0, 0, 0); |
347 | #endif |
348 | |
349 | for (i = 0; i < guard_page_count; i++) { |
350 | for (;;) { |
351 | mem = vm_page_grab_guard(); |
352 | |
353 | if (mem != VM_PAGE_NULL) |
354 | break; |
355 | if (flags & KMA_NOPAGEWAIT) { |
356 | kr = KERN_RESOURCE_SHORTAGE; |
357 | goto out; |
358 | } |
359 | vm_page_more_fictitious(); |
360 | } |
361 | mem->vmp_snext = guard_page_list; |
362 | guard_page_list = mem; |
363 | } |
364 | |
365 | if (!(flags & (KMA_VAONLY | KMA_PAGEABLE))) { |
366 | for (i = 0; i < wired_page_count; i++) { |
367 | uint64_t unavailable; |
368 | |
369 | for (;;) { |
370 | if (flags & KMA_LOMEM) |
371 | mem = vm_page_grablo(); |
372 | else |
373 | mem = vm_page_grab(); |
374 | |
375 | if (mem != VM_PAGE_NULL) |
376 | break; |
377 | |
378 | if (flags & KMA_NOPAGEWAIT) { |
379 | kr = KERN_RESOURCE_SHORTAGE; |
380 | goto out; |
381 | } |
382 | if ((flags & KMA_LOMEM) && (vm_lopage_needed == TRUE)) { |
383 | kr = KERN_RESOURCE_SHORTAGE; |
384 | goto out; |
385 | } |
386 | unavailable = (vm_page_wire_count + vm_page_free_target) * PAGE_SIZE; |
387 | |
388 | if (unavailable > max_mem || map_size > (max_mem - unavailable)) { |
389 | kr = KERN_RESOURCE_SHORTAGE; |
390 | goto out; |
391 | } |
392 | VM_PAGE_WAIT(); |
393 | } |
394 | page_grab_count++; |
395 | if (KMA_ZERO & flags) vm_page_zero_fill(mem); |
396 | mem->vmp_snext = wired_page_list; |
397 | wired_page_list = mem; |
398 | } |
399 | } |
400 | |
401 | /* |
402 | * Allocate a new object (if necessary). We must do this before |
403 | * locking the map, or risk deadlock with the default pager. |
404 | */ |
405 | if ((flags & KMA_KOBJECT) != 0) { |
406 | object = kernel_object; |
407 | vm_object_reference(object); |
408 | } else if ((flags & KMA_COMPRESSOR) != 0) { |
409 | object = compressor_object; |
410 | vm_object_reference(object); |
411 | } else { |
412 | object = vm_object_allocate(map_size); |
413 | } |
414 | |
415 | if (flags & KMA_ATOMIC) |
416 | vmk_flags.vmkf_atomic_entry = TRUE; |
417 | |
418 | kr = vm_map_find_space(map, &map_addr, |
419 | fill_size, map_mask, |
420 | vm_alloc_flags, vmk_flags, tag, &entry); |
421 | if (KERN_SUCCESS != kr) { |
422 | vm_object_deallocate(object); |
423 | goto out; |
424 | } |
425 | |
426 | if (object == kernel_object || object == compressor_object) { |
427 | offset = map_addr; |
428 | } else { |
429 | offset = 0; |
430 | } |
431 | VME_OBJECT_SET(entry, object); |
432 | VME_OFFSET_SET(entry, offset); |
433 | |
434 | if (!(flags & (KMA_COMPRESSOR | KMA_PAGEABLE))) |
435 | entry->wired_count++; |
436 | |
437 | if (flags & KMA_PERMANENT) |
438 | entry->permanent = TRUE; |
439 | |
440 | if (object != kernel_object && object != compressor_object) |
441 | vm_object_reference(object); |
442 | |
443 | vm_object_lock(object); |
444 | vm_map_unlock(map); |
445 | |
446 | pg_offset = 0; |
447 | |
448 | if (fill_start) { |
449 | if (guard_page_list == NULL) |
450 | panic("kernel_memory_allocate: guard_page_list == NULL" ); |
451 | |
452 | mem = guard_page_list; |
453 | guard_page_list = mem->vmp_snext; |
454 | mem->vmp_snext = NULL; |
455 | |
456 | vm_page_insert(mem, object, offset + pg_offset); |
457 | |
458 | mem->vmp_busy = FALSE; |
459 | pg_offset += PAGE_SIZE_64; |
460 | } |
461 | |
462 | kma_prot = VM_PROT_READ | VM_PROT_WRITE; |
463 | |
464 | #if KASAN |
465 | if (!(flags & KMA_VAONLY)) { |
466 | /* for VAONLY mappings we notify in populate only */ |
467 | kasan_notify_address(map_addr, size); |
468 | } |
469 | #endif |
470 | |
471 | if (flags & (KMA_VAONLY | KMA_PAGEABLE)) { |
472 | pg_offset = fill_start + fill_size; |
473 | } else { |
474 | for (pg_offset = fill_start; pg_offset < fill_start + fill_size; pg_offset += PAGE_SIZE_64) { |
475 | if (wired_page_list == NULL) |
476 | panic("kernel_memory_allocate: wired_page_list == NULL" ); |
477 | |
478 | mem = wired_page_list; |
479 | wired_page_list = mem->vmp_snext; |
480 | mem->vmp_snext = NULL; |
481 | |
482 | assert(mem->vmp_wire_count == 0); |
483 | assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q); |
484 | |
485 | mem->vmp_q_state = VM_PAGE_IS_WIRED; |
486 | mem->vmp_wire_count++; |
487 | if (__improbable(mem->vmp_wire_count == 0)) { |
488 | panic("kernel_memory_allocate(%p): wire_count overflow" , |
489 | mem); |
490 | } |
491 | |
492 | vm_page_insert_wired(mem, object, offset + pg_offset, tag); |
493 | |
494 | mem->vmp_busy = FALSE; |
495 | mem->vmp_pmapped = TRUE; |
496 | mem->vmp_wpmapped = TRUE; |
497 | |
498 | PMAP_ENTER_OPTIONS(kernel_pmap, map_addr + pg_offset, mem, |
499 | kma_prot, VM_PROT_NONE, ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE, |
500 | PMAP_OPTIONS_NOWAIT, pe_result); |
501 | |
502 | if (pe_result == KERN_RESOURCE_SHORTAGE) { |
503 | vm_object_unlock(object); |
504 | |
505 | PMAP_ENTER(kernel_pmap, map_addr + pg_offset, mem, |
506 | kma_prot, VM_PROT_NONE, ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE, |
507 | pe_result); |
508 | |
509 | vm_object_lock(object); |
510 | } |
511 | |
512 | assert(pe_result == KERN_SUCCESS); |
513 | |
514 | if (flags & KMA_NOENCRYPT) { |
515 | bzero(CAST_DOWN(void *, (map_addr + pg_offset)), PAGE_SIZE); |
516 | |
517 | pmap_set_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)); |
518 | } |
519 | } |
520 | if (kernel_object == object) vm_tag_update_size(tag, fill_size); |
521 | } |
522 | if ((fill_start + fill_size) < map_size) { |
523 | if (guard_page_list == NULL) |
524 | panic("kernel_memory_allocate: guard_page_list == NULL" ); |
525 | |
526 | mem = guard_page_list; |
527 | guard_page_list = mem->vmp_snext; |
528 | mem->vmp_snext = NULL; |
529 | |
530 | vm_page_insert(mem, object, offset + pg_offset); |
531 | |
532 | mem->vmp_busy = FALSE; |
533 | } |
534 | if (guard_page_list || wired_page_list) |
535 | panic("kernel_memory_allocate: non empty list\n" ); |
536 | |
537 | if (!(flags & (KMA_VAONLY | KMA_PAGEABLE))) { |
538 | vm_page_lockspin_queues(); |
539 | vm_page_wire_count += wired_page_count; |
540 | vm_page_unlock_queues(); |
541 | } |
542 | |
543 | vm_object_unlock(object); |
544 | |
545 | /* |
546 | * now that the pages are wired, we no longer have to fear coalesce |
547 | */ |
548 | if (object == kernel_object || object == compressor_object) |
549 | vm_map_simplify(map, map_addr); |
550 | else |
551 | vm_object_deallocate(object); |
552 | |
553 | #if DEBUG || DEVELOPMENT |
554 | VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0); |
555 | #endif |
556 | |
557 | /* |
558 | * Return the memory, not zeroed. |
559 | */ |
560 | *addrp = CAST_DOWN(vm_offset_t, map_addr); |
561 | return KERN_SUCCESS; |
562 | |
563 | out: |
564 | if (guard_page_list) |
565 | vm_page_free_list(guard_page_list, FALSE); |
566 | |
567 | if (wired_page_list) |
568 | vm_page_free_list(wired_page_list, FALSE); |
569 | |
570 | #if DEBUG || DEVELOPMENT |
571 | VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0); |
572 | #endif |
573 | |
574 | return kr; |
575 | } |
576 | |
577 | kern_return_t |
578 | kernel_memory_populate( |
579 | vm_map_t map, |
580 | vm_offset_t addr, |
581 | vm_size_t size, |
582 | int flags, |
583 | vm_tag_t tag) |
584 | { |
585 | vm_object_t object; |
586 | vm_object_offset_t offset, pg_offset; |
587 | kern_return_t kr, pe_result; |
588 | vm_page_t mem; |
589 | vm_page_t page_list = NULL; |
590 | int page_count = 0; |
591 | int page_grab_count = 0; |
592 | int i; |
593 | |
594 | #if DEBUG || DEVELOPMENT |
595 | VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_START, size, 0, 0, 0); |
596 | #endif |
597 | |
598 | page_count = (int) (size / PAGE_SIZE_64); |
599 | |
600 | assert((flags & (KMA_COMPRESSOR|KMA_KOBJECT)) != (KMA_COMPRESSOR|KMA_KOBJECT)); |
601 | |
602 | if (flags & KMA_COMPRESSOR) { |
603 | |
604 | pg_offset = page_count * PAGE_SIZE_64; |
605 | |
606 | do { |
607 | for (;;) { |
608 | mem = vm_page_grab(); |
609 | |
610 | if (mem != VM_PAGE_NULL) |
611 | break; |
612 | |
613 | VM_PAGE_WAIT(); |
614 | } |
615 | page_grab_count++; |
616 | if (KMA_ZERO & flags) vm_page_zero_fill(mem); |
617 | mem->vmp_snext = page_list; |
618 | page_list = mem; |
619 | |
620 | pg_offset -= PAGE_SIZE_64; |
621 | |
622 | kr = pmap_enter_options(kernel_pmap, |
623 | addr + pg_offset, VM_PAGE_GET_PHYS_PAGE(mem), |
624 | VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, TRUE, |
625 | PMAP_OPTIONS_INTERNAL, NULL); |
626 | assert(kr == KERN_SUCCESS); |
627 | |
628 | } while (pg_offset); |
629 | |
630 | offset = addr; |
631 | object = compressor_object; |
632 | |
633 | vm_object_lock(object); |
634 | |
635 | for (pg_offset = 0; |
636 | pg_offset < size; |
637 | pg_offset += PAGE_SIZE_64) { |
638 | |
639 | mem = page_list; |
640 | page_list = mem->vmp_snext; |
641 | mem->vmp_snext = NULL; |
642 | |
643 | vm_page_insert(mem, object, offset + pg_offset); |
644 | assert(mem->vmp_busy); |
645 | |
646 | mem->vmp_busy = FALSE; |
647 | mem->vmp_pmapped = TRUE; |
648 | mem->vmp_wpmapped = TRUE; |
649 | mem->vmp_q_state = VM_PAGE_USED_BY_COMPRESSOR; |
650 | } |
651 | vm_object_unlock(object); |
652 | |
653 | #if KASAN |
654 | if (map == compressor_map) { |
655 | kasan_notify_address_nopoison(addr, size); |
656 | } else { |
657 | kasan_notify_address(addr, size); |
658 | } |
659 | #endif |
660 | |
661 | #if DEBUG || DEVELOPMENT |
662 | VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0); |
663 | #endif |
664 | return KERN_SUCCESS; |
665 | } |
666 | |
667 | for (i = 0; i < page_count; i++) { |
668 | for (;;) { |
669 | if (flags & KMA_LOMEM) |
670 | mem = vm_page_grablo(); |
671 | else |
672 | mem = vm_page_grab(); |
673 | |
674 | if (mem != VM_PAGE_NULL) |
675 | break; |
676 | |
677 | if (flags & KMA_NOPAGEWAIT) { |
678 | kr = KERN_RESOURCE_SHORTAGE; |
679 | goto out; |
680 | } |
681 | if ((flags & KMA_LOMEM) && |
682 | (vm_lopage_needed == TRUE)) { |
683 | kr = KERN_RESOURCE_SHORTAGE; |
684 | goto out; |
685 | } |
686 | VM_PAGE_WAIT(); |
687 | } |
688 | page_grab_count++; |
689 | if (KMA_ZERO & flags) vm_page_zero_fill(mem); |
690 | mem->vmp_snext = page_list; |
691 | page_list = mem; |
692 | } |
693 | if (flags & KMA_KOBJECT) { |
694 | offset = addr; |
695 | object = kernel_object; |
696 | |
697 | vm_object_lock(object); |
698 | } else { |
699 | /* |
700 | * If it's not the kernel object, we need to: |
701 | * lock map; |
702 | * lookup entry; |
703 | * lock object; |
704 | * take reference on object; |
705 | * unlock map; |
706 | */ |
707 | panic("kernel_memory_populate(%p,0x%llx,0x%llx,0x%x): " |
708 | "!KMA_KOBJECT" , |
709 | map, (uint64_t) addr, (uint64_t) size, flags); |
710 | } |
711 | |
712 | for (pg_offset = 0; |
713 | pg_offset < size; |
714 | pg_offset += PAGE_SIZE_64) { |
715 | |
716 | if (page_list == NULL) |
717 | panic("kernel_memory_populate: page_list == NULL" ); |
718 | |
719 | mem = page_list; |
720 | page_list = mem->vmp_snext; |
721 | mem->vmp_snext = NULL; |
722 | |
723 | assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q); |
724 | mem->vmp_q_state = VM_PAGE_IS_WIRED; |
725 | mem->vmp_wire_count++; |
726 | if (__improbable(mem->vmp_wire_count == 0)) { |
727 | panic("kernel_memory_populate(%p): wire_count overflow" , mem); |
728 | } |
729 | |
730 | vm_page_insert_wired(mem, object, offset + pg_offset, tag); |
731 | |
732 | mem->vmp_busy = FALSE; |
733 | mem->vmp_pmapped = TRUE; |
734 | mem->vmp_wpmapped = TRUE; |
735 | |
736 | PMAP_ENTER_OPTIONS(kernel_pmap, addr + pg_offset, mem, |
737 | VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, |
738 | ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE, |
739 | PMAP_OPTIONS_NOWAIT, pe_result); |
740 | |
741 | if (pe_result == KERN_RESOURCE_SHORTAGE) { |
742 | |
743 | vm_object_unlock(object); |
744 | |
745 | PMAP_ENTER(kernel_pmap, addr + pg_offset, mem, |
746 | VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, |
747 | ((flags & KMA_KSTACK) ? VM_MEM_STACK : 0), TRUE, |
748 | pe_result); |
749 | |
750 | vm_object_lock(object); |
751 | } |
752 | |
753 | assert(pe_result == KERN_SUCCESS); |
754 | |
755 | if (flags & KMA_NOENCRYPT) { |
756 | bzero(CAST_DOWN(void *, (addr + pg_offset)), PAGE_SIZE); |
757 | pmap_set_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem)); |
758 | } |
759 | } |
760 | vm_page_lockspin_queues(); |
761 | vm_page_wire_count += page_count; |
762 | vm_page_unlock_queues(); |
763 | |
764 | #if DEBUG || DEVELOPMENT |
765 | VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0); |
766 | #endif |
767 | |
768 | if (kernel_object == object) vm_tag_update_size(tag, size); |
769 | |
770 | vm_object_unlock(object); |
771 | |
772 | #if KASAN |
773 | if (map == compressor_map) { |
774 | kasan_notify_address_nopoison(addr, size); |
775 | } else { |
776 | kasan_notify_address(addr, size); |
777 | } |
778 | #endif |
779 | return KERN_SUCCESS; |
780 | |
781 | out: |
782 | if (page_list) |
783 | vm_page_free_list(page_list, FALSE); |
784 | |
785 | #if DEBUG || DEVELOPMENT |
786 | VM_DEBUG_CONSTANT_EVENT(vm_kern_request, VM_KERN_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0); |
787 | #endif |
788 | |
789 | return kr; |
790 | } |
791 | |
792 | |
793 | void |
794 | kernel_memory_depopulate( |
795 | vm_map_t map, |
796 | vm_offset_t addr, |
797 | vm_size_t size, |
798 | int flags) |
799 | { |
800 | vm_object_t object; |
801 | vm_object_offset_t offset, pg_offset; |
802 | vm_page_t mem; |
803 | vm_page_t local_freeq = NULL; |
804 | |
805 | assert((flags & (KMA_COMPRESSOR|KMA_KOBJECT)) != (KMA_COMPRESSOR|KMA_KOBJECT)); |
806 | |
807 | if (flags & KMA_COMPRESSOR) { |
808 | offset = addr; |
809 | object = compressor_object; |
810 | |
811 | vm_object_lock(object); |
812 | } else if (flags & KMA_KOBJECT) { |
813 | offset = addr; |
814 | object = kernel_object; |
815 | vm_object_lock(object); |
816 | } else { |
817 | offset = 0; |
818 | object = NULL; |
819 | /* |
820 | * If it's not the kernel object, we need to: |
821 | * lock map; |
822 | * lookup entry; |
823 | * lock object; |
824 | * unlock map; |
825 | */ |
826 | panic("kernel_memory_depopulate(%p,0x%llx,0x%llx,0x%x): " |
827 | "!KMA_KOBJECT" , |
828 | map, (uint64_t) addr, (uint64_t) size, flags); |
829 | } |
830 | pmap_protect(kernel_map->pmap, offset, offset + size, VM_PROT_NONE); |
831 | |
832 | for (pg_offset = 0; |
833 | pg_offset < size; |
834 | pg_offset += PAGE_SIZE_64) { |
835 | |
836 | mem = vm_page_lookup(object, offset + pg_offset); |
837 | |
838 | assert(mem); |
839 | |
840 | if (mem->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR) |
841 | pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(mem)); |
842 | |
843 | mem->vmp_busy = TRUE; |
844 | |
845 | assert(mem->vmp_tabled); |
846 | vm_page_remove(mem, TRUE); |
847 | assert(mem->vmp_busy); |
848 | |
849 | assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0); |
850 | assert((mem->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) || |
851 | (mem->vmp_q_state == VM_PAGE_NOT_ON_Q)); |
852 | |
853 | mem->vmp_q_state = VM_PAGE_NOT_ON_Q; |
854 | mem->vmp_snext = local_freeq; |
855 | local_freeq = mem; |
856 | } |
857 | vm_object_unlock(object); |
858 | |
859 | if (local_freeq) |
860 | vm_page_free_list(local_freeq, TRUE); |
861 | } |
862 | |
863 | /* |
864 | * kmem_alloc: |
865 | * |
866 | * Allocate wired-down memory in the kernel's address map |
867 | * or a submap. The memory is not zero-filled. |
868 | */ |
869 | |
870 | kern_return_t |
871 | kmem_alloc_external( |
872 | vm_map_t map, |
873 | vm_offset_t *addrp, |
874 | vm_size_t size) |
875 | { |
876 | return (kmem_alloc(map, addrp, size, vm_tag_bt())); |
877 | } |
878 | |
879 | |
880 | kern_return_t |
881 | kmem_alloc( |
882 | vm_map_t map, |
883 | vm_offset_t *addrp, |
884 | vm_size_t size, |
885 | vm_tag_t tag) |
886 | { |
887 | return kmem_alloc_flags(map, addrp, size, tag, 0); |
888 | } |
889 | |
890 | kern_return_t |
891 | kmem_alloc_flags( |
892 | vm_map_t map, |
893 | vm_offset_t *addrp, |
894 | vm_size_t size, |
895 | vm_tag_t tag, |
896 | int flags) |
897 | { |
898 | kern_return_t kr = kernel_memory_allocate(map, addrp, size, 0, flags, tag); |
899 | TRACE_MACHLEAKS(KMEM_ALLOC_CODE, KMEM_ALLOC_CODE_2, size, *addrp); |
900 | return kr; |
901 | } |
902 | |
903 | /* |
904 | * kmem_realloc: |
905 | * |
906 | * Reallocate wired-down memory in the kernel's address map |
907 | * or a submap. Newly allocated pages are not zeroed. |
908 | * This can only be used on regions allocated with kmem_alloc. |
909 | * |
910 | * If successful, the pages in the old region are mapped twice. |
911 | * The old region is unchanged. Use kmem_free to get rid of it. |
912 | */ |
913 | kern_return_t |
914 | kmem_realloc( |
915 | vm_map_t map, |
916 | vm_offset_t oldaddr, |
917 | vm_size_t oldsize, |
918 | vm_offset_t *newaddrp, |
919 | vm_size_t newsize, |
920 | vm_tag_t tag) |
921 | { |
922 | vm_object_t object; |
923 | vm_object_offset_t offset; |
924 | vm_map_offset_t oldmapmin; |
925 | vm_map_offset_t oldmapmax; |
926 | vm_map_offset_t newmapaddr; |
927 | vm_map_size_t oldmapsize; |
928 | vm_map_size_t newmapsize; |
929 | vm_map_entry_t oldentry; |
930 | vm_map_entry_t newentry; |
931 | vm_page_t mem; |
932 | kern_return_t kr; |
933 | |
934 | oldmapmin = vm_map_trunc_page(oldaddr, |
935 | VM_MAP_PAGE_MASK(map)); |
936 | oldmapmax = vm_map_round_page(oldaddr + oldsize, |
937 | VM_MAP_PAGE_MASK(map)); |
938 | oldmapsize = oldmapmax - oldmapmin; |
939 | newmapsize = vm_map_round_page(newsize, |
940 | VM_MAP_PAGE_MASK(map)); |
941 | if (newmapsize < newsize) { |
942 | /* overflow */ |
943 | *newaddrp = 0; |
944 | return KERN_INVALID_ARGUMENT; |
945 | } |
946 | |
947 | /* |
948 | * Find the VM object backing the old region. |
949 | */ |
950 | |
951 | vm_map_lock(map); |
952 | |
953 | if (!vm_map_lookup_entry(map, oldmapmin, &oldentry)) |
954 | panic("kmem_realloc" ); |
955 | object = VME_OBJECT(oldentry); |
956 | |
957 | /* |
958 | * Increase the size of the object and |
959 | * fill in the new region. |
960 | */ |
961 | |
962 | vm_object_reference(object); |
963 | /* by grabbing the object lock before unlocking the map */ |
964 | /* we guarantee that we will panic if more than one */ |
965 | /* attempt is made to realloc a kmem_alloc'd area */ |
966 | vm_object_lock(object); |
967 | vm_map_unlock(map); |
968 | if (object->vo_size != oldmapsize) |
969 | panic("kmem_realloc" ); |
970 | object->vo_size = newmapsize; |
971 | vm_object_unlock(object); |
972 | |
973 | /* allocate the new pages while expanded portion of the */ |
974 | /* object is still not mapped */ |
975 | kmem_alloc_pages(object, vm_object_round_page(oldmapsize), |
976 | vm_object_round_page(newmapsize-oldmapsize)); |
977 | |
978 | /* |
979 | * Find space for the new region. |
980 | */ |
981 | |
982 | kr = vm_map_find_space(map, &newmapaddr, newmapsize, |
983 | (vm_map_offset_t) 0, 0, |
984 | VM_MAP_KERNEL_FLAGS_NONE, |
985 | tag, |
986 | &newentry); |
987 | if (kr != KERN_SUCCESS) { |
988 | vm_object_lock(object); |
989 | for(offset = oldmapsize; |
990 | offset < newmapsize; offset += PAGE_SIZE) { |
991 | if ((mem = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { |
992 | VM_PAGE_FREE(mem); |
993 | } |
994 | } |
995 | object->vo_size = oldmapsize; |
996 | vm_object_unlock(object); |
997 | vm_object_deallocate(object); |
998 | return kr; |
999 | } |
1000 | VME_OBJECT_SET(newentry, object); |
1001 | VME_OFFSET_SET(newentry, 0); |
1002 | assert(newentry->wired_count == 0); |
1003 | |
1004 | |
1005 | /* add an extra reference in case we have someone doing an */ |
1006 | /* unexpected deallocate */ |
1007 | vm_object_reference(object); |
1008 | vm_map_unlock(map); |
1009 | |
1010 | kr = vm_map_wire_kernel(map, newmapaddr, newmapaddr + newmapsize, |
1011 | VM_PROT_DEFAULT, tag, FALSE); |
1012 | if (KERN_SUCCESS != kr) { |
1013 | vm_map_remove(map, newmapaddr, newmapaddr + newmapsize, VM_MAP_REMOVE_NO_FLAGS); |
1014 | vm_object_lock(object); |
1015 | for(offset = oldsize; offset < newmapsize; offset += PAGE_SIZE) { |
1016 | if ((mem = vm_page_lookup(object, offset)) != VM_PAGE_NULL) { |
1017 | VM_PAGE_FREE(mem); |
1018 | } |
1019 | } |
1020 | object->vo_size = oldmapsize; |
1021 | vm_object_unlock(object); |
1022 | vm_object_deallocate(object); |
1023 | return (kr); |
1024 | } |
1025 | vm_object_deallocate(object); |
1026 | |
1027 | if (kernel_object == object) vm_tag_update_size(tag, newmapsize); |
1028 | |
1029 | *newaddrp = CAST_DOWN(vm_offset_t, newmapaddr); |
1030 | return KERN_SUCCESS; |
1031 | } |
1032 | |
1033 | /* |
1034 | * kmem_alloc_kobject: |
1035 | * |
1036 | * Allocate wired-down memory in the kernel's address map |
1037 | * or a submap. The memory is not zero-filled. |
1038 | * |
1039 | * The memory is allocated in the kernel_object. |
1040 | * It may not be copied with vm_map_copy, and |
1041 | * it may not be reallocated with kmem_realloc. |
1042 | */ |
1043 | |
1044 | kern_return_t |
1045 | kmem_alloc_kobject_external( |
1046 | vm_map_t map, |
1047 | vm_offset_t *addrp, |
1048 | vm_size_t size) |
1049 | { |
1050 | return (kmem_alloc_kobject(map, addrp, size, vm_tag_bt())); |
1051 | } |
1052 | |
1053 | kern_return_t |
1054 | kmem_alloc_kobject( |
1055 | vm_map_t map, |
1056 | vm_offset_t *addrp, |
1057 | vm_size_t size, |
1058 | vm_tag_t tag) |
1059 | { |
1060 | return kernel_memory_allocate(map, addrp, size, 0, KMA_KOBJECT, tag); |
1061 | } |
1062 | |
1063 | /* |
1064 | * kmem_alloc_aligned: |
1065 | * |
1066 | * Like kmem_alloc_kobject, except that the memory is aligned. |
1067 | * The size should be a power-of-2. |
1068 | */ |
1069 | |
1070 | kern_return_t |
1071 | kmem_alloc_aligned( |
1072 | vm_map_t map, |
1073 | vm_offset_t *addrp, |
1074 | vm_size_t size, |
1075 | vm_tag_t tag) |
1076 | { |
1077 | if ((size & (size - 1)) != 0) |
1078 | panic("kmem_alloc_aligned: size not aligned" ); |
1079 | return kernel_memory_allocate(map, addrp, size, size - 1, KMA_KOBJECT, tag); |
1080 | } |
1081 | |
1082 | /* |
1083 | * kmem_alloc_pageable: |
1084 | * |
1085 | * Allocate pageable memory in the kernel's address map. |
1086 | */ |
1087 | |
1088 | kern_return_t |
1089 | kmem_alloc_pageable_external( |
1090 | vm_map_t map, |
1091 | vm_offset_t *addrp, |
1092 | vm_size_t size) |
1093 | { |
1094 | return (kmem_alloc_pageable(map, addrp, size, vm_tag_bt())); |
1095 | } |
1096 | |
1097 | kern_return_t |
1098 | kmem_alloc_pageable( |
1099 | vm_map_t map, |
1100 | vm_offset_t *addrp, |
1101 | vm_size_t size, |
1102 | vm_tag_t tag) |
1103 | { |
1104 | vm_map_offset_t map_addr; |
1105 | vm_map_size_t map_size; |
1106 | kern_return_t kr; |
1107 | |
1108 | #ifndef normal |
1109 | map_addr = (vm_map_min(map)) + PAGE_SIZE; |
1110 | #else |
1111 | map_addr = vm_map_min(map); |
1112 | #endif |
1113 | map_size = vm_map_round_page(size, |
1114 | VM_MAP_PAGE_MASK(map)); |
1115 | if (map_size < size) { |
1116 | /* overflow */ |
1117 | *addrp = 0; |
1118 | return KERN_INVALID_ARGUMENT; |
1119 | } |
1120 | |
1121 | kr = vm_map_enter(map, &map_addr, map_size, |
1122 | (vm_map_offset_t) 0, |
1123 | VM_FLAGS_ANYWHERE, |
1124 | VM_MAP_KERNEL_FLAGS_NONE, |
1125 | tag, |
1126 | VM_OBJECT_NULL, (vm_object_offset_t) 0, FALSE, |
1127 | VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); |
1128 | |
1129 | if (kr != KERN_SUCCESS) |
1130 | return kr; |
1131 | |
1132 | #if KASAN |
1133 | kasan_notify_address(map_addr, map_size); |
1134 | #endif |
1135 | *addrp = CAST_DOWN(vm_offset_t, map_addr); |
1136 | return KERN_SUCCESS; |
1137 | } |
1138 | |
1139 | /* |
1140 | * kmem_free: |
1141 | * |
1142 | * Release a region of kernel virtual memory allocated |
1143 | * with kmem_alloc, kmem_alloc_kobject, or kmem_alloc_pageable, |
1144 | * and return the physical pages associated with that region. |
1145 | */ |
1146 | |
1147 | void |
1148 | kmem_free( |
1149 | vm_map_t map, |
1150 | vm_offset_t addr, |
1151 | vm_size_t size) |
1152 | { |
1153 | kern_return_t kr; |
1154 | |
1155 | assert(addr >= VM_MIN_KERNEL_AND_KEXT_ADDRESS); |
1156 | |
1157 | TRACE_MACHLEAKS(KMEM_FREE_CODE, KMEM_FREE_CODE_2, size, addr); |
1158 | |
1159 | if(size == 0) { |
1160 | #if MACH_ASSERT |
1161 | printf("kmem_free called with size==0 for map: %p with addr: 0x%llx\n" ,map,(uint64_t)addr); |
1162 | #endif |
1163 | return; |
1164 | } |
1165 | |
1166 | kr = vm_map_remove(map, |
1167 | vm_map_trunc_page(addr, |
1168 | VM_MAP_PAGE_MASK(map)), |
1169 | vm_map_round_page(addr + size, |
1170 | VM_MAP_PAGE_MASK(map)), |
1171 | VM_MAP_REMOVE_KUNWIRE); |
1172 | if (kr != KERN_SUCCESS) |
1173 | panic("kmem_free" ); |
1174 | } |
1175 | |
1176 | /* |
1177 | * Allocate new pages in an object. |
1178 | */ |
1179 | |
1180 | kern_return_t |
1181 | kmem_alloc_pages( |
1182 | vm_object_t object, |
1183 | vm_object_offset_t offset, |
1184 | vm_object_size_t size) |
1185 | { |
1186 | vm_object_size_t alloc_size; |
1187 | |
1188 | alloc_size = vm_object_round_page(size); |
1189 | vm_object_lock(object); |
1190 | while (alloc_size) { |
1191 | vm_page_t mem; |
1192 | |
1193 | |
1194 | /* |
1195 | * Allocate a page |
1196 | */ |
1197 | while (VM_PAGE_NULL == |
1198 | (mem = vm_page_alloc(object, offset))) { |
1199 | vm_object_unlock(object); |
1200 | VM_PAGE_WAIT(); |
1201 | vm_object_lock(object); |
1202 | } |
1203 | mem->vmp_busy = FALSE; |
1204 | |
1205 | alloc_size -= PAGE_SIZE; |
1206 | offset += PAGE_SIZE; |
1207 | } |
1208 | vm_object_unlock(object); |
1209 | return KERN_SUCCESS; |
1210 | } |
1211 | |
1212 | /* |
1213 | * kmem_suballoc: |
1214 | * |
1215 | * Allocates a map to manage a subrange |
1216 | * of the kernel virtual address space. |
1217 | * |
1218 | * Arguments are as follows: |
1219 | * |
1220 | * parent Map to take range from |
1221 | * addr Address of start of range (IN/OUT) |
1222 | * size Size of range to find |
1223 | * pageable Can region be paged |
1224 | * anywhere Can region be located anywhere in map |
1225 | * new_map Pointer to new submap |
1226 | */ |
1227 | kern_return_t |
1228 | kmem_suballoc( |
1229 | vm_map_t parent, |
1230 | vm_offset_t *addr, |
1231 | vm_size_t size, |
1232 | boolean_t pageable, |
1233 | int flags, |
1234 | vm_map_kernel_flags_t vmk_flags, |
1235 | vm_tag_t tag, |
1236 | vm_map_t *new_map) |
1237 | { |
1238 | vm_map_t map; |
1239 | vm_map_offset_t map_addr; |
1240 | vm_map_size_t map_size; |
1241 | kern_return_t kr; |
1242 | |
1243 | map_size = vm_map_round_page(size, |
1244 | VM_MAP_PAGE_MASK(parent)); |
1245 | if (map_size < size) { |
1246 | /* overflow */ |
1247 | *addr = 0; |
1248 | return KERN_INVALID_ARGUMENT; |
1249 | } |
1250 | |
1251 | /* |
1252 | * Need reference on submap object because it is internal |
1253 | * to the vm_system. vm_object_enter will never be called |
1254 | * on it (usual source of reference for vm_map_enter). |
1255 | */ |
1256 | vm_object_reference(vm_submap_object); |
1257 | |
1258 | map_addr = ((flags & VM_FLAGS_ANYWHERE) |
1259 | ? vm_map_min(parent) |
1260 | : vm_map_trunc_page(*addr, |
1261 | VM_MAP_PAGE_MASK(parent))); |
1262 | |
1263 | kr = vm_map_enter(parent, &map_addr, map_size, |
1264 | (vm_map_offset_t) 0, flags, vmk_flags, tag, |
1265 | vm_submap_object, (vm_object_offset_t) 0, FALSE, |
1266 | VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); |
1267 | if (kr != KERN_SUCCESS) { |
1268 | vm_object_deallocate(vm_submap_object); |
1269 | return (kr); |
1270 | } |
1271 | |
1272 | pmap_reference(vm_map_pmap(parent)); |
1273 | map = vm_map_create(vm_map_pmap(parent), map_addr, map_addr + map_size, pageable); |
1274 | if (map == VM_MAP_NULL) |
1275 | panic("kmem_suballoc: vm_map_create failed" ); /* "can't happen" */ |
1276 | /* inherit the parent map's page size */ |
1277 | vm_map_set_page_shift(map, VM_MAP_PAGE_SHIFT(parent)); |
1278 | |
1279 | kr = vm_map_submap(parent, map_addr, map_addr + map_size, map, map_addr, FALSE); |
1280 | if (kr != KERN_SUCCESS) { |
1281 | /* |
1282 | * See comment preceding vm_map_submap(). |
1283 | */ |
1284 | vm_map_remove(parent, map_addr, map_addr + map_size, |
1285 | VM_MAP_REMOVE_NO_FLAGS); |
1286 | vm_map_deallocate(map); /* also removes ref to pmap */ |
1287 | vm_object_deallocate(vm_submap_object); |
1288 | return (kr); |
1289 | } |
1290 | *addr = CAST_DOWN(vm_offset_t, map_addr); |
1291 | *new_map = map; |
1292 | return (KERN_SUCCESS); |
1293 | } |
1294 | |
1295 | /* |
1296 | * kmem_init: |
1297 | * |
1298 | * Initialize the kernel's virtual memory map, taking |
1299 | * into account all memory allocated up to this time. |
1300 | */ |
1301 | void |
1302 | kmem_init( |
1303 | vm_offset_t start, |
1304 | vm_offset_t end) |
1305 | { |
1306 | vm_map_offset_t map_start; |
1307 | vm_map_offset_t map_end; |
1308 | vm_map_kernel_flags_t vmk_flags; |
1309 | |
1310 | vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; |
1311 | vmk_flags.vmkf_permanent = TRUE; |
1312 | vmk_flags.vmkf_no_pmap_check = TRUE; |
1313 | |
1314 | map_start = vm_map_trunc_page(start, |
1315 | VM_MAP_PAGE_MASK(kernel_map)); |
1316 | map_end = vm_map_round_page(end, |
1317 | VM_MAP_PAGE_MASK(kernel_map)); |
1318 | |
1319 | #if defined(__arm__) || defined(__arm64__) |
1320 | kernel_map = vm_map_create(pmap_kernel(),VM_MIN_KERNEL_AND_KEXT_ADDRESS, |
1321 | VM_MAX_KERNEL_ADDRESS, FALSE); |
1322 | /* |
1323 | * Reserve virtual memory allocated up to this time. |
1324 | */ |
1325 | { |
1326 | unsigned int region_select = 0; |
1327 | vm_map_offset_t region_start; |
1328 | vm_map_size_t region_size; |
1329 | vm_map_offset_t map_addr; |
1330 | kern_return_t kr; |
1331 | |
1332 | while (pmap_virtual_region(region_select, ®ion_start, ®ion_size)) { |
1333 | |
1334 | map_addr = region_start; |
1335 | kr = vm_map_enter(kernel_map, &map_addr, |
1336 | vm_map_round_page(region_size, |
1337 | VM_MAP_PAGE_MASK(kernel_map)), |
1338 | (vm_map_offset_t) 0, |
1339 | VM_FLAGS_FIXED, |
1340 | vmk_flags, |
1341 | VM_KERN_MEMORY_NONE, |
1342 | VM_OBJECT_NULL, |
1343 | (vm_object_offset_t) 0, FALSE, VM_PROT_NONE, VM_PROT_NONE, |
1344 | VM_INHERIT_DEFAULT); |
1345 | |
1346 | if (kr != KERN_SUCCESS) { |
1347 | panic("kmem_init(0x%llx,0x%llx): vm_map_enter(0x%llx,0x%llx) error 0x%x\n" , |
1348 | (uint64_t) start, (uint64_t) end, (uint64_t) region_start, |
1349 | (uint64_t) region_size, kr); |
1350 | } |
1351 | |
1352 | region_select++; |
1353 | } |
1354 | } |
1355 | #else |
1356 | kernel_map = vm_map_create(pmap_kernel(),VM_MIN_KERNEL_AND_KEXT_ADDRESS, |
1357 | map_end, FALSE); |
1358 | /* |
1359 | * Reserve virtual memory allocated up to this time. |
1360 | */ |
1361 | if (start != VM_MIN_KERNEL_AND_KEXT_ADDRESS) { |
1362 | vm_map_offset_t map_addr; |
1363 | kern_return_t kr; |
1364 | |
1365 | vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; |
1366 | vmk_flags.vmkf_no_pmap_check = TRUE; |
1367 | |
1368 | map_addr = VM_MIN_KERNEL_AND_KEXT_ADDRESS; |
1369 | kr = vm_map_enter(kernel_map, |
1370 | &map_addr, |
1371 | (vm_map_size_t)(map_start - VM_MIN_KERNEL_AND_KEXT_ADDRESS), |
1372 | (vm_map_offset_t) 0, |
1373 | VM_FLAGS_FIXED, |
1374 | vmk_flags, |
1375 | VM_KERN_MEMORY_NONE, |
1376 | VM_OBJECT_NULL, |
1377 | (vm_object_offset_t) 0, FALSE, |
1378 | VM_PROT_NONE, VM_PROT_NONE, |
1379 | VM_INHERIT_DEFAULT); |
1380 | |
1381 | if (kr != KERN_SUCCESS) { |
1382 | panic("kmem_init(0x%llx,0x%llx): vm_map_enter(0x%llx,0x%llx) error 0x%x\n" , |
1383 | (uint64_t) start, (uint64_t) end, |
1384 | (uint64_t) VM_MIN_KERNEL_AND_KEXT_ADDRESS, |
1385 | (uint64_t) (map_start - VM_MIN_KERNEL_AND_KEXT_ADDRESS), |
1386 | kr); |
1387 | } |
1388 | } |
1389 | #endif |
1390 | |
1391 | /* |
1392 | * Set the default global user wire limit which limits the amount of |
1393 | * memory that can be locked via mlock(). We set this to the total |
1394 | * amount of memory that are potentially usable by a user app (max_mem) |
1395 | * minus a certain amount. This can be overridden via a sysctl. |
1396 | */ |
1397 | vm_global_no_user_wire_amount = MIN(max_mem*20/100, |
1398 | VM_NOT_USER_WIREABLE); |
1399 | vm_global_user_wire_limit = max_mem - vm_global_no_user_wire_amount; |
1400 | |
1401 | /* the default per user limit is the same as the global limit */ |
1402 | vm_user_wire_limit = vm_global_user_wire_limit; |
1403 | } |
1404 | |
1405 | |
1406 | /* |
1407 | * Routine: copyinmap |
1408 | * Purpose: |
1409 | * Like copyin, except that fromaddr is an address |
1410 | * in the specified VM map. This implementation |
1411 | * is incomplete; it handles the current user map |
1412 | * and the kernel map/submaps. |
1413 | */ |
1414 | kern_return_t |
1415 | copyinmap( |
1416 | vm_map_t map, |
1417 | vm_map_offset_t fromaddr, |
1418 | void *todata, |
1419 | vm_size_t length) |
1420 | { |
1421 | kern_return_t kr = KERN_SUCCESS; |
1422 | vm_map_t oldmap; |
1423 | |
1424 | if (vm_map_pmap(map) == pmap_kernel()) |
1425 | { |
1426 | /* assume a correct copy */ |
1427 | memcpy(todata, CAST_DOWN(void *, fromaddr), length); |
1428 | } |
1429 | else if (current_map() == map) |
1430 | { |
1431 | if (copyin(fromaddr, todata, length) != 0) |
1432 | kr = KERN_INVALID_ADDRESS; |
1433 | } |
1434 | else |
1435 | { |
1436 | vm_map_reference(map); |
1437 | oldmap = vm_map_switch(map); |
1438 | if (copyin(fromaddr, todata, length) != 0) |
1439 | kr = KERN_INVALID_ADDRESS; |
1440 | vm_map_switch(oldmap); |
1441 | vm_map_deallocate(map); |
1442 | } |
1443 | return kr; |
1444 | } |
1445 | |
1446 | /* |
1447 | * Routine: copyoutmap |
1448 | * Purpose: |
1449 | * Like copyout, except that toaddr is an address |
1450 | * in the specified VM map. This implementation |
1451 | * is incomplete; it handles the current user map |
1452 | * and the kernel map/submaps. |
1453 | */ |
1454 | kern_return_t |
1455 | copyoutmap( |
1456 | vm_map_t map, |
1457 | void *fromdata, |
1458 | vm_map_address_t toaddr, |
1459 | vm_size_t length) |
1460 | { |
1461 | if (vm_map_pmap(map) == pmap_kernel()) { |
1462 | /* assume a correct copy */ |
1463 | memcpy(CAST_DOWN(void *, toaddr), fromdata, length); |
1464 | return KERN_SUCCESS; |
1465 | } |
1466 | |
1467 | if (current_map() != map) |
1468 | return KERN_NOT_SUPPORTED; |
1469 | |
1470 | if (copyout(fromdata, toaddr, length) != 0) |
1471 | return KERN_INVALID_ADDRESS; |
1472 | |
1473 | return KERN_SUCCESS; |
1474 | } |
1475 | |
1476 | /* |
1477 | * |
1478 | * The following two functions are to be used when exposing kernel |
1479 | * addresses to userspace via any of the various debug or info |
1480 | * facilities that exist. These are basically the same as VM_KERNEL_ADDRPERM() |
1481 | * and VM_KERNEL_UNSLIDE_OR_PERM() except they use a different random seed and |
1482 | * are exported to KEXTs. |
1483 | * |
1484 | * NOTE: USE THE MACRO VERSIONS OF THESE FUNCTIONS (in vm_param.h) FROM WITHIN THE KERNEL |
1485 | */ |
1486 | |
1487 | static void |
1488 | vm_kernel_addrhash_internal( |
1489 | vm_offset_t addr, |
1490 | vm_offset_t *hash_addr, |
1491 | uint64_t salt) |
1492 | { |
1493 | assert(salt != 0); |
1494 | |
1495 | if (addr == 0) { |
1496 | *hash_addr = 0; |
1497 | return; |
1498 | } |
1499 | |
1500 | if (VM_KERNEL_IS_SLID(addr)) { |
1501 | *hash_addr = VM_KERNEL_UNSLIDE(addr); |
1502 | return; |
1503 | } |
1504 | |
1505 | vm_offset_t sha_digest[SHA256_DIGEST_LENGTH/sizeof(vm_offset_t)]; |
1506 | SHA256_CTX sha_ctx; |
1507 | |
1508 | SHA256_Init(&sha_ctx); |
1509 | SHA256_Update(&sha_ctx, &salt, sizeof(salt)); |
1510 | SHA256_Update(&sha_ctx, &addr, sizeof(addr)); |
1511 | SHA256_Final(sha_digest, &sha_ctx); |
1512 | |
1513 | *hash_addr = sha_digest[0]; |
1514 | } |
1515 | |
1516 | void |
1517 | vm_kernel_addrhash_external( |
1518 | vm_offset_t addr, |
1519 | vm_offset_t *hash_addr) |
1520 | { |
1521 | return vm_kernel_addrhash_internal(addr, hash_addr, vm_kernel_addrhash_salt_ext); |
1522 | } |
1523 | |
1524 | vm_offset_t |
1525 | vm_kernel_addrhash(vm_offset_t addr) |
1526 | { |
1527 | vm_offset_t hash_addr; |
1528 | vm_kernel_addrhash_internal(addr, &hash_addr, vm_kernel_addrhash_salt); |
1529 | return hash_addr; |
1530 | } |
1531 | |
1532 | void |
1533 | vm_kernel_addrhide( |
1534 | vm_offset_t addr, |
1535 | vm_offset_t *hide_addr) |
1536 | { |
1537 | *hide_addr = VM_KERNEL_ADDRHIDE(addr); |
1538 | } |
1539 | |
1540 | /* |
1541 | * vm_kernel_addrperm_external: |
1542 | * vm_kernel_unslide_or_perm_external: |
1543 | * |
1544 | * Use these macros when exposing an address to userspace that could come from |
1545 | * either kernel text/data *or* the heap. |
1546 | */ |
1547 | void |
1548 | vm_kernel_addrperm_external( |
1549 | vm_offset_t addr, |
1550 | vm_offset_t *perm_addr) |
1551 | { |
1552 | if (VM_KERNEL_IS_SLID(addr)) { |
1553 | *perm_addr = VM_KERNEL_UNSLIDE(addr); |
1554 | } else if (VM_KERNEL_ADDRESS(addr)) { |
1555 | *perm_addr = addr + vm_kernel_addrperm_ext; |
1556 | } else { |
1557 | *perm_addr = addr; |
1558 | } |
1559 | } |
1560 | |
1561 | void |
1562 | vm_kernel_unslide_or_perm_external( |
1563 | vm_offset_t addr, |
1564 | vm_offset_t *up_addr) |
1565 | { |
1566 | vm_kernel_addrperm_external(addr, up_addr); |
1567 | } |
1568 | |