1/*
2 * Copyright (c) 2000-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm/vm_user.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * User-exported virtual memory functions.
63 */
64
65/*
66 * There are three implementations of the "XXX_allocate" functionality in
67 * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
68 * (for a task with the same address space size, especially the current task),
69 * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
70 * in the kernel should only be used on the kernel_task. vm32_vm_allocate only
71 * makes sense on platforms where a user task can either be 32 or 64, or the kernel
72 * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
73 * for new code.
74 *
75 * The entrypoints into the kernel are more complex. All platforms support a
76 * mach_vm_allocate-style API (subsystem 4800) which operates with the largest
77 * size types for the platform. On platforms that only support U32/K32,
78 * subsystem 4800 is all you need. On platforms that support both U32 and U64,
79 * subsystem 3800 is used disambiguate the size of parameters, and they will
80 * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
81 * the MIG glue should never call into vm_allocate directly, because the calling
82 * task and kernel_task are unlikely to use the same size parameters
83 *
84 * New VM call implementations should be added here and to mach_vm.defs
85 * (subsystem 4800), and use mach_vm_* "wide" types.
86 */
87
88#include <debug.h>
89
90#include <vm_cpm.h>
91#include <mach/boolean.h>
92#include <mach/kern_return.h>
93#include <mach/mach_types.h> /* to get vm_address_t */
94#include <mach/memory_object.h>
95#include <mach/std_types.h> /* to get pointer_t */
96#include <mach/upl.h>
97#include <mach/vm_attributes.h>
98#include <mach/vm_param.h>
99#include <mach/vm_statistics.h>
100#include <mach/mach_syscalls.h>
101#include <mach/sdt.h>
102
103#include <mach/host_priv_server.h>
104#include <mach/mach_vm_server.h>
105#include <mach/memory_entry_server.h>
106#include <mach/vm_map_server.h>
107
108#include <kern/host.h>
109#include <kern/kalloc.h>
110#include <kern/task.h>
111#include <kern/misc_protos.h>
112#include <vm/vm_fault.h>
113#include <vm/vm_map_internal.h>
114#include <vm/vm_object.h>
115#include <vm/vm_page.h>
116#include <vm/memory_object.h>
117#include <vm/vm_pageout.h>
118#include <vm/vm_protos.h>
119#include <vm/vm_purgeable_internal.h>
120#if CONFIG_DEFERRED_RECLAIM
121#include <vm/vm_reclaim_internal.h>
122#endif /* CONFIG_DEFERRED_RECLAIM */
123#include <vm/vm_init.h>
124
125#include <san/kasan.h>
126
127#include <libkern/OSDebug.h>
128#include <IOKit/IOBSD.h>
129#include <sys/kdebug_triage.h>
130
131#if VM_CPM
132#include <vm/cpm.h>
133#endif /* VM_CPM */
134
135static void mach_memory_entry_no_senders(ipc_port_t, mach_port_mscount_t);
136
137__attribute__((always_inline))
138int
139vm_map_kernel_flags_vmflags(vm_map_kernel_flags_t vmk_flags)
140{
141 int flags = vmk_flags.__vm_flags & VM_FLAGS_ANY_MASK;
142
143 /* in vmk flags the meaning of fixed/anywhere is inverted */
144 return flags ^ (VM_FLAGS_FIXED | VM_FLAGS_ANYWHERE);
145}
146
147__attribute__((always_inline, overloadable))
148void
149vm_map_kernel_flags_set_vmflags(
150 vm_map_kernel_flags_t *vmk_flags,
151 int vm_flags,
152 vm_tag_t vm_tag)
153{
154 vm_flags ^= (VM_FLAGS_FIXED | VM_FLAGS_ANYWHERE);
155 vmk_flags->__vm_flags &= ~VM_FLAGS_ANY_MASK;
156 vmk_flags->__vm_flags |= (vm_flags & VM_FLAGS_ANY_MASK);
157 vmk_flags->vm_tag = vm_tag;
158}
159
160__attribute__((always_inline, overloadable))
161void
162vm_map_kernel_flags_set_vmflags(
163 vm_map_kernel_flags_t *vmk_flags,
164 int vm_flags_and_tag)
165{
166 vm_flags_and_tag ^= (VM_FLAGS_FIXED | VM_FLAGS_ANYWHERE);
167 vmk_flags->__vm_flags &= ~VM_FLAGS_ANY_MASK;
168 vmk_flags->__vm_flags |= (vm_flags_and_tag & VM_FLAGS_ANY_MASK);
169 VM_GET_FLAGS_ALIAS(vm_flags_and_tag, vmk_flags->vm_tag);
170}
171
172__attribute__((always_inline))
173void
174vm_map_kernel_flags_and_vmflags(
175 vm_map_kernel_flags_t *vmk_flags,
176 int vm_flags_mask)
177{
178 /* this function doesn't handle the inverted FIXED/ANYWHERE */
179 assert(vm_flags_mask & VM_FLAGS_ANYWHERE);
180 vmk_flags->__vm_flags &= vm_flags_mask;
181}
182
183bool
184vm_map_kernel_flags_check_vmflags(
185 vm_map_kernel_flags_t vmk_flags,
186 int vm_flags_mask)
187{
188 int vmflags = vmk_flags.__vm_flags & VM_FLAGS_ANY_MASK;
189
190 /* Note: up to 16 still has good calling conventions */
191 static_assert(sizeof(vm_map_kernel_flags_t) == 8);
192
193#if DEBUG || DEVELOPMENT
194 /*
195 * All of this compiles to nothing if all checks pass.
196 */
197#define check(field, value) ({ \
198 vm_map_kernel_flags_t fl = VM_MAP_KERNEL_FLAGS_NONE; \
199 fl.__vm_flags = (value); \
200 fl.field = 0; \
201 assert(fl.__vm_flags == 0); \
202})
203
204 /* bits 0-7 */
205 check(vmf_fixed, VM_FLAGS_ANYWHERE); // kind of a lie this is inverted
206 check(vmf_purgeable, VM_FLAGS_PURGABLE);
207 check(vmf_4gb_chunk, VM_FLAGS_4GB_CHUNK);
208 check(vmf_random_addr, VM_FLAGS_RANDOM_ADDR);
209 check(vmf_no_cache, VM_FLAGS_NO_CACHE);
210 check(vmf_resilient_codesign, VM_FLAGS_RESILIENT_CODESIGN);
211 check(vmf_resilient_media, VM_FLAGS_RESILIENT_MEDIA);
212 check(vmf_permanent, VM_FLAGS_PERMANENT);
213
214 /* bits 8-15 */
215 check(vmf_tpro, VM_FLAGS_TPRO);
216 check(vmf_overwrite, VM_FLAGS_OVERWRITE);
217
218 /* bits 16-23 */
219 check(vmf_superpage_size, VM_FLAGS_SUPERPAGE_MASK);
220 check(vmf_return_data_addr, VM_FLAGS_RETURN_DATA_ADDR);
221 check(vmf_return_4k_data_addr, VM_FLAGS_RETURN_4K_DATA_ADDR);
222
223 {
224 vm_map_kernel_flags_t fl = VM_MAP_KERNEL_FLAGS_NONE;
225
226 /* check user tags will never clip */
227 fl.vm_tag = VM_MEMORY_COUNT - 1;
228 assert(fl.vm_tag == VM_MEMORY_COUNT - 1);
229
230 /* check kernel tags will never clip */
231 fl.vm_tag = VM_MAX_TAG_VALUE - 1;
232 assert(fl.vm_tag == VM_MAX_TAG_VALUE - 1);
233 }
234
235
236#undef check
237#endif /* DEBUG || DEVELOPMENT */
238
239 return (vmflags & ~vm_flags_mask) == 0;
240}
241
242kern_return_t
243vm_purgable_control(
244 vm_map_t map,
245 vm_offset_t address,
246 vm_purgable_t control,
247 int *state);
248
249kern_return_t
250mach_vm_purgable_control(
251 vm_map_t map,
252 mach_vm_offset_t address,
253 vm_purgable_t control,
254 int *state);
255
256kern_return_t
257mach_memory_entry_ownership(
258 ipc_port_t entry_port,
259 task_t owner,
260 int ledger_tag,
261 int ledger_flags);
262
263IPC_KOBJECT_DEFINE(IKOT_NAMED_ENTRY,
264 .iko_op_stable = true,
265 .iko_op_no_senders = mach_memory_entry_no_senders);
266
267/*
268 * mach_vm_allocate allocates "zero fill" memory in the specfied
269 * map.
270 */
271kern_return_t
272mach_vm_allocate_external(
273 vm_map_t map,
274 mach_vm_offset_t *addr,
275 mach_vm_size_t size,
276 int flags)
277{
278 vm_tag_t tag;
279
280 VM_GET_FLAGS_ALIAS(flags, tag);
281 return mach_vm_allocate_kernel(map, addr, size, flags, tag);
282}
283
284kern_return_t
285mach_vm_allocate_kernel(
286 vm_map_t map,
287 mach_vm_offset_t *addr,
288 mach_vm_size_t size,
289 int flags,
290 vm_tag_t tag)
291{
292 vm_map_offset_t map_addr;
293 vm_map_size_t map_size;
294 kern_return_t result;
295 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
296
297 /* filter out any kernel-only flags */
298 if (flags & ~VM_FLAGS_USER_ALLOCATE) {
299 ktriage_record(thread_id: thread_tid(thread: current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_ALLOCATE_KERNEL_BADFLAGS_ERROR), KERN_INVALID_ARGUMENT /* arg */);
300 return KERN_INVALID_ARGUMENT;
301 }
302
303 vm_map_kernel_flags_set_vmflags(vmk_flags: &vmk_flags, vm_flags: flags, vm_tag: tag);
304
305 if (map == VM_MAP_NULL) {
306 ktriage_record(thread_id: thread_tid(thread: current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_ALLOCATE_KERNEL_BADMAP_ERROR), KERN_INVALID_ARGUMENT /* arg */);
307 return KERN_INVALID_ARGUMENT;
308 }
309 if (size == 0) {
310 *addr = 0;
311 return KERN_SUCCESS;
312 }
313
314 if (vmk_flags.vmf_fixed) {
315 map_addr = vm_map_trunc_page(*addr, VM_MAP_PAGE_MASK(map));
316 } else {
317 map_addr = 0;
318 }
319 map_size = vm_map_round_page(size,
320 VM_MAP_PAGE_MASK(map));
321 if (map_size == 0) {
322 ktriage_record(thread_id: thread_tid(thread: current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_ALLOCATE_KERNEL_BADSIZE_ERROR), KERN_INVALID_ARGUMENT /* arg */);
323 return KERN_INVALID_ARGUMENT;
324 }
325
326 vm_map_kernel_flags_update_range_id(flags: &vmk_flags, map);
327
328 result = vm_map_enter(
329 map,
330 address: &map_addr,
331 size: map_size,
332 mask: (vm_map_offset_t)0,
333 vmk_flags,
334 VM_OBJECT_NULL,
335 offset: (vm_object_offset_t)0,
336 FALSE,
337 VM_PROT_DEFAULT,
338 VM_PROT_ALL,
339 VM_INHERIT_DEFAULT);
340
341#if KASAN
342 if (result == KERN_SUCCESS && map->pmap == kernel_pmap) {
343 kasan_notify_address(map_addr, map_size);
344 }
345#endif
346 if (result != KERN_SUCCESS) {
347 ktriage_record(thread_id: thread_tid(thread: current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_ALLOCATE_KERNEL_VMMAPENTER_ERROR), arg: result /* arg */);
348 }
349 *addr = map_addr;
350 return result;
351}
352
353/*
354 * vm_allocate
355 * Legacy routine that allocates "zero fill" memory in the specfied
356 * map (which is limited to the same size as the kernel).
357 */
358kern_return_t
359vm_allocate_external(
360 vm_map_t map,
361 vm_offset_t *addr,
362 vm_size_t size,
363 int flags)
364{
365 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
366 vm_map_offset_t map_addr;
367 vm_map_size_t map_size;
368 kern_return_t result;
369
370 /* filter out any kernel-only flags */
371 if (flags & ~VM_FLAGS_USER_ALLOCATE) {
372 return KERN_INVALID_ARGUMENT;
373 }
374
375 vm_map_kernel_flags_set_vmflags(vmk_flags: &vmk_flags, vm_flags_and_tag: flags);
376
377 if (map == VM_MAP_NULL) {
378 return KERN_INVALID_ARGUMENT;
379 }
380 if (size == 0) {
381 *addr = 0;
382 return KERN_SUCCESS;
383 }
384
385 if (vmk_flags.vmf_fixed) {
386 map_addr = vm_map_trunc_page(*addr, VM_MAP_PAGE_MASK(map));
387 } else {
388 map_addr = 0;
389 }
390 map_size = vm_map_round_page(size,
391 VM_MAP_PAGE_MASK(map));
392 if (map_size == 0) {
393 return KERN_INVALID_ARGUMENT;
394 }
395
396 vm_map_kernel_flags_update_range_id(flags: &vmk_flags, map);
397
398 result = vm_map_enter(
399 map,
400 address: &map_addr,
401 size: map_size,
402 mask: (vm_map_offset_t)0,
403 vmk_flags,
404 VM_OBJECT_NULL,
405 offset: (vm_object_offset_t)0,
406 FALSE,
407 VM_PROT_DEFAULT,
408 VM_PROT_ALL,
409 VM_INHERIT_DEFAULT);
410
411#if KASAN
412 if (result == KERN_SUCCESS && map->pmap == kernel_pmap) {
413 kasan_notify_address(map_addr, map_size);
414 }
415#endif
416
417 *addr = CAST_DOWN(vm_offset_t, map_addr);
418 return result;
419}
420
421/*
422 * mach_vm_deallocate -
423 * deallocates the specified range of addresses in the
424 * specified address map.
425 */
426kern_return_t
427mach_vm_deallocate(
428 vm_map_t map,
429 mach_vm_offset_t start,
430 mach_vm_size_t size)
431{
432 if ((map == VM_MAP_NULL) || (start + size < start)) {
433 return KERN_INVALID_ARGUMENT;
434 }
435
436 if (size == (mach_vm_offset_t) 0) {
437 return KERN_SUCCESS;
438 }
439
440 return vm_map_remove_guard(map,
441 vm_map_trunc_page(start,
442 VM_MAP_PAGE_MASK(map)),
443 vm_map_round_page(start + size,
444 VM_MAP_PAGE_MASK(map)),
445 flags: VM_MAP_REMOVE_NO_FLAGS,
446 KMEM_GUARD_NONE).kmr_return;
447}
448
449/*
450 * vm_deallocate -
451 * deallocates the specified range of addresses in the
452 * specified address map (limited to addresses the same
453 * size as the kernel).
454 */
455kern_return_t
456vm_deallocate(
457 vm_map_t map,
458 vm_offset_t start,
459 vm_size_t size)
460{
461 if ((map == VM_MAP_NULL) || (start + size < start)) {
462 return KERN_INVALID_ARGUMENT;
463 }
464
465 if (size == (vm_offset_t) 0) {
466 return KERN_SUCCESS;
467 }
468
469 return vm_map_remove_guard(map,
470 vm_map_trunc_page(start,
471 VM_MAP_PAGE_MASK(map)),
472 vm_map_round_page(start + size,
473 VM_MAP_PAGE_MASK(map)),
474 flags: VM_MAP_REMOVE_NO_FLAGS,
475 KMEM_GUARD_NONE).kmr_return;
476}
477
478/*
479 * mach_vm_inherit -
480 * Sets the inheritance of the specified range in the
481 * specified map.
482 */
483kern_return_t
484mach_vm_inherit(
485 vm_map_t map,
486 mach_vm_offset_t start,
487 mach_vm_size_t size,
488 vm_inherit_t new_inheritance)
489{
490 if ((map == VM_MAP_NULL) || (start + size < start) ||
491 (new_inheritance > VM_INHERIT_LAST_VALID)) {
492 return KERN_INVALID_ARGUMENT;
493 }
494
495 if (size == 0) {
496 return KERN_SUCCESS;
497 }
498
499 return vm_map_inherit(map,
500 vm_map_trunc_page(start,
501 VM_MAP_PAGE_MASK(map)),
502 vm_map_round_page(start + size,
503 VM_MAP_PAGE_MASK(map)),
504 new_inheritance);
505}
506
507/*
508 * vm_inherit -
509 * Sets the inheritance of the specified range in the
510 * specified map (range limited to addresses
511 */
512kern_return_t
513vm_inherit(
514 vm_map_t map,
515 vm_offset_t start,
516 vm_size_t size,
517 vm_inherit_t new_inheritance)
518{
519 if ((map == VM_MAP_NULL) || (start + size < start) ||
520 (new_inheritance > VM_INHERIT_LAST_VALID)) {
521 return KERN_INVALID_ARGUMENT;
522 }
523
524 if (size == 0) {
525 return KERN_SUCCESS;
526 }
527
528 return vm_map_inherit(map,
529 vm_map_trunc_page(start,
530 VM_MAP_PAGE_MASK(map)),
531 vm_map_round_page(start + size,
532 VM_MAP_PAGE_MASK(map)),
533 new_inheritance);
534}
535
536/*
537 * mach_vm_protect -
538 * Sets the protection of the specified range in the
539 * specified map.
540 */
541
542kern_return_t
543mach_vm_protect(
544 vm_map_t map,
545 mach_vm_offset_t start,
546 mach_vm_size_t size,
547 boolean_t set_maximum,
548 vm_prot_t new_protection)
549{
550 if ((map == VM_MAP_NULL) || (start + size < start) ||
551 (new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) {
552 return KERN_INVALID_ARGUMENT;
553 }
554
555 if (size == 0) {
556 return KERN_SUCCESS;
557 }
558
559 return vm_map_protect(map,
560 vm_map_trunc_page(start,
561 VM_MAP_PAGE_MASK(map)),
562 vm_map_round_page(start + size,
563 VM_MAP_PAGE_MASK(map)),
564 new_prot: new_protection,
565 set_max: set_maximum);
566}
567
568/*
569 * vm_protect -
570 * Sets the protection of the specified range in the
571 * specified map. Addressability of the range limited
572 * to the same size as the kernel.
573 */
574
575kern_return_t
576vm_protect(
577 vm_map_t map,
578 vm_offset_t start,
579 vm_size_t size,
580 boolean_t set_maximum,
581 vm_prot_t new_protection)
582{
583 if ((map == VM_MAP_NULL) || (start + size < start) ||
584 (new_protection & ~VM_VALID_VMPROTECT_FLAGS)
585#if defined(__x86_64__)
586 || ((new_protection & VM_PROT_UEXEC) && !pmap_supported_feature(map->pmap, PMAP_FEAT_UEXEC))
587#endif
588 ) {
589 return KERN_INVALID_ARGUMENT;
590 }
591
592 if (size == 0) {
593 return KERN_SUCCESS;
594 }
595
596 return vm_map_protect(map,
597 vm_map_trunc_page(start,
598 VM_MAP_PAGE_MASK(map)),
599 vm_map_round_page(start + size,
600 VM_MAP_PAGE_MASK(map)),
601 new_prot: new_protection,
602 set_max: set_maximum);
603}
604
605/*
606 * mach_vm_machine_attributes -
607 * Handle machine-specific attributes for a mapping, such
608 * as cachability, migrability, etc.
609 */
610kern_return_t
611mach_vm_machine_attribute(
612 vm_map_t map,
613 mach_vm_address_t addr,
614 mach_vm_size_t size,
615 vm_machine_attribute_t attribute,
616 vm_machine_attribute_val_t* value) /* IN/OUT */
617{
618 if ((map == VM_MAP_NULL) || (addr + size < addr)) {
619 return KERN_INVALID_ARGUMENT;
620 }
621
622 if (size == 0) {
623 return KERN_SUCCESS;
624 }
625
626 return vm_map_machine_attribute(
627 map,
628 vm_map_trunc_page(addr,
629 VM_MAP_PAGE_MASK(map)),
630 vm_map_round_page(addr + size,
631 VM_MAP_PAGE_MASK(map)),
632 attribute,
633 value);
634}
635
636/*
637 * vm_machine_attribute -
638 * Handle machine-specific attributes for a mapping, such
639 * as cachability, migrability, etc. Limited addressability
640 * (same range limits as for the native kernel map).
641 */
642kern_return_t
643vm_machine_attribute(
644 vm_map_t map,
645 vm_address_t addr,
646 vm_size_t size,
647 vm_machine_attribute_t attribute,
648 vm_machine_attribute_val_t* value) /* IN/OUT */
649{
650 if ((map == VM_MAP_NULL) || (addr + size < addr)) {
651 return KERN_INVALID_ARGUMENT;
652 }
653
654 if (size == 0) {
655 return KERN_SUCCESS;
656 }
657
658 return vm_map_machine_attribute(
659 map,
660 vm_map_trunc_page(addr,
661 VM_MAP_PAGE_MASK(map)),
662 vm_map_round_page(addr + size,
663 VM_MAP_PAGE_MASK(map)),
664 attribute,
665 value);
666}
667
668/*
669 * mach_vm_read -
670 * Read/copy a range from one address space and return it to the caller.
671 *
672 * It is assumed that the address for the returned memory is selected by
673 * the IPC implementation as part of receiving the reply to this call.
674 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
675 * that gets returned.
676 *
677 * JMM - because of mach_msg_type_number_t, this call is limited to a
678 * single 4GB region at this time.
679 *
680 */
681kern_return_t
682mach_vm_read(
683 vm_map_t map,
684 mach_vm_address_t addr,
685 mach_vm_size_t size,
686 pointer_t *data,
687 mach_msg_type_number_t *data_size)
688{
689 kern_return_t error;
690 vm_map_copy_t ipc_address;
691
692 if (map == VM_MAP_NULL) {
693 return KERN_INVALID_ARGUMENT;
694 }
695
696 if ((mach_msg_type_number_t) size != size) {
697 return KERN_INVALID_ARGUMENT;
698 }
699
700 error = vm_map_copyin(src_map: map,
701 src_addr: (vm_map_address_t)addr,
702 len: (vm_map_size_t)size,
703 FALSE, /* src_destroy */
704 copy_result: &ipc_address);
705
706 if (KERN_SUCCESS == error) {
707 *data = (pointer_t) ipc_address;
708 *data_size = (mach_msg_type_number_t) size;
709 assert(*data_size == size);
710 }
711 return error;
712}
713
714/*
715 * vm_read -
716 * Read/copy a range from one address space and return it to the caller.
717 * Limited addressability (same range limits as for the native kernel map).
718 *
719 * It is assumed that the address for the returned memory is selected by
720 * the IPC implementation as part of receiving the reply to this call.
721 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
722 * that gets returned.
723 */
724kern_return_t
725vm_read(
726 vm_map_t map,
727 vm_address_t addr,
728 vm_size_t size,
729 pointer_t *data,
730 mach_msg_type_number_t *data_size)
731{
732 kern_return_t error;
733 vm_map_copy_t ipc_address;
734
735 if (map == VM_MAP_NULL) {
736 return KERN_INVALID_ARGUMENT;
737 }
738
739 mach_msg_type_number_t dsize;
740 if (os_convert_overflow(size, &dsize)) {
741 /*
742 * The kernel could handle a 64-bit "size" value, but
743 * it could not return the size of the data in "*data_size"
744 * without overflowing.
745 * Let's reject this "size" as invalid.
746 */
747 return KERN_INVALID_ARGUMENT;
748 }
749
750 error = vm_map_copyin(src_map: map,
751 src_addr: (vm_map_address_t)addr,
752 len: (vm_map_size_t)size,
753 FALSE, /* src_destroy */
754 copy_result: &ipc_address);
755
756 if (KERN_SUCCESS == error) {
757 *data = (pointer_t) ipc_address;
758 *data_size = dsize;
759 assert(*data_size == size);
760 }
761 return error;
762}
763
764/*
765 * mach_vm_read_list -
766 * Read/copy a list of address ranges from specified map.
767 *
768 * MIG does not know how to deal with a returned array of
769 * vm_map_copy_t structures, so we have to do the copyout
770 * manually here.
771 */
772kern_return_t
773mach_vm_read_list(
774 vm_map_t map,
775 mach_vm_read_entry_t data_list,
776 natural_t count)
777{
778 mach_msg_type_number_t i;
779 kern_return_t error;
780 vm_map_copy_t copy;
781
782 if (map == VM_MAP_NULL ||
783 count > VM_MAP_ENTRY_MAX) {
784 return KERN_INVALID_ARGUMENT;
785 }
786
787 error = KERN_SUCCESS;
788 for (i = 0; i < count; i++) {
789 vm_map_address_t map_addr;
790 vm_map_size_t map_size;
791
792 map_addr = (vm_map_address_t)(data_list[i].address);
793 map_size = (vm_map_size_t)(data_list[i].size);
794
795 if (map_size != 0) {
796 error = vm_map_copyin(src_map: map,
797 src_addr: map_addr,
798 len: map_size,
799 FALSE, /* src_destroy */
800 copy_result: &copy);
801 if (KERN_SUCCESS == error) {
802 error = vm_map_copyout(
803 dst_map: current_task()->map,
804 dst_addr: &map_addr,
805 copy);
806 if (KERN_SUCCESS == error) {
807 data_list[i].address = map_addr;
808 continue;
809 }
810 vm_map_copy_discard(copy);
811 }
812 }
813 data_list[i].address = (mach_vm_address_t)0;
814 data_list[i].size = (mach_vm_size_t)0;
815 }
816 return error;
817}
818
819/*
820 * vm_read_list -
821 * Read/copy a list of address ranges from specified map.
822 *
823 * MIG does not know how to deal with a returned array of
824 * vm_map_copy_t structures, so we have to do the copyout
825 * manually here.
826 *
827 * The source and destination ranges are limited to those
828 * that can be described with a vm_address_t (i.e. same
829 * size map as the kernel).
830 *
831 * JMM - If the result of the copyout is an address range
832 * that cannot be described with a vm_address_t (i.e. the
833 * caller had a larger address space but used this call
834 * anyway), it will result in a truncated address being
835 * returned (and a likely confused caller).
836 */
837
838kern_return_t
839vm_read_list(
840 vm_map_t map,
841 vm_read_entry_t data_list,
842 natural_t count)
843{
844 mach_msg_type_number_t i;
845 kern_return_t error;
846 vm_map_copy_t copy;
847
848 if (map == VM_MAP_NULL ||
849 count > VM_MAP_ENTRY_MAX) {
850 return KERN_INVALID_ARGUMENT;
851 }
852
853 error = KERN_SUCCESS;
854 for (i = 0; i < count; i++) {
855 vm_map_address_t map_addr;
856 vm_map_size_t map_size;
857
858 map_addr = (vm_map_address_t)(data_list[i].address);
859 map_size = (vm_map_size_t)(data_list[i].size);
860
861 if (map_size != 0) {
862 error = vm_map_copyin(src_map: map,
863 src_addr: map_addr,
864 len: map_size,
865 FALSE, /* src_destroy */
866 copy_result: &copy);
867 if (KERN_SUCCESS == error) {
868 error = vm_map_copyout(dst_map: current_task()->map,
869 dst_addr: &map_addr,
870 copy);
871 if (KERN_SUCCESS == error) {
872 data_list[i].address =
873 CAST_DOWN(vm_offset_t, map_addr);
874 continue;
875 }
876 vm_map_copy_discard(copy);
877 }
878 }
879 data_list[i].address = (mach_vm_address_t)0;
880 data_list[i].size = (mach_vm_size_t)0;
881 }
882 return error;
883}
884
885/*
886 * mach_vm_read_overwrite -
887 * Overwrite a range of the current map with data from the specified
888 * map/address range.
889 *
890 * In making an assumption that the current thread is local, it is
891 * no longer cluster-safe without a fully supportive local proxy
892 * thread/task (but we don't support cluster's anymore so this is moot).
893 */
894
895kern_return_t
896mach_vm_read_overwrite(
897 vm_map_t map,
898 mach_vm_address_t address,
899 mach_vm_size_t size,
900 mach_vm_address_t data,
901 mach_vm_size_t *data_size)
902{
903 kern_return_t error;
904 vm_map_copy_t copy;
905
906 if (map == VM_MAP_NULL) {
907 return KERN_INVALID_ARGUMENT;
908 }
909
910 error = vm_map_copyin(src_map: map, src_addr: (vm_map_address_t)address,
911 len: (vm_map_size_t)size, FALSE, copy_result: &copy);
912
913 if (KERN_SUCCESS == error) {
914 if (copy) {
915 assertf(copy->size == (vm_map_size_t) size, "Req size: 0x%llx, Copy size: 0x%llx\n", (uint64_t) size, (uint64_t) copy->size);
916 }
917
918 error = vm_map_copy_overwrite(dst_map: current_thread()->map,
919 dst_addr: (vm_map_address_t)data,
920 copy, copy_size: (vm_map_size_t) size, FALSE);
921 if (KERN_SUCCESS == error) {
922 *data_size = size;
923 return error;
924 }
925 vm_map_copy_discard(copy);
926 }
927 return error;
928}
929
930/*
931 * vm_read_overwrite -
932 * Overwrite a range of the current map with data from the specified
933 * map/address range.
934 *
935 * This routine adds the additional limitation that the source and
936 * destination ranges must be describable with vm_address_t values
937 * (i.e. the same size address spaces as the kernel, or at least the
938 * the ranges are in that first portion of the respective address
939 * spaces).
940 */
941
942kern_return_t
943vm_read_overwrite(
944 vm_map_t map,
945 vm_address_t address,
946 vm_size_t size,
947 vm_address_t data,
948 vm_size_t *data_size)
949{
950 kern_return_t error;
951 vm_map_copy_t copy;
952
953 if (map == VM_MAP_NULL) {
954 return KERN_INVALID_ARGUMENT;
955 }
956
957 error = vm_map_copyin(src_map: map, src_addr: (vm_map_address_t)address,
958 len: (vm_map_size_t)size, FALSE, copy_result: &copy);
959
960 if (KERN_SUCCESS == error) {
961 if (copy) {
962 assertf(copy->size == (vm_map_size_t) size, "Req size: 0x%llx, Copy size: 0x%llx\n", (uint64_t) size, (uint64_t) copy->size);
963 }
964
965 error = vm_map_copy_overwrite(dst_map: current_thread()->map,
966 dst_addr: (vm_map_address_t)data,
967 copy, copy_size: (vm_map_size_t) size, FALSE);
968 if (KERN_SUCCESS == error) {
969 *data_size = size;
970 return error;
971 }
972 vm_map_copy_discard(copy);
973 }
974 return error;
975}
976
977
978/*
979 * mach_vm_write -
980 * Overwrite the specified address range with the data provided
981 * (from the current map).
982 */
983kern_return_t
984mach_vm_write(
985 vm_map_t map,
986 mach_vm_address_t address,
987 pointer_t data,
988 mach_msg_type_number_t size)
989{
990 if (map == VM_MAP_NULL) {
991 return KERN_INVALID_ARGUMENT;
992 }
993
994 return vm_map_copy_overwrite(dst_map: map, dst_addr: (vm_map_address_t)address,
995 copy: (vm_map_copy_t) data, copy_size: size, FALSE /* interruptible XXX */);
996}
997
998/*
999 * vm_write -
1000 * Overwrite the specified address range with the data provided
1001 * (from the current map).
1002 *
1003 * The addressability of the range of addresses to overwrite is
1004 * limited bu the use of a vm_address_t (same size as kernel map).
1005 * Either the target map is also small, or the range is in the
1006 * low addresses within it.
1007 */
1008kern_return_t
1009vm_write(
1010 vm_map_t map,
1011 vm_address_t address,
1012 pointer_t data,
1013 mach_msg_type_number_t size)
1014{
1015 if (map == VM_MAP_NULL) {
1016 return KERN_INVALID_ARGUMENT;
1017 }
1018
1019 return vm_map_copy_overwrite(dst_map: map, dst_addr: (vm_map_address_t)address,
1020 copy: (vm_map_copy_t) data, copy_size: size, FALSE /* interruptible XXX */);
1021}
1022
1023/*
1024 * mach_vm_copy -
1025 * Overwrite one range of the specified map with the contents of
1026 * another range within that same map (i.e. both address ranges
1027 * are "over there").
1028 */
1029kern_return_t
1030mach_vm_copy(
1031 vm_map_t map,
1032 mach_vm_address_t source_address,
1033 mach_vm_size_t size,
1034 mach_vm_address_t dest_address)
1035{
1036 vm_map_copy_t copy;
1037 kern_return_t kr;
1038
1039 if (map == VM_MAP_NULL) {
1040 return KERN_INVALID_ARGUMENT;
1041 }
1042
1043 kr = vm_map_copyin(src_map: map, src_addr: (vm_map_address_t)source_address,
1044 len: (vm_map_size_t)size, FALSE, copy_result: &copy);
1045
1046 if (KERN_SUCCESS == kr) {
1047 if (copy) {
1048 assertf(copy->size == (vm_map_size_t) size, "Req size: 0x%llx, Copy size: 0x%llx\n", (uint64_t) size, (uint64_t) copy->size);
1049 }
1050
1051 kr = vm_map_copy_overwrite(dst_map: map,
1052 dst_addr: (vm_map_address_t)dest_address,
1053 copy, copy_size: (vm_map_size_t) size, FALSE /* interruptible XXX */);
1054
1055 if (KERN_SUCCESS != kr) {
1056 vm_map_copy_discard(copy);
1057 }
1058 }
1059 return kr;
1060}
1061
1062kern_return_t
1063vm_copy(
1064 vm_map_t map,
1065 vm_address_t source_address,
1066 vm_size_t size,
1067 vm_address_t dest_address)
1068{
1069 vm_map_copy_t copy;
1070 kern_return_t kr;
1071
1072 if (map == VM_MAP_NULL) {
1073 return KERN_INVALID_ARGUMENT;
1074 }
1075
1076 kr = vm_map_copyin(src_map: map, src_addr: (vm_map_address_t)source_address,
1077 len: (vm_map_size_t)size, FALSE, copy_result: &copy);
1078
1079 if (KERN_SUCCESS == kr) {
1080 if (copy) {
1081 assertf(copy->size == (vm_map_size_t) size, "Req size: 0x%llx, Copy size: 0x%llx\n", (uint64_t) size, (uint64_t) copy->size);
1082 }
1083
1084 kr = vm_map_copy_overwrite(dst_map: map,
1085 dst_addr: (vm_map_address_t)dest_address,
1086 copy, copy_size: (vm_map_size_t) size, FALSE /* interruptible XXX */);
1087
1088 if (KERN_SUCCESS != kr) {
1089 vm_map_copy_discard(copy);
1090 }
1091 }
1092 return kr;
1093}
1094
1095/*
1096 * mach_vm_map -
1097 * Map some range of an object into an address space.
1098 *
1099 * The object can be one of several types of objects:
1100 * NULL - anonymous memory
1101 * a named entry - a range within another address space
1102 * or a range within a memory object
1103 * a whole memory object
1104 *
1105 */
1106kern_return_t
1107mach_vm_map_external(
1108 vm_map_t target_map,
1109 mach_vm_offset_t *address,
1110 mach_vm_size_t initial_size,
1111 mach_vm_offset_t mask,
1112 int flags,
1113 ipc_port_t port,
1114 vm_object_offset_t offset,
1115 boolean_t copy,
1116 vm_prot_t cur_protection,
1117 vm_prot_t max_protection,
1118 vm_inherit_t inheritance)
1119{
1120 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
1121
1122 /* filter out any kernel-only flags */
1123 if (flags & ~VM_FLAGS_USER_MAP) {
1124 return KERN_INVALID_ARGUMENT;
1125 }
1126
1127 vm_map_kernel_flags_set_vmflags(vmk_flags: &vmk_flags, vm_flags_and_tag: flags);
1128 /* range_id is set by mach_vm_map_kernel */
1129 return mach_vm_map_kernel(target_map, address, initial_size, mask,
1130 vmk_flags, port, offset, copy,
1131 cur_protection, max_protection,
1132 inheritance);
1133}
1134
1135kern_return_t
1136mach_vm_map_kernel(
1137 vm_map_t target_map,
1138 mach_vm_offset_t *address,
1139 mach_vm_size_t initial_size,
1140 mach_vm_offset_t mask,
1141 vm_map_kernel_flags_t vmk_flags,
1142 ipc_port_t port,
1143 vm_object_offset_t offset,
1144 boolean_t copy,
1145 vm_prot_t cur_protection,
1146 vm_prot_t max_protection,
1147 vm_inherit_t inheritance)
1148{
1149 kern_return_t kr;
1150 vm_map_offset_t vmmaddr;
1151
1152 vmmaddr = (vm_map_offset_t) *address;
1153
1154 /* filter out any kernel-only flags */
1155 if (!vm_map_kernel_flags_check_vmflags(vmk_flags, VM_FLAGS_USER_MAP)) {
1156 return KERN_INVALID_ARGUMENT;
1157 }
1158
1159 /* range_id is set by vm_map_enter_mem_object */
1160 kr = vm_map_enter_mem_object(map: target_map,
1161 address: &vmmaddr,
1162 size: initial_size,
1163 mask,
1164 vmk_flags,
1165 port,
1166 offset,
1167 needs_copy: copy,
1168 cur_protection,
1169 max_protection,
1170 inheritance);
1171
1172#if KASAN
1173 if (kr == KERN_SUCCESS && target_map->pmap == kernel_pmap) {
1174 kasan_notify_address(vmmaddr, initial_size);
1175 }
1176#endif
1177
1178 *address = vmmaddr;
1179 return kr;
1180}
1181
1182
1183/* legacy interface */
1184__attribute__((always_inline))
1185kern_return_t
1186vm_map_64_external(
1187 vm_map_t target_map,
1188 vm_offset_t *address,
1189 vm_size_t size,
1190 vm_offset_t mask,
1191 int flags,
1192 ipc_port_t port,
1193 vm_object_offset_t offset,
1194 boolean_t copy,
1195 vm_prot_t cur_protection,
1196 vm_prot_t max_protection,
1197 vm_inherit_t inheritance)
1198{
1199 static_assert(sizeof(vm_offset_t) == sizeof(mach_vm_offset_t));
1200
1201 return mach_vm_map_external(target_map, address: (mach_vm_offset_t *)address,
1202 initial_size: size, mask, flags, port, offset, copy,
1203 cur_protection, max_protection, inheritance);
1204}
1205
1206/* temporary, until world build */
1207__attribute__((always_inline))
1208kern_return_t
1209vm_map_external(
1210 vm_map_t target_map,
1211 vm_offset_t *address,
1212 vm_size_t size,
1213 vm_offset_t mask,
1214 int flags,
1215 ipc_port_t port,
1216 vm_offset_t offset,
1217 boolean_t copy,
1218 vm_prot_t cur_protection,
1219 vm_prot_t max_protection,
1220 vm_inherit_t inheritance)
1221{
1222 static_assert(sizeof(vm_offset_t) == sizeof(mach_vm_offset_t));
1223
1224 return mach_vm_map_external(target_map, address: (mach_vm_offset_t *)address,
1225 initial_size: size, mask, flags, port, offset, copy,
1226 cur_protection, max_protection, inheritance);
1227}
1228
1229/*
1230 * mach_vm_remap_new -
1231 * Behaves like mach_vm_remap, except that VM_FLAGS_RETURN_DATA_ADDR is always set
1232 * and {cur,max}_protection are in/out.
1233 */
1234kern_return_t
1235mach_vm_remap_new_external(
1236 vm_map_t target_map,
1237 mach_vm_offset_t *address,
1238 mach_vm_size_t size,
1239 mach_vm_offset_t mask,
1240 int flags,
1241 mach_port_t src_tport,
1242 mach_vm_offset_t memory_address,
1243 boolean_t copy,
1244 vm_prot_t *cur_protection, /* IN/OUT */
1245 vm_prot_t *max_protection, /* IN/OUT */
1246 vm_inherit_t inheritance)
1247{
1248 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
1249 vm_map_t src_map;
1250 kern_return_t kr;
1251
1252 /* filter out any kernel-only flags */
1253 if (flags & ~VM_FLAGS_USER_REMAP) {
1254 return KERN_INVALID_ARGUMENT;
1255 }
1256
1257 vm_map_kernel_flags_set_vmflags(vmk_flags: &vmk_flags,
1258 vm_flags_and_tag: flags | VM_FLAGS_RETURN_DATA_ADDR);
1259
1260 if (target_map == VM_MAP_NULL) {
1261 return KERN_INVALID_ARGUMENT;
1262 }
1263
1264 if ((*cur_protection & ~VM_PROT_ALL) ||
1265 (*max_protection & ~VM_PROT_ALL) ||
1266 (*cur_protection & *max_protection) != *cur_protection) {
1267 return KERN_INVALID_ARGUMENT;
1268 }
1269 if ((*max_protection & (VM_PROT_WRITE | VM_PROT_EXECUTE)) ==
1270 (VM_PROT_WRITE | VM_PROT_EXECUTE)) {
1271 /*
1272 * XXX FBDP TODO
1273 * enforce target's "wx" policies
1274 */
1275 return KERN_PROTECTION_FAILURE;
1276 }
1277
1278 if (copy || *max_protection == VM_PROT_READ || *max_protection == VM_PROT_NONE) {
1279 src_map = convert_port_to_map_read(port: src_tport);
1280 } else {
1281 src_map = convert_port_to_map(port: src_tport);
1282 }
1283
1284 if (src_map == VM_MAP_NULL) {
1285 return KERN_INVALID_ARGUMENT;
1286 }
1287
1288 static_assert(sizeof(mach_vm_offset_t) == sizeof(vm_map_address_t));
1289
1290 /* range_id is set by vm_map_remap */
1291 kr = vm_map_remap(target_map,
1292 address,
1293 size,
1294 mask,
1295 vmk_flags,
1296 src_map,
1297 memory_address,
1298 copy,
1299 cur_protection, /* IN/OUT */
1300 max_protection, /* IN/OUT */
1301 inheritance);
1302
1303 vm_map_deallocate(map: src_map);
1304
1305 if (kr == KERN_SUCCESS) {
1306 ipc_port_release_send(port: src_tport); /* consume on success */
1307 }
1308 return kr;
1309}
1310
1311/*
1312 * mach_vm_remap -
1313 * Remap a range of memory from one task into another,
1314 * to another address range within the same task, or
1315 * over top of itself (with altered permissions and/or
1316 * as an in-place copy of itself).
1317 */
1318kern_return_t
1319mach_vm_remap_external(
1320 vm_map_t target_map,
1321 mach_vm_offset_t *address,
1322 mach_vm_size_t size,
1323 mach_vm_offset_t mask,
1324 int flags,
1325 vm_map_t src_map,
1326 mach_vm_offset_t memory_address,
1327 boolean_t copy,
1328 vm_prot_t *cur_protection, /* OUT */
1329 vm_prot_t *max_protection, /* OUT */
1330 vm_inherit_t inheritance)
1331{
1332 vm_tag_t tag;
1333 VM_GET_FLAGS_ALIAS(flags, tag);
1334
1335 return mach_vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map, memory_address,
1336 copy, cur_protection, max_protection, inheritance);
1337}
1338
1339static kern_return_t
1340mach_vm_remap_kernel_helper(
1341 vm_map_t target_map,
1342 mach_vm_offset_t *address,
1343 mach_vm_size_t size,
1344 mach_vm_offset_t mask,
1345 int flags,
1346 vm_tag_t tag,
1347 vm_map_t src_map,
1348 mach_vm_offset_t memory_address,
1349 boolean_t copy,
1350 vm_prot_t *cur_protection, /* IN/OUT */
1351 vm_prot_t *max_protection, /* IN/OUT */
1352 vm_inherit_t inheritance)
1353{
1354 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
1355 kern_return_t kr;
1356
1357 if (VM_MAP_NULL == target_map || VM_MAP_NULL == src_map) {
1358 return KERN_INVALID_ARGUMENT;
1359 }
1360
1361 /* filter out any kernel-only flags */
1362 if (flags & ~VM_FLAGS_USER_REMAP) {
1363 return KERN_INVALID_ARGUMENT;
1364 }
1365
1366 vm_map_kernel_flags_set_vmflags(vmk_flags: &vmk_flags, vm_flags: flags, vm_tag: tag);
1367
1368 static_assert(sizeof(mach_vm_offset_t) == sizeof(vm_map_address_t));
1369
1370 /* range_id is set by vm_map_remap */
1371 kr = vm_map_remap(target_map,
1372 address,
1373 size,
1374 mask,
1375 vmk_flags,
1376 src_map,
1377 memory_address,
1378 copy,
1379 cur_protection, /* IN/OUT */
1380 max_protection, /* IN/OUT */
1381 inheritance);
1382
1383#if KASAN
1384 if (kr == KERN_SUCCESS && target_map->pmap == kernel_pmap) {
1385 kasan_notify_address(*address, size);
1386 }
1387#endif
1388 return kr;
1389}
1390
1391kern_return_t
1392mach_vm_remap_kernel(
1393 vm_map_t target_map,
1394 mach_vm_offset_t *address,
1395 mach_vm_size_t size,
1396 mach_vm_offset_t mask,
1397 int flags,
1398 vm_tag_t tag,
1399 vm_map_t src_map,
1400 mach_vm_offset_t memory_address,
1401 boolean_t copy,
1402 vm_prot_t *cur_protection, /* OUT */
1403 vm_prot_t *max_protection, /* OUT */
1404 vm_inherit_t inheritance)
1405{
1406 *cur_protection = VM_PROT_NONE;
1407 *max_protection = VM_PROT_NONE;
1408
1409 return mach_vm_remap_kernel_helper(target_map,
1410 address,
1411 size,
1412 mask,
1413 flags,
1414 tag,
1415 src_map,
1416 memory_address,
1417 copy,
1418 cur_protection,
1419 max_protection,
1420 inheritance);
1421}
1422
1423kern_return_t
1424mach_vm_remap_new_kernel(
1425 vm_map_t target_map,
1426 mach_vm_offset_t *address,
1427 mach_vm_size_t size,
1428 mach_vm_offset_t mask,
1429 int flags,
1430 vm_tag_t tag,
1431 vm_map_t src_map,
1432 mach_vm_offset_t memory_address,
1433 boolean_t copy,
1434 vm_prot_t *cur_protection, /* IN/OUT */
1435 vm_prot_t *max_protection, /* IN/OUT */
1436 vm_inherit_t inheritance)
1437{
1438 if ((*cur_protection & ~VM_PROT_ALL) ||
1439 (*max_protection & ~VM_PROT_ALL) ||
1440 (*cur_protection & *max_protection) != *cur_protection) {
1441 return KERN_INVALID_ARGUMENT;
1442 }
1443
1444 flags |= VM_FLAGS_RETURN_DATA_ADDR;
1445
1446 return mach_vm_remap_kernel_helper(target_map,
1447 address,
1448 size,
1449 mask,
1450 flags,
1451 tag,
1452 src_map,
1453 memory_address,
1454 copy,
1455 cur_protection,
1456 max_protection,
1457 inheritance);
1458}
1459
1460/*
1461 * vm_remap_new -
1462 * Behaves like vm_remap, except that VM_FLAGS_RETURN_DATA_ADDR is always set
1463 * and {cur,max}_protection are in/out.
1464 */
1465kern_return_t
1466vm_remap_new_external(
1467 vm_map_t target_map,
1468 vm_offset_t *address,
1469 vm_size_t size,
1470 vm_offset_t mask,
1471 int flags,
1472 mach_port_t src_tport,
1473 vm_offset_t memory_address,
1474 boolean_t copy,
1475 vm_prot_t *cur_protection, /* IN/OUT */
1476 vm_prot_t *max_protection, /* IN/OUT */
1477 vm_inherit_t inheritance)
1478{
1479 static_assert(sizeof(vm_map_offset_t) == sizeof(vm_offset_t));
1480
1481 return mach_vm_remap_new_external(target_map,
1482 address: (vm_map_offset_t *)address,
1483 size,
1484 mask,
1485 flags,
1486 src_tport,
1487 memory_address,
1488 copy,
1489 cur_protection, /* IN/OUT */
1490 max_protection, /* IN/OUT */
1491 inheritance);
1492}
1493
1494/*
1495 * vm_remap -
1496 * Remap a range of memory from one task into another,
1497 * to another address range within the same task, or
1498 * over top of itself (with altered permissions and/or
1499 * as an in-place copy of itself).
1500 *
1501 * The addressability of the source and target address
1502 * range is limited by the size of vm_address_t (in the
1503 * kernel context).
1504 */
1505kern_return_t
1506vm_remap_external(
1507 vm_map_t target_map,
1508 vm_offset_t *address,
1509 vm_size_t size,
1510 vm_offset_t mask,
1511 int flags,
1512 vm_map_t src_map,
1513 vm_offset_t memory_address,
1514 boolean_t copy,
1515 vm_prot_t *cur_protection, /* OUT */
1516 vm_prot_t *max_protection, /* OUT */
1517 vm_inherit_t inheritance)
1518{
1519 static_assert(sizeof(vm_offset_t) == sizeof(mach_vm_offset_t));
1520
1521 return mach_vm_remap_external(target_map, address: (mach_vm_offset_t *)address,
1522 size, mask, flags, src_map, memory_address, copy,
1523 cur_protection, max_protection, inheritance);
1524}
1525
1526/*
1527 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1528 * when mach_vm_wire and vm_wire are changed to use ledgers.
1529 */
1530#include <mach/mach_host_server.h>
1531/*
1532 * mach_vm_wire
1533 * Specify that the range of the virtual address space
1534 * of the target task must not cause page faults for
1535 * the indicated accesses.
1536 *
1537 * [ To unwire the pages, specify VM_PROT_NONE. ]
1538 */
1539kern_return_t
1540mach_vm_wire_external(
1541 host_priv_t host_priv,
1542 vm_map_t map,
1543 mach_vm_offset_t start,
1544 mach_vm_size_t size,
1545 vm_prot_t access)
1546{
1547 if (host_priv == HOST_PRIV_NULL) {
1548 return KERN_INVALID_HOST;
1549 }
1550
1551 return mach_vm_wire_kernel(map, start, size, access, VM_KERN_MEMORY_MLOCK);
1552}
1553
1554kern_return_t
1555mach_vm_wire_kernel(
1556 vm_map_t map,
1557 mach_vm_offset_t start,
1558 mach_vm_size_t size,
1559 vm_prot_t access,
1560 vm_tag_t tag)
1561{
1562 kern_return_t rc;
1563
1564 if (map == VM_MAP_NULL) {
1565 return KERN_INVALID_TASK;
1566 }
1567
1568 if (access & ~VM_PROT_ALL || (start + size < start)) {
1569 return KERN_INVALID_ARGUMENT;
1570 }
1571
1572 if (access != VM_PROT_NONE) {
1573 rc = vm_map_wire_kernel(map,
1574 vm_map_trunc_page(start,
1575 VM_MAP_PAGE_MASK(map)),
1576 vm_map_round_page(start + size,
1577 VM_MAP_PAGE_MASK(map)),
1578 access_type: access, tag,
1579 TRUE);
1580 } else {
1581 rc = vm_map_unwire(map,
1582 vm_map_trunc_page(start,
1583 VM_MAP_PAGE_MASK(map)),
1584 vm_map_round_page(start + size,
1585 VM_MAP_PAGE_MASK(map)),
1586 TRUE);
1587 }
1588 return rc;
1589}
1590
1591/*
1592 * vm_wire -
1593 * Specify that the range of the virtual address space
1594 * of the target task must not cause page faults for
1595 * the indicated accesses.
1596 *
1597 * [ To unwire the pages, specify VM_PROT_NONE. ]
1598 */
1599kern_return_t
1600vm_wire(
1601 host_priv_t host_priv,
1602 vm_map_t map,
1603 vm_offset_t start,
1604 vm_size_t size,
1605 vm_prot_t access)
1606{
1607 kern_return_t rc;
1608
1609 if (host_priv == HOST_PRIV_NULL) {
1610 return KERN_INVALID_HOST;
1611 }
1612
1613 if (map == VM_MAP_NULL) {
1614 return KERN_INVALID_TASK;
1615 }
1616
1617 if ((access & ~VM_PROT_ALL) || (start + size < start)) {
1618 return KERN_INVALID_ARGUMENT;
1619 }
1620
1621 if (size == 0) {
1622 rc = KERN_SUCCESS;
1623 } else if (access != VM_PROT_NONE) {
1624 rc = vm_map_wire_kernel(map,
1625 vm_map_trunc_page(start,
1626 VM_MAP_PAGE_MASK(map)),
1627 vm_map_round_page(start + size,
1628 VM_MAP_PAGE_MASK(map)),
1629 access_type: access, VM_KERN_MEMORY_OSFMK,
1630 TRUE);
1631 } else {
1632 rc = vm_map_unwire(map,
1633 vm_map_trunc_page(start,
1634 VM_MAP_PAGE_MASK(map)),
1635 vm_map_round_page(start + size,
1636 VM_MAP_PAGE_MASK(map)),
1637 TRUE);
1638 }
1639 return rc;
1640}
1641
1642/*
1643 * vm_msync
1644 *
1645 * Synchronises the memory range specified with its backing store
1646 * image by either flushing or cleaning the contents to the appropriate
1647 * memory manager.
1648 *
1649 * interpretation of sync_flags
1650 * VM_SYNC_INVALIDATE - discard pages, only return precious
1651 * pages to manager.
1652 *
1653 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1654 * - discard pages, write dirty or precious
1655 * pages back to memory manager.
1656 *
1657 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1658 * - write dirty or precious pages back to
1659 * the memory manager.
1660 *
1661 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1662 * is a hole in the region, and we would
1663 * have returned KERN_SUCCESS, return
1664 * KERN_INVALID_ADDRESS instead.
1665 *
1666 * RETURNS
1667 * KERN_INVALID_TASK Bad task parameter
1668 * KERN_INVALID_ARGUMENT both sync and async were specified.
1669 * KERN_SUCCESS The usual.
1670 * KERN_INVALID_ADDRESS There was a hole in the region.
1671 */
1672
1673kern_return_t
1674mach_vm_msync(
1675 vm_map_t map,
1676 mach_vm_address_t address,
1677 mach_vm_size_t size,
1678 vm_sync_t sync_flags)
1679{
1680 if (map == VM_MAP_NULL) {
1681 return KERN_INVALID_TASK;
1682 }
1683
1684 return vm_map_msync(map, address: (vm_map_address_t)address,
1685 size: (vm_map_size_t)size, sync_flags);
1686}
1687
1688/*
1689 * vm_msync
1690 *
1691 * Synchronises the memory range specified with its backing store
1692 * image by either flushing or cleaning the contents to the appropriate
1693 * memory manager.
1694 *
1695 * interpretation of sync_flags
1696 * VM_SYNC_INVALIDATE - discard pages, only return precious
1697 * pages to manager.
1698 *
1699 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1700 * - discard pages, write dirty or precious
1701 * pages back to memory manager.
1702 *
1703 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1704 * - write dirty or precious pages back to
1705 * the memory manager.
1706 *
1707 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1708 * is a hole in the region, and we would
1709 * have returned KERN_SUCCESS, return
1710 * KERN_INVALID_ADDRESS instead.
1711 *
1712 * The addressability of the range is limited to that which can
1713 * be described by a vm_address_t.
1714 *
1715 * RETURNS
1716 * KERN_INVALID_TASK Bad task parameter
1717 * KERN_INVALID_ARGUMENT both sync and async were specified.
1718 * KERN_SUCCESS The usual.
1719 * KERN_INVALID_ADDRESS There was a hole in the region.
1720 */
1721
1722kern_return_t
1723vm_msync(
1724 vm_map_t map,
1725 vm_address_t address,
1726 vm_size_t size,
1727 vm_sync_t sync_flags)
1728{
1729 if (map == VM_MAP_NULL) {
1730 return KERN_INVALID_TASK;
1731 }
1732
1733 return vm_map_msync(map, address: (vm_map_address_t)address,
1734 size: (vm_map_size_t)size, sync_flags);
1735}
1736
1737
1738int
1739vm_toggle_entry_reuse(int toggle, int *old_value)
1740{
1741 vm_map_t map = current_map();
1742
1743 assert(!map->is_nested_map);
1744 if (toggle == VM_TOGGLE_GETVALUE && old_value != NULL) {
1745 *old_value = map->disable_vmentry_reuse;
1746 } else if (toggle == VM_TOGGLE_SET) {
1747 vm_map_entry_t map_to_entry;
1748
1749 vm_map_lock(map);
1750 vm_map_disable_hole_optimization(map);
1751 map->disable_vmentry_reuse = TRUE;
1752 __IGNORE_WCASTALIGN(map_to_entry = vm_map_to_entry(map));
1753 if (map->first_free == map_to_entry) {
1754 map->highest_entry_end = vm_map_min(map);
1755 } else {
1756 map->highest_entry_end = map->first_free->vme_end;
1757 }
1758 vm_map_unlock(map);
1759 } else if (toggle == VM_TOGGLE_CLEAR) {
1760 vm_map_lock(map);
1761 map->disable_vmentry_reuse = FALSE;
1762 vm_map_unlock(map);
1763 } else {
1764 return KERN_INVALID_ARGUMENT;
1765 }
1766
1767 return KERN_SUCCESS;
1768}
1769
1770/*
1771 * mach_vm_behavior_set
1772 *
1773 * Sets the paging behavior attribute for the specified range
1774 * in the specified map.
1775 *
1776 * This routine will fail with KERN_INVALID_ADDRESS if any address
1777 * in [start,start+size) is not a valid allocated memory region.
1778 */
1779kern_return_t
1780mach_vm_behavior_set(
1781 vm_map_t map,
1782 mach_vm_offset_t start,
1783 mach_vm_size_t size,
1784 vm_behavior_t new_behavior)
1785{
1786 vm_map_offset_t align_mask;
1787
1788 if ((map == VM_MAP_NULL) || (start + size < start)) {
1789 return KERN_INVALID_ARGUMENT;
1790 }
1791
1792 if (size == 0) {
1793 return KERN_SUCCESS;
1794 }
1795
1796 switch (new_behavior) {
1797 case VM_BEHAVIOR_REUSABLE:
1798 case VM_BEHAVIOR_REUSE:
1799 case VM_BEHAVIOR_CAN_REUSE:
1800 case VM_BEHAVIOR_ZERO:
1801 /*
1802 * Align to the hardware page size, to allow
1803 * malloc() to maximize the amount of re-usability,
1804 * even on systems with larger software page size.
1805 */
1806 align_mask = PAGE_MASK;
1807 break;
1808 default:
1809 align_mask = VM_MAP_PAGE_MASK(map);
1810 break;
1811 }
1812
1813 return vm_map_behavior_set(map,
1814 vm_map_trunc_page(start, align_mask),
1815 vm_map_round_page(start + size, align_mask),
1816 new_behavior);
1817}
1818
1819/*
1820 * vm_behavior_set
1821 *
1822 * Sets the paging behavior attribute for the specified range
1823 * in the specified map.
1824 *
1825 * This routine will fail with KERN_INVALID_ADDRESS if any address
1826 * in [start,start+size) is not a valid allocated memory region.
1827 *
1828 * This routine is potentially limited in addressibility by the
1829 * use of vm_offset_t (if the map provided is larger than the
1830 * kernel's).
1831 */
1832kern_return_t
1833vm_behavior_set(
1834 vm_map_t map,
1835 vm_offset_t start,
1836 vm_size_t size,
1837 vm_behavior_t new_behavior)
1838{
1839 if (start + size < start) {
1840 return KERN_INVALID_ARGUMENT;
1841 }
1842
1843 return mach_vm_behavior_set(map,
1844 start: (mach_vm_offset_t) start,
1845 size: (mach_vm_size_t) size,
1846 new_behavior);
1847}
1848
1849/*
1850 * mach_vm_region:
1851 *
1852 * User call to obtain information about a region in
1853 * a task's address map. Currently, only one flavor is
1854 * supported.
1855 *
1856 * XXX The reserved and behavior fields cannot be filled
1857 * in until the vm merge from the IK is completed, and
1858 * vm_reserve is implemented.
1859 *
1860 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1861 */
1862
1863kern_return_t
1864mach_vm_region(
1865 vm_map_t map,
1866 mach_vm_offset_t *address, /* IN/OUT */
1867 mach_vm_size_t *size, /* OUT */
1868 vm_region_flavor_t flavor, /* IN */
1869 vm_region_info_t info, /* OUT */
1870 mach_msg_type_number_t *count, /* IN/OUT */
1871 mach_port_t *object_name) /* OUT */
1872{
1873 vm_map_offset_t map_addr;
1874 vm_map_size_t map_size;
1875 kern_return_t kr;
1876
1877 if (VM_MAP_NULL == map) {
1878 return KERN_INVALID_ARGUMENT;
1879 }
1880
1881 map_addr = (vm_map_offset_t)*address;
1882 map_size = (vm_map_size_t)*size;
1883
1884 /* legacy conversion */
1885 if (VM_REGION_BASIC_INFO == flavor) {
1886 flavor = VM_REGION_BASIC_INFO_64;
1887 }
1888
1889 kr = vm_map_region(map,
1890 address: &map_addr, size: &map_size,
1891 flavor, info, count,
1892 object_name);
1893
1894 *address = map_addr;
1895 *size = map_size;
1896 return kr;
1897}
1898
1899/*
1900 * vm_region_64 and vm_region:
1901 *
1902 * User call to obtain information about a region in
1903 * a task's address map. Currently, only one flavor is
1904 * supported.
1905 *
1906 * XXX The reserved and behavior fields cannot be filled
1907 * in until the vm merge from the IK is completed, and
1908 * vm_reserve is implemented.
1909 *
1910 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1911 */
1912
1913kern_return_t
1914vm_region_64(
1915 vm_map_t map,
1916 vm_offset_t *address, /* IN/OUT */
1917 vm_size_t *size, /* OUT */
1918 vm_region_flavor_t flavor, /* IN */
1919 vm_region_info_t info, /* OUT */
1920 mach_msg_type_number_t *count, /* IN/OUT */
1921 mach_port_t *object_name) /* OUT */
1922{
1923 vm_map_offset_t map_addr;
1924 vm_map_size_t map_size;
1925 kern_return_t kr;
1926
1927 if (VM_MAP_NULL == map) {
1928 return KERN_INVALID_ARGUMENT;
1929 }
1930
1931 map_addr = (vm_map_offset_t)*address;
1932 map_size = (vm_map_size_t)*size;
1933
1934 /* legacy conversion */
1935 if (VM_REGION_BASIC_INFO == flavor) {
1936 flavor = VM_REGION_BASIC_INFO_64;
1937 }
1938
1939 kr = vm_map_region(map,
1940 address: &map_addr, size: &map_size,
1941 flavor, info, count,
1942 object_name);
1943
1944 *address = CAST_DOWN(vm_offset_t, map_addr);
1945 *size = CAST_DOWN(vm_size_t, map_size);
1946
1947 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
1948 return KERN_INVALID_ADDRESS;
1949 }
1950 return kr;
1951}
1952
1953kern_return_t
1954vm_region(
1955 vm_map_t map,
1956 vm_address_t *address, /* IN/OUT */
1957 vm_size_t *size, /* OUT */
1958 vm_region_flavor_t flavor, /* IN */
1959 vm_region_info_t info, /* OUT */
1960 mach_msg_type_number_t *count, /* IN/OUT */
1961 mach_port_t *object_name) /* OUT */
1962{
1963 vm_map_address_t map_addr;
1964 vm_map_size_t map_size;
1965 kern_return_t kr;
1966
1967 if (VM_MAP_NULL == map) {
1968 return KERN_INVALID_ARGUMENT;
1969 }
1970
1971 map_addr = (vm_map_address_t)*address;
1972 map_size = (vm_map_size_t)*size;
1973
1974 kr = vm_map_region(map,
1975 address: &map_addr, size: &map_size,
1976 flavor, info, count,
1977 object_name);
1978
1979 *address = CAST_DOWN(vm_address_t, map_addr);
1980 *size = CAST_DOWN(vm_size_t, map_size);
1981
1982 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
1983 return KERN_INVALID_ADDRESS;
1984 }
1985 return kr;
1986}
1987
1988/*
1989 * vm_region_recurse: A form of vm_region which follows the
1990 * submaps in a target map
1991 *
1992 */
1993kern_return_t
1994mach_vm_region_recurse(
1995 vm_map_t map,
1996 mach_vm_address_t *address,
1997 mach_vm_size_t *size,
1998 uint32_t *depth,
1999 vm_region_recurse_info_t info,
2000 mach_msg_type_number_t *infoCnt)
2001{
2002 vm_map_address_t map_addr;
2003 vm_map_size_t map_size;
2004 kern_return_t kr;
2005
2006 if (VM_MAP_NULL == map) {
2007 return KERN_INVALID_ARGUMENT;
2008 }
2009
2010 map_addr = (vm_map_address_t)*address;
2011 map_size = (vm_map_size_t)*size;
2012
2013 kr = vm_map_region_recurse_64(
2014 map,
2015 address: &map_addr,
2016 size: &map_size,
2017 nesting_depth: depth,
2018 info: (vm_region_submap_info_64_t)info,
2019 count: infoCnt);
2020
2021 *address = map_addr;
2022 *size = map_size;
2023 return kr;
2024}
2025
2026/*
2027 * vm_region_recurse: A form of vm_region which follows the
2028 * submaps in a target map
2029 *
2030 */
2031kern_return_t
2032vm_region_recurse_64(
2033 vm_map_t map,
2034 vm_address_t *address,
2035 vm_size_t *size,
2036 uint32_t *depth,
2037 vm_region_recurse_info_64_t info,
2038 mach_msg_type_number_t *infoCnt)
2039{
2040 vm_map_address_t map_addr;
2041 vm_map_size_t map_size;
2042 kern_return_t kr;
2043
2044 if (VM_MAP_NULL == map) {
2045 return KERN_INVALID_ARGUMENT;
2046 }
2047
2048 map_addr = (vm_map_address_t)*address;
2049 map_size = (vm_map_size_t)*size;
2050
2051 kr = vm_map_region_recurse_64(
2052 map,
2053 address: &map_addr,
2054 size: &map_size,
2055 nesting_depth: depth,
2056 info: (vm_region_submap_info_64_t)info,
2057 count: infoCnt);
2058
2059 *address = CAST_DOWN(vm_address_t, map_addr);
2060 *size = CAST_DOWN(vm_size_t, map_size);
2061
2062 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
2063 return KERN_INVALID_ADDRESS;
2064 }
2065 return kr;
2066}
2067
2068kern_return_t
2069vm_region_recurse(
2070 vm_map_t map,
2071 vm_offset_t *address, /* IN/OUT */
2072 vm_size_t *size, /* OUT */
2073 natural_t *depth, /* IN/OUT */
2074 vm_region_recurse_info_t info32, /* IN/OUT */
2075 mach_msg_type_number_t *infoCnt) /* IN/OUT */
2076{
2077 vm_region_submap_info_data_64_t info64;
2078 vm_region_submap_info_t info;
2079 vm_map_address_t map_addr;
2080 vm_map_size_t map_size;
2081 kern_return_t kr;
2082
2083 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT) {
2084 return KERN_INVALID_ARGUMENT;
2085 }
2086
2087
2088 map_addr = (vm_map_address_t)*address;
2089 map_size = (vm_map_size_t)*size;
2090 info = (vm_region_submap_info_t)info32;
2091 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
2092
2093 kr = vm_map_region_recurse_64(map, address: &map_addr, size: &map_size,
2094 nesting_depth: depth, info: &info64, count: infoCnt);
2095
2096 info->protection = info64.protection;
2097 info->max_protection = info64.max_protection;
2098 info->inheritance = info64.inheritance;
2099 info->offset = (uint32_t)info64.offset; /* trouble-maker */
2100 info->user_tag = info64.user_tag;
2101 info->pages_resident = info64.pages_resident;
2102 info->pages_shared_now_private = info64.pages_shared_now_private;
2103 info->pages_swapped_out = info64.pages_swapped_out;
2104 info->pages_dirtied = info64.pages_dirtied;
2105 info->ref_count = info64.ref_count;
2106 info->shadow_depth = info64.shadow_depth;
2107 info->external_pager = info64.external_pager;
2108 info->share_mode = info64.share_mode;
2109 info->is_submap = info64.is_submap;
2110 info->behavior = info64.behavior;
2111 info->object_id = info64.object_id;
2112 info->user_wired_count = info64.user_wired_count;
2113
2114 *address = CAST_DOWN(vm_address_t, map_addr);
2115 *size = CAST_DOWN(vm_size_t, map_size);
2116 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
2117
2118 if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
2119 return KERN_INVALID_ADDRESS;
2120 }
2121 return kr;
2122}
2123
2124kern_return_t
2125mach_vm_purgable_control(
2126 vm_map_t map,
2127 mach_vm_offset_t address,
2128 vm_purgable_t control,
2129 int *state)
2130{
2131 if (VM_MAP_NULL == map) {
2132 return KERN_INVALID_ARGUMENT;
2133 }
2134
2135 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
2136 /* not allowed from user-space */
2137 return KERN_INVALID_ARGUMENT;
2138 }
2139
2140 return vm_map_purgable_control(map,
2141 vm_map_trunc_page(address, VM_MAP_PAGE_MASK(map)),
2142 control,
2143 state);
2144}
2145
2146kern_return_t
2147mach_vm_purgable_control_external(
2148 mach_port_t target_tport,
2149 mach_vm_offset_t address,
2150 vm_purgable_t control,
2151 int *state)
2152{
2153 vm_map_t map;
2154 kern_return_t kr;
2155
2156 if (control == VM_PURGABLE_GET_STATE) {
2157 map = convert_port_to_map_read(port: target_tport);
2158 } else {
2159 map = convert_port_to_map(port: target_tport);
2160 }
2161
2162 kr = mach_vm_purgable_control(map, address, control, state);
2163 vm_map_deallocate(map);
2164
2165 return kr;
2166}
2167
2168kern_return_t
2169vm_purgable_control(
2170 vm_map_t map,
2171 vm_offset_t address,
2172 vm_purgable_t control,
2173 int *state)
2174{
2175 if (VM_MAP_NULL == map) {
2176 return KERN_INVALID_ARGUMENT;
2177 }
2178
2179 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
2180 /* not allowed from user-space */
2181 return KERN_INVALID_ARGUMENT;
2182 }
2183
2184 return vm_map_purgable_control(map,
2185 vm_map_trunc_page(address, VM_MAP_PAGE_MASK(map)),
2186 control,
2187 state);
2188}
2189
2190kern_return_t
2191vm_purgable_control_external(
2192 mach_port_t target_tport,
2193 vm_offset_t address,
2194 vm_purgable_t control,
2195 int *state)
2196{
2197 vm_map_t map;
2198 kern_return_t kr;
2199
2200 if (control == VM_PURGABLE_GET_STATE) {
2201 map = convert_port_to_map_read(port: target_tport);
2202 } else {
2203 map = convert_port_to_map(port: target_tport);
2204 }
2205
2206 kr = vm_purgable_control(map, address, control, state);
2207 vm_map_deallocate(map);
2208
2209 return kr;
2210}
2211
2212
2213/*
2214 * Ordinarily, the right to allocate CPM is restricted
2215 * to privileged applications (those that can gain access
2216 * to the host priv port). Set this variable to zero if
2217 * you want to let any application allocate CPM.
2218 */
2219unsigned int vm_allocate_cpm_privileged = 0;
2220
2221/*
2222 * Allocate memory in the specified map, with the caveat that
2223 * the memory is physically contiguous. This call may fail
2224 * if the system can't find sufficient contiguous memory.
2225 * This call may cause or lead to heart-stopping amounts of
2226 * paging activity.
2227 *
2228 * Memory obtained from this call should be freed in the
2229 * normal way, viz., via vm_deallocate.
2230 */
2231kern_return_t
2232vm_allocate_cpm(
2233 host_priv_t host_priv,
2234 vm_map_t map,
2235 vm_address_t *addr,
2236 vm_size_t size,
2237 int flags)
2238{
2239 vm_map_address_t map_addr;
2240 vm_map_size_t map_size;
2241 kern_return_t kr;
2242 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
2243
2244 if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv) {
2245 return KERN_INVALID_HOST;
2246 }
2247
2248 if (VM_MAP_NULL == map) {
2249 return KERN_INVALID_ARGUMENT;
2250 }
2251
2252 map_addr = (vm_map_address_t)*addr;
2253 map_size = (vm_map_size_t)size;
2254
2255 vm_map_kernel_flags_set_vmflags(vmk_flags: &vmk_flags, vm_flags_and_tag: flags);
2256 vm_map_kernel_flags_update_range_id(flags: &vmk_flags, map);
2257
2258 kr = vm_map_enter_cpm(map, addr: &map_addr, size: map_size, vmk_flags);
2259
2260 *addr = CAST_DOWN(vm_address_t, map_addr);
2261 return kr;
2262}
2263
2264
2265kern_return_t
2266mach_vm_page_query(
2267 vm_map_t map,
2268 mach_vm_offset_t offset,
2269 int *disposition,
2270 int *ref_count)
2271{
2272 if (VM_MAP_NULL == map) {
2273 return KERN_INVALID_ARGUMENT;
2274 }
2275
2276 return vm_map_page_query_internal(
2277 map,
2278 vm_map_trunc_page(offset, PAGE_MASK),
2279 disposition, ref_count);
2280}
2281
2282kern_return_t
2283vm_map_page_query(
2284 vm_map_t map,
2285 vm_offset_t offset,
2286 int *disposition,
2287 int *ref_count)
2288{
2289 if (VM_MAP_NULL == map) {
2290 return KERN_INVALID_ARGUMENT;
2291 }
2292
2293 return vm_map_page_query_internal(
2294 map,
2295 vm_map_trunc_page(offset, PAGE_MASK),
2296 disposition, ref_count);
2297}
2298
2299kern_return_t
2300mach_vm_page_range_query(
2301 vm_map_t map,
2302 mach_vm_offset_t address,
2303 mach_vm_size_t size,
2304 mach_vm_address_t dispositions_addr,
2305 mach_vm_size_t *dispositions_count)
2306{
2307 kern_return_t kr = KERN_SUCCESS;
2308 int num_pages = 0, i = 0;
2309 mach_vm_size_t curr_sz = 0, copy_sz = 0;
2310 mach_vm_size_t disp_buf_req_size = 0, disp_buf_total_size = 0;
2311 mach_msg_type_number_t count = 0;
2312
2313 void *info = NULL;
2314 void *local_disp = NULL;
2315 vm_map_size_t info_size = 0, local_disp_size = 0;
2316 mach_vm_offset_t start = 0, end = 0;
2317 int effective_page_shift, effective_page_size, effective_page_mask;
2318
2319 if (map == VM_MAP_NULL || dispositions_count == NULL) {
2320 return KERN_INVALID_ARGUMENT;
2321 }
2322
2323 effective_page_shift = vm_self_region_page_shift_safely(target_map: map);
2324 if (effective_page_shift == -1) {
2325 return KERN_INVALID_ARGUMENT;
2326 }
2327 effective_page_size = (1 << effective_page_shift);
2328 effective_page_mask = effective_page_size - 1;
2329
2330 if (os_mul_overflow(*dispositions_count, sizeof(int), &disp_buf_req_size)) {
2331 return KERN_INVALID_ARGUMENT;
2332 }
2333
2334 start = vm_map_trunc_page(address, effective_page_mask);
2335 end = vm_map_round_page(address + size, effective_page_mask);
2336
2337 if (end < start) {
2338 return KERN_INVALID_ARGUMENT;
2339 }
2340
2341 if ((end - start) < size) {
2342 /*
2343 * Aligned size is less than unaligned size.
2344 */
2345 return KERN_INVALID_ARGUMENT;
2346 }
2347
2348 if (disp_buf_req_size == 0 || (end == start)) {
2349 return KERN_SUCCESS;
2350 }
2351
2352 /*
2353 * For large requests, we will go through them
2354 * MAX_PAGE_RANGE_QUERY chunk at a time.
2355 */
2356
2357 curr_sz = MIN(end - start, MAX_PAGE_RANGE_QUERY);
2358 num_pages = (int) (curr_sz >> effective_page_shift);
2359
2360 info_size = num_pages * sizeof(vm_page_info_basic_data_t);
2361 info = kalloc_data(info_size, Z_WAITOK);
2362
2363 local_disp_size = num_pages * sizeof(int);
2364 local_disp = kalloc_data(local_disp_size, Z_WAITOK);
2365
2366 if (info == NULL || local_disp == NULL) {
2367 kr = KERN_RESOURCE_SHORTAGE;
2368 goto out;
2369 }
2370
2371 while (size) {
2372 count = VM_PAGE_INFO_BASIC_COUNT;
2373 kr = vm_map_page_range_info_internal(
2374 map,
2375 start_offset: start,
2376 vm_map_round_page(start + curr_sz, effective_page_mask),
2377 effective_page_shift,
2378 VM_PAGE_INFO_BASIC,
2379 info: (vm_page_info_t) info,
2380 count: &count);
2381
2382 assert(kr == KERN_SUCCESS);
2383
2384 for (i = 0; i < num_pages; i++) {
2385 ((int*)local_disp)[i] = ((vm_page_info_basic_t)info)[i].disposition;
2386 }
2387
2388 copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int) /* an int per page */);
2389 kr = copyout(local_disp, (mach_vm_address_t)dispositions_addr, copy_sz);
2390
2391 start += curr_sz;
2392 disp_buf_req_size -= copy_sz;
2393 disp_buf_total_size += copy_sz;
2394
2395 if (kr != 0) {
2396 break;
2397 }
2398
2399 if ((disp_buf_req_size == 0) || (curr_sz >= size)) {
2400 /*
2401 * We might have inspected the full range OR
2402 * more than it esp. if the user passed in
2403 * non-page aligned start/size and/or if we
2404 * descended into a submap. We are done here.
2405 */
2406
2407 size = 0;
2408 } else {
2409 dispositions_addr += copy_sz;
2410
2411 size -= curr_sz;
2412
2413 curr_sz = MIN(vm_map_round_page(size, effective_page_mask), MAX_PAGE_RANGE_QUERY);
2414 num_pages = (int)(curr_sz >> effective_page_shift);
2415 }
2416 }
2417
2418 *dispositions_count = disp_buf_total_size / sizeof(int);
2419
2420out:
2421 kfree_data(local_disp, local_disp_size);
2422 kfree_data(info, info_size);
2423 return kr;
2424}
2425
2426kern_return_t
2427mach_vm_page_info(
2428 vm_map_t map,
2429 mach_vm_address_t address,
2430 vm_page_info_flavor_t flavor,
2431 vm_page_info_t info,
2432 mach_msg_type_number_t *count)
2433{
2434 kern_return_t kr;
2435
2436 if (map == VM_MAP_NULL) {
2437 return KERN_INVALID_ARGUMENT;
2438 }
2439
2440 kr = vm_map_page_info(map, offset: address, flavor, info, count);
2441 return kr;
2442}
2443
2444/* map a (whole) upl into an address space */
2445kern_return_t
2446vm_upl_map(
2447 vm_map_t map,
2448 upl_t upl,
2449 vm_address_t *dst_addr)
2450{
2451 vm_map_offset_t map_addr;
2452 kern_return_t kr;
2453
2454 if (VM_MAP_NULL == map) {
2455 return KERN_INVALID_ARGUMENT;
2456 }
2457
2458 kr = vm_map_enter_upl(map, upl, dst_addr: &map_addr);
2459 *dst_addr = CAST_DOWN(vm_address_t, map_addr);
2460 return kr;
2461}
2462
2463kern_return_t
2464vm_upl_unmap(
2465 vm_map_t map,
2466 upl_t upl)
2467{
2468 if (VM_MAP_NULL == map) {
2469 return KERN_INVALID_ARGUMENT;
2470 }
2471
2472 return vm_map_remove_upl(map, upl);
2473}
2474
2475/* map a part of a upl into an address space with requested protection. */
2476kern_return_t
2477vm_upl_map_range(
2478 vm_map_t map,
2479 upl_t upl,
2480 vm_offset_t offset_to_map,
2481 vm_size_t size_to_map,
2482 vm_prot_t prot_to_map,
2483 vm_address_t *dst_addr)
2484{
2485 vm_map_offset_t map_addr, aligned_offset_to_map, adjusted_offset;
2486 kern_return_t kr;
2487
2488 if (VM_MAP_NULL == map) {
2489 return KERN_INVALID_ARGUMENT;
2490 }
2491 aligned_offset_to_map = VM_MAP_TRUNC_PAGE(offset_to_map, VM_MAP_PAGE_MASK(map));
2492 adjusted_offset = offset_to_map - aligned_offset_to_map;
2493 size_to_map = VM_MAP_ROUND_PAGE(size_to_map + adjusted_offset, VM_MAP_PAGE_MASK(map));
2494
2495 kr = vm_map_enter_upl_range(map, upl, offset: aligned_offset_to_map, size: size_to_map, prot: prot_to_map, dst_addr: &map_addr);
2496 *dst_addr = CAST_DOWN(vm_address_t, (map_addr + adjusted_offset));
2497 return kr;
2498}
2499
2500/* unmap a part of a upl that was mapped in the address space. */
2501kern_return_t
2502vm_upl_unmap_range(
2503 vm_map_t map,
2504 upl_t upl,
2505 vm_offset_t offset_to_unmap,
2506 vm_size_t size_to_unmap)
2507{
2508 vm_map_offset_t aligned_offset_to_unmap, page_offset;
2509
2510 if (VM_MAP_NULL == map) {
2511 return KERN_INVALID_ARGUMENT;
2512 }
2513
2514 aligned_offset_to_unmap = VM_MAP_TRUNC_PAGE(offset_to_unmap, VM_MAP_PAGE_MASK(map));
2515 page_offset = offset_to_unmap - aligned_offset_to_unmap;
2516 size_to_unmap = VM_MAP_ROUND_PAGE(size_to_unmap + page_offset, VM_MAP_PAGE_MASK(map));
2517
2518 return vm_map_remove_upl_range(map, upl, offset: aligned_offset_to_unmap, size: size_to_unmap);
2519}
2520
2521/* Retrieve a upl for an object underlying an address range in a map */
2522
2523kern_return_t
2524vm_map_get_upl(
2525 vm_map_t map,
2526 vm_map_offset_t map_offset,
2527 upl_size_t *upl_size,
2528 upl_t *upl,
2529 upl_page_info_array_t page_list,
2530 unsigned int *count,
2531 upl_control_flags_t *flags,
2532 vm_tag_t tag,
2533 int force_data_sync)
2534{
2535 upl_control_flags_t map_flags;
2536 kern_return_t kr;
2537
2538 if (VM_MAP_NULL == map) {
2539 return KERN_INVALID_ARGUMENT;
2540 }
2541
2542 map_flags = *flags & ~UPL_NOZEROFILL;
2543 if (force_data_sync) {
2544 map_flags |= UPL_FORCE_DATA_SYNC;
2545 }
2546
2547 kr = vm_map_create_upl(map,
2548 offset: map_offset,
2549 upl_size,
2550 upl,
2551 page_list,
2552 count,
2553 flags: &map_flags,
2554 tag);
2555
2556 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
2557 return kr;
2558}
2559
2560/*
2561 * mach_make_memory_entry_64
2562 *
2563 * Think of it as a two-stage vm_remap() operation. First
2564 * you get a handle. Second, you get map that handle in
2565 * somewhere else. Rather than doing it all at once (and
2566 * without needing access to the other whole map).
2567 */
2568kern_return_t
2569mach_make_memory_entry_64(
2570 vm_map_t target_map,
2571 memory_object_size_t *size,
2572 memory_object_offset_t offset,
2573 vm_prot_t permission,
2574 ipc_port_t *object_handle,
2575 ipc_port_t parent_handle)
2576{
2577 vm_named_entry_kernel_flags_t vmne_kflags;
2578
2579 if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_USER) {
2580 /*
2581 * Unknown flag: reject for forward compatibility.
2582 */
2583 return KERN_INVALID_VALUE;
2584 }
2585
2586 vmne_kflags = VM_NAMED_ENTRY_KERNEL_FLAGS_NONE;
2587 if (permission & MAP_MEM_LEDGER_TAGGED) {
2588 vmne_kflags.vmnekf_ledger_tag = VM_LEDGER_TAG_DEFAULT;
2589 }
2590 return mach_make_memory_entry_internal(target_map,
2591 size,
2592 offset,
2593 permission,
2594 vmne_kflags,
2595 object_handle,
2596 parent_handle);
2597}
2598
2599kern_return_t
2600mach_make_memory_entry_internal(
2601 vm_map_t target_map,
2602 memory_object_size_t *size,
2603 memory_object_offset_t offset,
2604 vm_prot_t permission,
2605 vm_named_entry_kernel_flags_t vmne_kflags,
2606 ipc_port_t *object_handle,
2607 ipc_port_t parent_handle)
2608{
2609 vm_named_entry_t parent_entry;
2610 vm_named_entry_t user_entry;
2611 kern_return_t kr = KERN_SUCCESS;
2612 vm_object_t object;
2613 vm_map_size_t map_size;
2614 vm_map_offset_t map_start, map_end;
2615 vm_map_offset_t tmp;
2616
2617 /*
2618 * Stash the offset in the page for use by vm_map_enter_mem_object()
2619 * in the VM_FLAGS_RETURN_DATA_ADDR/MAP_MEM_USE_DATA_ADDR case.
2620 */
2621 vm_object_offset_t offset_in_page;
2622
2623 unsigned int access;
2624 vm_prot_t protections;
2625 vm_prot_t original_protections, mask_protections;
2626 unsigned int wimg_mode;
2627 boolean_t use_data_addr;
2628 boolean_t use_4K_compat;
2629
2630 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x\n", target_map, offset, *size, permission);
2631
2632 user_entry = NULL;
2633
2634 if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_ALL) {
2635 /*
2636 * Unknown flag: reject for forward compatibility.
2637 */
2638 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_INVALID_VALUE);
2639 return KERN_INVALID_VALUE;
2640 }
2641
2642 parent_entry = mach_memory_entry_from_port(port: parent_handle);
2643 if (parent_entry && parent_entry->is_copy) {
2644 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_INVALID_ARGUMENT);
2645 return KERN_INVALID_ARGUMENT;
2646 }
2647
2648 if (target_map == NULL || target_map->pmap == kernel_pmap) {
2649 offset = pgz_decode(offset, *size);
2650 }
2651
2652 if (__improbable(vm_map_range_overflows(target_map, offset, *size))) {
2653 return KERN_INVALID_ARGUMENT;
2654 }
2655
2656 original_protections = permission & VM_PROT_ALL;
2657 protections = original_protections;
2658 mask_protections = permission & VM_PROT_IS_MASK;
2659 access = GET_MAP_MEM(permission);
2660 use_data_addr = ((permission & MAP_MEM_USE_DATA_ADDR) != 0);
2661 use_4K_compat = ((permission & MAP_MEM_4K_DATA_ADDR) != 0);
2662
2663 user_entry = NULL;
2664
2665 map_start = vm_map_trunc_page(offset, VM_MAP_PAGE_MASK(target_map));
2666
2667 if (permission & MAP_MEM_ONLY) {
2668 boolean_t parent_is_object;
2669
2670 if (__improbable(os_add_overflow(offset, *size, &map_end))) {
2671 return KERN_INVALID_ARGUMENT;
2672 }
2673 map_end = vm_map_round_page(offset + *size, VM_MAP_PAGE_MASK(target_map));
2674 if (__improbable(map_end == 0 && *size != 0)) {
2675 return KERN_INVALID_ARGUMENT;
2676 }
2677 map_size = map_end - map_start;
2678
2679 if (use_data_addr || use_4K_compat || parent_entry == NULL) {
2680 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_INVALID_ARGUMENT);
2681 return KERN_INVALID_ARGUMENT;
2682 }
2683
2684 parent_is_object = parent_entry->is_object;
2685 if (!parent_is_object) {
2686 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_INVALID_ARGUMENT);
2687 return KERN_INVALID_ARGUMENT;
2688 }
2689 object = vm_named_entry_to_vm_object(named_entry: parent_entry);
2690 if (parent_is_object && object != VM_OBJECT_NULL) {
2691 wimg_mode = object->wimg_bits;
2692 } else {
2693 wimg_mode = VM_WIMG_USE_DEFAULT;
2694 }
2695 if ((access != parent_entry->access) &&
2696 !(parent_entry->protection & VM_PROT_WRITE)) {
2697 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_INVALID_RIGHT);
2698 return KERN_INVALID_RIGHT;
2699 }
2700 vm_prot_to_wimg(prot: access, wimg: &wimg_mode);
2701 if (access != MAP_MEM_NOOP) {
2702 parent_entry->access = access;
2703 }
2704 if (parent_is_object && object &&
2705 (access != MAP_MEM_NOOP) &&
2706 (!(object->nophyscache))) {
2707 if (object->wimg_bits != wimg_mode) {
2708 vm_object_lock(object);
2709 vm_object_change_wimg_mode(object, wimg_mode);
2710 vm_object_unlock(object);
2711 }
2712 }
2713 if (object_handle) {
2714 *object_handle = IP_NULL;
2715 }
2716 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_SUCCESS);
2717 return KERN_SUCCESS;
2718 } else if (permission & MAP_MEM_NAMED_CREATE) {
2719 int ledger_flags = 0;
2720 task_t owner;
2721 bool fully_owned = false;
2722
2723 if (__improbable(os_add_overflow(offset, *size, &map_end))) {
2724 return KERN_INVALID_ARGUMENT;
2725 }
2726 map_end = vm_map_round_page(map_end, VM_MAP_PAGE_MASK(target_map));
2727 map_size = map_end - map_start;
2728 if (__improbable(map_size == 0)) {
2729 *size = 0;
2730 *object_handle = IPC_PORT_NULL;
2731 return KERN_SUCCESS;
2732 }
2733 if (__improbable(map_end == 0)) {
2734 return KERN_INVALID_ARGUMENT;
2735 }
2736
2737 if (use_data_addr || use_4K_compat) {
2738 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_INVALID_ARGUMENT);
2739 return KERN_INVALID_ARGUMENT;
2740 }
2741
2742 /*
2743 * Force the creation of the VM object now.
2744 */
2745#if __LP64__
2746 if (map_size > ANON_MAX_SIZE) {
2747 kr = KERN_FAILURE;
2748 goto make_mem_done;
2749 }
2750#endif /* __LP64__ */
2751
2752 object = vm_object_allocate(size: map_size);
2753 assert(object != VM_OBJECT_NULL);
2754 vm_object_lock(object);
2755
2756 /*
2757 * XXX
2758 * We use this path when we want to make sure that
2759 * nobody messes with the object (coalesce, for
2760 * example) before we map it.
2761 * We might want to use these objects for transposition via
2762 * vm_object_transpose() too, so we don't want any copy or
2763 * shadow objects either...
2764 */
2765 object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2766 VM_OBJECT_SET_TRUE_SHARE(object, TRUE);
2767
2768 owner = current_task();
2769 if ((permission & MAP_MEM_PURGABLE) ||
2770 vmne_kflags.vmnekf_ledger_tag) {
2771 assert(object->vo_owner == NULL);
2772 assert(object->resident_page_count == 0);
2773 assert(object->wired_page_count == 0);
2774 assert(owner != TASK_NULL);
2775 if (vmne_kflags.vmnekf_ledger_no_footprint) {
2776 ledger_flags |= VM_LEDGER_FLAG_NO_FOOTPRINT;
2777 object->vo_no_footprint = TRUE;
2778 }
2779 if (permission & MAP_MEM_PURGABLE) {
2780 if (!(permission & VM_PROT_WRITE)) {
2781 /* if we can't write, we can't purge */
2782 vm_object_unlock(object);
2783 vm_object_deallocate(object);
2784 kr = KERN_INVALID_ARGUMENT;
2785 goto make_mem_done;
2786 }
2787 VM_OBJECT_SET_PURGABLE(object, VM_PURGABLE_NONVOLATILE);
2788 if (permission & MAP_MEM_PURGABLE_KERNEL_ONLY) {
2789 VM_OBJECT_SET_PURGEABLE_ONLY_BY_KERNEL(object, TRUE);
2790 }
2791#if __arm64__
2792 if (owner->task_legacy_footprint) {
2793 /*
2794 * For ios11, we failed to account for
2795 * this memory. Keep doing that for
2796 * legacy apps (built before ios12),
2797 * for backwards compatibility's sake...
2798 */
2799 owner = kernel_task;
2800 }
2801#endif /* __arm64__ */
2802 vm_purgeable_nonvolatile_enqueue(object, task: owner);
2803 /* all memory in this named entry is "owned" */
2804 fully_owned = true;
2805 }
2806 }
2807
2808 if (vmne_kflags.vmnekf_ledger_tag) {
2809 /*
2810 * Bill this object to the current task's
2811 * ledgers for the given tag.
2812 */
2813 if (vmne_kflags.vmnekf_ledger_no_footprint) {
2814 ledger_flags |= VM_LEDGER_FLAG_NO_FOOTPRINT;
2815 }
2816 object->vo_ledger_tag = vmne_kflags.vmnekf_ledger_tag;
2817 kr = vm_object_ownership_change(
2818 object,
2819 new_ledger_tag: vmne_kflags.vmnekf_ledger_tag,
2820 new_owner: owner, /* new owner */
2821 new_ledger_flags: ledger_flags,
2822 FALSE); /* task_objq locked? */
2823 if (kr != KERN_SUCCESS) {
2824 vm_object_unlock(object);
2825 vm_object_deallocate(object);
2826 goto make_mem_done;
2827 }
2828 /* all memory in this named entry is "owned" */
2829 fully_owned = true;
2830 }
2831
2832#if CONFIG_SECLUDED_MEMORY
2833 if (secluded_for_iokit && /* global boot-arg */
2834 ((permission & MAP_MEM_GRAB_SECLUDED))) {
2835 object->can_grab_secluded = TRUE;
2836 assert(!object->eligible_for_secluded);
2837 }
2838#endif /* CONFIG_SECLUDED_MEMORY */
2839
2840 /*
2841 * The VM object is brand new and nobody else knows about it,
2842 * so we don't need to lock it.
2843 */
2844
2845 wimg_mode = object->wimg_bits;
2846 vm_prot_to_wimg(prot: access, wimg: &wimg_mode);
2847 if (access != MAP_MEM_NOOP) {
2848 object->wimg_bits = wimg_mode;
2849 }
2850
2851 vm_object_unlock(object);
2852
2853 /* the object has no pages, so no WIMG bits to update here */
2854
2855 user_entry = mach_memory_entry_allocate(user_handle_p: object_handle);
2856 vm_named_entry_associate_vm_object(
2857 named_entry: user_entry,
2858 object,
2859 offset: 0,
2860 size: map_size,
2861 prot: (protections & VM_PROT_ALL));
2862 user_entry->internal = TRUE;
2863 user_entry->is_sub_map = FALSE;
2864 user_entry->offset = 0;
2865 user_entry->data_offset = 0;
2866 user_entry->protection = protections;
2867 user_entry->access = access;
2868 user_entry->size = map_size;
2869 user_entry->is_fully_owned = fully_owned;
2870
2871 /* user_object pager and internal fields are not used */
2872 /* when the object field is filled in. */
2873
2874 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2875 user_entry->data_offset));
2876 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_SUCCESS);
2877 return KERN_SUCCESS;
2878 }
2879
2880 if (permission & MAP_MEM_VM_COPY) {
2881 vm_map_copy_t copy;
2882
2883 if (target_map == VM_MAP_NULL) {
2884 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_INVALID_TASK);
2885 return KERN_INVALID_TASK;
2886 }
2887
2888 if (__improbable(os_add_overflow(offset, *size, &map_end))) {
2889 return KERN_INVALID_ARGUMENT;
2890 }
2891 map_end = vm_map_round_page(map_end, VM_MAP_PAGE_MASK(target_map));
2892 map_size = map_end - map_start;
2893 if (__improbable(map_size == 0)) {
2894 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_INVALID_ARGUMENT);
2895 return KERN_INVALID_ARGUMENT;
2896 }
2897 if (__improbable(map_end == 0)) {
2898 return KERN_INVALID_ARGUMENT;
2899 }
2900
2901 if (use_data_addr || use_4K_compat) {
2902 offset_in_page = offset - map_start;
2903 if (use_4K_compat) {
2904 offset_in_page &= ~((signed)(0xFFF));
2905 }
2906 } else {
2907 offset_in_page = 0;
2908 }
2909
2910 kr = vm_map_copyin_internal(src_map: target_map,
2911 src_addr: map_start,
2912 len: map_size,
2913 VM_MAP_COPYIN_ENTRY_LIST,
2914 copy_result: &copy);
2915 if (kr != KERN_SUCCESS) {
2916 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, kr);
2917 return kr;
2918 }
2919 assert(copy != VM_MAP_COPY_NULL);
2920
2921 user_entry = mach_memory_entry_allocate(user_handle_p: object_handle);
2922 user_entry->backing.copy = copy;
2923 user_entry->internal = FALSE;
2924 user_entry->is_sub_map = FALSE;
2925 user_entry->is_copy = TRUE;
2926 user_entry->offset = 0;
2927 user_entry->protection = protections;
2928 user_entry->size = map_size;
2929 user_entry->data_offset = offset_in_page;
2930
2931 /* is all memory in this named entry "owned"? */
2932 vm_map_entry_t entry;
2933 user_entry->is_fully_owned = TRUE;
2934 for (entry = vm_map_copy_first_entry(copy);
2935 entry != vm_map_copy_to_entry(copy);
2936 entry = entry->vme_next) {
2937 if (entry->is_sub_map ||
2938 VME_OBJECT(entry) == VM_OBJECT_NULL ||
2939 VM_OBJECT_OWNER(VME_OBJECT(entry)) == TASK_NULL) {
2940 /* this memory is not "owned" */
2941 user_entry->is_fully_owned = FALSE;
2942 break;
2943 }
2944 }
2945
2946 *size = CAST_DOWN(vm_size_t, (user_entry->size -
2947 user_entry->data_offset));
2948 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_SUCCESS);
2949 return KERN_SUCCESS;
2950 }
2951
2952 if ((permission & MAP_MEM_VM_SHARE)
2953 || parent_entry == NULL
2954 || (permission & MAP_MEM_NAMED_REUSE)) {
2955 vm_map_copy_t copy;
2956 vm_prot_t cur_prot, max_prot;
2957 vm_map_kernel_flags_t vmk_flags;
2958 vm_map_entry_t parent_copy_entry;
2959
2960 if (target_map == VM_MAP_NULL) {
2961 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_INVALID_TASK);
2962 return KERN_INVALID_TASK;
2963 }
2964
2965 if (__improbable(os_add_overflow(offset, *size, &map_end))) {
2966 return KERN_INVALID_ARGUMENT;
2967 }
2968 map_end = vm_map_round_page(map_end, VM_MAP_PAGE_MASK(target_map));
2969 map_size = map_end - map_start;
2970 if (__improbable(map_size == 0)) {
2971 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_INVALID_ARGUMENT);
2972 return KERN_INVALID_ARGUMENT;
2973 }
2974 if (__improbable(map_end == 0)) {
2975 /* rounding overflow */
2976 return KERN_INVALID_ARGUMENT;
2977 }
2978
2979 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
2980 vmk_flags.vmkf_range_id = KMEM_RANGE_ID_DATA;
2981 parent_copy_entry = VM_MAP_ENTRY_NULL;
2982 if (!(permission & MAP_MEM_VM_SHARE)) {
2983 vm_map_t tmp_map, real_map;
2984 vm_map_version_t version;
2985 vm_object_t tmp_object;
2986 vm_object_offset_t obj_off;
2987 vm_prot_t prot;
2988 boolean_t wired;
2989 bool contended;
2990
2991 /* resolve any pending submap copy-on-write... */
2992 if (protections & VM_PROT_WRITE) {
2993 tmp_map = target_map;
2994 vm_map_lock_read(tmp_map);
2995 kr = vm_map_lookup_and_lock_object(var_map: &tmp_map,
2996 vaddr: map_start,
2997 fault_type: protections | mask_protections,
2998 OBJECT_LOCK_EXCLUSIVE,
2999 out_version: &version,
3000 object: &tmp_object,
3001 offset: &obj_off,
3002 out_prot: &prot,
3003 wired: &wired,
3004 NULL, /* fault_info */
3005 real_map: &real_map,
3006 contended: &contended);
3007 if (kr != KERN_SUCCESS) {
3008 vm_map_unlock_read(tmp_map);
3009 } else {
3010 vm_object_unlock(tmp_object);
3011 vm_map_unlock_read(tmp_map);
3012 if (real_map != tmp_map) {
3013 vm_map_unlock_read(real_map);
3014 }
3015 }
3016 }
3017 /* ... and carry on */
3018
3019 /* stop extracting if VM object changes */
3020 vmk_flags.vmkf_copy_single_object = TRUE;
3021 if ((permission & MAP_MEM_NAMED_REUSE) &&
3022 parent_entry != NULL &&
3023 parent_entry->is_object) {
3024 vm_map_copy_t parent_copy;
3025 parent_copy = parent_entry->backing.copy;
3026 /*
3027 * Assert that the vm_map_copy is coming from the right
3028 * zone and hasn't been forged
3029 */
3030 vm_map_copy_require(copy: parent_copy);
3031 assert(parent_copy->cpy_hdr.nentries == 1);
3032 parent_copy_entry = vm_map_copy_first_entry(parent_copy);
3033 assert(!parent_copy_entry->is_sub_map);
3034 }
3035 }
3036
3037 if (use_data_addr || use_4K_compat) {
3038 offset_in_page = offset - map_start;
3039 if (use_4K_compat) {
3040 offset_in_page &= ~((signed)(0xFFF));
3041 }
3042 } else {
3043 offset_in_page = 0;
3044 }
3045
3046 if (mask_protections) {
3047 /*
3048 * caller is asking for whichever proctections are
3049 * available: no required protections.
3050 */
3051 cur_prot = VM_PROT_NONE;
3052 max_prot = VM_PROT_NONE;
3053 } else {
3054 /*
3055 * Caller wants a memory entry with "protections".
3056 * Make sure we extract only memory that matches that.
3057 */
3058 cur_prot = protections;
3059 max_prot = protections;
3060 }
3061 if (target_map->pmap == kernel_pmap) {
3062 /*
3063 * Get "reserved" map entries to avoid deadlocking
3064 * on the kernel map or a kernel submap if we
3065 * run out of VM map entries and need to refill that
3066 * zone.
3067 */
3068 vmk_flags.vmkf_copy_pageable = FALSE;
3069 } else {
3070 vmk_flags.vmkf_copy_pageable = TRUE;
3071 }
3072 vmk_flags.vmkf_copy_same_map = FALSE;
3073 assert(map_size != 0);
3074 kr = vm_map_copy_extract(src_map: target_map,
3075 src_addr: map_start,
3076 len: map_size,
3077 FALSE, /* copy */
3078 copy_result: &copy,
3079 cur_prot: &cur_prot,
3080 max_prot: &max_prot,
3081 VM_INHERIT_SHARE,
3082 vmk_flags);
3083 if (kr != KERN_SUCCESS) {
3084 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, kr);
3085 if (VM_MAP_PAGE_SHIFT(map: target_map) < PAGE_SHIFT) {
3086// panic("DEBUG4K %s:%d kr 0x%x", __FUNCTION__, __LINE__, kr);
3087 }
3088 return kr;
3089 }
3090 assert(copy != VM_MAP_COPY_NULL);
3091
3092 if (mask_protections) {
3093 /*
3094 * We just want as much of "original_protections"
3095 * as we can get out of the actual "cur_prot".
3096 */
3097 protections &= cur_prot;
3098 if (protections == VM_PROT_NONE) {
3099 /* no access at all: fail */
3100 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_PROTECTION_FAILURE);
3101 if (VM_MAP_PAGE_SHIFT(map: target_map) < PAGE_SHIFT) {
3102// panic("DEBUG4K %s:%d kr 0x%x", __FUNCTION__, __LINE__, kr);
3103 }
3104 vm_map_copy_discard(copy);
3105 return KERN_PROTECTION_FAILURE;
3106 }
3107 } else {
3108 /*
3109 * We want exactly "original_protections"
3110 * out of "cur_prot".
3111 */
3112 assert((cur_prot & protections) == protections);
3113 assert((max_prot & protections) == protections);
3114 /* XXX FBDP TODO: no longer needed? */
3115 if ((cur_prot & protections) != protections) {
3116 if (VM_MAP_PAGE_SHIFT(map: target_map) < PAGE_SHIFT) {
3117// panic("DEBUG4K %s:%d kr 0x%x", __FUNCTION__, __LINE__, KERN_PROTECTION_FAILURE);
3118 }
3119 vm_map_copy_discard(copy);
3120 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_PROTECTION_FAILURE);
3121 return KERN_PROTECTION_FAILURE;
3122 }
3123 }
3124
3125 if (!(permission & MAP_MEM_VM_SHARE)) {
3126 vm_map_entry_t copy_entry;
3127
3128 /* limit size to what's actually covered by "copy" */
3129 assert(copy->cpy_hdr.nentries == 1);
3130 copy_entry = vm_map_copy_first_entry(copy);
3131 map_size = copy_entry->vme_end - copy_entry->vme_start;
3132
3133 if ((permission & MAP_MEM_NAMED_REUSE) &&
3134 parent_copy_entry != VM_MAP_ENTRY_NULL &&
3135 VME_OBJECT(copy_entry) == VME_OBJECT(parent_copy_entry) &&
3136 VME_OFFSET(entry: copy_entry) == VME_OFFSET(entry: parent_copy_entry) &&
3137 parent_entry->offset == 0 &&
3138 parent_entry->size == map_size &&
3139 (parent_entry->data_offset == offset_in_page)) {
3140 /* we have a match: re-use "parent_entry" */
3141
3142 /* release our new "copy" */
3143 vm_map_copy_discard(copy);
3144 /* get extra send right on handle */
3145 parent_handle = ipc_port_copy_send_any(port: parent_handle);
3146
3147 *size = CAST_DOWN(vm_size_t,
3148 (parent_entry->size -
3149 parent_entry->data_offset));
3150 *object_handle = parent_handle;
3151 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_SUCCESS);
3152 return KERN_SUCCESS;
3153 }
3154
3155 /* no match: we need to create a new entry */
3156 object = VME_OBJECT(copy_entry);
3157 vm_object_lock(object);
3158 wimg_mode = object->wimg_bits;
3159 if (!(object->nophyscache)) {
3160 vm_prot_to_wimg(prot: access, wimg: &wimg_mode);
3161 }
3162 if (object->wimg_bits != wimg_mode) {
3163 vm_object_change_wimg_mode(object, wimg_mode);
3164 }
3165 vm_object_unlock(object);
3166 }
3167
3168 user_entry = mach_memory_entry_allocate(user_handle_p: object_handle);
3169 user_entry->backing.copy = copy;
3170 user_entry->is_sub_map = FALSE;
3171 user_entry->is_object = FALSE;
3172 user_entry->internal = FALSE;
3173 user_entry->protection = protections;
3174 user_entry->size = map_size;
3175 user_entry->data_offset = offset_in_page;
3176
3177 if (permission & MAP_MEM_VM_SHARE) {
3178 vm_map_entry_t copy_entry;
3179
3180 user_entry->is_copy = TRUE;
3181 user_entry->offset = 0;
3182
3183 /* is all memory in this named entry "owned"? */
3184 user_entry->is_fully_owned = TRUE;
3185 for (copy_entry = vm_map_copy_first_entry(copy);
3186 copy_entry != vm_map_copy_to_entry(copy);
3187 copy_entry = copy_entry->vme_next) {
3188 if (copy_entry->is_sub_map ||
3189 VM_OBJECT_OWNER(VME_OBJECT(copy_entry)) == TASK_NULL) {
3190 /* this memory is not "owned" */
3191 user_entry->is_fully_owned = FALSE;
3192 break;
3193 }
3194 }
3195 } else {
3196 user_entry->is_object = TRUE;
3197 user_entry->internal = object->internal;
3198 user_entry->offset = VME_OFFSET(vm_map_copy_first_entry(copy));
3199 user_entry->access = GET_MAP_MEM(permission);
3200 /* is all memory in this named entry "owned"? */
3201 if (VM_OBJECT_OWNER(vm_named_entry_to_vm_object(user_entry)) != TASK_NULL) {
3202 user_entry->is_fully_owned = TRUE;
3203 }
3204 }
3205
3206 *size = CAST_DOWN(vm_size_t, (user_entry->size -
3207 user_entry->data_offset));
3208 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_SUCCESS);
3209 return KERN_SUCCESS;
3210 }
3211
3212 /* The new object will be based on an existing named object */
3213 if (parent_entry == NULL) {
3214 kr = KERN_INVALID_ARGUMENT;
3215 goto make_mem_done;
3216 }
3217
3218 if (parent_entry->is_copy) {
3219 panic("parent_entry %p is_copy not supported", parent_entry);
3220 kr = KERN_INVALID_ARGUMENT;
3221 goto make_mem_done;
3222 }
3223
3224 if (use_data_addr || use_4K_compat) {
3225 /*
3226 * submaps and pagers should only be accessible from within
3227 * the kernel, which shouldn't use the data address flag, so can fail here.
3228 */
3229 if (parent_entry->is_sub_map) {
3230 panic("Shouldn't be using data address with a parent entry that is a submap.");
3231 }
3232 /*
3233 * Account for offset to data in parent entry and
3234 * compute our own offset to data.
3235 */
3236 if (__improbable(os_add3_overflow(offset, *size, parent_entry->data_offset, &map_size))) {
3237 kr = KERN_INVALID_ARGUMENT;
3238 goto make_mem_done;
3239 }
3240 if (map_size > parent_entry->size) {
3241 kr = KERN_INVALID_ARGUMENT;
3242 goto make_mem_done;
3243 }
3244
3245 if (__improbable(os_add_overflow(offset, parent_entry->data_offset, &map_start))) {
3246 kr = KERN_INVALID_ARGUMENT;
3247 goto make_mem_done;
3248 }
3249 map_start = vm_map_trunc_page(map_start, PAGE_MASK);
3250 offset_in_page = (offset + parent_entry->data_offset) - map_start;
3251 if (use_4K_compat) {
3252 offset_in_page &= ~((signed)(0xFFF));
3253 }
3254 if (__improbable(os_add3_overflow(offset, parent_entry->data_offset, *size, &map_end))) {
3255 kr = KERN_INVALID_ARGUMENT;
3256 goto make_mem_done;
3257 }
3258 map_end = vm_map_round_page(map_end, PAGE_MASK);
3259 if (__improbable(map_end == 0 && *size != 0)) {
3260 /* rounding overflow */
3261 kr = KERN_INVALID_ARGUMENT;
3262 goto make_mem_done;
3263 }
3264 map_size = map_end - map_start;
3265 } else {
3266 if (__improbable(os_add_overflow(offset, *size, &map_end))) {
3267 kr = KERN_INVALID_ARGUMENT;
3268 goto make_mem_done;
3269 }
3270 map_end = vm_map_round_page(map_end, PAGE_MASK);
3271 if (__improbable(map_end == 0 && *size != 0)) {
3272 kr = KERN_INVALID_ARGUMENT;
3273 goto make_mem_done;
3274 }
3275 map_size = map_end - map_start;
3276 offset_in_page = 0;
3277
3278 if (__improbable(os_add_overflow(offset, map_size, &tmp))) {
3279 kr = KERN_INVALID_ARGUMENT;
3280 goto make_mem_done;
3281 }
3282 if ((offset + map_size) > parent_entry->size) {
3283 kr = KERN_INVALID_ARGUMENT;
3284 goto make_mem_done;
3285 }
3286 }
3287
3288 if (mask_protections) {
3289 /*
3290 * The caller asked us to use the "protections" as
3291 * a mask, so restrict "protections" to what this
3292 * mapping actually allows.
3293 */
3294 protections &= parent_entry->protection;
3295 }
3296 if ((protections & parent_entry->protection) != protections) {
3297 kr = KERN_PROTECTION_FAILURE;
3298 goto make_mem_done;
3299 }
3300
3301 if (__improbable(os_add_overflow(parent_entry->offset, map_start, &tmp))) {
3302 kr = KERN_INVALID_ARGUMENT;
3303 goto make_mem_done;
3304 }
3305 user_entry = mach_memory_entry_allocate(user_handle_p: object_handle);
3306 user_entry->size = map_size;
3307 user_entry->offset = parent_entry->offset + map_start;
3308 user_entry->data_offset = offset_in_page;
3309 user_entry->is_sub_map = parent_entry->is_sub_map;
3310 user_entry->is_copy = parent_entry->is_copy;
3311 user_entry->protection = protections;
3312
3313 if (access != MAP_MEM_NOOP) {
3314 user_entry->access = access;
3315 }
3316
3317 if (parent_entry->is_sub_map) {
3318 vm_map_t map = parent_entry->backing.map;
3319 vm_map_reference(map);
3320 user_entry->backing.map = map;
3321 } else {
3322 object = vm_named_entry_to_vm_object(named_entry: parent_entry);
3323 assert(object != VM_OBJECT_NULL);
3324 assert(object->copy_strategy != MEMORY_OBJECT_COPY_SYMMETRIC);
3325 vm_named_entry_associate_vm_object(
3326 named_entry: user_entry,
3327 object,
3328 offset: user_entry->offset,
3329 size: user_entry->size,
3330 prot: (user_entry->protection & VM_PROT_ALL));
3331 assert(user_entry->is_object);
3332 /* we now point to this object, hold on */
3333 vm_object_lock(object);
3334 vm_object_reference_locked(object);
3335#if VM_OBJECT_TRACKING_OP_TRUESHARE
3336 if (!object->true_share &&
3337 vm_object_tracking_btlog) {
3338 btlog_record(vm_object_tracking_btlog, object,
3339 VM_OBJECT_TRACKING_OP_TRUESHARE,
3340 btref_get(__builtin_frame_address(0), 0));
3341 }
3342#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
3343
3344 VM_OBJECT_SET_TRUE_SHARE(object, TRUE);
3345 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
3346 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
3347 }
3348 vm_object_unlock(object);
3349 }
3350 *size = CAST_DOWN(vm_size_t, (user_entry->size -
3351 user_entry->data_offset));
3352 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, KERN_SUCCESS);
3353 return KERN_SUCCESS;
3354
3355make_mem_done:
3356 DEBUG4K_MEMENTRY("map %p offset 0x%llx size 0x%llx prot 0x%x -> entry %p kr 0x%x\n", target_map, offset, *size, permission, user_entry, kr);
3357 return kr;
3358}
3359
3360kern_return_t
3361_mach_make_memory_entry(
3362 vm_map_t target_map,
3363 memory_object_size_t *size,
3364 memory_object_offset_t offset,
3365 vm_prot_t permission,
3366 ipc_port_t *object_handle,
3367 ipc_port_t parent_entry)
3368{
3369 memory_object_size_t mo_size;
3370 kern_return_t kr;
3371
3372 mo_size = (memory_object_size_t)*size;
3373 kr = mach_make_memory_entry_64(target_map, size: &mo_size,
3374 offset: (memory_object_offset_t)offset, permission, object_handle,
3375 parent_handle: parent_entry);
3376 *size = mo_size;
3377 return kr;
3378}
3379
3380kern_return_t
3381mach_make_memory_entry(
3382 vm_map_t target_map,
3383 vm_size_t *size,
3384 vm_offset_t offset,
3385 vm_prot_t permission,
3386 ipc_port_t *object_handle,
3387 ipc_port_t parent_entry)
3388{
3389 memory_object_size_t mo_size;
3390 kern_return_t kr;
3391
3392 mo_size = (memory_object_size_t)*size;
3393 kr = mach_make_memory_entry_64(target_map, size: &mo_size,
3394 offset: (memory_object_offset_t)offset, permission, object_handle,
3395 parent_handle: parent_entry);
3396 *size = CAST_DOWN(vm_size_t, mo_size);
3397 return kr;
3398}
3399
3400/*
3401 * task_wire
3402 *
3403 * Set or clear the map's wiring_required flag. This flag, if set,
3404 * will cause all future virtual memory allocation to allocate
3405 * user wired memory. Unwiring pages wired down as a result of
3406 * this routine is done with the vm_wire interface.
3407 */
3408kern_return_t
3409task_wire(
3410 vm_map_t map,
3411 boolean_t must_wire __unused)
3412{
3413 if (map == VM_MAP_NULL) {
3414 return KERN_INVALID_ARGUMENT;
3415 }
3416
3417 return KERN_NOT_SUPPORTED;
3418}
3419
3420kern_return_t
3421vm_map_exec_lockdown(
3422 vm_map_t map)
3423{
3424 if (map == VM_MAP_NULL) {
3425 return KERN_INVALID_ARGUMENT;
3426 }
3427
3428 vm_map_lock(map);
3429 map->map_disallow_new_exec = TRUE;
3430 vm_map_unlock(map);
3431
3432 return KERN_SUCCESS;
3433}
3434
3435__private_extern__ vm_named_entry_t
3436mach_memory_entry_allocate(ipc_port_t *user_handle_p)
3437{
3438 vm_named_entry_t user_entry;
3439
3440 user_entry = kalloc_type(struct vm_named_entry,
3441 Z_WAITOK | Z_ZERO | Z_NOFAIL);
3442 named_entry_lock_init(user_entry);
3443
3444 *user_handle_p = ipc_kobject_alloc_port(kobject: (ipc_kobject_t)user_entry,
3445 type: IKOT_NAMED_ENTRY,
3446 options: IPC_KOBJECT_ALLOC_MAKE_SEND | IPC_KOBJECT_ALLOC_NSREQUEST);
3447
3448#if VM_NAMED_ENTRY_DEBUG
3449 /* backtrace at allocation time, for debugging only */
3450 user_entry->named_entry_bt = btref_get(__builtin_frame_address(0), 0);
3451#endif /* VM_NAMED_ENTRY_DEBUG */
3452 return user_entry;
3453}
3454
3455/*
3456 * mach_memory_object_memory_entry_64
3457 *
3458 * Create a named entry backed by the provided pager.
3459 *
3460 */
3461kern_return_t
3462mach_memory_object_memory_entry_64(
3463 host_t host,
3464 boolean_t internal,
3465 vm_object_offset_t size,
3466 vm_prot_t permission,
3467 memory_object_t pager,
3468 ipc_port_t *entry_handle)
3469{
3470 vm_named_entry_t user_entry;
3471 ipc_port_t user_handle;
3472 vm_object_t object;
3473
3474 if (host == HOST_NULL) {
3475 return KERN_INVALID_HOST;
3476 }
3477
3478 size = vm_object_round_page(size);
3479
3480 if (pager == MEMORY_OBJECT_NULL && internal) {
3481 object = vm_object_allocate(size);
3482 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
3483 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
3484 }
3485 } else {
3486 object = memory_object_to_vm_object(mem_obj: pager);
3487 if (object != VM_OBJECT_NULL) {
3488 vm_object_reference(object);
3489 }
3490 }
3491 if (object == VM_OBJECT_NULL) {
3492 return KERN_INVALID_ARGUMENT;
3493 }
3494
3495 user_entry = mach_memory_entry_allocate(user_handle_p: &user_handle);
3496 user_entry->size = size;
3497 user_entry->offset = 0;
3498 user_entry->protection = permission & VM_PROT_ALL;
3499 user_entry->access = GET_MAP_MEM(permission);
3500 user_entry->is_sub_map = FALSE;
3501
3502 vm_named_entry_associate_vm_object(named_entry: user_entry, object, offset: 0, size,
3503 prot: (user_entry->protection & VM_PROT_ALL));
3504 user_entry->internal = object->internal;
3505 assert(object->internal == internal);
3506 if (VM_OBJECT_OWNER(object) != TASK_NULL) {
3507 /* all memory in this entry is "owned" */
3508 user_entry->is_fully_owned = TRUE;
3509 }
3510
3511 *entry_handle = user_handle;
3512 return KERN_SUCCESS;
3513}
3514
3515kern_return_t
3516mach_memory_object_memory_entry(
3517 host_t host,
3518 boolean_t internal,
3519 vm_size_t size,
3520 vm_prot_t permission,
3521 memory_object_t pager,
3522 ipc_port_t *entry_handle)
3523{
3524 return mach_memory_object_memory_entry_64( host, internal,
3525 size: (vm_object_offset_t)size, permission, pager, entry_handle);
3526}
3527
3528
3529kern_return_t
3530mach_memory_entry_purgable_control(
3531 ipc_port_t entry_port,
3532 vm_purgable_t control,
3533 int *state)
3534{
3535 if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
3536 /* not allowed from user-space */
3537 return KERN_INVALID_ARGUMENT;
3538 }
3539
3540 return memory_entry_purgeable_control_internal(entry_port, control, state);
3541}
3542
3543kern_return_t
3544memory_entry_purgeable_control_internal(
3545 ipc_port_t entry_port,
3546 vm_purgable_t control,
3547 int *state)
3548{
3549 kern_return_t kr;
3550 vm_named_entry_t mem_entry;
3551 vm_object_t object;
3552
3553 mem_entry = mach_memory_entry_from_port(port: entry_port);
3554 if (mem_entry == NULL) {
3555 return KERN_INVALID_ARGUMENT;
3556 }
3557
3558 if (control != VM_PURGABLE_SET_STATE &&
3559 control != VM_PURGABLE_GET_STATE &&
3560 control != VM_PURGABLE_SET_STATE_FROM_KERNEL) {
3561 return KERN_INVALID_ARGUMENT;
3562 }
3563
3564 if ((control == VM_PURGABLE_SET_STATE ||
3565 control == VM_PURGABLE_SET_STATE_FROM_KERNEL) &&
3566 (((*state & ~(VM_PURGABLE_ALL_MASKS)) != 0) ||
3567 ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK))) {
3568 return KERN_INVALID_ARGUMENT;
3569 }
3570
3571 named_entry_lock(mem_entry);
3572
3573 if (mem_entry->is_sub_map ||
3574 mem_entry->is_copy) {
3575 named_entry_unlock(mem_entry);
3576 return KERN_INVALID_ARGUMENT;
3577 }
3578
3579 assert(mem_entry->is_object);
3580 object = vm_named_entry_to_vm_object(named_entry: mem_entry);
3581 if (object == VM_OBJECT_NULL) {
3582 named_entry_unlock(mem_entry);
3583 return KERN_INVALID_ARGUMENT;
3584 }
3585
3586 vm_object_lock(object);
3587
3588 /* check that named entry covers entire object ? */
3589 if (mem_entry->offset != 0 || object->vo_size != mem_entry->size) {
3590 vm_object_unlock(object);
3591 named_entry_unlock(mem_entry);
3592 return KERN_INVALID_ARGUMENT;
3593 }
3594
3595 named_entry_unlock(mem_entry);
3596
3597 kr = vm_object_purgable_control(object, control, state);
3598
3599 vm_object_unlock(object);
3600
3601 return kr;
3602}
3603
3604kern_return_t
3605mach_memory_entry_access_tracking(
3606 ipc_port_t entry_port,
3607 int *access_tracking,
3608 uint32_t *access_tracking_reads,
3609 uint32_t *access_tracking_writes)
3610{
3611 return memory_entry_access_tracking_internal(entry_port,
3612 access_tracking,
3613 access_tracking_reads,
3614 access_tracking_writes);
3615}
3616
3617kern_return_t
3618memory_entry_access_tracking_internal(
3619 ipc_port_t entry_port,
3620 int *access_tracking,
3621 uint32_t *access_tracking_reads,
3622 uint32_t *access_tracking_writes)
3623{
3624 vm_named_entry_t mem_entry;
3625 vm_object_t object;
3626 kern_return_t kr;
3627
3628 mem_entry = mach_memory_entry_from_port(port: entry_port);
3629 if (mem_entry == NULL) {
3630 return KERN_INVALID_ARGUMENT;
3631 }
3632
3633 named_entry_lock(mem_entry);
3634
3635 if (mem_entry->is_sub_map ||
3636 mem_entry->is_copy) {
3637 named_entry_unlock(mem_entry);
3638 return KERN_INVALID_ARGUMENT;
3639 }
3640
3641 assert(mem_entry->is_object);
3642 object = vm_named_entry_to_vm_object(named_entry: mem_entry);
3643 if (object == VM_OBJECT_NULL) {
3644 named_entry_unlock(mem_entry);
3645 return KERN_INVALID_ARGUMENT;
3646 }
3647
3648#if VM_OBJECT_ACCESS_TRACKING
3649 vm_object_access_tracking(object,
3650 access_tracking,
3651 access_tracking_reads,
3652 access_tracking_writes);
3653 kr = KERN_SUCCESS;
3654#else /* VM_OBJECT_ACCESS_TRACKING */
3655 (void) access_tracking;
3656 (void) access_tracking_reads;
3657 (void) access_tracking_writes;
3658 kr = KERN_NOT_SUPPORTED;
3659#endif /* VM_OBJECT_ACCESS_TRACKING */
3660
3661 named_entry_unlock(mem_entry);
3662
3663 return kr;
3664}
3665
3666#if DEVELOPMENT || DEBUG
3667/* For dtrace probe in mach_memory_entry_ownership */
3668extern int proc_selfpid(void);
3669extern char *proc_name_address(void *p);
3670#endif /* DEVELOPMENT || DEBUG */
3671
3672/* Kernel call only, MIG uses *_from_user() below */
3673kern_return_t
3674mach_memory_entry_ownership(
3675 ipc_port_t entry_port,
3676 task_t owner,
3677 int ledger_tag,
3678 int ledger_flags)
3679{
3680 task_t cur_task;
3681 kern_return_t kr;
3682 vm_named_entry_t mem_entry;
3683 vm_object_t object;
3684
3685 cur_task = current_task();
3686 if (cur_task != kernel_task &&
3687 ((owner != cur_task && owner != TASK_NULL) ||
3688 (ledger_flags & VM_LEDGER_FLAG_NO_FOOTPRINT) ||
3689 (ledger_flags & VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG) ||
3690 ledger_tag == VM_LEDGER_TAG_NETWORK)) {
3691 bool transfer_ok = false;
3692
3693 /*
3694 * An entitlement is required to:
3695 * + tranfer memory ownership to someone else,
3696 * + request that the memory not count against the footprint,
3697 * + tag as "network" (since that implies "no footprint")
3698 *
3699 * Exception: task with task_no_footprint_for_debug == 1 on internal build
3700 */
3701 if (!cur_task->task_can_transfer_memory_ownership &&
3702 IOCurrentTaskHasEntitlement(entitlement: "com.apple.private.memory.ownership_transfer")) {
3703 cur_task->task_can_transfer_memory_ownership = TRUE;
3704 }
3705 if (cur_task->task_can_transfer_memory_ownership) {
3706 /* we're allowed to transfer ownership to any task */
3707 transfer_ok = true;
3708 }
3709#if DEVELOPMENT || DEBUG
3710 if (!transfer_ok &&
3711 ledger_tag == VM_LEDGER_TAG_DEFAULT &&
3712 (ledger_flags & VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG) &&
3713 cur_task->task_no_footprint_for_debug) {
3714 int to_panic = 0;
3715 static bool init_bootarg = false;
3716
3717 /*
3718 * Allow performance tools running on internal builds to hide memory usage from phys_footprint even
3719 * WITHOUT an entitlement. This can be enabled by per task sysctl vm.task_no_footprint_for_debug=1
3720 * with the ledger tag VM_LEDGER_TAG_DEFAULT and flag VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG.
3721 *
3722 * If the boot-arg "panic_on_no_footprint_for_debug" is set, the kernel will
3723 * panic here in order to detect any abuse of this feature, which is intended solely for
3724 * memory debugging purpose.
3725 */
3726 if (!init_bootarg) {
3727 PE_parse_boot_argn("panic_on_no_footprint_for_debug", &to_panic, sizeof(to_panic));
3728 init_bootarg = true;
3729 }
3730 if (to_panic) {
3731 panic("%s: panic_on_no_footprint_for_debug is triggered by pid %d procname %s", __func__, proc_selfpid(), get_bsdtask_info(cur_task)? proc_name_address(get_bsdtask_info(cur_task)) : "?");
3732 }
3733
3734 /*
3735 * Flushing out user space processes using this interface:
3736 * $ dtrace -n 'task_no_footprint_for_debug {printf("%d[%s]\n", pid, execname); stack(); ustack();}'
3737 */
3738 DTRACE_VM(task_no_footprint_for_debug);
3739 transfer_ok = true;
3740 }
3741#endif /* DEVELOPMENT || DEBUG */
3742 if (!transfer_ok) {
3743#define TRANSFER_ENTITLEMENT_MAX_LENGTH 1024 /* XXX ? */
3744 const char *our_id, *their_id;
3745 our_id = IOTaskGetEntitlement(task: current_task(), entitlement: "com.apple.developer.memory.transfer-send");
3746 their_id = IOTaskGetEntitlement(task: owner, entitlement: "com.apple.developer.memory.transfer-accept");
3747 if (our_id && their_id &&
3748 !strncmp(s1: our_id, s2: their_id, TRANSFER_ENTITLEMENT_MAX_LENGTH)) {
3749 /* allow transfer between tasks that have matching entitlements */
3750 if (strnlen(s: our_id, TRANSFER_ENTITLEMENT_MAX_LENGTH) < TRANSFER_ENTITLEMENT_MAX_LENGTH &&
3751 strnlen(s: their_id, TRANSFER_ENTITLEMENT_MAX_LENGTH) < TRANSFER_ENTITLEMENT_MAX_LENGTH) {
3752 transfer_ok = true;
3753 } else {
3754 /* complain about entitlement(s) being too long... */
3755 assertf((strlen(our_id) <= TRANSFER_ENTITLEMENT_MAX_LENGTH &&
3756 strlen(their_id) <= TRANSFER_ENTITLEMENT_MAX_LENGTH),
3757 "our_id:%lu their_id:%lu",
3758 strlen(our_id), strlen(their_id));
3759 }
3760 }
3761 }
3762 if (!transfer_ok) {
3763 /* transfer denied */
3764 return KERN_NO_ACCESS;
3765 }
3766
3767 if (ledger_flags & VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG) {
3768 /*
3769 * We've made it past the checks above, so we either
3770 * have the entitlement or the sysctl.
3771 * Convert to VM_LEDGER_FLAG_NO_FOOTPRINT.
3772 */
3773 ledger_flags &= ~VM_LEDGER_FLAG_NO_FOOTPRINT_FOR_DEBUG;
3774 ledger_flags |= VM_LEDGER_FLAG_NO_FOOTPRINT;
3775 }
3776 }
3777
3778 if (ledger_flags & ~VM_LEDGER_FLAGS) {
3779 return KERN_INVALID_ARGUMENT;
3780 }
3781 if (ledger_tag == VM_LEDGER_TAG_UNCHANGED) {
3782 /* leave "ledger_tag" unchanged */
3783 } else if (ledger_tag < 0 ||
3784 ledger_tag > VM_LEDGER_TAG_MAX) {
3785 return KERN_INVALID_ARGUMENT;
3786 }
3787 if (owner == TASK_NULL) {
3788 /* leave "owner" unchanged */
3789 owner = VM_OBJECT_OWNER_UNCHANGED;
3790 }
3791
3792 mem_entry = mach_memory_entry_from_port(port: entry_port);
3793 if (mem_entry == NULL) {
3794 return KERN_INVALID_ARGUMENT;
3795 }
3796
3797 named_entry_lock(mem_entry);
3798
3799 if (mem_entry->is_sub_map ||
3800 !mem_entry->is_fully_owned) {
3801 named_entry_unlock(mem_entry);
3802 return KERN_INVALID_ARGUMENT;
3803 }
3804
3805 if (mem_entry->is_object) {
3806 object = vm_named_entry_to_vm_object(named_entry: mem_entry);
3807 if (object == VM_OBJECT_NULL) {
3808 named_entry_unlock(mem_entry);
3809 return KERN_INVALID_ARGUMENT;
3810 }
3811 vm_object_lock(object);
3812 /* check that named entry covers entire object ? */
3813 if (mem_entry->offset != 0 || object->vo_size != mem_entry->size) {
3814 vm_object_unlock(object);
3815 named_entry_unlock(mem_entry);
3816 return KERN_INVALID_ARGUMENT;
3817 }
3818 named_entry_unlock(mem_entry);
3819 kr = vm_object_ownership_change(object,
3820 new_ledger_tag: ledger_tag,
3821 new_owner: owner,
3822 new_ledger_flags: ledger_flags,
3823 FALSE); /* task_objq_locked */
3824 vm_object_unlock(object);
3825 } else if (mem_entry->is_copy) {
3826 vm_map_copy_t copy;
3827 vm_map_entry_t entry;
3828
3829 copy = mem_entry->backing.copy;
3830 named_entry_unlock(mem_entry);
3831 for (entry = vm_map_copy_first_entry(copy);
3832 entry != vm_map_copy_to_entry(copy);
3833 entry = entry->vme_next) {
3834 object = VME_OBJECT(entry);
3835 if (entry->is_sub_map ||
3836 object == VM_OBJECT_NULL) {
3837 kr = KERN_INVALID_ARGUMENT;
3838 break;
3839 }
3840 vm_object_lock(object);
3841 if (VME_OFFSET(entry) != 0 ||
3842 entry->vme_end - entry->vme_start != object->vo_size) {
3843 vm_object_unlock(object);
3844 kr = KERN_INVALID_ARGUMENT;
3845 break;
3846 }
3847 kr = vm_object_ownership_change(object,
3848 new_ledger_tag: ledger_tag,
3849 new_owner: owner,
3850 new_ledger_flags: ledger_flags,
3851 FALSE); /* task_objq_locked */
3852 vm_object_unlock(object);
3853 if (kr != KERN_SUCCESS) {
3854 kr = KERN_INVALID_ARGUMENT;
3855 break;
3856 }
3857 }
3858 } else {
3859 named_entry_unlock(mem_entry);
3860 return KERN_INVALID_ARGUMENT;
3861 }
3862
3863 return kr;
3864}
3865
3866/* MIG call from userspace */
3867kern_return_t
3868mach_memory_entry_ownership_from_user(
3869 ipc_port_t entry_port,
3870 mach_port_t owner_port,
3871 int ledger_tag,
3872 int ledger_flags)
3873{
3874 task_t owner = TASK_NULL;
3875 kern_return_t kr;
3876
3877 if (IP_VALID(owner_port)) {
3878 if (ip_kotype(owner_port) == IKOT_TASK_ID_TOKEN) {
3879 task_id_token_t token = convert_port_to_task_id_token(port: owner_port);
3880 (void)task_identity_token_get_task_grp(token, taskp: &owner, grp: TASK_GRP_MIG);
3881 task_id_token_release(token);
3882 /* token ref released */
3883 } else {
3884 owner = convert_port_to_task_mig(port: owner_port);
3885 }
3886 }
3887 /* hold task ref on owner (Nullable) */
3888
3889 if (owner && task_is_a_corpse(task: owner)) {
3890 /* identity token can represent a corpse, disallow it */
3891 task_deallocate_mig(owner);
3892 owner = TASK_NULL;
3893 }
3894
3895 /* mach_memory_entry_ownership() will handle TASK_NULL owner */
3896 kr = mach_memory_entry_ownership(entry_port, owner, /* Nullable */
3897 ledger_tag, ledger_flags);
3898
3899 if (owner) {
3900 task_deallocate_mig(owner);
3901 }
3902
3903 if (kr == KERN_SUCCESS) {
3904 /* MIG rule, consume port right on success */
3905 ipc_port_release_send(port: owner_port);
3906 }
3907 return kr;
3908}
3909
3910kern_return_t
3911mach_memory_entry_get_page_counts(
3912 ipc_port_t entry_port,
3913 unsigned int *resident_page_count,
3914 unsigned int *dirty_page_count)
3915{
3916 kern_return_t kr;
3917 vm_named_entry_t mem_entry;
3918 vm_object_t object;
3919 vm_object_offset_t offset;
3920 vm_object_size_t size;
3921
3922 mem_entry = mach_memory_entry_from_port(port: entry_port);
3923 if (mem_entry == NULL) {
3924 return KERN_INVALID_ARGUMENT;
3925 }
3926
3927 named_entry_lock(mem_entry);
3928
3929 if (mem_entry->is_sub_map ||
3930 mem_entry->is_copy) {
3931 named_entry_unlock(mem_entry);
3932 return KERN_INVALID_ARGUMENT;
3933 }
3934
3935 assert(mem_entry->is_object);
3936 object = vm_named_entry_to_vm_object(named_entry: mem_entry);
3937 if (object == VM_OBJECT_NULL) {
3938 named_entry_unlock(mem_entry);
3939 return KERN_INVALID_ARGUMENT;
3940 }
3941
3942 vm_object_lock(object);
3943
3944 offset = mem_entry->offset;
3945 size = mem_entry->size;
3946 size = vm_object_round_page(offset + size) - vm_object_trunc_page(offset);
3947 offset = vm_object_trunc_page(offset);
3948
3949 named_entry_unlock(mem_entry);
3950
3951 kr = vm_object_get_page_counts(object, offset, size, resident_page_count, dirty_page_count);
3952
3953 vm_object_unlock(object);
3954
3955 return kr;
3956}
3957
3958kern_return_t
3959mach_memory_entry_phys_page_offset(
3960 ipc_port_t entry_port,
3961 vm_object_offset_t *offset_p)
3962{
3963 vm_named_entry_t mem_entry;
3964 vm_object_t object;
3965 vm_object_offset_t offset;
3966 vm_object_offset_t data_offset;
3967
3968 mem_entry = mach_memory_entry_from_port(port: entry_port);
3969 if (mem_entry == NULL) {
3970 return KERN_INVALID_ARGUMENT;
3971 }
3972
3973 named_entry_lock(mem_entry);
3974
3975 if (mem_entry->is_sub_map ||
3976 mem_entry->is_copy) {
3977 named_entry_unlock(mem_entry);
3978 return KERN_INVALID_ARGUMENT;
3979 }
3980
3981 assert(mem_entry->is_object);
3982 object = vm_named_entry_to_vm_object(named_entry: mem_entry);
3983 if (object == VM_OBJECT_NULL) {
3984 named_entry_unlock(mem_entry);
3985 return KERN_INVALID_ARGUMENT;
3986 }
3987
3988 offset = mem_entry->offset;
3989 data_offset = mem_entry->data_offset;
3990
3991 named_entry_unlock(mem_entry);
3992
3993 *offset_p = offset - vm_object_trunc_page(offset) + data_offset;
3994 return KERN_SUCCESS;
3995}
3996
3997kern_return_t
3998mach_memory_entry_map_size(
3999 ipc_port_t entry_port,
4000 vm_map_t map,
4001 memory_object_offset_t offset,
4002 memory_object_offset_t size,
4003 mach_vm_size_t *map_size)
4004{
4005 vm_named_entry_t mem_entry;
4006 vm_object_t object;
4007 vm_object_offset_t object_offset_start, object_offset_end;
4008 vm_map_copy_t copy_map, target_copy_map;
4009 vm_map_offset_t overmap_start, overmap_end, trimmed_start;
4010 kern_return_t kr;
4011
4012 mem_entry = mach_memory_entry_from_port(port: entry_port);
4013 if (mem_entry == NULL) {
4014 return KERN_INVALID_ARGUMENT;
4015 }
4016
4017 named_entry_lock(mem_entry);
4018
4019 if (mem_entry->is_sub_map) {
4020 named_entry_unlock(mem_entry);
4021 return KERN_INVALID_ARGUMENT;
4022 }
4023
4024 if (mem_entry->is_object) {
4025 object = vm_named_entry_to_vm_object(named_entry: mem_entry);
4026 if (object == VM_OBJECT_NULL) {
4027 named_entry_unlock(mem_entry);
4028 return KERN_INVALID_ARGUMENT;
4029 }
4030
4031 object_offset_start = mem_entry->offset;
4032 object_offset_start += mem_entry->data_offset;
4033 object_offset_start += offset;
4034 object_offset_end = object_offset_start + size;
4035 object_offset_start = vm_map_trunc_page(object_offset_start,
4036 VM_MAP_PAGE_MASK(map));
4037 object_offset_end = vm_map_round_page(object_offset_end,
4038 VM_MAP_PAGE_MASK(map));
4039
4040 named_entry_unlock(mem_entry);
4041
4042 *map_size = object_offset_end - object_offset_start;
4043 return KERN_SUCCESS;
4044 }
4045
4046 if (!mem_entry->is_copy) {
4047 panic("unsupported type of mem_entry %p", mem_entry);
4048 }
4049
4050 assert(mem_entry->is_copy);
4051 if (VM_MAP_COPY_PAGE_MASK(mem_entry->backing.copy) == VM_MAP_PAGE_MASK(map)) {
4052 *map_size = vm_map_round_page(mem_entry->offset + mem_entry->data_offset + offset + size, VM_MAP_PAGE_MASK(map)) - vm_map_trunc_page(mem_entry->offset + mem_entry->data_offset + offset, VM_MAP_PAGE_MASK(map));
4053 DEBUG4K_SHARE("map %p (%d) mem_entry %p offset 0x%llx + 0x%llx + 0x%llx size 0x%llx -> map_size 0x%llx\n", map, VM_MAP_PAGE_MASK(map), mem_entry, mem_entry->offset, mem_entry->data_offset, offset, size, *map_size);
4054 named_entry_unlock(mem_entry);
4055 return KERN_SUCCESS;
4056 }
4057
4058 DEBUG4K_SHARE("mem_entry %p copy %p (%d) map %p (%d) offset 0x%llx size 0x%llx\n", mem_entry, mem_entry->backing.copy, VM_MAP_COPY_PAGE_SHIFT(mem_entry->backing.copy), map, VM_MAP_PAGE_SHIFT(map), offset, size);
4059 copy_map = mem_entry->backing.copy;
4060 target_copy_map = VM_MAP_COPY_NULL;
4061 DEBUG4K_ADJUST("adjusting...\n");
4062 kr = vm_map_copy_adjust_to_target(copy_map,
4063 offset: mem_entry->data_offset + offset,
4064 size,
4065 target_map: map,
4066 FALSE,
4067 target_copy_map_p: &target_copy_map,
4068 overmap_start_p: &overmap_start,
4069 overmap_end_p: &overmap_end,
4070 trimmed_start_p: &trimmed_start);
4071 if (kr == KERN_SUCCESS) {
4072 if (target_copy_map->size != copy_map->size) {
4073 DEBUG4K_ADJUST("copy %p (%d) map %p (%d) offset 0x%llx size 0x%llx overmap_start 0x%llx overmap_end 0x%llx trimmed_start 0x%llx map_size 0x%llx -> 0x%llx\n", copy_map, VM_MAP_COPY_PAGE_SHIFT(copy_map), map, VM_MAP_PAGE_SHIFT(map), (uint64_t)offset, (uint64_t)size, (uint64_t)overmap_start, (uint64_t)overmap_end, (uint64_t)trimmed_start, (uint64_t)copy_map->size, (uint64_t)target_copy_map->size);
4074 }
4075 *map_size = target_copy_map->size;
4076 if (target_copy_map != copy_map) {
4077 vm_map_copy_discard(copy: target_copy_map);
4078 }
4079 target_copy_map = VM_MAP_COPY_NULL;
4080 }
4081 named_entry_unlock(mem_entry);
4082 return kr;
4083}
4084
4085/*
4086 * mach_memory_entry_port_release:
4087 *
4088 * Release a send right on a named entry port. This is the correct
4089 * way to destroy a named entry. When the last right on the port is
4090 * released, mach_memory_entry_no_senders() willl be called.
4091 */
4092void
4093mach_memory_entry_port_release(
4094 ipc_port_t port)
4095{
4096 assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
4097 ipc_port_release_send(port);
4098}
4099
4100vm_named_entry_t
4101mach_memory_entry_from_port(ipc_port_t port)
4102{
4103 if (IP_VALID(port)) {
4104 return ipc_kobject_get_stable(port, type: IKOT_NAMED_ENTRY);
4105 }
4106 return NULL;
4107}
4108
4109/*
4110 * mach_memory_entry_no_senders:
4111 *
4112 * Destroys the memory entry associated with a mach port.
4113 * Memory entries have the exact same lifetime as their owning port.
4114 *
4115 * Releasing a memory entry is done by calling
4116 * mach_memory_entry_port_release() on its owning port.
4117 */
4118static void
4119mach_memory_entry_no_senders(ipc_port_t port, mach_port_mscount_t mscount)
4120{
4121 vm_named_entry_t named_entry;
4122
4123 named_entry = ipc_kobject_dealloc_port(port, mscount, type: IKOT_NAMED_ENTRY);
4124
4125 if (named_entry->is_sub_map) {
4126 vm_map_deallocate(map: named_entry->backing.map);
4127 } else if (named_entry->is_copy) {
4128 vm_map_copy_discard(copy: named_entry->backing.copy);
4129 } else if (named_entry->is_object) {
4130 assert(named_entry->backing.copy->cpy_hdr.nentries == 1);
4131 vm_map_copy_discard(copy: named_entry->backing.copy);
4132 } else {
4133 assert(named_entry->backing.copy == VM_MAP_COPY_NULL);
4134 }
4135
4136#if VM_NAMED_ENTRY_DEBUG
4137 btref_put(named_entry->named_entry_bt);
4138#endif /* VM_NAMED_ENTRY_DEBUG */
4139
4140 named_entry_lock_destroy(named_entry);
4141 kfree_type(struct vm_named_entry, named_entry);
4142}
4143
4144/* Allow manipulation of individual page state. This is actually part of */
4145/* the UPL regimen but takes place on the memory entry rather than on a UPL */
4146
4147kern_return_t
4148mach_memory_entry_page_op(
4149 ipc_port_t entry_port,
4150 vm_object_offset_t offset,
4151 int ops,
4152 ppnum_t *phys_entry,
4153 int *flags)
4154{
4155 vm_named_entry_t mem_entry;
4156 vm_object_t object;
4157 kern_return_t kr;
4158
4159 mem_entry = mach_memory_entry_from_port(port: entry_port);
4160 if (mem_entry == NULL) {
4161 return KERN_INVALID_ARGUMENT;
4162 }
4163
4164 named_entry_lock(mem_entry);
4165
4166 if (mem_entry->is_sub_map ||
4167 mem_entry->is_copy) {
4168 named_entry_unlock(mem_entry);
4169 return KERN_INVALID_ARGUMENT;
4170 }
4171
4172 assert(mem_entry->is_object);
4173 object = vm_named_entry_to_vm_object(named_entry: mem_entry);
4174 if (object == VM_OBJECT_NULL) {
4175 named_entry_unlock(mem_entry);
4176 return KERN_INVALID_ARGUMENT;
4177 }
4178
4179 vm_object_reference(object);
4180 named_entry_unlock(mem_entry);
4181
4182 kr = vm_object_page_op(object, offset, ops, phys_entry, flags);
4183
4184 vm_object_deallocate(object);
4185
4186 return kr;
4187}
4188
4189/*
4190 * mach_memory_entry_range_op offers performance enhancement over
4191 * mach_memory_entry_page_op for page_op functions which do not require page
4192 * level state to be returned from the call. Page_op was created to provide
4193 * a low-cost alternative to page manipulation via UPLs when only a single
4194 * page was involved. The range_op call establishes the ability in the _op
4195 * family of functions to work on multiple pages where the lack of page level
4196 * state handling allows the caller to avoid the overhead of the upl structures.
4197 */
4198
4199kern_return_t
4200mach_memory_entry_range_op(
4201 ipc_port_t entry_port,
4202 vm_object_offset_t offset_beg,
4203 vm_object_offset_t offset_end,
4204 int ops,
4205 int *range)
4206{
4207 vm_named_entry_t mem_entry;
4208 vm_object_t object;
4209 kern_return_t kr;
4210
4211 mem_entry = mach_memory_entry_from_port(port: entry_port);
4212 if (mem_entry == NULL) {
4213 return KERN_INVALID_ARGUMENT;
4214 }
4215
4216 named_entry_lock(mem_entry);
4217
4218 if (mem_entry->is_sub_map ||
4219 mem_entry->is_copy) {
4220 named_entry_unlock(mem_entry);
4221 return KERN_INVALID_ARGUMENT;
4222 }
4223
4224 assert(mem_entry->is_object);
4225 object = vm_named_entry_to_vm_object(named_entry: mem_entry);
4226 if (object == VM_OBJECT_NULL) {
4227 named_entry_unlock(mem_entry);
4228 return KERN_INVALID_ARGUMENT;
4229 }
4230
4231 vm_object_reference(object);
4232 named_entry_unlock(mem_entry);
4233
4234 kr = vm_object_range_op(object,
4235 offset_beg,
4236 offset_end,
4237 ops,
4238 range: (uint32_t *) range);
4239
4240 vm_object_deallocate(object);
4241
4242 return kr;
4243}
4244
4245/* ******* Temporary Internal calls to UPL for BSD ***** */
4246
4247extern int kernel_upl_map(
4248 vm_map_t map,
4249 upl_t upl,
4250 vm_offset_t *dst_addr);
4251
4252extern int kernel_upl_unmap(
4253 vm_map_t map,
4254 upl_t upl);
4255
4256extern int kernel_upl_commit(
4257 upl_t upl,
4258 upl_page_info_t *pl,
4259 mach_msg_type_number_t count);
4260
4261extern int kernel_upl_commit_range(
4262 upl_t upl,
4263 upl_offset_t offset,
4264 upl_size_t size,
4265 int flags,
4266 upl_page_info_array_t pl,
4267 mach_msg_type_number_t count);
4268
4269extern int kernel_upl_abort(
4270 upl_t upl,
4271 int abort_type);
4272
4273extern int kernel_upl_abort_range(
4274 upl_t upl,
4275 upl_offset_t offset,
4276 upl_size_t size,
4277 int abort_flags);
4278
4279
4280kern_return_t
4281kernel_upl_map(
4282 vm_map_t map,
4283 upl_t upl,
4284 vm_offset_t *dst_addr)
4285{
4286 return vm_upl_map(map, upl, dst_addr);
4287}
4288
4289
4290kern_return_t
4291kernel_upl_unmap(
4292 vm_map_t map,
4293 upl_t upl)
4294{
4295 return vm_upl_unmap(map, upl);
4296}
4297
4298kern_return_t
4299kernel_upl_commit(
4300 upl_t upl,
4301 upl_page_info_t *pl,
4302 mach_msg_type_number_t count)
4303{
4304 kern_return_t kr;
4305
4306 kr = upl_commit(upl_object: upl, page_list: pl, page_listCnt: count);
4307 upl_deallocate(upl);
4308 return kr;
4309}
4310
4311
4312kern_return_t
4313kernel_upl_commit_range(
4314 upl_t upl,
4315 upl_offset_t offset,
4316 upl_size_t size,
4317 int flags,
4318 upl_page_info_array_t pl,
4319 mach_msg_type_number_t count)
4320{
4321 boolean_t finished = FALSE;
4322 kern_return_t kr;
4323
4324 if (flags & UPL_COMMIT_FREE_ON_EMPTY) {
4325 flags |= UPL_COMMIT_NOTIFY_EMPTY;
4326 }
4327
4328 if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) {
4329 return KERN_INVALID_ARGUMENT;
4330 }
4331
4332 kr = upl_commit_range(upl_object: upl, offset, size, cntrl_flags: flags, page_list: pl, page_listCnt: count, empty: &finished);
4333
4334 if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished) {
4335 upl_deallocate(upl);
4336 }
4337
4338 return kr;
4339}
4340
4341kern_return_t
4342kernel_upl_abort_range(
4343 upl_t upl,
4344 upl_offset_t offset,
4345 upl_size_t size,
4346 int abort_flags)
4347{
4348 kern_return_t kr;
4349 boolean_t finished = FALSE;
4350
4351 if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY) {
4352 abort_flags |= UPL_COMMIT_NOTIFY_EMPTY;
4353 }
4354
4355 kr = upl_abort_range(upl_object: upl, offset, size, abort_cond: abort_flags, empty: &finished);
4356
4357 if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished) {
4358 upl_deallocate(upl);
4359 }
4360
4361 return kr;
4362}
4363
4364kern_return_t
4365kernel_upl_abort(
4366 upl_t upl,
4367 int abort_type)
4368{
4369 kern_return_t kr;
4370
4371 kr = upl_abort(upl_object: upl, abort_cond: abort_type);
4372 upl_deallocate(upl);
4373 return kr;
4374}
4375
4376/*
4377 * Now a kernel-private interface (for BootCache
4378 * use only). Need a cleaner way to create an
4379 * empty vm_map() and return a handle to it.
4380 */
4381
4382kern_return_t
4383vm_region_object_create(
4384 vm_map_t target_map,
4385 vm_size_t size,
4386 ipc_port_t *object_handle)
4387{
4388 vm_named_entry_t user_entry;
4389 vm_map_t new_map;
4390
4391 user_entry = mach_memory_entry_allocate(user_handle_p: object_handle);
4392
4393 /* Create a named object based on a submap of specified size */
4394
4395 new_map = vm_map_create_options(PMAP_NULL, VM_MAP_MIN_ADDRESS,
4396 vm_map_round_page(size, VM_MAP_PAGE_MASK(target_map)),
4397 options: VM_MAP_CREATE_PAGEABLE);
4398 vm_map_set_page_shift(map: new_map, pageshift: VM_MAP_PAGE_SHIFT(map: target_map));
4399
4400 user_entry->backing.map = new_map;
4401 user_entry->internal = TRUE;
4402 user_entry->is_sub_map = TRUE;
4403 user_entry->offset = 0;
4404 user_entry->protection = VM_PROT_ALL;
4405 user_entry->size = size;
4406
4407 return KERN_SUCCESS;
4408}
4409
4410ppnum_t vm_map_get_phys_page( /* forward */
4411 vm_map_t map,
4412 vm_offset_t offset);
4413
4414ppnum_t
4415vm_map_get_phys_page(
4416 vm_map_t map,
4417 vm_offset_t addr)
4418{
4419 vm_object_offset_t offset;
4420 vm_object_t object;
4421 vm_map_offset_t map_offset;
4422 vm_map_entry_t entry;
4423 ppnum_t phys_page = 0;
4424
4425 map_offset = vm_map_trunc_page(addr, PAGE_MASK);
4426
4427 vm_map_lock(map);
4428 while (vm_map_lookup_entry(map, address: map_offset, entry: &entry)) {
4429 if (entry->is_sub_map) {
4430 vm_map_t old_map;
4431 vm_map_lock(VME_SUBMAP(entry));
4432 old_map = map;
4433 map = VME_SUBMAP(entry);
4434 map_offset = (VME_OFFSET(entry) +
4435 (map_offset - entry->vme_start));
4436 vm_map_unlock(old_map);
4437 continue;
4438 }
4439 if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
4440 vm_map_unlock(map);
4441 return (ppnum_t) 0;
4442 }
4443 if (VME_OBJECT(entry)->phys_contiguous) {
4444 /* These are not standard pageable memory mappings */
4445 /* If they are not present in the object they will */
4446 /* have to be picked up from the pager through the */
4447 /* fault mechanism. */
4448 if (VME_OBJECT(entry)->vo_shadow_offset == 0) {
4449 /* need to call vm_fault */
4450 vm_map_unlock(map);
4451 vm_fault(map, vaddr: map_offset, VM_PROT_NONE,
4452 FALSE /* change_wiring */, VM_KERN_MEMORY_NONE,
4453 THREAD_UNINT, NULL, pmap_addr: 0);
4454 vm_map_lock(map);
4455 continue;
4456 }
4457 offset = (VME_OFFSET(entry) +
4458 (map_offset - entry->vme_start));
4459 phys_page = (ppnum_t)
4460 ((VME_OBJECT(entry)->vo_shadow_offset
4461 + offset) >> PAGE_SHIFT);
4462 break;
4463 }
4464 offset = (VME_OFFSET(entry) + (map_offset - entry->vme_start));
4465 object = VME_OBJECT(entry);
4466 vm_object_lock(object);
4467 while (TRUE) {
4468 vm_page_t dst_page = vm_page_lookup(object, offset);
4469 if (dst_page == VM_PAGE_NULL) {
4470 if (object->shadow) {
4471 vm_object_t old_object;
4472 vm_object_lock(object->shadow);
4473 old_object = object;
4474 offset = offset + object->vo_shadow_offset;
4475 object = object->shadow;
4476 vm_object_unlock(old_object);
4477 } else {
4478 vm_object_unlock(object);
4479 break;
4480 }
4481 } else {
4482 phys_page = (ppnum_t)(VM_PAGE_GET_PHYS_PAGE(m: dst_page));
4483 vm_object_unlock(object);
4484 break;
4485 }
4486 }
4487 break;
4488 }
4489
4490 vm_map_unlock(map);
4491 return phys_page;
4492}
4493
4494kern_return_t
4495mach_vm_deferred_reclamation_buffer_init(
4496 task_t task,
4497 mach_vm_offset_t address,
4498 mach_vm_size_t size)
4499{
4500#if CONFIG_DEFERRED_RECLAIM
4501 return vm_deferred_reclamation_buffer_init_internal(task, address, size);
4502#else
4503 (void) task;
4504 (void) address;
4505 (void) size;
4506 (void) indices;
4507 return KERN_NOT_SUPPORTED;
4508#endif /* CONFIG_DEFERRED_RECLAIM */
4509}
4510
4511kern_return_t
4512mach_vm_deferred_reclamation_buffer_synchronize(
4513 task_t task,
4514 mach_vm_size_t num_entries_to_reclaim)
4515{
4516#if CONFIG_DEFERRED_RECLAIM
4517 return vm_deferred_reclamation_buffer_synchronize_internal(task, max_entries_to_reclaim: num_entries_to_reclaim);
4518#else
4519 (void) task;
4520 (void) num_entries_to_reclaim;
4521 return KERN_NOT_SUPPORTED;
4522#endif /* CONFIG_DEFERRED_RECLAIM */
4523}
4524
4525kern_return_t
4526mach_vm_deferred_reclamation_buffer_update_reclaimable_bytes(task_t task, mach_vm_size_t reclaimable_bytes)
4527{
4528#if CONFIG_DEFERRED_RECLAIM
4529 return vm_deferred_reclamation_buffer_update_reclaimable_bytes_internal(task, reclaimable_bytes);
4530#else
4531 (void) task;
4532 (void) reclaimable_bytes;
4533 return KERN_NOT_SUPPORTED;
4534#endif /* CONFIG_DEFERRED_RECLAIM */
4535}
4536
4537#if 0
4538kern_return_t kernel_object_iopl_request( /* forward */
4539 vm_named_entry_t named_entry,
4540 memory_object_offset_t offset,
4541 upl_size_t *upl_size,
4542 upl_t *upl_ptr,
4543 upl_page_info_array_t user_page_list,
4544 unsigned int *page_list_count,
4545 int *flags);
4546
4547kern_return_t
4548kernel_object_iopl_request(
4549 vm_named_entry_t named_entry,
4550 memory_object_offset_t offset,
4551 upl_size_t *upl_size,
4552 upl_t *upl_ptr,
4553 upl_page_info_array_t user_page_list,
4554 unsigned int *page_list_count,
4555 int *flags)
4556{
4557 vm_object_t object;
4558 kern_return_t ret;
4559
4560 int caller_flags;
4561
4562 caller_flags = *flags;
4563
4564 if (caller_flags & ~UPL_VALID_FLAGS) {
4565 /*
4566 * For forward compatibility's sake,
4567 * reject any unknown flag.
4568 */
4569 return KERN_INVALID_VALUE;
4570 }
4571
4572 /* a few checks to make sure user is obeying rules */
4573 if (*upl_size == 0) {
4574 if (offset >= named_entry->size) {
4575 return KERN_INVALID_RIGHT;
4576 }
4577 *upl_size = (upl_size_t) (named_entry->size - offset);
4578 if (*upl_size != named_entry->size - offset) {
4579 return KERN_INVALID_ARGUMENT;
4580 }
4581 }
4582 if (caller_flags & UPL_COPYOUT_FROM) {
4583 if ((named_entry->protection & VM_PROT_READ)
4584 != VM_PROT_READ) {
4585 return KERN_INVALID_RIGHT;
4586 }
4587 } else {
4588 if ((named_entry->protection &
4589 (VM_PROT_READ | VM_PROT_WRITE))
4590 != (VM_PROT_READ | VM_PROT_WRITE)) {
4591 return KERN_INVALID_RIGHT;
4592 }
4593 }
4594 if (named_entry->size < (offset + *upl_size)) {
4595 return KERN_INVALID_ARGUMENT;
4596 }
4597
4598 /* the callers parameter offset is defined to be the */
4599 /* offset from beginning of named entry offset in object */
4600 offset = offset + named_entry->offset;
4601
4602 if (named_entry->is_sub_map ||
4603 named_entry->is_copy) {
4604 return KERN_INVALID_ARGUMENT;
4605 }
4606
4607 named_entry_lock(named_entry);
4608
4609 /* This is the case where we are going to operate */
4610 /* on an already known object. If the object is */
4611 /* not ready it is internal. An external */
4612 /* object cannot be mapped until it is ready */
4613 /* we can therefore avoid the ready check */
4614 /* in this case. */
4615 assert(named_entry->is_object);
4616 object = vm_named_entry_to_vm_object(named_entry);
4617 vm_object_reference(object);
4618 named_entry_unlock(named_entry);
4619
4620 if (!object->private) {
4621 if (*upl_size > MAX_UPL_TRANSFER_BYTES) {
4622 *upl_size = MAX_UPL_TRANSFER_BYTES;
4623 }
4624 if (object->phys_contiguous) {
4625 *flags = UPL_PHYS_CONTIG;
4626 } else {
4627 *flags = 0;
4628 }
4629 } else {
4630 *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
4631 }
4632
4633 ret = vm_object_iopl_request(object,
4634 offset,
4635 *upl_size,
4636 upl_ptr,
4637 user_page_list,
4638 page_list_count,
4639 (upl_control_flags_t)(unsigned int)caller_flags);
4640 vm_object_deallocate(object);
4641 return ret;
4642}
4643#endif
4644
4645/*
4646 * These symbols are looked up at runtime by vmware, VirtualBox,
4647 * despite not being exported in the symbol sets.
4648 */
4649
4650#if defined(__x86_64__)
4651
4652kern_return_t
4653mach_vm_map(
4654 vm_map_t target_map,
4655 mach_vm_offset_t *address,
4656 mach_vm_size_t initial_size,
4657 mach_vm_offset_t mask,
4658 int flags,
4659 ipc_port_t port,
4660 vm_object_offset_t offset,
4661 boolean_t copy,
4662 vm_prot_t cur_protection,
4663 vm_prot_t max_protection,
4664 vm_inherit_t inheritance);
4665
4666kern_return_t
4667mach_vm_remap(
4668 vm_map_t target_map,
4669 mach_vm_offset_t *address,
4670 mach_vm_size_t size,
4671 mach_vm_offset_t mask,
4672 int flags,
4673 vm_map_t src_map,
4674 mach_vm_offset_t memory_address,
4675 boolean_t copy,
4676 vm_prot_t *cur_protection,
4677 vm_prot_t *max_protection,
4678 vm_inherit_t inheritance);
4679
4680kern_return_t
4681mach_vm_map(
4682 vm_map_t target_map,
4683 mach_vm_offset_t *address,
4684 mach_vm_size_t initial_size,
4685 mach_vm_offset_t mask,
4686 int flags,
4687 ipc_port_t port,
4688 vm_object_offset_t offset,
4689 boolean_t copy,
4690 vm_prot_t cur_protection,
4691 vm_prot_t max_protection,
4692 vm_inherit_t inheritance)
4693{
4694 return mach_vm_map_external(target_map, address, initial_size, mask, flags, port,
4695 offset, copy, cur_protection, max_protection, inheritance);
4696}
4697
4698kern_return_t
4699mach_vm_remap(
4700 vm_map_t target_map,
4701 mach_vm_offset_t *address,
4702 mach_vm_size_t size,
4703 mach_vm_offset_t mask,
4704 int flags,
4705 vm_map_t src_map,
4706 mach_vm_offset_t memory_address,
4707 boolean_t copy,
4708 vm_prot_t *cur_protection, /* OUT */
4709 vm_prot_t *max_protection, /* OUT */
4710 vm_inherit_t inheritance)
4711{
4712 return mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address,
4713 copy, cur_protection, max_protection, inheritance);
4714}
4715
4716kern_return_t
4717vm_map(
4718 vm_map_t target_map,
4719 vm_offset_t *address,
4720 vm_size_t size,
4721 vm_offset_t mask,
4722 int flags,
4723 ipc_port_t port,
4724 vm_offset_t offset,
4725 boolean_t copy,
4726 vm_prot_t cur_protection,
4727 vm_prot_t max_protection,
4728 vm_inherit_t inheritance);
4729
4730kern_return_t
4731vm_map(
4732 vm_map_t target_map,
4733 vm_offset_t *address,
4734 vm_size_t size,
4735 vm_offset_t mask,
4736 int flags,
4737 ipc_port_t port,
4738 vm_offset_t offset,
4739 boolean_t copy,
4740 vm_prot_t cur_protection,
4741 vm_prot_t max_protection,
4742 vm_inherit_t inheritance)
4743{
4744 static_assert(sizeof(vm_offset_t) == sizeof(mach_vm_offset_t));
4745
4746 return mach_vm_map(target_map, (mach_vm_offset_t *)address,
4747 size, mask, flags, port, offset, copy,
4748 cur_protection, max_protection, inheritance);
4749}
4750
4751#endif /* __x86_64__ */
4752