vm_user.c source code [xnu/osfmk/vm/vm_user.c]

1	/*
2	* Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3	*
4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5	*
6	* This file contains Original Code and/or Modifications of Original Code
7	* as defined in and that are subject to the Apple Public Source License
8	* Version 2.0 (the 'License'). You may not use this file except in
9	* compliance with the License. The rights granted to you under the License
10	* may not be used to create, or enable the creation or redistribution of,
11	* unlawful or unlicensed copies of an Apple operating system, or to
12	* circumvent, violate, or enable the circumvention or violation of, any
13	* terms of an Apple operating system software license agreement.
14	*
15	* Please obtain a copy of the License at
16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
17	*
18	* The Original Code and all software distributed under the License are
19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23	* Please see the License for the specific language governing rights and
24	* limitations under the License.
25	*
26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27	*/
28	/*
29	* @OSF_COPYRIGHT@
30	*/
31	/*
32	* Mach Operating System
33	* Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34	* All Rights Reserved.
35	*
36	* Permission to use, copy, modify and distribute this software and its
37	* documentation is hereby granted, provided that both the copyright
38	* notice and this permission notice appear in all copies of the
39	* software, derivative works or modified versions, and any portions
40	* thereof, and that both notices appear in supporting documentation.
41	*
42	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45	*
46	* Carnegie Mellon requests users of this software to return to
47	*
48	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49	* School of Computer Science
50	* Carnegie Mellon University
51	* Pittsburgh PA 15213-3890
52	*
53	* any improvements or extensions that they make and grant Carnegie Mellon
54	* the rights to redistribute these changes.
55	*/
56	/*
57	*/
58	/*
59	* File: vm/vm_user.c
60	* Author: Avadis Tevanian, Jr., Michael Wayne Young
61	*
62	* User-exported virtual memory functions.
63	*/
64
65	/*
66	* There are three implementations of the "XXX_allocate" functionality in
67	* the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
68	* (for a task with the same address space size, especially the current task),
69	* and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
70	* in the kernel should only be used on the kernel_task. vm32_vm_allocate only
71	* makes sense on platforms where a user task can either be 32 or 64, or the kernel
72	* task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
73	* for new code.
74	*
75	* The entrypoints into the kernel are more complex. All platforms support a
76	* mach_vm_allocate-style API (subsystem 4800) which operates with the largest
77	* size types for the platform. On platforms that only support U32/K32,
78	* subsystem 4800 is all you need. On platforms that support both U32 and U64,
79	* subsystem 3800 is used disambiguate the size of parameters, and they will
80	* always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
81	* the MIG glue should never call into vm_allocate directly, because the calling
82	* task and kernel_task are unlikely to use the same size parameters
83	*
84	* New VM call implementations should be added here and to mach_vm.defs
85	* (subsystem 4800), and use mach_vm_* "wide" types.
86	*/
87
88	#include <debug.h>
89
90	#include <vm_cpm.h>
91	#include <mach/boolean.h>
92	#include <mach/kern_return.h>
93	#include <mach/mach_types.h> /* to get vm_address_t */
94	#include <mach/memory_object.h>
95	#include <mach/std_types.h> /* to get pointer_t */
96	#include <mach/upl.h>
97	#include <mach/vm_attributes.h>
98	#include <mach/vm_param.h>
99	#include <mach/vm_statistics.h>
100	#include <mach/mach_syscalls.h>
101	#include <mach/sdt.h>
102
103	#include <mach/host_priv_server.h>
104	#include <mach/mach_vm_server.h>
105	#include <mach/memory_entry_server.h>
106	#include <mach/vm_map_server.h>
107
108	#include <kern/host.h>
109	#include <kern/kalloc.h>
110	#include <kern/task.h>
111	#include <kern/misc_protos.h>
112	#include <vm/vm_fault.h>
113	#include <vm/vm_map.h>
114	#include <vm/vm_object.h>
115	#include <vm/vm_page.h>
116	#include <vm/memory_object.h>
117	#include <vm/vm_pageout.h>
118	#include <vm/vm_protos.h>
119	#include <vm/vm_purgeable_internal.h>
120	#include <vm/vm_init.h>
121
122	#include <san/kasan.h>
123
124	#include <libkern/OSDebug.h>
125
126	vm_size_t upl_offset_to_pagelist = `0`;
127
128	#if VM_CPM
129	#include <vm/cpm.h>
130	#endif /* VM_CPM */
131
132	/*
133	* mach_vm_allocate allocates "zero fill" memory in the specfied
134	* map.
135	*/
136	kern_return_t
137	mach_vm_allocate_external(
138	vm_map_t map,
139	mach_vm_offset_t *addr,
140	mach_vm_size_t size,
141	int flags)
142	{
143	vm_tag_t tag;
144
145	VM_GET_FLAGS_ALIAS(flags, tag);
146	return (mach_vm_allocate_kernel(map, addr, size, flags, tag));
147	}
148
149	kern_return_t
150	mach_vm_allocate_kernel(
151	vm_map_t map,
152	mach_vm_offset_t *addr,
153	mach_vm_size_t size,
154	int flags,
155	vm_tag_t tag)
156	{
157	vm_map_offset_t map_addr;
158	vm_map_size_t map_size;
159	kern_return_t result;
160	boolean_t anywhere;
161
162	/ filter out any kernel-only flags /
163	if (flags & ~VM_FLAGS_USER_ALLOCATE)
164	return KERN_INVALID_ARGUMENT;
165
166	if (map == VM_MAP_NULL)
167	return(KERN_INVALID_ARGUMENT);
168	if (size == `0`) {
169	*addr = `0`;
170	return(KERN_SUCCESS);
171	}
172
173	anywhere = ((VM_FLAGS_ANYWHERE & flags) != `0`);
174	if (anywhere) {
175	/*
176	* No specific address requested, so start candidate address
177	* search at the minimum address in the map. However, if that
178	* minimum is 0, bump it up by PAGE_SIZE. We want to limit
179	* allocations of PAGEZERO to explicit requests since its
180	* normal use is to catch dereferences of NULL and many
181	* applications also treat pointers with a value of 0 as
182	* special and suddenly having address 0 contain useable
183	* memory would tend to confuse those applications.
184	*/
185	map_addr = vm_map_min(map);
186	if (map_addr == `0`)
187	map_addr += VM_MAP_PAGE_SIZE(map);
188	} else
189	map_addr = vm_map_trunc_page(*addr,
190	VM_MAP_PAGE_MASK(map));
191	map_size = vm_map_round_page(size,
192	VM_MAP_PAGE_MASK(map));
193	if (map_size == `0`) {
194	return(KERN_INVALID_ARGUMENT);
195	}
196
197	result = vm_map_enter(
198	map,
199	&map_addr,
200	map_size,
201	(vm_map_offset_t)`0`,
202	flags,
203	VM_MAP_KERNEL_FLAGS_NONE,
204	tag,
205	VM_OBJECT_NULL,
206	(vm_object_offset_t)`0`,
207	FALSE,
208	VM_PROT_DEFAULT,
209	VM_PROT_ALL,
210	VM_INHERIT_DEFAULT);
211
212	*addr = map_addr;
213	return(result);
214	}
215
216	/*
217	* vm_allocate
218	* Legacy routine that allocates "zero fill" memory in the specfied
219	* map (which is limited to the same size as the kernel).
220	*/
221	kern_return_t
222	vm_allocate_external(
223	vm_map_t map,
224	vm_offset_t *addr,
225	vm_size_t size,
226	int flags)
227	{
228	vm_tag_t tag;
229
230	VM_GET_FLAGS_ALIAS(flags, tag);
231	return (vm_allocate_kernel(map, addr, size, flags, tag));
232	}
233
234	kern_return_t
235	vm_allocate_kernel(
236	vm_map_t map,
237	vm_offset_t *addr,
238	vm_size_t size,
239	int flags,
240	vm_tag_t tag)
241	{
242	vm_map_offset_t map_addr;
243	vm_map_size_t map_size;
244	kern_return_t result;
245	boolean_t anywhere;
246
247	/ filter out any kernel-only flags /
248	if (flags & ~VM_FLAGS_USER_ALLOCATE)
249	return KERN_INVALID_ARGUMENT;
250
251	if (map == VM_MAP_NULL)
252	return(KERN_INVALID_ARGUMENT);
253	if (size == `0`) {
254	*addr = `0`;
255	return(KERN_SUCCESS);
256	}
257
258	anywhere = ((VM_FLAGS_ANYWHERE & flags) != `0`);
259	if (anywhere) {
260	/*
261	* No specific address requested, so start candidate address
262	* search at the minimum address in the map. However, if that
263	* minimum is 0, bump it up by PAGE_SIZE. We want to limit
264	* allocations of PAGEZERO to explicit requests since its
265	* normal use is to catch dereferences of NULL and many
266	* applications also treat pointers with a value of 0 as
267	* special and suddenly having address 0 contain useable
268	* memory would tend to confuse those applications.
269	*/
270	map_addr = vm_map_min(map);
271	if (map_addr == `0`)
272	map_addr += VM_MAP_PAGE_SIZE(map);
273	} else
274	map_addr = vm_map_trunc_page(*addr,
275	VM_MAP_PAGE_MASK(map));
276	map_size = vm_map_round_page(size,
277	VM_MAP_PAGE_MASK(map));
278	if (map_size == `0`) {
279	return(KERN_INVALID_ARGUMENT);
280	}
281
282	result = vm_map_enter(
283	map,
284	&map_addr,
285	map_size,
286	(vm_map_offset_t)`0`,
287	flags,
288	VM_MAP_KERNEL_FLAGS_NONE,
289	tag,
290	VM_OBJECT_NULL,
291	(vm_object_offset_t)`0`,
292	FALSE,
293	VM_PROT_DEFAULT,
294	VM_PROT_ALL,
295	VM_INHERIT_DEFAULT);
296
297	#if KASAN
298	if (result == KERN_SUCCESS && map->pmap == kernel_pmap) {
299	kasan_notify_address(map_addr, map_size);
300	}
301	#endif
302
303	*addr = CAST_DOWN(vm_offset_t, map_addr);
304	return(result);
305	}
306
307	/*
308	* mach_vm_deallocate -
309	* deallocates the specified range of addresses in the
310	* specified address map.
311	*/
312	kern_return_t
313	mach_vm_deallocate(
314	vm_map_t map,
315	mach_vm_offset_t start,
316	mach_vm_size_t size)
317	{
318	if ((map == VM_MAP_NULL) \|\| (start + size < start))
319	return(KERN_INVALID_ARGUMENT);
320
321	if (size == (mach_vm_offset_t) `0`)
322	return(KERN_SUCCESS);
323
324	return vm_map_remove(map,
325	vm_map_trunc_page(start,
326	VM_MAP_PAGE_MASK(map)),
327	vm_map_round_page(start+size,
328	VM_MAP_PAGE_MASK(map)),
329	VM_MAP_REMOVE_NO_FLAGS);
330	}
331
332	/*
333	* vm_deallocate -
334	* deallocates the specified range of addresses in the
335	* specified address map (limited to addresses the same
336	* size as the kernel).
337	*/
338	kern_return_t
339	vm_deallocate(
340	vm_map_t map,
341	vm_offset_t start,
342	vm_size_t size)
343	{
344	if ((map == VM_MAP_NULL) \|\| (start + size < start))
345	return(KERN_INVALID_ARGUMENT);
346
347	if (size == (vm_offset_t) `0`)
348	return(KERN_SUCCESS);
349
350	return vm_map_remove(map,
351	vm_map_trunc_page(start,
352	VM_MAP_PAGE_MASK(map)),
353	vm_map_round_page(start+size,
354	VM_MAP_PAGE_MASK(map)),
355	VM_MAP_REMOVE_NO_FLAGS);
356	}
357
358	/*
359	* mach_vm_inherit -
360	* Sets the inheritance of the specified range in the
361	* specified map.
362	*/
363	kern_return_t
364	mach_vm_inherit(
365	vm_map_t map,
366	mach_vm_offset_t start,
367	mach_vm_size_t size,
368	vm_inherit_t new_inheritance)
369	{
370	if ((map == VM_MAP_NULL) \|\| (start + size < start) \|\|
371	(new_inheritance > VM_INHERIT_LAST_VALID))
372	return(KERN_INVALID_ARGUMENT);
373
374	if (size == `0`)
375	return KERN_SUCCESS;
376
377	return(vm_map_inherit(map,
378	vm_map_trunc_page(start,
379	VM_MAP_PAGE_MASK(map)),
380	vm_map_round_page(start+size,
381	VM_MAP_PAGE_MASK(map)),
382	new_inheritance));
383	}
384
385	/*
386	* vm_inherit -
387	* Sets the inheritance of the specified range in the
388	* specified map (range limited to addresses
389	*/
390	kern_return_t
391	vm_inherit(
392	vm_map_t map,
393	vm_offset_t start,
394	vm_size_t size,
395	vm_inherit_t new_inheritance)
396	{
397	if ((map == VM_MAP_NULL) \|\| (start + size < start) \|\|
398	(new_inheritance > VM_INHERIT_LAST_VALID))
399	return(KERN_INVALID_ARGUMENT);
400
401	if (size == `0`)
402	return KERN_SUCCESS;
403
404	return(vm_map_inherit(map,
405	vm_map_trunc_page(start,
406	VM_MAP_PAGE_MASK(map)),
407	vm_map_round_page(start+size,
408	VM_MAP_PAGE_MASK(map)),
409	new_inheritance));
410	}
411
412	/*
413	* mach_vm_protect -
414	* Sets the protection of the specified range in the
415	* specified map.
416	*/
417
418	kern_return_t
419	mach_vm_protect(
420	vm_map_t map,
421	mach_vm_offset_t start,
422	mach_vm_size_t size,
423	boolean_t set_maximum,
424	vm_prot_t new_protection)
425	{
426	if ((map == VM_MAP_NULL) \|\| (start + size < start) \|\|
427	(new_protection & ~(VM_PROT_ALL \| VM_PROT_COPY)))
428	return(KERN_INVALID_ARGUMENT);
429
430	if (size == `0`)
431	return KERN_SUCCESS;
432
433	return(vm_map_protect(map,
434	vm_map_trunc_page(start,
435	VM_MAP_PAGE_MASK(map)),
436	vm_map_round_page(start+size,
437	VM_MAP_PAGE_MASK(map)),
438	new_protection,
439	set_maximum));
440	}
441
442	/*
443	* vm_protect -
444	* Sets the protection of the specified range in the
445	* specified map. Addressability of the range limited
446	* to the same size as the kernel.
447	*/
448
449	kern_return_t
450	vm_protect(
451	vm_map_t map,
452	vm_offset_t start,
453	vm_size_t size,
454	boolean_t set_maximum,
455	vm_prot_t new_protection)
456	{
457	if ((map == VM_MAP_NULL) \|\| (start + size < start) \|\|
458	(new_protection & ~(VM_PROT_ALL \| VM_PROT_COPY)))
459	return(KERN_INVALID_ARGUMENT);
460
461	if (size == `0`)
462	return KERN_SUCCESS;
463
464	return(vm_map_protect(map,
465	vm_map_trunc_page(start,
466	VM_MAP_PAGE_MASK(map)),
467	vm_map_round_page(start+size,
468	VM_MAP_PAGE_MASK(map)),
469	new_protection,
470	set_maximum));
471	}
472
473	/*
474	* mach_vm_machine_attributes -
475	* Handle machine-specific attributes for a mapping, such
476	* as cachability, migrability, etc.
477	*/
478	kern_return_t
479	mach_vm_machine_attribute(
480	vm_map_t map,
481	mach_vm_address_t addr,
482	mach_vm_size_t size,
483	vm_machine_attribute_t attribute,
484	vm_machine_attribute_val_t* value) / IN/OUT /
485	{
486	if ((map == VM_MAP_NULL) \|\| (addr + size < addr))
487	return(KERN_INVALID_ARGUMENT);
488
489	if (size == `0`)
490	return KERN_SUCCESS;
491
492	return vm_map_machine_attribute(
493	map,
494	vm_map_trunc_page(addr,
495	VM_MAP_PAGE_MASK(map)),
496	vm_map_round_page(addr+size,
497	VM_MAP_PAGE_MASK(map)),
498	attribute,
499	value);
500	}
501
502	/*
503	* vm_machine_attribute -
504	* Handle machine-specific attributes for a mapping, such
505	* as cachability, migrability, etc. Limited addressability
506	* (same range limits as for the native kernel map).
507	*/
508	kern_return_t
509	vm_machine_attribute(
510	vm_map_t map,
511	vm_address_t addr,
512	vm_size_t size,
513	vm_machine_attribute_t attribute,
514	vm_machine_attribute_val_t* value) / IN/OUT /
515	{
516	if ((map == VM_MAP_NULL) \|\| (addr + size < addr))
517	return(KERN_INVALID_ARGUMENT);
518
519	if (size == `0`)
520	return KERN_SUCCESS;
521
522	return vm_map_machine_attribute(
523	map,
524	vm_map_trunc_page(addr,
525	VM_MAP_PAGE_MASK(map)),
526	vm_map_round_page(addr+size,
527	VM_MAP_PAGE_MASK(map)),
528	attribute,
529	value);
530	}
531
532	/*
533	* mach_vm_read -
534	* Read/copy a range from one address space and return it to the caller.
535	*
536	* It is assumed that the address for the returned memory is selected by
537	* the IPC implementation as part of receiving the reply to this call.
538	* If IPC isn't used, the caller must deal with the vm_map_copy_t object
539	* that gets returned.
540	*
541	* JMM - because of mach_msg_type_number_t, this call is limited to a
542	* single 4GB region at this time.
543	*
544	*/
545	kern_return_t
546	mach_vm_read(
547	vm_map_t map,
548	mach_vm_address_t addr,
549	mach_vm_size_t size,
550	pointer_t *data,
551	mach_msg_type_number_t *data_size)
552	{
553	kern_return_t error;
554	vm_map_copy_t ipc_address;
555
556	if (map == VM_MAP_NULL)
557	return(KERN_INVALID_ARGUMENT);
558
559	if ((mach_msg_type_number_t) size != size)
560	return KERN_INVALID_ARGUMENT;
561
562	error = vm_map_copyin(map,
563	(vm_map_address_t)addr,
564	(vm_map_size_t)size,
565	FALSE, / src_destroy /
566	&ipc_address);
567
568	if (KERN_SUCCESS == error) {
569	*data = (pointer_t) ipc_address;
570	*data_size = (mach_msg_type_number_t) size;
571	assert(*data_size == size);
572	}
573	return(error);
574	}
575
576	/*
577	* vm_read -
578	* Read/copy a range from one address space and return it to the caller.
579	* Limited addressability (same range limits as for the native kernel map).
580	*
581	* It is assumed that the address for the returned memory is selected by
582	* the IPC implementation as part of receiving the reply to this call.
583	* If IPC isn't used, the caller must deal with the vm_map_copy_t object
584	* that gets returned.
585	*/
586	kern_return_t
587	vm_read(
588	vm_map_t map,
589	vm_address_t addr,
590	vm_size_t size,
591	pointer_t *data,
592	mach_msg_type_number_t *data_size)
593	{
594	kern_return_t error;
595	vm_map_copy_t ipc_address;
596
597	if (map == VM_MAP_NULL)
598	return(KERN_INVALID_ARGUMENT);
599
600	mach_msg_type_number_t dsize;
601	if (os_convert_overflow(size, &dsize)) {
602	/*
603	* The kernel could handle a 64-bit "size" value, but
604	* it could not return the size of the data in "*data_size"
605	* without overflowing.
606	* Let's reject this "size" as invalid.
607	*/
608	return KERN_INVALID_ARGUMENT;
609	}
610
611	error = vm_map_copyin(map,
612	(vm_map_address_t)addr,
613	(vm_map_size_t)size,
614	FALSE, / src_destroy /
615	&ipc_address);
616
617	if (KERN_SUCCESS == error) {
618	*data = (pointer_t) ipc_address;
619	*data_size = dsize;
620	assert(*data_size == size);
621	}
622	return(error);
623	}
624
625	/*
626	* mach_vm_read_list -
627	* Read/copy a list of address ranges from specified map.
628	*
629	* MIG does not know how to deal with a returned array of
630	* vm_map_copy_t structures, so we have to do the copyout
631	* manually here.
632	*/
633	kern_return_t
634	mach_vm_read_list(
635	vm_map_t map,
636	mach_vm_read_entry_t data_list,
637	natural_t count)
638	{
639	mach_msg_type_number_t i;
640	kern_return_t error;
641	vm_map_copy_t copy;
642
643	if (map == VM_MAP_NULL \|\|
644	count > VM_MAP_ENTRY_MAX)
645	return(KERN_INVALID_ARGUMENT);
646
647	error = KERN_SUCCESS;
648	for(i=`0`; i<count; i++) {
649	vm_map_address_t map_addr;
650	vm_map_size_t map_size;
651
652	map_addr = (vm_map_address_t)(data_list[i].address);
653	map_size = (vm_map_size_t)(data_list[i].size);
654
655	if(map_size != `0`) {
656	error = vm_map_copyin(map,
657	map_addr,
658	map_size,
659	FALSE, / src_destroy /
660	&copy);
661	if (KERN_SUCCESS == error) {
662	error = vm_map_copyout(
663	current_task()->map,
664	&map_addr,
665	copy);
666	if (KERN_SUCCESS == error) {
667	data_list[i].address = map_addr;
668	continue;
669	}
670	vm_map_copy_discard(copy);
671	}
672	}
673	data_list[i].address = (mach_vm_address_t)`0`;
674	data_list[i].size = (mach_vm_size_t)`0`;
675	}
676	return(error);
677	}
678
679	/*
680	* vm_read_list -
681	* Read/copy a list of address ranges from specified map.
682	*
683	* MIG does not know how to deal with a returned array of
684	* vm_map_copy_t structures, so we have to do the copyout
685	* manually here.
686	*
687	* The source and destination ranges are limited to those
688	* that can be described with a vm_address_t (i.e. same
689	* size map as the kernel).
690	*
691	* JMM - If the result of the copyout is an address range
692	* that cannot be described with a vm_address_t (i.e. the
693	* caller had a larger address space but used this call
694	* anyway), it will result in a truncated address being
695	* returned (and a likely confused caller).
696	*/
697
698	kern_return_t
699	vm_read_list(
700	vm_map_t map,
701	vm_read_entry_t data_list,
702	natural_t count)
703	{
704	mach_msg_type_number_t i;
705	kern_return_t error;
706	vm_map_copy_t copy;
707
708	if (map == VM_MAP_NULL \|\|
709	count > VM_MAP_ENTRY_MAX)
710	return(KERN_INVALID_ARGUMENT);
711
712	error = KERN_SUCCESS;
713	for(i=`0`; i<count; i++) {
714	vm_map_address_t map_addr;
715	vm_map_size_t map_size;
716
717	map_addr = (vm_map_address_t)(data_list[i].address);
718	map_size = (vm_map_size_t)(data_list[i].size);
719
720	if(map_size != `0`) {
721	error = vm_map_copyin(map,
722	map_addr,
723	map_size,
724	FALSE, / src_destroy /
725	&copy);
726	if (KERN_SUCCESS == error) {
727	error = vm_map_copyout(current_task()->map,
728	&map_addr,
729	copy);
730	if (KERN_SUCCESS == error) {
731	data_list[i].address =
732	CAST_DOWN(vm_offset_t, map_addr);
733	continue;
734	}
735	vm_map_copy_discard(copy);
736	}
737	}
738	data_list[i].address = (mach_vm_address_t)`0`;
739	data_list[i].size = (mach_vm_size_t)`0`;
740	}
741	return(error);
742	}
743
744	/*
745	* mach_vm_read_overwrite -
746	* Overwrite a range of the current map with data from the specified
747	* map/address range.
748	*
749	* In making an assumption that the current thread is local, it is
750	* no longer cluster-safe without a fully supportive local proxy
751	* thread/task (but we don't support cluster's anymore so this is moot).
752	*/
753
754	kern_return_t
755	mach_vm_read_overwrite(
756	vm_map_t map,
757	mach_vm_address_t address,
758	mach_vm_size_t size,
759	mach_vm_address_t data,
760	mach_vm_size_t *data_size)
761	{
762	kern_return_t error;
763	vm_map_copy_t copy;
764
765	if (map == VM_MAP_NULL)
766	return(KERN_INVALID_ARGUMENT);
767
768	error = vm_map_copyin(map, (vm_map_address_t)address,
769	(vm_map_size_t)size, FALSE, &copy);
770
771	if (KERN_SUCCESS == error) {
772	error = vm_map_copy_overwrite(current_thread()->map,
773	(vm_map_address_t)data,
774	copy, FALSE);
775	if (KERN_SUCCESS == error) {
776	*data_size = size;
777	return error;
778	}
779	vm_map_copy_discard(copy);
780	}
781	return(error);
782	}
783
784	/*
785	* vm_read_overwrite -
786	* Overwrite a range of the current map with data from the specified
787	* map/address range.
788	*
789	* This routine adds the additional limitation that the source and
790	* destination ranges must be describable with vm_address_t values
791	* (i.e. the same size address spaces as the kernel, or at least the
792	* the ranges are in that first portion of the respective address
793	* spaces).
794	*/
795
796	kern_return_t
797	vm_read_overwrite(
798	vm_map_t map,
799	vm_address_t address,
800	vm_size_t size,
801	vm_address_t data,
802	vm_size_t *data_size)
803	{
804	kern_return_t error;
805	vm_map_copy_t copy;
806
807	if (map == VM_MAP_NULL)
808	return(KERN_INVALID_ARGUMENT);
809
810	error = vm_map_copyin(map, (vm_map_address_t)address,
811	(vm_map_size_t)size, FALSE, &copy);
812
813	if (KERN_SUCCESS == error) {
814	error = vm_map_copy_overwrite(current_thread()->map,
815	(vm_map_address_t)data,
816	copy, FALSE);
817	if (KERN_SUCCESS == error) {
818	*data_size = size;
819	return error;
820	}
821	vm_map_copy_discard(copy);
822	}
823	return(error);
824	}
825
826
827	/*
828	* mach_vm_write -
829	* Overwrite the specified address range with the data provided
830	* (from the current map).
831	*/
832	kern_return_t
833	mach_vm_write(
834	vm_map_t map,
835	mach_vm_address_t address,
836	pointer_t data,
837	__unused mach_msg_type_number_t size)
838	{
839	if (map == VM_MAP_NULL)
840	return KERN_INVALID_ARGUMENT;
841
842	return vm_map_copy_overwrite(map, (vm_map_address_t)address,
843	(vm_map_copy_t) data, FALSE / interruptible XXX /);
844	}
845
846	/*
847	* vm_write -
848	* Overwrite the specified address range with the data provided
849	* (from the current map).
850	*
851	* The addressability of the range of addresses to overwrite is
852	* limited bu the use of a vm_address_t (same size as kernel map).
853	* Either the target map is also small, or the range is in the
854	* low addresses within it.
855	*/
856	kern_return_t
857	vm_write(
858	vm_map_t map,
859	vm_address_t address,
860	pointer_t data,
861	__unused mach_msg_type_number_t size)
862	{
863	if (map == VM_MAP_NULL)
864	return KERN_INVALID_ARGUMENT;
865
866	return vm_map_copy_overwrite(map, (vm_map_address_t)address,
867	(vm_map_copy_t) data, FALSE / interruptible XXX /);
868	}
869
870	/*
871	* mach_vm_copy -
872	* Overwrite one range of the specified map with the contents of
873	* another range within that same map (i.e. both address ranges
874	* are "over there").
875	*/
876	kern_return_t
877	mach_vm_copy(
878	vm_map_t map,
879	mach_vm_address_t source_address,
880	mach_vm_size_t size,
881	mach_vm_address_t dest_address)
882	{
883	vm_map_copy_t copy;
884	kern_return_t kr;
885
886	if (map == VM_MAP_NULL)
887	return KERN_INVALID_ARGUMENT;
888
889	kr = vm_map_copyin(map, (vm_map_address_t)source_address,
890	(vm_map_size_t)size, FALSE, &copy);
891
892	if (KERN_SUCCESS == kr) {
893	kr = vm_map_copy_overwrite(map,
894	(vm_map_address_t)dest_address,
895	copy, FALSE / interruptible XXX /);
896
897	if (KERN_SUCCESS != kr)
898	vm_map_copy_discard(copy);
899	}
900	return kr;
901	}
902
903	kern_return_t
904	vm_copy(
905	vm_map_t map,
906	vm_address_t source_address,
907	vm_size_t size,
908	vm_address_t dest_address)
909	{
910	vm_map_copy_t copy;
911	kern_return_t kr;
912
913	if (map == VM_MAP_NULL)
914	return KERN_INVALID_ARGUMENT;
915
916	kr = vm_map_copyin(map, (vm_map_address_t)source_address,
917	(vm_map_size_t)size, FALSE, &copy);
918
919	if (KERN_SUCCESS == kr) {
920	kr = vm_map_copy_overwrite(map,
921	(vm_map_address_t)dest_address,
922	copy, FALSE / interruptible XXX /);
923
924	if (KERN_SUCCESS != kr)
925	vm_map_copy_discard(copy);
926	}
927	return kr;
928	}
929
930	/*
931	* mach_vm_map -
932	* Map some range of an object into an address space.
933	*
934	* The object can be one of several types of objects:
935	* NULL - anonymous memory
936	* a named entry - a range within another address space
937	* or a range within a memory object
938	* a whole memory object
939	*
940	*/
941	kern_return_t
942	mach_vm_map_external(
943	vm_map_t target_map,
944	mach_vm_offset_t *address,
945	mach_vm_size_t initial_size,
946	mach_vm_offset_t mask,
947	int flags,
948	ipc_port_t port,
949	vm_object_offset_t offset,
950	boolean_t copy,
951	vm_prot_t cur_protection,
952	vm_prot_t max_protection,
953	vm_inherit_t inheritance)
954	{
955	vm_tag_t tag;
956
957	VM_GET_FLAGS_ALIAS(flags, tag);
958	return (mach_vm_map_kernel(target_map, address, initial_size, mask,
959	flags, VM_MAP_KERNEL_FLAGS_NONE, tag,
960	port, offset, copy,
961	cur_protection, max_protection,
962	inheritance));
963	}
964
965	kern_return_t
966	mach_vm_map_kernel(
967	vm_map_t target_map,
968	mach_vm_offset_t *address,
969	mach_vm_size_t initial_size,
970	mach_vm_offset_t mask,
971	int flags,
972	vm_map_kernel_flags_t vmk_flags,
973	vm_tag_t tag,
974	ipc_port_t port,
975	vm_object_offset_t offset,
976	boolean_t copy,
977	vm_prot_t cur_protection,
978	vm_prot_t max_protection,
979	vm_inherit_t inheritance)
980	{
981	kern_return_t kr;
982	vm_map_offset_t vmmaddr;
983
984	vmmaddr = (vm_map_offset_t) *address;
985
986	/ filter out any kernel-only flags /
987	if (flags & ~VM_FLAGS_USER_MAP)
988	return KERN_INVALID_ARGUMENT;
989
990	kr = vm_map_enter_mem_object(target_map,
991	&vmmaddr,
992	initial_size,
993	mask,
994	flags,
995	vmk_flags,
996	tag,
997	port,
998	offset,
999	copy,
1000	cur_protection,
1001	max_protection,
1002	inheritance);
1003
1004	#if KASAN
1005	if (kr == KERN_SUCCESS && target_map->pmap == kernel_pmap) {
1006	kasan_notify_address(vmmaddr, initial_size);
1007	}
1008	#endif
1009
1010	*address = vmmaddr;
1011	return kr;
1012	}
1013
1014
1015	/ legacy interface /
1016	kern_return_t
1017	vm_map_64_external(
1018	vm_map_t target_map,
1019	vm_offset_t *address,
1020	vm_size_t size,
1021	vm_offset_t mask,
1022	int flags,
1023	ipc_port_t port,
1024	vm_object_offset_t offset,
1025	boolean_t copy,
1026	vm_prot_t cur_protection,
1027	vm_prot_t max_protection,
1028	vm_inherit_t inheritance)
1029	{
1030	vm_tag_t tag;
1031
1032	VM_GET_FLAGS_ALIAS(flags, tag);
1033	return (vm_map_64_kernel(target_map, address, size, mask,
1034	flags, VM_MAP_KERNEL_FLAGS_NONE,
1035	tag, port, offset, copy,
1036	cur_protection, max_protection,
1037	inheritance));
1038	}
1039
1040	kern_return_t
1041	vm_map_64_kernel(
1042	vm_map_t target_map,
1043	vm_offset_t *address,
1044	vm_size_t size,
1045	vm_offset_t mask,
1046	int flags,
1047	vm_map_kernel_flags_t vmk_flags,
1048	vm_tag_t tag,
1049	ipc_port_t port,
1050	vm_object_offset_t offset,
1051	boolean_t copy,
1052	vm_prot_t cur_protection,
1053	vm_prot_t max_protection,
1054	vm_inherit_t inheritance)
1055	{
1056	mach_vm_address_t map_addr;
1057	mach_vm_size_t map_size;
1058	mach_vm_offset_t map_mask;
1059	kern_return_t kr;
1060
1061	map_addr = (mach_vm_address_t)*address;
1062	map_size = (mach_vm_size_t)size;
1063	map_mask = (mach_vm_offset_t)mask;
1064
1065	kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask,
1066	flags, vmk_flags, tag,
1067	port, offset, copy,
1068	cur_protection, max_protection, inheritance);
1069	*address = CAST_DOWN(vm_offset_t, map_addr);
1070	return kr;
1071	}
1072
1073	/ temporary, until world build /
1074	kern_return_t
1075	vm_map_external(
1076	vm_map_t target_map,
1077	vm_offset_t *address,
1078	vm_size_t size,
1079	vm_offset_t mask,
1080	int flags,
1081	ipc_port_t port,
1082	vm_offset_t offset,
1083	boolean_t copy,
1084	vm_prot_t cur_protection,
1085	vm_prot_t max_protection,
1086	vm_inherit_t inheritance)
1087	{
1088	vm_tag_t tag;
1089
1090	VM_GET_FLAGS_ALIAS(flags, tag);
1091	return (vm_map_kernel(target_map, address, size, mask,
1092	flags, VM_MAP_KERNEL_FLAGS_NONE, tag,
1093	port, offset, copy,
1094	cur_protection, max_protection, inheritance));
1095	}
1096
1097	kern_return_t
1098	vm_map_kernel(
1099	vm_map_t target_map,
1100	vm_offset_t *address,
1101	vm_size_t size,
1102	vm_offset_t mask,
1103	int flags,
1104	vm_map_kernel_flags_t vmk_flags,
1105	vm_tag_t tag,
1106	ipc_port_t port,
1107	vm_offset_t offset,
1108	boolean_t copy,
1109	vm_prot_t cur_protection,
1110	vm_prot_t max_protection,
1111	vm_inherit_t inheritance)
1112	{
1113	mach_vm_address_t map_addr;
1114	mach_vm_size_t map_size;
1115	mach_vm_offset_t map_mask;
1116	vm_object_offset_t obj_offset;
1117	kern_return_t kr;
1118
1119	map_addr = (mach_vm_address_t)*address;
1120	map_size = (mach_vm_size_t)size;
1121	map_mask = (mach_vm_offset_t)mask;
1122	obj_offset = (vm_object_offset_t)offset;
1123
1124	kr = mach_vm_map_kernel(target_map, &map_addr, map_size, map_mask,
1125	flags, vmk_flags, tag,
1126	port, obj_offset, copy,
1127	cur_protection, max_protection, inheritance);
1128	*address = CAST_DOWN(vm_offset_t, map_addr);
1129	return kr;
1130	}
1131
1132	/*
1133	* mach_vm_remap -
1134	* Remap a range of memory from one task into another,
1135	* to another address range within the same task, or
1136	* over top of itself (with altered permissions and/or
1137	* as an in-place copy of itself).
1138	*/
1139	kern_return_t
1140	mach_vm_remap_external(
1141	vm_map_t target_map,
1142	mach_vm_offset_t *address,
1143	mach_vm_size_t size,
1144	mach_vm_offset_t mask,
1145	int flags,
1146	vm_map_t src_map,
1147	mach_vm_offset_t memory_address,
1148	boolean_t copy,
1149	vm_prot_t *cur_protection,
1150	vm_prot_t *max_protection,
1151	vm_inherit_t inheritance)
1152	{
1153	vm_tag_t tag;
1154	VM_GET_FLAGS_ALIAS(flags, tag);
1155
1156	return (mach_vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map, memory_address,
1157	copy, cur_protection, max_protection, inheritance));
1158	}
1159
1160	kern_return_t
1161	mach_vm_remap_kernel(
1162	vm_map_t target_map,
1163	mach_vm_offset_t *address,
1164	mach_vm_size_t size,
1165	mach_vm_offset_t mask,
1166	int flags,
1167	vm_tag_t tag,
1168	vm_map_t src_map,
1169	mach_vm_offset_t memory_address,
1170	boolean_t copy,
1171	vm_prot_t *cur_protection,
1172	vm_prot_t *max_protection,
1173	vm_inherit_t inheritance)
1174	{
1175	vm_map_offset_t map_addr;
1176	kern_return_t kr;
1177
1178	if (VM_MAP_NULL == target_map \|\| VM_MAP_NULL == src_map)
1179	return KERN_INVALID_ARGUMENT;
1180
1181	/ filter out any kernel-only flags /
1182	if (flags & ~VM_FLAGS_USER_REMAP)
1183	return KERN_INVALID_ARGUMENT;
1184
1185	map_addr = (vm_map_offset_t)*address;
1186
1187	kr = vm_map_remap(target_map,
1188	&map_addr,
1189	size,
1190	mask,
1191	flags,
1192	VM_MAP_KERNEL_FLAGS_NONE,
1193	tag,
1194	src_map,
1195	memory_address,
1196	copy,
1197	cur_protection,
1198	max_protection,
1199	inheritance);
1200	*address = map_addr;
1201	return kr;
1202	}
1203
1204	/*
1205	* vm_remap -
1206	* Remap a range of memory from one task into another,
1207	* to another address range within the same task, or
1208	* over top of itself (with altered permissions and/or
1209	* as an in-place copy of itself).
1210	*
1211	* The addressability of the source and target address
1212	* range is limited by the size of vm_address_t (in the
1213	* kernel context).
1214	*/
1215	kern_return_t
1216	vm_remap_external(
1217	vm_map_t target_map,
1218	vm_offset_t *address,
1219	vm_size_t size,
1220	vm_offset_t mask,
1221	int flags,
1222	vm_map_t src_map,
1223	vm_offset_t memory_address,
1224	boolean_t copy,
1225	vm_prot_t *cur_protection,
1226	vm_prot_t *max_protection,
1227	vm_inherit_t inheritance)
1228	{
1229	vm_tag_t tag;
1230	VM_GET_FLAGS_ALIAS(flags, tag);
1231
1232	return (vm_remap_kernel(target_map, address, size, mask, flags, tag, src_map,
1233	memory_address, copy, cur_protection, max_protection, inheritance));
1234	}
1235
1236	kern_return_t
1237	vm_remap_kernel(
1238	vm_map_t target_map,
1239	vm_offset_t *address,
1240	vm_size_t size,
1241	vm_offset_t mask,
1242	int flags,
1243	vm_tag_t tag,
1244	vm_map_t src_map,
1245	vm_offset_t memory_address,
1246	boolean_t copy,
1247	vm_prot_t *cur_protection,
1248	vm_prot_t *max_protection,
1249	vm_inherit_t inheritance)
1250	{
1251	vm_map_offset_t map_addr;
1252	kern_return_t kr;
1253
1254	if (VM_MAP_NULL == target_map \|\| VM_MAP_NULL == src_map)
1255	return KERN_INVALID_ARGUMENT;
1256
1257	/ filter out any kernel-only flags /
1258	if (flags & ~VM_FLAGS_USER_REMAP)
1259	return KERN_INVALID_ARGUMENT;
1260
1261	map_addr = (vm_map_offset_t)*address;
1262
1263	kr = vm_map_remap(target_map,
1264	&map_addr,
1265	size,
1266	mask,
1267	flags,
1268	VM_MAP_KERNEL_FLAGS_NONE,
1269	tag,
1270	src_map,
1271	memory_address,
1272	copy,
1273	cur_protection,
1274	max_protection,
1275	inheritance);
1276	*address = CAST_DOWN(vm_offset_t, map_addr);
1277	return kr;
1278	}
1279
1280	/*
1281	* NOTE: these routine (and this file) will no longer require mach_host_server.h
1282	* when mach_vm_wire and vm_wire are changed to use ledgers.
1283	*/
1284	#include <mach/mach_host_server.h>
1285	/*
1286	* mach_vm_wire
1287	* Specify that the range of the virtual address space
1288	* of the target task must not cause page faults for
1289	* the indicated accesses.
1290	*
1291	* [ To unwire the pages, specify VM_PROT_NONE. ]
1292	*/
1293	kern_return_t
1294	mach_vm_wire_external(
1295	host_priv_t host_priv,
1296	vm_map_t map,
1297	mach_vm_offset_t start,
1298	mach_vm_size_t size,
1299	vm_prot_t access)
1300	{
1301	return (mach_vm_wire_kernel(host_priv, map, start, size, access, VM_KERN_MEMORY_MLOCK));
1302	}
1303
1304	kern_return_t
1305	mach_vm_wire_kernel(
1306	host_priv_t host_priv,
1307	vm_map_t map,
1308	mach_vm_offset_t start,
1309	mach_vm_size_t size,
1310	vm_prot_t access,
1311	vm_tag_t tag)
1312	{
1313	kern_return_t rc;
1314
1315	if (host_priv == HOST_PRIV_NULL)
1316	return KERN_INVALID_HOST;
1317
1318	assert(host_priv == &realhost);
1319
1320	if (map == VM_MAP_NULL)
1321	return KERN_INVALID_TASK;
1322
1323	if (access & ~VM_PROT_ALL \|\| (start + size < start))
1324	return KERN_INVALID_ARGUMENT;
1325
1326	if (access != VM_PROT_NONE) {
1327	rc = vm_map_wire_kernel(map,
1328	vm_map_trunc_page(start,
1329	VM_MAP_PAGE_MASK(map)),
1330	vm_map_round_page(start+size,
1331	VM_MAP_PAGE_MASK(map)),
1332	access, tag,
1333	TRUE);
1334	} else {
1335	rc = vm_map_unwire(map,
1336	vm_map_trunc_page(start,
1337	VM_MAP_PAGE_MASK(map)),
1338	vm_map_round_page(start+size,
1339	VM_MAP_PAGE_MASK(map)),
1340	TRUE);
1341	}
1342	return rc;
1343	}
1344
1345	/*
1346	* vm_wire -
1347	* Specify that the range of the virtual address space
1348	* of the target task must not cause page faults for
1349	* the indicated accesses.
1350	*
1351	* [ To unwire the pages, specify VM_PROT_NONE. ]
1352	*/
1353	kern_return_t
1354	vm_wire(
1355	host_priv_t host_priv,
1356	vm_map_t map,
1357	vm_offset_t start,
1358	vm_size_t size,
1359	vm_prot_t access)
1360	{
1361	kern_return_t rc;
1362
1363	if (host_priv == HOST_PRIV_NULL)
1364	return KERN_INVALID_HOST;
1365
1366	assert(host_priv == &realhost);
1367
1368	if (map == VM_MAP_NULL)
1369	return KERN_INVALID_TASK;
1370
1371	if ((access & ~VM_PROT_ALL) \|\| (start + size < start))
1372	return KERN_INVALID_ARGUMENT;
1373
1374	if (size == `0`) {
1375	rc = KERN_SUCCESS;
1376	} else if (access != VM_PROT_NONE) {
1377	rc = vm_map_wire_kernel(map,
1378	vm_map_trunc_page(start,
1379	VM_MAP_PAGE_MASK(map)),
1380	vm_map_round_page(start+size,
1381	VM_MAP_PAGE_MASK(map)),
1382	access, VM_KERN_MEMORY_OSFMK,
1383	TRUE);
1384	} else {
1385	rc = vm_map_unwire(map,
1386	vm_map_trunc_page(start,
1387	VM_MAP_PAGE_MASK(map)),
1388	vm_map_round_page(start+size,
1389	VM_MAP_PAGE_MASK(map)),
1390	TRUE);
1391	}
1392	return rc;
1393	}
1394
1395	/*
1396	* vm_msync
1397	*
1398	* Synchronises the memory range specified with its backing store
1399	* image by either flushing or cleaning the contents to the appropriate
1400	* memory manager.
1401	*
1402	* interpretation of sync_flags
1403	* VM_SYNC_INVALIDATE - discard pages, only return precious
1404	* pages to manager.
1405	*
1406	* VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS \| VM_SYNC_ASYNCHRONOUS)
1407	* - discard pages, write dirty or precious
1408	* pages back to memory manager.
1409	*
1410	* VM_SYNC_SYNCHRONOUS \| VM_SYNC_ASYNCHRONOUS
1411	* - write dirty or precious pages back to
1412	* the memory manager.
1413	*
1414	* VM_SYNC_CONTIGUOUS - does everything normally, but if there
1415	* is a hole in the region, and we would
1416	* have returned KERN_SUCCESS, return
1417	* KERN_INVALID_ADDRESS instead.
1418	*
1419	* RETURNS
1420	* KERN_INVALID_TASK Bad task parameter
1421	* KERN_INVALID_ARGUMENT both sync and async were specified.
1422	* KERN_SUCCESS The usual.
1423	* KERN_INVALID_ADDRESS There was a hole in the region.
1424	*/
1425
1426	kern_return_t
1427	mach_vm_msync(
1428	vm_map_t map,
1429	mach_vm_address_t address,
1430	mach_vm_size_t size,
1431	vm_sync_t sync_flags)
1432	{
1433
1434	if (map == VM_MAP_NULL)
1435	return(KERN_INVALID_TASK);
1436
1437	return vm_map_msync(map, (vm_map_address_t)address,
1438	(vm_map_size_t)size, sync_flags);
1439	}
1440
1441	/*
1442	* vm_msync
1443	*
1444	* Synchronises the memory range specified with its backing store
1445	* image by either flushing or cleaning the contents to the appropriate
1446	* memory manager.
1447	*
1448	* interpretation of sync_flags
1449	* VM_SYNC_INVALIDATE - discard pages, only return precious
1450	* pages to manager.
1451	*
1452	* VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS \| VM_SYNC_ASYNCHRONOUS)
1453	* - discard pages, write dirty or precious
1454	* pages back to memory manager.
1455	*
1456	* VM_SYNC_SYNCHRONOUS \| VM_SYNC_ASYNCHRONOUS
1457	* - write dirty or precious pages back to
1458	* the memory manager.
1459	*
1460	* VM_SYNC_CONTIGUOUS - does everything normally, but if there
1461	* is a hole in the region, and we would
1462	* have returned KERN_SUCCESS, return
1463	* KERN_INVALID_ADDRESS instead.
1464	*
1465	* The addressability of the range is limited to that which can
1466	* be described by a vm_address_t.
1467	*
1468	* RETURNS
1469	* KERN_INVALID_TASK Bad task parameter
1470	* KERN_INVALID_ARGUMENT both sync and async were specified.
1471	* KERN_SUCCESS The usual.
1472	* KERN_INVALID_ADDRESS There was a hole in the region.
1473	*/
1474
1475	kern_return_t
1476	vm_msync(
1477	vm_map_t map,
1478	vm_address_t address,
1479	vm_size_t size,
1480	vm_sync_t sync_flags)
1481	{
1482
1483	if (map == VM_MAP_NULL)
1484	return(KERN_INVALID_TASK);
1485
1486	return vm_map_msync(map, (vm_map_address_t)address,
1487	(vm_map_size_t)size, sync_flags);
1488	}
1489
1490
1491	int
1492	vm_toggle_entry_reuse(int toggle, int *old_value)
1493	{
1494	vm_map_t map = current_map();
1495
1496	assert(!map->is_nested_map);
1497	if(toggle == VM_TOGGLE_GETVALUE && old_value != NULL){
1498	*old_value = map->disable_vmentry_reuse;
1499	} else if(toggle == VM_TOGGLE_SET){
1500	vm_map_entry_t map_to_entry;
1501
1502	vm_map_lock(map);
1503	vm_map_disable_hole_optimization(map);
1504	map->disable_vmentry_reuse = TRUE;
1505	__IGNORE_WCASTALIGN(map_to_entry = vm_map_to_entry(map));
1506	if (map->first_free == map_to_entry) {
1507	map->highest_entry_end = vm_map_min(map);
1508	} else {
1509	map->highest_entry_end = map->first_free->vme_end;
1510	}
1511	vm_map_unlock(map);
1512	} else if (toggle == VM_TOGGLE_CLEAR){
1513	vm_map_lock(map);
1514	map->disable_vmentry_reuse = FALSE;
1515	vm_map_unlock(map);
1516	} else
1517	return KERN_INVALID_ARGUMENT;
1518
1519	return KERN_SUCCESS;
1520	}
1521
1522	/*
1523	* mach_vm_behavior_set
1524	*
1525	* Sets the paging behavior attribute for the specified range
1526	* in the specified map.
1527	*
1528	* This routine will fail with KERN_INVALID_ADDRESS if any address
1529	* in [start,start+size) is not a valid allocated memory region.
1530	*/
1531	kern_return_t
1532	mach_vm_behavior_set(
1533	vm_map_t map,
1534	mach_vm_offset_t start,
1535	mach_vm_size_t size,
1536	vm_behavior_t new_behavior)
1537	{
1538	vm_map_offset_t align_mask;
1539
1540	if ((map == VM_MAP_NULL) \|\| (start + size < start))
1541	return(KERN_INVALID_ARGUMENT);
1542
1543	if (size == `0`)
1544	return KERN_SUCCESS;
1545
1546	switch (new_behavior) {
1547	case VM_BEHAVIOR_REUSABLE:
1548	case VM_BEHAVIOR_REUSE:
1549	case VM_BEHAVIOR_CAN_REUSE:
1550	/*
1551	* Align to the hardware page size, to allow
1552	* malloc() to maximize the amount of re-usability,
1553	* even on systems with larger software page size.
1554	*/
1555	align_mask = PAGE_MASK;
1556	break;
1557	default:
1558	align_mask = VM_MAP_PAGE_MASK(map);
1559	break;
1560	}
1561
1562	return vm_map_behavior_set(map,
1563	vm_map_trunc_page(start, align_mask),
1564	vm_map_round_page(start+size, align_mask),
1565	new_behavior);
1566	}
1567
1568	/*
1569	* vm_behavior_set
1570	*
1571	* Sets the paging behavior attribute for the specified range
1572	* in the specified map.
1573	*
1574	* This routine will fail with KERN_INVALID_ADDRESS if any address
1575	* in [start,start+size) is not a valid allocated memory region.
1576	*
1577	* This routine is potentially limited in addressibility by the
1578	* use of vm_offset_t (if the map provided is larger than the
1579	* kernel's).
1580	*/
1581	kern_return_t
1582	vm_behavior_set(
1583	vm_map_t map,
1584	vm_offset_t start,
1585	vm_size_t size,
1586	vm_behavior_t new_behavior)
1587	{
1588	if (start + size < start)
1589	return KERN_INVALID_ARGUMENT;
1590
1591	return mach_vm_behavior_set(map,
1592	(mach_vm_offset_t) start,
1593	(mach_vm_size_t) size,
1594	new_behavior);
1595	}
1596
1597	/*
1598	* mach_vm_region:
1599	*
1600	* User call to obtain information about a region in
1601	* a task's address map. Currently, only one flavor is
1602	* supported.
1603	*
1604	* XXX The reserved and behavior fields cannot be filled
1605	* in until the vm merge from the IK is completed, and
1606	* vm_reserve is implemented.
1607	*
1608	* XXX Dependency: syscall_vm_region() also supports only one flavor.
1609	*/
1610
1611	kern_return_t
1612	mach_vm_region(
1613	vm_map_t map,
1614	mach_vm_offset_t address, /* IN/OUT /
1615	mach_vm_size_t size, /* OUT /
1616	vm_region_flavor_t flavor, / IN /
1617	vm_region_info_t info, / OUT /
1618	mach_msg_type_number_t count, /* IN/OUT /
1619	mach_port_t object_name) /* OUT /
1620	{
1621	vm_map_offset_t map_addr;
1622	vm_map_size_t map_size;
1623	kern_return_t kr;
1624
1625	if (VM_MAP_NULL == map)
1626	return KERN_INVALID_ARGUMENT;
1627
1628	map_addr = (vm_map_offset_t)*address;
1629	map_size = (vm_map_size_t)*size;
1630
1631	/ legacy conversion /
1632	if (VM_REGION_BASIC_INFO == flavor)
1633	flavor = VM_REGION_BASIC_INFO_64;
1634
1635	kr = vm_map_region(map,
1636	&map_addr, &map_size,
1637	flavor, info, count,
1638	object_name);
1639
1640	*address = map_addr;
1641	*size = map_size;
1642	return kr;
1643	}
1644
1645	/*
1646	* vm_region_64 and vm_region:
1647	*
1648	* User call to obtain information about a region in
1649	* a task's address map. Currently, only one flavor is
1650	* supported.
1651	*
1652	* XXX The reserved and behavior fields cannot be filled
1653	* in until the vm merge from the IK is completed, and
1654	* vm_reserve is implemented.
1655	*
1656	* XXX Dependency: syscall_vm_region() also supports only one flavor.
1657	*/
1658
1659	kern_return_t
1660	vm_region_64(
1661	vm_map_t map,
1662	vm_offset_t address, /* IN/OUT /
1663	vm_size_t size, /* OUT /
1664	vm_region_flavor_t flavor, / IN /
1665	vm_region_info_t info, / OUT /
1666	mach_msg_type_number_t count, /* IN/OUT /
1667	mach_port_t object_name) /* OUT /
1668	{
1669	vm_map_offset_t map_addr;
1670	vm_map_size_t map_size;
1671	kern_return_t kr;
1672
1673	if (VM_MAP_NULL == map)
1674	return KERN_INVALID_ARGUMENT;
1675
1676	map_addr = (vm_map_offset_t)*address;
1677	map_size = (vm_map_size_t)*size;
1678
1679	/ legacy conversion /
1680	if (VM_REGION_BASIC_INFO == flavor)
1681	flavor = VM_REGION_BASIC_INFO_64;
1682
1683	kr = vm_map_region(map,
1684	&map_addr, &map_size,
1685	flavor, info, count,
1686	object_name);
1687
1688	*address = CAST_DOWN(vm_offset_t, map_addr);
1689	*size = CAST_DOWN(vm_size_t, map_size);
1690
1691	if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1692	return KERN_INVALID_ADDRESS;
1693	return kr;
1694	}
1695
1696	kern_return_t
1697	vm_region(
1698	vm_map_t map,
1699	vm_address_t address, /* IN/OUT /
1700	vm_size_t size, /* OUT /
1701	vm_region_flavor_t flavor, / IN /
1702	vm_region_info_t info, / OUT /
1703	mach_msg_type_number_t count, /* IN/OUT /
1704	mach_port_t object_name) /* OUT /
1705	{
1706	vm_map_address_t map_addr;
1707	vm_map_size_t map_size;
1708	kern_return_t kr;
1709
1710	if (VM_MAP_NULL == map)
1711	return KERN_INVALID_ARGUMENT;
1712
1713	map_addr = (vm_map_address_t)*address;
1714	map_size = (vm_map_size_t)*size;
1715
1716	kr = vm_map_region(map,
1717	&map_addr, &map_size,
1718	flavor, info, count,
1719	object_name);
1720
1721	*address = CAST_DOWN(vm_address_t, map_addr);
1722	*size = CAST_DOWN(vm_size_t, map_size);
1723
1724	if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1725	return KERN_INVALID_ADDRESS;
1726	return kr;
1727	}
1728
1729	/*
1730	* vm_region_recurse: A form of vm_region which follows the
1731	* submaps in a target map
1732	*
1733	*/
1734	kern_return_t
1735	mach_vm_region_recurse(
1736	vm_map_t map,
1737	mach_vm_address_t *address,
1738	mach_vm_size_t *size,
1739	uint32_t *depth,
1740	vm_region_recurse_info_t info,
1741	mach_msg_type_number_t *infoCnt)
1742	{
1743	vm_map_address_t map_addr;
1744	vm_map_size_t map_size;
1745	kern_return_t kr;
1746
1747	if (VM_MAP_NULL == map)
1748	return KERN_INVALID_ARGUMENT;
1749
1750	map_addr = (vm_map_address_t)*address;
1751	map_size = (vm_map_size_t)*size;
1752
1753	kr = vm_map_region_recurse_64(
1754	map,
1755	&map_addr,
1756	&map_size,
1757	depth,
1758	(vm_region_submap_info_64_t)info,
1759	infoCnt);
1760
1761	*address = map_addr;
1762	*size = map_size;
1763	return kr;
1764	}
1765
1766	/*
1767	* vm_region_recurse: A form of vm_region which follows the
1768	* submaps in a target map
1769	*
1770	*/
1771	kern_return_t
1772	vm_region_recurse_64(
1773	vm_map_t map,
1774	vm_address_t *address,
1775	vm_size_t *size,
1776	uint32_t *depth,
1777	vm_region_recurse_info_64_t info,
1778	mach_msg_type_number_t *infoCnt)
1779	{
1780	vm_map_address_t map_addr;
1781	vm_map_size_t map_size;
1782	kern_return_t kr;
1783
1784	if (VM_MAP_NULL == map)
1785	return KERN_INVALID_ARGUMENT;
1786
1787	map_addr = (vm_map_address_t)*address;
1788	map_size = (vm_map_size_t)*size;
1789
1790	kr = vm_map_region_recurse_64(
1791	map,
1792	&map_addr,
1793	&map_size,
1794	depth,
1795	(vm_region_submap_info_64_t)info,
1796	infoCnt);
1797
1798	*address = CAST_DOWN(vm_address_t, map_addr);
1799	*size = CAST_DOWN(vm_size_t, map_size);
1800
1801	if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1802	return KERN_INVALID_ADDRESS;
1803	return kr;
1804	}
1805
1806	kern_return_t
1807	vm_region_recurse(
1808	vm_map_t map,
1809	vm_offset_t address, /* IN/OUT /
1810	vm_size_t size, /* OUT /
1811	natural_t depth, /* IN/OUT /
1812	vm_region_recurse_info_t info32, / IN/OUT /
1813	mach_msg_type_number_t infoCnt) /* IN/OUT /
1814	{
1815	vm_region_submap_info_data_64_t info64;
1816	vm_region_submap_info_t info;
1817	vm_map_address_t map_addr;
1818	vm_map_size_t map_size;
1819	kern_return_t kr;
1820
1821	if (VM_MAP_NULL == map \|\| *infoCnt < VM_REGION_SUBMAP_INFO_COUNT)
1822	return KERN_INVALID_ARGUMENT;
1823
1824
1825	map_addr = (vm_map_address_t)*address;
1826	map_size = (vm_map_size_t)*size;
1827	info = (vm_region_submap_info_t)info32;
1828	*infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1829
1830	kr = vm_map_region_recurse_64(map, &map_addr,&map_size,
1831	depth, &info64, infoCnt);
1832
1833	info->protection = info64.protection;
1834	info->max_protection = info64.max_protection;
1835	info->inheritance = info64.inheritance;
1836	info->offset = (uint32_t)info64.offset; / trouble-maker /
1837	info->user_tag = info64.user_tag;
1838	info->pages_resident = info64.pages_resident;
1839	info->pages_shared_now_private = info64.pages_shared_now_private;
1840	info->pages_swapped_out = info64.pages_swapped_out;
1841	info->pages_dirtied = info64.pages_dirtied;
1842	info->ref_count = info64.ref_count;
1843	info->shadow_depth = info64.shadow_depth;
1844	info->external_pager = info64.external_pager;
1845	info->share_mode = info64.share_mode;
1846	info->is_submap = info64.is_submap;
1847	info->behavior = info64.behavior;
1848	info->object_id = info64.object_id;
1849	info->user_wired_count = info64.user_wired_count;
1850
1851	*address = CAST_DOWN(vm_address_t, map_addr);
1852	*size = CAST_DOWN(vm_size_t, map_size);
1853	*infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1854
1855	if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS)
1856	return KERN_INVALID_ADDRESS;
1857	return kr;
1858	}
1859
1860	kern_return_t
1861	mach_vm_purgable_control(
1862	vm_map_t map,
1863	mach_vm_offset_t address,
1864	vm_purgable_t control,
1865	int *state)
1866	{
1867	if (VM_MAP_NULL == map)
1868	return KERN_INVALID_ARGUMENT;
1869
1870	if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
1871	/ not allowed from user-space /
1872	return KERN_INVALID_ARGUMENT;
1873	}
1874
1875	return vm_map_purgable_control(map,
1876	vm_map_trunc_page(address, PAGE_MASK),
1877	control,
1878	state);
1879	}
1880
1881	kern_return_t
1882	vm_purgable_control(
1883	vm_map_t map,
1884	vm_offset_t address,
1885	vm_purgable_t control,
1886	int *state)
1887	{
1888	if (VM_MAP_NULL == map)
1889	return KERN_INVALID_ARGUMENT;
1890
1891	if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
1892	/ not allowed from user-space /
1893	return KERN_INVALID_ARGUMENT;
1894	}
1895
1896	return vm_map_purgable_control(map,
1897	vm_map_trunc_page(address, PAGE_MASK),
1898	control,
1899	state);
1900	}
1901
1902
1903	/*
1904	* Ordinarily, the right to allocate CPM is restricted
1905	* to privileged applications (those that can gain access
1906	* to the host priv port). Set this variable to zero if
1907	* you want to let any application allocate CPM.
1908	*/
1909	unsigned int vm_allocate_cpm_privileged = `0`;
1910
1911	/*
1912	* Allocate memory in the specified map, with the caveat that
1913	* the memory is physically contiguous. This call may fail
1914	* if the system can't find sufficient contiguous memory.
1915	* This call may cause or lead to heart-stopping amounts of
1916	* paging activity.
1917	*
1918	* Memory obtained from this call should be freed in the
1919	* normal way, viz., via vm_deallocate.
1920	*/
1921	kern_return_t
1922	vm_allocate_cpm(
1923	host_priv_t host_priv,
1924	vm_map_t map,
1925	vm_address_t *addr,
1926	vm_size_t size,
1927	int flags)
1928	{
1929	vm_map_address_t map_addr;
1930	vm_map_size_t map_size;
1931	kern_return_t kr;
1932
1933	if (vm_allocate_cpm_privileged && HOST_PRIV_NULL == host_priv)
1934	return KERN_INVALID_HOST;
1935
1936	if (VM_MAP_NULL == map)
1937	return KERN_INVALID_ARGUMENT;
1938
1939	map_addr = (vm_map_address_t)*addr;
1940	map_size = (vm_map_size_t)size;
1941
1942	kr = vm_map_enter_cpm(map,
1943	&map_addr,
1944	map_size,
1945	flags);
1946
1947	*addr = CAST_DOWN(vm_address_t, map_addr);
1948	return kr;
1949	}
1950
1951
1952	kern_return_t
1953	mach_vm_page_query(
1954	vm_map_t map,
1955	mach_vm_offset_t offset,
1956	int *disposition,
1957	int *ref_count)
1958	{
1959	if (VM_MAP_NULL == map)
1960	return KERN_INVALID_ARGUMENT;
1961
1962	return vm_map_page_query_internal(
1963	map,
1964	vm_map_trunc_page(offset, PAGE_MASK),
1965	disposition, ref_count);
1966	}
1967
1968	kern_return_t
1969	vm_map_page_query(
1970	vm_map_t map,
1971	vm_offset_t offset,
1972	int *disposition,
1973	int *ref_count)
1974	{
1975	if (VM_MAP_NULL == map)
1976	return KERN_INVALID_ARGUMENT;
1977
1978	return vm_map_page_query_internal(
1979	map,
1980	vm_map_trunc_page(offset, PAGE_MASK),
1981	disposition, ref_count);
1982	}
1983
1984	kern_return_t
1985	mach_vm_page_range_query(
1986	vm_map_t map,
1987	mach_vm_offset_t address,
1988	mach_vm_size_t size,
1989	mach_vm_address_t dispositions_addr,
1990	mach_vm_size_t *dispositions_count)
1991	{
1992	kern_return_t kr = KERN_SUCCESS;
1993	int num_pages = `0`, i = `0`;
1994	mach_vm_size_t curr_sz = `0`, copy_sz = `0`;
1995	mach_vm_size_t disp_buf_req_size = `0`, disp_buf_total_size = `0`;
1996	mach_msg_type_number_t count = `0`;
1997
1998	void *info = NULL;
1999	void *local_disp = NULL;;
2000	vm_map_size_t info_size = `0`, local_disp_size = `0`;
2001	mach_vm_offset_t start = `0`, end = `0`;
2002
2003	if (map == VM_MAP_NULL \|\| dispositions_count == NULL) {
2004	return KERN_INVALID_ARGUMENT;
2005	}
2006
2007	disp_buf_req_size = ( dispositions_count sizeof(int));
2008	start = mach_vm_trunc_page(address);
2009	end = mach_vm_round_page(address + size);
2010
2011	if (end < start) {
2012	return KERN_INVALID_ARGUMENT;
2013	}
2014
2015	if (disp_buf_req_size == `0` \|\| (end == start)) {
2016	return KERN_SUCCESS;
2017	}
2018
2019	/*
2020	* For large requests, we will go through them
2021	* MAX_PAGE_RANGE_QUERY chunk at a time.
2022	*/
2023
2024	curr_sz = MIN(end - start, MAX_PAGE_RANGE_QUERY);
2025	num_pages = (int) (curr_sz >> PAGE_SHIFT);
2026
2027	info_size = num_pages * sizeof(vm_page_info_basic_data_t);
2028	info = kalloc(info_size);
2029
2030	if (info == NULL) {
2031	return KERN_RESOURCE_SHORTAGE;
2032	}
2033
2034	local_disp_size = num_pages * sizeof(int);
2035	local_disp = kalloc(local_disp_size);
2036
2037	if (local_disp == NULL) {
2038
2039	kfree(info, info_size);
2040	info = NULL;
2041	return KERN_RESOURCE_SHORTAGE;
2042	}
2043
2044	while (size) {
2045
2046	count = VM_PAGE_INFO_BASIC_COUNT;
2047	kr = vm_map_page_range_info_internal(
2048	map,
2049	start,
2050	mach_vm_round_page(start + curr_sz),
2051	VM_PAGE_INFO_BASIC,
2052	(vm_page_info_t) info,
2053	&count);
2054
2055	assert(kr == KERN_SUCCESS);
2056
2057	for (i = `0`; i < num_pages; i++) {
2058
2059	((int*)local_disp)[i] = ((vm_page_info_basic_t)info)[i].disposition;
2060	}
2061
2062	copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int)/ an int per page /);
2063	kr = copyout(local_disp, (mach_vm_address_t)dispositions_addr, copy_sz);
2064
2065	start += curr_sz;
2066	disp_buf_req_size -= copy_sz;
2067	disp_buf_total_size += copy_sz;
2068
2069	if (kr != `0`) {
2070	break;
2071	}
2072
2073	if ((disp_buf_req_size == `0`) \|\| (curr_sz >= size)) {
2074
2075	/*
2076	* We might have inspected the full range OR
2077	* more than it esp. if the user passed in
2078	* non-page aligned start/size and/or if we
2079	* descended into a submap. We are done here.
2080	*/
2081
2082	size = `0`;
2083
2084	} else {
2085
2086	dispositions_addr += copy_sz;
2087
2088	size -= curr_sz;
2089
2090	curr_sz = MIN(mach_vm_round_page(size), MAX_PAGE_RANGE_QUERY);
2091	num_pages = (int)(curr_sz >> PAGE_SHIFT);
2092	}
2093	}
2094
2095	dispositions_count = disp_buf_total_size / sizeof(int*);
2096
2097	kfree(local_disp, local_disp_size);
2098	local_disp = NULL;
2099
2100	kfree(info, info_size);
2101	info = NULL;
2102
2103	return kr;
2104	}
2105
2106	kern_return_t
2107	mach_vm_page_info(
2108	vm_map_t map,
2109	mach_vm_address_t address,
2110	vm_page_info_flavor_t flavor,
2111	vm_page_info_t info,
2112	mach_msg_type_number_t *count)
2113	{
2114	kern_return_t kr;
2115
2116	if (map == VM_MAP_NULL) {
2117	return KERN_INVALID_ARGUMENT;
2118	}
2119
2120	kr = vm_map_page_info(map, address, flavor, info, count);
2121	return kr;
2122	}
2123
2124	/ map a (whole) upl into an address space /
2125	kern_return_t
2126	vm_upl_map(
2127	vm_map_t map,
2128	upl_t upl,
2129	vm_address_t *dst_addr)
2130	{
2131	vm_map_offset_t map_addr;
2132	kern_return_t kr;
2133
2134	if (VM_MAP_NULL == map)
2135	return KERN_INVALID_ARGUMENT;
2136
2137	kr = vm_map_enter_upl(map, upl, &map_addr);
2138	*dst_addr = CAST_DOWN(vm_address_t, map_addr);
2139	return kr;
2140	}
2141
2142	kern_return_t
2143	vm_upl_unmap(
2144	vm_map_t map,
2145	upl_t upl)
2146	{
2147	if (VM_MAP_NULL == map)
2148	return KERN_INVALID_ARGUMENT;
2149
2150	return (vm_map_remove_upl(map, upl));
2151	}
2152
2153	/ Retrieve a upl for an object underlying an address range in a map /
2154
2155	kern_return_t
2156	vm_map_get_upl(
2157	vm_map_t map,
2158	vm_map_offset_t map_offset,
2159	upl_size_t *upl_size,
2160	upl_t *upl,
2161	upl_page_info_array_t page_list,
2162	unsigned int *count,
2163	upl_control_flags_t *flags,
2164	vm_tag_t tag,
2165	int force_data_sync)
2166	{
2167	upl_control_flags_t map_flags;
2168	kern_return_t kr;
2169
2170	if (VM_MAP_NULL == map)
2171	return KERN_INVALID_ARGUMENT;
2172
2173	map_flags = *flags & ~UPL_NOZEROFILL;
2174	if (force_data_sync)
2175	map_flags \|= UPL_FORCE_DATA_SYNC;
2176
2177	kr = vm_map_create_upl(map,
2178	map_offset,
2179	upl_size,
2180	upl,
2181	page_list,
2182	count,
2183	&map_flags,
2184	tag);
2185
2186	*flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
2187	return kr;
2188	}
2189
2190	#if CONFIG_EMBEDDED
2191	extern int proc_selfpid(void);
2192	extern char proc_name_address(void* *p);
2193	int cs_executable_mem_entry = `0`;
2194	int log_executable_mem_entry = `0`;
2195	#endif /* CONFIG_EMBEDDED */
2196
2197	/*
2198	* mach_make_memory_entry_64
2199	*
2200	* Think of it as a two-stage vm_remap() operation. First
2201	* you get a handle. Second, you get map that handle in
2202	* somewhere else. Rather than doing it all at once (and
2203	* without needing access to the other whole map).
2204	*/
2205	kern_return_t
2206	mach_make_memory_entry_64(
2207	vm_map_t target_map,
2208	memory_object_size_t *size,
2209	memory_object_offset_t offset,
2210	vm_prot_t permission,
2211	ipc_port_t *object_handle,
2212	ipc_port_t parent_handle)
2213	{
2214	if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_USER) {
2215	/*
2216	* Unknown flag: reject for forward compatibility.
2217	*/
2218	return KERN_INVALID_VALUE;
2219	}
2220
2221	return mach_make_memory_entry_internal(target_map,
2222	size,
2223	offset,
2224	permission,
2225	object_handle,
2226	parent_handle);
2227	}
2228
2229	kern_return_t
2230	mach_make_memory_entry_internal(
2231	vm_map_t target_map,
2232	memory_object_size_t *size,
2233	memory_object_offset_t offset,
2234	vm_prot_t permission,
2235	ipc_port_t *object_handle,
2236	ipc_port_t parent_handle)
2237	{
2238	vm_map_version_t version;
2239	vm_named_entry_t parent_entry;
2240	vm_named_entry_t user_entry;
2241	ipc_port_t user_handle;
2242	kern_return_t kr;
2243	vm_map_t real_map;
2244
2245	/ needed for call to vm_map_lookup_locked /
2246	boolean_t wired;
2247	boolean_t iskernel;
2248	vm_object_offset_t obj_off;
2249	vm_prot_t prot;
2250	struct vm_object_fault_info fault_info = {};
2251	vm_object_t object;
2252	vm_object_t shadow_object;
2253
2254	/ needed for direct map entry manipulation /
2255	vm_map_entry_t map_entry;
2256	vm_map_entry_t next_entry;
2257	vm_map_t local_map;
2258	vm_map_t original_map = target_map;
2259	vm_map_size_t total_size, map_size;
2260	vm_map_offset_t map_start, map_end;
2261	vm_map_offset_t local_offset;
2262	vm_object_size_t mappable_size;
2263
2264	/*
2265	* Stash the offset in the page for use by vm_map_enter_mem_object()
2266	* in the VM_FLAGS_RETURN_DATA_ADDR/MAP_MEM_USE_DATA_ADDR case.
2267	*/
2268	vm_object_offset_t offset_in_page;
2269
2270	unsigned int access;
2271	vm_prot_t protections;
2272	vm_prot_t original_protections, mask_protections;
2273	unsigned int wimg_mode;
2274
2275	boolean_t force_shadow = FALSE;
2276	boolean_t use_data_addr;
2277	boolean_t use_4K_compat;
2278	#if VM_NAMED_ENTRY_LIST
2279	int alias = -`1`;
2280	#endif /* VM_NAMED_ENTRY_LIST */
2281
2282	if ((permission & MAP_MEM_FLAGS_MASK) & ~MAP_MEM_FLAGS_ALL) {
2283	/*
2284	* Unknown flag: reject for forward compatibility.
2285	*/
2286	return KERN_INVALID_VALUE;
2287	}
2288
2289	if (IP_VALID(parent_handle) &&
2290	ip_kotype(parent_handle) == IKOT_NAMED_ENTRY) {
2291	parent_entry = (vm_named_entry_t) parent_handle->ip_kobject;
2292	} else {
2293	parent_entry = NULL;
2294	}
2295
2296	if (parent_entry && parent_entry->is_copy) {
2297	return KERN_INVALID_ARGUMENT;
2298	}
2299
2300	original_protections = permission & VM_PROT_ALL;
2301	protections = original_protections;
2302	mask_protections = permission & VM_PROT_IS_MASK;
2303	access = GET_MAP_MEM(permission);
2304	use_data_addr = ((permission & MAP_MEM_USE_DATA_ADDR) != `0`);
2305	use_4K_compat = ((permission & MAP_MEM_4K_DATA_ADDR) != `0`);
2306
2307	user_handle = IP_NULL;
2308	user_entry = NULL;
2309
2310	map_start = vm_map_trunc_page(offset, PAGE_MASK);
2311
2312	if (permission & MAP_MEM_ONLY) {
2313	boolean_t parent_is_object;
2314
2315	map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2316	map_size = map_end - map_start;
2317
2318	if (use_data_addr \|\| use_4K_compat \|\| parent_entry == NULL) {
2319	return KERN_INVALID_ARGUMENT;
2320	}
2321
2322	parent_is_object = !parent_entry->is_sub_map;
2323	object = parent_entry->backing.object;
2324	if(parent_is_object && object != VM_OBJECT_NULL)
2325	wimg_mode = object->wimg_bits;
2326	else
2327	wimg_mode = VM_WIMG_USE_DEFAULT;
2328	if((access != GET_MAP_MEM(parent_entry->protection)) &&
2329	!(parent_entry->protection & VM_PROT_WRITE)) {
2330	return KERN_INVALID_RIGHT;
2331	}
2332	vm_prot_to_wimg(access, &wimg_mode);
2333	if (access != MAP_MEM_NOOP)
2334	SET_MAP_MEM(access, parent_entry->protection);
2335	if (parent_is_object && object &&
2336	(access != MAP_MEM_NOOP) &&
2337	(!(object->nophyscache))) {
2338
2339	if (object->wimg_bits != wimg_mode) {
2340	vm_object_lock(object);
2341	vm_object_change_wimg_mode(object, wimg_mode);
2342	vm_object_unlock(object);
2343	}
2344	}
2345	if (object_handle)
2346	*object_handle = IP_NULL;
2347	return KERN_SUCCESS;
2348	} else if (permission & MAP_MEM_NAMED_CREATE) {
2349	map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2350	map_size = map_end - map_start;
2351
2352	if (use_data_addr \|\| use_4K_compat) {
2353	return KERN_INVALID_ARGUMENT;
2354	}
2355
2356	kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2357	if (kr != KERN_SUCCESS) {
2358	return KERN_FAILURE;
2359	}
2360
2361	/*
2362	* Force the creation of the VM object now.
2363	*/
2364	if (map_size > (vm_map_size_t) ANON_MAX_SIZE) {
2365	/*
2366	* LP64todo - for now, we can only allocate 4GB-4096
2367	* internal objects because the default pager can't
2368	* page bigger ones. Remove this when it can.
2369	*/
2370	kr = KERN_FAILURE;
2371	goto make_mem_done;
2372	}
2373
2374	object = vm_object_allocate(map_size);
2375	assert(object != VM_OBJECT_NULL);
2376
2377	if (permission & MAP_MEM_PURGABLE) {
2378	task_t owner;
2379
2380	if (! (permission & VM_PROT_WRITE)) {
2381	/ if we can't write, we can't purge /
2382	vm_object_deallocate(object);
2383	kr = KERN_INVALID_ARGUMENT;
2384	goto make_mem_done;
2385	}
2386	object->purgable = VM_PURGABLE_NONVOLATILE;
2387	if (permission & MAP_MEM_PURGABLE_KERNEL_ONLY) {
2388	object->purgeable_only_by_kernel = TRUE;
2389	}
2390	assert(object->vo_owner == NULL);
2391	assert(object->resident_page_count == `0`);
2392	assert(object->wired_page_count == `0`);
2393	vm_object_lock(object);
2394	owner = current_task();
2395	#if __arm64__
2396	if (owner->task_legacy_footprint) {
2397	/*
2398	* For ios11, we failed to account for
2399	* this memory. Keep doing that for
2400	* legacy apps (built before ios12),
2401	* for backwards compatibility's sake...
2402	*/
2403	owner = kernel_task;
2404	}
2405	#endif /* __arm64__ */
2406	vm_purgeable_nonvolatile_enqueue(object, owner);
2407	vm_object_unlock(object);
2408	}
2409
2410	if (permission & MAP_MEM_LEDGER_TAG_NETWORK) {
2411	/ make this object owned by the calling task /
2412	vm_object_lock(object);
2413	vm_object_ownership_change(
2414	object,
2415	VM_OBJECT_LEDGER_TAG_NETWORK,
2416	current_task(), / new owner /
2417	FALSE); / task_objq locked? /
2418	vm_object_unlock(object);
2419	}
2420
2421	#if CONFIG_SECLUDED_MEMORY
2422	if (secluded_for_iokit && / global boot-arg /
2423	((permission & MAP_MEM_GRAB_SECLUDED)
2424	#if 11
2425	/ XXX FBDP for my testing only /
2426	\|\| (secluded_for_fbdp && map_size == `97550336`)
2427	#endif
2428	)) {
2429	#if 11
2430	if (!(permission & MAP_MEM_GRAB_SECLUDED) &&
2431	secluded_for_fbdp) {
2432	printf("FBDP: object %p size %lld can grab secluded\n", object, (uint64_t) map_size);
2433	}
2434	#endif
2435	object->can_grab_secluded = TRUE;
2436	assert(!object->eligible_for_secluded);
2437	}
2438	#endif /* CONFIG_SECLUDED_MEMORY */
2439
2440	/*
2441	* The VM object is brand new and nobody else knows about it,
2442	* so we don't need to lock it.
2443	*/
2444
2445	wimg_mode = object->wimg_bits;
2446	vm_prot_to_wimg(access, &wimg_mode);
2447	if (access != MAP_MEM_NOOP) {
2448	object->wimg_bits = wimg_mode;
2449	}
2450
2451	/ the object has no pages, so no WIMG bits to update here /
2452
2453	/*
2454	* XXX
2455	* We use this path when we want to make sure that
2456	* nobody messes with the object (coalesce, for
2457	* example) before we map it.
2458	* We might want to use these objects for transposition via
2459	* vm_object_transpose() too, so we don't want any copy or
2460	* shadow objects either...
2461	*/
2462	object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
2463	object->true_share = TRUE;
2464
2465	user_entry->backing.object = object;
2466	user_entry->internal = TRUE;
2467	user_entry->is_sub_map = FALSE;
2468	user_entry->offset = `0`;
2469	user_entry->data_offset = `0`;
2470	user_entry->protection = protections;
2471	SET_MAP_MEM(access, user_entry->protection);
2472	user_entry->size = map_size;
2473
2474	/ user_object pager and internal fields are not used /
2475	/ when the object field is filled in. /
2476
2477	*size = CAST_DOWN(vm_size_t, (user_entry->size -
2478	user_entry->data_offset));
2479	*object_handle = user_handle;
2480	return KERN_SUCCESS;
2481	}
2482
2483	if (permission & MAP_MEM_VM_COPY) {
2484	vm_map_copy_t copy;
2485
2486	if (target_map == VM_MAP_NULL) {
2487	return KERN_INVALID_TASK;
2488	}
2489
2490	map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2491	map_size = map_end - map_start;
2492	if (use_data_addr \|\| use_4K_compat) {
2493	offset_in_page = offset - map_start;
2494	if (use_4K_compat)
2495	offset_in_page &= ~((signed)(`0xFFF`));
2496	} else {
2497	offset_in_page = `0`;
2498	}
2499
2500	kr = vm_map_copyin_internal(target_map,
2501	map_start,
2502	map_size,
2503	VM_MAP_COPYIN_ENTRY_LIST,
2504	&copy);
2505	if (kr != KERN_SUCCESS) {
2506	return kr;
2507	}
2508
2509	kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2510	if (kr != KERN_SUCCESS) {
2511	vm_map_copy_discard(copy);
2512	return KERN_FAILURE;
2513	}
2514
2515	user_entry->backing.copy = copy;
2516	user_entry->internal = FALSE;
2517	user_entry->is_sub_map = FALSE;
2518	user_entry->is_copy = TRUE;
2519	user_entry->offset = `0`;
2520	user_entry->protection = protections;
2521	user_entry->size = map_size;
2522	user_entry->data_offset = offset_in_page;
2523
2524	*size = CAST_DOWN(vm_size_t, (user_entry->size -
2525	user_entry->data_offset));
2526	*object_handle = user_handle;
2527	return KERN_SUCCESS;
2528	}
2529
2530	if (permission & MAP_MEM_VM_SHARE) {
2531	vm_map_copy_t copy;
2532	vm_prot_t cur_prot, max_prot;
2533
2534	if (target_map == VM_MAP_NULL) {
2535	return KERN_INVALID_TASK;
2536	}
2537
2538	map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2539	map_size = map_end - map_start;
2540	if (use_data_addr \|\| use_4K_compat) {
2541	offset_in_page = offset - map_start;
2542	if (use_4K_compat)
2543	offset_in_page &= ~((signed)(`0xFFF`));
2544	} else {
2545	offset_in_page = `0`;
2546	}
2547
2548	cur_prot = VM_PROT_ALL;
2549	kr = vm_map_copy_extract(target_map,
2550	map_start,
2551	map_size,
2552	&copy,
2553	&cur_prot,
2554	&max_prot);
2555	if (kr != KERN_SUCCESS) {
2556	return kr;
2557	}
2558
2559	if (mask_protections) {
2560	/*
2561	* We just want as much of "original_protections"
2562	* as we can get out of the actual "cur_prot".
2563	*/
2564	protections &= cur_prot;
2565	if (protections == VM_PROT_NONE) {
2566	/ no access at all: fail /
2567	vm_map_copy_discard(copy);
2568	return KERN_PROTECTION_FAILURE;
2569	}
2570	} else {
2571	/*
2572	* We want exactly "original_protections"
2573	* out of "cur_prot".
2574	*/
2575	if ((cur_prot & protections) != protections) {
2576	vm_map_copy_discard(copy);
2577	return KERN_PROTECTION_FAILURE;
2578	}
2579	}
2580
2581	kr = mach_memory_entry_allocate(&user_entry, &user_handle);
2582	if (kr != KERN_SUCCESS) {
2583	vm_map_copy_discard(copy);
2584	return KERN_FAILURE;
2585	}
2586
2587	user_entry->backing.copy = copy;
2588	user_entry->internal = FALSE;
2589	user_entry->is_sub_map = FALSE;
2590	user_entry->is_copy = TRUE;
2591	user_entry->offset = `0`;
2592	user_entry->protection = protections;
2593	user_entry->size = map_size;
2594	user_entry->data_offset = offset_in_page;
2595
2596	*size = CAST_DOWN(vm_size_t, (user_entry->size -
2597	user_entry->data_offset));
2598	*object_handle = user_handle;
2599	return KERN_SUCCESS;
2600	}
2601
2602	if (parent_entry == NULL \|\|
2603	(permission & MAP_MEM_NAMED_REUSE)) {
2604
2605	map_end = vm_map_round_page(offset + *size, PAGE_MASK);
2606	map_size = map_end - map_start;
2607	if (use_data_addr \|\| use_4K_compat) {
2608	offset_in_page = offset - map_start;
2609	if (use_4K_compat)
2610	offset_in_page &= ~((signed)(`0xFFF`));
2611	} else {
2612	offset_in_page = `0`;
2613	}
2614
2615	/ Create a named object based on address range within the task map /
2616	/ Go find the object at given address /
2617
2618	if (target_map == VM_MAP_NULL) {
2619	return KERN_INVALID_TASK;
2620	}
2621
2622	redo_lookup:
2623	protections = original_protections;
2624	vm_map_lock_read(target_map);
2625
2626	/ get the object associated with the target address /
2627	/ note we check the permission of the range against /
2628	/ that requested by the caller /
2629
2630	kr = vm_map_lookup_locked(&target_map, map_start,
2631	protections \| mask_protections,
2632	OBJECT_LOCK_EXCLUSIVE, &version,
2633	&object, &obj_off, &prot, &wired,
2634	&fault_info,
2635	&real_map);
2636	if (kr != KERN_SUCCESS) {
2637	vm_map_unlock_read(target_map);
2638	goto make_mem_done;
2639	}
2640	if (mask_protections) {
2641	/*
2642	* The caller asked us to use the "protections" as
2643	* a mask, so restrict "protections" to what this
2644	* mapping actually allows.
2645	*/
2646	protections &= prot;
2647	}
2648	#if CONFIG_EMBEDDED
2649	/*
2650	* Wiring would copy the pages to a shadow object.
2651	* The shadow object would not be code-signed so
2652	* attempting to execute code from these copied pages
2653	* would trigger a code-signing violation.
2654	*/
2655	if (prot & VM_PROT_EXECUTE) {
2656	if (log_executable_mem_entry) {
2657	void *bsd_info;
2658	bsd_info = current_task()->bsd_info;
2659	printf("pid %d[%s] making memory entry out of "
2660	"executable range from 0x%llx to 0x%llx:"
2661	"might cause code-signing issues "
2662	"later\n",
2663	proc_selfpid(),
2664	(bsd_info != NULL
2665	? proc_name_address(bsd_info)
2666	: "?"),
2667	(uint64_t) map_start,
2668	(uint64_t) map_end);
2669	}
2670	DTRACE_VM2(cs_executable_mem_entry,
2671	uint64_t, (uint64_t)map_start,
2672	uint64_t, (uint64_t)map_end);
2673	cs_executable_mem_entry++;
2674
2675	#if 11
2676	/*
2677	* We don't know how the memory entry will be used.
2678	* It might never get wired and might not cause any
2679	* trouble, so let's not reject this request...
2680	*/
2681	#else /* 11 */
2682	kr = KERN_PROTECTION_FAILURE;
2683	vm_object_unlock(object);
2684	vm_map_unlock_read(target_map);
2685	if(real_map != target_map)
2686	vm_map_unlock_read(real_map);
2687	goto make_mem_done;
2688	#endif /* 11 */
2689
2690	}
2691	#endif /* CONFIG_EMBEDDED */
2692
2693	if (((prot & protections) != protections)
2694	\|\| (object == kernel_object)) {
2695	kr = KERN_INVALID_RIGHT;
2696	vm_object_unlock(object);
2697	vm_map_unlock_read(target_map);
2698	if(real_map != target_map)
2699	vm_map_unlock_read(real_map);
2700	if(object == kernel_object) {
2701	printf("Warning: Attempt to create a named"
2702	" entry from the kernel_object\n");
2703	}
2704	goto make_mem_done;
2705	}
2706
2707	/ We have an object, now check to see if this object /
2708	/ is suitable. If not, create a shadow and share that /
2709
2710	/*
2711	* We have to unlock the VM object to avoid deadlocking with
2712	* a VM map lock (the lock ordering is map, the object), if we
2713	* need to modify the VM map to create a shadow object. Since
2714	* we might release the VM map lock below anyway, we have
2715	* to release the VM map lock now.
2716	* XXX FBDP There must be a way to avoid this double lookup...
2717	*
2718	* Take an extra reference on the VM object to make sure it's
2719	* not going to disappear.
2720	*/
2721	vm_object_reference_locked(object); / extra ref to hold obj /
2722	vm_object_unlock(object);
2723
2724	local_map = original_map;
2725	local_offset = map_start;
2726	if(target_map != local_map) {
2727	vm_map_unlock_read(target_map);
2728	if(real_map != target_map)
2729	vm_map_unlock_read(real_map);
2730	vm_map_lock_read(local_map);
2731	target_map = local_map;
2732	real_map = local_map;
2733	}
2734	while(TRUE) {
2735	if(!vm_map_lookup_entry(local_map,
2736	local_offset, &map_entry)) {
2737	kr = KERN_INVALID_ARGUMENT;
2738	vm_map_unlock_read(target_map);
2739	if(real_map != target_map)
2740	vm_map_unlock_read(real_map);
2741	vm_object_deallocate(object); / release extra ref /
2742	object = VM_OBJECT_NULL;
2743	goto make_mem_done;
2744	}
2745	iskernel = (local_map->pmap == kernel_pmap);
2746	if(!(map_entry->is_sub_map)) {
2747	if (VME_OBJECT(map_entry) != object) {
2748	kr = KERN_INVALID_ARGUMENT;
2749	vm_map_unlock_read(target_map);
2750	if(real_map != target_map)
2751	vm_map_unlock_read(real_map);
2752	vm_object_deallocate(object); / release extra ref /
2753	object = VM_OBJECT_NULL;
2754	goto make_mem_done;
2755	}
2756	break;
2757	} else {
2758	vm_map_t tmap;
2759	tmap = local_map;
2760	local_map = VME_SUBMAP(map_entry);
2761
2762	vm_map_lock_read(local_map);
2763	vm_map_unlock_read(tmap);
2764	target_map = local_map;
2765	real_map = local_map;
2766	local_offset = local_offset - map_entry->vme_start;
2767	local_offset += VME_OFFSET(map_entry);
2768	}
2769	}
2770
2771	#if VM_NAMED_ENTRY_LIST
2772	alias = VME_ALIAS(map_entry);
2773	#endif /* VM_NAMED_ENTRY_LIST */
2774
2775	/*
2776	* We found the VM map entry, lock the VM object again.
2777	*/
2778	vm_object_lock(object);
2779	if(map_entry->wired_count) {
2780	/ JMM - The check below should be reworked instead. /
2781	object->true_share = TRUE;
2782	}
2783	if (mask_protections) {
2784	/*
2785	* The caller asked us to use the "protections" as
2786	* a mask, so restrict "protections" to what this
2787	* mapping actually allows.
2788	*/
2789	protections &= map_entry->max_protection;
2790	}
2791	if(((map_entry->max_protection) & protections) != protections) {
2792	kr = KERN_INVALID_RIGHT;
2793	vm_object_unlock(object);
2794	vm_map_unlock_read(target_map);
2795	if(real_map != target_map)
2796	vm_map_unlock_read(real_map);
2797	vm_object_deallocate(object);
2798	object = VM_OBJECT_NULL;
2799	goto make_mem_done;
2800	}
2801
2802	mappable_size = fault_info.hi_offset - obj_off;
2803	total_size = map_entry->vme_end - map_entry->vme_start;
2804	if(map_size > mappable_size) {
2805	/ try to extend mappable size if the entries /
2806	/ following are from the same object and are /
2807	/ compatible /
2808	next_entry = map_entry->vme_next;
2809	/ lets see if the next map entry is still /
2810	/ pointing at this object and is contiguous /
2811	while(map_size > mappable_size) {
2812	if ((VME_OBJECT(next_entry) == object) &&
2813	(next_entry->vme_start ==
2814	next_entry->vme_prev->vme_end) &&
2815	(VME_OFFSET(next_entry) ==
2816	(VME_OFFSET(next_entry->vme_prev) +
2817	(next_entry->vme_prev->vme_end -
2818	next_entry->vme_prev->vme_start)))) {
2819	if (mask_protections) {
2820	/*
2821	* The caller asked us to use
2822	* the "protections" as a mask,
2823	* so restrict "protections" to
2824	* what this mapping actually
2825	* allows.
2826	*/
2827	protections &= next_entry->max_protection;
2828	}
2829	if ((next_entry->wired_count) &&
2830	(map_entry->wired_count == `0`)) {
2831	break;
2832	}
2833	if(((next_entry->max_protection)
2834	& protections) != protections) {
2835	break;
2836	}
2837	if (next_entry->needs_copy !=
2838	map_entry->needs_copy)
2839	break;
2840	mappable_size += next_entry->vme_end
2841	- next_entry->vme_start;
2842	total_size += next_entry->vme_end
2843	- next_entry->vme_start;
2844	next_entry = next_entry->vme_next;
2845	} else {
2846	break;
2847	}
2848
2849	}
2850	}
2851
2852	/ vm_map_entry_should_cow_for_true_share() checks for malloc tags,*
2853	* never true in kernel */
2854	if (!iskernel && vm_map_entry_should_cow_for_true_share(map_entry) &&
2855	object->vo_size > map_size &&
2856	map_size != `0`) {
2857	/*
2858	* Set up the targeted range for copy-on-write to
2859	* limit the impact of "true_share"/"copy_delay" to
2860	* that range instead of the entire VM object...
2861	*/
2862
2863	vm_object_unlock(object);
2864	if (vm_map_lock_read_to_write(target_map)) {
2865	vm_object_deallocate(object);
2866	target_map = original_map;
2867	goto redo_lookup;
2868	}
2869
2870	vm_map_clip_start(target_map,
2871	map_entry,
2872	vm_map_trunc_page(map_start,
2873	VM_MAP_PAGE_MASK(target_map)));
2874	vm_map_clip_end(target_map,
2875	map_entry,
2876	(vm_map_round_page(map_end,
2877	VM_MAP_PAGE_MASK(target_map))));
2878	force_shadow = TRUE;
2879
2880	if ((map_entry->vme_end - offset) < map_size) {
2881	map_size = map_entry->vme_end - map_start;
2882	}
2883	total_size = map_entry->vme_end - map_entry->vme_start;
2884
2885	vm_map_lock_write_to_read(target_map);
2886	vm_object_lock(object);
2887	}
2888
2889	if (object->internal) {
2890	/ vm_map_lookup_locked will create a shadow if /
2891	/ needs_copy is set but does not check for the /
2892	/ other two conditions shown. It is important to /
2893	/ set up an object which will not be pulled from /
2894	/ under us. /
2895
2896	if (force_shadow \|\|
2897	((map_entry->needs_copy \|\|
2898	object->shadowed \|\|
2899	(object->vo_size > total_size &&
2900	(VME_OFFSET(map_entry) != `0` \|\|
2901	object->vo_size >
2902	vm_map_round_page(total_size,
2903	VM_MAP_PAGE_MASK(target_map)))))
2904	&& !object->true_share
2905	&& object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)) {
2906	/*
2907	* We have to unlock the VM object before
2908	* trying to upgrade the VM map lock, to
2909	* honor lock ordering (map then object).
2910	* Otherwise, we would deadlock if another
2911	* thread holds a read lock on the VM map and
2912	* is trying to acquire the VM object's lock.
2913	* We still hold an extra reference on the
2914	* VM object, guaranteeing that it won't
2915	* disappear.
2916	*/
2917	vm_object_unlock(object);
2918
2919	if (vm_map_lock_read_to_write(target_map)) {
2920	/*
2921	* We couldn't upgrade our VM map lock
2922	* from "read" to "write" and we lost
2923	* our "read" lock.
2924	* Start all over again...
2925	*/
2926	vm_object_deallocate(object); / extra ref /
2927	target_map = original_map;
2928	goto redo_lookup;
2929	}
2930	#if 00
2931	vm_object_lock(object);
2932	#endif
2933
2934	/*
2935	* JMM - We need to avoid coming here when the object
2936	* is wired by anybody, not just the current map. Why
2937	* couldn't we use the standard vm_object_copy_quickly()
2938	* approach here?
2939	*/
2940
2941	/ create a shadow object /
2942	VME_OBJECT_SHADOW(map_entry, total_size);
2943	shadow_object = VME_OBJECT(map_entry);
2944	#if 00
2945	vm_object_unlock(object);
2946	#endif
2947
2948	prot = map_entry->protection & ~VM_PROT_WRITE;
2949
2950	if (override_nx(target_map,
2951	VME_ALIAS(map_entry))
2952	&& prot)
2953	prot \|= VM_PROT_EXECUTE;
2954
2955	vm_object_pmap_protect(
2956	object, VME_OFFSET(map_entry),
2957	total_size,
2958	((map_entry->is_shared
2959	\|\| target_map->mapped_in_other_pmaps)
2960	? PMAP_NULL :
2961	target_map->pmap),
2962	map_entry->vme_start,
2963	prot);
2964	total_size -= (map_entry->vme_end
2965	- map_entry->vme_start);
2966	next_entry = map_entry->vme_next;
2967	map_entry->needs_copy = FALSE;
2968
2969	vm_object_lock(shadow_object);
2970	while (total_size) {
2971	assert((next_entry->wired_count == `0`) \|\|
2972	(map_entry->wired_count));
2973
2974	if (VME_OBJECT(next_entry) == object) {
2975	vm_object_reference_locked(shadow_object);
2976	VME_OBJECT_SET(next_entry,
2977	shadow_object);
2978	vm_object_deallocate(object);
2979	VME_OFFSET_SET(
2980	next_entry,
2981	(VME_OFFSET(next_entry->vme_prev) +
2982	(next_entry->vme_prev->vme_end
2983	- next_entry->vme_prev->vme_start)));
2984	next_entry->use_pmap = TRUE;
2985	next_entry->needs_copy = FALSE;
2986	} else {
2987	panic("mach_make_memory_entry_64:"
2988	" map entries out of sync\n");
2989	}
2990	total_size -=
2991	next_entry->vme_end
2992	- next_entry->vme_start;
2993	next_entry = next_entry->vme_next;
2994	}
2995
2996	/*
2997	* Transfer our extra reference to the
2998	* shadow object.
2999	*/
3000	vm_object_reference_locked(shadow_object);
3001	vm_object_deallocate(object); / extra ref /
3002	object = shadow_object;
3003
3004	obj_off = ((local_offset - map_entry->vme_start)
3005	+ VME_OFFSET(map_entry));
3006
3007	vm_map_lock_write_to_read(target_map);
3008	}
3009	}
3010
3011	/ note: in the future we can (if necessary) allow for /
3012	/ memory object lists, this will better support /
3013	/ fragmentation, but is it necessary? The user should /
3014	/ be encouraged to create address space oriented /
3015	/ shared objects from CLEAN memory regions which have /
3016	/ a known and defined history. i.e. no inheritence /
3017	/ share, make this call before making the region the /
3018	/ target of ipc's, etc. The code above, protecting /
3019	/ against delayed copy, etc. is mostly defensive. /
3020
3021	wimg_mode = object->wimg_bits;
3022	if(!(object->nophyscache))
3023	vm_prot_to_wimg(access, &wimg_mode);
3024
3025	#if VM_OBJECT_TRACKING_OP_TRUESHARE
3026	if (!object->true_share &&
3027	vm_object_tracking_inited) {
3028	void *bt[VM_OBJECT_TRACKING_BTDEPTH];
3029	int num = `0`;
3030
3031	num = OSBacktrace(bt,
3032	VM_OBJECT_TRACKING_BTDEPTH);
3033	btlog_add_entry(vm_object_tracking_btlog,
3034	object,
3035	VM_OBJECT_TRACKING_OP_TRUESHARE,
3036	bt,
3037	num);
3038	}
3039	#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
3040
3041	vm_object_lock_assert_exclusive(object);
3042	object->true_share = TRUE;
3043	if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
3044	object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
3045
3046	/*
3047	* The memory entry now points to this VM object and we
3048	* need to hold a reference on the VM object. Use the extra
3049	* reference we took earlier to keep the object alive when we
3050	* had to unlock it.
3051	*/
3052
3053	vm_map_unlock_read(target_map);
3054	if(real_map != target_map)
3055	vm_map_unlock_read(real_map);
3056
3057	if (object->wimg_bits != wimg_mode)
3058	vm_object_change_wimg_mode(object, wimg_mode);
3059
3060	/ the size of mapped entry that overlaps with our region /
3061	/ which is targeted for share. /
3062	/ (entry_end - entry_start) - /
3063	/ offset of our beg addr within entry /
3064	/ it corresponds to this: /
3065
3066	if(map_size > mappable_size)
3067	map_size = mappable_size;
3068
3069	if (permission & MAP_MEM_NAMED_REUSE) {
3070	/*
3071	* Compare what we got with the "parent_entry".
3072	* If they match, re-use the "parent_entry" instead
3073	* of creating a new one.
3074	*/
3075	if (parent_entry != NULL &&
3076	parent_entry->backing.object == object &&
3077	parent_entry->internal == object->internal &&
3078	parent_entry->is_sub_map == FALSE &&
3079	parent_entry->offset == obj_off &&
3080	parent_entry->protection == protections &&
3081	parent_entry->size == map_size &&
3082	((!(use_data_addr \|\| use_4K_compat) &&
3083	(parent_entry->data_offset == `0`)) \|\|
3084	((use_data_addr \|\| use_4K_compat) &&
3085	(parent_entry->data_offset == offset_in_page)))) {
3086	/*
3087	* We have a match: re-use "parent_entry".
3088	*/
3089	/ release our extra reference on object /
3090	vm_object_unlock(object);
3091	vm_object_deallocate(object);
3092	/ parent_entry->ref_count++; XXX ? /
3093	/ Get an extra send-right on handle /
3094	ipc_port_copy_send(parent_handle);
3095
3096	*size = CAST_DOWN(vm_size_t,
3097	(parent_entry->size -
3098	parent_entry->data_offset));
3099	*object_handle = parent_handle;
3100	return KERN_SUCCESS;
3101	} else {
3102	/*
3103	* No match: we need to create a new entry.
3104	* fall through...
3105	*/
3106	}
3107	}
3108
3109	vm_object_unlock(object);
3110	if (mach_memory_entry_allocate(&user_entry, &user_handle)
3111	!= KERN_SUCCESS) {
3112	/ release our unused reference on the object /
3113	vm_object_deallocate(object);
3114	return KERN_FAILURE;
3115	}
3116
3117	user_entry->backing.object = object;
3118	user_entry->internal = object->internal;
3119	user_entry->is_sub_map = FALSE;
3120	user_entry->offset = obj_off;
3121	user_entry->data_offset = offset_in_page;
3122	user_entry->protection = protections;
3123	SET_MAP_MEM(GET_MAP_MEM(permission), user_entry->protection);
3124	user_entry->size = map_size;
3125	#if VM_NAMED_ENTRY_LIST
3126	user_entry->named_entry_alias = alias;
3127	#endif /* VM_NAMED_ENTRY_LIST */
3128
3129	/ user_object pager and internal fields are not used /
3130	/ when the object field is filled in. /
3131
3132	*size = CAST_DOWN(vm_size_t, (user_entry->size -
3133	user_entry->data_offset));
3134	*object_handle = user_handle;
3135	return KERN_SUCCESS;
3136
3137	} else {
3138	/ The new object will be base on an existing named object /
3139	if (parent_entry == NULL) {
3140	kr = KERN_INVALID_ARGUMENT;
3141	goto make_mem_done;
3142	}
3143
3144	if (use_data_addr \|\| use_4K_compat) {
3145	/*
3146	* submaps and pagers should only be accessible from within
3147	* the kernel, which shouldn't use the data address flag, so can fail here.
3148	*/
3149	if (parent_entry->is_sub_map) {
3150	panic("Shouldn't be using data address with a parent entry that is a submap.");
3151	}
3152	/*
3153	* Account for offset to data in parent entry and
3154	* compute our own offset to data.
3155	*/
3156	if((offset + *size + parent_entry->data_offset) > parent_entry->size) {
3157	kr = KERN_INVALID_ARGUMENT;
3158	goto make_mem_done;
3159	}
3160
3161	map_start = vm_map_trunc_page(offset + parent_entry->data_offset, PAGE_MASK);
3162	offset_in_page = (offset + parent_entry->data_offset) - map_start;
3163	if (use_4K_compat)
3164	offset_in_page &= ~((signed)(`0xFFF`));
3165	map_end = vm_map_round_page(offset + parent_entry->data_offset + *size, PAGE_MASK);
3166	map_size = map_end - map_start;
3167	} else {
3168	map_end = vm_map_round_page(offset + *size, PAGE_MASK);
3169	map_size = map_end - map_start;
3170	offset_in_page = `0`;
3171
3172	if((offset + map_size) > parent_entry->size) {
3173	kr = KERN_INVALID_ARGUMENT;
3174	goto make_mem_done;
3175	}
3176	}
3177
3178	if (mask_protections) {
3179	/*
3180	* The caller asked us to use the "protections" as
3181	* a mask, so restrict "protections" to what this
3182	* mapping actually allows.
3183	*/
3184	protections &= parent_entry->protection;
3185	}
3186	if((protections & parent_entry->protection) != protections) {
3187	kr = KERN_PROTECTION_FAILURE;
3188	goto make_mem_done;
3189	}
3190
3191	if (mach_memory_entry_allocate(&user_entry, &user_handle)
3192	!= KERN_SUCCESS) {
3193	kr = KERN_FAILURE;
3194	goto make_mem_done;
3195	}
3196
3197	user_entry->size = map_size;
3198	user_entry->offset = parent_entry->offset + map_start;
3199	user_entry->data_offset = offset_in_page;
3200	user_entry->is_sub_map = parent_entry->is_sub_map;
3201	user_entry->is_copy = parent_entry->is_copy;
3202	user_entry->internal = parent_entry->internal;
3203	user_entry->protection = protections;
3204
3205	if(access != MAP_MEM_NOOP) {
3206	SET_MAP_MEM(access, user_entry->protection);
3207	}
3208
3209	if(parent_entry->is_sub_map) {
3210	user_entry->backing.map = parent_entry->backing.map;
3211	vm_map_lock(user_entry->backing.map);
3212	user_entry->backing.map->map_refcnt++;
3213	vm_map_unlock(user_entry->backing.map);
3214	} else {
3215	object = parent_entry->backing.object;
3216	assert(object != VM_OBJECT_NULL);
3217	user_entry->backing.object = object;
3218	/ we now point to this object, hold on /
3219	vm_object_lock(object);
3220	vm_object_reference_locked(object);
3221	#if VM_OBJECT_TRACKING_OP_TRUESHARE
3222	if (!object->true_share &&
3223	vm_object_tracking_inited) {
3224	void *bt[VM_OBJECT_TRACKING_BTDEPTH];
3225	int num = `0`;
3226
3227	num = OSBacktrace(bt,
3228	VM_OBJECT_TRACKING_BTDEPTH);
3229	btlog_add_entry(vm_object_tracking_btlog,
3230	object,
3231	VM_OBJECT_TRACKING_OP_TRUESHARE,
3232	bt,
3233	num);
3234	}
3235	#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
3236
3237	object->true_share = TRUE;
3238	if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
3239	object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
3240	vm_object_unlock(object);
3241	}
3242	*size = CAST_DOWN(vm_size_t, (user_entry->size -
3243	user_entry->data_offset));
3244	*object_handle = user_handle;
3245	return KERN_SUCCESS;
3246	}
3247
3248	make_mem_done:
3249	if (user_handle != IP_NULL) {
3250	/*
3251	* Releasing "user_handle" causes the kernel object
3252	* associated with it ("user_entry" here) to also be
3253	* released and freed.
3254	*/
3255	mach_memory_entry_port_release(user_handle);
3256	}
3257	return kr;
3258	}
3259
3260	kern_return_t
3261	_mach_make_memory_entry(
3262	vm_map_t target_map,
3263	memory_object_size_t *size,
3264	memory_object_offset_t offset,
3265	vm_prot_t permission,
3266	ipc_port_t *object_handle,
3267	ipc_port_t parent_entry)
3268	{
3269	memory_object_size_t mo_size;
3270	kern_return_t kr;
3271
3272	mo_size = (memory_object_size_t)*size;
3273	kr = mach_make_memory_entry_64(target_map, &mo_size,
3274	(memory_object_offset_t)offset, permission, object_handle,
3275	parent_entry);
3276	*size = mo_size;
3277	return kr;
3278	}
3279
3280	kern_return_t
3281	mach_make_memory_entry(
3282	vm_map_t target_map,
3283	vm_size_t *size,
3284	vm_offset_t offset,
3285	vm_prot_t permission,
3286	ipc_port_t *object_handle,
3287	ipc_port_t parent_entry)
3288	{
3289	memory_object_size_t mo_size;
3290	kern_return_t kr;
3291
3292	mo_size = (memory_object_size_t)*size;
3293	kr = mach_make_memory_entry_64(target_map, &mo_size,
3294	(memory_object_offset_t)offset, permission, object_handle,
3295	parent_entry);
3296	*size = CAST_DOWN(vm_size_t, mo_size);
3297	return kr;
3298	}
3299
3300	/*
3301	* task_wire
3302	*
3303	* Set or clear the map's wiring_required flag. This flag, if set,
3304	* will cause all future virtual memory allocation to allocate
3305	* user wired memory. Unwiring pages wired down as a result of
3306	* this routine is done with the vm_wire interface.
3307	*/
3308	kern_return_t
3309	task_wire(
3310	vm_map_t map,
3311	boolean_t must_wire)
3312	{
3313	if (map == VM_MAP_NULL)
3314	return(KERN_INVALID_ARGUMENT);
3315
3316	vm_map_lock(map);
3317	map->wiring_required = (must_wire == TRUE);
3318	vm_map_unlock(map);
3319
3320	return(KERN_SUCCESS);
3321	}
3322
3323	kern_return_t
3324	vm_map_exec_lockdown(
3325	vm_map_t map)
3326	{
3327	if (map == VM_MAP_NULL)
3328	return(KERN_INVALID_ARGUMENT);
3329
3330	vm_map_lock(map);
3331	map->map_disallow_new_exec = TRUE;
3332	vm_map_unlock(map);
3333
3334	return(KERN_SUCCESS);
3335	}
3336
3337	#if VM_NAMED_ENTRY_LIST
3338	queue_head_t vm_named_entry_list;
3339	int vm_named_entry_count = `0`;
3340	lck_mtx_t vm_named_entry_list_lock_data;
3341	lck_mtx_ext_t vm_named_entry_list_lock_data_ext;
3342	#endif /* VM_NAMED_ENTRY_LIST */
3343
3344	void vm_named_entry_init(void);
3345	void
3346	vm_named_entry_init(void)
3347	{
3348	#if VM_NAMED_ENTRY_LIST
3349	queue_init(&vm_named_entry_list);
3350	vm_named_entry_count = `0`;
3351	lck_mtx_init_ext(&vm_named_entry_list_lock_data,
3352	&vm_named_entry_list_lock_data_ext,
3353	&vm_object_lck_grp,
3354	&vm_object_lck_attr);
3355	#endif /* VM_NAMED_ENTRY_LIST */
3356	}
3357
3358	__private_extern__ kern_return_t
3359	mach_memory_entry_allocate(
3360	vm_named_entry_t *user_entry_p,
3361	ipc_port_t *user_handle_p)
3362	{
3363	vm_named_entry_t user_entry;
3364	ipc_port_t user_handle;
3365	ipc_port_t previous;
3366
3367	user_entry = (vm_named_entry_t) kalloc(sizeof *user_entry);
3368	if (user_entry == NULL)
3369	return KERN_FAILURE;
3370	bzero(user_entry, sizeof (*user_entry));
3371
3372	named_entry_lock_init(user_entry);
3373
3374	user_handle = ipc_port_alloc_kernel();
3375	if (user_handle == IP_NULL) {
3376	kfree(user_entry, sizeof *user_entry);
3377	return KERN_FAILURE;
3378	}
3379	ip_lock(user_handle);
3380
3381	/ make a sonce right /
3382	user_handle->ip_sorights++;
3383	ip_reference(user_handle);
3384
3385	/ make a send right /
3386	user_handle->ip_mscount++;
3387	user_handle->ip_srights++;
3388	ip_reference(user_handle);
3389
3390	ipc_port_nsrequest(user_handle, `1`, user_handle, &previous);
3391	/ nsrequest unlocks user_handle /
3392
3393	user_entry->backing.object = NULL;
3394	user_entry->is_sub_map = FALSE;
3395	user_entry->is_copy = FALSE;
3396	user_entry->internal = FALSE;
3397	user_entry->size = `0`;
3398	user_entry->offset = `0`;
3399	user_entry->data_offset = `0`;
3400	user_entry->protection = VM_PROT_NONE;
3401	user_entry->ref_count = `1`;
3402
3403	ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
3404	IKOT_NAMED_ENTRY);
3405
3406	*user_entry_p = user_entry;
3407	*user_handle_p = user_handle;
3408
3409	#if VM_NAMED_ENTRY_LIST
3410	/ keep a loose (no reference) pointer to the Mach port, for debugging only /
3411	user_entry->named_entry_port = user_handle;
3412	/ backtrace at allocation time, for debugging only /
3413	OSBacktrace(&user_entry->named_entry_bt[`0`],
3414	NAMED_ENTRY_BT_DEPTH);
3415
3416	/ add this new named entry to the global list /
3417	lck_mtx_lock_spin(&vm_named_entry_list_lock_data);
3418	queue_enter(&vm_named_entry_list, user_entry,
3419	vm_named_entry_t, named_entry_list);
3420	vm_named_entry_count++;
3421	lck_mtx_unlock(&vm_named_entry_list_lock_data);
3422	#endif /* VM_NAMED_ENTRY_LIST */
3423
3424	return KERN_SUCCESS;
3425	}
3426
3427	/*
3428	* mach_memory_object_memory_entry_64
3429	*
3430	* Create a named entry backed by the provided pager.
3431	*
3432	*/
3433	kern_return_t
3434	mach_memory_object_memory_entry_64(
3435	host_t host,
3436	boolean_t internal,
3437	vm_object_offset_t size,
3438	vm_prot_t permission,
3439	memory_object_t pager,
3440	ipc_port_t *entry_handle)
3441	{
3442	unsigned int access;
3443	vm_named_entry_t user_entry;
3444	ipc_port_t user_handle;
3445	vm_object_t object;
3446
3447	if (host == HOST_NULL)
3448	return(KERN_INVALID_HOST);
3449
3450	if (pager == MEMORY_OBJECT_NULL && internal) {
3451	object = vm_object_allocate(size);
3452	if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
3453	object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
3454	}
3455	} else {
3456	object = memory_object_to_vm_object(pager);
3457	if (object != VM_OBJECT_NULL) {
3458	vm_object_reference(object);
3459	}
3460	}
3461	if (object == VM_OBJECT_NULL) {
3462	return KERN_INVALID_ARGUMENT;
3463	}
3464
3465	if (mach_memory_entry_allocate(&user_entry, &user_handle)
3466	!= KERN_SUCCESS) {
3467	vm_object_deallocate(object);
3468	return KERN_FAILURE;
3469	}
3470
3471	user_entry->size = size;
3472	user_entry->offset = `0`;
3473	user_entry->protection = permission & VM_PROT_ALL;
3474	access = GET_MAP_MEM(permission);
3475	SET_MAP_MEM(access, user_entry->protection);
3476	user_entry->is_sub_map = FALSE;
3477	assert(user_entry->ref_count == `1`);
3478
3479	user_entry->backing.object = object;
3480	user_entry->internal = object->internal;
3481	assert(object->internal == internal);
3482
3483	*entry_handle = user_handle;
3484	return KERN_SUCCESS;
3485	}
3486
3487	kern_return_t
3488	mach_memory_object_memory_entry(
3489	host_t host,
3490	boolean_t internal,
3491	vm_size_t size,
3492	vm_prot_t permission,
3493	memory_object_t pager,
3494	ipc_port_t *entry_handle)
3495	{
3496	return mach_memory_object_memory_entry_64( host, internal,
3497	(vm_object_offset_t)size, permission, pager, entry_handle);
3498	}
3499
3500
3501	kern_return_t
3502	mach_memory_entry_purgable_control(
3503	ipc_port_t entry_port,
3504	vm_purgable_t control,
3505	int *state)
3506	{
3507	if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
3508	/ not allowed from user-space /
3509	return KERN_INVALID_ARGUMENT;
3510	}
3511
3512	return memory_entry_purgeable_control_internal(entry_port, control, state);
3513	}
3514
3515	kern_return_t
3516	memory_entry_purgeable_control_internal(
3517	ipc_port_t entry_port,
3518	vm_purgable_t control,
3519	int *state)
3520	{
3521	kern_return_t kr;
3522	vm_named_entry_t mem_entry;
3523	vm_object_t object;
3524
3525	if (!IP_VALID(entry_port) \|\|
3526	ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3527	return KERN_INVALID_ARGUMENT;
3528	}
3529	if (control != VM_PURGABLE_SET_STATE &&
3530	control != VM_PURGABLE_GET_STATE &&
3531	control != VM_PURGABLE_SET_STATE_FROM_KERNEL)
3532	return(KERN_INVALID_ARGUMENT);
3533
3534	if ((control == VM_PURGABLE_SET_STATE \|\|
3535	control == VM_PURGABLE_SET_STATE_FROM_KERNEL) &&
3536	(((*state & ~(VM_PURGABLE_ALL_MASKS)) != `0`) \|\|
3537	((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK)))
3538	return(KERN_INVALID_ARGUMENT);
3539
3540	mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3541
3542	named_entry_lock(mem_entry);
3543
3544	if (mem_entry->is_sub_map \|\|
3545	mem_entry->is_copy) {
3546	named_entry_unlock(mem_entry);
3547	return KERN_INVALID_ARGUMENT;
3548	}
3549
3550	object = mem_entry->backing.object;
3551	if (object == VM_OBJECT_NULL) {
3552	named_entry_unlock(mem_entry);
3553	return KERN_INVALID_ARGUMENT;
3554	}
3555
3556	vm_object_lock(object);
3557
3558	/ check that named entry covers entire object ? /
3559	if (mem_entry->offset != `0` \|\| object->vo_size != mem_entry->size) {
3560	vm_object_unlock(object);
3561	named_entry_unlock(mem_entry);
3562	return KERN_INVALID_ARGUMENT;
3563	}
3564
3565	named_entry_unlock(mem_entry);
3566
3567	kr = vm_object_purgable_control(object, control, state);
3568
3569	vm_object_unlock(object);
3570
3571	return kr;
3572	}
3573
3574	kern_return_t
3575	mach_memory_entry_access_tracking(
3576	ipc_port_t entry_port,
3577	int *access_tracking,
3578	uint32_t *access_tracking_reads,
3579	uint32_t *access_tracking_writes)
3580	{
3581	return memory_entry_access_tracking_internal(entry_port,
3582	access_tracking,
3583	access_tracking_reads,
3584	access_tracking_writes);
3585	}
3586
3587	kern_return_t
3588	memory_entry_access_tracking_internal(
3589	ipc_port_t entry_port,
3590	int *access_tracking,
3591	uint32_t *access_tracking_reads,
3592	uint32_t *access_tracking_writes)
3593	{
3594	vm_named_entry_t mem_entry;
3595	vm_object_t object;
3596	kern_return_t kr;
3597
3598	if (!IP_VALID(entry_port) \|\|
3599	ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3600	return KERN_INVALID_ARGUMENT;
3601	}
3602
3603	mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3604
3605	named_entry_lock(mem_entry);
3606
3607	if (mem_entry->is_sub_map \|\|
3608	mem_entry->is_copy) {
3609	named_entry_unlock(mem_entry);
3610	return KERN_INVALID_ARGUMENT;
3611	}
3612
3613	object = mem_entry->backing.object;
3614	if (object == VM_OBJECT_NULL) {
3615	named_entry_unlock(mem_entry);
3616	return KERN_INVALID_ARGUMENT;
3617	}
3618
3619	#if VM_OBJECT_ACCESS_TRACKING
3620	vm_object_access_tracking(object,
3621	access_tracking,
3622	access_tracking_reads,
3623	access_tracking_writes);
3624	kr = KERN_SUCCESS;
3625	#else /* VM_OBJECT_ACCESS_TRACKING */
3626	(void) access_tracking;
3627	(void) access_tracking_reads;
3628	(void) access_tracking_writes;
3629	kr = KERN_NOT_SUPPORTED;
3630	#endif /* VM_OBJECT_ACCESS_TRACKING */
3631
3632	named_entry_unlock(mem_entry);
3633
3634	return kr;
3635	}
3636
3637	kern_return_t
3638	mach_memory_entry_get_page_counts(
3639	ipc_port_t entry_port,
3640	unsigned int *resident_page_count,
3641	unsigned int *dirty_page_count)
3642	{
3643	kern_return_t kr;
3644	vm_named_entry_t mem_entry;
3645	vm_object_t object;
3646	vm_object_offset_t offset;
3647	vm_object_size_t size;
3648
3649	if (!IP_VALID(entry_port) \|\|
3650	ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3651	return KERN_INVALID_ARGUMENT;
3652	}
3653
3654	mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3655
3656	named_entry_lock(mem_entry);
3657
3658	if (mem_entry->is_sub_map \|\|
3659	mem_entry->is_copy) {
3660	named_entry_unlock(mem_entry);
3661	return KERN_INVALID_ARGUMENT;
3662	}
3663
3664	object = mem_entry->backing.object;
3665	if (object == VM_OBJECT_NULL) {
3666	named_entry_unlock(mem_entry);
3667	return KERN_INVALID_ARGUMENT;
3668	}
3669
3670	vm_object_lock(object);
3671
3672	offset = mem_entry->offset;
3673	size = mem_entry->size;
3674
3675	named_entry_unlock(mem_entry);
3676
3677	kr = vm_object_get_page_counts(object, offset, size, resident_page_count, dirty_page_count);
3678
3679	vm_object_unlock(object);
3680
3681	return kr;
3682	}
3683
3684	/*
3685	* mach_memory_entry_port_release:
3686	*
3687	* Release a send right on a named entry port. This is the correct
3688	* way to destroy a named entry. When the last right on the port is
3689	* released, ipc_kobject_destroy() will call mach_destroy_memory_entry().
3690	*/
3691	void
3692	mach_memory_entry_port_release(
3693	ipc_port_t port)
3694	{
3695	assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
3696	ipc_port_release_send(port);
3697	}
3698
3699	/*
3700	* mach_destroy_memory_entry:
3701	*
3702	* Drops a reference on a memory entry and destroys the memory entry if
3703	* there are no more references on it.
3704	* NOTE: This routine should not be called to destroy a memory entry from the
3705	* kernel, as it will not release the Mach port associated with the memory
3706	* entry. The proper way to destroy a memory entry in the kernel is to
3707	* call mach_memort_entry_port_release() to release the kernel's send-right on
3708	* the memory entry's port. When the last send right is released, the memory
3709	* entry will be destroyed via ipc_kobject_destroy().
3710	*/
3711	void
3712	mach_destroy_memory_entry(
3713	ipc_port_t port)
3714	{
3715	vm_named_entry_t named_entry;
3716	#if MACH_ASSERT
3717	assert(ip_kotype(port) == IKOT_NAMED_ENTRY);
3718	#endif /* MACH_ASSERT */
3719	named_entry = (vm_named_entry_t)port->ip_kobject;
3720
3721	named_entry_lock(named_entry);
3722	named_entry->ref_count -= `1`;
3723
3724	if(named_entry->ref_count == `0`) {
3725	if (named_entry->is_sub_map) {
3726	vm_map_deallocate(named_entry->backing.map);
3727	} else if (named_entry->is_copy) {
3728	vm_map_copy_discard(named_entry->backing.copy);
3729	} else {
3730	/ release the VM object we've been pointing to /
3731	vm_object_deallocate(named_entry->backing.object);
3732	}
3733
3734	named_entry_unlock(named_entry);
3735	named_entry_lock_destroy(named_entry);
3736
3737	#if VM_NAMED_ENTRY_LIST
3738	lck_mtx_lock_spin(&vm_named_entry_list_lock_data);
3739	queue_remove(&vm_named_entry_list, named_entry,
3740	vm_named_entry_t, named_entry_list);
3741	assert(vm_named_entry_count > `0`);
3742	vm_named_entry_count--;
3743	lck_mtx_unlock(&vm_named_entry_list_lock_data);
3744	#endif /* VM_NAMED_ENTRY_LIST */
3745
3746	kfree((void *) port->ip_kobject,
3747	sizeof (struct vm_named_entry));
3748	} else
3749	named_entry_unlock(named_entry);
3750	}
3751
3752	/ Allow manipulation of individual page state. This is actually part of /
3753	/ the UPL regimen but takes place on the memory entry rather than on a UPL /
3754
3755	kern_return_t
3756	mach_memory_entry_page_op(
3757	ipc_port_t entry_port,
3758	vm_object_offset_t offset,
3759	int ops,
3760	ppnum_t *phys_entry,
3761	int *flags)
3762	{
3763	vm_named_entry_t mem_entry;
3764	vm_object_t object;
3765	kern_return_t kr;
3766
3767	if (!IP_VALID(entry_port) \|\|
3768	ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3769	return KERN_INVALID_ARGUMENT;
3770	}
3771
3772	mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3773
3774	named_entry_lock(mem_entry);
3775
3776	if (mem_entry->is_sub_map \|\|
3777	mem_entry->is_copy) {
3778	named_entry_unlock(mem_entry);
3779	return KERN_INVALID_ARGUMENT;
3780	}
3781
3782	object = mem_entry->backing.object;
3783	if (object == VM_OBJECT_NULL) {
3784	named_entry_unlock(mem_entry);
3785	return KERN_INVALID_ARGUMENT;
3786	}
3787
3788	vm_object_reference(object);
3789	named_entry_unlock(mem_entry);
3790
3791	kr = vm_object_page_op(object, offset, ops, phys_entry, flags);
3792
3793	vm_object_deallocate(object);
3794
3795	return kr;
3796	}
3797
3798	/*
3799	* mach_memory_entry_range_op offers performance enhancement over
3800	* mach_memory_entry_page_op for page_op functions which do not require page
3801	* level state to be returned from the call. Page_op was created to provide
3802	* a low-cost alternative to page manipulation via UPLs when only a single
3803	* page was involved. The range_op call establishes the ability in the _op
3804	* family of functions to work on multiple pages where the lack of page level
3805	* state handling allows the caller to avoid the overhead of the upl structures.
3806	*/
3807
3808	kern_return_t
3809	mach_memory_entry_range_op(
3810	ipc_port_t entry_port,
3811	vm_object_offset_t offset_beg,
3812	vm_object_offset_t offset_end,
3813	int ops,
3814	int *range)
3815	{
3816	vm_named_entry_t mem_entry;
3817	vm_object_t object;
3818	kern_return_t kr;
3819
3820	if (!IP_VALID(entry_port) \|\|
3821	ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
3822	return KERN_INVALID_ARGUMENT;
3823	}
3824
3825	mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
3826
3827	named_entry_lock(mem_entry);
3828
3829	if (mem_entry->is_sub_map \|\|
3830	mem_entry->is_copy) {
3831	named_entry_unlock(mem_entry);
3832	return KERN_INVALID_ARGUMENT;
3833	}
3834
3835	object = mem_entry->backing.object;
3836	if (object == VM_OBJECT_NULL) {
3837	named_entry_unlock(mem_entry);
3838	return KERN_INVALID_ARGUMENT;
3839	}
3840
3841	vm_object_reference(object);
3842	named_entry_unlock(mem_entry);
3843
3844	kr = vm_object_range_op(object,
3845	offset_beg,
3846	offset_end,
3847	ops,
3848	(uint32_t *) range);
3849
3850	vm_object_deallocate(object);
3851
3852	return kr;
3853	}
3854
3855	/ ***** Temporary Internal calls to UPL for BSD *** /
3856
3857	extern int kernel_upl_map(
3858	vm_map_t map,
3859	upl_t upl,
3860	vm_offset_t *dst_addr);
3861
3862	extern int kernel_upl_unmap(
3863	vm_map_t map,
3864	upl_t upl);
3865
3866	extern int kernel_upl_commit(
3867	upl_t upl,
3868	upl_page_info_t *pl,
3869	mach_msg_type_number_t count);
3870
3871	extern int kernel_upl_commit_range(
3872	upl_t upl,
3873	upl_offset_t offset,
3874	upl_size_t size,
3875	int flags,
3876	upl_page_info_array_t pl,
3877	mach_msg_type_number_t count);
3878
3879	extern int kernel_upl_abort(
3880	upl_t upl,
3881	int abort_type);
3882
3883	extern int kernel_upl_abort_range(
3884	upl_t upl,
3885	upl_offset_t offset,
3886	upl_size_t size,
3887	int abort_flags);
3888
3889
3890	kern_return_t
3891	kernel_upl_map(
3892	vm_map_t map,
3893	upl_t upl,
3894	vm_offset_t *dst_addr)
3895	{
3896	return vm_upl_map(map, upl, dst_addr);
3897	}
3898
3899
3900	kern_return_t
3901	kernel_upl_unmap(
3902	vm_map_t map,
3903	upl_t upl)
3904	{
3905	return vm_upl_unmap(map, upl);
3906	}
3907
3908	kern_return_t
3909	kernel_upl_commit(
3910	upl_t upl,
3911	upl_page_info_t *pl,
3912	mach_msg_type_number_t count)
3913	{
3914	kern_return_t kr;
3915
3916	kr = upl_commit(upl, pl, count);
3917	upl_deallocate(upl);
3918	return kr;
3919	}
3920
3921
3922	kern_return_t
3923	kernel_upl_commit_range(
3924	upl_t upl,
3925	upl_offset_t offset,
3926	upl_size_t size,
3927	int flags,
3928	upl_page_info_array_t pl,
3929	mach_msg_type_number_t count)
3930	{
3931	boolean_t finished = FALSE;
3932	kern_return_t kr;
3933
3934	if (flags & UPL_COMMIT_FREE_ON_EMPTY)
3935	flags \|= UPL_COMMIT_NOTIFY_EMPTY;
3936
3937	if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) {
3938	return KERN_INVALID_ARGUMENT;
3939	}
3940
3941	kr = upl_commit_range(upl, offset, size, flags, pl, count, &finished);
3942
3943	if ((flags & UPL_COMMIT_NOTIFY_EMPTY) && finished)
3944	upl_deallocate(upl);
3945
3946	return kr;
3947	}
3948
3949	kern_return_t
3950	kernel_upl_abort_range(
3951	upl_t upl,
3952	upl_offset_t offset,
3953	upl_size_t size,
3954	int abort_flags)
3955	{
3956	kern_return_t kr;
3957	boolean_t finished = FALSE;
3958
3959	if (abort_flags & UPL_COMMIT_FREE_ON_EMPTY)
3960	abort_flags \|= UPL_COMMIT_NOTIFY_EMPTY;
3961
3962	kr = upl_abort_range(upl, offset, size, abort_flags, &finished);
3963
3964	if ((abort_flags & UPL_COMMIT_FREE_ON_EMPTY) && finished)
3965	upl_deallocate(upl);
3966
3967	return kr;
3968	}
3969
3970	kern_return_t
3971	kernel_upl_abort(
3972	upl_t upl,
3973	int abort_type)
3974	{
3975	kern_return_t kr;
3976
3977	kr = upl_abort(upl, abort_type);
3978	upl_deallocate(upl);
3979	return kr;
3980	}
3981
3982	/*
3983	* Now a kernel-private interface (for BootCache
3984	* use only). Need a cleaner way to create an
3985	* empty vm_map() and return a handle to it.
3986	*/
3987
3988	kern_return_t
3989	vm_region_object_create(
3990	__unused vm_map_t target_map,
3991	vm_size_t size,
3992	ipc_port_t *object_handle)
3993	{
3994	vm_named_entry_t user_entry;
3995	ipc_port_t user_handle;
3996
3997	vm_map_t new_map;
3998
3999	if (mach_memory_entry_allocate(&user_entry, &user_handle)
4000	!= KERN_SUCCESS) {
4001	return KERN_FAILURE;
4002	}
4003
4004	/ Create a named object based on a submap of specified size /
4005
4006	new_map = vm_map_create(PMAP_NULL, VM_MAP_MIN_ADDRESS,
4007	vm_map_round_page(size,
4008	VM_MAP_PAGE_MASK(target_map)),
4009	TRUE);
4010	vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map));
4011
4012	user_entry->backing.map = new_map;
4013	user_entry->internal = TRUE;
4014	user_entry->is_sub_map = TRUE;
4015	user_entry->offset = `0`;
4016	user_entry->protection = VM_PROT_ALL;
4017	user_entry->size = size;
4018	assert(user_entry->ref_count == `1`);
4019
4020	*object_handle = user_handle;
4021	return KERN_SUCCESS;
4022
4023	}
4024
4025	ppnum_t vm_map_get_phys_page( / forward /
4026	vm_map_t map,
4027	vm_offset_t offset);
4028
4029	ppnum_t
4030	vm_map_get_phys_page(
4031	vm_map_t map,
4032	vm_offset_t addr)
4033	{
4034	vm_object_offset_t offset;
4035	vm_object_t object;
4036	vm_map_offset_t map_offset;
4037	vm_map_entry_t entry;
4038	ppnum_t phys_page = `0`;
4039
4040	map_offset = vm_map_trunc_page(addr, PAGE_MASK);
4041
4042	vm_map_lock(map);
4043	while (vm_map_lookup_entry(map, map_offset, &entry)) {
4044
4045	if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
4046	vm_map_unlock(map);
4047	return (ppnum_t) `0`;
4048	}
4049	if (entry->is_sub_map) {
4050	vm_map_t old_map;
4051	vm_map_lock(VME_SUBMAP(entry));
4052	old_map = map;
4053	map = VME_SUBMAP(entry);
4054	map_offset = (VME_OFFSET(entry) +
4055	(map_offset - entry->vme_start));
4056	vm_map_unlock(old_map);
4057	continue;
4058	}
4059	if (VME_OBJECT(entry)->phys_contiguous) {
4060	/ These are not standard pageable memory mappings /
4061	/ If they are not present in the object they will /
4062	/ have to be picked up from the pager through the /
4063	/ fault mechanism. /
4064	if (VME_OBJECT(entry)->vo_shadow_offset == `0`) {
4065	/ need to call vm_fault /
4066	vm_map_unlock(map);
4067	vm_fault(map, map_offset, VM_PROT_NONE,
4068	FALSE / change_wiring /, VM_KERN_MEMORY_NONE,
4069	THREAD_UNINT, NULL, `0`);
4070	vm_map_lock(map);
4071	continue;
4072	}
4073	offset = (VME_OFFSET(entry) +
4074	(map_offset - entry->vme_start));
4075	phys_page = (ppnum_t)
4076	((VME_OBJECT(entry)->vo_shadow_offset
4077	+ offset) >> PAGE_SHIFT);
4078	break;
4079
4080	}
4081	offset = (VME_OFFSET(entry) + (map_offset - entry->vme_start));
4082	object = VME_OBJECT(entry);
4083	vm_object_lock(object);
4084	while (TRUE) {
4085	vm_page_t dst_page = vm_page_lookup(object,offset);
4086	if(dst_page == VM_PAGE_NULL) {
4087	if(object->shadow) {
4088	vm_object_t old_object;
4089	vm_object_lock(object->shadow);
4090	old_object = object;
4091	offset = offset + object->vo_shadow_offset;
4092	object = object->shadow;
4093	vm_object_unlock(old_object);
4094	} else {
4095	vm_object_unlock(object);
4096	break;
4097	}
4098	} else {
4099	phys_page = (ppnum_t)(VM_PAGE_GET_PHYS_PAGE(dst_page));
4100	vm_object_unlock(object);
4101	break;
4102	}
4103	}
4104	break;
4105
4106	}
4107
4108	vm_map_unlock(map);
4109	return phys_page;
4110	}
4111
4112	#if 0
4113	kern_return_t kernel_object_iopl_request( / forward /
4114	vm_named_entry_t named_entry,
4115	memory_object_offset_t offset,
4116	upl_size_t *upl_size,
4117	upl_t *upl_ptr,
4118	upl_page_info_array_t user_page_list,
4119	unsigned int *page_list_count,
4120	int *flags);
4121
4122	kern_return_t
4123	kernel_object_iopl_request(
4124	vm_named_entry_t named_entry,
4125	memory_object_offset_t offset,
4126	upl_size_t *upl_size,
4127	upl_t *upl_ptr,
4128	upl_page_info_array_t user_page_list,
4129	unsigned int *page_list_count,
4130	int *flags)
4131	{
4132	vm_object_t object;
4133	kern_return_t ret;
4134
4135	int caller_flags;
4136
4137	caller_flags = *flags;
4138
4139	if (caller_flags & ~UPL_VALID_FLAGS) {
4140	/*
4141	* For forward compatibility's sake,
4142	* reject any unknown flag.
4143	*/
4144	return KERN_INVALID_VALUE;
4145	}
4146
4147	/ a few checks to make sure user is obeying rules /
4148	if(*upl_size == `0`) {
4149	if(offset >= named_entry->size)
4150	return(KERN_INVALID_RIGHT);
4151	*upl_size = (upl_size_t) (named_entry->size - offset);
4152	if (*upl_size != named_entry->size - offset)
4153	return KERN_INVALID_ARGUMENT;
4154	}
4155	if(caller_flags & UPL_COPYOUT_FROM) {
4156	if((named_entry->protection & VM_PROT_READ)
4157	!= VM_PROT_READ) {
4158	return(KERN_INVALID_RIGHT);
4159	}
4160	} else {
4161	if((named_entry->protection &
4162	(VM_PROT_READ \| VM_PROT_WRITE))
4163	!= (VM_PROT_READ \| VM_PROT_WRITE)) {
4164	return(KERN_INVALID_RIGHT);
4165	}
4166	}
4167	if(named_entry->size < (offset + *upl_size))
4168	return(KERN_INVALID_ARGUMENT);
4169
4170	/ the callers parameter offset is defined to be the /
4171	/ offset from beginning of named entry offset in object /
4172	offset = offset + named_entry->offset;
4173
4174	if (named_entry->is_sub_map \|\|
4175	named_entry->is_copy)
4176	return KERN_INVALID_ARGUMENT;
4177
4178	named_entry_lock(named_entry);
4179
4180	/ This is the case where we are going to operate /
4181	/ on an already known object. If the object is /
4182	/ not ready it is internal. An external /
4183	/ object cannot be mapped until it is ready /
4184	/ we can therefore avoid the ready check /
4185	/ in this case. /
4186	object = named_entry->backing.object;
4187	vm_object_reference(object);
4188	named_entry_unlock(named_entry);
4189
4190	if (!object->private) {
4191	if (*upl_size > MAX_UPL_TRANSFER_BYTES)
4192	*upl_size = MAX_UPL_TRANSFER_BYTES;
4193	if (object->phys_contiguous) {
4194	*flags = UPL_PHYS_CONTIG;
4195	} else {
4196	*flags = `0`;
4197	}
4198	} else {
4199	*flags = UPL_DEV_MEMORY \| UPL_PHYS_CONTIG;
4200	}
4201
4202	ret = vm_object_iopl_request(object,
4203	offset,
4204	*upl_size,
4205	upl_ptr,
4206	user_page_list,
4207	page_list_count,
4208	(upl_control_flags_t)(unsigned int)caller_flags);
4209	vm_object_deallocate(object);
4210	return ret;
4211	}
4212	#endif
4213
4214	/*
4215	* These symbols are looked up at runtime by vmware, VirtualBox,
4216	* despite not being exported in the symbol sets.
4217	*/
4218
4219	#if defined(__x86_64__)
4220
4221	kern_return_t
4222	mach_vm_map(
4223	vm_map_t target_map,
4224	mach_vm_offset_t *address,
4225	mach_vm_size_t initial_size,
4226	mach_vm_offset_t mask,
4227	int flags,
4228	ipc_port_t port,
4229	vm_object_offset_t offset,
4230	boolean_t copy,
4231	vm_prot_t cur_protection,
4232	vm_prot_t max_protection,
4233	vm_inherit_t inheritance);
4234
4235	kern_return_t
4236	mach_vm_remap(
4237	vm_map_t target_map,
4238	mach_vm_offset_t *address,
4239	mach_vm_size_t size,
4240	mach_vm_offset_t mask,
4241	int flags,
4242	vm_map_t src_map,
4243	mach_vm_offset_t memory_address,
4244	boolean_t copy,
4245	vm_prot_t *cur_protection,
4246	vm_prot_t *max_protection,
4247	vm_inherit_t inheritance);
4248
4249	kern_return_t
4250	mach_vm_map(
4251	vm_map_t target_map,
4252	mach_vm_offset_t *address,
4253	mach_vm_size_t initial_size,
4254	mach_vm_offset_t mask,
4255	int flags,
4256	ipc_port_t port,
4257	vm_object_offset_t offset,
4258	boolean_t copy,
4259	vm_prot_t cur_protection,
4260	vm_prot_t max_protection,
4261	vm_inherit_t inheritance)
4262	{
4263	return (mach_vm_map_external(target_map, address, initial_size, mask, flags, port,
4264	offset, copy, cur_protection, max_protection, inheritance));
4265	}
4266
4267	kern_return_t
4268	mach_vm_remap(
4269	vm_map_t target_map,
4270	mach_vm_offset_t *address,
4271	mach_vm_size_t size,
4272	mach_vm_offset_t mask,
4273	int flags,
4274	vm_map_t src_map,
4275	mach_vm_offset_t memory_address,
4276	boolean_t copy,
4277	vm_prot_t *cur_protection,
4278	vm_prot_t *max_protection,
4279	vm_inherit_t inheritance)
4280	{
4281	return (mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address,
4282	copy, cur_protection, max_protection, inheritance));
4283	}
4284
4285	kern_return_t
4286	vm_map(
4287	vm_map_t target_map,
4288	vm_offset_t *address,
4289	vm_size_t size,
4290	vm_offset_t mask,
4291	int flags,
4292	ipc_port_t port,
4293	vm_offset_t offset,
4294	boolean_t copy,
4295	vm_prot_t cur_protection,
4296	vm_prot_t max_protection,
4297	vm_inherit_t inheritance);
4298
4299	kern_return_t
4300	vm_map(
4301	vm_map_t target_map,
4302	vm_offset_t *address,
4303	vm_size_t size,
4304	vm_offset_t mask,
4305	int flags,
4306	ipc_port_t port,
4307	vm_offset_t offset,
4308	boolean_t copy,
4309	vm_prot_t cur_protection,
4310	vm_prot_t max_protection,
4311	vm_inherit_t inheritance)
4312	{
4313	vm_tag_t tag;
4314
4315	VM_GET_FLAGS_ALIAS(flags, tag);
4316	return vm_map_kernel(target_map, address, size, mask,
4317	flags, VM_MAP_KERNEL_FLAGS_NONE, tag,
4318	port, offset, copy,
4319	cur_protection, max_protection, inheritance);
4320	}
4321
4322	#endif /* __x86_64__ */
4323

Browse the source code of xnu/osfmk/vm/vm_user.c