pmap_data.h source code [xnu/osfmk/arm/pmap/pmap_data.h]

1	/*
2	* Copyright (c) 2020 Apple Inc. All rights reserved.
3	*
4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5	*
6	* This file contains Original Code and/or Modifications of Original Code
7	* as defined in and that are subject to the Apple Public Source License
8	* Version 2.0 (the 'License'). You may not use this file except in
9	* compliance with the License. The rights granted to you under the License
10	* may not be used to create, or enable the creation or redistribution of,
11	* unlawful or unlicensed copies of an Apple operating system, or to
12	* circumvent, violate, or enable the circumvention or violation of, any
13	* terms of an Apple operating system software license agreement.
14	*
15	* Please obtain a copy of the License at
16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
17	*
18	* The Original Code and all software distributed under the License are
19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23	* Please see the License for the specific language governing rights and
24	* limitations under the License.
25	*
26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27	*/
28	/**
29	* This header file is used to store the types, prototypes, and inline functions
30	* that define some of the most important data structures used in the pmap. This
31	* header is only meant for sharing types within the pmap; if a type is meant to
32	* be used by the rest of the kernel, then put it into osfmk/arm/pmap.h.
33	*/
34	#ifndef _ARM_PMAP_PMAP_DATA_H_
35	#define _ARM_PMAP_PMAP_DATA_H_
36
37	#include <stdint.h>
38
39	#include <kern/ledger.h>
40	#include <mach/vm_types.h>
41	#include <mach_assert.h>
42	#include <vm/vm_page.h>
43
44	#include <arm/cpu_data.h>
45	#include <arm/machine_routines.h>
46	#include <arm64/proc_reg.h>
47
48	/ Temporary include before moving all ledger functions into pmap_data.c /
49	#include <os/refcnt.h>
50
51	/**
52	* These headers are safe to be included in this file since they shouldn't rely
53	* on any of the internal pmap header files (so no circular dependencies).
54	*/
55	#include <arm/pmap.h>
56	#include <arm/pmap/pmap_pt_geometry.h>
57
58	/**
59	* These values represent the first and last kernel-managed physical addresses.
60	* We keep track of extra metadata on kernel-managed pages compared to other
61	* pages (usually iBoot carved out memory or I/O).
62	*/
63	extern pmap_paddr_t vm_first_phys, vm_last_phys;
64
65	/**
66	* Return whether the given address represents a kernel-managed physical page.
67	*
68	* Whether a page is considered "kernel-managed" is determined by the BootArgs
69	* passed by the bootloader. Typically memory carved out by the bootloader as
70	* well as I/O memory should return false.
71	*
72	* @param pa The physical address to check.
73	*/
74	static inline bool
75	pa_valid(pmap_paddr_t pa)
76	{
77	return (pa >= vm_first_phys) && (pa < vm_last_phys);
78	}
79
80	/**
81	* The pmap has a variety of data structures (pv_head_table/pp_attr_table) that
82	* contain an entry for every kernel-managed page in the system. These systems
83	* are indexed with physical address indices ("pai") generated by this function.
84	*
85	* The logic is simple since there should be one entry in each of these data
86	* structures for each kernel-managed physical page in the system. These data
87	* structures are allocated on boot based on the amount of memory available.
88	*
89	* @note PAIs are defined using the VM page size, which might not be identical
90	* to the underlying hardware page size for an arbitrary address space.
91	* This means that the data structures relying on PAIs will contain one
92	* entry for each VM page, not hardware page.
93	*
94	* @note This function is only valid for physical addresses that are
95	* kernel-managed.
96	*/
97	static inline unsigned int
98	pa_index(pmap_paddr_t pa)
99	{
100	return (unsigned int)atop(pa - vm_first_phys);
101	}
102
103	/ See the definition of pv_head_table for more information. /
104	extern pv_entry_t **pv_head_table;
105
106	/ Represents a NULL entry in the pv_head_table. /
107	#define PV_ENTRY_NULL ((pv_entry_t *) 0)
108
109	/**
110	* Given a physical address index, return the corresponding pv_head_table entry.
111	*
112	* @note Despite returning a pointer to a pv_entry_t pointer, the entry might
113	* actually be a different type of pointer (pt_entry_t or pt_desc_t)
114	* depending on the type for this entry. Determine the type using
115	* pvh_test_type().
116	*
117	* @param pai The index returned by pa_index() for the page whose pv_head_table
118	* entry should be retrieved.
119	*/
120	static inline pv_entry_t **
121	pai_to_pvh(unsigned int pai)
122	{
123	return &pv_head_table[pai];
124	}
125
126	/**
127	* Each pv_head_table entry can be one of four different types:
128	*
129	* - PVH_TYPE_NULL: No mappings to the physical page exist outside of the
130	* physical aperture. Physical aperture mappings are not
131	* tracked in the pv_head_table.
132	*
133	* - PVH_TYPE_PVEP: There are multiple mappings to the physical page.
134	* These entries are linked lists of pv_entry_t objects (which
135	* each contain a pointer to the associated PTE and a pointer
136	* to the next entry in the list).
137	*
138	* - PVH_TYPE_PTEP: There is a single mapping to the physical page. Once more
139	* mappings are created, this entry will get upgraded to an
140	* entry of type PVH_TYPE_PVEP. These entries are pointers
141	* directly to the page table entry that contain the mapping
142	* (pt_entry_t*).
143	*
144	* - PVH_TYPE_PTDP: The physical page is being used as a page table. These
145	* entries are pointers to page table descriptor structures
146	* (pt_desc_t) which contain metadata related to each page
147	* table.
148	*
149	* The type is stored in the bottom two bits of each pv_head_table entry. That
150	* type needs to be checked before dereferencing the pointer to determine which
151	* pointer type to dereference as.
152	*/
153	#define PVH_TYPE_NULL 0x0UL
154	#define PVH_TYPE_PVEP 0x1UL
155	#define PVH_TYPE_PTEP 0x2UL
156	#define PVH_TYPE_PTDP 0x3UL
157
158	#define PVH_TYPE_MASK (0x3UL)
159
160	#if defined(__arm64__)
161
162	/**
163	* PV_HEAD_TABLE Flags.
164	*
165	* All flags listed below are stored in the pv_head_table entry/pointer
166	* (per-physical-page) unless otherwise noted.
167	*
168	* Please update the pv_walk LLDB macro if these flags are changed or added to.
169	*/
170
171	/**
172	* This flag is set for every mapping created by an IOMMU.
173	*
174	* Stored in each PTE pointer (for PVH_TYPE_PVEP lists), or in the pv_head_table
175	* entry/pointer for single-PTE entries (PVH_TYPE_PTEP).
176	*/
177	#define PVH_FLAG_IOMMU 0x4UL
178
179	/**
180	* This flag is only valid when PVH_FLAG_IOMMU is set. For an IOMMU mapping, if
181	* this bit is set, then the PTE pointer points directly into the IOMMU page
182	* table for this mapping. If this bit is cleared, then the "PTE pointer" is
183	* actually a pointer to the IOMMU descriptor object that owns this mapping.
184	*
185	* There are cases where it's not easy to tie an IOMMU mapping directly to a
186	* specific page table, so this allows us to at least get a pointer to which
187	* IOMMU created this mapping which is useful for debugging purposes.
188	*
189	* Stored in each PTE pointer (for PVH_TYPE_PVEP lists), or in the pv_head_table
190	* entry/pointer for single-PTE entries (PVH_TYPE_PTEP).
191	*/
192	#define PVH_FLAG_IOMMU_TABLE (1ULL << 63)
193
194	/**
195	* This flag is set when the first CPU (non-IOMMU) mapping is created. This is
196	* important to keep track of because various accounting statistics are based on
197	* the options specified for the first CPU mapping. This flag, and thus the
198	* accounting statistics, will persist as long as there any mappings of the
199	* page (including IOMMU mappings). This works because the accounting for a page
200	* should not need to change until the page is recycled by the VM layer, and we
201	* double-check that there are no mappings (CPU or IOMMU) when a page is
202	* recycled (see: pmap_verify_free()).
203	*/
204	#define PVH_FLAG_CPU (1ULL << 62)
205
206	/ This bit is used as a lock when modifying a pv_head_table entry. /
207	#define PVH_LOCK_BIT 61
208	#define PVH_FLAG_LOCK (1ULL << PVH_LOCK_BIT)
209
210	/**
211	* This flag is set when there are any executable mappings to this physical
212	* page. This is used to prevent any writable mappings from being created at
213	* the same time an executable mapping exists.
214	*/
215	#define PVH_FLAG_EXEC (1ULL << 60)
216
217	/**
218	* Marking a pv_head_table entry with this flag denotes that this page is a
219	* kernelcache text or data page that shouldn't have dynamically-created
220	* mappings. See PVH_FLAG_LOCKDOWN_MASK for more details.
221	*/
222	#define PVH_FLAG_LOCKDOWN_KC (1ULL << 59)
223
224	/**
225	* This flag is used to mark that a page has been hashed into the hibernation
226	* image.
227	*
228	* The hibernation driver will use this to ensure that all PPL-owned memory is
229	* correctly included into the hibernation image (a missing PPL page could be
230	* a security concern when coming out of hibernation).
231	*/
232	#define PVH_FLAG_HASHED (1ULL << 58)
233
234	/**
235	* Marking a pv_head_table entry with this flag denotes that this page is a
236	* code signature page that shouldn't have dynamically-created mappings.
237	* See PVH_FLAG_LOCKDOWN_MASK for more details.
238	*/
239	#define PVH_FLAG_LOCKDOWN_CS (1ULL << 57)
240
241	/**
242	* Marking a pv_head_table entry with this flag denotes that this page is a
243	* read-only allocator page that shouldn't have dynamically-created mappings.
244	* See PVH_FLAG_LOCKDOWN_MASK for more details.
245	*/
246	#define PVH_FLAG_LOCKDOWN_RO (1ULL << 56)
247
248
249	/**
250	* Marking a pv_head_table entry with this flag denotes that this page has
251	* been mapped into a non-coherent coprocessor address space and requires a
252	* cache flush operation once all mappings have been removed.
253	*/
254	#define PVH_FLAG_FLUSH_NEEDED (1ULL << 54)
255
256	/**
257	* Marking a pv_head_table entry with any bit in this mask denotes that this page
258	* has been locked down by the PPL. Locked down pages can't have new mappings
259	* created or existing mappings removed, and all existing mappings will have been
260	* converted to read-only. This essentially makes the page immutable.
261	*/
262	#define PVH_FLAG_LOCKDOWN_MASK (PVH_FLAG_LOCKDOWN_KC \| PVH_FLAG_LOCKDOWN_CS \| PVH_FLAG_LOCKDOWN_RO)
263
264	/**
265	* These bits need to be set to safely dereference a pv_head_table
266	* entry/pointer.
267	*
268	* Any change to this #define should also update the copy located in the pmap.py
269	* LLDB macros file.
270	*/
271	#define PVH_HIGH_FLAGS (PVH_FLAG_CPU \| PVH_FLAG_LOCK \| PVH_FLAG_EXEC \| PVH_FLAG_LOCKDOWN_MASK \| \
272	PVH_FLAG_HASHED \| PVH_FLAG_FLUSH_NEEDED)
273
274	#endif /* defined(__arm64__) */
275
276	/ Mask used to clear out the TYPE bits from a pv_head_table entry/pointer. /
277	#define PVH_LIST_MASK (~PVH_TYPE_MASK)
278
279	/ Which 32-bit word in each pv_head_table entry/pointer contains the LOCK bit. /
280	#if defined(__arm64__)
281	#define PVH_LOCK_WORD 1 /* Assumes little-endian */
282	#endif /* defined(__arm64__) */
283
284	/**
285	* Assert that a pv_head_table entry is locked. Will panic if the lock isn't
286	* acquired.
287	*
288	* @param index The physical address index to check.
289	*/
290	static inline void
291	pvh_assert_locked(__assert_only unsigned int index)
292	{
293	assert((vm_offset_t)(pv_head_table[index]) & PVH_FLAG_LOCK);
294	}
295
296
297	/**
298	* Lock a pv_head_table entry.
299	*
300	* @param index The physical address index of the pv_head_table entry to lock.
301	*/
302	static inline void
303	pvh_lock(unsigned int index)
304	{
305	pmap_lock_bit((uint32_t*)(&pv_head_table[index]) + PVH_LOCK_WORD,
306	PVH_LOCK_BIT - (PVH_LOCK_WORD * `32`));
307	}
308
309	/**
310	* Unlock a pv_head_table entry.
311	*
312	* @param index The physical address index of the pv_head_table entry to unlock.
313	*/
314	static inline void
315	pvh_unlock(unsigned int index)
316	{
317	pvh_assert_locked(index);
318
319	pmap_unlock_bit((uint32_t*)(&pv_head_table[index]) + PVH_LOCK_WORD,
320	PVH_LOCK_BIT - (PVH_LOCK_WORD * `32`));
321	}
322
323	/**
324	* Check that a pv_head_table entry/pointer is a specific type.
325	*
326	* @param pvh The pv_head_table entry/pointer to check.
327	* @param type The type to check for.
328	*
329	* @return True if the pv_head_table entry is of the passed in type, false
330	* otherwise.
331	*/
332	static inline bool
333	pvh_test_type(pv_entry_t **pvh, vm_offset_t type)
334	{
335	return (((vm_offset_t )pvh) & PVH_TYPE_MASK) == type;
336	}
337
338	/**
339	* Convert a pv_head_table entry/pointer into a page table entry pointer. This
340	* should only be done if the type of this entry is PVH_TYPE_PTEP.
341	*
342	* @param pvh The pv_head_table entry/pointer to convert into a pt_entry_t*.
343	*
344	* @return Return back a safe to derefence pointer to the single mapping of this
345	* physical page by masking off the TYPE bits and adding any missing
346	* flags to the upper portion of the pointer.
347	*/
348	static inline pt_entry_t*
349	pvh_ptep(pv_entry_t **pvh)
350	{
351	return (pt_entry_t )((((vm_offset_t *)pvh) & PVH_LIST_MASK) \| PVH_HIGH_FLAGS);
352	}
353
354	/**
355	* Convert a pv_head_table entry/pointer into a PVE list pointer. This
356	* should only be done if the type of this entry is PVH_TYPE_PVEP.
357	*
358	* @param pvh The pv_head_table entry/pointer to convert into a safe to
359	* dereference pv_entry_t*.
360	*
361	* @return Return back a safe to derefence pointer to the first mapping of this
362	* physical page by masking off the TYPE bits and adding any missing
363	* flags to the upper portion of the pointer.
364	*/
365	static inline pv_entry_t*
366	pvh_pve_list(pv_entry_t **pvh)
367	{
368	return (pv_entry_t )((((vm_offset_t *)pvh) & PVH_LIST_MASK) \| PVH_HIGH_FLAGS);
369	}
370
371	/**
372	* Return the flags associated with a pv_head_table entry/pointer.
373	*
374	* @param pvh The pv_head_table entry whose flags to get.
375	*/
376	static inline vm_offset_t
377	pvh_get_flags(pv_entry_t **pvh)
378	{
379	return ((vm_offset_t )pvh) & PVH_HIGH_FLAGS;
380	}
381
382	/**
383	* Atomically set the flags associated with a pv_head_table entry/pointer.
384	*
385	* @param pvh The pv_head_table entry whose flags are getting set.
386	*/
387	static inline void
388	pvh_set_flags(pv_entry_t **pvh, vm_offset_t flags)
389	{
390	os_atomic_store((vm_offset_t )pvh, (((vm_offset_t *)pvh) & ~PVH_HIGH_FLAGS) \| flags, relaxed);
391	}
392
393	/**
394	* Update a pv_head_table entry/pointer to be a different type and/or point to
395	* a different object.
396	*
397	* @note The pv_head_table entry MUST already be locked.
398	*
399	* @note This function will clobber any existing flags stored in the PVH pointer
400	* (except PVH_FLAG_LOCK). It's up to the caller to preserve flags if that
401	* functionality is needed (either by ensuring `pvep` contains those
402	* flags, or by manually setting the flags after this call).
403	*
404	* @param pvh The pv_head_table entry/pointer to update.
405	* @param pvep The new entry to use. This could be either a pt_entry_t*,
406	* pv_entry_t, or pt_desc_t depending on the type.
407	* @param type The type of the new entry.
408	*/
409	static inline void
410	pvh_update_head(pv_entry_t *pvh, void* pvep, unsigned* int type)
411	{
412	assert(((vm_offset_t )pvh) & PVH_FLAG_LOCK);
413	os_atomic_store((vm_offset_t *)pvh, (vm_offset_t)pvep \| type \| PVH_FLAG_LOCK, relaxed);
414	}
415
416	/**
417	* Update a pv_head_table entry/pointer to be a different type and/or point to
418	* a different object.
419	*
420	* @note The pv_head_table entry CAN'T already be locked.
421	*
422	* @note This function will clobber any existing flags stored in the PVH
423	* pointer. It's up to the caller to preserve flags if that functionality
424	* is needed (either by ensuring `pvep` contains those flags, or by
425	* manually setting the flags after this call).
426	*
427	* @param pvh The pv_head_table entry/pointer to update.
428	* @param pvep The new entry to use. This could be either a pt_entry_t*,
429	* pv_entry_t, or pt_desc_t depending on the type.
430	* @param type The type of the new entry.
431	*/
432	static inline void
433	pvh_update_head_unlocked(pv_entry_t *pvh, void* pvep, unsigned* int type)
434	{
435	assert(!(((vm_offset_t )pvh) & PVH_FLAG_LOCK));
436	(vm_offset_t )pvh = ((vm_offset_t)pvep \| type) & ~PVH_FLAG_LOCK;
437	}
438
439	/**
440	* Given a page table entry pointer retrieved from the pv_head_table (from an
441	* entry of type PVH_TYPE_PTEP or PVH_TYPE_PVEP), return back whether the PTE is
442	* an IOMMU mapping.
443	*
444	* @note The way this function determines whether the passed in pointer is
445	* pointing to an IOMMU PTE, is by checking for a special flag stored in
446	* the lower bits of the pointer. This flag is only set on pointers stored
447	* in the pv_head_table, and as such, this function will only work on
448	* pointers retrieved from the pv_head_table. If a pointer to a PTE was
449	* directly retrieved from an IOMMU's page tables, this function would
450	* always return false despite actually being an IOMMU PTE.
451	*
452	* @param ptep A PTE pointer obtained from the pv_head_table to check.
453	*
454	* @return True if the entry is an IOMMU mapping, false otherwise.
455	*/
456	static inline bool
457	pvh_ptep_is_iommu(const pt_entry_t *ptep)
458	{
459	#ifdef PVH_FLAG_IOMMU
460	return (vm_offset_t)ptep & PVH_FLAG_IOMMU;
461	#else /* PVH_FLAG_IOMMU */
462	#pragma unused(ptep)
463	return false;
464	#endif /* PVH_FLAG_IOMMU */
465	}
466
467	/**
468	* Sometimes the PTE pointers retrieved from the pv_head_table (from an entry of
469	* type PVH_TYPE_PTEP or PVH_TYPE_PVEP) contain flags themselves. This function
470	* strips out those flags and returns back a dereferencable pointer.
471	*
472	* @param ptep The PTE pointer to strip out the unwanted flags.
473	*
474	* @return A valid dereferencable pointer to the page table entry.
475	*/
476	static inline const pt_entry_t*
477	pvh_strip_ptep(const pt_entry_t *ptep)
478	{
479	#ifdef PVH_FLAG_IOMMU
480	const vm_offset_t pte_va = (vm_offset_t)ptep;
481	return (const pt_entry_t*)((pte_va & ~PVH_FLAG_IOMMU) \| PVH_FLAG_IOMMU_TABLE);
482	#else /* PVH_FLAG_IOMMU */
483	return ptep;
484	#endif /* PVH_FLAG_IOMMU */
485	}
486
487	/**
488	* PVH_TYPE_PVEP Helper Functions.
489	*
490	* The following are methods used to manipulate PVE lists. This is the type of
491	* pv_head_table entry used when there are multiple mappings to a single
492	* physical page.
493	*/
494
495	/**
496	* Whether a physical page is using "alternate accounting" (ALTACCT) for its
497	* ledger statistics is something that needs to be tracked on a per-mapping
498	* basis, not on a per-physical-page basis. Because of that, it's tracked
499	* differently depending on whether there's a single mapping to a page
500	* (PVH_TYPE_PTEP) or multiple (PVH_TYPE_PVEP). For single mappings, the bit is
501	* tracked in the pp_attr_table. But when there are multiple mappings, the least
502	* significant bit of the corresponding "pve_pte" pointer in each pv_entry object
503	* is used as a marker for pages using alternate accounting.
504	*
505	* @note See the definition for PP_ATTR_ALTACCT for a more detailed description
506	* of what "alternate accounting" actually means in respect to the
507	* footprint ledger.
508	*
509	* Since some code (KernelDiskImages, e.g.) might map a phsyical page as
510	* "device" memory (i.e. external) while it's also being used as regular
511	* "anonymous" memory (i.e. internal) in user space, we have to manage the
512	* "internal" attribute per mapping rather than per physical page.
513	* When there are multiple mappings, we use the next least significant bit of
514	* the corresponding "pve_pte" pointer for that.
515	*/
516	#define PVE_PTEP_ALTACCT ((uintptr_t) 0x1)
517	#define PVE_PTEP_INTERNAL ((uintptr_t) 0x2)
518	#define PVE_PTEP_FLAGS (PVE_PTEP_ALTACCT \| PVE_PTEP_INTERNAL)
519
520	/**
521	* Set the ALTACCT bit for a specific PTE pointer.
522	*
523	* @param pvep A pointer to the current pv_entry mapping in the linked list of
524	* mappings.
525	* @param idx Index of the chosen PTE pointer inside the PVE.
526	*/
527	static inline void
528	pve_set_altacct(pv_entry_t pvep, unsigned* idx)
529	{
530	assert(idx < PTE_PER_PVE);
531	pvep->pve_ptep[idx] = (pt_entry_t *)((uintptr_t)pvep->pve_ptep[idx] \| PVE_PTEP_ALTACCT);
532	}
533	/**
534	* Set the INTERNAL bit for a specific PTE pointer.
535	*
536	* @param pvep A pointer to the current pv_entry mapping in the linked list of
537	* mappings.
538	* @param idx Index of the chosen PTE pointer inside the PVE.
539	*/
540	static inline void
541	pve_set_internal(pv_entry_t pvep, unsigned* idx)
542	{
543	assert(idx < PTE_PER_PVE);
544	pvep->pve_ptep[idx] = (pt_entry_t *)((uintptr_t)pvep->pve_ptep[idx] \| PVE_PTEP_INTERNAL);
545	}
546
547	/**
548	* Clear the ALTACCT bit for a specific PTE pointer.
549	*
550	* @param pvep A pointer to the current pv_entry mapping in the linked list of
551	* mappings.
552	* @param idx Index of the chosen PTE pointer inside the PVE.
553	*/
554	static inline void
555	pve_clr_altacct(pv_entry_t pvep, unsigned* idx)
556	{
557	assert(idx < PTE_PER_PVE);
558	pvep->pve_ptep[idx] = (pt_entry_t *)((uintptr_t)pvep->pve_ptep[idx] & ~PVE_PTEP_ALTACCT);
559	}
560	/**
561	* Clear the INTERNAL bit for a specific PTE pointer.
562	*
563	* @param pvep A pointer to the current pv_entry mapping in the linked list of
564	* mappings.
565	* @param idx Index of the chosen PTE pointer inside the PVE.
566	*/
567	static inline void
568	pve_clr_internal(pv_entry_t pvep, unsigned* idx)
569	{
570	assert(idx < PTE_PER_PVE);
571	pvep->pve_ptep[idx] = (pt_entry_t *)((uintptr_t)pvep->pve_ptep[idx] & ~PVE_PTEP_INTERNAL);
572	}
573
574	/**
575	* Return the ALTACCT bit for a specific PTE pointer.
576	*
577	* @param pvep A pointer to the current pv_entry mapping in the linked list of
578	* mappings.
579	* @param idx Index of the chosen PTE pointer inside the PVE.
580	*/
581	static inline bool
582	pve_get_altacct(pv_entry_t pvep, unsigned* idx)
583	{
584	assert(idx < PTE_PER_PVE);
585	return (uintptr_t)pvep->pve_ptep[idx] & PVE_PTEP_ALTACCT;
586	}
587	/**
588	* Return the INTERNAL bit for a specific PTE pointer.
589	*
590	* @param pvep A pointer to the current pv_entry mapping in the linked list of
591	* mappings.
592	* @param idx Index of the chosen PTE pointer inside the PVE.
593	*/
594	static inline bool
595	pve_get_internal(pv_entry_t pvep, unsigned* idx)
596	{
597	assert(idx < PTE_PER_PVE);
598	return (uintptr_t)pvep->pve_ptep[idx] & PVE_PTEP_INTERNAL;
599	}
600
601	/**
602	* Return the next mapping (pv_entry) in a linked list of mappings. This applies
603	* to pv_head_table entries of type PVH_TYPE_PVEP.
604	*
605	* @param pvep A pointer to the current pv_entry mapping in the linked list of
606	* mappings.
607	*
608	* @return The next virtual mapping for a physical page, or PV_ENTRY_NULL if the
609	* end of the list has been reached.
610	*/
611	static inline pv_entry_t *
612	pve_next(pv_entry_t *pvep)
613	{
614	return pvep->pve_next;
615	}
616
617	/**
618	* Return a pointer to the pve_next field in a pv_entry. This value is used
619	* when adding and removing entries to a PVE list.
620	*
621	* @param pvep The pv_entry whose pve_next field is being accessed.
622	*
623	* @return Pointer to the pve_next field.
624	*/
625	static inline pv_entry_t **
626	pve_next_ptr(pv_entry_t *pvep)
627	{
628	return &pvep->pve_next;
629	}
630
631	/**
632	* Return a pointer to the page table entry for this mapping.
633	*
634	* @param pvep The pv_entry whose pve_ptep field is to be returned.
635	* @param idx Index of the chosen PTE pointer inside the PVE.
636	*
637	* @return Pointer to the page table entry.
638	*/
639	static inline pt_entry_t *
640	pve_get_ptep(pv_entry_t pvep, unsigned* idx)
641	{
642	assert(idx < PTE_PER_PVE);
643	return (pt_entry_t *)((uintptr_t)pvep->pve_ptep[idx] & ~PVE_PTEP_FLAGS);
644	}
645
646	/**
647	* Update the page table entry for a specific physical to virtual mapping.
648	*
649	* @param pvep The pv_entry to update.
650	* @param idx Index of the chosen PTE pointer inside the PVE.
651	* @param ptep_new The new page table entry.
652	*/
653	static inline void
654	pve_set_ptep(pv_entry_t pvep, unsigned* idx, pt_entry_t *ptep_new)
655	{
656	assert(idx < PTE_PER_PVE);
657	pvep->pve_ptep[idx] = ptep_new;
658	}
659
660	/**
661	* Initialize all fields in a PVE to NULL.
662	*
663	* @param pvep The pv_entry to initialize.
664	*/
665	static inline void
666	pve_init(pv_entry_t *pvep)
667	{
668	pvep->pve_next = PV_ENTRY_NULL;
669	for (int i = `0`; i < PTE_PER_PVE; i++) {
670	pvep->pve_ptep[i] = PT_ENTRY_NULL;
671	}
672	}
673
674	/**
675	* Find PTE pointer in PVE and return its index.
676	*
677	* @param pvep The PVE to search.
678	* @param ptep PTE to search for.
679	*
680	* @return Index of the found entry, or -1 if no entry exists.
681	*/
682	static inline int
683	pve_find_ptep_index(pv_entry_t pvep, pt_entry_t ptep)
684	{
685	for (unsigned int i = `0`; i < PTE_PER_PVE; i++) {
686	if (pve_get_ptep(pvep, idx: i) == ptep) {
687	return (int)i;
688	}
689	}
690
691	return -`1`;
692	}
693
694	/**
695	* Checks if no PTEs are currently associated with this PVE.
696	*
697	* @param pvep The PVE to search.
698	*
699	* @return True if no PTEs are currently associated with this PVE, or false.
700	*/
701	static inline bool
702	pve_is_empty(pv_entry_t *pvep)
703	{
704	for (unsigned int i = `0`; i < PTE_PER_PVE; i++) {
705	if (pve_get_ptep(pvep, idx: i) != PT_ENTRY_NULL) {
706	return false;
707	}
708	}
709
710	return true;
711	}
712
713	/**
714	* Prepend a new pv_entry node to a PVE list.
715	*
716	* @note This function will clobber any existing flags stored in the PVH
717	* pointer. It's up to the caller to preserve flags if that functionality
718	* is needed (either by ensuring `pvep` contains those flags, or by
719	* manually setting the flags after this call).
720	*
721	* @param pvh The linked list of mappings to update.
722	* @param pvep The new mapping to add to the linked list.
723	*/
724	static inline void
725	pve_add(pv_entry_t *pvh, pv_entry_t pvep)
726	{
727	assert(pvh_test_type(pvh, PVH_TYPE_PVEP));
728
729	pvep->pve_next = pvh_pve_list(pvh);
730	pvh_update_head(pvh, pvep, PVH_TYPE_PVEP);
731	}
732
733	/**
734	* Remove an entry from a PVE list of mappings.
735	*
736	* @note This function will clobber any existing flags stored in the PVH
737	* pointer. It's up to the caller to preserve flags if that functionality
738	* is needed.
739	*
740	* @param pvh The pv_head_table entry of the PVE list to remove a mapping from.
741	* This is the first entry in the list of pv_entry_t mappings.
742	* @param pvepp A pointer to the pv_entry_t* that's being removed. If this entry
743	* is the first in the linked list of mappings, then this should be
744	* identical to the pv_head_table entry. If the mapping isn't the
745	* first, then this is a pointer to the pve_next field in the
746	* previous mapping.
747	* @param pvep The entry that should be removed. Should be identical to a
748	* dereference of the pvepp parameter (unless it's the pv_head_table
749	* entry).
750	*/
751	static inline void
752	pve_remove(pv_entry_t pvh, pv_entry_t pvepp, pv_entry_t *pvep)
753	{
754	assert(pvh_test_type(pvh, PVH_TYPE_PVEP));
755
756	if (pvepp == pvh) {
757	if (pve_next(pvep) == PV_ENTRY_NULL) {
758	/ The last mapping to this page is being removed. /
759	pvh_update_head(pvh, PV_ENTRY_NULL, PVH_TYPE_NULL);
760	} else {
761	/**
762	* There are still mappings left, make the next one the new head of
763	* the list. This effectively removes the first entry from the list.
764	*/
765	pvh_update_head(pvh, pvep: pve_next(pvep), PVH_TYPE_PVEP);
766	}
767	} else {
768	/**
769	* Move the previous entry's next field to the entry after the one being
770	* removed. This will clobber the ALTACCT and INTERNAL bits.
771	*/
772	*pvepp = pve_next(pvep);
773	}
774	}
775
776	/**
777	* PVH_TYPE_PTDP Types and Helper Functions.
778	*
779	* The following are types and methods used to manipulate page table descriptor
780	* (PTD) objects. This is the type of pv_head_table entry used when a page is
781	* being used as a page table.
782	*/
783
784	/**
785	* When the pmap layer allocates memory, it always does so in chunks of the VM
786	* page size (which are represented by the PAGE_SIZE/PAGE_SHIFT macros). The VM
787	* page size might not match up with the hardware page size for a given address
788	* space (this is especially true on systems that support more than one page
789	* size).
790	*
791	* The pv_head_table is allocated to have one entry per VM page, not hardware
792	* page (which can change depending on the address space). Because of that, a
793	* single VM-page-sized region (single pv_head_table entry) can potentially hold
794	* up to four page tables. Only one page table descriptor (PTD) is allocated per
795	* pv_head_table entry (per VM page), so on some systems, one PTD might have to
796	* keep track of up to four different page tables.
797	*/
798
799	#if __ARM_MIXED_PAGE_SIZE__
800	#define PT_INDEX_MAX (ARM_PGBYTES / 4096)
801	#elif (ARM_PGSHIFT == 14)
802	#define PT_INDEX_MAX 1
803	#elif (ARM_PGSHIFT == 12)
804	#define PT_INDEX_MAX 4
805	#else
806	#error Unsupported ARM_PGSHIFT
807	#endif /* __ARM_MIXED_PAGE_SIZE__ \|\| ARM_PGSHIFT == 14 \|\| ARM_PGSHIFT == 12 */
808
809
810	/**
811	* Page table descriptor (PTD) info structure.
812	*
813	* Contains information about a page table. These pieces of data are separate
814	* from the PTD itself because in address spaces where the VM page size doesn't
815	* match the underlying hardware page size, one PTD could represent multiple
816	* page tables (and so will need multiple PTD info structures).
817	*
818	* These fields are also in their own struct so that they can be allocated
819	* separately from the associated pt_desc_t object. This allows us to allocate
820	* the counts in this structure in a way that ensures they don't fall within the
821	* same cache line as the main pt_desc_t object. This is important because the
822	* fields in this structure are atomically updated which could cause false
823	* sharing cache performance issues with the "va" field in pt_desc_t if all of
824	* the fields were within the same structure.
825	*/
826	typedef struct {
827	/**
828	* Pre-defined sentinel values for ptd_info_t.refcnt. If these refcnt values
829	* change, make sure to update the showpte LLDB macro to reflect the
830	* changes.
831	*/
832	#define PT_DESC_REFCOUNT 0x4000U
833	#define PT_DESC_IOMMU_GRANTED_REFCOUNT 0x8000U
834	#define PT_DESC_IOMMU_ACCEPTED_REFCOUNT 0x8001U
835
836	/*
837	* For non-leaf pagetables, should always be PT_DESC_REFCOUNT.
838	* For leaf pagetables, should reflect the number of non-empty PTEs.
839	* For IOMMU pages, should always be either PT_DESC_IOMMU_GRANTED_REFCOUNT
840	* or PT_DESC_IOMMU_ACCEPTED_REFCOUNT.
841	*/
842	unsigned short refcnt;
843
844	/*
845	* For non-leaf pagetables, should be 0.
846	* For leaf pagetables, should reflect the number of wired entries.
847	* For IOMMU pages, may optionally reflect a driver-defined refcount (IOMMU
848	* operations are implicitly wired).
849	*/
850	unsigned short wiredcnt;
851	} ptd_info_t;
852
853	/**
854	* Page Table Descriptor (PTD).
855	*
856	* Provides a per-table data structure and a way of keeping track of all page
857	* tables in the system.
858	*
859	* This structure is also used as a convenient way of keeping track of IOMMU
860	* pages (which may or may not be used as page tables). In that case the "iommu"
861	* field will point to the owner of the page, ptd_info[0].refcnt will be
862	* PT_DESC_IOMMU_GRANTED_REFCOUNT or PT_DESC_IOMMU_ACCEPTED_REFCOUNT, and
863	* ptd_info[0].wiredcnt can be used as an arbitrary refcnt controlled by the
864	* IOMMU driver.
865	*/
866	typedef struct pt_desc {
867	/**
868	* This queue chain provides a mechanism for keeping a list of pages
869	* being used as page tables. This is used to potentially reclaim userspace
870	* page tables as a fast way of "allocating" a page.
871	*
872	* Refer to osfmk/kern/queue.h for more information about queue chains.
873	*/
874	queue_chain_t pt_page;
875
876	/ Each page table is either owned by a pmap or a specific IOMMU. /
877	union {
878	struct pmap *pmap;
879	};
880
881	/**
882	* The following fields contain per-page-table properties, and as such,
883	* might have multiple elements each. This is due to a single PTD
884	* potentially representing multiple page tables (in address spaces where
885	* the VM page size differs from the hardware page size). Use the
886	* ptd_get_index() function to get the correct index for a specific page
887	* table.
888	*/
889
890	/**
891	* The first address of the virtual address space this page table is
892	* translating for, or a value set by an IOMMU driver if this PTD is being
893	* used to track an IOMMU page.
894	*/
895	vm_offset_t va[PT_INDEX_MAX];
896
897	/**
898	* ptd_info_t's are allocated separately so as to reduce false sharing
899	* with the va field. This is desirable because ptd_info_t's are updated
900	* atomically from all CPUs.
901	*/
902	ptd_info_t *ptd_info;
903	} pt_desc_t;
904
905	/**
906	* Convert a pv_head_table entry/pointer into a page table descriptor pointer.
907	* This should only be done if the type of this entry is PVH_TYPE_PTDP.
908	*
909	* @param pvh The pv_head_table entry/pointer to convert into a safe to
910	* dereference pt_desc_t*.
911	*
912	* @return Return back a safe to derefence pointer to the page table descriptor
913	* for this physical page by masking off the TYPE bits and adding any
914	* missing flags to the upper portion of the pointer.
915	*/
916	static inline pt_desc_t*
917	pvh_ptd(pv_entry_t **pvh)
918	{
919	return (pt_desc_t )((((vm_offset_t *)pvh) & PVH_LIST_MASK) \| PVH_HIGH_FLAGS);
920	}
921
922	/**
923	* Given an arbitrary page table entry, return back the page table descriptor
924	* (PTD) object for the page table that contains that entry.
925	*
926	* @param ptep Pointer to a PTE whose page table descriptor object to return.
927	*
928	* @return The PTD object for the passed in page table.
929	*/
930	static inline pt_desc_t *
931	ptep_get_ptd(const pt_entry_t *ptep)
932	{
933	assert(ptep != NULL);
934
935	const vm_offset_t pt_base_va = (vm_offset_t)ptep;
936	pv_entry_t **pvh = pai_to_pvh(pai: pa_index(pa: ml_static_vtop(pt_base_va)));
937
938	if (__improbable(!pvh_test_type(pvh, PVH_TYPE_PTDP))) {
939	panic("%s: invalid PV head 0x%llx for PTE %p", __func__, (uint64_t)(*pvh), ptep);
940	}
941
942	return pvh_ptd(pvh);
943	}
944
945	/**
946	* Given an arbitrary page table entry, return back the pmap that owns that
947	* page table.
948	*
949	* @note This won't work correctly for page tables owned by IOMMUs, because
950	* those table aren't owned by any specific pmap.
951	*
952	* @param ptep Pointer to a page table entry whose owner we're trying to return.
953	*
954	* @return The pmap that owns the given page table entry.
955	*/
956	static inline struct pmap *
957	ptep_get_pmap(const pt_entry_t *ptep)
958	{
959	return ptep_get_ptd(ptep)->pmap;
960	}
961
962
963	/**
964	* Given an arbitrary translation table entry, get the page table descriptor
965	* (PTD) object for the page table pointed to by the TTE.
966	*
967	* @param tte The translation table entry to parse. For instance, if this is an
968	* L2 TTE, then the PTD for the L3 table this entry points to will be
969	* returned.
970	*
971	* @return The page table descriptor (PTD) for the page table pointed to by this
972	* TTE.
973	*/
974	static inline pt_desc_t *
975	tte_get_ptd(const tt_entry_t tte)
976	{
977	const vm_offset_t pt_base_va = (vm_offset_t)(tte & ~((tt_entry_t)PAGE_MASK));
978	pv_entry_t **pvh = pai_to_pvh(pai: pa_index(pa: pt_base_va));
979
980	if (__improbable(!pvh_test_type(pvh, PVH_TYPE_PTDP))) {
981	panic("%s: invalid PV head 0x%llx for TTE 0x%llx", __func__, (uint64_t)(*pvh), (uint64_t)tte);
982	}
983
984	return pvh_ptd(pvh);
985	}
986
987	/**
988	* In address spaces where the VM page size doesn't match the underlying
989	* hardware page size, one PTD could represent multiple page tables. This
990	* function returns the correct index value depending on which page table is
991	* being accessed. That index value can then be used to access the
992	* per-page-table properties stored within a PTD.
993	*
994	* @note See the description above the PT_INDEX_MAX definition for a more
995	* detailed explanation of why multiple page tables can be represented
996	* by a single PTD object in the pv_head_table.
997	*
998	* @param ptd The page table descriptor that's being accessed.
999	* @param ttep Pointer to the translation table entry that's being accessed.
1000	*
1001	* @return The correct index value for a specific, hardware-sized page
1002	* table.
1003	*/
1004	static inline unsigned
1005	ptd_get_index(__unused const pt_desc_t ptd, __unused const* tt_entry_t *ttep)
1006	{
1007	#if PT_INDEX_MAX == 1
1008	return `0`;
1009	#else
1010	assert(ptd != NULL);
1011
1012	const uint64_t pmap_page_shift = pt_attr_leaf_shift(pmap_get_pt_attr(ptd->pmap));
1013	const vm_offset_t ttep_page = (vm_offset_t)ttep >> pmap_page_shift;
1014
1015	/**
1016	* Use the difference between the VM page shift and the hardware page shift
1017	* to get the index of the correct page table. In practice, this equates to
1018	* masking out the bottom two bits of the L3 table index in address spaces
1019	* where the VM page size is greater than the hardware page size. In address
1020	* spaces where they're identical, the index will always be zero.
1021	*/
1022	const unsigned int ttep_index = ttep_page & ((`1U` << (PAGE_SHIFT - pmap_page_shift)) - `1`);
1023	assert(ttep_index < PT_INDEX_MAX);
1024
1025	return ttep_index;
1026	#endif
1027	}
1028
1029	/**
1030	* In address spaces where the VM page size doesn't match the underlying
1031	* hardware page size, one PTD could represent multiple page tables. This
1032	* function returns the correct ptd_info_t structure depending on which page
1033	* table is being accessed.
1034	*
1035	* @note See the description above the PT_INDEX_MAX definition for a more
1036	* detailed explanation of why multiple page tables can be represented
1037	* by a single PTD object in the pv_head_table.
1038	*
1039	* @param ptd The page table descriptor that's being accessed.
1040	* @param ttep Pointer to the translation table entry that's being accessed.
1041	*
1042	* @return The correct ptd_info_t structure for a specific, hardware-sized page
1043	* table.
1044	*/
1045	static inline ptd_info_t *
1046	ptd_get_info(pt_desc_t ptd, const* tt_entry_t *ttep)
1047	{
1048	assert((ptd != NULL) && (ptd->ptd_info[`0`].refcnt < PT_DESC_IOMMU_GRANTED_REFCOUNT));
1049
1050	return &ptd->ptd_info[ptd_get_index(ptd, ttep)];
1051	}
1052
1053	/**
1054	* Given a pointer to a page table entry, return back the ptd_info structure
1055	* for the page table that contains that entry.
1056	*
1057	* @param ptep Pointer to a PTE whose ptd_info object to return.
1058	*
1059	* @return The ptd_info object for the page table that contains the passed in
1060	* page table entry.
1061	*/
1062	static inline ptd_info_t *
1063	ptep_get_info(const pt_entry_t *ptep)
1064	{
1065	return ptd_get_info(ptd: ptep_get_ptd(ptep), ttep: ptep);
1066	}
1067
1068	/**
1069	* Return the virtual address mapped by the passed in leaf page table entry,
1070	* using an already-retrieved pagetable descriptor.
1071	*
1072	* @param ptdp pointer to the descriptor for the pagetable containing ptep
1073	* @param ptep Pointer to a PTE to parse
1074	*/
1075	static inline vm_map_address_t
1076	ptd_get_va(const pt_desc_t ptdp, const* pt_entry_t *ptep)
1077	{
1078	const pt_attr_t * const pt_attr = pmap_get_pt_attr(ptdp->pmap);
1079
1080	vm_map_address_t va = ptdp->va[ptd_get_index(ptd: ptdp, ttep: ptep)];
1081	vm_offset_t ptep_index = ((vm_offset_t)ptep & pt_attr_leaf_offmask(pt_attr)) / sizeof(*ptep);
1082
1083	va += (ptep_index << pt_attr_leaf_shift(pt_attr));
1084
1085	return va;
1086	}
1087
1088	/**
1089	* Return the virtual address that is being mapped by the passed in leaf page
1090	* table entry.
1091	*
1092	* @param ptep Pointer to a PTE to parse.
1093	*/
1094	static inline vm_map_address_t
1095	ptep_get_va(const pt_entry_t *ptep)
1096	{
1097	return ptd_get_va(ptdp: ptep_get_ptd(ptep), ptep);
1098	}
1099
1100	/**
1101	* Physical Page Attribute Table (pp_attr_table) defines and helper functions.
1102	*/
1103
1104	/ How many bits to use for flags on a per-VM-page basis. /
1105	typedef uint16_t pp_attr_t;
1106
1107	/ See the definition of pp_attr_table for more information. /
1108	extern volatile pp_attr_t* pp_attr_table;
1109
1110	/**
1111	* Flags stored in the pp_attr_table on a per-physical-page basis.
1112	*
1113	* Please update the pv_walk LLDB macro if these flags are changed or added to.
1114	*/
1115
1116	/**
1117	* The bottom 6-bits are used to store the default WIMG (cacheability and memory
1118	* type) setting for this physical page. This can be changed by calling
1119	* pmap_set_cache_attributes().
1120	*
1121	* If a default WIMG setting isn't set for a page, then the default is Normal,
1122	* Cached memory (VM_WIMG_DEFAULT).
1123	*/
1124	#define PP_ATTR_WIMG_MASK 0x003F
1125	#define PP_ATTR_WIMG(x) ((x) & PP_ATTR_WIMG_MASK)
1126
1127	/**
1128	* The reference and modify bits keep track of whether a page has been accessed
1129	* or modified since the last time the bits were cleared. These bits are used to
1130	* enforce policy decisions in the VM layer.
1131	*/
1132	#define PP_ATTR_REFERENCED 0x0040
1133	#define PP_ATTR_MODIFIED 0x0080
1134
1135	/**
1136	* This physical page is being used as anonymous memory that's internally
1137	* managed by the VM and is not connected to an external pager. This flag is
1138	* only set/cleared on the first CPU mapping of a page (see PVH_FLAG_CPU). Any
1139	* subsequent mappings won't set/clear this flag until all mappings are removed
1140	* and a new CPU mapping is added.
1141	*/
1142	#define PP_ATTR_INTERNAL 0x0100
1143
1144	/**
1145	* This flag is used to keep track of pages that are still resident but are not
1146	* considered dirty and can be reclaimed under memory pressure. These pages do
1147	* not count as a part of the memory footprint, so the footprint ledger does not
1148	* need to be updated for these pages. This is hinted to the VM by the
1149	* `madvise(MADV_FREE_REUSABLE)` system call.
1150	*/
1151	#define PP_ATTR_REUSABLE 0x0200
1152
1153	/**
1154	* This flag denotes that a page is utilizing "alternate accounting". This means
1155	* that the pmap doesn't need to keep track of these pages with regards to the
1156	* footprint ledger because the VM is already accounting for them in a different
1157	* way. These include IOKit mappings (VM adds their entire virtual size to the
1158	* footprint), and purgeable pages (VM counts them only when non-volatile and
1159	* only for one "owner"), among others.
1160	*
1161	* Note that alternate accounting status is tracked on a per-mapping basis (not
1162	* per-page). Because of that the ALTACCT flag in the pp_attr_table is only used
1163	* when there's a single mapping to a page. When there are multiple mappings,
1164	* the status of this flag is tracked in the pv_head_table (see PVE_PTEP_ALTACCT
1165	* above).
1166	*/
1167	#define PP_ATTR_ALTACCT 0x0400
1168
1169	/**
1170	* This bit was originally used on x86 to keep track of what pages to not
1171	* encrypt during the hibernation process as a performance optimization when
1172	* encryption was done in software. This doesn't apply to the ARM
1173	* hibernation process because all pages are automatically encrypted using
1174	* hardware acceleration. Despite that, the pmap still keeps track of this flag
1175	* as a debugging aid on internal builds.
1176	*
1177	* TODO: This bit can probably be reclaimed:
1178	* rdar://70740650 (PMAP Cleanup: Potentially reclaim the PP_ATTR_NOENCRYPT bit on ARM)
1179	*/
1180	#define PP_ATTR_NOENCRYPT 0x0800
1181
1182	/**
1183	* These bits denote that a physical page is expecting the next access or
1184	* modification to set the PP_ATTR_REFERENCED and PP_ATTR_MODIFIED flags
1185	* respectively.
1186	*/
1187	#define PP_ATTR_REFFAULT 0x1000
1188	#define PP_ATTR_MODFAULT 0x2000
1189
1190	#if XNU_MONITOR
1191	/**
1192	* Denotes that a page is owned by the PPL. This is modified/checked with the
1193	* PVH lock held, to avoid ownership related races. This does not need to be a
1194	* PP_ATTR bit (as we have the lock), but for now this is a convenient place to
1195	* put the bit.
1196	*/
1197	#define PP_ATTR_MONITOR 0x4000
1198
1199	/**
1200	* Denotes that a page cannot be owned by the PPL. This is required in order
1201	* to temporarily 'pin' kernel pages that are used to store PPL output
1202	* parameters. Otherwise a malicious or buggy caller could pass PPL-owned memory
1203	* for these parameters and in so doing stage a write gadget against the PPL.
1204	*/
1205	#define PP_ATTR_NO_MONITOR 0x8000
1206
1207	/**
1208	* All of the bits owned by the PPL; kernel requests to set or clear these bits
1209	* are illegal.
1210	*/
1211	#define PP_ATTR_PPL_OWNED_BITS (PP_ATTR_MONITOR \| PP_ATTR_NO_MONITOR)
1212	#endif /* XNU_MONITOR */
1213
1214	/**
1215	* Atomically set some flags in a pp_attr_table entry.
1216	*
1217	* @param pai The physical address index for the entry to update.
1218	* @param bits The flags to set in the entry.
1219	*/
1220	static inline void
1221	ppattr_set_bits(unsigned int pai, pp_attr_t bits)
1222	{
1223	volatile pp_attr_t *ppattr = &pp_attr_table[pai];
1224	os_atomic_or(ppattr, bits, acq_rel);
1225	}
1226
1227	/**
1228	* Atomically clear some flags in a pp_attr_table entry.
1229	*
1230	* @param pai The physical address index for the entry to update.
1231	* @param bits The flags to clear in the entry.
1232	*/
1233	static inline void
1234	ppattr_clear_bits(unsigned int pai, pp_attr_t bits)
1235	{
1236	volatile pp_attr_t *ppattr = &pp_attr_table[pai];
1237	os_atomic_andnot(ppattr, bits, acq_rel);
1238	}
1239
1240	/**
1241	* Return true if the pp_attr_table entry contains the passed in bits.
1242	*
1243	* @param pai The physical address index for the entry to test.
1244	* @param bits The flags to check for.
1245	*/
1246	static inline bool
1247	ppattr_test_bits(unsigned int pai, pp_attr_t bits)
1248	{
1249	const volatile pp_attr_t *ppattr = &pp_attr_table[pai];
1250	return (*ppattr & bits) == bits;
1251	}
1252
1253	/**
1254	* Only set some flags in a pp_attr_table entry if the passed in physical
1255	* address is a kernel-managed address.
1256	*
1257	* @param pa The physical address for the entry to update.
1258	* @param bits The flags to set in the entry.
1259	*/
1260	static inline void
1261	ppattr_pa_set_bits(pmap_paddr_t pa, pp_attr_t bits)
1262	{
1263	if (pa_valid(pa)) {
1264	ppattr_set_bits(pai: pa_index(pa), bits);
1265	}
1266	}
1267
1268	/**
1269	* Only clear some flags in a pp_attr_table entry if the passed in physical
1270	* address is a kernel-managed address.
1271	*
1272	* @param pa The physical address for the entry to update.
1273	* @param bits The flags to clear in the entry.
1274	*/
1275	static inline void
1276	ppattr_pa_clear_bits(pmap_paddr_t pa, pp_attr_t bits)
1277	{
1278	if (pa_valid(pa)) {
1279	ppattr_clear_bits(pai: pa_index(pa), bits);
1280	}
1281	}
1282
1283	/**
1284	* Only test flags in a pp_attr_table entry if the passed in physical address
1285	* is a kernel-managed page.
1286	*
1287	* @param pa The physical address for the entry to test.
1288	* @param bits The flags to check for.
1289	*
1290	* @return False if the PA isn't a kernel-managed page, otherwise true/false
1291	* depending on whether the bits are set.
1292	*/
1293	static inline bool
1294	ppattr_pa_test_bits(pmap_paddr_t pa, pp_attr_t bits)
1295	{
1296	return pa_valid(pa) ? ppattr_test_bits(pai: pa_index(pa), bits) : false;
1297	}
1298
1299	/**
1300	* Set the PP_ATTR_MODIFIED flag on a specific pp_attr_table entry if the passed
1301	* in physical address is a kernel-managed page.
1302	*
1303	* @param pa The physical address for the entry to update.
1304	*/
1305	static inline void
1306	ppattr_pa_set_modify(pmap_paddr_t pa)
1307	{
1308	ppattr_pa_set_bits(pa, PP_ATTR_MODIFIED);
1309	}
1310
1311	/**
1312	* Clear the PP_ATTR_MODIFIED flag on a specific pp_attr_table entry if the
1313	* passed in physical address is a kernel-managed page.
1314	*
1315	* @param pa The physical address for the entry to update.
1316	*/
1317	static inline void
1318	ppattr_pa_clear_modify(pmap_paddr_t pa)
1319	{
1320	ppattr_pa_clear_bits(pa, PP_ATTR_MODIFIED);
1321	}
1322
1323	/**
1324	* Set the PP_ATTR_REFERENCED flag on a specific pp_attr_table entry if the
1325	* passed in physical address is a kernel-managed page.
1326	*
1327	* @param pa The physical address for the entry to update.
1328	*/
1329	static inline void
1330	ppattr_pa_set_reference(pmap_paddr_t pa)
1331	{
1332	ppattr_pa_set_bits(pa, PP_ATTR_REFERENCED);
1333	}
1334
1335	/**
1336	* Clear the PP_ATTR_REFERENCED flag on a specific pp_attr_table entry if the
1337	* passed in physical address is a kernel-managed page.
1338	*
1339	* @param pa The physical address for the entry to update.
1340	*/
1341	static inline void
1342	ppattr_pa_clear_reference(pmap_paddr_t pa)
1343	{
1344	ppattr_pa_clear_bits(pa, PP_ATTR_REFERENCED);
1345	}
1346
1347	#if XNU_MONITOR
1348
1349	/**
1350	* Set the PP_ATTR_MONITOR flag on a specific pp_attr_table entry if the passed
1351	* in physical address is a kernel-managed page.
1352	*
1353	* @param pa The physical address for the entry to update.
1354	*/
1355	static inline void
1356	ppattr_pa_set_monitor(pmap_paddr_t pa)
1357	{
1358	ppattr_pa_set_bits(pa, PP_ATTR_MONITOR);
1359	}
1360
1361	/**
1362	* Clear the PP_ATTR_MONITOR flag on a specific pp_attr_table entry if the
1363	* passed in physical address is a kernel-managed page.
1364	*
1365	* @param pa The physical address for the entry to update.
1366	*/
1367	static inline void
1368	ppattr_pa_clear_monitor(pmap_paddr_t pa)
1369	{
1370	ppattr_pa_clear_bits(pa, PP_ATTR_MONITOR);
1371	}
1372
1373	/**
1374	* Only test for the PP_ATTR_MONITOR flag in a pp_attr_table entry if the passed
1375	* in physical address is a kernel-managed page.
1376	*
1377	* @param pa The physical address for the entry to test.
1378	*
1379	* @return False if the PA isn't a kernel-managed page, otherwise true/false
1380	* depending on whether the PP_ATTR_MONITOR is set.
1381	*/
1382	static inline bool
1383	ppattr_pa_test_monitor(pmap_paddr_t pa)
1384	{
1385	return ppattr_pa_test_bits(pa, PP_ATTR_MONITOR);
1386	}
1387
1388	/**
1389	* Set the PP_ATTR_NO_MONITOR flag on a specific pp_attr_table entry if the
1390	* passed in physical address is a kernel-managed page.
1391	*
1392	* @param pa The physical address for the entry to update.
1393	*/
1394	static inline void
1395	ppattr_pa_set_no_monitor(pmap_paddr_t pa)
1396	{
1397	ppattr_pa_set_bits(pa, PP_ATTR_NO_MONITOR);
1398	}
1399
1400	/**
1401	* Clear the PP_ATTR_NO_MONITOR flag on a specific pp_attr_table entry if the
1402	* passed in physical address is a kernel-managed page.
1403	*
1404	* @param pa The physical address for the entry to update.
1405	*/
1406	static inline void
1407	ppattr_pa_clear_no_monitor(pmap_paddr_t pa)
1408	{
1409	ppattr_pa_clear_bits(pa, PP_ATTR_NO_MONITOR);
1410	}
1411
1412	/**
1413	* Only test for the PP_ATTR_NO_MONITOR flag in a pp_attr_table entry if the
1414	* passed in physical address is a kernel-managed page.
1415	*
1416	* @param pa The physical address for the entry to test.
1417	*
1418	* @return False if the PA isn't a kernel-managed page, otherwise true/false
1419	* depending on whether the PP_ATTR_NO_MONITOR is set.
1420	*/
1421	static inline bool
1422	ppattr_pa_test_no_monitor(pmap_paddr_t pa)
1423	{
1424	return ppattr_pa_test_bits(pa, PP_ATTR_NO_MONITOR);
1425	}
1426
1427	#endif /* XNU_MONITOR */
1428
1429	/**
1430	* Set the PP_ATTR_INTERNAL flag on a specific pp_attr_table entry.
1431	*
1432	* @param pai The physical address index for the entry to update.
1433	*/
1434	static inline void
1435	ppattr_set_internal(unsigned int pai)
1436	{
1437	ppattr_set_bits(pai, PP_ATTR_INTERNAL);
1438	}
1439
1440	/**
1441	* Clear the PP_ATTR_INTERNAL flag on a specific pp_attr_table entry.
1442	*
1443	* @param pai The physical address index for the entry to update.
1444	*/
1445	static inline void
1446	ppattr_clear_internal(unsigned int pai)
1447	{
1448	ppattr_clear_bits(pai, PP_ATTR_INTERNAL);
1449	}
1450
1451	/**
1452	* Return true if the pp_attr_table entry has the PP_ATTR_INTERNAL flag set.
1453	*
1454	* @param pai The physical address index for the entry to test.
1455	*/
1456	static inline bool
1457	ppattr_test_internal(unsigned int pai)
1458	{
1459	return ppattr_test_bits(pai, PP_ATTR_INTERNAL);
1460	}
1461
1462	/**
1463	* Set the PP_ATTR_REUSABLE flag on a specific pp_attr_table entry.
1464	*
1465	* @param pai The physical address index for the entry to update.
1466	*/
1467	static inline void
1468	ppattr_set_reusable(unsigned int pai)
1469	{
1470	ppattr_set_bits(pai, PP_ATTR_REUSABLE);
1471	}
1472
1473	/**
1474	* Clear the PP_ATTR_REUSABLE flag on a specific pp_attr_table entry.
1475	*
1476	* @param pai The physical address index for the entry to update.
1477	*/
1478	static inline void
1479	ppattr_clear_reusable(unsigned int pai)
1480	{
1481	ppattr_clear_bits(pai, PP_ATTR_REUSABLE);
1482	}
1483
1484	/**
1485	* Return true if the pp_attr_table entry has the PP_ATTR_REUSABLE flag set.
1486	*
1487	* @param pai The physical address index for the entry to test.
1488	*/
1489	static inline bool
1490	ppattr_test_reusable(unsigned int pai)
1491	{
1492	return ppattr_test_bits(pai, PP_ATTR_REUSABLE);
1493	}
1494
1495	/**
1496	* Set the PP_ATTR_ALTACCT flag on a specific pp_attr_table entry.
1497	*
1498	* @note This is only valid when the ALTACCT flag is being tracked using the
1499	* pp_attr_table. See the descriptions above the PVE_PTEP_ALTACCT and
1500	* PP_ATTR_ALTACCT definitions for more information.
1501	*
1502	* @param pai The physical address index for the entry to update.
1503	*/
1504	static inline void
1505	ppattr_set_altacct(unsigned int pai)
1506	{
1507	ppattr_set_bits(pai, PP_ATTR_ALTACCT);
1508	}
1509
1510	/**
1511	* Clear the PP_ATTR_ALTACCT flag on a specific pp_attr_table entry.
1512	*
1513	* @note This is only valid when the ALTACCT flag is being tracked using the
1514	* pp_attr_table. See the descriptions above the PVE_PTEP_ALTACCT and
1515	* PP_ATTR_ALTACCT definitions for more information.
1516	*
1517	* @param pai The physical address index for the entry to update.
1518	*/
1519	static inline void
1520	ppattr_clear_altacct(unsigned int pai)
1521	{
1522	ppattr_clear_bits(pai, PP_ATTR_ALTACCT);
1523	}
1524
1525	/**
1526	* Get the PP_ATTR_ALTACCT flag on a specific pp_attr_table entry.
1527	*
1528	* @note This is only valid when the ALTACCT flag is being tracked using the
1529	* pp_attr_table. See the descriptions above the PVE_PTEP_ALTACCT and
1530	* PP_ATTR_ALTACCT definitions for more information.
1531	*
1532	* @param pai The physical address index for the entry to test.
1533	*
1534	* @return True if the passed in page uses alternate accounting, false
1535	* otherwise.
1536	*/
1537	static inline bool
1538	ppattr_is_altacct(unsigned int pai)
1539	{
1540	return ppattr_test_bits(pai, PP_ATTR_ALTACCT);
1541	}
1542	/**
1543	* Get the PP_ATTR_INTERNAL flag on a specific pp_attr_table entry.
1544	*
1545	* @note This is only valid when the INTERNAL flag is being tracked using the
1546	* pp_attr_table. See the descriptions above the PVE_PTEP_INTERNAL and
1547	* PP_ATTR_INTERNAL definitions for more information.
1548	*
1549	* @param pai The physical address index for the entry to test.
1550	*
1551	* @return True if the passed in page is accounted for as "internal", false
1552	* otherwise.
1553	*/
1554	static inline bool
1555	ppattr_is_internal(unsigned int pai)
1556	{
1557	return ppattr_test_bits(pai, PP_ATTR_INTERNAL);
1558	}
1559
1560	/**
1561	* The "alternate accounting" (ALTACCT) status for a page is tracked differently
1562	* depending on whether there are one or multiple mappings to a page. This
1563	* function abstracts out the difference between single and multiple mappings to
1564	* a page and provides a single function for determining whether alternate
1565	* accounting is set for a mapping.
1566	*
1567	* @note See the descriptions above the PVE_PTEP_ALTACCT and PP_ATTR_ALTACCT
1568	* definitions for more information.
1569	*
1570	* @param pai The physical address index for the entry to test.
1571	* @param pvep Pointer to the pv_entry_t object containing that mapping.
1572	* @param idx Index of the chosen PTE pointer inside the PVE.
1573	*
1574	* @return True if the passed in page uses alternate accounting, false
1575	* otherwise.
1576	*/
1577	static inline bool
1578	ppattr_pve_is_altacct(unsigned int pai, pv_entry_t pvep, unsigned* idx)
1579	{
1580	return (pvep == PV_ENTRY_NULL) ? ppattr_is_altacct(pai) : pve_get_altacct(pvep, idx);
1581	}
1582	/**
1583	* The "internal" (INTERNAL) status for a page is tracked differently
1584	* depending on whether there are one or multiple mappings to a page. This
1585	* function abstracts out the difference between single and multiple mappings to
1586	* a page and provides a single function for determining whether "internal"
1587	* is set for a mapping.
1588	*
1589	* @note See the descriptions above the PVE_PTEP_INTERNAL and PP_ATTR_INTERNAL
1590	* definitions for more information.
1591	*
1592	* @param pai The physical address index for the entry to test.
1593	* @param pvep Pointer to the pv_entry_t object containing that mapping.
1594	* @param idx Index of the chosen PTE pointer inside the PVE.
1595	*
1596	* @return True if the passed in page is "internal", false otherwise.
1597	*/
1598	static inline bool
1599	ppattr_pve_is_internal(unsigned int pai, pv_entry_t pvep, unsigned* idx)
1600	{
1601	return (pvep == PV_ENTRY_NULL) ? ppattr_is_internal(pai) : pve_get_internal(pvep, idx);
1602	}
1603
1604	/**
1605	* The "alternate accounting" (ALTACCT) status for a page is tracked differently
1606	* depending on whether there are one or multiple mappings to a page. This
1607	* function abstracts out the difference between single and multiple mappings to
1608	* a page and provides a single function for setting the alternate accounting status
1609	* for a mapping.
1610	*
1611	* @note See the descriptions above the PVE_PTEP_ALTACCT and PP_ATTR_ALTACCT
1612	* definitions for more information.
1613	*
1614	* @param pai The physical address index for the entry to update.
1615	* @param pvep Pointer to the pv_entry_t object containing that mapping.
1616	* @param idx Index of the chosen PTE pointer inside the PVE.
1617	*/
1618	static inline void
1619	ppattr_pve_set_altacct(unsigned int pai, pv_entry_t pvep, unsigned* idx)
1620	{
1621	if (pvep == PV_ENTRY_NULL) {
1622	ppattr_set_altacct(pai);
1623	} else {
1624	pve_set_altacct(pvep, idx);
1625	}
1626	}
1627	/**
1628	* The "internal" (INTERNAL) status for a page is tracked differently
1629	* depending on whether there are one or multiple mappings to a page. This
1630	* function abstracts out the difference between single and multiple mappings to
1631	* a page and provides a single function for setting the "internal" status
1632	* for a mapping.
1633	*
1634	* @note See the descriptions above the PVE_PTEP_INTERNAL and PP_ATTR_INTERNAL
1635	* definitions for more information.
1636	*
1637	* @param pai The physical address index for the entry to update.
1638	* @param pvep Pointer to the pv_entry_t object containing that mapping.
1639	* @param idx Index of the chosen PTE pointer inside the PVE.
1640	*/
1641	static inline void
1642	ppattr_pve_set_internal(unsigned int pai, pv_entry_t pvep, unsigned* idx)
1643	{
1644	if (pvep == PV_ENTRY_NULL) {
1645	ppattr_set_internal(pai);
1646	} else {
1647	pve_set_internal(pvep, idx);
1648	}
1649	}
1650
1651	/**
1652	* The "alternate accounting" (ALTACCT) status for a page is tracked differently
1653	* depending on whether there are one or multiple mappings to a page. This
1654	* function abstracts out the difference between single and multiple mappings to
1655	* a page and provides a single function for clearing the alternate accounting status
1656	* for a mapping.
1657	*
1658	* @note See the descriptions above the PVE_PTEP_ALTACCT and PP_ATTR_ALTACCT
1659	* definitions for more information.
1660	*
1661	* @param pai The physical address index for the entry to update.
1662	* @param pvep Pointer to the pv_entry_t object containing that mapping.
1663	* @param idx Index of the chosen PTE pointer inside the PVE.
1664	*/
1665	static inline void
1666	ppattr_pve_clr_altacct(unsigned int pai, pv_entry_t pvep, unsigned* idx)
1667	{
1668	if (pvep == PV_ENTRY_NULL) {
1669	ppattr_clear_altacct(pai);
1670	} else {
1671	pve_clr_altacct(pvep, idx);
1672	}
1673	}
1674	/**
1675	* The "internal" (INTERNAL) status for a page is tracked differently
1676	* depending on whether there are one or multiple mappings to a page. This
1677	* function abstracts out the difference between single and multiple mappings to
1678	* a page and provides a single function for clearing the "internal" status
1679	* for a mapping.
1680	*
1681	* @note See the descriptions above the PVE_PTEP_INTERNAL and PP_ATTR_INTERNAL
1682	* definitions for more information.
1683	*
1684	* @param pai The physical address index for the entry to update.
1685	* @param pvep Pointer to the pv_entry_t object containing that mapping.
1686	* @param idx Index of the chosen PTE pointer inside the PVE.
1687	*/
1688	static inline void
1689	ppattr_pve_clr_internal(unsigned int pai, pv_entry_t pvep, unsigned* idx)
1690	{
1691	if (pvep == PV_ENTRY_NULL) {
1692	ppattr_clear_internal(pai);
1693	} else {
1694	pve_clr_internal(pvep, idx);
1695	}
1696	}
1697
1698	/**
1699	* Set the PP_ATTR_REFFAULT flag on a specific pp_attr_table entry.
1700	*
1701	* @param pai The physical address index for the entry to update.
1702	*/
1703	static inline void
1704	ppattr_set_reffault(unsigned int pai)
1705	{
1706	ppattr_set_bits(pai, PP_ATTR_REFFAULT);
1707	}
1708
1709	/**
1710	* Clear the PP_ATTR_REFFAULT flag on a specific pp_attr_table entry.
1711	*
1712	* @param pai The physical address index for the entry to update.
1713	*/
1714	static inline void
1715	ppattr_clear_reffault(unsigned int pai)
1716	{
1717	ppattr_clear_bits(pai, PP_ATTR_REFFAULT);
1718	}
1719
1720	/**
1721	* Return true if the pp_attr_table entry has the PP_ATTR_REFFAULT flag set.
1722	*
1723	* @param pai The physical address index for the entry to test.
1724	*/
1725	static inline bool
1726	ppattr_test_reffault(unsigned int pai)
1727	{
1728	return ppattr_test_bits(pai, PP_ATTR_REFFAULT);
1729	}
1730
1731	/**
1732	* Set the PP_ATTR_MODFAULT flag on a specific pp_attr_table entry.
1733	*
1734	* @param pai The physical address index for the entry to update.
1735	*/
1736	static inline void
1737	ppattr_set_modfault(unsigned int pai)
1738	{
1739	ppattr_set_bits(pai, PP_ATTR_MODFAULT);
1740	}
1741
1742	/**
1743	* Clear the PP_ATTR_MODFAULT flag on a specific pp_attr_table entry.
1744	*
1745	* @param pai The physical address index for the entry to update.
1746	*/
1747	static inline void
1748	ppattr_clear_modfault(unsigned int pai)
1749	{
1750	ppattr_clear_bits(pai, PP_ATTR_MODFAULT);
1751	}
1752
1753	/**
1754	* Return true if the pp_attr_table entry has the PP_ATTR_MODFAULT flag set.
1755	*
1756	* @param pai The physical address index for the entry to test.
1757	*/
1758	static inline bool
1759	ppattr_test_modfault(unsigned int pai)
1760	{
1761	return ppattr_test_bits(pai, PP_ATTR_MODFAULT);
1762	}
1763
1764	/**
1765	* The minimum number of pages to keep in the PPL page free list.
1766	*
1767	* We define our target as 8 pages: enough for 2 page table pages, a PTD page,
1768	* and a PV page; in essence, twice as many pages as may be necessary to satisfy
1769	* a single pmap_enter request.
1770	*/
1771	#define PMAP_MIN_FREE_PPL_PAGES 8
1772
1773	/**
1774	* Flags passed to various page allocation functions, usually accessed through
1775	* the pmap_pages_alloc_zeroed() API. Each function that can take these flags as
1776	* a part of its option field, will describe these flags in its function header.
1777	*/
1778
1779	/**
1780	* Instruct the allocation function to return immediately if no pages are
1781	* current available. Without this flag, the function will spin and wait for a
1782	* page to become available. This flag can be required in some circumstances
1783	* (for instance, when allocating pages from within the PPL).
1784	*/
1785	#define PMAP_PAGES_ALLOCATE_NOWAIT 0x1
1786
1787	/**
1788	* Instructs an allocation function to fallback to reclaiming a userspace page
1789	* table if it failed to allocate a page from the free lists. This can be useful
1790	* when allocating from within the PPL because refilling the free lists requires
1791	* exiting and re-entering the PPL (which incurs extra latency).
1792	*
1793	* This is a quick way of allocating a page at the expense of having to
1794	* reallocate the table the next time one of its mappings is accessed.
1795	*/
1796	#define PMAP_PAGE_RECLAIM_NOWAIT 0x2
1797
1798	/**
1799	* Global variables exported to the rest of the internal pmap implementation.
1800	*/
1801	#if XNU_MONITOR
1802	extern uint64_t pmap_ppl_free_page_count;
1803	extern pmap_paddr_t pmap_stacks_start_pa;
1804	extern pmap_paddr_t pmap_stacks_end_pa;
1805	extern pmap_paddr_t ppl_cpu_save_area_start;
1806	extern pmap_paddr_t ppl_cpu_save_area_end;
1807	#endif /* XNU_MONITOR */
1808	extern unsigned int inuse_pmap_pages_count;
1809	extern vm_object_t pmap_object;
1810	extern uint32_t pv_alloc_initial_target;
1811	extern uint32_t pv_kern_alloc_initial_target;
1812
1813	/**
1814	* Functions exported to the rest of the internal pmap implementation.
1815	*/
1816	extern void pmap_data_bootstrap(void);
1817	extern void pmap_enqueue_pages(vm_page_t);
1818	extern kern_return_t pmap_pages_alloc_zeroed(pmap_paddr_t , unsigned, unsigned*);
1819	extern void pmap_pages_free(pmap_paddr_t, unsigned);
1820
1821	#if XNU_MONITOR
1822
1823	extern void pmap_mark_page_as_ppl_page_internal(pmap_paddr_t, bool);
1824	extern void pmap_mark_page_as_ppl_page(pmap_paddr_t);
1825	extern void pmap_mark_page_as_kernel_page(pmap_paddr_t);
1826	extern pmap_paddr_t pmap_alloc_page_for_kern(unsigned int);
1827	extern void pmap_alloc_page_for_ppl(unsigned int);
1828	extern uint64_t pmap_release_ppl_pages_to_kernel(void);
1829
1830	extern uint64_t pmap_ledger_validate(const volatile void *);
1831	void pmap_ledger_retain(ledger_t ledger);
1832	void pmap_ledger_release(ledger_t ledger);
1833	extern void pmap_ledger_check_balance(pmap_t pmap);
1834
1835	kern_return_t pmap_alloc_pmap(pmap_t *pmap);
1836	void pmap_free_pmap(pmap_t pmap);
1837
1838	#endif /* XNU_MONITOR */
1839
1840	/**
1841	* The modes in which a pmap lock can be acquired. Note that shared access
1842	* doesn't necessarily mean "read-only". As long as data is atomically updated
1843	* correctly (to account for multi-cpu accesses) data can still get written with
1844	* a shared lock held. Care just needs to be taken so as to not introduce any
1845	* race conditions when there are multiple writers.
1846	*
1847	* This is here in pmap_data.h because it's a needed parameter for pv_alloc()
1848	* and pmap_enter_pv(). This header is always included in pmap_internal.h before
1849	* the rest of the pmap locking code is defined so there shouldn't be any issues
1850	* with missing types.
1851	*/
1852	OS_ENUM(pmap_lock_mode, uint8_t,
1853	PMAP_LOCK_SHARED,
1854	PMAP_LOCK_EXCLUSIVE);
1855
1856	/**
1857	* Possible return values for pv_alloc(). See the pv_alloc() function header for
1858	* a description of each of these values.
1859	*/
1860	typedef enum {
1861	PV_ALLOC_SUCCESS,
1862	PV_ALLOC_RETRY,
1863	PV_ALLOC_FAIL
1864	} pv_alloc_return_t;
1865
1866	extern pv_alloc_return_t pv_alloc(
1867	pmap_t, unsigned int, pmap_lock_mode_t, unsigned int, pv_entry_t **);
1868	extern void pv_free(pv_entry_t *);
1869	extern void pv_list_free(pv_entry_t , pv_entry_t , int);
1870	extern void pmap_compute_pv_targets(void);
1871	extern pv_alloc_return_t pmap_enter_pv(
1872	pmap_t, pt_entry_t , int, unsigned* int, pmap_lock_mode_t, pv_entry_t *, int* *new_pve_ptep_idx);
1873	extern void pmap_remove_pv(pmap_t, pt_entry_t , int, bool, bool , bool *);
1874
1875	extern void ptd_bootstrap(pt_desc_t , unsigned* int);
1876	extern pt_desc_t ptd_alloc_unlinked(void*);
1877	extern pt_desc_t *ptd_alloc(pmap_t);
1878	extern void ptd_deallocate(pt_desc_t *);
1879	extern void ptd_info_init(
1880	pt_desc_t , pmap_t, vm_map_address_t, unsigned* int, pt_entry_t *);
1881
1882	extern kern_return_t pmap_ledger_credit(pmap_t, int, ledger_amount_t);
1883	extern kern_return_t pmap_ledger_debit(pmap_t, int, ledger_amount_t);
1884
1885	extern void validate_pmap_internal(const volatile struct pmap , const* char *);
1886	extern void validate_pmap_mutable_internal(const volatile struct pmap , const* char *);
1887
1888	/**
1889	* Macro function wrappers around pmap validation so that the calling function
1890	* can be printed in the panic strings for easier validation failure debugging.
1891	*/
1892	#define validate_pmap(x) validate_pmap_internal(x, __func__)
1893	#define validate_pmap_mutable(x) validate_pmap_mutable_internal(x, __func__)
1894
1895	/**
1896	* This structure describes a PPL-owned I/O range.
1897	*
1898	* @note This doesn't necessarily have to represent "I/O" only, this can also
1899	* represent non-kernel-managed DRAM (e.g., iBoot carveouts). Any physical
1900	* address region that isn't considered "kernel-managed" is fair game.
1901	*
1902	* @note The layout of this structure needs to map 1-to-1 with the pmap-io-range
1903	* device tree nodes. Astris (through the LowGlobals) also depends on the
1904	* consistency of this structure.
1905	*/
1906	typedef struct pmap_io_range {
1907	/ Physical address of the PPL-owned I/O range. /
1908	uint64_t addr;
1909
1910	/**
1911	* Length (in bytes) of the PPL-owned I/O range. Has to be the size
1912	* of a page if the range will be refered to by pmap_io_filter_entries.
1913	*/
1914	uint64_t len;
1915
1916	/ Strong DSB required for pages in this range. /
1917	#define PMAP_IO_RANGE_STRONG_SYNC (1UL << 31)
1918
1919	/ Corresponds to memory carved out by bootloader. /
1920	#define PMAP_IO_RANGE_CARVEOUT (1UL << 30)
1921
1922	/ Pages in this range need to be included in the hibernation image /
1923	#define PMAP_IO_RANGE_NEEDS_HIBERNATING (1UL << 29)
1924
1925	/ Mark the range as 'owned' by a given subsystem /
1926	#define PMAP_IO_RANGE_OWNED (1UL << 28)
1927
1928	/**
1929	* Lower 16 bits treated as pp_attr_t, upper 16 bits contain additional
1930	* mapping flags (defined above).
1931	*/
1932	uint32_t wimg;
1933
1934	/**
1935	* 4 Character Code (4CC) describing what this range is.
1936	*
1937	* This has to be unique for each "type" of pages, meaning pages sharing
1938	* the same register layout, if it is used for the I/O filter descriptors
1939	* below. Otherwise it doesn't matter.
1940	*/
1941	uint32_t signature;
1942	} pmap_io_range_t;
1943
1944	/ Reminder: be sure to change all relevant device trees if you change the layout of pmap_io_range_t /
1945	_Static_assert(sizeof(pmap_io_range_t) == `24`, "unexpected size for pmap_io_range_t");
1946
1947	extern pmap_io_range_t* pmap_find_io_attr(pmap_paddr_t);
1948
1949	/**
1950	* This structure describes a sub-page-size I/O region owned by PPL but the kernel can write to.
1951	*
1952	* @note I/O filter software will use a collection of such data structures to determine access
1953	* permissions to a page owned by PPL.
1954	*
1955	* @note The {signature, offset} key is used to index a collection of such data structures to
1956	* optimize for space in the case where one page layout is repeated for many devices, such
1957	* as the memory controller channels.
1958	*/
1959	typedef struct pmap_io_filter_entry {
1960	/ 4 Character Code (4CC) describing what this range (page) is. /
1961	uint32_t signature;
1962
1963	/ Offset within the page. It has to be within [0, PAGE_SIZE). /
1964	uint16_t offset;
1965
1966	/ Length of the range, and (offset + length) has to be within [0, PAGE_SIZE). /
1967	uint16_t length;
1968	} pmap_io_filter_entry_t;
1969
1970	_Static_assert(sizeof(pmap_io_filter_entry_t) == `8`, "unexpected size for pmap_io_filter_entry_t");
1971
1972	extern pmap_io_filter_entry_t pmap_find_io_filter_entry(pmap_paddr_t, uint64_t, const* pmap_io_range_t **);
1973
1974	extern void pmap_cpu_data_init_internal(unsigned int);
1975
1976	/**
1977	* Flush a single 16K page from noncoherent coprocessor caches.
1978	*
1979	* @note Nonocoherent cache flushes are only guaranteed to work if the participating coprocessor(s)
1980	* do not have any active VA translations for the page being flushed. Since coprocessor
1981	* mappings should always be controlled by some PPL IOMMU extension, they should always
1982	* have PV list entries. This flush should therefore be performed at a point when the PV
1983	* list is known to be either empty or at least to not contain any IOMMU entries. For
1984	* the purposes of our security model, it is sufficient to wait for the PV list to become
1985	* empty, as we really want to protect PPL-sensitive pages from malicious/accidental
1986	* coprocessor cacheline evictions, and the PV list must be empty before a page can be
1987	* handed to the PPL.
1988	*
1989	* @param paddr The base physical address of the page to flush.
1990	*/
1991	extern void pmap_flush_noncoherent_page(pmap_paddr_t paddr);
1992
1993	#endif /* _ARM_PMAP_PMAP_DATA_H_ */
1994

Browse the source code of xnu/osfmk/arm/pmap/pmap_data.h