1/*
2 * Copyright (c) 2019-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57/*
58 * Compressor Pager.
59 * Memory Object Management.
60 */
61
62#include <kern/host_statistics.h>
63#include <kern/kalloc.h>
64#include <kern/ipc_kobject.h>
65
66#include <machine/atomic.h>
67
68#include <mach/memory_object_control.h>
69#include <mach/memory_object_types.h>
70#include <mach/upl.h>
71
72#include <vm/memory_object.h>
73#include <vm/vm_compressor_pager.h>
74#include <vm/vm_external.h>
75#include <vm/vm_pageout.h>
76#include <vm/vm_protos.h>
77
78#include <sys/kdebug_triage.h>
79
80/* memory_object interfaces */
81void compressor_memory_object_reference(memory_object_t mem_obj);
82void compressor_memory_object_deallocate(memory_object_t mem_obj);
83kern_return_t compressor_memory_object_init(
84 memory_object_t mem_obj,
85 memory_object_control_t control,
86 memory_object_cluster_size_t pager_page_size);
87kern_return_t compressor_memory_object_terminate(memory_object_t mem_obj);
88kern_return_t compressor_memory_object_data_request(
89 memory_object_t mem_obj,
90 memory_object_offset_t offset,
91 memory_object_cluster_size_t length,
92 __unused vm_prot_t protection_required,
93 memory_object_fault_info_t fault_info);
94kern_return_t compressor_memory_object_data_return(
95 memory_object_t mem_obj,
96 memory_object_offset_t offset,
97 memory_object_cluster_size_t size,
98 __unused memory_object_offset_t *resid_offset,
99 __unused int *io_error,
100 __unused boolean_t dirty,
101 __unused boolean_t kernel_copy,
102 __unused int upl_flags);
103kern_return_t compressor_memory_object_data_initialize(
104 memory_object_t mem_obj,
105 memory_object_offset_t offset,
106 memory_object_cluster_size_t size);
107kern_return_t compressor_memory_object_map(
108 __unused memory_object_t mem_obj,
109 __unused vm_prot_t prot);
110kern_return_t compressor_memory_object_last_unmap(memory_object_t mem_obj);
111
112const struct memory_object_pager_ops compressor_pager_ops = {
113 .memory_object_reference = compressor_memory_object_reference,
114 .memory_object_deallocate = compressor_memory_object_deallocate,
115 .memory_object_init = compressor_memory_object_init,
116 .memory_object_terminate = compressor_memory_object_terminate,
117 .memory_object_data_request = compressor_memory_object_data_request,
118 .memory_object_data_return = compressor_memory_object_data_return,
119 .memory_object_data_initialize = compressor_memory_object_data_initialize,
120 .memory_object_map = compressor_memory_object_map,
121 .memory_object_last_unmap = compressor_memory_object_last_unmap,
122 .memory_object_backing_object = NULL,
123 .memory_object_pager_name = "compressor pager"
124};
125
126/* internal data structures */
127
128struct {
129 uint64_t data_returns;
130 uint64_t data_requests;
131 uint64_t put;
132 uint64_t get;
133 uint64_t state_clr;
134 uint64_t state_get;
135 uint64_t transfer;
136} compressor_pager_stats;
137
138typedef int compressor_slot_t;
139
140typedef struct compressor_pager {
141 /* mandatory generic header */
142 struct memory_object cpgr_hdr;
143
144 /* pager-specific data */
145 lck_mtx_t cpgr_lock;
146#if MEMORY_OBJECT_HAS_REFCOUNT
147#define cpgr_references cpgr_hdr.mo_ref
148#else
149 os_ref_atomic_t cpgr_references;
150#endif
151 unsigned int cpgr_num_slots;
152 unsigned int cpgr_num_slots_occupied;
153 union {
154 compressor_slot_t cpgr_eslots[2]; /* embedded slots */
155 compressor_slot_t *cpgr_dslots; /* direct slots */
156 compressor_slot_t **cpgr_islots; /* indirect slots */
157 } cpgr_slots;
158} *compressor_pager_t;
159
160#define compressor_pager_lookup(_mem_obj_, _cpgr_) \
161 MACRO_BEGIN \
162 if (_mem_obj_ == NULL || \
163 _mem_obj_->mo_pager_ops != &compressor_pager_ops) { \
164 _cpgr_ = NULL; \
165 } else { \
166 _cpgr_ = (compressor_pager_t) _mem_obj_; \
167 } \
168 MACRO_END
169
170/* embedded slot pointers in compressor_pager get packed, so VA restricted */
171static ZONE_DEFINE_TYPE(compressor_pager_zone, "compressor_pager",
172 struct compressor_pager, ZC_NOENCRYPT | ZC_VM);
173
174LCK_GRP_DECLARE(compressor_pager_lck_grp, "compressor_pager");
175
176#define compressor_pager_lock(_cpgr_) \
177 lck_mtx_lock(&(_cpgr_)->cpgr_lock)
178#define compressor_pager_unlock(_cpgr_) \
179 lck_mtx_unlock(&(_cpgr_)->cpgr_lock)
180#define compressor_pager_lock_init(_cpgr_) \
181 lck_mtx_init(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp, LCK_ATTR_NULL)
182#define compressor_pager_lock_destroy(_cpgr_) \
183 lck_mtx_destroy(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp)
184
185#define COMPRESSOR_SLOTS_CHUNK_SIZE (512)
186#define COMPRESSOR_SLOTS_PER_CHUNK (COMPRESSOR_SLOTS_CHUNK_SIZE / sizeof (compressor_slot_t))
187
188/* forward declarations */
189unsigned int compressor_pager_slots_chunk_free(compressor_slot_t *chunk,
190 int num_slots,
191 int flags,
192 int *failures);
193void compressor_pager_slot_lookup(
194 compressor_pager_t pager,
195 boolean_t do_alloc,
196 memory_object_offset_t offset,
197 compressor_slot_t **slot_pp);
198
199#if defined(__LP64__)
200
201/* restricted VA zones for slots */
202
203#define NUM_SLOTS_ZONES 3
204
205static const size_t compressor_slots_zones_sizes[NUM_SLOTS_ZONES] = {
206 16,
207 64,
208 COMPRESSOR_SLOTS_CHUNK_SIZE
209};
210
211static const char * compressor_slots_zones_names[NUM_SLOTS_ZONES] = {
212 "compressor_slots.16",
213 "compressor_slots.64",
214 "compressor_slots.512"
215};
216
217static zone_t
218 compressor_slots_zones[NUM_SLOTS_ZONES];
219
220#endif /* defined(__LP64__) */
221
222static void
223zfree_slot_array(compressor_slot_t *slots, size_t size);
224static compressor_slot_t *
225zalloc_slot_array(size_t size, zalloc_flags_t);
226
227static inline unsigned int
228compressor_pager_num_chunks(
229 compressor_pager_t pager)
230{
231 unsigned int num_chunks;
232
233 num_chunks = pager->cpgr_num_slots / COMPRESSOR_SLOTS_PER_CHUNK;
234 if (num_chunks * COMPRESSOR_SLOTS_PER_CHUNK < pager->cpgr_num_slots) {
235 num_chunks++;
236 }
237 return num_chunks;
238}
239
240kern_return_t
241compressor_memory_object_init(
242 memory_object_t mem_obj,
243 memory_object_control_t control,
244 __unused memory_object_cluster_size_t pager_page_size)
245{
246 compressor_pager_t pager;
247
248 assert(pager_page_size == PAGE_SIZE);
249
250 memory_object_control_reference(control);
251
252 compressor_pager_lookup(mem_obj, pager);
253 compressor_pager_lock(pager);
254
255 if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) {
256 panic("compressor_memory_object_init: bad request");
257 }
258 pager->cpgr_hdr.mo_control = control;
259
260 compressor_pager_unlock(pager);
261
262 return KERN_SUCCESS;
263}
264
265kern_return_t
266compressor_memory_object_map(
267 __unused memory_object_t mem_obj,
268 __unused vm_prot_t prot)
269{
270 panic("compressor_memory_object_map");
271 return KERN_FAILURE;
272}
273
274kern_return_t
275compressor_memory_object_last_unmap(
276 __unused memory_object_t mem_obj)
277{
278 panic("compressor_memory_object_last_unmap");
279 return KERN_FAILURE;
280}
281
282kern_return_t
283compressor_memory_object_terminate(
284 memory_object_t mem_obj)
285{
286 memory_object_control_t control;
287 compressor_pager_t pager;
288
289 /*
290 * control port is a receive right, not a send right.
291 */
292
293 compressor_pager_lookup(mem_obj, pager);
294 compressor_pager_lock(pager);
295
296 /*
297 * After memory_object_terminate both memory_object_init
298 * and a no-senders notification are possible, so we need
299 * to clean up our reference to the memory_object_control
300 * to prepare for a new init.
301 */
302
303 control = pager->cpgr_hdr.mo_control;
304 pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
305
306 compressor_pager_unlock(pager);
307
308 /*
309 * Now we deallocate our reference on the control.
310 */
311 memory_object_control_deallocate(control);
312 return KERN_SUCCESS;
313}
314
315void
316compressor_memory_object_reference(
317 memory_object_t mem_obj)
318{
319 compressor_pager_t pager;
320
321 compressor_pager_lookup(mem_obj, pager);
322 if (pager == NULL) {
323 return;
324 }
325
326 compressor_pager_lock(pager);
327 os_ref_retain_locked_raw(&pager->cpgr_references, NULL);
328 compressor_pager_unlock(pager);
329}
330
331void
332compressor_memory_object_deallocate(
333 memory_object_t mem_obj)
334{
335 compressor_pager_t pager;
336 unsigned int num_slots_freed;
337
338 /*
339 * Because we don't give out multiple first references
340 * for a memory object, there can't be a race
341 * between getting a deallocate call and creating
342 * a new reference for the object.
343 */
344
345 compressor_pager_lookup(mem_obj, pager);
346 if (pager == NULL) {
347 return;
348 }
349
350 compressor_pager_lock(pager);
351 if (os_ref_release_locked_raw(&pager->cpgr_references, NULL) > 0) {
352 compressor_pager_unlock(pager);
353 return;
354 }
355
356 /*
357 * We shouldn't get a deallocation call
358 * when the kernel has the object cached.
359 */
360 if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) {
361 panic("compressor_memory_object_deallocate(): bad request");
362 }
363
364 /*
365 * Unlock the pager (though there should be no one
366 * waiting for it).
367 */
368 compressor_pager_unlock(pager);
369
370 /* free the compressor slots */
371 unsigned int num_chunks;
372 unsigned int i;
373 compressor_slot_t *chunk;
374
375 num_chunks = compressor_pager_num_chunks(pager);
376 if (num_chunks > 1) {
377 /* we have an array of chunks */
378 for (i = 0; i < num_chunks; i++) {
379 chunk = pager->cpgr_slots.cpgr_islots[i];
380 if (chunk != NULL) {
381 num_slots_freed =
382 compressor_pager_slots_chunk_free(
383 chunk,
384 COMPRESSOR_SLOTS_PER_CHUNK,
385 flags: 0,
386 NULL);
387 pager->cpgr_slots.cpgr_islots[i] = NULL;
388 zfree_slot_array(slots: chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
389 }
390 }
391 kfree_type(compressor_slot_t *, num_chunks,
392 pager->cpgr_slots.cpgr_islots);
393 pager->cpgr_slots.cpgr_islots = NULL;
394 } else if (pager->cpgr_num_slots > 2) {
395 chunk = pager->cpgr_slots.cpgr_dslots;
396 num_slots_freed =
397 compressor_pager_slots_chunk_free(
398 chunk,
399 num_slots: pager->cpgr_num_slots,
400 flags: 0,
401 NULL);
402 pager->cpgr_slots.cpgr_dslots = NULL;
403 zfree_slot_array(slots: chunk,
404 size: (pager->cpgr_num_slots *
405 sizeof(pager->cpgr_slots.cpgr_dslots[0])));
406 } else {
407 chunk = &pager->cpgr_slots.cpgr_eslots[0];
408 num_slots_freed =
409 compressor_pager_slots_chunk_free(
410 chunk,
411 num_slots: pager->cpgr_num_slots,
412 flags: 0,
413 NULL);
414 }
415
416 compressor_pager_lock_destroy(pager);
417 zfree(compressor_pager_zone, pager);
418}
419
420kern_return_t
421compressor_memory_object_data_request(
422 memory_object_t mem_obj,
423 memory_object_offset_t offset,
424 memory_object_cluster_size_t length,
425 __unused vm_prot_t protection_required,
426 __unused memory_object_fault_info_t fault_info)
427{
428 compressor_pager_t pager;
429 kern_return_t kr;
430 compressor_slot_t *slot_p;
431
432 compressor_pager_stats.data_requests++;
433
434 /*
435 * Request must be on a page boundary and a multiple of pages.
436 */
437 if ((offset & PAGE_MASK) != 0 || (length & PAGE_MASK) != 0) {
438 panic("compressor_memory_object_data_request(): bad alignment");
439 }
440
441 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
442 panic("%s: offset 0x%llx overflow",
443 __FUNCTION__, (uint64_t) offset);
444 return KERN_FAILURE;
445 }
446
447 compressor_pager_lookup(mem_obj, pager);
448
449 if (length == 0) {
450 /* we're only querying the pager for this page */
451 } else {
452 panic("compressor: data_request");
453 }
454
455 /* find the compressor slot for that page */
456 compressor_pager_slot_lookup(pager, FALSE, offset, slot_pp: &slot_p);
457
458 if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {
459 /* out of range */
460 kr = KERN_FAILURE;
461 } else if (slot_p == NULL || *slot_p == 0) {
462 /* compressor does not have this page */
463 kr = KERN_FAILURE;
464 } else {
465 /* compressor does have this page */
466 kr = KERN_SUCCESS;
467 }
468 return kr;
469}
470
471/*
472 * memory_object_data_initialize: check whether we already have each page, and
473 * write it if we do not. The implementation is far from optimized, and
474 * also assumes that the default_pager is single-threaded.
475 */
476/* It is questionable whether or not a pager should decide what is relevant */
477/* and what is not in data sent from the kernel. Data initialize has been */
478/* changed to copy back all data sent to it in preparation for its eventual */
479/* merge with data return. It is the kernel that should decide what pages */
480/* to write back. As of the writing of this note, this is indeed the case */
481/* the kernel writes back one page at a time through this interface */
482
483kern_return_t
484compressor_memory_object_data_initialize(
485 memory_object_t mem_obj,
486 memory_object_offset_t offset,
487 memory_object_cluster_size_t size)
488{
489 compressor_pager_t pager;
490 memory_object_offset_t cur_offset;
491
492 compressor_pager_lookup(mem_obj, pager);
493 compressor_pager_lock(pager);
494
495 for (cur_offset = offset;
496 cur_offset < offset + size;
497 cur_offset += PAGE_SIZE) {
498 panic("do a data_return() if slot for this page is empty");
499 }
500
501 compressor_pager_unlock(pager);
502
503 return KERN_SUCCESS;
504}
505
506
507/*ARGSUSED*/
508kern_return_t
509compressor_memory_object_data_return(
510 __unused memory_object_t mem_obj,
511 __unused memory_object_offset_t offset,
512 __unused memory_object_cluster_size_t size,
513 __unused memory_object_offset_t *resid_offset,
514 __unused int *io_error,
515 __unused boolean_t dirty,
516 __unused boolean_t kernel_copy,
517 __unused int upl_flags)
518{
519 panic("compressor: data_return");
520 return KERN_FAILURE;
521}
522
523/*
524 * Routine: default_pager_memory_object_create
525 * Purpose:
526 * Handle requests for memory objects from the
527 * kernel.
528 * Notes:
529 * Because we only give out the default memory
530 * manager port to the kernel, we don't have to
531 * be so paranoid about the contents.
532 */
533kern_return_t
534compressor_memory_object_create(
535 memory_object_size_t new_size,
536 memory_object_t *new_mem_obj)
537{
538 compressor_pager_t pager;
539 unsigned int num_chunks;
540
541 if ((uint32_t)(new_size / PAGE_SIZE) != (new_size / PAGE_SIZE)) {
542 /* 32-bit overflow for number of pages */
543 panic("%s: size 0x%llx overflow",
544 __FUNCTION__, (uint64_t) new_size);
545 return KERN_INVALID_ARGUMENT;
546 }
547
548 pager = zalloc_flags(compressor_pager_zone, Z_WAITOK | Z_NOFAIL);
549
550 compressor_pager_lock_init(pager);
551 os_ref_init_raw(&pager->cpgr_references, NULL);
552 pager->cpgr_num_slots = (uint32_t)(new_size / PAGE_SIZE);
553 pager->cpgr_num_slots_occupied = 0;
554
555 num_chunks = compressor_pager_num_chunks(pager);
556 if (num_chunks > 1) {
557 pager->cpgr_slots.cpgr_islots = kalloc_type(compressor_slot_t *,
558 num_chunks, Z_WAITOK | Z_ZERO);
559 } else if (pager->cpgr_num_slots > 2) {
560 pager->cpgr_slots.cpgr_dslots = zalloc_slot_array(size: pager->cpgr_num_slots *
561 sizeof(pager->cpgr_slots.cpgr_dslots[0]), Z_WAITOK | Z_ZERO);
562 } else {
563 pager->cpgr_slots.cpgr_eslots[0] = 0;
564 pager->cpgr_slots.cpgr_eslots[1] = 0;
565 }
566
567 /*
568 * Set up associations between this memory object
569 * and this compressor_pager structure
570 */
571 pager->cpgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
572 pager->cpgr_hdr.mo_pager_ops = &compressor_pager_ops;
573 pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
574
575 *new_mem_obj = (memory_object_t) pager;
576 return KERN_SUCCESS;
577}
578
579
580unsigned int
581compressor_pager_slots_chunk_free(
582 compressor_slot_t *chunk,
583 int num_slots,
584 int flags,
585 int *failures)
586{
587 int i;
588 int retval;
589 unsigned int num_slots_freed;
590
591 if (failures) {
592 *failures = 0;
593 }
594 num_slots_freed = 0;
595 for (i = 0; i < num_slots; i++) {
596 if (chunk[i] != 0) {
597 retval = vm_compressor_free(slot: &chunk[i], flags);
598
599 if (retval == 0) {
600 num_slots_freed++;
601 } else {
602 if (retval == -2) {
603 assert(flags & C_DONT_BLOCK);
604 }
605
606 if (failures) {
607 *failures += 1;
608 }
609 }
610 }
611 }
612 return num_slots_freed;
613}
614
615void
616compressor_pager_slot_lookup(
617 compressor_pager_t pager,
618 boolean_t do_alloc,
619 memory_object_offset_t offset,
620 compressor_slot_t **slot_pp)
621{
622 unsigned int num_chunks;
623 uint32_t page_num;
624 unsigned int chunk_idx;
625 int slot_idx;
626 compressor_slot_t *chunk;
627 compressor_slot_t *t_chunk;
628
629 page_num = (uint32_t)(offset / PAGE_SIZE);
630 if (page_num != (offset / PAGE_SIZE)) {
631 /* overflow */
632 panic("%s: offset 0x%llx overflow",
633 __FUNCTION__, (uint64_t) offset);
634 *slot_pp = NULL;
635 return;
636 }
637 if (page_num >= pager->cpgr_num_slots) {
638 /* out of range */
639 *slot_pp = NULL;
640 return;
641 }
642 num_chunks = compressor_pager_num_chunks(pager);
643 if (num_chunks > 1) {
644 /* we have an array of chunks */
645 chunk_idx = page_num / COMPRESSOR_SLOTS_PER_CHUNK;
646 chunk = pager->cpgr_slots.cpgr_islots[chunk_idx];
647
648 if (chunk == NULL && do_alloc) {
649 t_chunk = zalloc_slot_array(COMPRESSOR_SLOTS_CHUNK_SIZE,
650 Z_WAITOK | Z_ZERO);
651
652 compressor_pager_lock(pager);
653
654 if ((chunk = pager->cpgr_slots.cpgr_islots[chunk_idx]) == NULL) {
655 /*
656 * On some platforms, the memory stores from
657 * the bzero(t_chunk) above might not have been
658 * made visible and another thread might see
659 * the contents of this new chunk before it's
660 * been fully zero-filled.
661 * This memory barrier should take care of this
662 * according to the platform requirements.
663 */
664 os_atomic_thread_fence(release);
665
666 chunk = pager->cpgr_slots.cpgr_islots[chunk_idx] = t_chunk;
667 t_chunk = NULL;
668 }
669 compressor_pager_unlock(pager);
670
671 if (t_chunk) {
672 zfree_slot_array(slots: t_chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
673 }
674 }
675 if (chunk == NULL) {
676 *slot_pp = NULL;
677 } else {
678 slot_idx = page_num % COMPRESSOR_SLOTS_PER_CHUNK;
679 *slot_pp = &chunk[slot_idx];
680 }
681 } else if (pager->cpgr_num_slots > 2) {
682 slot_idx = page_num;
683 *slot_pp = &pager->cpgr_slots.cpgr_dslots[slot_idx];
684 } else {
685 slot_idx = page_num;
686 *slot_pp = &pager->cpgr_slots.cpgr_eslots[slot_idx];
687 }
688}
689
690#if defined(__LP64__)
691__startup_func
692static void
693vm_compressor_slots_init(void)
694{
695 for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
696 compressor_slots_zones[idx] = zone_create(
697 name: compressor_slots_zones_names[idx],
698 size: compressor_slots_zones_sizes[idx],
699 flags: ZC_PGZ_USE_GUARDS | ZC_VM);
700 }
701}
702STARTUP(ZALLOC, STARTUP_RANK_MIDDLE, vm_compressor_slots_init);
703#endif /* defined(__LP64__) */
704
705static compressor_slot_t *
706zalloc_slot_array(size_t size, zalloc_flags_t flags)
707{
708#if defined(__LP64__)
709 compressor_slot_t *slots = NULL;
710
711 assert(size <= COMPRESSOR_SLOTS_CHUNK_SIZE);
712 for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
713 if (size > compressor_slots_zones_sizes[idx]) {
714 continue;
715 }
716 slots = zalloc_flags(compressor_slots_zones[idx], flags);
717 break;
718 }
719 return slots;
720#else /* defined(__LP64__) */
721 return kalloc_data(size, flags);
722#endif /* !defined(__LP64__) */
723}
724
725static void
726zfree_slot_array(compressor_slot_t *slots, size_t size)
727{
728#if defined(__LP64__)
729 assert(size <= COMPRESSOR_SLOTS_CHUNK_SIZE);
730 for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
731 if (size > compressor_slots_zones_sizes[idx]) {
732 continue;
733 }
734 zfree(compressor_slots_zones[idx], slots);
735 break;
736 }
737#else /* defined(__LP64__) */
738 kfree_data(slots, size);
739#endif /* !defined(__LP64__) */
740}
741
742kern_return_t
743vm_compressor_pager_put(
744 memory_object_t mem_obj,
745 memory_object_offset_t offset,
746 ppnum_t ppnum,
747 bool unmodified,
748 void **current_chead,
749 char *scratch_buf,
750 int *compressed_count_delta_p)
751{
752 compressor_pager_t pager;
753 compressor_slot_t *slot_p;
754
755 compressor_pager_stats.put++;
756
757 *compressed_count_delta_p = 0;
758
759 /* This routine is called by the pageout thread. The pageout thread */
760 /* cannot be blocked by read activities unless the read activities */
761 /* Therefore the grant of vs lock must be done on a try versus a */
762 /* blocking basis. The code below relies on the fact that the */
763 /* interface is synchronous. Should this interface be again async */
764 /* for some type of pager in the future the pages will have to be */
765 /* returned through a separate, asynchronous path. */
766
767 compressor_pager_lookup(mem_obj, pager);
768
769 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
770 /* overflow */
771 panic("%s: offset 0x%llx overflow",
772 __FUNCTION__, (uint64_t) offset);
773 return KERN_RESOURCE_SHORTAGE;
774 }
775
776 compressor_pager_slot_lookup(pager, TRUE, offset, slot_pp: &slot_p);
777
778 if (slot_p == NULL) {
779 /* out of range ? */
780 panic("vm_compressor_pager_put: out of range");
781 }
782 if (*slot_p != 0) {
783 /*
784 * Already compressed: forget about the old one.
785 *
786 * This can happen after a vm_object_do_collapse() when
787 * the "backing_object" had some pages paged out and the
788 * "object" had an equivalent page resident.
789 */
790 vm_compressor_free(slot: slot_p, flags: (unmodified ? C_PAGE_UNMODIFIED : 0));
791 *compressed_count_delta_p -= 1;
792 }
793
794 /*
795 * If the compressor operation succeeds, we presumably don't need to
796 * undo any previous WIMG update, as all live mappings should be
797 * disconnected.
798 */
799
800 if (vm_compressor_put(pn: ppnum, slot: slot_p, current_chead, scratch_buf, unmodified)) {
801 return KERN_RESOURCE_SHORTAGE;
802 }
803 *compressed_count_delta_p += 1;
804
805 return KERN_SUCCESS;
806}
807
808
809kern_return_t
810vm_compressor_pager_get(
811 memory_object_t mem_obj,
812 memory_object_offset_t offset,
813 ppnum_t ppnum,
814 int *my_fault_type,
815 int flags,
816 int *compressed_count_delta_p)
817{
818 compressor_pager_t pager;
819 kern_return_t kr;
820 compressor_slot_t *slot_p;
821
822 compressor_pager_stats.get++;
823
824 *compressed_count_delta_p = 0;
825
826 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
827 panic("%s: offset 0x%llx overflow",
828 __FUNCTION__, (uint64_t) offset);
829 return KERN_MEMORY_ERROR;
830 }
831
832 compressor_pager_lookup(mem_obj, pager);
833
834 /* find the compressor slot for that page */
835 compressor_pager_slot_lookup(pager, FALSE, offset, slot_pp: &slot_p);
836
837 if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {
838 /* out of range */
839 ktriage_record(thread_id: thread_tid(thread: current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_GET_OUT_OF_RANGE), arg: 0 /* arg */);
840 kr = KERN_MEMORY_FAILURE;
841 } else if (slot_p == NULL || *slot_p == 0) {
842 /* compressor does not have this page */
843 ktriage_record(thread_id: thread_tid(thread: current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_GET_NO_PAGE), arg: 0 /* arg */);
844 kr = KERN_MEMORY_ERROR;
845 } else {
846 /* compressor does have this page */
847 kr = KERN_SUCCESS;
848 }
849 *my_fault_type = DBG_COMPRESSOR_FAULT;
850
851 if (kr == KERN_SUCCESS) {
852 int retval;
853 bool unmodified = (vm_compressor_is_slot_compressed(slot: slot_p) == false);
854 /* get the page from the compressor */
855 retval = vm_compressor_get(pn: ppnum, slot: slot_p, flags: (unmodified ? (flags | C_PAGE_UNMODIFIED) : flags));
856 if (retval == -1) {
857 ktriage_record(thread_id: thread_tid(thread: current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_DECOMPRESS_FAILED), arg: 0 /* arg */);
858 kr = KERN_MEMORY_FAILURE;
859 } else if (retval == 1) {
860 *my_fault_type = DBG_COMPRESSOR_SWAPIN_FAULT;
861 } else if (retval == -2) {
862 assert((flags & C_DONT_BLOCK));
863 /*
864 * Not a fatal failure because we just retry with a blocking get later. So we skip ktriage to avoid noise.
865 */
866 kr = KERN_FAILURE;
867 }
868 }
869
870 if (kr == KERN_SUCCESS) {
871 assert(slot_p != NULL);
872 if (*slot_p != 0) {
873 /*
874 * We got the page for a copy-on-write fault
875 * and we kept the original in place. Slot
876 * is still occupied.
877 */
878 } else {
879 *compressed_count_delta_p -= 1;
880 }
881 }
882
883 return kr;
884}
885
886unsigned int
887vm_compressor_pager_state_clr(
888 memory_object_t mem_obj,
889 memory_object_offset_t offset)
890{
891 compressor_pager_t pager;
892 compressor_slot_t *slot_p;
893 unsigned int num_slots_freed;
894
895 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
896
897 compressor_pager_stats.state_clr++;
898
899 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
900 /* overflow */
901 panic("%s: offset 0x%llx overflow",
902 __FUNCTION__, (uint64_t) offset);
903 return 0;
904 }
905
906 compressor_pager_lookup(mem_obj, pager);
907
908 /* find the compressor slot for that page */
909 compressor_pager_slot_lookup(pager, FALSE, offset, slot_pp: &slot_p);
910
911 num_slots_freed = 0;
912 if (slot_p && *slot_p != 0) {
913 vm_compressor_free(slot: slot_p, flags: 0);
914 num_slots_freed++;
915 assert(*slot_p == 0);
916 }
917
918 return num_slots_freed;
919}
920
921vm_external_state_t
922vm_compressor_pager_state_get(
923 memory_object_t mem_obj,
924 memory_object_offset_t offset)
925{
926 compressor_pager_t pager;
927 compressor_slot_t *slot_p;
928
929 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
930
931 compressor_pager_stats.state_get++;
932
933 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
934 /* overflow */
935 panic("%s: offset 0x%llx overflow",
936 __FUNCTION__, (uint64_t) offset);
937 return VM_EXTERNAL_STATE_ABSENT;
938 }
939
940 compressor_pager_lookup(mem_obj, pager);
941
942 /* find the compressor slot for that page */
943 compressor_pager_slot_lookup(pager, FALSE, offset, slot_pp: &slot_p);
944
945 if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {
946 /* out of range */
947 return VM_EXTERNAL_STATE_ABSENT;
948 } else if (slot_p == NULL || *slot_p == 0) {
949 /* compressor does not have this page */
950 return VM_EXTERNAL_STATE_ABSENT;
951 } else {
952 /* compressor does have this page */
953 return VM_EXTERNAL_STATE_EXISTS;
954 }
955}
956
957unsigned int
958vm_compressor_pager_reap_pages(
959 memory_object_t mem_obj,
960 int flags)
961{
962 compressor_pager_t pager;
963 unsigned int num_chunks;
964 int failures;
965 unsigned int i;
966 compressor_slot_t *chunk;
967 unsigned int num_slots_freed;
968
969 compressor_pager_lookup(mem_obj, pager);
970 if (pager == NULL) {
971 return 0;
972 }
973
974 compressor_pager_lock(pager);
975
976 /* reap the compressor slots */
977 num_slots_freed = 0;
978
979 num_chunks = compressor_pager_num_chunks(pager);
980 if (num_chunks > 1) {
981 /* we have an array of chunks */
982 for (i = 0; i < num_chunks; i++) {
983 chunk = pager->cpgr_slots.cpgr_islots[i];
984 if (chunk != NULL) {
985 num_slots_freed +=
986 compressor_pager_slots_chunk_free(
987 chunk,
988 COMPRESSOR_SLOTS_PER_CHUNK,
989 flags,
990 failures: &failures);
991 if (failures == 0) {
992 pager->cpgr_slots.cpgr_islots[i] = NULL;
993 zfree_slot_array(slots: chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
994 }
995 }
996 }
997 } else if (pager->cpgr_num_slots > 2) {
998 chunk = pager->cpgr_slots.cpgr_dslots;
999 num_slots_freed +=
1000 compressor_pager_slots_chunk_free(
1001 chunk,
1002 num_slots: pager->cpgr_num_slots,
1003 flags,
1004 NULL);
1005 } else {
1006 chunk = &pager->cpgr_slots.cpgr_eslots[0];
1007 num_slots_freed +=
1008 compressor_pager_slots_chunk_free(
1009 chunk,
1010 num_slots: pager->cpgr_num_slots,
1011 flags,
1012 NULL);
1013 }
1014
1015 compressor_pager_unlock(pager);
1016
1017 return num_slots_freed;
1018}
1019
1020void
1021vm_compressor_pager_transfer(
1022 memory_object_t dst_mem_obj,
1023 memory_object_offset_t dst_offset,
1024 memory_object_t src_mem_obj,
1025 memory_object_offset_t src_offset)
1026{
1027 compressor_pager_t src_pager, dst_pager;
1028 compressor_slot_t *src_slot_p, *dst_slot_p;
1029
1030 compressor_pager_stats.transfer++;
1031
1032 /* find the compressor slot for the destination */
1033 compressor_pager_lookup(dst_mem_obj, dst_pager);
1034 assert(dst_offset / PAGE_SIZE < dst_pager->cpgr_num_slots);
1035 compressor_pager_slot_lookup(pager: dst_pager, TRUE, offset: dst_offset, slot_pp: &dst_slot_p);
1036 assert(dst_slot_p != NULL);
1037 assert(*dst_slot_p == 0);
1038
1039 /* find the compressor slot for the source */
1040 compressor_pager_lookup(src_mem_obj, src_pager);
1041 assert(src_offset / PAGE_SIZE < src_pager->cpgr_num_slots);
1042 compressor_pager_slot_lookup(pager: src_pager, FALSE, offset: src_offset, slot_pp: &src_slot_p);
1043 assert(src_slot_p != NULL);
1044 assert(*src_slot_p != 0);
1045
1046 /* transfer the slot from source to destination */
1047 vm_compressor_transfer(dst_slot_p, src_slot_p);
1048 OSAddAtomic(-1, &src_pager->cpgr_num_slots_occupied);
1049 OSAddAtomic(+1, &dst_pager->cpgr_num_slots_occupied);
1050}
1051
1052memory_object_offset_t
1053vm_compressor_pager_next_compressed(
1054 memory_object_t mem_obj,
1055 memory_object_offset_t offset)
1056{
1057 compressor_pager_t pager;
1058 unsigned int num_chunks;
1059 uint32_t page_num;
1060 unsigned int chunk_idx;
1061 uint32_t slot_idx;
1062 compressor_slot_t *chunk;
1063
1064 compressor_pager_lookup(mem_obj, pager);
1065
1066 page_num = (uint32_t)(offset / PAGE_SIZE);
1067 if (page_num != (offset / PAGE_SIZE)) {
1068 /* overflow */
1069 return (memory_object_offset_t) -1;
1070 }
1071 if (page_num >= pager->cpgr_num_slots) {
1072 /* out of range */
1073 return (memory_object_offset_t) -1;
1074 }
1075
1076 num_chunks = compressor_pager_num_chunks(pager);
1077 if (num_chunks == 1) {
1078 if (pager->cpgr_num_slots > 2) {
1079 chunk = pager->cpgr_slots.cpgr_dslots;
1080 } else {
1081 chunk = &pager->cpgr_slots.cpgr_eslots[0];
1082 }
1083 for (slot_idx = page_num;
1084 slot_idx < pager->cpgr_num_slots;
1085 slot_idx++) {
1086 if (chunk[slot_idx] != 0) {
1087 /* found a non-NULL slot in this chunk */
1088 return (memory_object_offset_t) slot_idx *
1089 PAGE_SIZE;
1090 }
1091 }
1092 return (memory_object_offset_t) -1;
1093 }
1094
1095 /* we have an array of chunks; find the next non-NULL chunk */
1096 chunk = NULL;
1097 for (chunk_idx = page_num / COMPRESSOR_SLOTS_PER_CHUNK,
1098 slot_idx = page_num % COMPRESSOR_SLOTS_PER_CHUNK;
1099 chunk_idx < num_chunks;
1100 chunk_idx++,
1101 slot_idx = 0) {
1102 chunk = pager->cpgr_slots.cpgr_islots[chunk_idx];
1103 if (chunk == NULL) {
1104 /* no chunk here: try the next one */
1105 continue;
1106 }
1107 /* search for an occupied slot in this chunk */
1108 for (;
1109 slot_idx < COMPRESSOR_SLOTS_PER_CHUNK;
1110 slot_idx++) {
1111 if (chunk[slot_idx] != 0) {
1112 /* found an occupied slot in this chunk */
1113 uint32_t next_slot;
1114
1115 next_slot = ((chunk_idx *
1116 COMPRESSOR_SLOTS_PER_CHUNK) +
1117 slot_idx);
1118 if (next_slot >= pager->cpgr_num_slots) {
1119 /* went beyond end of object */
1120 return (memory_object_offset_t) -1;
1121 }
1122 return (memory_object_offset_t) next_slot *
1123 PAGE_SIZE;
1124 }
1125 }
1126 }
1127 return (memory_object_offset_t) -1;
1128}
1129
1130unsigned int
1131vm_compressor_pager_get_count(
1132 memory_object_t mem_obj)
1133{
1134 compressor_pager_t pager;
1135
1136 compressor_pager_lookup(mem_obj, pager);
1137 if (pager == NULL) {
1138 return 0;
1139 }
1140
1141 /*
1142 * The caller should have the VM object locked and one
1143 * needs that lock to do a page-in or page-out, so no
1144 * need to lock the pager here.
1145 */
1146 assert(pager->cpgr_num_slots_occupied >= 0);
1147
1148 return pager->cpgr_num_slots_occupied;
1149}
1150
1151void
1152vm_compressor_pager_count(
1153 memory_object_t mem_obj,
1154 int compressed_count_delta,
1155 boolean_t shared_lock,
1156 vm_object_t object __unused)
1157{
1158 compressor_pager_t pager;
1159
1160 if (compressed_count_delta == 0) {
1161 return;
1162 }
1163
1164 compressor_pager_lookup(mem_obj, pager);
1165 if (pager == NULL) {
1166 return;
1167 }
1168
1169 if (compressed_count_delta < 0) {
1170 assert(pager->cpgr_num_slots_occupied >=
1171 (unsigned int) -compressed_count_delta);
1172 }
1173
1174 /*
1175 * The caller should have the VM object locked,
1176 * shared or exclusive.
1177 */
1178 if (shared_lock) {
1179 vm_object_lock_assert_shared(object);
1180 OSAddAtomic(compressed_count_delta,
1181 &pager->cpgr_num_slots_occupied);
1182 } else {
1183 vm_object_lock_assert_exclusive(object);
1184 pager->cpgr_num_slots_occupied += compressed_count_delta;
1185 }
1186}
1187
1188#if CONFIG_FREEZE
1189kern_return_t
1190vm_compressor_pager_relocate(
1191 memory_object_t mem_obj,
1192 memory_object_offset_t offset,
1193 void **current_chead)
1194{
1195 /*
1196 * Has the page at this offset been compressed?
1197 */
1198
1199 compressor_slot_t *slot_p;
1200 compressor_pager_t dst_pager;
1201
1202 assert(mem_obj);
1203
1204 compressor_pager_lookup(mem_obj, dst_pager);
1205 if (dst_pager == NULL) {
1206 return KERN_FAILURE;
1207 }
1208
1209 compressor_pager_slot_lookup(dst_pager, FALSE, offset, &slot_p);
1210 return vm_compressor_relocate(current_chead, slot_p);
1211}
1212#endif /* CONFIG_FREEZE */
1213
1214#if DEVELOPMENT || DEBUG
1215
1216kern_return_t
1217vm_compressor_pager_inject_error(memory_object_t mem_obj,
1218 memory_object_offset_t offset)
1219{
1220 kern_return_t result = KERN_FAILURE;
1221 compressor_slot_t *slot_p;
1222 compressor_pager_t pager;
1223
1224 assert(mem_obj);
1225
1226 compressor_pager_lookup(mem_obj, pager);
1227 if (pager != NULL) {
1228 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
1229 if (slot_p != NULL && *slot_p != 0) {
1230 vm_compressor_inject_error(slot_p);
1231 result = KERN_SUCCESS;
1232 }
1233 }
1234
1235 return result;
1236}
1237
1238#endif
1239