1 | /* |
2 | * Copyright (c) 2019-2020 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | |
57 | /* |
58 | * Compressor Pager. |
59 | * Memory Object Management. |
60 | */ |
61 | |
62 | #include <kern/host_statistics.h> |
63 | #include <kern/kalloc.h> |
64 | #include <kern/ipc_kobject.h> |
65 | |
66 | #include <machine/atomic.h> |
67 | |
68 | #include <mach/memory_object_control.h> |
69 | #include <mach/memory_object_types.h> |
70 | #include <mach/upl.h> |
71 | |
72 | #include <vm/memory_object.h> |
73 | #include <vm/vm_compressor_pager.h> |
74 | #include <vm/vm_external.h> |
75 | #include <vm/vm_pageout.h> |
76 | #include <vm/vm_protos.h> |
77 | |
78 | #include <sys/kdebug_triage.h> |
79 | |
80 | /* memory_object interfaces */ |
81 | void compressor_memory_object_reference(memory_object_t mem_obj); |
82 | void compressor_memory_object_deallocate(memory_object_t mem_obj); |
83 | kern_return_t compressor_memory_object_init( |
84 | memory_object_t mem_obj, |
85 | memory_object_control_t control, |
86 | memory_object_cluster_size_t ); |
87 | kern_return_t compressor_memory_object_terminate(memory_object_t mem_obj); |
88 | kern_return_t compressor_memory_object_data_request( |
89 | memory_object_t mem_obj, |
90 | memory_object_offset_t offset, |
91 | memory_object_cluster_size_t length, |
92 | __unused vm_prot_t protection_required, |
93 | memory_object_fault_info_t fault_info); |
94 | kern_return_t compressor_memory_object_data_return( |
95 | memory_object_t mem_obj, |
96 | memory_object_offset_t offset, |
97 | memory_object_cluster_size_t size, |
98 | __unused memory_object_offset_t *resid_offset, |
99 | __unused int *io_error, |
100 | __unused boolean_t dirty, |
101 | __unused boolean_t kernel_copy, |
102 | __unused int upl_flags); |
103 | kern_return_t compressor_memory_object_data_initialize( |
104 | memory_object_t mem_obj, |
105 | memory_object_offset_t offset, |
106 | memory_object_cluster_size_t size); |
107 | kern_return_t compressor_memory_object_map( |
108 | __unused memory_object_t mem_obj, |
109 | __unused vm_prot_t prot); |
110 | kern_return_t compressor_memory_object_last_unmap(memory_object_t mem_obj); |
111 | |
112 | const struct memory_object_pager_ops = { |
113 | .memory_object_reference = compressor_memory_object_reference, |
114 | .memory_object_deallocate = compressor_memory_object_deallocate, |
115 | .memory_object_init = compressor_memory_object_init, |
116 | .memory_object_terminate = compressor_memory_object_terminate, |
117 | .memory_object_data_request = compressor_memory_object_data_request, |
118 | .memory_object_data_return = compressor_memory_object_data_return, |
119 | .memory_object_data_initialize = compressor_memory_object_data_initialize, |
120 | .memory_object_map = compressor_memory_object_map, |
121 | .memory_object_last_unmap = compressor_memory_object_last_unmap, |
122 | .memory_object_backing_object = NULL, |
123 | .memory_object_pager_name = "compressor pager" |
124 | }; |
125 | |
126 | /* internal data structures */ |
127 | |
128 | struct { |
129 | uint64_t data_returns; |
130 | uint64_t data_requests; |
131 | uint64_t put; |
132 | uint64_t get; |
133 | uint64_t state_clr; |
134 | uint64_t state_get; |
135 | uint64_t transfer; |
136 | } ; |
137 | |
138 | typedef int compressor_slot_t; |
139 | |
140 | typedef struct { |
141 | /* mandatory generic header */ |
142 | struct memory_object ; |
143 | |
144 | /* pager-specific data */ |
145 | lck_mtx_t ; |
146 | #if MEMORY_OBJECT_HAS_REFCOUNT |
147 | #define cpgr_references cpgr_hdr.mo_ref |
148 | #else |
149 | os_ref_atomic_t cpgr_references; |
150 | #endif |
151 | unsigned int ; |
152 | unsigned int ; |
153 | union { |
154 | compressor_slot_t [2]; /* embedded slots */ |
155 | compressor_slot_t *; /* direct slots */ |
156 | compressor_slot_t **; /* indirect slots */ |
157 | } ; |
158 | } *; |
159 | |
160 | #define (_mem_obj_, _cpgr_) \ |
161 | MACRO_BEGIN \ |
162 | if (_mem_obj_ == NULL || \ |
163 | _mem_obj_->mo_pager_ops != &compressor_pager_ops) { \ |
164 | _cpgr_ = NULL; \ |
165 | } else { \ |
166 | _cpgr_ = (compressor_pager_t) _mem_obj_; \ |
167 | } \ |
168 | MACRO_END |
169 | |
170 | /* embedded slot pointers in compressor_pager get packed, so VA restricted */ |
171 | static ZONE_DEFINE_TYPE(, "compressor_pager" , |
172 | struct compressor_pager, ZC_NOENCRYPT | ZC_VM); |
173 | |
174 | LCK_GRP_DECLARE(, "compressor_pager" ); |
175 | |
176 | #define (_cpgr_) \ |
177 | lck_mtx_lock(&(_cpgr_)->cpgr_lock) |
178 | #define (_cpgr_) \ |
179 | lck_mtx_unlock(&(_cpgr_)->cpgr_lock) |
180 | #define (_cpgr_) \ |
181 | lck_mtx_init(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp, LCK_ATTR_NULL) |
182 | #define (_cpgr_) \ |
183 | lck_mtx_destroy(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp) |
184 | |
185 | #define COMPRESSOR_SLOTS_CHUNK_SIZE (512) |
186 | #define COMPRESSOR_SLOTS_PER_CHUNK (COMPRESSOR_SLOTS_CHUNK_SIZE / sizeof (compressor_slot_t)) |
187 | |
188 | /* forward declarations */ |
189 | unsigned int compressor_pager_slots_chunk_free(compressor_slot_t *chunk, |
190 | int num_slots, |
191 | int flags, |
192 | int *failures); |
193 | void compressor_pager_slot_lookup( |
194 | compressor_pager_t , |
195 | boolean_t do_alloc, |
196 | memory_object_offset_t offset, |
197 | compressor_slot_t **slot_pp); |
198 | |
199 | #if defined(__LP64__) |
200 | |
201 | /* restricted VA zones for slots */ |
202 | |
203 | #define NUM_SLOTS_ZONES 3 |
204 | |
205 | static const size_t compressor_slots_zones_sizes[NUM_SLOTS_ZONES] = { |
206 | 16, |
207 | 64, |
208 | COMPRESSOR_SLOTS_CHUNK_SIZE |
209 | }; |
210 | |
211 | static const char * compressor_slots_zones_names[NUM_SLOTS_ZONES] = { |
212 | "compressor_slots.16" , |
213 | "compressor_slots.64" , |
214 | "compressor_slots.512" |
215 | }; |
216 | |
217 | static zone_t |
218 | compressor_slots_zones[NUM_SLOTS_ZONES]; |
219 | |
220 | #endif /* defined(__LP64__) */ |
221 | |
222 | static void |
223 | zfree_slot_array(compressor_slot_t *slots, size_t size); |
224 | static compressor_slot_t * |
225 | zalloc_slot_array(size_t size, zalloc_flags_t); |
226 | |
227 | static inline unsigned int |
228 | ( |
229 | compressor_pager_t ) |
230 | { |
231 | unsigned int num_chunks; |
232 | |
233 | num_chunks = pager->cpgr_num_slots / COMPRESSOR_SLOTS_PER_CHUNK; |
234 | if (num_chunks * COMPRESSOR_SLOTS_PER_CHUNK < pager->cpgr_num_slots) { |
235 | num_chunks++; |
236 | } |
237 | return num_chunks; |
238 | } |
239 | |
240 | kern_return_t |
241 | compressor_memory_object_init( |
242 | memory_object_t mem_obj, |
243 | memory_object_control_t control, |
244 | __unused memory_object_cluster_size_t ) |
245 | { |
246 | compressor_pager_t ; |
247 | |
248 | assert(pager_page_size == PAGE_SIZE); |
249 | |
250 | memory_object_control_reference(control); |
251 | |
252 | compressor_pager_lookup(mem_obj, pager); |
253 | compressor_pager_lock(pager); |
254 | |
255 | if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { |
256 | panic("compressor_memory_object_init: bad request" ); |
257 | } |
258 | pager->cpgr_hdr.mo_control = control; |
259 | |
260 | compressor_pager_unlock(pager); |
261 | |
262 | return KERN_SUCCESS; |
263 | } |
264 | |
265 | kern_return_t |
266 | compressor_memory_object_map( |
267 | __unused memory_object_t mem_obj, |
268 | __unused vm_prot_t prot) |
269 | { |
270 | panic("compressor_memory_object_map" ); |
271 | return KERN_FAILURE; |
272 | } |
273 | |
274 | kern_return_t |
275 | compressor_memory_object_last_unmap( |
276 | __unused memory_object_t mem_obj) |
277 | { |
278 | panic("compressor_memory_object_last_unmap" ); |
279 | return KERN_FAILURE; |
280 | } |
281 | |
282 | kern_return_t |
283 | compressor_memory_object_terminate( |
284 | memory_object_t mem_obj) |
285 | { |
286 | memory_object_control_t control; |
287 | compressor_pager_t ; |
288 | |
289 | /* |
290 | * control port is a receive right, not a send right. |
291 | */ |
292 | |
293 | compressor_pager_lookup(mem_obj, pager); |
294 | compressor_pager_lock(pager); |
295 | |
296 | /* |
297 | * After memory_object_terminate both memory_object_init |
298 | * and a no-senders notification are possible, so we need |
299 | * to clean up our reference to the memory_object_control |
300 | * to prepare for a new init. |
301 | */ |
302 | |
303 | control = pager->cpgr_hdr.mo_control; |
304 | pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; |
305 | |
306 | compressor_pager_unlock(pager); |
307 | |
308 | /* |
309 | * Now we deallocate our reference on the control. |
310 | */ |
311 | memory_object_control_deallocate(control); |
312 | return KERN_SUCCESS; |
313 | } |
314 | |
315 | void |
316 | compressor_memory_object_reference( |
317 | memory_object_t mem_obj) |
318 | { |
319 | compressor_pager_t ; |
320 | |
321 | compressor_pager_lookup(mem_obj, pager); |
322 | if (pager == NULL) { |
323 | return; |
324 | } |
325 | |
326 | compressor_pager_lock(pager); |
327 | os_ref_retain_locked_raw(&pager->cpgr_references, NULL); |
328 | compressor_pager_unlock(pager); |
329 | } |
330 | |
331 | void |
332 | compressor_memory_object_deallocate( |
333 | memory_object_t mem_obj) |
334 | { |
335 | compressor_pager_t ; |
336 | unsigned int num_slots_freed; |
337 | |
338 | /* |
339 | * Because we don't give out multiple first references |
340 | * for a memory object, there can't be a race |
341 | * between getting a deallocate call and creating |
342 | * a new reference for the object. |
343 | */ |
344 | |
345 | compressor_pager_lookup(mem_obj, pager); |
346 | if (pager == NULL) { |
347 | return; |
348 | } |
349 | |
350 | compressor_pager_lock(pager); |
351 | if (os_ref_release_locked_raw(&pager->cpgr_references, NULL) > 0) { |
352 | compressor_pager_unlock(pager); |
353 | return; |
354 | } |
355 | |
356 | /* |
357 | * We shouldn't get a deallocation call |
358 | * when the kernel has the object cached. |
359 | */ |
360 | if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) { |
361 | panic("compressor_memory_object_deallocate(): bad request" ); |
362 | } |
363 | |
364 | /* |
365 | * Unlock the pager (though there should be no one |
366 | * waiting for it). |
367 | */ |
368 | compressor_pager_unlock(pager); |
369 | |
370 | /* free the compressor slots */ |
371 | unsigned int num_chunks; |
372 | unsigned int i; |
373 | compressor_slot_t *chunk; |
374 | |
375 | num_chunks = compressor_pager_num_chunks(pager); |
376 | if (num_chunks > 1) { |
377 | /* we have an array of chunks */ |
378 | for (i = 0; i < num_chunks; i++) { |
379 | chunk = pager->cpgr_slots.cpgr_islots[i]; |
380 | if (chunk != NULL) { |
381 | num_slots_freed = |
382 | compressor_pager_slots_chunk_free( |
383 | chunk, |
384 | COMPRESSOR_SLOTS_PER_CHUNK, |
385 | flags: 0, |
386 | NULL); |
387 | pager->cpgr_slots.cpgr_islots[i] = NULL; |
388 | zfree_slot_array(slots: chunk, COMPRESSOR_SLOTS_CHUNK_SIZE); |
389 | } |
390 | } |
391 | kfree_type(compressor_slot_t *, num_chunks, |
392 | pager->cpgr_slots.cpgr_islots); |
393 | pager->cpgr_slots.cpgr_islots = NULL; |
394 | } else if (pager->cpgr_num_slots > 2) { |
395 | chunk = pager->cpgr_slots.cpgr_dslots; |
396 | num_slots_freed = |
397 | compressor_pager_slots_chunk_free( |
398 | chunk, |
399 | num_slots: pager->cpgr_num_slots, |
400 | flags: 0, |
401 | NULL); |
402 | pager->cpgr_slots.cpgr_dslots = NULL; |
403 | zfree_slot_array(slots: chunk, |
404 | size: (pager->cpgr_num_slots * |
405 | sizeof(pager->cpgr_slots.cpgr_dslots[0]))); |
406 | } else { |
407 | chunk = &pager->cpgr_slots.cpgr_eslots[0]; |
408 | num_slots_freed = |
409 | compressor_pager_slots_chunk_free( |
410 | chunk, |
411 | num_slots: pager->cpgr_num_slots, |
412 | flags: 0, |
413 | NULL); |
414 | } |
415 | |
416 | compressor_pager_lock_destroy(pager); |
417 | zfree(compressor_pager_zone, pager); |
418 | } |
419 | |
420 | kern_return_t |
421 | compressor_memory_object_data_request( |
422 | memory_object_t mem_obj, |
423 | memory_object_offset_t offset, |
424 | memory_object_cluster_size_t length, |
425 | __unused vm_prot_t protection_required, |
426 | __unused memory_object_fault_info_t fault_info) |
427 | { |
428 | compressor_pager_t ; |
429 | kern_return_t kr; |
430 | compressor_slot_t *slot_p; |
431 | |
432 | compressor_pager_stats.data_requests++; |
433 | |
434 | /* |
435 | * Request must be on a page boundary and a multiple of pages. |
436 | */ |
437 | if ((offset & PAGE_MASK) != 0 || (length & PAGE_MASK) != 0) { |
438 | panic("compressor_memory_object_data_request(): bad alignment" ); |
439 | } |
440 | |
441 | if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) { |
442 | panic("%s: offset 0x%llx overflow" , |
443 | __FUNCTION__, (uint64_t) offset); |
444 | return KERN_FAILURE; |
445 | } |
446 | |
447 | compressor_pager_lookup(mem_obj, pager); |
448 | |
449 | if (length == 0) { |
450 | /* we're only querying the pager for this page */ |
451 | } else { |
452 | panic("compressor: data_request" ); |
453 | } |
454 | |
455 | /* find the compressor slot for that page */ |
456 | compressor_pager_slot_lookup(pager, FALSE, offset, slot_pp: &slot_p); |
457 | |
458 | if (offset / PAGE_SIZE >= pager->cpgr_num_slots) { |
459 | /* out of range */ |
460 | kr = KERN_FAILURE; |
461 | } else if (slot_p == NULL || *slot_p == 0) { |
462 | /* compressor does not have this page */ |
463 | kr = KERN_FAILURE; |
464 | } else { |
465 | /* compressor does have this page */ |
466 | kr = KERN_SUCCESS; |
467 | } |
468 | return kr; |
469 | } |
470 | |
471 | /* |
472 | * memory_object_data_initialize: check whether we already have each page, and |
473 | * write it if we do not. The implementation is far from optimized, and |
474 | * also assumes that the default_pager is single-threaded. |
475 | */ |
476 | /* It is questionable whether or not a pager should decide what is relevant */ |
477 | /* and what is not in data sent from the kernel. Data initialize has been */ |
478 | /* changed to copy back all data sent to it in preparation for its eventual */ |
479 | /* merge with data return. It is the kernel that should decide what pages */ |
480 | /* to write back. As of the writing of this note, this is indeed the case */ |
481 | /* the kernel writes back one page at a time through this interface */ |
482 | |
483 | kern_return_t |
484 | compressor_memory_object_data_initialize( |
485 | memory_object_t mem_obj, |
486 | memory_object_offset_t offset, |
487 | memory_object_cluster_size_t size) |
488 | { |
489 | compressor_pager_t ; |
490 | memory_object_offset_t cur_offset; |
491 | |
492 | compressor_pager_lookup(mem_obj, pager); |
493 | compressor_pager_lock(pager); |
494 | |
495 | for (cur_offset = offset; |
496 | cur_offset < offset + size; |
497 | cur_offset += PAGE_SIZE) { |
498 | panic("do a data_return() if slot for this page is empty" ); |
499 | } |
500 | |
501 | compressor_pager_unlock(pager); |
502 | |
503 | return KERN_SUCCESS; |
504 | } |
505 | |
506 | |
507 | /*ARGSUSED*/ |
508 | kern_return_t |
509 | compressor_memory_object_data_return( |
510 | __unused memory_object_t mem_obj, |
511 | __unused memory_object_offset_t offset, |
512 | __unused memory_object_cluster_size_t size, |
513 | __unused memory_object_offset_t *resid_offset, |
514 | __unused int *io_error, |
515 | __unused boolean_t dirty, |
516 | __unused boolean_t kernel_copy, |
517 | __unused int upl_flags) |
518 | { |
519 | panic("compressor: data_return" ); |
520 | return KERN_FAILURE; |
521 | } |
522 | |
523 | /* |
524 | * Routine: default_pager_memory_object_create |
525 | * Purpose: |
526 | * Handle requests for memory objects from the |
527 | * kernel. |
528 | * Notes: |
529 | * Because we only give out the default memory |
530 | * manager port to the kernel, we don't have to |
531 | * be so paranoid about the contents. |
532 | */ |
533 | kern_return_t |
534 | compressor_memory_object_create( |
535 | memory_object_size_t new_size, |
536 | memory_object_t *new_mem_obj) |
537 | { |
538 | compressor_pager_t ; |
539 | unsigned int num_chunks; |
540 | |
541 | if ((uint32_t)(new_size / PAGE_SIZE) != (new_size / PAGE_SIZE)) { |
542 | /* 32-bit overflow for number of pages */ |
543 | panic("%s: size 0x%llx overflow" , |
544 | __FUNCTION__, (uint64_t) new_size); |
545 | return KERN_INVALID_ARGUMENT; |
546 | } |
547 | |
548 | pager = zalloc_flags(compressor_pager_zone, Z_WAITOK | Z_NOFAIL); |
549 | |
550 | compressor_pager_lock_init(pager); |
551 | os_ref_init_raw(&pager->cpgr_references, NULL); |
552 | pager->cpgr_num_slots = (uint32_t)(new_size / PAGE_SIZE); |
553 | pager->cpgr_num_slots_occupied = 0; |
554 | |
555 | num_chunks = compressor_pager_num_chunks(pager); |
556 | if (num_chunks > 1) { |
557 | pager->cpgr_slots.cpgr_islots = kalloc_type(compressor_slot_t *, |
558 | num_chunks, Z_WAITOK | Z_ZERO); |
559 | } else if (pager->cpgr_num_slots > 2) { |
560 | pager->cpgr_slots.cpgr_dslots = zalloc_slot_array(size: pager->cpgr_num_slots * |
561 | sizeof(pager->cpgr_slots.cpgr_dslots[0]), Z_WAITOK | Z_ZERO); |
562 | } else { |
563 | pager->cpgr_slots.cpgr_eslots[0] = 0; |
564 | pager->cpgr_slots.cpgr_eslots[1] = 0; |
565 | } |
566 | |
567 | /* |
568 | * Set up associations between this memory object |
569 | * and this compressor_pager structure |
570 | */ |
571 | pager->cpgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT; |
572 | pager->cpgr_hdr.mo_pager_ops = &compressor_pager_ops; |
573 | pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL; |
574 | |
575 | *new_mem_obj = (memory_object_t) pager; |
576 | return KERN_SUCCESS; |
577 | } |
578 | |
579 | |
580 | unsigned int |
581 | ( |
582 | compressor_slot_t *chunk, |
583 | int num_slots, |
584 | int flags, |
585 | int *failures) |
586 | { |
587 | int i; |
588 | int retval; |
589 | unsigned int num_slots_freed; |
590 | |
591 | if (failures) { |
592 | *failures = 0; |
593 | } |
594 | num_slots_freed = 0; |
595 | for (i = 0; i < num_slots; i++) { |
596 | if (chunk[i] != 0) { |
597 | retval = vm_compressor_free(slot: &chunk[i], flags); |
598 | |
599 | if (retval == 0) { |
600 | num_slots_freed++; |
601 | } else { |
602 | if (retval == -2) { |
603 | assert(flags & C_DONT_BLOCK); |
604 | } |
605 | |
606 | if (failures) { |
607 | *failures += 1; |
608 | } |
609 | } |
610 | } |
611 | } |
612 | return num_slots_freed; |
613 | } |
614 | |
615 | void |
616 | ( |
617 | compressor_pager_t , |
618 | boolean_t do_alloc, |
619 | memory_object_offset_t offset, |
620 | compressor_slot_t **slot_pp) |
621 | { |
622 | unsigned int num_chunks; |
623 | uint32_t page_num; |
624 | unsigned int chunk_idx; |
625 | int slot_idx; |
626 | compressor_slot_t *chunk; |
627 | compressor_slot_t *t_chunk; |
628 | |
629 | page_num = (uint32_t)(offset / PAGE_SIZE); |
630 | if (page_num != (offset / PAGE_SIZE)) { |
631 | /* overflow */ |
632 | panic("%s: offset 0x%llx overflow" , |
633 | __FUNCTION__, (uint64_t) offset); |
634 | *slot_pp = NULL; |
635 | return; |
636 | } |
637 | if (page_num >= pager->cpgr_num_slots) { |
638 | /* out of range */ |
639 | *slot_pp = NULL; |
640 | return; |
641 | } |
642 | num_chunks = compressor_pager_num_chunks(pager); |
643 | if (num_chunks > 1) { |
644 | /* we have an array of chunks */ |
645 | chunk_idx = page_num / COMPRESSOR_SLOTS_PER_CHUNK; |
646 | chunk = pager->cpgr_slots.cpgr_islots[chunk_idx]; |
647 | |
648 | if (chunk == NULL && do_alloc) { |
649 | t_chunk = zalloc_slot_array(COMPRESSOR_SLOTS_CHUNK_SIZE, |
650 | Z_WAITOK | Z_ZERO); |
651 | |
652 | compressor_pager_lock(pager); |
653 | |
654 | if ((chunk = pager->cpgr_slots.cpgr_islots[chunk_idx]) == NULL) { |
655 | /* |
656 | * On some platforms, the memory stores from |
657 | * the bzero(t_chunk) above might not have been |
658 | * made visible and another thread might see |
659 | * the contents of this new chunk before it's |
660 | * been fully zero-filled. |
661 | * This memory barrier should take care of this |
662 | * according to the platform requirements. |
663 | */ |
664 | os_atomic_thread_fence(release); |
665 | |
666 | chunk = pager->cpgr_slots.cpgr_islots[chunk_idx] = t_chunk; |
667 | t_chunk = NULL; |
668 | } |
669 | compressor_pager_unlock(pager); |
670 | |
671 | if (t_chunk) { |
672 | zfree_slot_array(slots: t_chunk, COMPRESSOR_SLOTS_CHUNK_SIZE); |
673 | } |
674 | } |
675 | if (chunk == NULL) { |
676 | *slot_pp = NULL; |
677 | } else { |
678 | slot_idx = page_num % COMPRESSOR_SLOTS_PER_CHUNK; |
679 | *slot_pp = &chunk[slot_idx]; |
680 | } |
681 | } else if (pager->cpgr_num_slots > 2) { |
682 | slot_idx = page_num; |
683 | *slot_pp = &pager->cpgr_slots.cpgr_dslots[slot_idx]; |
684 | } else { |
685 | slot_idx = page_num; |
686 | *slot_pp = &pager->cpgr_slots.cpgr_eslots[slot_idx]; |
687 | } |
688 | } |
689 | |
690 | #if defined(__LP64__) |
691 | __startup_func |
692 | static void |
693 | vm_compressor_slots_init(void) |
694 | { |
695 | for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) { |
696 | compressor_slots_zones[idx] = zone_create( |
697 | name: compressor_slots_zones_names[idx], |
698 | size: compressor_slots_zones_sizes[idx], |
699 | flags: ZC_PGZ_USE_GUARDS | ZC_VM); |
700 | } |
701 | } |
702 | STARTUP(ZALLOC, STARTUP_RANK_MIDDLE, vm_compressor_slots_init); |
703 | #endif /* defined(__LP64__) */ |
704 | |
705 | static compressor_slot_t * |
706 | zalloc_slot_array(size_t size, zalloc_flags_t flags) |
707 | { |
708 | #if defined(__LP64__) |
709 | compressor_slot_t *slots = NULL; |
710 | |
711 | assert(size <= COMPRESSOR_SLOTS_CHUNK_SIZE); |
712 | for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) { |
713 | if (size > compressor_slots_zones_sizes[idx]) { |
714 | continue; |
715 | } |
716 | slots = zalloc_flags(compressor_slots_zones[idx], flags); |
717 | break; |
718 | } |
719 | return slots; |
720 | #else /* defined(__LP64__) */ |
721 | return kalloc_data(size, flags); |
722 | #endif /* !defined(__LP64__) */ |
723 | } |
724 | |
725 | static void |
726 | zfree_slot_array(compressor_slot_t *slots, size_t size) |
727 | { |
728 | #if defined(__LP64__) |
729 | assert(size <= COMPRESSOR_SLOTS_CHUNK_SIZE); |
730 | for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) { |
731 | if (size > compressor_slots_zones_sizes[idx]) { |
732 | continue; |
733 | } |
734 | zfree(compressor_slots_zones[idx], slots); |
735 | break; |
736 | } |
737 | #else /* defined(__LP64__) */ |
738 | kfree_data(slots, size); |
739 | #endif /* !defined(__LP64__) */ |
740 | } |
741 | |
742 | kern_return_t |
743 | ( |
744 | memory_object_t mem_obj, |
745 | memory_object_offset_t offset, |
746 | ppnum_t ppnum, |
747 | bool unmodified, |
748 | void **current_chead, |
749 | char *scratch_buf, |
750 | int *compressed_count_delta_p) |
751 | { |
752 | compressor_pager_t ; |
753 | compressor_slot_t *slot_p; |
754 | |
755 | compressor_pager_stats.put++; |
756 | |
757 | *compressed_count_delta_p = 0; |
758 | |
759 | /* This routine is called by the pageout thread. The pageout thread */ |
760 | /* cannot be blocked by read activities unless the read activities */ |
761 | /* Therefore the grant of vs lock must be done on a try versus a */ |
762 | /* blocking basis. The code below relies on the fact that the */ |
763 | /* interface is synchronous. Should this interface be again async */ |
764 | /* for some type of pager in the future the pages will have to be */ |
765 | /* returned through a separate, asynchronous path. */ |
766 | |
767 | compressor_pager_lookup(mem_obj, pager); |
768 | |
769 | if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) { |
770 | /* overflow */ |
771 | panic("%s: offset 0x%llx overflow" , |
772 | __FUNCTION__, (uint64_t) offset); |
773 | return KERN_RESOURCE_SHORTAGE; |
774 | } |
775 | |
776 | compressor_pager_slot_lookup(pager, TRUE, offset, slot_pp: &slot_p); |
777 | |
778 | if (slot_p == NULL) { |
779 | /* out of range ? */ |
780 | panic("vm_compressor_pager_put: out of range" ); |
781 | } |
782 | if (*slot_p != 0) { |
783 | /* |
784 | * Already compressed: forget about the old one. |
785 | * |
786 | * This can happen after a vm_object_do_collapse() when |
787 | * the "backing_object" had some pages paged out and the |
788 | * "object" had an equivalent page resident. |
789 | */ |
790 | vm_compressor_free(slot: slot_p, flags: (unmodified ? C_PAGE_UNMODIFIED : 0)); |
791 | *compressed_count_delta_p -= 1; |
792 | } |
793 | |
794 | /* |
795 | * If the compressor operation succeeds, we presumably don't need to |
796 | * undo any previous WIMG update, as all live mappings should be |
797 | * disconnected. |
798 | */ |
799 | |
800 | if (vm_compressor_put(pn: ppnum, slot: slot_p, current_chead, scratch_buf, unmodified)) { |
801 | return KERN_RESOURCE_SHORTAGE; |
802 | } |
803 | *compressed_count_delta_p += 1; |
804 | |
805 | return KERN_SUCCESS; |
806 | } |
807 | |
808 | |
809 | kern_return_t |
810 | ( |
811 | memory_object_t mem_obj, |
812 | memory_object_offset_t offset, |
813 | ppnum_t ppnum, |
814 | int *my_fault_type, |
815 | int flags, |
816 | int *compressed_count_delta_p) |
817 | { |
818 | compressor_pager_t ; |
819 | kern_return_t kr; |
820 | compressor_slot_t *slot_p; |
821 | |
822 | compressor_pager_stats.get++; |
823 | |
824 | *compressed_count_delta_p = 0; |
825 | |
826 | if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) { |
827 | panic("%s: offset 0x%llx overflow" , |
828 | __FUNCTION__, (uint64_t) offset); |
829 | return KERN_MEMORY_ERROR; |
830 | } |
831 | |
832 | compressor_pager_lookup(mem_obj, pager); |
833 | |
834 | /* find the compressor slot for that page */ |
835 | compressor_pager_slot_lookup(pager, FALSE, offset, slot_pp: &slot_p); |
836 | |
837 | if (offset / PAGE_SIZE >= pager->cpgr_num_slots) { |
838 | /* out of range */ |
839 | ktriage_record(thread_id: thread_tid(thread: current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_GET_OUT_OF_RANGE), arg: 0 /* arg */); |
840 | kr = KERN_MEMORY_FAILURE; |
841 | } else if (slot_p == NULL || *slot_p == 0) { |
842 | /* compressor does not have this page */ |
843 | ktriage_record(thread_id: thread_tid(thread: current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_GET_NO_PAGE), arg: 0 /* arg */); |
844 | kr = KERN_MEMORY_ERROR; |
845 | } else { |
846 | /* compressor does have this page */ |
847 | kr = KERN_SUCCESS; |
848 | } |
849 | *my_fault_type = DBG_COMPRESSOR_FAULT; |
850 | |
851 | if (kr == KERN_SUCCESS) { |
852 | int retval; |
853 | bool unmodified = (vm_compressor_is_slot_compressed(slot: slot_p) == false); |
854 | /* get the page from the compressor */ |
855 | retval = vm_compressor_get(pn: ppnum, slot: slot_p, flags: (unmodified ? (flags | C_PAGE_UNMODIFIED) : flags)); |
856 | if (retval == -1) { |
857 | ktriage_record(thread_id: thread_tid(thread: current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_DECOMPRESS_FAILED), arg: 0 /* arg */); |
858 | kr = KERN_MEMORY_FAILURE; |
859 | } else if (retval == 1) { |
860 | *my_fault_type = DBG_COMPRESSOR_SWAPIN_FAULT; |
861 | } else if (retval == -2) { |
862 | assert((flags & C_DONT_BLOCK)); |
863 | /* |
864 | * Not a fatal failure because we just retry with a blocking get later. So we skip ktriage to avoid noise. |
865 | */ |
866 | kr = KERN_FAILURE; |
867 | } |
868 | } |
869 | |
870 | if (kr == KERN_SUCCESS) { |
871 | assert(slot_p != NULL); |
872 | if (*slot_p != 0) { |
873 | /* |
874 | * We got the page for a copy-on-write fault |
875 | * and we kept the original in place. Slot |
876 | * is still occupied. |
877 | */ |
878 | } else { |
879 | *compressed_count_delta_p -= 1; |
880 | } |
881 | } |
882 | |
883 | return kr; |
884 | } |
885 | |
886 | unsigned int |
887 | ( |
888 | memory_object_t mem_obj, |
889 | memory_object_offset_t offset) |
890 | { |
891 | compressor_pager_t ; |
892 | compressor_slot_t *slot_p; |
893 | unsigned int num_slots_freed; |
894 | |
895 | assert(VM_CONFIG_COMPRESSOR_IS_PRESENT); |
896 | |
897 | compressor_pager_stats.state_clr++; |
898 | |
899 | if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) { |
900 | /* overflow */ |
901 | panic("%s: offset 0x%llx overflow" , |
902 | __FUNCTION__, (uint64_t) offset); |
903 | return 0; |
904 | } |
905 | |
906 | compressor_pager_lookup(mem_obj, pager); |
907 | |
908 | /* find the compressor slot for that page */ |
909 | compressor_pager_slot_lookup(pager, FALSE, offset, slot_pp: &slot_p); |
910 | |
911 | num_slots_freed = 0; |
912 | if (slot_p && *slot_p != 0) { |
913 | vm_compressor_free(slot: slot_p, flags: 0); |
914 | num_slots_freed++; |
915 | assert(*slot_p == 0); |
916 | } |
917 | |
918 | return num_slots_freed; |
919 | } |
920 | |
921 | vm_external_state_t |
922 | ( |
923 | memory_object_t mem_obj, |
924 | memory_object_offset_t offset) |
925 | { |
926 | compressor_pager_t ; |
927 | compressor_slot_t *slot_p; |
928 | |
929 | assert(VM_CONFIG_COMPRESSOR_IS_PRESENT); |
930 | |
931 | compressor_pager_stats.state_get++; |
932 | |
933 | if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) { |
934 | /* overflow */ |
935 | panic("%s: offset 0x%llx overflow" , |
936 | __FUNCTION__, (uint64_t) offset); |
937 | return VM_EXTERNAL_STATE_ABSENT; |
938 | } |
939 | |
940 | compressor_pager_lookup(mem_obj, pager); |
941 | |
942 | /* find the compressor slot for that page */ |
943 | compressor_pager_slot_lookup(pager, FALSE, offset, slot_pp: &slot_p); |
944 | |
945 | if (offset / PAGE_SIZE >= pager->cpgr_num_slots) { |
946 | /* out of range */ |
947 | return VM_EXTERNAL_STATE_ABSENT; |
948 | } else if (slot_p == NULL || *slot_p == 0) { |
949 | /* compressor does not have this page */ |
950 | return VM_EXTERNAL_STATE_ABSENT; |
951 | } else { |
952 | /* compressor does have this page */ |
953 | return VM_EXTERNAL_STATE_EXISTS; |
954 | } |
955 | } |
956 | |
957 | unsigned int |
958 | ( |
959 | memory_object_t mem_obj, |
960 | int flags) |
961 | { |
962 | compressor_pager_t ; |
963 | unsigned int num_chunks; |
964 | int failures; |
965 | unsigned int i; |
966 | compressor_slot_t *chunk; |
967 | unsigned int num_slots_freed; |
968 | |
969 | compressor_pager_lookup(mem_obj, pager); |
970 | if (pager == NULL) { |
971 | return 0; |
972 | } |
973 | |
974 | compressor_pager_lock(pager); |
975 | |
976 | /* reap the compressor slots */ |
977 | num_slots_freed = 0; |
978 | |
979 | num_chunks = compressor_pager_num_chunks(pager); |
980 | if (num_chunks > 1) { |
981 | /* we have an array of chunks */ |
982 | for (i = 0; i < num_chunks; i++) { |
983 | chunk = pager->cpgr_slots.cpgr_islots[i]; |
984 | if (chunk != NULL) { |
985 | num_slots_freed += |
986 | compressor_pager_slots_chunk_free( |
987 | chunk, |
988 | COMPRESSOR_SLOTS_PER_CHUNK, |
989 | flags, |
990 | failures: &failures); |
991 | if (failures == 0) { |
992 | pager->cpgr_slots.cpgr_islots[i] = NULL; |
993 | zfree_slot_array(slots: chunk, COMPRESSOR_SLOTS_CHUNK_SIZE); |
994 | } |
995 | } |
996 | } |
997 | } else if (pager->cpgr_num_slots > 2) { |
998 | chunk = pager->cpgr_slots.cpgr_dslots; |
999 | num_slots_freed += |
1000 | compressor_pager_slots_chunk_free( |
1001 | chunk, |
1002 | num_slots: pager->cpgr_num_slots, |
1003 | flags, |
1004 | NULL); |
1005 | } else { |
1006 | chunk = &pager->cpgr_slots.cpgr_eslots[0]; |
1007 | num_slots_freed += |
1008 | compressor_pager_slots_chunk_free( |
1009 | chunk, |
1010 | num_slots: pager->cpgr_num_slots, |
1011 | flags, |
1012 | NULL); |
1013 | } |
1014 | |
1015 | compressor_pager_unlock(pager); |
1016 | |
1017 | return num_slots_freed; |
1018 | } |
1019 | |
1020 | void |
1021 | ( |
1022 | memory_object_t dst_mem_obj, |
1023 | memory_object_offset_t dst_offset, |
1024 | memory_object_t src_mem_obj, |
1025 | memory_object_offset_t src_offset) |
1026 | { |
1027 | compressor_pager_t , ; |
1028 | compressor_slot_t *src_slot_p, *dst_slot_p; |
1029 | |
1030 | compressor_pager_stats.transfer++; |
1031 | |
1032 | /* find the compressor slot for the destination */ |
1033 | compressor_pager_lookup(dst_mem_obj, dst_pager); |
1034 | assert(dst_offset / PAGE_SIZE < dst_pager->cpgr_num_slots); |
1035 | compressor_pager_slot_lookup(pager: dst_pager, TRUE, offset: dst_offset, slot_pp: &dst_slot_p); |
1036 | assert(dst_slot_p != NULL); |
1037 | assert(*dst_slot_p == 0); |
1038 | |
1039 | /* find the compressor slot for the source */ |
1040 | compressor_pager_lookup(src_mem_obj, src_pager); |
1041 | assert(src_offset / PAGE_SIZE < src_pager->cpgr_num_slots); |
1042 | compressor_pager_slot_lookup(pager: src_pager, FALSE, offset: src_offset, slot_pp: &src_slot_p); |
1043 | assert(src_slot_p != NULL); |
1044 | assert(*src_slot_p != 0); |
1045 | |
1046 | /* transfer the slot from source to destination */ |
1047 | vm_compressor_transfer(dst_slot_p, src_slot_p); |
1048 | OSAddAtomic(-1, &src_pager->cpgr_num_slots_occupied); |
1049 | OSAddAtomic(+1, &dst_pager->cpgr_num_slots_occupied); |
1050 | } |
1051 | |
1052 | memory_object_offset_t |
1053 | ( |
1054 | memory_object_t mem_obj, |
1055 | memory_object_offset_t offset) |
1056 | { |
1057 | compressor_pager_t ; |
1058 | unsigned int num_chunks; |
1059 | uint32_t page_num; |
1060 | unsigned int chunk_idx; |
1061 | uint32_t slot_idx; |
1062 | compressor_slot_t *chunk; |
1063 | |
1064 | compressor_pager_lookup(mem_obj, pager); |
1065 | |
1066 | page_num = (uint32_t)(offset / PAGE_SIZE); |
1067 | if (page_num != (offset / PAGE_SIZE)) { |
1068 | /* overflow */ |
1069 | return (memory_object_offset_t) -1; |
1070 | } |
1071 | if (page_num >= pager->cpgr_num_slots) { |
1072 | /* out of range */ |
1073 | return (memory_object_offset_t) -1; |
1074 | } |
1075 | |
1076 | num_chunks = compressor_pager_num_chunks(pager); |
1077 | if (num_chunks == 1) { |
1078 | if (pager->cpgr_num_slots > 2) { |
1079 | chunk = pager->cpgr_slots.cpgr_dslots; |
1080 | } else { |
1081 | chunk = &pager->cpgr_slots.cpgr_eslots[0]; |
1082 | } |
1083 | for (slot_idx = page_num; |
1084 | slot_idx < pager->cpgr_num_slots; |
1085 | slot_idx++) { |
1086 | if (chunk[slot_idx] != 0) { |
1087 | /* found a non-NULL slot in this chunk */ |
1088 | return (memory_object_offset_t) slot_idx * |
1089 | PAGE_SIZE; |
1090 | } |
1091 | } |
1092 | return (memory_object_offset_t) -1; |
1093 | } |
1094 | |
1095 | /* we have an array of chunks; find the next non-NULL chunk */ |
1096 | chunk = NULL; |
1097 | for (chunk_idx = page_num / COMPRESSOR_SLOTS_PER_CHUNK, |
1098 | slot_idx = page_num % COMPRESSOR_SLOTS_PER_CHUNK; |
1099 | chunk_idx < num_chunks; |
1100 | chunk_idx++, |
1101 | slot_idx = 0) { |
1102 | chunk = pager->cpgr_slots.cpgr_islots[chunk_idx]; |
1103 | if (chunk == NULL) { |
1104 | /* no chunk here: try the next one */ |
1105 | continue; |
1106 | } |
1107 | /* search for an occupied slot in this chunk */ |
1108 | for (; |
1109 | slot_idx < COMPRESSOR_SLOTS_PER_CHUNK; |
1110 | slot_idx++) { |
1111 | if (chunk[slot_idx] != 0) { |
1112 | /* found an occupied slot in this chunk */ |
1113 | uint32_t next_slot; |
1114 | |
1115 | next_slot = ((chunk_idx * |
1116 | COMPRESSOR_SLOTS_PER_CHUNK) + |
1117 | slot_idx); |
1118 | if (next_slot >= pager->cpgr_num_slots) { |
1119 | /* went beyond end of object */ |
1120 | return (memory_object_offset_t) -1; |
1121 | } |
1122 | return (memory_object_offset_t) next_slot * |
1123 | PAGE_SIZE; |
1124 | } |
1125 | } |
1126 | } |
1127 | return (memory_object_offset_t) -1; |
1128 | } |
1129 | |
1130 | unsigned int |
1131 | ( |
1132 | memory_object_t mem_obj) |
1133 | { |
1134 | compressor_pager_t ; |
1135 | |
1136 | compressor_pager_lookup(mem_obj, pager); |
1137 | if (pager == NULL) { |
1138 | return 0; |
1139 | } |
1140 | |
1141 | /* |
1142 | * The caller should have the VM object locked and one |
1143 | * needs that lock to do a page-in or page-out, so no |
1144 | * need to lock the pager here. |
1145 | */ |
1146 | assert(pager->cpgr_num_slots_occupied >= 0); |
1147 | |
1148 | return pager->cpgr_num_slots_occupied; |
1149 | } |
1150 | |
1151 | void |
1152 | ( |
1153 | memory_object_t mem_obj, |
1154 | int compressed_count_delta, |
1155 | boolean_t shared_lock, |
1156 | vm_object_t object __unused) |
1157 | { |
1158 | compressor_pager_t ; |
1159 | |
1160 | if (compressed_count_delta == 0) { |
1161 | return; |
1162 | } |
1163 | |
1164 | compressor_pager_lookup(mem_obj, pager); |
1165 | if (pager == NULL) { |
1166 | return; |
1167 | } |
1168 | |
1169 | if (compressed_count_delta < 0) { |
1170 | assert(pager->cpgr_num_slots_occupied >= |
1171 | (unsigned int) -compressed_count_delta); |
1172 | } |
1173 | |
1174 | /* |
1175 | * The caller should have the VM object locked, |
1176 | * shared or exclusive. |
1177 | */ |
1178 | if (shared_lock) { |
1179 | vm_object_lock_assert_shared(object); |
1180 | OSAddAtomic(compressed_count_delta, |
1181 | &pager->cpgr_num_slots_occupied); |
1182 | } else { |
1183 | vm_object_lock_assert_exclusive(object); |
1184 | pager->cpgr_num_slots_occupied += compressed_count_delta; |
1185 | } |
1186 | } |
1187 | |
1188 | #if CONFIG_FREEZE |
1189 | kern_return_t |
1190 | vm_compressor_pager_relocate( |
1191 | memory_object_t mem_obj, |
1192 | memory_object_offset_t offset, |
1193 | void **current_chead) |
1194 | { |
1195 | /* |
1196 | * Has the page at this offset been compressed? |
1197 | */ |
1198 | |
1199 | compressor_slot_t *slot_p; |
1200 | compressor_pager_t dst_pager; |
1201 | |
1202 | assert(mem_obj); |
1203 | |
1204 | compressor_pager_lookup(mem_obj, dst_pager); |
1205 | if (dst_pager == NULL) { |
1206 | return KERN_FAILURE; |
1207 | } |
1208 | |
1209 | compressor_pager_slot_lookup(dst_pager, FALSE, offset, &slot_p); |
1210 | return vm_compressor_relocate(current_chead, slot_p); |
1211 | } |
1212 | #endif /* CONFIG_FREEZE */ |
1213 | |
1214 | #if DEVELOPMENT || DEBUG |
1215 | |
1216 | kern_return_t |
1217 | vm_compressor_pager_inject_error(memory_object_t mem_obj, |
1218 | memory_object_offset_t offset) |
1219 | { |
1220 | kern_return_t result = KERN_FAILURE; |
1221 | compressor_slot_t *slot_p; |
1222 | compressor_pager_t pager; |
1223 | |
1224 | assert(mem_obj); |
1225 | |
1226 | compressor_pager_lookup(mem_obj, pager); |
1227 | if (pager != NULL) { |
1228 | compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p); |
1229 | if (slot_p != NULL && *slot_p != 0) { |
1230 | vm_compressor_inject_error(slot_p); |
1231 | result = KERN_SUCCESS; |
1232 | } |
1233 | } |
1234 | |
1235 | return result; |
1236 | } |
1237 | |
1238 | #endif |
1239 | |