1/*
2 * Copyright (c) 2008-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#if !FS_COMPRESSION
29
30/* We need these symbols even though compression is turned off */
31
32#define UNUSED_SYMBOL(x) asm(".global _" #x "\n.set _" #x ", 0\n");
33
34UNUSED_SYMBOL(register_decmpfs_decompressor)
35UNUSED_SYMBOL(unregister_decmpfs_decompressor)
36UNUSED_SYMBOL(decmpfs_init)
37UNUSED_SYMBOL(decmpfs_read_compressed)
38UNUSED_SYMBOL(decmpfs_cnode_cmp_type)
39UNUSED_SYMBOL(decmpfs_cnode_get_vnode_state)
40UNUSED_SYMBOL(decmpfs_cnode_get_vnode_cached_size)
41UNUSED_SYMBOL(decmpfs_cnode_get_vnode_cached_nchildren)
42UNUSED_SYMBOL(decmpfs_cnode_get_vnode_cached_total_size)
43UNUSED_SYMBOL(decmpfs_lock_compressed_data)
44UNUSED_SYMBOL(decmpfs_cnode_free)
45UNUSED_SYMBOL(decmpfs_cnode_alloc)
46UNUSED_SYMBOL(decmpfs_cnode_destroy)
47UNUSED_SYMBOL(decmpfs_decompress_file)
48UNUSED_SYMBOL(decmpfs_unlock_compressed_data)
49UNUSED_SYMBOL(decmpfs_cnode_init)
50UNUSED_SYMBOL(decmpfs_cnode_set_vnode_state)
51UNUSED_SYMBOL(decmpfs_hides_xattr)
52UNUSED_SYMBOL(decmpfs_ctx)
53UNUSED_SYMBOL(decmpfs_file_is_compressed)
54UNUSED_SYMBOL(decmpfs_update_attributes)
55UNUSED_SYMBOL(decmpfs_hides_rsrc)
56UNUSED_SYMBOL(decmpfs_pagein_compressed)
57UNUSED_SYMBOL(decmpfs_validate_compressed_file)
58
59#else /* FS_COMPRESSION */
60#include <sys/kernel.h>
61#include <sys/vnode_internal.h>
62#include <sys/file_internal.h>
63#include <sys/stat.h>
64#include <sys/fcntl.h>
65#include <sys/xattr.h>
66#include <sys/namei.h>
67#include <sys/user.h>
68#include <sys/mount_internal.h>
69#include <sys/ubc.h>
70#include <sys/decmpfs.h>
71#include <sys/uio_internal.h>
72#include <libkern/OSByteOrder.h>
73#include <libkern/section_keywords.h>
74#include <sys/fsctl.h>
75
76#include <sys/kdebug_triage.h>
77
78#include <ptrauth.h>
79
80#pragma mark --- debugging ---
81
82#define COMPRESSION_DEBUG 0
83#define COMPRESSION_DEBUG_VERBOSE 0
84#define MALLOC_DEBUG 0
85
86#if COMPRESSION_DEBUG
87static char*
88vnpath(vnode_t vp, char *path, int len)
89{
90 int origlen = len;
91 path[0] = 0;
92 vn_getpath(vp, path, &len);
93 path[origlen - 1] = 0;
94 return path;
95}
96#endif
97
98#define ErrorLog(x, args...) \
99 printf("%s:%d:%s: " x, __FILE_NAME__, __LINE__, __FUNCTION__, ## args)
100#if COMPRESSION_DEBUG
101#define ErrorLogWithPath(x, args...) do { \
102 char *path = zalloc(ZV_NAMEI); \
103 printf("%s:%d:%s: %s: " x, __FILE_NAME__, __LINE__, __FUNCTION__, \
104 vnpath(vp, path, PATH_MAX), ## args); \
105 zfree(ZV_NAMEI, path); \
106} while(0)
107#else
108#define ErrorLogWithPath(x, args...) do { \
109 (void*)vp; \
110 printf("%s:%d:%s: %s: " x, __FILE_NAME__, __LINE__, __FUNCTION__, \
111 "<private>", ## args); \
112} while(0)
113#endif
114
115#if COMPRESSION_DEBUG
116#define DebugLog ErrorLog
117#define DebugLogWithPath ErrorLogWithPath
118#else
119#define DebugLog(x...) do { } while(0)
120#define DebugLogWithPath(x...) do { } while(0)
121#endif
122
123#if COMPRESSION_DEBUG_VERBOSE
124#define VerboseLog ErrorLog
125#define VerboseLogWithPath ErrorLogWithPath
126#else
127#define VerboseLog(x...) do { } while(0)
128#define VerboseLogWithPath(x...) do { } while(0)
129#endif
130
131#define decmpfs_ktriage_record(code, arg) ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_DECMPFS, KDBG_TRIAGE_RESERVED, code), arg);
132
133enum ktriage_decmpfs_error_codes {
134 KTRIAGE_DECMPFS_PREFIX = 0,
135 KTRIAGE_DECMPFS_IVALID_OFFSET,
136 KTRIAGE_DECMPFS_COMPRESSOR_NOT_REGISTERED,
137 KTRIAGE_DECMPFS_FETCH_CALLBACK_FAILED,
138 KTRIAGE_DECMPFS_FETCH_HEADER_FAILED,
139 KTRIAGE_DECMPFS_UBC_UPL_MAP_FAILED,
140 KTRIAGE_DECMPFS_FETCH_UNCOMPRESSED_DATA_FAILED,
141
142 KTRIAGE_DECMPFS_MAX
143};
144
145const char *ktriage_decmpfs_strings[] = {
146 [KTRIAGE_DECMPFS_PREFIX] = "decmpfs - ",
147 [KTRIAGE_DECMPFS_IVALID_OFFSET] = "pagein offset is invalid\n",
148 [KTRIAGE_DECMPFS_COMPRESSOR_NOT_REGISTERED] = "compressor is not registered\n",
149 [KTRIAGE_DECMPFS_FETCH_CALLBACK_FAILED] = "fetch callback failed\n",
150 [KTRIAGE_DECMPFS_FETCH_HEADER_FAILED] = "fetch decmpfs xattr failed\n",
151 [KTRIAGE_DECMPFS_UBC_UPL_MAP_FAILED] = "failed to map a UBC UPL\n",
152 [KTRIAGE_DECMPFS_FETCH_UNCOMPRESSED_DATA_FAILED] = "failed to fetch uncompressed data\n",
153};
154
155ktriage_strings_t ktriage_decmpfs_subsystem_strings = {KTRIAGE_DECMPFS_MAX, ktriage_decmpfs_strings};
156
157#pragma mark --- globals ---
158
159static LCK_GRP_DECLARE(decmpfs_lockgrp, "VFSCOMP");
160static LCK_RW_DECLARE(decompressorsLock, &decmpfs_lockgrp);
161static LCK_MTX_DECLARE(decompress_channel_mtx, &decmpfs_lockgrp);
162
163static const decmpfs_registration *decompressors[CMP_MAX]; /* the registered compressors */
164static int decompress_channel; /* channel used by decompress_file to wake up waiters */
165
166vfs_context_t decmpfs_ctx;
167
168#pragma mark --- decmp_get_func ---
169
170#define offsetof_func(func) ((uintptr_t)offsetof(decmpfs_registration, func))
171
172static void *
173_func_from_offset(uint32_t type, uintptr_t offset, uint32_t discriminator)
174{
175 /* get the function at the given offset in the registration for the given type */
176 const decmpfs_registration *reg = decompressors[type];
177
178 switch (reg->decmpfs_registration) {
179 case DECMPFS_REGISTRATION_VERSION_V1:
180 if (offset > offsetof_func(free_data)) {
181 return NULL;
182 }
183 break;
184 case DECMPFS_REGISTRATION_VERSION_V3:
185 if (offset > offsetof_func(get_flags)) {
186 return NULL;
187 }
188 break;
189 default:
190 return NULL;
191 }
192
193 void *ptr = *(void * const *)((uintptr_t)reg + offset);
194 if (ptr != NULL) {
195 /* Resign as a function-in-void* */
196 ptr = ptrauth_auth_and_resign(ptr, ptrauth_key_asia, discriminator, ptrauth_key_asia, 0);
197 }
198 return ptr;
199}
200
201extern void IOServicePublishResource( const char * property, boolean_t value );
202extern boolean_t IOServiceWaitForMatchingResource( const char * property, uint64_t timeout );
203extern boolean_t IOCatalogueMatchingDriversPresent( const char * property );
204
205static void *
206_decmp_get_func(vnode_t vp, uint32_t type, uintptr_t offset, uint32_t discriminator)
207{
208 /*
209 * this function should be called while holding a shared lock to decompressorsLock,
210 * and will return with the lock held
211 */
212
213 if (type >= CMP_MAX) {
214 return NULL;
215 }
216
217 if (decompressors[type] != NULL) {
218 // the compressor has already registered but the function might be null
219 return _func_from_offset(type, offset, discriminator);
220 }
221
222 // does IOKit know about a kext that is supposed to provide this type?
223 char providesName[80];
224 snprintf(providesName, count: sizeof(providesName), "com.apple.AppleFSCompression.providesType%u", type);
225 if (IOCatalogueMatchingDriversPresent(property: providesName)) {
226 // there is a kext that says it will register for this type, so let's wait for it
227 char resourceName[80];
228 uint64_t delay = 10000000ULL; // 10 milliseconds.
229 snprintf(resourceName, count: sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", type);
230 ErrorLogWithPath("waiting for %s\n", resourceName);
231 while (decompressors[type] == NULL) {
232 lck_rw_unlock_shared(lck: &decompressorsLock); // we have to unlock to allow the kext to register
233 if (IOServiceWaitForMatchingResource(property: resourceName, timeout: delay)) {
234 lck_rw_lock_shared(lck: &decompressorsLock);
235 break;
236 }
237 if (!IOCatalogueMatchingDriversPresent(property: providesName)) {
238 //
239 ErrorLogWithPath("the kext with %s is no longer present\n", providesName);
240 lck_rw_lock_shared(lck: &decompressorsLock);
241 break;
242 }
243 ErrorLogWithPath("still waiting for %s\n", resourceName);
244 delay *= 2;
245 lck_rw_lock_shared(lck: &decompressorsLock);
246 }
247 // IOKit says the kext is loaded, so it should be registered too!
248 if (decompressors[type] == NULL) {
249 ErrorLogWithPath("we found %s, but the type still isn't registered\n", providesName);
250 return NULL;
251 }
252 // it's now registered, so let's return the function
253 return _func_from_offset(type, offset, discriminator);
254 }
255
256 // the compressor hasn't registered, so it never will unless someone manually kextloads it
257 ErrorLogWithPath("tried to access a compressed file of unregistered type %d\n", type);
258 return NULL;
259}
260
261#define decmp_get_func(vp, type, func) (typeof(decompressors[0]->func))_decmp_get_func(vp, type, offsetof_func(func), ptrauth_function_pointer_type_discriminator(typeof(decompressors[0]->func)))
262
263#pragma mark --- utilities ---
264
265#if COMPRESSION_DEBUG
266static int
267vnsize(vnode_t vp, uint64_t *size)
268{
269 struct vnode_attr va;
270 VATTR_INIT(&va);
271 VATTR_WANTED(&va, va_data_size);
272 int error = vnode_getattr(vp, &va, decmpfs_ctx);
273 if (error != 0) {
274 ErrorLogWithPath("vnode_getattr err %d\n", error);
275 return error;
276 }
277 *size = va.va_data_size;
278 return 0;
279}
280#endif /* COMPRESSION_DEBUG */
281
282#pragma mark --- cnode routines ---
283
284ZONE_DEFINE(decmpfs_cnode_zone, "decmpfs_cnode",
285 sizeof(struct decmpfs_cnode), ZC_NONE);
286
287decmpfs_cnode *
288decmpfs_cnode_alloc(void)
289{
290 return zalloc(zone: decmpfs_cnode_zone);
291}
292
293void
294decmpfs_cnode_free(decmpfs_cnode *dp)
295{
296 zfree(decmpfs_cnode_zone, dp);
297}
298
299void
300decmpfs_cnode_init(decmpfs_cnode *cp)
301{
302 memset(s: cp, c: 0, n: sizeof(*cp));
303 lck_rw_init(lck: &cp->compressed_data_lock, grp: &decmpfs_lockgrp, NULL);
304}
305
306void
307decmpfs_cnode_destroy(decmpfs_cnode *cp)
308{
309 lck_rw_destroy(lck: &cp->compressed_data_lock, grp: &decmpfs_lockgrp);
310}
311
312bool
313decmpfs_trylock_compressed_data(decmpfs_cnode *cp, int exclusive)
314{
315 void *thread = current_thread();
316 bool retval = false;
317
318 if (cp->lockowner == thread) {
319 /* this thread is already holding an exclusive lock, so bump the count */
320 cp->lockcount++;
321 retval = true;
322 } else if (exclusive) {
323 if ((retval = lck_rw_try_lock_exclusive(lck: &cp->compressed_data_lock))) {
324 cp->lockowner = thread;
325 cp->lockcount = 1;
326 }
327 } else {
328 if ((retval = lck_rw_try_lock_shared(lck: &cp->compressed_data_lock))) {
329 cp->lockowner = (void *)-1;
330 }
331 }
332 return retval;
333}
334
335void
336decmpfs_lock_compressed_data(decmpfs_cnode *cp, int exclusive)
337{
338 void *thread = current_thread();
339
340 if (cp->lockowner == thread) {
341 /* this thread is already holding an exclusive lock, so bump the count */
342 cp->lockcount++;
343 } else if (exclusive) {
344 lck_rw_lock_exclusive(lck: &cp->compressed_data_lock);
345 cp->lockowner = thread;
346 cp->lockcount = 1;
347 } else {
348 lck_rw_lock_shared(lck: &cp->compressed_data_lock);
349 cp->lockowner = (void *)-1;
350 }
351}
352
353void
354decmpfs_unlock_compressed_data(decmpfs_cnode *cp, __unused int exclusive)
355{
356 void *thread = current_thread();
357
358 if (cp->lockowner == thread) {
359 /* this thread is holding an exclusive lock, so decrement the count */
360 if ((--cp->lockcount) > 0) {
361 /* the caller still has outstanding locks, so we're done */
362 return;
363 }
364 cp->lockowner = NULL;
365 }
366
367 lck_rw_done(lck: &cp->compressed_data_lock);
368}
369
370uint32_t
371decmpfs_cnode_get_vnode_state(decmpfs_cnode *cp)
372{
373 return cp->cmp_state;
374}
375
376void
377decmpfs_cnode_set_vnode_state(decmpfs_cnode *cp, uint32_t state, int skiplock)
378{
379 if (!skiplock) {
380 decmpfs_lock_compressed_data(cp, exclusive: 1);
381 }
382 cp->cmp_state = (uint8_t)state;
383 if (state == FILE_TYPE_UNKNOWN) {
384 /* clear out the compression type too */
385 cp->cmp_type = 0;
386 }
387 if (!skiplock) {
388 decmpfs_unlock_compressed_data(cp, exclusive: 1);
389 }
390}
391
392static void
393decmpfs_cnode_set_vnode_cmp_type(decmpfs_cnode *cp, uint32_t cmp_type, int skiplock)
394{
395 if (!skiplock) {
396 decmpfs_lock_compressed_data(cp, exclusive: 1);
397 }
398 cp->cmp_type = cmp_type;
399 if (!skiplock) {
400 decmpfs_unlock_compressed_data(cp, exclusive: 1);
401 }
402}
403
404static void
405decmpfs_cnode_set_vnode_minimal_xattr(decmpfs_cnode *cp, int minimal_xattr, int skiplock)
406{
407 if (!skiplock) {
408 decmpfs_lock_compressed_data(cp, exclusive: 1);
409 }
410 cp->cmp_minimal_xattr = !!minimal_xattr;
411 if (!skiplock) {
412 decmpfs_unlock_compressed_data(cp, exclusive: 1);
413 }
414}
415
416uint64_t
417decmpfs_cnode_get_vnode_cached_size(decmpfs_cnode *cp)
418{
419 return cp->uncompressed_size;
420}
421
422uint64_t
423decmpfs_cnode_get_vnode_cached_nchildren(decmpfs_cnode *cp)
424{
425 return cp->nchildren;
426}
427
428uint64_t
429decmpfs_cnode_get_vnode_cached_total_size(decmpfs_cnode *cp)
430{
431 return cp->total_size;
432}
433
434void
435decmpfs_cnode_set_vnode_cached_size(decmpfs_cnode *cp, uint64_t size)
436{
437 while (1) {
438 uint64_t old = cp->uncompressed_size;
439 if (OSCompareAndSwap64(old, size, (UInt64*)&cp->uncompressed_size)) {
440 return;
441 } else {
442 /* failed to write our value, so loop */
443 }
444 }
445}
446
447void
448decmpfs_cnode_set_vnode_cached_nchildren(decmpfs_cnode *cp, uint64_t nchildren)
449{
450 while (1) {
451 uint64_t old = cp->nchildren;
452 if (OSCompareAndSwap64(old, nchildren, (UInt64*)&cp->nchildren)) {
453 return;
454 } else {
455 /* failed to write our value, so loop */
456 }
457 }
458}
459
460void
461decmpfs_cnode_set_vnode_cached_total_size(decmpfs_cnode *cp, uint64_t total_sz)
462{
463 while (1) {
464 uint64_t old = cp->total_size;
465 if (OSCompareAndSwap64(old, total_sz, (UInt64*)&cp->total_size)) {
466 return;
467 } else {
468 /* failed to write our value, so loop */
469 }
470 }
471}
472
473static uint64_t
474decmpfs_cnode_get_decompression_flags(decmpfs_cnode *cp)
475{
476 return cp->decompression_flags;
477}
478
479static void
480decmpfs_cnode_set_decompression_flags(decmpfs_cnode *cp, uint64_t flags)
481{
482 while (1) {
483 uint64_t old = cp->decompression_flags;
484 if (OSCompareAndSwap64(old, flags, (UInt64*)&cp->decompression_flags)) {
485 return;
486 } else {
487 /* failed to write our value, so loop */
488 }
489 }
490}
491
492uint32_t
493decmpfs_cnode_cmp_type(decmpfs_cnode *cp)
494{
495 return cp->cmp_type;
496}
497
498#pragma mark --- decmpfs state routines ---
499
500static int
501decmpfs_fetch_compressed_header(vnode_t vp, decmpfs_cnode *cp, decmpfs_header **hdrOut, int returnInvalid, size_t *hdr_size)
502{
503 /*
504 * fetches vp's compression xattr, converting it into a decmpfs_header; returns 0 or errno
505 * if returnInvalid == 1, returns the header even if the type was invalid (out of range),
506 * and return ERANGE in that case
507 */
508
509 size_t read_size = 0;
510 size_t attr_size = 0;
511 size_t alloc_size = 0;
512 uio_t attr_uio = NULL;
513 int err = 0;
514 char *data = NULL;
515 const bool no_additional_data = ((cp != NULL)
516 && (cp->cmp_type != 0)
517 && (cp->cmp_minimal_xattr != 0));
518 UIO_STACKBUF(uio_buf, 1);
519 decmpfs_header *hdr = NULL;
520
521 /*
522 * Trace the following parameters on entry with event-id 0x03120004
523 *
524 * @vp->v_id: vnode-id for which to fetch compressed header.
525 * @no_additional_data: If set true then xattr didn't have any extra data.
526 * @returnInvalid: return the header even though the type is out of range.
527 */
528 DECMPFS_EMIT_TRACE_ENTRY(DECMPDBG_FETCH_COMPRESSED_HEADER, vp->v_id,
529 no_additional_data, returnInvalid);
530
531 if (no_additional_data) {
532 /* this file's xattr didn't have any extra data when we fetched it, so we can synthesize a header from the data in the cnode */
533
534 alloc_size = sizeof(decmpfs_header);
535 data = kalloc_data(sizeof(decmpfs_header), Z_WAITOK);
536 if (!data) {
537 err = ENOMEM;
538 goto out;
539 }
540 hdr = (decmpfs_header*)data;
541 hdr->attr_size = sizeof(decmpfs_disk_header);
542 hdr->compression_magic = DECMPFS_MAGIC;
543 hdr->compression_type = cp->cmp_type;
544 if (hdr->compression_type == DATALESS_PKG_CMPFS_TYPE) {
545 if (!vnode_isdir(vp)) {
546 err = EINVAL;
547 goto out;
548 }
549 hdr->_size.value = DECMPFS_PKG_VALUE_FROM_SIZE_COUNT(
550 decmpfs_cnode_get_vnode_cached_size(cp),
551 decmpfs_cnode_get_vnode_cached_nchildren(cp));
552 } else if (vnode_isdir(vp)) {
553 hdr->_size.value = decmpfs_cnode_get_vnode_cached_nchildren(cp);
554 } else {
555 hdr->_size.value = decmpfs_cnode_get_vnode_cached_size(cp);
556 }
557 } else {
558 /* figure out how big the xattr is on disk */
559 err = vn_getxattr(vp, DECMPFS_XATTR_NAME, NULL, &attr_size, XATTR_NOSECURITY, decmpfs_ctx);
560 if (err != 0) {
561 goto out;
562 }
563 alloc_size = attr_size + sizeof(hdr->attr_size);
564
565 if (attr_size < sizeof(decmpfs_disk_header) || attr_size > MAX_DECMPFS_XATTR_SIZE) {
566 err = EINVAL;
567 goto out;
568 }
569
570 /* allocation includes space for the extra attr_size field of a compressed_header */
571 data = (char *)kalloc_data(alloc_size, Z_WAITOK);
572 if (!data) {
573 err = ENOMEM;
574 goto out;
575 }
576
577 /* read the xattr into our buffer, skipping over the attr_size field at the beginning */
578 attr_uio = uio_createwithbuffer(a_iovcount: 1, a_offset: 0, a_spacetype: UIO_SYSSPACE, a_iodirection: UIO_READ, a_buf_p: &uio_buf[0], a_buffer_size: sizeof(uio_buf));
579 uio_addiov(a_uio: attr_uio, CAST_USER_ADDR_T(data + sizeof(hdr->attr_size)), a_length: attr_size);
580
581 err = vn_getxattr(vp, DECMPFS_XATTR_NAME, attr_uio, &read_size, XATTR_NOSECURITY, decmpfs_ctx);
582 if (err != 0) {
583 goto out;
584 }
585 if (read_size != attr_size) {
586 err = EINVAL;
587 goto out;
588 }
589 hdr = (decmpfs_header*)data;
590 hdr->attr_size = (uint32_t)attr_size;
591 /* swap the fields to native endian */
592 hdr->compression_magic = OSSwapLittleToHostInt32(hdr->compression_magic);
593 hdr->compression_type = OSSwapLittleToHostInt32(hdr->compression_type);
594 hdr->uncompressed_size = OSSwapLittleToHostInt64(hdr->uncompressed_size);
595 }
596
597 if (hdr->compression_magic != DECMPFS_MAGIC) {
598 ErrorLogWithPath("invalid compression_magic 0x%08x, should be 0x%08x\n", hdr->compression_magic, DECMPFS_MAGIC);
599 err = EINVAL;
600 goto out;
601 }
602
603 /*
604 * Special-case the DATALESS compressor here; that is a valid type,
605 * even through there will never be an entry in the decompressor
606 * handler table for it. If we don't do this, then the cmp_state
607 * for this cnode will end up being marked NOT_COMPRESSED, and
608 * we'll be stuck in limbo.
609 */
610 if (hdr->compression_type >= CMP_MAX && !decmpfs_type_is_dataless(cmp_type: hdr->compression_type)) {
611 if (returnInvalid) {
612 /* return the header even though the type is out of range */
613 err = ERANGE;
614 } else {
615 ErrorLogWithPath("compression_type %d out of range\n", hdr->compression_type);
616 err = EINVAL;
617 }
618 goto out;
619 }
620
621out:
622 if (err && (err != ERANGE)) {
623 DebugLogWithPath("err %d\n", err);
624 kfree_data(data, alloc_size);
625 *hdrOut = NULL;
626 } else {
627 *hdrOut = hdr;
628 *hdr_size = alloc_size;
629 }
630 /*
631 * Trace the following parameters on return with event-id 0x03120004.
632 *
633 * @vp->v_id: vnode-id for which to fetch compressed header.
634 * @err: value returned from this function.
635 */
636 DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FETCH_COMPRESSED_HEADER, vp->v_id, err);
637 return err;
638}
639
640static int
641decmpfs_fast_get_state(decmpfs_cnode *cp)
642{
643 /*
644 * return the cached state
645 * this should *only* be called when we know that decmpfs_file_is_compressed has already been called,
646 * because this implies that the cached state is valid
647 */
648 int cmp_state = decmpfs_cnode_get_vnode_state(cp);
649
650 switch (cmp_state) {
651 case FILE_IS_NOT_COMPRESSED:
652 case FILE_IS_COMPRESSED:
653 case FILE_IS_CONVERTING:
654 return cmp_state;
655 case FILE_TYPE_UNKNOWN:
656 /*
657 * we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode,
658 * which should not be possible
659 */
660 ErrorLog("decmpfs_fast_get_state called on unknown file\n");
661 return FILE_IS_NOT_COMPRESSED;
662 default:
663 /* */
664 ErrorLog("unknown cmp_state %d\n", cmp_state);
665 return FILE_IS_NOT_COMPRESSED;
666 }
667}
668
669static int
670decmpfs_fast_file_is_compressed(decmpfs_cnode *cp)
671{
672 int cmp_state = decmpfs_cnode_get_vnode_state(cp);
673
674 switch (cmp_state) {
675 case FILE_IS_NOT_COMPRESSED:
676 return 0;
677 case FILE_IS_COMPRESSED:
678 case FILE_IS_CONVERTING:
679 return 1;
680 case FILE_TYPE_UNKNOWN:
681 /*
682 * we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode,
683 * which should not be possible
684 */
685 ErrorLog("decmpfs_fast_get_state called on unknown file\n");
686 return 0;
687 default:
688 /* */
689 ErrorLog("unknown cmp_state %d\n", cmp_state);
690 return 0;
691 }
692}
693
694errno_t
695decmpfs_validate_compressed_file(vnode_t vp, decmpfs_cnode *cp)
696{
697 /* give a compressor a chance to indicate that a compressed file is invalid */
698 decmpfs_header *hdr = NULL;
699 size_t alloc_size = 0;
700 errno_t err = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 0, hdr_size: &alloc_size);
701
702 if (err) {
703 /* we couldn't get the header */
704 if (decmpfs_fast_get_state(cp) == FILE_IS_NOT_COMPRESSED) {
705 /* the file is no longer compressed, so return success */
706 err = 0;
707 }
708 goto out;
709 }
710
711 if (!decmpfs_type_is_dataless(cmp_type: hdr->compression_type)) {
712 lck_rw_lock_shared(lck: &decompressorsLock);
713 decmpfs_validate_compressed_file_func validate = decmp_get_func(vp, hdr->compression_type, validate);
714 if (validate) { /* make sure this validation function is valid */
715 /* is the data okay? */
716 err = validate(vp, decmpfs_ctx, hdr);
717 } else if (decmp_get_func(vp, hdr->compression_type, fetch) == NULL) {
718 /* the type isn't registered */
719 err = EIO;
720 } else {
721 /* no validate registered, so nothing to do */
722 err = 0;
723 }
724 lck_rw_unlock_shared(lck: &decompressorsLock);
725 }
726out:
727 if (hdr != NULL) {
728 kfree_data(hdr, alloc_size);
729 }
730#if COMPRESSION_DEBUG
731 if (err) {
732 DebugLogWithPath("decmpfs_validate_compressed_file ret %d, vp->v_flag %d\n", err, vp->v_flag);
733 }
734#endif
735 return err;
736}
737
738int
739decmpfs_file_is_compressed(vnode_t vp, decmpfs_cnode *cp)
740{
741 /*
742 * determines whether vp points to a compressed file
743 *
744 * to speed up this operation, we cache the result in the cnode, and do as little as possible
745 * in the case where the cnode already has a valid cached state
746 *
747 */
748
749 int ret = 0;
750 int error = 0;
751 uint32_t cmp_state;
752 struct vnode_attr va_fetch;
753 decmpfs_header *hdr = NULL;
754 size_t alloc_size = 0;
755 mount_t mp = NULL;
756 int cnode_locked = 0;
757 int saveInvalid = 0; // save the header data even though the type was out of range
758 uint64_t decompression_flags = 0;
759 bool is_mounted, is_local_fs;
760
761 if (vnode_isnamedstream(vp)) {
762 /*
763 * named streams can't be compressed
764 * since named streams of the same file share the same cnode,
765 * we don't want to get/set the state in the cnode, just return 0
766 */
767 return 0;
768 }
769
770 /* examine the cached a state in this cnode */
771 cmp_state = decmpfs_cnode_get_vnode_state(cp);
772 switch (cmp_state) {
773 case FILE_IS_NOT_COMPRESSED:
774 return 0;
775 case FILE_IS_COMPRESSED:
776 return 1;
777 case FILE_IS_CONVERTING:
778 /* treat the file as compressed, because this gives us a way to block future reads until decompression is done */
779 return 1;
780 case FILE_TYPE_UNKNOWN:
781 /* the first time we encountered this vnode, so we need to check it out */
782 break;
783 default:
784 /* unknown state, assume file is not compressed */
785 ErrorLogWithPath("unknown cmp_state %d\n", cmp_state);
786 return 0;
787 }
788
789 is_mounted = false;
790 is_local_fs = false;
791 mp = vnode_mount(vp);
792 if (mp) {
793 is_mounted = true;
794 }
795 if (is_mounted) {
796 is_local_fs = ((mp->mnt_flag & MNT_LOCAL));
797 }
798 /*
799 * Trace the following parameters on entry with event-id 0x03120014.
800 *
801 * @vp->v_id: vnode-id of the file being queried.
802 * @is_mounted: set to true if @vp belongs to a mounted fs.
803 * @is_local_fs: set to true if @vp belongs to local fs.
804 */
805 DECMPFS_EMIT_TRACE_ENTRY(DECMPDBG_FILE_IS_COMPRESSED, vp->v_id,
806 is_mounted, is_local_fs);
807
808 if (!is_mounted) {
809 /*
810 * this should only be true before we mount the root filesystem
811 * we short-cut this return to avoid the call to getattr below, which
812 * will fail before root is mounted
813 */
814 ret = FILE_IS_NOT_COMPRESSED;
815 goto done;
816 }
817
818 if (!is_local_fs) {
819 /* compression only supported on local filesystems */
820 ret = FILE_IS_NOT_COMPRESSED;
821 goto done;
822 }
823
824 /* lock our cnode data so that another caller doesn't change the state under us */
825 decmpfs_lock_compressed_data(cp, exclusive: 1);
826 cnode_locked = 1;
827
828 VATTR_INIT(&va_fetch);
829 VATTR_WANTED(&va_fetch, va_flags);
830 error = vnode_getattr(vp, vap: &va_fetch, ctx: decmpfs_ctx);
831 if (error) {
832 /* failed to get the bsd flags so the file is not compressed */
833 ret = FILE_IS_NOT_COMPRESSED;
834 goto done;
835 }
836 if (va_fetch.va_flags & UF_COMPRESSED) {
837 /* UF_COMPRESSED is on, make sure the file has the DECMPFS_XATTR_NAME xattr */
838 error = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 1, hdr_size: &alloc_size);
839 if ((hdr != NULL) && (error == ERANGE)) {
840 saveInvalid = 1;
841 }
842 if (error) {
843 /* failed to get the xattr so the file is not compressed */
844 ret = FILE_IS_NOT_COMPRESSED;
845 goto done;
846 }
847 /*
848 * We got the xattr, so the file is at least tagged compressed.
849 * For DATALESS, regular files and directories can be "compressed".
850 * For all other types, only files are allowed.
851 */
852 if (!vnode_isreg(vp) &&
853 !(decmpfs_type_is_dataless(cmp_type: hdr->compression_type) && vnode_isdir(vp))) {
854 ret = FILE_IS_NOT_COMPRESSED;
855 goto done;
856 }
857 ret = FILE_IS_COMPRESSED;
858 goto done;
859 }
860 /* UF_COMPRESSED isn't on, so the file isn't compressed */
861 ret = FILE_IS_NOT_COMPRESSED;
862
863done:
864 if (((ret == FILE_IS_COMPRESSED) || saveInvalid) && hdr) {
865 /*
866 * cache the uncompressed size away in the cnode
867 */
868
869 if (!cnode_locked) {
870 /*
871 * we should never get here since the only place ret is set to FILE_IS_COMPRESSED
872 * is after the call to decmpfs_lock_compressed_data above
873 */
874 decmpfs_lock_compressed_data(cp, exclusive: 1);
875 cnode_locked = 1;
876 }
877
878 if (vnode_isdir(vp)) {
879 decmpfs_cnode_set_vnode_cached_size(cp, size: 64);
880 decmpfs_cnode_set_vnode_cached_nchildren(cp, nchildren: decmpfs_get_directory_entries(hdr));
881 if (hdr->compression_type == DATALESS_PKG_CMPFS_TYPE) {
882 decmpfs_cnode_set_vnode_cached_total_size(cp, DECMPFS_PKG_SIZE(hdr->_size));
883 }
884 } else {
885 decmpfs_cnode_set_vnode_cached_size(cp, size: hdr->uncompressed_size);
886 }
887 decmpfs_cnode_set_vnode_state(cp, state: ret, skiplock: 1);
888 decmpfs_cnode_set_vnode_cmp_type(cp, cmp_type: hdr->compression_type, skiplock: 1);
889 /* remember if the xattr's size was equal to the minimal xattr */
890 if (hdr->attr_size == sizeof(decmpfs_disk_header)) {
891 decmpfs_cnode_set_vnode_minimal_xattr(cp, minimal_xattr: 1, skiplock: 1);
892 }
893 if (ret == FILE_IS_COMPRESSED) {
894 /* update the ubc's size for this file */
895 ubc_setsize(vp, hdr->uncompressed_size);
896
897 /* update the decompression flags in the decmpfs cnode */
898 lck_rw_lock_shared(lck: &decompressorsLock);
899 decmpfs_get_decompression_flags_func get_flags = decmp_get_func(vp, hdr->compression_type, get_flags);
900 if (get_flags) {
901 decompression_flags = get_flags(vp, decmpfs_ctx, hdr);
902 }
903 lck_rw_unlock_shared(lck: &decompressorsLock);
904 decmpfs_cnode_set_decompression_flags(cp, flags: decompression_flags);
905 }
906 } else {
907 /* we might have already taken the lock above; if so, skip taking it again by passing cnode_locked as the skiplock parameter */
908 decmpfs_cnode_set_vnode_state(cp, state: ret, skiplock: cnode_locked);
909 }
910
911 if (cnode_locked) {
912 decmpfs_unlock_compressed_data(cp, exclusive: 1);
913 }
914
915 if (hdr != NULL) {
916 kfree_data(hdr, alloc_size);
917 }
918
919 /*
920 * Trace the following parameters on return with event-id 0x03120014.
921 *
922 * @vp->v_id: vnode-id of the file being queried.
923 * @return: set to 1 is file is compressed.
924 */
925 switch (ret) {
926 case FILE_IS_NOT_COMPRESSED:
927 DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FILE_IS_COMPRESSED, vp->v_id, 0);
928 return 0;
929 case FILE_IS_COMPRESSED:
930 case FILE_IS_CONVERTING:
931 DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FILE_IS_COMPRESSED, vp->v_id, 1);
932 return 1;
933 default:
934 /* unknown state, assume file is not compressed */
935 DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FILE_IS_COMPRESSED, vp->v_id, 0);
936 ErrorLogWithPath("unknown ret %d\n", ret);
937 return 0;
938 }
939}
940
941int
942decmpfs_update_attributes(vnode_t vp, struct vnode_attr *vap)
943{
944 int error = 0;
945
946 if (VATTR_IS_ACTIVE(vap, va_flags)) {
947 /* the BSD flags are being updated */
948 if (vap->va_flags & UF_COMPRESSED) {
949 /* the compressed bit is being set, did it change? */
950 struct vnode_attr va_fetch;
951 int old_flags = 0;
952 VATTR_INIT(&va_fetch);
953 VATTR_WANTED(&va_fetch, va_flags);
954 error = vnode_getattr(vp, vap: &va_fetch, ctx: decmpfs_ctx);
955 if (error) {
956 return error;
957 }
958
959 old_flags = va_fetch.va_flags;
960
961 if (!(old_flags & UF_COMPRESSED)) {
962 /*
963 * Compression bit was turned on, make sure the file has the DECMPFS_XATTR_NAME attribute.
964 * This precludes anyone from using the UF_COMPRESSED bit for anything else, and it enforces
965 * an order of operation -- you must first do the setxattr and then the chflags.
966 */
967
968 if (VATTR_IS_ACTIVE(vap, va_data_size)) {
969 /*
970 * don't allow the caller to set the BSD flag and the size in the same call
971 * since this doesn't really make sense
972 */
973 vap->va_flags &= ~UF_COMPRESSED;
974 return 0;
975 }
976
977 decmpfs_header *hdr = NULL;
978 size_t alloc_size = 0;
979 error = decmpfs_fetch_compressed_header(vp, NULL, hdrOut: &hdr, returnInvalid: 1, hdr_size: &alloc_size);
980 if (error == 0) {
981 /*
982 * Allow the flag to be set since the decmpfs attribute
983 * is present.
984 *
985 * If we're creating a dataless file we do not want to
986 * truncate it to zero which allows the file resolver to
987 * have more control over when truncation should happen.
988 * All other types of compressed files are truncated to
989 * zero.
990 */
991 if (!decmpfs_type_is_dataless(cmp_type: hdr->compression_type)) {
992 VATTR_SET_ACTIVE(vap, va_data_size);
993 vap->va_data_size = 0;
994 }
995 } else if (error == ERANGE) {
996 /* the file had a decmpfs attribute but the type was out of range, so don't muck with the file's data size */
997 } else {
998 /* no DECMPFS_XATTR_NAME attribute, so deny the update */
999 vap->va_flags &= ~UF_COMPRESSED;
1000 }
1001 if (hdr != NULL) {
1002 kfree_data(hdr, alloc_size);
1003 }
1004 }
1005 }
1006 }
1007
1008 return 0;
1009}
1010
1011static int
1012wait_for_decompress(decmpfs_cnode *cp)
1013{
1014 int state;
1015 lck_mtx_lock(lck: &decompress_channel_mtx);
1016 do {
1017 state = decmpfs_fast_get_state(cp);
1018 if (state != FILE_IS_CONVERTING) {
1019 /* file is not decompressing */
1020 lck_mtx_unlock(lck: &decompress_channel_mtx);
1021 return state;
1022 }
1023 msleep(chan: (caddr_t)&decompress_channel, mtx: &decompress_channel_mtx, PINOD, wmesg: "wait_for_decompress", NULL);
1024 } while (1);
1025}
1026
1027#pragma mark --- decmpfs hide query routines ---
1028
1029int
1030decmpfs_hides_rsrc(vfs_context_t ctx, decmpfs_cnode *cp)
1031{
1032 /*
1033 * WARNING!!!
1034 * callers may (and do) pass NULL for ctx, so we should only use it
1035 * for this equality comparison
1036 *
1037 * This routine should only be called after a file has already been through decmpfs_file_is_compressed
1038 */
1039
1040 if (ctx == decmpfs_ctx) {
1041 return 0;
1042 }
1043
1044 if (!decmpfs_fast_file_is_compressed(cp)) {
1045 return 0;
1046 }
1047
1048 /* all compressed files hide their resource fork */
1049 return 1;
1050}
1051
1052int
1053decmpfs_hides_xattr(vfs_context_t ctx, decmpfs_cnode *cp, const char *xattr)
1054{
1055 /*
1056 * WARNING!!!
1057 * callers may (and do) pass NULL for ctx, so we should only use it
1058 * for this equality comparison
1059 *
1060 * This routine should only be called after a file has already been through decmpfs_file_is_compressed
1061 */
1062
1063 if (ctx == decmpfs_ctx) {
1064 return 0;
1065 }
1066 if (strncmp(s1: xattr, XATTR_RESOURCEFORK_NAME, n: sizeof(XATTR_RESOURCEFORK_NAME) - 1) == 0) {
1067 return decmpfs_hides_rsrc(ctx, cp);
1068 }
1069 if (!decmpfs_fast_file_is_compressed(cp)) {
1070 /* file is not compressed, so don't hide this xattr */
1071 return 0;
1072 }
1073 if (strncmp(s1: xattr, DECMPFS_XATTR_NAME, n: sizeof(DECMPFS_XATTR_NAME) - 1) == 0) {
1074 /* it's our xattr, so hide it */
1075 return 1;
1076 }
1077 /* don't hide this xattr */
1078 return 0;
1079}
1080
1081#pragma mark --- registration/validation routines ---
1082
1083static inline int
1084registration_valid(const decmpfs_registration *registration)
1085{
1086 return registration && ((registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V1) || (registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V3));
1087}
1088
1089errno_t
1090register_decmpfs_decompressor(uint32_t compression_type, const decmpfs_registration *registration)
1091{
1092 /* called by kexts to register decompressors */
1093
1094 errno_t ret = 0;
1095 int locked = 0;
1096 char resourceName[80];
1097
1098 if ((compression_type >= CMP_MAX) || !registration_valid(registration)) {
1099 ret = EINVAL;
1100 goto out;
1101 }
1102
1103 lck_rw_lock_exclusive(lck: &decompressorsLock); locked = 1;
1104
1105 /* make sure the registration for this type is zero */
1106 if (decompressors[compression_type] != NULL) {
1107 ret = EEXIST;
1108 goto out;
1109 }
1110 decompressors[compression_type] = registration;
1111 snprintf(resourceName, count: sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", compression_type);
1112 IOServicePublishResource(property: resourceName, TRUE);
1113
1114out:
1115 if (locked) {
1116 lck_rw_unlock_exclusive(lck: &decompressorsLock);
1117 }
1118 return ret;
1119}
1120
1121errno_t
1122unregister_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration)
1123{
1124 /* called by kexts to unregister decompressors */
1125
1126 errno_t ret = 0;
1127 int locked = 0;
1128 char resourceName[80];
1129
1130 if ((compression_type >= CMP_MAX) || !registration_valid(registration)) {
1131 ret = EINVAL;
1132 goto out;
1133 }
1134
1135 lck_rw_lock_exclusive(lck: &decompressorsLock); locked = 1;
1136 if (decompressors[compression_type] != registration) {
1137 ret = EEXIST;
1138 goto out;
1139 }
1140 decompressors[compression_type] = NULL;
1141 snprintf(resourceName, count: sizeof(resourceName), "com.apple.AppleFSCompression.Type%u", compression_type);
1142 IOServicePublishResource(property: resourceName, FALSE);
1143
1144out:
1145 if (locked) {
1146 lck_rw_unlock_exclusive(lck: &decompressorsLock);
1147 }
1148 return ret;
1149}
1150
1151static int
1152compression_type_valid(vnode_t vp, decmpfs_header *hdr)
1153{
1154 /* fast pre-check to determine if the given compressor has checked in */
1155 int ret = 0;
1156
1157 /* every compressor must have at least a fetch function */
1158 lck_rw_lock_shared(lck: &decompressorsLock);
1159 if (decmp_get_func(vp, hdr->compression_type, fetch) != NULL) {
1160 ret = 1;
1161 }
1162 lck_rw_unlock_shared(lck: &decompressorsLock);
1163
1164 return ret;
1165}
1166
1167#pragma mark --- compression/decompression routines ---
1168
1169static int
1170decmpfs_fetch_uncompressed_data(vnode_t vp, decmpfs_cnode *cp, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read)
1171{
1172 /* get the uncompressed bytes for the specified region of vp by calling out to the registered compressor */
1173
1174 int err = 0;
1175
1176 *bytes_read = 0;
1177
1178 if (offset >= (off_t)hdr->uncompressed_size) {
1179 /* reading past end of file; nothing to do */
1180 err = 0;
1181 goto out;
1182 }
1183 if (offset < 0) {
1184 /* tried to read from before start of file */
1185 err = EINVAL;
1186 goto out;
1187 }
1188 if (hdr->uncompressed_size - offset < size) {
1189 /* adjust size so we don't read past the end of the file */
1190 size = (user_ssize_t)(hdr->uncompressed_size - offset);
1191 }
1192 if (size == 0) {
1193 /* nothing to read */
1194 err = 0;
1195 goto out;
1196 }
1197
1198 /*
1199 * Trace the following parameters on entry with event-id 0x03120008.
1200 *
1201 * @vp->v_id: vnode-id of the file being decompressed.
1202 * @hdr->compression_type: compression type.
1203 * @offset: offset from where to fetch uncompressed data.
1204 * @size: amount of uncompressed data to fetch.
1205 *
1206 * Please NOTE: @offset and @size can overflow in theory but
1207 * here it is safe.
1208 */
1209 DECMPFS_EMIT_TRACE_ENTRY(DECMPDBG_FETCH_UNCOMPRESSED_DATA, vp->v_id,
1210 hdr->compression_type, (int)offset, (int)size);
1211 lck_rw_lock_shared(lck: &decompressorsLock);
1212 decmpfs_fetch_uncompressed_data_func fetch = decmp_get_func(vp, hdr->compression_type, fetch);
1213 if (fetch) {
1214 err = fetch(vp, decmpfs_ctx, hdr, offset, size, nvec, vec, bytes_read);
1215 lck_rw_unlock_shared(lck: &decompressorsLock);
1216 if (err == 0) {
1217 uint64_t decompression_flags = decmpfs_cnode_get_decompression_flags(cp);
1218 if (decompression_flags & DECMPFS_FLAGS_FORCE_FLUSH_ON_DECOMPRESS) {
1219#if !defined(__i386__) && !defined(__x86_64__)
1220 int i;
1221 for (i = 0; i < nvec; i++) {
1222 assert(vec[i].size >= 0 && vec[i].size <= UINT_MAX);
1223 flush_dcache64((addr64_t)(uintptr_t)vec[i].buf, (unsigned int)vec[i].size, FALSE);
1224 }
1225#endif
1226 }
1227 } else {
1228 decmpfs_ktriage_record(KTRIAGE_DECMPFS_FETCH_CALLBACK_FAILED, err);
1229 }
1230 } else {
1231 decmpfs_ktriage_record(KTRIAGE_DECMPFS_COMPRESSOR_NOT_REGISTERED, hdr->compression_type);
1232 err = ENOTSUP;
1233 lck_rw_unlock_shared(lck: &decompressorsLock);
1234 }
1235 /*
1236 * Trace the following parameters on return with event-id 0x03120008.
1237 *
1238 * @vp->v_id: vnode-id of the file being decompressed.
1239 * @bytes_read: amount of uncompressed bytes fetched in bytes.
1240 * @err: value returned from this function.
1241 *
1242 * Please NOTE: @bytes_read can overflow in theory but here it is safe.
1243 */
1244 DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FETCH_UNCOMPRESSED_DATA, vp->v_id,
1245 (int)*bytes_read, err);
1246out:
1247 return err;
1248}
1249
1250static kern_return_t
1251commit_upl(upl_t upl, upl_offset_t pl_offset, size_t uplSize, int flags, int abort)
1252{
1253 kern_return_t kr = 0;
1254
1255#if CONFIG_IOSCHED
1256 upl_unmark_decmp(upl);
1257#endif /* CONFIG_IOSCHED */
1258
1259 /* commit the upl pages */
1260 if (abort) {
1261 VerboseLog("aborting upl, flags 0x%08x\n", flags);
1262 kr = ubc_upl_abort_range(upl, pl_offset, (upl_size_t)uplSize, flags);
1263 if (kr != KERN_SUCCESS) {
1264 ErrorLog("ubc_upl_abort_range error %d\n", (int)kr);
1265 }
1266 } else {
1267 VerboseLog("committing upl, flags 0x%08x\n", flags | UPL_COMMIT_CLEAR_DIRTY);
1268 kr = ubc_upl_commit_range(upl, pl_offset, (upl_size_t)uplSize, flags | UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_WRITTEN_BY_KERNEL);
1269 if (kr != KERN_SUCCESS) {
1270 ErrorLog("ubc_upl_commit_range error %d\n", (int)kr);
1271 }
1272 }
1273 return kr;
1274}
1275
1276
1277errno_t
1278decmpfs_pagein_compressed(struct vnop_pagein_args *ap, int *is_compressed, decmpfs_cnode *cp)
1279{
1280 /* handles a page-in request from vfs for a compressed file */
1281
1282 int err = 0;
1283 vnode_t vp = ap->a_vp;
1284 upl_t pl = ap->a_pl;
1285 upl_offset_t pl_offset = ap->a_pl_offset;
1286 off_t f_offset = ap->a_f_offset;
1287 size_t size = ap->a_size;
1288 int flags = ap->a_flags;
1289 off_t uplPos = 0;
1290 user_ssize_t uplSize = 0;
1291 user_ssize_t rounded_uplSize = 0;
1292 size_t verify_block_size = 0;
1293 void *data = NULL;
1294 decmpfs_header *hdr = NULL;
1295 size_t alloc_size = 0;
1296 uint64_t cachedSize = 0;
1297 uint32_t fs_bsize = 0;
1298 int cmpdata_locked = 0;
1299 int num_valid_pages = 0;
1300 int num_invalid_pages = 0;
1301 bool file_tail_page_valid = false;
1302
1303 if (!decmpfs_trylock_compressed_data(cp, exclusive: 0)) {
1304 return EAGAIN;
1305 }
1306 cmpdata_locked = 1;
1307
1308
1309 if (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD)) {
1310 DebugLogWithPath("pagein: unknown flags 0x%08x\n", (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD)));
1311 }
1312
1313 err = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 0, hdr_size: &alloc_size);
1314 if (err != 0) {
1315 decmpfs_ktriage_record(KTRIAGE_DECMPFS_FETCH_HEADER_FAILED, err);
1316 goto out;
1317 }
1318
1319 cachedSize = hdr->uncompressed_size;
1320
1321 if (!compression_type_valid(vp, hdr)) {
1322 /* compressor not registered */
1323 decmpfs_ktriage_record(KTRIAGE_DECMPFS_COMPRESSOR_NOT_REGISTERED, hdr->compression_type);
1324 err = ENOTSUP;
1325 goto out;
1326 }
1327
1328 /*
1329 * can't page-in from a negative offset
1330 * or if we're starting beyond the EOF
1331 * or if the file offset isn't page aligned
1332 * or the size requested isn't a multiple of PAGE_SIZE
1333 */
1334 if (f_offset < 0 || f_offset >= cachedSize ||
1335 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK) || (pl_offset & PAGE_MASK)) {
1336 decmpfs_ktriage_record(KTRIAGE_DECMPFS_IVALID_OFFSET, 0);
1337 err = EINVAL;
1338 goto out;
1339 }
1340
1341 /*
1342 * If the verify block size is larger than the page size, the UPL needs
1343 * to be aligned to it, Since the UPL has been created by the filesystem,
1344 * we will only check if the passed in UPL length conforms to the
1345 * alignment requirements.
1346 */
1347 err = VNOP_VERIFY(vp, f_offset, NULL, 0, &verify_block_size, NULL,
1348 VNODE_VERIFY_DEFAULT, NULL);
1349 if (err) {
1350 ErrorLogWithPath("VNOP_VERIFY returned error = %d\n", err);
1351 goto out;
1352 } else if (verify_block_size) {
1353 if (vp->v_mount->mnt_vfsstat.f_bsize > PAGE_SIZE) {
1354 fs_bsize = vp->v_mount->mnt_vfsstat.f_bsize;
1355 }
1356 if (verify_block_size & (verify_block_size - 1)) {
1357 ErrorLogWithPath("verify block size (%zu) is not power of 2, no verification will be done\n", verify_block_size);
1358 err = EINVAL;
1359 } else if (size % verify_block_size) {
1360 ErrorLogWithPath("upl size (%zu) is not a multiple of verify block size (%zu)\n", (size_t)size, verify_block_size);
1361 err = EINVAL;
1362 } else if (fs_bsize) {
1363 /*
1364 * Filesystems requesting verification have to provide
1365 * values for block sizes which are powers of 2.
1366 */
1367 if (fs_bsize & (fs_bsize - 1)) {
1368 ErrorLogWithPath("FS block size (%u) is greater than PAGE_SIZE (%d) and is not power of 2, no verification will be done\n",
1369 fs_bsize, PAGE_SIZE);
1370 err = EINVAL;
1371 } else if (fs_bsize > verify_block_size) {
1372 ErrorLogWithPath("FS block size (%u) is greater than verify block size (%zu), no verification will be done\n",
1373 fs_bsize, verify_block_size);
1374 err = EINVAL;
1375 }
1376 }
1377 if (err) {
1378 goto out;
1379 }
1380 }
1381
1382#if CONFIG_IOSCHED
1383 /* Mark the UPL as the requesting UPL for decompression */
1384 upl_mark_decmp(upl: pl);
1385#endif /* CONFIG_IOSCHED */
1386
1387 /* map the upl so we can fetch into it */
1388 kern_return_t kr = ubc_upl_map(pl, (vm_offset_t*)&data);
1389 if ((kr != KERN_SUCCESS) || (data == NULL)) {
1390 decmpfs_ktriage_record(KTRIAGE_DECMPFS_UBC_UPL_MAP_FAILED, kr);
1391 err = ENOSPC;
1392 data = NULL;
1393#if CONFIG_IOSCHED
1394 upl_unmark_decmp(upl: pl);
1395#endif /* CONFIG_IOSCHED */
1396 goto out;
1397 }
1398
1399 uplPos = f_offset;
1400 off_t max_size = cachedSize - f_offset;
1401
1402 if (size < max_size) {
1403 rounded_uplSize = uplSize = size;
1404 file_tail_page_valid = true;
1405 } else {
1406 uplSize = (user_ssize_t)max_size;
1407 if (fs_bsize) {
1408 /* First round up to fs_bsize */
1409 rounded_uplSize = (uplSize + (fs_bsize - 1)) & ~(fs_bsize - 1);
1410 /* then to PAGE_SIZE */
1411 rounded_uplSize = MIN(size, round_page((vm_offset_t)rounded_uplSize));
1412 } else {
1413 rounded_uplSize = round_page(x: (vm_offset_t)uplSize);
1414 }
1415 }
1416
1417 /* do the fetch */
1418 decmpfs_vector vec;
1419
1420decompress:
1421 /* the mapped data pointer points to the first page of the page list, so we want to start filling in at an offset of pl_offset */
1422 vec = (decmpfs_vector) {
1423 .buf = (char*)data + pl_offset,
1424 .size = size,
1425 };
1426
1427 uint64_t did_read = 0;
1428 if (decmpfs_fast_get_state(cp) == FILE_IS_CONVERTING) {
1429 ErrorLogWithPath("unexpected pagein during decompress\n");
1430 /*
1431 * if the file is converting, this must be a recursive call to pagein from underneath a call to decmpfs_decompress_file;
1432 * pretend that it succeeded but don't do anything since we're just going to write over the pages anyway
1433 */
1434 err = 0;
1435 } else {
1436 if (verify_block_size <= PAGE_SIZE) {
1437 err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset: uplPos, size: uplSize, nvec: 1, vec: &vec, bytes_read: &did_read);
1438 /* zero out whatever wasn't read */
1439 if (did_read < rounded_uplSize) {
1440 memset(s: (char*)vec.buf + did_read, c: 0, n: (size_t)(rounded_uplSize - did_read));
1441 }
1442 } else {
1443 off_t l_uplPos = uplPos;
1444 off_t l_pl_offset = pl_offset;
1445 user_ssize_t l_uplSize = uplSize;
1446 upl_page_info_t *pl_info = ubc_upl_pageinfo(pl);
1447
1448 err = 0;
1449 /*
1450 * When the system page size is less than the "verify block size",
1451 * the UPL passed may not consist solely of absent pages.
1452 * We have to detect the "absent" pages and only decompress
1453 * into those absent/invalid page ranges.
1454 *
1455 * Things that will change in each iteration of the loop :
1456 *
1457 * l_pl_offset = where we are inside the UPL [0, caller_upl_created_size)
1458 * l_uplPos = the file offset the l_pl_offset corresponds to.
1459 * l_uplSize = the size of the upl still unprocessed;
1460 *
1461 * In this picture, we have to do the transfer on 2 ranges
1462 * (One 2 page range and one 3 page range) and the loop
1463 * below will skip the first two pages and then identify
1464 * the next two as invalid and fill those in and
1465 * then skip the next one and then do the last pages.
1466 *
1467 * uplPos(file_offset)
1468 * | uplSize
1469 * 0 V<--------------> file_size
1470 * |--------------------------------------------------->
1471 * | | |V|V|I|I|V|I|I|I|
1472 * ^
1473 * | upl
1474 * <------------------->
1475 * |
1476 * pl_offset
1477 *
1478 * uplSize will be clipped in case the UPL range exceeds
1479 * the file size.
1480 *
1481 */
1482 while (l_uplSize) {
1483 uint64_t l_did_read = 0;
1484 int pl_offset_pg = (int)(l_pl_offset / PAGE_SIZE);
1485 int pages_left_in_upl;
1486 int start_pg;
1487 int last_pg;
1488
1489 /*
1490 * l_uplSize may start off less than the size of the upl,
1491 * we have to round it up to PAGE_SIZE to calculate
1492 * how many more pages are left.
1493 */
1494 pages_left_in_upl = (int)(round_page(x: (vm_offset_t)l_uplSize) / PAGE_SIZE);
1495
1496 /*
1497 * scan from the beginning of the upl looking for the first
1498 * non-valid page.... this will become the first page in
1499 * the request we're going to make to
1500 * 'decmpfs_fetch_uncompressed_data'... if all
1501 * of the pages are valid, we won't call through
1502 * to 'decmpfs_fetch_uncompressed_data'
1503 */
1504 for (start_pg = 0; start_pg < pages_left_in_upl; start_pg++) {
1505 if (!upl_valid_page(upl: pl_info, index: pl_offset_pg + start_pg)) {
1506 break;
1507 }
1508 }
1509
1510 num_valid_pages += start_pg;
1511
1512 /*
1513 * scan from the starting invalid page looking for
1514 * a valid page before the end of the upl is
1515 * reached, if we find one, then it will be the
1516 * last page of the request to 'decmpfs_fetch_uncompressed_data'
1517 */
1518 for (last_pg = start_pg; last_pg < pages_left_in_upl; last_pg++) {
1519 if (upl_valid_page(upl: pl_info, index: pl_offset_pg + last_pg)) {
1520 break;
1521 }
1522 }
1523
1524 if (start_pg < last_pg) {
1525 off_t inval_offset = start_pg * PAGE_SIZE;
1526 int inval_pages = last_pg - start_pg;
1527 int inval_size = inval_pages * PAGE_SIZE;
1528 decmpfs_vector l_vec;
1529
1530 num_invalid_pages += inval_pages;
1531 if (inval_offset) {
1532 did_read += inval_offset;
1533 l_pl_offset += inval_offset;
1534 l_uplPos += inval_offset;
1535 l_uplSize -= inval_offset;
1536 }
1537
1538 l_vec = (decmpfs_vector) {
1539 .buf = (char*)data + l_pl_offset,
1540 .size = inval_size,
1541 };
1542
1543 err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset: l_uplPos,
1544 MIN(l_uplSize, inval_size), nvec: 1, vec: &l_vec, bytes_read: &l_did_read);
1545
1546 if (!err && (l_did_read != inval_size) && (l_uplSize > inval_size)) {
1547 ErrorLogWithPath("Unexpected size fetch of decompressed data, l_uplSize = %d, l_did_read = %d, inval_size = %d\n",
1548 (int)l_uplSize, (int)l_did_read, (int)inval_size);
1549 err = EINVAL;
1550 }
1551 } else {
1552 /* no invalid pages left */
1553 l_did_read = l_uplSize;
1554 if (!file_tail_page_valid) {
1555 file_tail_page_valid = true;
1556 }
1557 }
1558
1559 if (err) {
1560 break;
1561 }
1562
1563 did_read += l_did_read;
1564 l_pl_offset += l_did_read;
1565 l_uplPos += l_did_read;
1566 l_uplSize -= l_did_read;
1567 }
1568
1569 /* Zero out the region after EOF in the last page (if needed) */
1570 if (!err && !file_tail_page_valid && (uplSize < rounded_uplSize)) {
1571 memset(s: (char*)vec.buf + uplSize, c: 0, n: (size_t)(rounded_uplSize - uplSize));
1572 }
1573 }
1574 }
1575 if (err) {
1576 decmpfs_ktriage_record(KTRIAGE_DECMPFS_FETCH_UNCOMPRESSED_DATA_FAILED, err)
1577 DebugLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err);
1578 int cmp_state = decmpfs_fast_get_state(cp);
1579 if (cmp_state == FILE_IS_CONVERTING) {
1580 DebugLogWithPath("cmp_state == FILE_IS_CONVERTING\n");
1581 cmp_state = wait_for_decompress(cp);
1582 if (cmp_state == FILE_IS_COMPRESSED) {
1583 DebugLogWithPath("cmp_state == FILE_IS_COMPRESSED\n");
1584 /* a decompress was attempted but it failed, let's try calling fetch again */
1585 goto decompress;
1586 }
1587 }
1588 if (cmp_state == FILE_IS_NOT_COMPRESSED) {
1589 DebugLogWithPath("cmp_state == FILE_IS_NOT_COMPRESSED\n");
1590 /* the file was decompressed after we started reading it */
1591 *is_compressed = 0; /* instruct caller to fall back to its normal path */
1592 }
1593 }
1594
1595 if (!err && verify_block_size) {
1596 size_t cur_verify_block_size = verify_block_size;
1597
1598 if ((err = VNOP_VERIFY(vp, uplPos, vec.buf, rounded_uplSize, &cur_verify_block_size, NULL, 0, NULL))) {
1599 ErrorLogWithPath("Verification failed with error %d, uplPos = %lld, uplSize = %d, did_read = %d, valid_pages = %d, invalid_pages = %d, tail_page_valid = %d\n",
1600 err, (long long)uplPos, (int)rounded_uplSize, (int)did_read, num_valid_pages, num_invalid_pages, file_tail_page_valid);
1601 }
1602 /* XXX : If the verify block size changes, redo the read */
1603 }
1604
1605#if CONFIG_IOSCHED
1606 upl_unmark_decmp(upl: pl);
1607#endif /* CONFIG_IOSCHED */
1608
1609 kr = ubc_upl_unmap(pl); data = NULL; /* make sure to set data to NULL so we don't try to unmap again below */
1610 if (kr != KERN_SUCCESS) {
1611 ErrorLogWithPath("ubc_upl_unmap error %d\n", (int)kr);
1612 } else {
1613 if (!err) {
1614 /* commit our pages */
1615 kr = commit_upl(upl: pl, pl_offset, uplSize: (size_t)rounded_uplSize, UPL_COMMIT_FREE_ON_EMPTY, abort: 0 /* commit */);
1616 /* If there were any pages after the page containing EOF, abort them. */
1617 if (rounded_uplSize < size) {
1618 kr = commit_upl(upl: pl, pl_offset: (upl_offset_t)(pl_offset + rounded_uplSize), uplSize: (size_t)(size - rounded_uplSize),
1619 UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR, abort: 1 /* abort */);
1620 }
1621 }
1622 }
1623
1624out:
1625 if (data) {
1626 ubc_upl_unmap(pl);
1627 }
1628 if (hdr != NULL) {
1629 kfree_data(hdr, alloc_size);
1630 }
1631 if (cmpdata_locked) {
1632 decmpfs_unlock_compressed_data(cp, exclusive: 0);
1633 }
1634 if (err) {
1635#if 0
1636 if (err != ENXIO && err != ENOSPC) {
1637 char *path = zalloc(ZV_NAMEI);
1638 panic("%s: decmpfs_pagein_compressed: err %d", vnpath(vp, path, PATH_MAX), err);
1639 zfree(ZV_NAMEI, path);
1640 }
1641#endif /* 0 */
1642 ErrorLogWithPath("err %d\n", err);
1643 }
1644 return err;
1645}
1646
1647errno_t
1648decmpfs_read_compressed(struct vnop_read_args *ap, int *is_compressed, decmpfs_cnode *cp)
1649{
1650 /* handles a read request from vfs for a compressed file */
1651
1652 uio_t uio = ap->a_uio;
1653 vnode_t vp = ap->a_vp;
1654 int err = 0;
1655 int countInt = 0;
1656 off_t uplPos = 0;
1657 user_ssize_t uplSize = 0;
1658 user_ssize_t uplRemaining = 0;
1659 off_t curUplPos = 0;
1660 user_ssize_t curUplSize = 0;
1661 kern_return_t kr = KERN_SUCCESS;
1662 int abort_read = 0;
1663 void *data = NULL;
1664 uint64_t did_read = 0;
1665 upl_t upl = NULL;
1666 upl_page_info_t *pli = NULL;
1667 decmpfs_header *hdr = NULL;
1668 size_t alloc_size = 0;
1669 uint64_t cachedSize = 0;
1670 off_t uioPos = 0;
1671 user_ssize_t uioRemaining = 0;
1672 size_t verify_block_size = 0;
1673 size_t alignment_size = PAGE_SIZE;
1674 int cmpdata_locked = 0;
1675
1676 decmpfs_lock_compressed_data(cp, exclusive: 0); cmpdata_locked = 1;
1677
1678 uplPos = uio_offset(a_uio: uio);
1679 uplSize = uio_resid(a_uio: uio);
1680 VerboseLogWithPath("uplPos %lld uplSize %lld\n", uplPos, uplSize);
1681
1682 cachedSize = decmpfs_cnode_get_vnode_cached_size(cp);
1683
1684 if ((uint64_t)uplPos + uplSize > cachedSize) {
1685 /* truncate the read to the size of the file */
1686 uplSize = (user_ssize_t)(cachedSize - uplPos);
1687 }
1688
1689 /* give the cluster layer a chance to fill in whatever it already has */
1690 countInt = (uplSize > INT_MAX) ? INT_MAX : (int)uplSize;
1691 err = cluster_copy_ubc_data(vp, uio, &countInt, 0);
1692 if (err != 0) {
1693 goto out;
1694 }
1695
1696 /* figure out what's left */
1697 uioPos = uio_offset(a_uio: uio);
1698 uioRemaining = uio_resid(a_uio: uio);
1699 if ((uint64_t)uioPos + uioRemaining > cachedSize) {
1700 /* truncate the read to the size of the file */
1701 uioRemaining = (user_ssize_t)(cachedSize - uioPos);
1702 }
1703
1704 if (uioRemaining <= 0) {
1705 /* nothing left */
1706 goto out;
1707 }
1708
1709 err = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 0, hdr_size: &alloc_size);
1710 if (err != 0) {
1711 goto out;
1712 }
1713 if (!compression_type_valid(vp, hdr)) {
1714 err = ENOTSUP;
1715 goto out;
1716 }
1717
1718 uplPos = uioPos;
1719 uplSize = uioRemaining;
1720#if COMPRESSION_DEBUG
1721 DebugLogWithPath("uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize);
1722#endif
1723
1724 lck_rw_lock_shared(lck: &decompressorsLock);
1725 decmpfs_adjust_fetch_region_func adjust_fetch = decmp_get_func(vp, hdr->compression_type, adjust_fetch);
1726 if (adjust_fetch) {
1727 /* give the compressor a chance to adjust the portion of the file that we read */
1728 adjust_fetch(vp, decmpfs_ctx, hdr, &uplPos, &uplSize);
1729 VerboseLogWithPath("adjusted uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize);
1730 }
1731 lck_rw_unlock_shared(lck: &decompressorsLock);
1732
1733 /* clip the adjusted size to the size of the file */
1734 if ((uint64_t)uplPos + uplSize > cachedSize) {
1735 /* truncate the read to the size of the file */
1736 uplSize = (user_ssize_t)(cachedSize - uplPos);
1737 }
1738
1739 if (uplSize <= 0) {
1740 /* nothing left */
1741 goto out;
1742 }
1743
1744 /*
1745 * since we're going to create a upl for the given region of the file,
1746 * make sure we're on page boundaries
1747 */
1748
1749 /* If the verify block size is larger than the page size, the UPL needs to aligned to it */
1750 err = VNOP_VERIFY(vp, uplPos, NULL, 0, &verify_block_size, NULL, VNODE_VERIFY_DEFAULT, NULL);
1751 if (err) {
1752 goto out;
1753 } else if (verify_block_size) {
1754 if (verify_block_size & (verify_block_size - 1)) {
1755 ErrorLogWithPath("verify block size is not power of 2, no verification will be done\n");
1756 verify_block_size = 0;
1757 } else if (verify_block_size > PAGE_SIZE) {
1758 alignment_size = verify_block_size;
1759 }
1760 }
1761
1762 if (uplPos & (alignment_size - 1)) {
1763 /* round position down to page boundary */
1764 uplSize += (uplPos & (alignment_size - 1));
1765 uplPos &= ~(alignment_size - 1);
1766 }
1767
1768 /* round size up to alignement_size multiple */
1769 uplSize = (uplSize + (alignment_size - 1)) & ~(alignment_size - 1);
1770
1771 VerboseLogWithPath("new uplPos %lld uplSize %lld\n", (uint64_t)uplPos, (uint64_t)uplSize);
1772
1773 uplRemaining = uplSize;
1774 curUplPos = uplPos;
1775 curUplSize = 0;
1776
1777 while (uplRemaining > 0) {
1778 /* start after the last upl */
1779 curUplPos += curUplSize;
1780
1781 /* clip to max upl size */
1782 curUplSize = uplRemaining;
1783 if (curUplSize > MAX_UPL_SIZE_BYTES) {
1784 curUplSize = MAX_UPL_SIZE_BYTES;
1785 }
1786
1787 /* create the upl */
1788 kr = ubc_create_upl_kernel(vp, curUplPos, (int)curUplSize, &upl, &pli, UPL_SET_LITE, VM_KERN_MEMORY_FILE);
1789 if (kr != KERN_SUCCESS) {
1790 ErrorLogWithPath("ubc_create_upl error %d\n", (int)kr);
1791 err = EINVAL;
1792 goto out;
1793 }
1794 VerboseLogWithPath("curUplPos %lld curUplSize %lld\n", (uint64_t)curUplPos, (uint64_t)curUplSize);
1795
1796#if CONFIG_IOSCHED
1797 /* Mark the UPL as the requesting UPL for decompression */
1798 upl_mark_decmp(upl);
1799#endif /* CONFIG_IOSCHED */
1800
1801 /* map the upl */
1802 kr = ubc_upl_map(upl, (vm_offset_t*)&data);
1803 if (kr != KERN_SUCCESS) {
1804 commit_upl(upl, pl_offset: 0, uplSize: curUplSize, UPL_ABORT_FREE_ON_EMPTY, abort: 1);
1805#if 0
1806 char *path = zalloc(ZV_NAMEI);
1807 panic("%s: decmpfs_read_compressed: ubc_upl_map error %d", vnpath(vp, path, PATH_MAX), (int)kr);
1808 zfree(ZV_NAMEI, path);
1809#else /* 0 */
1810 ErrorLogWithPath("ubc_upl_map kr=0x%x\n", (int)kr);
1811#endif /* 0 */
1812 err = EINVAL;
1813 goto out;
1814 }
1815
1816 /* make sure the map succeeded */
1817 if (!data) {
1818 commit_upl(upl, pl_offset: 0, uplSize: curUplSize, UPL_ABORT_FREE_ON_EMPTY, abort: 1);
1819
1820 ErrorLogWithPath("ubc_upl_map mapped null\n");
1821 err = EINVAL;
1822 goto out;
1823 }
1824
1825 /* fetch uncompressed data into the mapped upl */
1826 decmpfs_vector vec;
1827decompress:
1828 vec = (decmpfs_vector){ .buf = data, .size = curUplSize };
1829 err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset: curUplPos, size: curUplSize, nvec: 1, vec: &vec, bytes_read: &did_read);
1830 if (err) {
1831 ErrorLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err);
1832
1833 /* maybe the file is converting to decompressed */
1834 int cmp_state = decmpfs_fast_get_state(cp);
1835 if (cmp_state == FILE_IS_CONVERTING) {
1836 ErrorLogWithPath("cmp_state == FILE_IS_CONVERTING\n");
1837 cmp_state = wait_for_decompress(cp);
1838 if (cmp_state == FILE_IS_COMPRESSED) {
1839 ErrorLogWithPath("cmp_state == FILE_IS_COMPRESSED\n");
1840 /* a decompress was attempted but it failed, let's try fetching again */
1841 goto decompress;
1842 }
1843 }
1844 if (cmp_state == FILE_IS_NOT_COMPRESSED) {
1845 ErrorLogWithPath("cmp_state == FILE_IS_NOT_COMPRESSED\n");
1846 /* the file was decompressed after we started reading it */
1847 abort_read = 1; /* we're not going to commit our data */
1848 *is_compressed = 0; /* instruct caller to fall back to its normal path */
1849 }
1850 kr = KERN_FAILURE;
1851 did_read = 0;
1852 }
1853
1854 /* zero out the remainder of the last page */
1855 memset(s: (char*)data + did_read, c: 0, n: (size_t)(curUplSize - did_read));
1856 if (!err && verify_block_size) {
1857 size_t cur_verify_block_size = verify_block_size;
1858
1859 if ((err = VNOP_VERIFY(vp, curUplPos, data, curUplSize, &cur_verify_block_size, NULL, 0, NULL))) {
1860 ErrorLogWithPath("Verification failed with error %d\n", err);
1861 abort_read = 1;
1862 }
1863 /* XXX : If the verify block size changes, redo the read */
1864 }
1865
1866 kr = ubc_upl_unmap(upl);
1867 if (kr == KERN_SUCCESS) {
1868 if (abort_read) {
1869 kr = commit_upl(upl, pl_offset: 0, uplSize: curUplSize, UPL_ABORT_FREE_ON_EMPTY, abort: 1);
1870 } else {
1871 VerboseLogWithPath("uioPos %lld uioRemaining %lld\n", (uint64_t)uioPos, (uint64_t)uioRemaining);
1872 if (uioRemaining) {
1873 off_t uplOff = uioPos - curUplPos;
1874 if (uplOff < 0) {
1875 ErrorLogWithPath("uplOff %lld should never be negative\n", (int64_t)uplOff);
1876 err = EINVAL;
1877 } else if (uplOff > INT_MAX) {
1878 ErrorLogWithPath("uplOff %lld too large\n", (int64_t)uplOff);
1879 err = EINVAL;
1880 } else {
1881 off_t count = curUplPos + curUplSize - uioPos;
1882 if (count < 0) {
1883 /* this upl is entirely before the uio */
1884 } else {
1885 if (count > uioRemaining) {
1886 count = uioRemaining;
1887 }
1888 int icount = (count > INT_MAX) ? INT_MAX : (int)count;
1889 int io_resid = icount;
1890 err = cluster_copy_upl_data(uio, upl, (int)uplOff, &io_resid);
1891 int copied = icount - io_resid;
1892 VerboseLogWithPath("uplOff %lld count %lld copied %lld\n", (uint64_t)uplOff, (uint64_t)count, (uint64_t)copied);
1893 if (err) {
1894 ErrorLogWithPath("cluster_copy_upl_data err %d\n", err);
1895 }
1896 uioPos += copied;
1897 uioRemaining -= copied;
1898 }
1899 }
1900 }
1901 kr = commit_upl(upl, pl_offset: 0, uplSize: curUplSize, UPL_COMMIT_FREE_ON_EMPTY | UPL_COMMIT_INACTIVATE, abort: 0);
1902 if (err) {
1903 goto out;
1904 }
1905 }
1906 } else {
1907 ErrorLogWithPath("ubc_upl_unmap error %d\n", (int)kr);
1908 }
1909
1910 uplRemaining -= curUplSize;
1911 }
1912
1913out:
1914
1915 if (hdr != NULL) {
1916 kfree_data(hdr, alloc_size);
1917 }
1918 if (cmpdata_locked) {
1919 decmpfs_unlock_compressed_data(cp, exclusive: 0);
1920 }
1921 if (err) {/* something went wrong */
1922 ErrorLogWithPath("err %d\n", err);
1923 return err;
1924 }
1925
1926#if COMPRESSION_DEBUG
1927 uplSize = uio_resid(uio);
1928 if (uplSize) {
1929 VerboseLogWithPath("still %lld bytes to copy\n", uplSize);
1930 }
1931#endif
1932 return 0;
1933}
1934
1935int
1936decmpfs_free_compressed_data(vnode_t vp, decmpfs_cnode *cp)
1937{
1938 /*
1939 * call out to the decompressor to free remove any data associated with this compressed file
1940 * then delete the file's compression xattr
1941 */
1942 decmpfs_header *hdr = NULL;
1943 size_t alloc_size = 0;
1944
1945 /*
1946 * Trace the following parameters on entry with event-id 0x03120010.
1947 *
1948 * @vp->v_id: vnode-id of the file for which to free compressed data.
1949 */
1950 DECMPFS_EMIT_TRACE_ENTRY(DECMPDBG_FREE_COMPRESSED_DATA, vp->v_id);
1951
1952 int err = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 0, hdr_size: &alloc_size);
1953 if (err) {
1954 ErrorLogWithPath("decmpfs_fetch_compressed_header err %d\n", err);
1955 } else {
1956 lck_rw_lock_shared(lck: &decompressorsLock);
1957 decmpfs_free_compressed_data_func free_data = decmp_get_func(vp, hdr->compression_type, free_data);
1958 if (free_data) {
1959 err = free_data(vp, decmpfs_ctx, hdr);
1960 } else {
1961 /* nothing to do, so no error */
1962 err = 0;
1963 }
1964 lck_rw_unlock_shared(lck: &decompressorsLock);
1965
1966 if (err != 0) {
1967 ErrorLogWithPath("decompressor err %d\n", err);
1968 }
1969 }
1970 /*
1971 * Trace the following parameters on return with event-id 0x03120010.
1972 *
1973 * @vp->v_id: vnode-id of the file for which to free compressed data.
1974 * @err: value returned from this function.
1975 */
1976 DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FREE_COMPRESSED_DATA, vp->v_id, err);
1977
1978 /* delete the xattr */
1979 err = vn_removexattr(vp, DECMPFS_XATTR_NAME, 0, decmpfs_ctx);
1980
1981 if (hdr != NULL) {
1982 kfree_data(hdr, alloc_size);
1983 }
1984 return err;
1985}
1986
1987#pragma mark --- file conversion routines ---
1988
1989static int
1990unset_compressed_flag(vnode_t vp)
1991{
1992 int err = 0;
1993 struct vnode_attr va;
1994 struct fsioc_cas_bsdflags cas;
1995 int i;
1996
1997# define MAX_CAS_BSDFLAGS_LOOPS 4
1998 /* UF_COMPRESSED should be manipulated only with FSIOC_CAS_BSDFLAGS */
1999 for (i = 0; i < MAX_CAS_BSDFLAGS_LOOPS; i++) {
2000 VATTR_INIT(&va);
2001 VATTR_WANTED(&va, va_flags);
2002 err = vnode_getattr(vp, vap: &va, ctx: decmpfs_ctx);
2003 if (err != 0) {
2004 ErrorLogWithPath("vnode_getattr err %d, num retries %d\n", err, i);
2005 goto out;
2006 }
2007
2008 cas.expected_flags = va.va_flags;
2009 cas.new_flags = va.va_flags & ~UF_COMPRESSED;
2010 err = VNOP_IOCTL(vp, FSIOC_CAS_BSDFLAGS, data: (caddr_t)&cas, FWRITE, ctx: decmpfs_ctx);
2011
2012 if ((err == 0) && (va.va_flags == cas.actual_flags)) {
2013 goto out;
2014 }
2015
2016 if ((err != 0) && (err != EAGAIN)) {
2017 break;
2018 }
2019 }
2020
2021 /* fallback to regular chflags if FSIOC_CAS_BSDFLAGS is not supported */
2022 if (err == ENOTTY) {
2023 VATTR_INIT(&va);
2024 VATTR_SET(&va, va_flags, cas.new_flags);
2025 err = vnode_setattr(vp, vap: &va, ctx: decmpfs_ctx);
2026 if (err != 0) {
2027 ErrorLogWithPath("vnode_setattr err %d\n", err);
2028 }
2029 } else if (va.va_flags != cas.actual_flags) {
2030 ErrorLogWithPath("FSIOC_CAS_BSDFLAGS err: flags mismatc. actual (%x) expected (%x), num retries %d\n", cas.actual_flags, va.va_flags, i);
2031 } else if (err != 0) {
2032 ErrorLogWithPath("FSIOC_CAS_BSDFLAGS err %d, num retries %d\n", err, i);
2033 }
2034
2035out:
2036 return err;
2037}
2038
2039int
2040decmpfs_decompress_file(vnode_t vp, decmpfs_cnode *cp, off_t toSize, int truncate_okay, int skiplock)
2041{
2042 /* convert a compressed file to an uncompressed file */
2043
2044 int err = 0;
2045 char *data = NULL;
2046 uio_t uio_w = 0;
2047 off_t offset = 0;
2048 uint32_t old_state = 0;
2049 uint32_t new_state = 0;
2050 int update_file_state = 0;
2051 size_t allocSize = 0;
2052 decmpfs_header *hdr = NULL;
2053 size_t hdr_size = 0;
2054 int cmpdata_locked = 0;
2055 off_t remaining = 0;
2056 uint64_t uncompressed_size = 0;
2057
2058 /*
2059 * Trace the following parameters on entry with event-id 0x03120000.
2060 *
2061 * @vp->v_id: vnode-id of the file being decompressed.
2062 * @toSize: uncompress given bytes of the file.
2063 * @truncate_okay: on error it is OK to truncate.
2064 * @skiplock: compressed data is locked, skip locking again.
2065 *
2066 * Please NOTE: @toSize can overflow in theory but here it is safe.
2067 */
2068 DECMPFS_EMIT_TRACE_ENTRY(DECMPDBG_DECOMPRESS_FILE, vp->v_id,
2069 (int)toSize, truncate_okay, skiplock);
2070
2071 if (!skiplock) {
2072 decmpfs_lock_compressed_data(cp, exclusive: 1); cmpdata_locked = 1;
2073 }
2074
2075decompress:
2076 old_state = decmpfs_fast_get_state(cp);
2077
2078 switch (old_state) {
2079 case FILE_IS_NOT_COMPRESSED:
2080 {
2081 /* someone else decompressed the file */
2082 err = 0;
2083 goto out;
2084 }
2085
2086 case FILE_TYPE_UNKNOWN:
2087 {
2088 /* the file is in an unknown state, so update the state and retry */
2089 (void)decmpfs_file_is_compressed(vp, cp);
2090
2091 /* try again */
2092 goto decompress;
2093 }
2094
2095 case FILE_IS_COMPRESSED:
2096 {
2097 /* the file is compressed, so decompress it */
2098 break;
2099 }
2100
2101 default:
2102 {
2103 /*
2104 * this shouldn't happen since multiple calls to decmpfs_decompress_file lock each other out,
2105 * and when decmpfs_decompress_file returns, the state should be always be set back to
2106 * FILE_IS_NOT_COMPRESSED or FILE_IS_UNKNOWN
2107 */
2108 err = EINVAL;
2109 goto out;
2110 }
2111 }
2112
2113 err = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 0, hdr_size: &hdr_size);
2114 if (err != 0) {
2115 goto out;
2116 }
2117
2118 uncompressed_size = hdr->uncompressed_size;
2119 if (toSize == -1) {
2120 toSize = hdr->uncompressed_size;
2121 }
2122
2123 if (toSize == 0) {
2124 /* special case truncating the file to zero bytes */
2125 goto nodecmp;
2126 } else if ((uint64_t)toSize > hdr->uncompressed_size) {
2127 /* the caller is trying to grow the file, so we should decompress all the data */
2128 toSize = hdr->uncompressed_size;
2129 }
2130
2131 allocSize = MIN(64 * 1024, (size_t)toSize);
2132 data = (char *)kalloc_data(allocSize, Z_WAITOK);
2133 if (!data) {
2134 err = ENOMEM;
2135 goto out;
2136 }
2137
2138 uio_w = uio_create(a_iovcount: 1, a_offset: 0LL, a_spacetype: UIO_SYSSPACE, a_iodirection: UIO_WRITE);
2139 if (!uio_w) {
2140 err = ENOMEM;
2141 goto out;
2142 }
2143 uio_w->uio_flags |= UIO_FLAGS_IS_COMPRESSED_FILE;
2144
2145 remaining = toSize;
2146
2147 /* tell the buffer cache that this is an empty file */
2148 ubc_setsize(vp, 0);
2149
2150 /* if we got here, we need to decompress the file */
2151 decmpfs_cnode_set_vnode_state(cp, state: FILE_IS_CONVERTING, skiplock: 1);
2152
2153 while (remaining > 0) {
2154 /* loop decompressing data from the file and writing it into the data fork */
2155
2156 uint64_t bytes_read = 0;
2157 decmpfs_vector vec = { .buf = data, .size = (user_ssize_t)MIN(allocSize, remaining) };
2158 err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset, size: vec.size, nvec: 1, vec: &vec, bytes_read: &bytes_read);
2159 if (err != 0) {
2160 ErrorLogWithPath("decmpfs_fetch_uncompressed_data err %d\n", err);
2161 goto out;
2162 }
2163
2164 if (bytes_read == 0) {
2165 /* we're done reading data */
2166 break;
2167 }
2168
2169 uio_reset(a_uio: uio_w, a_offset: offset, a_spacetype: UIO_SYSSPACE, a_iodirection: UIO_WRITE);
2170 err = uio_addiov(a_uio: uio_w, CAST_USER_ADDR_T(data), a_length: (user_size_t)bytes_read);
2171 if (err != 0) {
2172 ErrorLogWithPath("uio_addiov err %d\n", err);
2173 err = ENOMEM;
2174 goto out;
2175 }
2176
2177 err = VNOP_WRITE(vp, uio: uio_w, ioflag: 0, ctx: decmpfs_ctx);
2178 if (err != 0) {
2179 /* if the write failed, truncate the file to zero bytes */
2180 ErrorLogWithPath("VNOP_WRITE err %d\n", err);
2181 break;
2182 }
2183 offset += bytes_read;
2184 remaining -= bytes_read;
2185 }
2186
2187 if (err == 0) {
2188 if (offset != toSize) {
2189 ErrorLogWithPath("file decompressed to %lld instead of %lld\n", offset, toSize);
2190 err = EINVAL;
2191 goto out;
2192 }
2193 }
2194
2195 if (err == 0) {
2196 /* sync the data and metadata */
2197 err = VNOP_FSYNC(vp, MNT_WAIT, ctx: decmpfs_ctx);
2198 if (err != 0) {
2199 ErrorLogWithPath("VNOP_FSYNC err %d\n", err);
2200 goto out;
2201 }
2202 }
2203
2204 if (err != 0) {
2205 /* write, setattr, or fsync failed */
2206 ErrorLogWithPath("aborting decompress, err %d\n", err);
2207 if (truncate_okay) {
2208 /* truncate anything we might have written */
2209 int error = vnode_setsize(vp, 0, ioflag: 0, decmpfs_ctx);
2210 ErrorLogWithPath("vnode_setsize err %d\n", error);
2211 }
2212 goto out;
2213 }
2214
2215nodecmp:
2216 /* if we're truncating the file to zero bytes, we'll skip ahead to here */
2217
2218 /* unset the compressed flag */
2219 unset_compressed_flag(vp);
2220
2221 /* free the compressed data associated with this file */
2222 err = decmpfs_free_compressed_data(vp, cp);
2223 if (err != 0) {
2224 ErrorLogWithPath("decmpfs_free_compressed_data err %d\n", err);
2225 }
2226
2227 /*
2228 * even if free_compressed_data or vnode_getattr/vnode_setattr failed, return success
2229 * since we succeeded in writing all of the file data to the data fork
2230 */
2231 err = 0;
2232
2233 /* if we got this far, the file was successfully decompressed */
2234 update_file_state = 1;
2235 new_state = FILE_IS_NOT_COMPRESSED;
2236
2237#if COMPRESSION_DEBUG
2238 {
2239 uint64_t filesize = 0;
2240 vnsize(vp, &filesize);
2241 DebugLogWithPath("new file size %lld\n", filesize);
2242 }
2243#endif
2244
2245out:
2246 if (hdr != NULL) {
2247 kfree_data(hdr, hdr_size);
2248 }
2249 kfree_data(data, allocSize);
2250
2251 if (uio_w) {
2252 uio_free(a_uio: uio_w);
2253 }
2254
2255 if (err != 0) {
2256 /* if there was a failure, reset compression flags to unknown and clear the buffer cache data */
2257 update_file_state = 1;
2258 new_state = FILE_TYPE_UNKNOWN;
2259 if (uncompressed_size) {
2260 ubc_setsize(vp, 0);
2261 ubc_setsize(vp, uncompressed_size);
2262 }
2263 }
2264
2265 if (update_file_state) {
2266 lck_mtx_lock(lck: &decompress_channel_mtx);
2267 decmpfs_cnode_set_vnode_state(cp, state: new_state, skiplock: 1);
2268 wakeup(chan: (caddr_t)&decompress_channel); /* wake up anyone who might have been waiting for decompression */
2269 lck_mtx_unlock(lck: &decompress_channel_mtx);
2270 }
2271
2272 if (cmpdata_locked) {
2273 decmpfs_unlock_compressed_data(cp, exclusive: 1);
2274 }
2275 /*
2276 * Trace the following parameters on return with event-id 0x03120000.
2277 *
2278 * @vp->v_id: vnode-id of the file being decompressed.
2279 * @err: value returned from this function.
2280 */
2281 DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_DECOMPRESS_FILE, vp->v_id, err);
2282 return err;
2283}
2284
2285#pragma mark --- Type1 compressor ---
2286
2287/*
2288 * The "Type1" compressor stores the data fork directly in the compression xattr
2289 */
2290
2291static int
2292decmpfs_validate_compressed_file_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr)
2293{
2294 int err = 0;
2295
2296 if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) {
2297 err = EINVAL;
2298 goto out;
2299 }
2300out:
2301 return err;
2302}
2303
2304static int
2305decmpfs_fetch_uncompressed_data_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read)
2306{
2307 int err = 0;
2308 int i;
2309 user_ssize_t remaining;
2310
2311 if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) {
2312 err = EINVAL;
2313 goto out;
2314 }
2315
2316#if COMPRESSION_DEBUG
2317 static int dummy = 0; // prevent syslog from coalescing printfs
2318 DebugLogWithPath("%d memcpy %lld at %lld\n", dummy++, size, (uint64_t)offset);
2319#endif
2320
2321 remaining = size;
2322 for (i = 0; (i < nvec) && (remaining > 0); i++) {
2323 user_ssize_t curCopy = vec[i].size;
2324 if (curCopy > remaining) {
2325 curCopy = remaining;
2326 }
2327 memcpy(dst: vec[i].buf, src: hdr->attr_bytes + offset, n: curCopy);
2328 offset += curCopy;
2329 remaining -= curCopy;
2330 }
2331
2332 if ((bytes_read) && (err == 0)) {
2333 *bytes_read = (size - remaining);
2334 }
2335
2336out:
2337 return err;
2338}
2339
2340SECURITY_READ_ONLY_EARLY(static decmpfs_registration) Type1Reg =
2341{
2342 .decmpfs_registration = DECMPFS_REGISTRATION_VERSION,
2343 .validate = decmpfs_validate_compressed_file_Type1,
2344 .adjust_fetch = NULL,/* no adjust necessary */
2345 .fetch = decmpfs_fetch_uncompressed_data_Type1,
2346 .free_data = NULL,/* no free necessary */
2347 .get_flags = NULL/* no flags */
2348};
2349
2350#pragma mark --- decmpfs initialization ---
2351
2352void
2353decmpfs_init(void)
2354{
2355 static int done = 0;
2356 if (done) {
2357 return;
2358 }
2359
2360 decmpfs_ctx = vfs_context_create(ctx: vfs_context_kernel());
2361
2362 register_decmpfs_decompressor(compression_type: CMP_Type1, registration: &Type1Reg);
2363
2364 ktriage_register_subsystem_strings(KDBG_TRIAGE_SUBSYS_DECMPFS, subsystem_strings: &ktriage_decmpfs_subsystem_strings);
2365
2366 done = 1;
2367}
2368#endif /* FS_COMPRESSION */
2369