1 | /* |
2 | * Copyright (c) 2008-2018 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | #if !FS_COMPRESSION |
29 | |
30 | /* We need these symbols even though compression is turned off */ |
31 | |
32 | #define UNUSED_SYMBOL(x) asm(".global _" #x "\n.set _" #x ", 0\n"); |
33 | |
34 | UNUSED_SYMBOL(register_decmpfs_decompressor) |
35 | UNUSED_SYMBOL(unregister_decmpfs_decompressor) |
36 | UNUSED_SYMBOL(decmpfs_init) |
37 | UNUSED_SYMBOL(decmpfs_read_compressed) |
38 | UNUSED_SYMBOL(decmpfs_cnode_cmp_type) |
39 | UNUSED_SYMBOL(decmpfs_cnode_get_vnode_state) |
40 | UNUSED_SYMBOL(decmpfs_cnode_get_vnode_cached_size) |
41 | UNUSED_SYMBOL(decmpfs_cnode_get_vnode_cached_nchildren) |
42 | UNUSED_SYMBOL(decmpfs_cnode_get_vnode_cached_total_size) |
43 | UNUSED_SYMBOL(decmpfs_lock_compressed_data) |
44 | UNUSED_SYMBOL(decmpfs_cnode_free) |
45 | UNUSED_SYMBOL(decmpfs_cnode_alloc) |
46 | UNUSED_SYMBOL(decmpfs_cnode_destroy) |
47 | UNUSED_SYMBOL(decmpfs_decompress_file) |
48 | UNUSED_SYMBOL(decmpfs_unlock_compressed_data) |
49 | UNUSED_SYMBOL(decmpfs_cnode_init) |
50 | UNUSED_SYMBOL(decmpfs_cnode_set_vnode_state) |
51 | UNUSED_SYMBOL(decmpfs_hides_xattr) |
52 | UNUSED_SYMBOL(decmpfs_ctx) |
53 | UNUSED_SYMBOL(decmpfs_file_is_compressed) |
54 | UNUSED_SYMBOL(decmpfs_update_attributes) |
55 | UNUSED_SYMBOL(decmpfs_hides_rsrc) |
56 | UNUSED_SYMBOL(decmpfs_pagein_compressed) |
57 | UNUSED_SYMBOL(decmpfs_validate_compressed_file) |
58 | |
59 | #else /* FS_COMPRESSION */ |
60 | #include <sys/kernel.h> |
61 | #include <sys/vnode_internal.h> |
62 | #include <sys/file_internal.h> |
63 | #include <sys/stat.h> |
64 | #include <sys/fcntl.h> |
65 | #include <sys/xattr.h> |
66 | #include <sys/namei.h> |
67 | #include <sys/user.h> |
68 | #include <sys/mount_internal.h> |
69 | #include <sys/ubc.h> |
70 | #include <sys/decmpfs.h> |
71 | #include <sys/uio_internal.h> |
72 | #include <libkern/OSByteOrder.h> |
73 | #include <libkern/section_keywords.h> |
74 | #include <sys/fsctl.h> |
75 | |
76 | #include <sys/kdebug_triage.h> |
77 | |
78 | #include <ptrauth.h> |
79 | |
80 | #pragma mark --- debugging --- |
81 | |
82 | #define COMPRESSION_DEBUG 0 |
83 | #define COMPRESSION_DEBUG_VERBOSE 0 |
84 | #define MALLOC_DEBUG 0 |
85 | |
86 | #if COMPRESSION_DEBUG |
87 | static char* |
88 | vnpath(vnode_t vp, char *path, int len) |
89 | { |
90 | int origlen = len; |
91 | path[0] = 0; |
92 | vn_getpath(vp, path, &len); |
93 | path[origlen - 1] = 0; |
94 | return path; |
95 | } |
96 | #endif |
97 | |
98 | #define ErrorLog(x, args...) \ |
99 | printf("%s:%d:%s: " x, __FILE_NAME__, __LINE__, __FUNCTION__, ## args) |
100 | #if COMPRESSION_DEBUG |
101 | #define ErrorLogWithPath(x, args...) do { \ |
102 | char *path = zalloc(ZV_NAMEI); \ |
103 | printf("%s:%d:%s: %s: " x, __FILE_NAME__, __LINE__, __FUNCTION__, \ |
104 | vnpath(vp, path, PATH_MAX), ## args); \ |
105 | zfree(ZV_NAMEI, path); \ |
106 | } while(0) |
107 | #else |
108 | #define ErrorLogWithPath(x, args...) do { \ |
109 | (void*)vp; \ |
110 | printf("%s:%d:%s: %s: " x, __FILE_NAME__, __LINE__, __FUNCTION__, \ |
111 | "<private>", ## args); \ |
112 | } while(0) |
113 | #endif |
114 | |
115 | #if COMPRESSION_DEBUG |
116 | #define DebugLog ErrorLog |
117 | #define DebugLogWithPath ErrorLogWithPath |
118 | #else |
119 | #define DebugLog(x...) do { } while(0) |
120 | #define DebugLogWithPath(x...) do { } while(0) |
121 | #endif |
122 | |
123 | #if COMPRESSION_DEBUG_VERBOSE |
124 | #define VerboseLog ErrorLog |
125 | #define VerboseLogWithPath ErrorLogWithPath |
126 | #else |
127 | #define VerboseLog(x...) do { } while(0) |
128 | #define VerboseLogWithPath(x...) do { } while(0) |
129 | #endif |
130 | |
131 | #define decmpfs_ktriage_record(code, arg) ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_DECMPFS, KDBG_TRIAGE_RESERVED, code), arg); |
132 | |
133 | enum ktriage_decmpfs_error_codes { |
134 | KTRIAGE_DECMPFS_PREFIX = 0, |
135 | KTRIAGE_DECMPFS_IVALID_OFFSET, |
136 | KTRIAGE_DECMPFS_COMPRESSOR_NOT_REGISTERED, |
137 | KTRIAGE_DECMPFS_FETCH_CALLBACK_FAILED, |
138 | , |
139 | KTRIAGE_DECMPFS_UBC_UPL_MAP_FAILED, |
140 | KTRIAGE_DECMPFS_FETCH_UNCOMPRESSED_DATA_FAILED, |
141 | |
142 | KTRIAGE_DECMPFS_MAX |
143 | }; |
144 | |
145 | const char *ktriage_decmpfs_strings[] = { |
146 | [KTRIAGE_DECMPFS_PREFIX] = "decmpfs - " , |
147 | [KTRIAGE_DECMPFS_IVALID_OFFSET] = "pagein offset is invalid\n" , |
148 | [KTRIAGE_DECMPFS_COMPRESSOR_NOT_REGISTERED] = "compressor is not registered\n" , |
149 | [KTRIAGE_DECMPFS_FETCH_CALLBACK_FAILED] = "fetch callback failed\n" , |
150 | [KTRIAGE_DECMPFS_FETCH_HEADER_FAILED] = "fetch decmpfs xattr failed\n" , |
151 | [KTRIAGE_DECMPFS_UBC_UPL_MAP_FAILED] = "failed to map a UBC UPL\n" , |
152 | [KTRIAGE_DECMPFS_FETCH_UNCOMPRESSED_DATA_FAILED] = "failed to fetch uncompressed data\n" , |
153 | }; |
154 | |
155 | ktriage_strings_t ktriage_decmpfs_subsystem_strings = {KTRIAGE_DECMPFS_MAX, ktriage_decmpfs_strings}; |
156 | |
157 | #pragma mark --- globals --- |
158 | |
159 | static LCK_GRP_DECLARE(decmpfs_lockgrp, "VFSCOMP" ); |
160 | static LCK_RW_DECLARE(decompressorsLock, &decmpfs_lockgrp); |
161 | static LCK_MTX_DECLARE(decompress_channel_mtx, &decmpfs_lockgrp); |
162 | |
163 | static const decmpfs_registration *decompressors[CMP_MAX]; /* the registered compressors */ |
164 | static int decompress_channel; /* channel used by decompress_file to wake up waiters */ |
165 | |
166 | vfs_context_t decmpfs_ctx; |
167 | |
168 | #pragma mark --- decmp_get_func --- |
169 | |
170 | #define offsetof_func(func) ((uintptr_t)offsetof(decmpfs_registration, func)) |
171 | |
172 | static void * |
173 | _func_from_offset(uint32_t type, uintptr_t offset, uint32_t discriminator) |
174 | { |
175 | /* get the function at the given offset in the registration for the given type */ |
176 | const decmpfs_registration *reg = decompressors[type]; |
177 | |
178 | switch (reg->decmpfs_registration) { |
179 | case DECMPFS_REGISTRATION_VERSION_V1: |
180 | if (offset > offsetof_func(free_data)) { |
181 | return NULL; |
182 | } |
183 | break; |
184 | case DECMPFS_REGISTRATION_VERSION_V3: |
185 | if (offset > offsetof_func(get_flags)) { |
186 | return NULL; |
187 | } |
188 | break; |
189 | default: |
190 | return NULL; |
191 | } |
192 | |
193 | void *ptr = *(void * const *)((uintptr_t)reg + offset); |
194 | if (ptr != NULL) { |
195 | /* Resign as a function-in-void* */ |
196 | ptr = ptrauth_auth_and_resign(ptr, ptrauth_key_asia, discriminator, ptrauth_key_asia, 0); |
197 | } |
198 | return ptr; |
199 | } |
200 | |
201 | extern void IOServicePublishResource( const char * property, boolean_t value ); |
202 | extern boolean_t IOServiceWaitForMatchingResource( const char * property, uint64_t timeout ); |
203 | extern boolean_t IOCatalogueMatchingDriversPresent( const char * property ); |
204 | |
205 | static void * |
206 | _decmp_get_func(vnode_t vp, uint32_t type, uintptr_t offset, uint32_t discriminator) |
207 | { |
208 | /* |
209 | * this function should be called while holding a shared lock to decompressorsLock, |
210 | * and will return with the lock held |
211 | */ |
212 | |
213 | if (type >= CMP_MAX) { |
214 | return NULL; |
215 | } |
216 | |
217 | if (decompressors[type] != NULL) { |
218 | // the compressor has already registered but the function might be null |
219 | return _func_from_offset(type, offset, discriminator); |
220 | } |
221 | |
222 | // does IOKit know about a kext that is supposed to provide this type? |
223 | char providesName[80]; |
224 | snprintf(providesName, count: sizeof(providesName), "com.apple.AppleFSCompression.providesType%u" , type); |
225 | if (IOCatalogueMatchingDriversPresent(property: providesName)) { |
226 | // there is a kext that says it will register for this type, so let's wait for it |
227 | char resourceName[80]; |
228 | uint64_t delay = 10000000ULL; // 10 milliseconds. |
229 | snprintf(resourceName, count: sizeof(resourceName), "com.apple.AppleFSCompression.Type%u" , type); |
230 | ErrorLogWithPath("waiting for %s\n" , resourceName); |
231 | while (decompressors[type] == NULL) { |
232 | lck_rw_unlock_shared(lck: &decompressorsLock); // we have to unlock to allow the kext to register |
233 | if (IOServiceWaitForMatchingResource(property: resourceName, timeout: delay)) { |
234 | lck_rw_lock_shared(lck: &decompressorsLock); |
235 | break; |
236 | } |
237 | if (!IOCatalogueMatchingDriversPresent(property: providesName)) { |
238 | // |
239 | ErrorLogWithPath("the kext with %s is no longer present\n" , providesName); |
240 | lck_rw_lock_shared(lck: &decompressorsLock); |
241 | break; |
242 | } |
243 | ErrorLogWithPath("still waiting for %s\n" , resourceName); |
244 | delay *= 2; |
245 | lck_rw_lock_shared(lck: &decompressorsLock); |
246 | } |
247 | // IOKit says the kext is loaded, so it should be registered too! |
248 | if (decompressors[type] == NULL) { |
249 | ErrorLogWithPath("we found %s, but the type still isn't registered\n" , providesName); |
250 | return NULL; |
251 | } |
252 | // it's now registered, so let's return the function |
253 | return _func_from_offset(type, offset, discriminator); |
254 | } |
255 | |
256 | // the compressor hasn't registered, so it never will unless someone manually kextloads it |
257 | ErrorLogWithPath("tried to access a compressed file of unregistered type %d\n" , type); |
258 | return NULL; |
259 | } |
260 | |
261 | #define decmp_get_func(vp, type, func) (typeof(decompressors[0]->func))_decmp_get_func(vp, type, offsetof_func(func), ptrauth_function_pointer_type_discriminator(typeof(decompressors[0]->func))) |
262 | |
263 | #pragma mark --- utilities --- |
264 | |
265 | #if COMPRESSION_DEBUG |
266 | static int |
267 | vnsize(vnode_t vp, uint64_t *size) |
268 | { |
269 | struct vnode_attr va; |
270 | VATTR_INIT(&va); |
271 | VATTR_WANTED(&va, va_data_size); |
272 | int error = vnode_getattr(vp, &va, decmpfs_ctx); |
273 | if (error != 0) { |
274 | ErrorLogWithPath("vnode_getattr err %d\n" , error); |
275 | return error; |
276 | } |
277 | *size = va.va_data_size; |
278 | return 0; |
279 | } |
280 | #endif /* COMPRESSION_DEBUG */ |
281 | |
282 | #pragma mark --- cnode routines --- |
283 | |
284 | ZONE_DEFINE(decmpfs_cnode_zone, "decmpfs_cnode" , |
285 | sizeof(struct decmpfs_cnode), ZC_NONE); |
286 | |
287 | decmpfs_cnode * |
288 | decmpfs_cnode_alloc(void) |
289 | { |
290 | return zalloc(zone: decmpfs_cnode_zone); |
291 | } |
292 | |
293 | void |
294 | decmpfs_cnode_free(decmpfs_cnode *dp) |
295 | { |
296 | zfree(decmpfs_cnode_zone, dp); |
297 | } |
298 | |
299 | void |
300 | decmpfs_cnode_init(decmpfs_cnode *cp) |
301 | { |
302 | memset(s: cp, c: 0, n: sizeof(*cp)); |
303 | lck_rw_init(lck: &cp->compressed_data_lock, grp: &decmpfs_lockgrp, NULL); |
304 | } |
305 | |
306 | void |
307 | decmpfs_cnode_destroy(decmpfs_cnode *cp) |
308 | { |
309 | lck_rw_destroy(lck: &cp->compressed_data_lock, grp: &decmpfs_lockgrp); |
310 | } |
311 | |
312 | bool |
313 | decmpfs_trylock_compressed_data(decmpfs_cnode *cp, int exclusive) |
314 | { |
315 | void *thread = current_thread(); |
316 | bool retval = false; |
317 | |
318 | if (cp->lockowner == thread) { |
319 | /* this thread is already holding an exclusive lock, so bump the count */ |
320 | cp->lockcount++; |
321 | retval = true; |
322 | } else if (exclusive) { |
323 | if ((retval = lck_rw_try_lock_exclusive(lck: &cp->compressed_data_lock))) { |
324 | cp->lockowner = thread; |
325 | cp->lockcount = 1; |
326 | } |
327 | } else { |
328 | if ((retval = lck_rw_try_lock_shared(lck: &cp->compressed_data_lock))) { |
329 | cp->lockowner = (void *)-1; |
330 | } |
331 | } |
332 | return retval; |
333 | } |
334 | |
335 | void |
336 | decmpfs_lock_compressed_data(decmpfs_cnode *cp, int exclusive) |
337 | { |
338 | void *thread = current_thread(); |
339 | |
340 | if (cp->lockowner == thread) { |
341 | /* this thread is already holding an exclusive lock, so bump the count */ |
342 | cp->lockcount++; |
343 | } else if (exclusive) { |
344 | lck_rw_lock_exclusive(lck: &cp->compressed_data_lock); |
345 | cp->lockowner = thread; |
346 | cp->lockcount = 1; |
347 | } else { |
348 | lck_rw_lock_shared(lck: &cp->compressed_data_lock); |
349 | cp->lockowner = (void *)-1; |
350 | } |
351 | } |
352 | |
353 | void |
354 | decmpfs_unlock_compressed_data(decmpfs_cnode *cp, __unused int exclusive) |
355 | { |
356 | void *thread = current_thread(); |
357 | |
358 | if (cp->lockowner == thread) { |
359 | /* this thread is holding an exclusive lock, so decrement the count */ |
360 | if ((--cp->lockcount) > 0) { |
361 | /* the caller still has outstanding locks, so we're done */ |
362 | return; |
363 | } |
364 | cp->lockowner = NULL; |
365 | } |
366 | |
367 | lck_rw_done(lck: &cp->compressed_data_lock); |
368 | } |
369 | |
370 | uint32_t |
371 | decmpfs_cnode_get_vnode_state(decmpfs_cnode *cp) |
372 | { |
373 | return cp->cmp_state; |
374 | } |
375 | |
376 | void |
377 | decmpfs_cnode_set_vnode_state(decmpfs_cnode *cp, uint32_t state, int skiplock) |
378 | { |
379 | if (!skiplock) { |
380 | decmpfs_lock_compressed_data(cp, exclusive: 1); |
381 | } |
382 | cp->cmp_state = (uint8_t)state; |
383 | if (state == FILE_TYPE_UNKNOWN) { |
384 | /* clear out the compression type too */ |
385 | cp->cmp_type = 0; |
386 | } |
387 | if (!skiplock) { |
388 | decmpfs_unlock_compressed_data(cp, exclusive: 1); |
389 | } |
390 | } |
391 | |
392 | static void |
393 | decmpfs_cnode_set_vnode_cmp_type(decmpfs_cnode *cp, uint32_t cmp_type, int skiplock) |
394 | { |
395 | if (!skiplock) { |
396 | decmpfs_lock_compressed_data(cp, exclusive: 1); |
397 | } |
398 | cp->cmp_type = cmp_type; |
399 | if (!skiplock) { |
400 | decmpfs_unlock_compressed_data(cp, exclusive: 1); |
401 | } |
402 | } |
403 | |
404 | static void |
405 | decmpfs_cnode_set_vnode_minimal_xattr(decmpfs_cnode *cp, int minimal_xattr, int skiplock) |
406 | { |
407 | if (!skiplock) { |
408 | decmpfs_lock_compressed_data(cp, exclusive: 1); |
409 | } |
410 | cp->cmp_minimal_xattr = !!minimal_xattr; |
411 | if (!skiplock) { |
412 | decmpfs_unlock_compressed_data(cp, exclusive: 1); |
413 | } |
414 | } |
415 | |
416 | uint64_t |
417 | decmpfs_cnode_get_vnode_cached_size(decmpfs_cnode *cp) |
418 | { |
419 | return cp->uncompressed_size; |
420 | } |
421 | |
422 | uint64_t |
423 | decmpfs_cnode_get_vnode_cached_nchildren(decmpfs_cnode *cp) |
424 | { |
425 | return cp->nchildren; |
426 | } |
427 | |
428 | uint64_t |
429 | decmpfs_cnode_get_vnode_cached_total_size(decmpfs_cnode *cp) |
430 | { |
431 | return cp->total_size; |
432 | } |
433 | |
434 | void |
435 | decmpfs_cnode_set_vnode_cached_size(decmpfs_cnode *cp, uint64_t size) |
436 | { |
437 | while (1) { |
438 | uint64_t old = cp->uncompressed_size; |
439 | if (OSCompareAndSwap64(old, size, (UInt64*)&cp->uncompressed_size)) { |
440 | return; |
441 | } else { |
442 | /* failed to write our value, so loop */ |
443 | } |
444 | } |
445 | } |
446 | |
447 | void |
448 | decmpfs_cnode_set_vnode_cached_nchildren(decmpfs_cnode *cp, uint64_t nchildren) |
449 | { |
450 | while (1) { |
451 | uint64_t old = cp->nchildren; |
452 | if (OSCompareAndSwap64(old, nchildren, (UInt64*)&cp->nchildren)) { |
453 | return; |
454 | } else { |
455 | /* failed to write our value, so loop */ |
456 | } |
457 | } |
458 | } |
459 | |
460 | void |
461 | decmpfs_cnode_set_vnode_cached_total_size(decmpfs_cnode *cp, uint64_t total_sz) |
462 | { |
463 | while (1) { |
464 | uint64_t old = cp->total_size; |
465 | if (OSCompareAndSwap64(old, total_sz, (UInt64*)&cp->total_size)) { |
466 | return; |
467 | } else { |
468 | /* failed to write our value, so loop */ |
469 | } |
470 | } |
471 | } |
472 | |
473 | static uint64_t |
474 | decmpfs_cnode_get_decompression_flags(decmpfs_cnode *cp) |
475 | { |
476 | return cp->decompression_flags; |
477 | } |
478 | |
479 | static void |
480 | decmpfs_cnode_set_decompression_flags(decmpfs_cnode *cp, uint64_t flags) |
481 | { |
482 | while (1) { |
483 | uint64_t old = cp->decompression_flags; |
484 | if (OSCompareAndSwap64(old, flags, (UInt64*)&cp->decompression_flags)) { |
485 | return; |
486 | } else { |
487 | /* failed to write our value, so loop */ |
488 | } |
489 | } |
490 | } |
491 | |
492 | uint32_t |
493 | decmpfs_cnode_cmp_type(decmpfs_cnode *cp) |
494 | { |
495 | return cp->cmp_type; |
496 | } |
497 | |
498 | #pragma mark --- decmpfs state routines --- |
499 | |
500 | static int |
501 | (vnode_t vp, decmpfs_cnode *cp, decmpfs_header **hdrOut, int returnInvalid, size_t *hdr_size) |
502 | { |
503 | /* |
504 | * fetches vp's compression xattr, converting it into a decmpfs_header; returns 0 or errno |
505 | * if returnInvalid == 1, returns the header even if the type was invalid (out of range), |
506 | * and return ERANGE in that case |
507 | */ |
508 | |
509 | size_t read_size = 0; |
510 | size_t attr_size = 0; |
511 | size_t alloc_size = 0; |
512 | uio_t attr_uio = NULL; |
513 | int err = 0; |
514 | char *data = NULL; |
515 | const bool no_additional_data = ((cp != NULL) |
516 | && (cp->cmp_type != 0) |
517 | && (cp->cmp_minimal_xattr != 0)); |
518 | UIO_STACKBUF(uio_buf, 1); |
519 | decmpfs_header *hdr = NULL; |
520 | |
521 | /* |
522 | * Trace the following parameters on entry with event-id 0x03120004 |
523 | * |
524 | * @vp->v_id: vnode-id for which to fetch compressed header. |
525 | * @no_additional_data: If set true then xattr didn't have any extra data. |
526 | * @returnInvalid: return the header even though the type is out of range. |
527 | */ |
528 | DECMPFS_EMIT_TRACE_ENTRY(DECMPDBG_FETCH_COMPRESSED_HEADER, vp->v_id, |
529 | no_additional_data, returnInvalid); |
530 | |
531 | if (no_additional_data) { |
532 | /* this file's xattr didn't have any extra data when we fetched it, so we can synthesize a header from the data in the cnode */ |
533 | |
534 | alloc_size = sizeof(decmpfs_header); |
535 | data = kalloc_data(sizeof(decmpfs_header), Z_WAITOK); |
536 | if (!data) { |
537 | err = ENOMEM; |
538 | goto out; |
539 | } |
540 | hdr = (decmpfs_header*)data; |
541 | hdr->attr_size = sizeof(decmpfs_disk_header); |
542 | hdr->compression_magic = DECMPFS_MAGIC; |
543 | hdr->compression_type = cp->cmp_type; |
544 | if (hdr->compression_type == DATALESS_PKG_CMPFS_TYPE) { |
545 | if (!vnode_isdir(vp)) { |
546 | err = EINVAL; |
547 | goto out; |
548 | } |
549 | hdr->_size.value = DECMPFS_PKG_VALUE_FROM_SIZE_COUNT( |
550 | decmpfs_cnode_get_vnode_cached_size(cp), |
551 | decmpfs_cnode_get_vnode_cached_nchildren(cp)); |
552 | } else if (vnode_isdir(vp)) { |
553 | hdr->_size.value = decmpfs_cnode_get_vnode_cached_nchildren(cp); |
554 | } else { |
555 | hdr->_size.value = decmpfs_cnode_get_vnode_cached_size(cp); |
556 | } |
557 | } else { |
558 | /* figure out how big the xattr is on disk */ |
559 | err = vn_getxattr(vp, DECMPFS_XATTR_NAME, NULL, &attr_size, XATTR_NOSECURITY, decmpfs_ctx); |
560 | if (err != 0) { |
561 | goto out; |
562 | } |
563 | alloc_size = attr_size + sizeof(hdr->attr_size); |
564 | |
565 | if (attr_size < sizeof(decmpfs_disk_header) || attr_size > MAX_DECMPFS_XATTR_SIZE) { |
566 | err = EINVAL; |
567 | goto out; |
568 | } |
569 | |
570 | /* allocation includes space for the extra attr_size field of a compressed_header */ |
571 | data = (char *)kalloc_data(alloc_size, Z_WAITOK); |
572 | if (!data) { |
573 | err = ENOMEM; |
574 | goto out; |
575 | } |
576 | |
577 | /* read the xattr into our buffer, skipping over the attr_size field at the beginning */ |
578 | attr_uio = uio_createwithbuffer(a_iovcount: 1, a_offset: 0, a_spacetype: UIO_SYSSPACE, a_iodirection: UIO_READ, a_buf_p: &uio_buf[0], a_buffer_size: sizeof(uio_buf)); |
579 | uio_addiov(a_uio: attr_uio, CAST_USER_ADDR_T(data + sizeof(hdr->attr_size)), a_length: attr_size); |
580 | |
581 | err = vn_getxattr(vp, DECMPFS_XATTR_NAME, attr_uio, &read_size, XATTR_NOSECURITY, decmpfs_ctx); |
582 | if (err != 0) { |
583 | goto out; |
584 | } |
585 | if (read_size != attr_size) { |
586 | err = EINVAL; |
587 | goto out; |
588 | } |
589 | hdr = (decmpfs_header*)data; |
590 | hdr->attr_size = (uint32_t)attr_size; |
591 | /* swap the fields to native endian */ |
592 | hdr->compression_magic = OSSwapLittleToHostInt32(hdr->compression_magic); |
593 | hdr->compression_type = OSSwapLittleToHostInt32(hdr->compression_type); |
594 | hdr->uncompressed_size = OSSwapLittleToHostInt64(hdr->uncompressed_size); |
595 | } |
596 | |
597 | if (hdr->compression_magic != DECMPFS_MAGIC) { |
598 | ErrorLogWithPath("invalid compression_magic 0x%08x, should be 0x%08x\n" , hdr->compression_magic, DECMPFS_MAGIC); |
599 | err = EINVAL; |
600 | goto out; |
601 | } |
602 | |
603 | /* |
604 | * Special-case the DATALESS compressor here; that is a valid type, |
605 | * even through there will never be an entry in the decompressor |
606 | * handler table for it. If we don't do this, then the cmp_state |
607 | * for this cnode will end up being marked NOT_COMPRESSED, and |
608 | * we'll be stuck in limbo. |
609 | */ |
610 | if (hdr->compression_type >= CMP_MAX && !decmpfs_type_is_dataless(cmp_type: hdr->compression_type)) { |
611 | if (returnInvalid) { |
612 | /* return the header even though the type is out of range */ |
613 | err = ERANGE; |
614 | } else { |
615 | ErrorLogWithPath("compression_type %d out of range\n" , hdr->compression_type); |
616 | err = EINVAL; |
617 | } |
618 | goto out; |
619 | } |
620 | |
621 | out: |
622 | if (err && (err != ERANGE)) { |
623 | DebugLogWithPath("err %d\n" , err); |
624 | kfree_data(data, alloc_size); |
625 | *hdrOut = NULL; |
626 | } else { |
627 | *hdrOut = hdr; |
628 | *hdr_size = alloc_size; |
629 | } |
630 | /* |
631 | * Trace the following parameters on return with event-id 0x03120004. |
632 | * |
633 | * @vp->v_id: vnode-id for which to fetch compressed header. |
634 | * @err: value returned from this function. |
635 | */ |
636 | DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FETCH_COMPRESSED_HEADER, vp->v_id, err); |
637 | return err; |
638 | } |
639 | |
640 | static int |
641 | decmpfs_fast_get_state(decmpfs_cnode *cp) |
642 | { |
643 | /* |
644 | * return the cached state |
645 | * this should *only* be called when we know that decmpfs_file_is_compressed has already been called, |
646 | * because this implies that the cached state is valid |
647 | */ |
648 | int cmp_state = decmpfs_cnode_get_vnode_state(cp); |
649 | |
650 | switch (cmp_state) { |
651 | case FILE_IS_NOT_COMPRESSED: |
652 | case FILE_IS_COMPRESSED: |
653 | case FILE_IS_CONVERTING: |
654 | return cmp_state; |
655 | case FILE_TYPE_UNKNOWN: |
656 | /* |
657 | * we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode, |
658 | * which should not be possible |
659 | */ |
660 | ErrorLog("decmpfs_fast_get_state called on unknown file\n" ); |
661 | return FILE_IS_NOT_COMPRESSED; |
662 | default: |
663 | /* */ |
664 | ErrorLog("unknown cmp_state %d\n" , cmp_state); |
665 | return FILE_IS_NOT_COMPRESSED; |
666 | } |
667 | } |
668 | |
669 | static int |
670 | decmpfs_fast_file_is_compressed(decmpfs_cnode *cp) |
671 | { |
672 | int cmp_state = decmpfs_cnode_get_vnode_state(cp); |
673 | |
674 | switch (cmp_state) { |
675 | case FILE_IS_NOT_COMPRESSED: |
676 | return 0; |
677 | case FILE_IS_COMPRESSED: |
678 | case FILE_IS_CONVERTING: |
679 | return 1; |
680 | case FILE_TYPE_UNKNOWN: |
681 | /* |
682 | * we should only get here if decmpfs_file_is_compressed was not called earlier on this vnode, |
683 | * which should not be possible |
684 | */ |
685 | ErrorLog("decmpfs_fast_get_state called on unknown file\n" ); |
686 | return 0; |
687 | default: |
688 | /* */ |
689 | ErrorLog("unknown cmp_state %d\n" , cmp_state); |
690 | return 0; |
691 | } |
692 | } |
693 | |
694 | errno_t |
695 | decmpfs_validate_compressed_file(vnode_t vp, decmpfs_cnode *cp) |
696 | { |
697 | /* give a compressor a chance to indicate that a compressed file is invalid */ |
698 | decmpfs_header *hdr = NULL; |
699 | size_t alloc_size = 0; |
700 | errno_t err = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 0, hdr_size: &alloc_size); |
701 | |
702 | if (err) { |
703 | /* we couldn't get the header */ |
704 | if (decmpfs_fast_get_state(cp) == FILE_IS_NOT_COMPRESSED) { |
705 | /* the file is no longer compressed, so return success */ |
706 | err = 0; |
707 | } |
708 | goto out; |
709 | } |
710 | |
711 | if (!decmpfs_type_is_dataless(cmp_type: hdr->compression_type)) { |
712 | lck_rw_lock_shared(lck: &decompressorsLock); |
713 | decmpfs_validate_compressed_file_func validate = decmp_get_func(vp, hdr->compression_type, validate); |
714 | if (validate) { /* make sure this validation function is valid */ |
715 | /* is the data okay? */ |
716 | err = validate(vp, decmpfs_ctx, hdr); |
717 | } else if (decmp_get_func(vp, hdr->compression_type, fetch) == NULL) { |
718 | /* the type isn't registered */ |
719 | err = EIO; |
720 | } else { |
721 | /* no validate registered, so nothing to do */ |
722 | err = 0; |
723 | } |
724 | lck_rw_unlock_shared(lck: &decompressorsLock); |
725 | } |
726 | out: |
727 | if (hdr != NULL) { |
728 | kfree_data(hdr, alloc_size); |
729 | } |
730 | #if COMPRESSION_DEBUG |
731 | if (err) { |
732 | DebugLogWithPath("decmpfs_validate_compressed_file ret %d, vp->v_flag %d\n" , err, vp->v_flag); |
733 | } |
734 | #endif |
735 | return err; |
736 | } |
737 | |
738 | int |
739 | decmpfs_file_is_compressed(vnode_t vp, decmpfs_cnode *cp) |
740 | { |
741 | /* |
742 | * determines whether vp points to a compressed file |
743 | * |
744 | * to speed up this operation, we cache the result in the cnode, and do as little as possible |
745 | * in the case where the cnode already has a valid cached state |
746 | * |
747 | */ |
748 | |
749 | int ret = 0; |
750 | int error = 0; |
751 | uint32_t cmp_state; |
752 | struct vnode_attr va_fetch; |
753 | decmpfs_header *hdr = NULL; |
754 | size_t alloc_size = 0; |
755 | mount_t mp = NULL; |
756 | int cnode_locked = 0; |
757 | int saveInvalid = 0; // save the header data even though the type was out of range |
758 | uint64_t decompression_flags = 0; |
759 | bool is_mounted, is_local_fs; |
760 | |
761 | if (vnode_isnamedstream(vp)) { |
762 | /* |
763 | * named streams can't be compressed |
764 | * since named streams of the same file share the same cnode, |
765 | * we don't want to get/set the state in the cnode, just return 0 |
766 | */ |
767 | return 0; |
768 | } |
769 | |
770 | /* examine the cached a state in this cnode */ |
771 | cmp_state = decmpfs_cnode_get_vnode_state(cp); |
772 | switch (cmp_state) { |
773 | case FILE_IS_NOT_COMPRESSED: |
774 | return 0; |
775 | case FILE_IS_COMPRESSED: |
776 | return 1; |
777 | case FILE_IS_CONVERTING: |
778 | /* treat the file as compressed, because this gives us a way to block future reads until decompression is done */ |
779 | return 1; |
780 | case FILE_TYPE_UNKNOWN: |
781 | /* the first time we encountered this vnode, so we need to check it out */ |
782 | break; |
783 | default: |
784 | /* unknown state, assume file is not compressed */ |
785 | ErrorLogWithPath("unknown cmp_state %d\n" , cmp_state); |
786 | return 0; |
787 | } |
788 | |
789 | is_mounted = false; |
790 | is_local_fs = false; |
791 | mp = vnode_mount(vp); |
792 | if (mp) { |
793 | is_mounted = true; |
794 | } |
795 | if (is_mounted) { |
796 | is_local_fs = ((mp->mnt_flag & MNT_LOCAL)); |
797 | } |
798 | /* |
799 | * Trace the following parameters on entry with event-id 0x03120014. |
800 | * |
801 | * @vp->v_id: vnode-id of the file being queried. |
802 | * @is_mounted: set to true if @vp belongs to a mounted fs. |
803 | * @is_local_fs: set to true if @vp belongs to local fs. |
804 | */ |
805 | DECMPFS_EMIT_TRACE_ENTRY(DECMPDBG_FILE_IS_COMPRESSED, vp->v_id, |
806 | is_mounted, is_local_fs); |
807 | |
808 | if (!is_mounted) { |
809 | /* |
810 | * this should only be true before we mount the root filesystem |
811 | * we short-cut this return to avoid the call to getattr below, which |
812 | * will fail before root is mounted |
813 | */ |
814 | ret = FILE_IS_NOT_COMPRESSED; |
815 | goto done; |
816 | } |
817 | |
818 | if (!is_local_fs) { |
819 | /* compression only supported on local filesystems */ |
820 | ret = FILE_IS_NOT_COMPRESSED; |
821 | goto done; |
822 | } |
823 | |
824 | /* lock our cnode data so that another caller doesn't change the state under us */ |
825 | decmpfs_lock_compressed_data(cp, exclusive: 1); |
826 | cnode_locked = 1; |
827 | |
828 | VATTR_INIT(&va_fetch); |
829 | VATTR_WANTED(&va_fetch, va_flags); |
830 | error = vnode_getattr(vp, vap: &va_fetch, ctx: decmpfs_ctx); |
831 | if (error) { |
832 | /* failed to get the bsd flags so the file is not compressed */ |
833 | ret = FILE_IS_NOT_COMPRESSED; |
834 | goto done; |
835 | } |
836 | if (va_fetch.va_flags & UF_COMPRESSED) { |
837 | /* UF_COMPRESSED is on, make sure the file has the DECMPFS_XATTR_NAME xattr */ |
838 | error = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 1, hdr_size: &alloc_size); |
839 | if ((hdr != NULL) && (error == ERANGE)) { |
840 | saveInvalid = 1; |
841 | } |
842 | if (error) { |
843 | /* failed to get the xattr so the file is not compressed */ |
844 | ret = FILE_IS_NOT_COMPRESSED; |
845 | goto done; |
846 | } |
847 | /* |
848 | * We got the xattr, so the file is at least tagged compressed. |
849 | * For DATALESS, regular files and directories can be "compressed". |
850 | * For all other types, only files are allowed. |
851 | */ |
852 | if (!vnode_isreg(vp) && |
853 | !(decmpfs_type_is_dataless(cmp_type: hdr->compression_type) && vnode_isdir(vp))) { |
854 | ret = FILE_IS_NOT_COMPRESSED; |
855 | goto done; |
856 | } |
857 | ret = FILE_IS_COMPRESSED; |
858 | goto done; |
859 | } |
860 | /* UF_COMPRESSED isn't on, so the file isn't compressed */ |
861 | ret = FILE_IS_NOT_COMPRESSED; |
862 | |
863 | done: |
864 | if (((ret == FILE_IS_COMPRESSED) || saveInvalid) && hdr) { |
865 | /* |
866 | * cache the uncompressed size away in the cnode |
867 | */ |
868 | |
869 | if (!cnode_locked) { |
870 | /* |
871 | * we should never get here since the only place ret is set to FILE_IS_COMPRESSED |
872 | * is after the call to decmpfs_lock_compressed_data above |
873 | */ |
874 | decmpfs_lock_compressed_data(cp, exclusive: 1); |
875 | cnode_locked = 1; |
876 | } |
877 | |
878 | if (vnode_isdir(vp)) { |
879 | decmpfs_cnode_set_vnode_cached_size(cp, size: 64); |
880 | decmpfs_cnode_set_vnode_cached_nchildren(cp, nchildren: decmpfs_get_directory_entries(hdr)); |
881 | if (hdr->compression_type == DATALESS_PKG_CMPFS_TYPE) { |
882 | decmpfs_cnode_set_vnode_cached_total_size(cp, DECMPFS_PKG_SIZE(hdr->_size)); |
883 | } |
884 | } else { |
885 | decmpfs_cnode_set_vnode_cached_size(cp, size: hdr->uncompressed_size); |
886 | } |
887 | decmpfs_cnode_set_vnode_state(cp, state: ret, skiplock: 1); |
888 | decmpfs_cnode_set_vnode_cmp_type(cp, cmp_type: hdr->compression_type, skiplock: 1); |
889 | /* remember if the xattr's size was equal to the minimal xattr */ |
890 | if (hdr->attr_size == sizeof(decmpfs_disk_header)) { |
891 | decmpfs_cnode_set_vnode_minimal_xattr(cp, minimal_xattr: 1, skiplock: 1); |
892 | } |
893 | if (ret == FILE_IS_COMPRESSED) { |
894 | /* update the ubc's size for this file */ |
895 | ubc_setsize(vp, hdr->uncompressed_size); |
896 | |
897 | /* update the decompression flags in the decmpfs cnode */ |
898 | lck_rw_lock_shared(lck: &decompressorsLock); |
899 | decmpfs_get_decompression_flags_func get_flags = decmp_get_func(vp, hdr->compression_type, get_flags); |
900 | if (get_flags) { |
901 | decompression_flags = get_flags(vp, decmpfs_ctx, hdr); |
902 | } |
903 | lck_rw_unlock_shared(lck: &decompressorsLock); |
904 | decmpfs_cnode_set_decompression_flags(cp, flags: decompression_flags); |
905 | } |
906 | } else { |
907 | /* we might have already taken the lock above; if so, skip taking it again by passing cnode_locked as the skiplock parameter */ |
908 | decmpfs_cnode_set_vnode_state(cp, state: ret, skiplock: cnode_locked); |
909 | } |
910 | |
911 | if (cnode_locked) { |
912 | decmpfs_unlock_compressed_data(cp, exclusive: 1); |
913 | } |
914 | |
915 | if (hdr != NULL) { |
916 | kfree_data(hdr, alloc_size); |
917 | } |
918 | |
919 | /* |
920 | * Trace the following parameters on return with event-id 0x03120014. |
921 | * |
922 | * @vp->v_id: vnode-id of the file being queried. |
923 | * @return: set to 1 is file is compressed. |
924 | */ |
925 | switch (ret) { |
926 | case FILE_IS_NOT_COMPRESSED: |
927 | DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FILE_IS_COMPRESSED, vp->v_id, 0); |
928 | return 0; |
929 | case FILE_IS_COMPRESSED: |
930 | case FILE_IS_CONVERTING: |
931 | DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FILE_IS_COMPRESSED, vp->v_id, 1); |
932 | return 1; |
933 | default: |
934 | /* unknown state, assume file is not compressed */ |
935 | DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FILE_IS_COMPRESSED, vp->v_id, 0); |
936 | ErrorLogWithPath("unknown ret %d\n" , ret); |
937 | return 0; |
938 | } |
939 | } |
940 | |
941 | int |
942 | decmpfs_update_attributes(vnode_t vp, struct vnode_attr *vap) |
943 | { |
944 | int error = 0; |
945 | |
946 | if (VATTR_IS_ACTIVE(vap, va_flags)) { |
947 | /* the BSD flags are being updated */ |
948 | if (vap->va_flags & UF_COMPRESSED) { |
949 | /* the compressed bit is being set, did it change? */ |
950 | struct vnode_attr va_fetch; |
951 | int old_flags = 0; |
952 | VATTR_INIT(&va_fetch); |
953 | VATTR_WANTED(&va_fetch, va_flags); |
954 | error = vnode_getattr(vp, vap: &va_fetch, ctx: decmpfs_ctx); |
955 | if (error) { |
956 | return error; |
957 | } |
958 | |
959 | old_flags = va_fetch.va_flags; |
960 | |
961 | if (!(old_flags & UF_COMPRESSED)) { |
962 | /* |
963 | * Compression bit was turned on, make sure the file has the DECMPFS_XATTR_NAME attribute. |
964 | * This precludes anyone from using the UF_COMPRESSED bit for anything else, and it enforces |
965 | * an order of operation -- you must first do the setxattr and then the chflags. |
966 | */ |
967 | |
968 | if (VATTR_IS_ACTIVE(vap, va_data_size)) { |
969 | /* |
970 | * don't allow the caller to set the BSD flag and the size in the same call |
971 | * since this doesn't really make sense |
972 | */ |
973 | vap->va_flags &= ~UF_COMPRESSED; |
974 | return 0; |
975 | } |
976 | |
977 | decmpfs_header *hdr = NULL; |
978 | size_t alloc_size = 0; |
979 | error = decmpfs_fetch_compressed_header(vp, NULL, hdrOut: &hdr, returnInvalid: 1, hdr_size: &alloc_size); |
980 | if (error == 0) { |
981 | /* |
982 | * Allow the flag to be set since the decmpfs attribute |
983 | * is present. |
984 | * |
985 | * If we're creating a dataless file we do not want to |
986 | * truncate it to zero which allows the file resolver to |
987 | * have more control over when truncation should happen. |
988 | * All other types of compressed files are truncated to |
989 | * zero. |
990 | */ |
991 | if (!decmpfs_type_is_dataless(cmp_type: hdr->compression_type)) { |
992 | VATTR_SET_ACTIVE(vap, va_data_size); |
993 | vap->va_data_size = 0; |
994 | } |
995 | } else if (error == ERANGE) { |
996 | /* the file had a decmpfs attribute but the type was out of range, so don't muck with the file's data size */ |
997 | } else { |
998 | /* no DECMPFS_XATTR_NAME attribute, so deny the update */ |
999 | vap->va_flags &= ~UF_COMPRESSED; |
1000 | } |
1001 | if (hdr != NULL) { |
1002 | kfree_data(hdr, alloc_size); |
1003 | } |
1004 | } |
1005 | } |
1006 | } |
1007 | |
1008 | return 0; |
1009 | } |
1010 | |
1011 | static int |
1012 | wait_for_decompress(decmpfs_cnode *cp) |
1013 | { |
1014 | int state; |
1015 | lck_mtx_lock(lck: &decompress_channel_mtx); |
1016 | do { |
1017 | state = decmpfs_fast_get_state(cp); |
1018 | if (state != FILE_IS_CONVERTING) { |
1019 | /* file is not decompressing */ |
1020 | lck_mtx_unlock(lck: &decompress_channel_mtx); |
1021 | return state; |
1022 | } |
1023 | msleep(chan: (caddr_t)&decompress_channel, mtx: &decompress_channel_mtx, PINOD, wmesg: "wait_for_decompress" , NULL); |
1024 | } while (1); |
1025 | } |
1026 | |
1027 | #pragma mark --- decmpfs hide query routines --- |
1028 | |
1029 | int |
1030 | decmpfs_hides_rsrc(vfs_context_t ctx, decmpfs_cnode *cp) |
1031 | { |
1032 | /* |
1033 | * WARNING!!! |
1034 | * callers may (and do) pass NULL for ctx, so we should only use it |
1035 | * for this equality comparison |
1036 | * |
1037 | * This routine should only be called after a file has already been through decmpfs_file_is_compressed |
1038 | */ |
1039 | |
1040 | if (ctx == decmpfs_ctx) { |
1041 | return 0; |
1042 | } |
1043 | |
1044 | if (!decmpfs_fast_file_is_compressed(cp)) { |
1045 | return 0; |
1046 | } |
1047 | |
1048 | /* all compressed files hide their resource fork */ |
1049 | return 1; |
1050 | } |
1051 | |
1052 | int |
1053 | decmpfs_hides_xattr(vfs_context_t ctx, decmpfs_cnode *cp, const char *xattr) |
1054 | { |
1055 | /* |
1056 | * WARNING!!! |
1057 | * callers may (and do) pass NULL for ctx, so we should only use it |
1058 | * for this equality comparison |
1059 | * |
1060 | * This routine should only be called after a file has already been through decmpfs_file_is_compressed |
1061 | */ |
1062 | |
1063 | if (ctx == decmpfs_ctx) { |
1064 | return 0; |
1065 | } |
1066 | if (strncmp(s1: xattr, XATTR_RESOURCEFORK_NAME, n: sizeof(XATTR_RESOURCEFORK_NAME) - 1) == 0) { |
1067 | return decmpfs_hides_rsrc(ctx, cp); |
1068 | } |
1069 | if (!decmpfs_fast_file_is_compressed(cp)) { |
1070 | /* file is not compressed, so don't hide this xattr */ |
1071 | return 0; |
1072 | } |
1073 | if (strncmp(s1: xattr, DECMPFS_XATTR_NAME, n: sizeof(DECMPFS_XATTR_NAME) - 1) == 0) { |
1074 | /* it's our xattr, so hide it */ |
1075 | return 1; |
1076 | } |
1077 | /* don't hide this xattr */ |
1078 | return 0; |
1079 | } |
1080 | |
1081 | #pragma mark --- registration/validation routines --- |
1082 | |
1083 | static inline int |
1084 | registration_valid(const decmpfs_registration *registration) |
1085 | { |
1086 | return registration && ((registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V1) || (registration->decmpfs_registration == DECMPFS_REGISTRATION_VERSION_V3)); |
1087 | } |
1088 | |
1089 | errno_t |
1090 | register_decmpfs_decompressor(uint32_t compression_type, const decmpfs_registration *registration) |
1091 | { |
1092 | /* called by kexts to register decompressors */ |
1093 | |
1094 | errno_t ret = 0; |
1095 | int locked = 0; |
1096 | char resourceName[80]; |
1097 | |
1098 | if ((compression_type >= CMP_MAX) || !registration_valid(registration)) { |
1099 | ret = EINVAL; |
1100 | goto out; |
1101 | } |
1102 | |
1103 | lck_rw_lock_exclusive(lck: &decompressorsLock); locked = 1; |
1104 | |
1105 | /* make sure the registration for this type is zero */ |
1106 | if (decompressors[compression_type] != NULL) { |
1107 | ret = EEXIST; |
1108 | goto out; |
1109 | } |
1110 | decompressors[compression_type] = registration; |
1111 | snprintf(resourceName, count: sizeof(resourceName), "com.apple.AppleFSCompression.Type%u" , compression_type); |
1112 | IOServicePublishResource(property: resourceName, TRUE); |
1113 | |
1114 | out: |
1115 | if (locked) { |
1116 | lck_rw_unlock_exclusive(lck: &decompressorsLock); |
1117 | } |
1118 | return ret; |
1119 | } |
1120 | |
1121 | errno_t |
1122 | unregister_decmpfs_decompressor(uint32_t compression_type, decmpfs_registration *registration) |
1123 | { |
1124 | /* called by kexts to unregister decompressors */ |
1125 | |
1126 | errno_t ret = 0; |
1127 | int locked = 0; |
1128 | char resourceName[80]; |
1129 | |
1130 | if ((compression_type >= CMP_MAX) || !registration_valid(registration)) { |
1131 | ret = EINVAL; |
1132 | goto out; |
1133 | } |
1134 | |
1135 | lck_rw_lock_exclusive(lck: &decompressorsLock); locked = 1; |
1136 | if (decompressors[compression_type] != registration) { |
1137 | ret = EEXIST; |
1138 | goto out; |
1139 | } |
1140 | decompressors[compression_type] = NULL; |
1141 | snprintf(resourceName, count: sizeof(resourceName), "com.apple.AppleFSCompression.Type%u" , compression_type); |
1142 | IOServicePublishResource(property: resourceName, FALSE); |
1143 | |
1144 | out: |
1145 | if (locked) { |
1146 | lck_rw_unlock_exclusive(lck: &decompressorsLock); |
1147 | } |
1148 | return ret; |
1149 | } |
1150 | |
1151 | static int |
1152 | compression_type_valid(vnode_t vp, decmpfs_header *hdr) |
1153 | { |
1154 | /* fast pre-check to determine if the given compressor has checked in */ |
1155 | int ret = 0; |
1156 | |
1157 | /* every compressor must have at least a fetch function */ |
1158 | lck_rw_lock_shared(lck: &decompressorsLock); |
1159 | if (decmp_get_func(vp, hdr->compression_type, fetch) != NULL) { |
1160 | ret = 1; |
1161 | } |
1162 | lck_rw_unlock_shared(lck: &decompressorsLock); |
1163 | |
1164 | return ret; |
1165 | } |
1166 | |
1167 | #pragma mark --- compression/decompression routines --- |
1168 | |
1169 | static int |
1170 | decmpfs_fetch_uncompressed_data(vnode_t vp, decmpfs_cnode *cp, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read) |
1171 | { |
1172 | /* get the uncompressed bytes for the specified region of vp by calling out to the registered compressor */ |
1173 | |
1174 | int err = 0; |
1175 | |
1176 | *bytes_read = 0; |
1177 | |
1178 | if (offset >= (off_t)hdr->uncompressed_size) { |
1179 | /* reading past end of file; nothing to do */ |
1180 | err = 0; |
1181 | goto out; |
1182 | } |
1183 | if (offset < 0) { |
1184 | /* tried to read from before start of file */ |
1185 | err = EINVAL; |
1186 | goto out; |
1187 | } |
1188 | if (hdr->uncompressed_size - offset < size) { |
1189 | /* adjust size so we don't read past the end of the file */ |
1190 | size = (user_ssize_t)(hdr->uncompressed_size - offset); |
1191 | } |
1192 | if (size == 0) { |
1193 | /* nothing to read */ |
1194 | err = 0; |
1195 | goto out; |
1196 | } |
1197 | |
1198 | /* |
1199 | * Trace the following parameters on entry with event-id 0x03120008. |
1200 | * |
1201 | * @vp->v_id: vnode-id of the file being decompressed. |
1202 | * @hdr->compression_type: compression type. |
1203 | * @offset: offset from where to fetch uncompressed data. |
1204 | * @size: amount of uncompressed data to fetch. |
1205 | * |
1206 | * Please NOTE: @offset and @size can overflow in theory but |
1207 | * here it is safe. |
1208 | */ |
1209 | DECMPFS_EMIT_TRACE_ENTRY(DECMPDBG_FETCH_UNCOMPRESSED_DATA, vp->v_id, |
1210 | hdr->compression_type, (int)offset, (int)size); |
1211 | lck_rw_lock_shared(lck: &decompressorsLock); |
1212 | decmpfs_fetch_uncompressed_data_func fetch = decmp_get_func(vp, hdr->compression_type, fetch); |
1213 | if (fetch) { |
1214 | err = fetch(vp, decmpfs_ctx, hdr, offset, size, nvec, vec, bytes_read); |
1215 | lck_rw_unlock_shared(lck: &decompressorsLock); |
1216 | if (err == 0) { |
1217 | uint64_t decompression_flags = decmpfs_cnode_get_decompression_flags(cp); |
1218 | if (decompression_flags & DECMPFS_FLAGS_FORCE_FLUSH_ON_DECOMPRESS) { |
1219 | #if !defined(__i386__) && !defined(__x86_64__) |
1220 | int i; |
1221 | for (i = 0; i < nvec; i++) { |
1222 | assert(vec[i].size >= 0 && vec[i].size <= UINT_MAX); |
1223 | flush_dcache64((addr64_t)(uintptr_t)vec[i].buf, (unsigned int)vec[i].size, FALSE); |
1224 | } |
1225 | #endif |
1226 | } |
1227 | } else { |
1228 | decmpfs_ktriage_record(KTRIAGE_DECMPFS_FETCH_CALLBACK_FAILED, err); |
1229 | } |
1230 | } else { |
1231 | decmpfs_ktriage_record(KTRIAGE_DECMPFS_COMPRESSOR_NOT_REGISTERED, hdr->compression_type); |
1232 | err = ENOTSUP; |
1233 | lck_rw_unlock_shared(lck: &decompressorsLock); |
1234 | } |
1235 | /* |
1236 | * Trace the following parameters on return with event-id 0x03120008. |
1237 | * |
1238 | * @vp->v_id: vnode-id of the file being decompressed. |
1239 | * @bytes_read: amount of uncompressed bytes fetched in bytes. |
1240 | * @err: value returned from this function. |
1241 | * |
1242 | * Please NOTE: @bytes_read can overflow in theory but here it is safe. |
1243 | */ |
1244 | DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FETCH_UNCOMPRESSED_DATA, vp->v_id, |
1245 | (int)*bytes_read, err); |
1246 | out: |
1247 | return err; |
1248 | } |
1249 | |
1250 | static kern_return_t |
1251 | commit_upl(upl_t upl, upl_offset_t pl_offset, size_t uplSize, int flags, int abort) |
1252 | { |
1253 | kern_return_t kr = 0; |
1254 | |
1255 | #if CONFIG_IOSCHED |
1256 | upl_unmark_decmp(upl); |
1257 | #endif /* CONFIG_IOSCHED */ |
1258 | |
1259 | /* commit the upl pages */ |
1260 | if (abort) { |
1261 | VerboseLog("aborting upl, flags 0x%08x\n" , flags); |
1262 | kr = ubc_upl_abort_range(upl, pl_offset, (upl_size_t)uplSize, flags); |
1263 | if (kr != KERN_SUCCESS) { |
1264 | ErrorLog("ubc_upl_abort_range error %d\n" , (int)kr); |
1265 | } |
1266 | } else { |
1267 | VerboseLog("committing upl, flags 0x%08x\n" , flags | UPL_COMMIT_CLEAR_DIRTY); |
1268 | kr = ubc_upl_commit_range(upl, pl_offset, (upl_size_t)uplSize, flags | UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_WRITTEN_BY_KERNEL); |
1269 | if (kr != KERN_SUCCESS) { |
1270 | ErrorLog("ubc_upl_commit_range error %d\n" , (int)kr); |
1271 | } |
1272 | } |
1273 | return kr; |
1274 | } |
1275 | |
1276 | |
1277 | errno_t |
1278 | decmpfs_pagein_compressed(struct vnop_pagein_args *ap, int *is_compressed, decmpfs_cnode *cp) |
1279 | { |
1280 | /* handles a page-in request from vfs for a compressed file */ |
1281 | |
1282 | int err = 0; |
1283 | vnode_t vp = ap->a_vp; |
1284 | upl_t pl = ap->a_pl; |
1285 | upl_offset_t pl_offset = ap->a_pl_offset; |
1286 | off_t f_offset = ap->a_f_offset; |
1287 | size_t size = ap->a_size; |
1288 | int flags = ap->a_flags; |
1289 | off_t uplPos = 0; |
1290 | user_ssize_t uplSize = 0; |
1291 | user_ssize_t rounded_uplSize = 0; |
1292 | size_t verify_block_size = 0; |
1293 | void *data = NULL; |
1294 | decmpfs_header *hdr = NULL; |
1295 | size_t alloc_size = 0; |
1296 | uint64_t cachedSize = 0; |
1297 | uint32_t fs_bsize = 0; |
1298 | int cmpdata_locked = 0; |
1299 | int num_valid_pages = 0; |
1300 | int num_invalid_pages = 0; |
1301 | bool file_tail_page_valid = false; |
1302 | |
1303 | if (!decmpfs_trylock_compressed_data(cp, exclusive: 0)) { |
1304 | return EAGAIN; |
1305 | } |
1306 | cmpdata_locked = 1; |
1307 | |
1308 | |
1309 | if (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD)) { |
1310 | DebugLogWithPath("pagein: unknown flags 0x%08x\n" , (flags & ~(UPL_IOSYNC | UPL_NOCOMMIT | UPL_NORDAHEAD))); |
1311 | } |
1312 | |
1313 | err = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 0, hdr_size: &alloc_size); |
1314 | if (err != 0) { |
1315 | decmpfs_ktriage_record(KTRIAGE_DECMPFS_FETCH_HEADER_FAILED, err); |
1316 | goto out; |
1317 | } |
1318 | |
1319 | cachedSize = hdr->uncompressed_size; |
1320 | |
1321 | if (!compression_type_valid(vp, hdr)) { |
1322 | /* compressor not registered */ |
1323 | decmpfs_ktriage_record(KTRIAGE_DECMPFS_COMPRESSOR_NOT_REGISTERED, hdr->compression_type); |
1324 | err = ENOTSUP; |
1325 | goto out; |
1326 | } |
1327 | |
1328 | /* |
1329 | * can't page-in from a negative offset |
1330 | * or if we're starting beyond the EOF |
1331 | * or if the file offset isn't page aligned |
1332 | * or the size requested isn't a multiple of PAGE_SIZE |
1333 | */ |
1334 | if (f_offset < 0 || f_offset >= cachedSize || |
1335 | (f_offset & PAGE_MASK_64) || (size & PAGE_MASK) || (pl_offset & PAGE_MASK)) { |
1336 | decmpfs_ktriage_record(KTRIAGE_DECMPFS_IVALID_OFFSET, 0); |
1337 | err = EINVAL; |
1338 | goto out; |
1339 | } |
1340 | |
1341 | /* |
1342 | * If the verify block size is larger than the page size, the UPL needs |
1343 | * to be aligned to it, Since the UPL has been created by the filesystem, |
1344 | * we will only check if the passed in UPL length conforms to the |
1345 | * alignment requirements. |
1346 | */ |
1347 | err = VNOP_VERIFY(vp, f_offset, NULL, 0, &verify_block_size, NULL, |
1348 | VNODE_VERIFY_DEFAULT, NULL); |
1349 | if (err) { |
1350 | ErrorLogWithPath("VNOP_VERIFY returned error = %d\n" , err); |
1351 | goto out; |
1352 | } else if (verify_block_size) { |
1353 | if (vp->v_mount->mnt_vfsstat.f_bsize > PAGE_SIZE) { |
1354 | fs_bsize = vp->v_mount->mnt_vfsstat.f_bsize; |
1355 | } |
1356 | if (verify_block_size & (verify_block_size - 1)) { |
1357 | ErrorLogWithPath("verify block size (%zu) is not power of 2, no verification will be done\n" , verify_block_size); |
1358 | err = EINVAL; |
1359 | } else if (size % verify_block_size) { |
1360 | ErrorLogWithPath("upl size (%zu) is not a multiple of verify block size (%zu)\n" , (size_t)size, verify_block_size); |
1361 | err = EINVAL; |
1362 | } else if (fs_bsize) { |
1363 | /* |
1364 | * Filesystems requesting verification have to provide |
1365 | * values for block sizes which are powers of 2. |
1366 | */ |
1367 | if (fs_bsize & (fs_bsize - 1)) { |
1368 | ErrorLogWithPath("FS block size (%u) is greater than PAGE_SIZE (%d) and is not power of 2, no verification will be done\n" , |
1369 | fs_bsize, PAGE_SIZE); |
1370 | err = EINVAL; |
1371 | } else if (fs_bsize > verify_block_size) { |
1372 | ErrorLogWithPath("FS block size (%u) is greater than verify block size (%zu), no verification will be done\n" , |
1373 | fs_bsize, verify_block_size); |
1374 | err = EINVAL; |
1375 | } |
1376 | } |
1377 | if (err) { |
1378 | goto out; |
1379 | } |
1380 | } |
1381 | |
1382 | #if CONFIG_IOSCHED |
1383 | /* Mark the UPL as the requesting UPL for decompression */ |
1384 | upl_mark_decmp(upl: pl); |
1385 | #endif /* CONFIG_IOSCHED */ |
1386 | |
1387 | /* map the upl so we can fetch into it */ |
1388 | kern_return_t kr = ubc_upl_map(pl, (vm_offset_t*)&data); |
1389 | if ((kr != KERN_SUCCESS) || (data == NULL)) { |
1390 | decmpfs_ktriage_record(KTRIAGE_DECMPFS_UBC_UPL_MAP_FAILED, kr); |
1391 | err = ENOSPC; |
1392 | data = NULL; |
1393 | #if CONFIG_IOSCHED |
1394 | upl_unmark_decmp(upl: pl); |
1395 | #endif /* CONFIG_IOSCHED */ |
1396 | goto out; |
1397 | } |
1398 | |
1399 | uplPos = f_offset; |
1400 | off_t max_size = cachedSize - f_offset; |
1401 | |
1402 | if (size < max_size) { |
1403 | rounded_uplSize = uplSize = size; |
1404 | file_tail_page_valid = true; |
1405 | } else { |
1406 | uplSize = (user_ssize_t)max_size; |
1407 | if (fs_bsize) { |
1408 | /* First round up to fs_bsize */ |
1409 | rounded_uplSize = (uplSize + (fs_bsize - 1)) & ~(fs_bsize - 1); |
1410 | /* then to PAGE_SIZE */ |
1411 | rounded_uplSize = MIN(size, round_page((vm_offset_t)rounded_uplSize)); |
1412 | } else { |
1413 | rounded_uplSize = round_page(x: (vm_offset_t)uplSize); |
1414 | } |
1415 | } |
1416 | |
1417 | /* do the fetch */ |
1418 | decmpfs_vector vec; |
1419 | |
1420 | decompress: |
1421 | /* the mapped data pointer points to the first page of the page list, so we want to start filling in at an offset of pl_offset */ |
1422 | vec = (decmpfs_vector) { |
1423 | .buf = (char*)data + pl_offset, |
1424 | .size = size, |
1425 | }; |
1426 | |
1427 | uint64_t did_read = 0; |
1428 | if (decmpfs_fast_get_state(cp) == FILE_IS_CONVERTING) { |
1429 | ErrorLogWithPath("unexpected pagein during decompress\n" ); |
1430 | /* |
1431 | * if the file is converting, this must be a recursive call to pagein from underneath a call to decmpfs_decompress_file; |
1432 | * pretend that it succeeded but don't do anything since we're just going to write over the pages anyway |
1433 | */ |
1434 | err = 0; |
1435 | } else { |
1436 | if (verify_block_size <= PAGE_SIZE) { |
1437 | err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset: uplPos, size: uplSize, nvec: 1, vec: &vec, bytes_read: &did_read); |
1438 | /* zero out whatever wasn't read */ |
1439 | if (did_read < rounded_uplSize) { |
1440 | memset(s: (char*)vec.buf + did_read, c: 0, n: (size_t)(rounded_uplSize - did_read)); |
1441 | } |
1442 | } else { |
1443 | off_t l_uplPos = uplPos; |
1444 | off_t l_pl_offset = pl_offset; |
1445 | user_ssize_t l_uplSize = uplSize; |
1446 | upl_page_info_t *pl_info = ubc_upl_pageinfo(pl); |
1447 | |
1448 | err = 0; |
1449 | /* |
1450 | * When the system page size is less than the "verify block size", |
1451 | * the UPL passed may not consist solely of absent pages. |
1452 | * We have to detect the "absent" pages and only decompress |
1453 | * into those absent/invalid page ranges. |
1454 | * |
1455 | * Things that will change in each iteration of the loop : |
1456 | * |
1457 | * l_pl_offset = where we are inside the UPL [0, caller_upl_created_size) |
1458 | * l_uplPos = the file offset the l_pl_offset corresponds to. |
1459 | * l_uplSize = the size of the upl still unprocessed; |
1460 | * |
1461 | * In this picture, we have to do the transfer on 2 ranges |
1462 | * (One 2 page range and one 3 page range) and the loop |
1463 | * below will skip the first two pages and then identify |
1464 | * the next two as invalid and fill those in and |
1465 | * then skip the next one and then do the last pages. |
1466 | * |
1467 | * uplPos(file_offset) |
1468 | * | uplSize |
1469 | * 0 V<--------------> file_size |
1470 | * |---------------------------------------------------> |
1471 | * | | |V|V|I|I|V|I|I|I| |
1472 | * ^ |
1473 | * | upl |
1474 | * <-------------------> |
1475 | * | |
1476 | * pl_offset |
1477 | * |
1478 | * uplSize will be clipped in case the UPL range exceeds |
1479 | * the file size. |
1480 | * |
1481 | */ |
1482 | while (l_uplSize) { |
1483 | uint64_t l_did_read = 0; |
1484 | int pl_offset_pg = (int)(l_pl_offset / PAGE_SIZE); |
1485 | int pages_left_in_upl; |
1486 | int start_pg; |
1487 | int last_pg; |
1488 | |
1489 | /* |
1490 | * l_uplSize may start off less than the size of the upl, |
1491 | * we have to round it up to PAGE_SIZE to calculate |
1492 | * how many more pages are left. |
1493 | */ |
1494 | pages_left_in_upl = (int)(round_page(x: (vm_offset_t)l_uplSize) / PAGE_SIZE); |
1495 | |
1496 | /* |
1497 | * scan from the beginning of the upl looking for the first |
1498 | * non-valid page.... this will become the first page in |
1499 | * the request we're going to make to |
1500 | * 'decmpfs_fetch_uncompressed_data'... if all |
1501 | * of the pages are valid, we won't call through |
1502 | * to 'decmpfs_fetch_uncompressed_data' |
1503 | */ |
1504 | for (start_pg = 0; start_pg < pages_left_in_upl; start_pg++) { |
1505 | if (!upl_valid_page(upl: pl_info, index: pl_offset_pg + start_pg)) { |
1506 | break; |
1507 | } |
1508 | } |
1509 | |
1510 | num_valid_pages += start_pg; |
1511 | |
1512 | /* |
1513 | * scan from the starting invalid page looking for |
1514 | * a valid page before the end of the upl is |
1515 | * reached, if we find one, then it will be the |
1516 | * last page of the request to 'decmpfs_fetch_uncompressed_data' |
1517 | */ |
1518 | for (last_pg = start_pg; last_pg < pages_left_in_upl; last_pg++) { |
1519 | if (upl_valid_page(upl: pl_info, index: pl_offset_pg + last_pg)) { |
1520 | break; |
1521 | } |
1522 | } |
1523 | |
1524 | if (start_pg < last_pg) { |
1525 | off_t inval_offset = start_pg * PAGE_SIZE; |
1526 | int inval_pages = last_pg - start_pg; |
1527 | int inval_size = inval_pages * PAGE_SIZE; |
1528 | decmpfs_vector l_vec; |
1529 | |
1530 | num_invalid_pages += inval_pages; |
1531 | if (inval_offset) { |
1532 | did_read += inval_offset; |
1533 | l_pl_offset += inval_offset; |
1534 | l_uplPos += inval_offset; |
1535 | l_uplSize -= inval_offset; |
1536 | } |
1537 | |
1538 | l_vec = (decmpfs_vector) { |
1539 | .buf = (char*)data + l_pl_offset, |
1540 | .size = inval_size, |
1541 | }; |
1542 | |
1543 | err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset: l_uplPos, |
1544 | MIN(l_uplSize, inval_size), nvec: 1, vec: &l_vec, bytes_read: &l_did_read); |
1545 | |
1546 | if (!err && (l_did_read != inval_size) && (l_uplSize > inval_size)) { |
1547 | ErrorLogWithPath("Unexpected size fetch of decompressed data, l_uplSize = %d, l_did_read = %d, inval_size = %d\n" , |
1548 | (int)l_uplSize, (int)l_did_read, (int)inval_size); |
1549 | err = EINVAL; |
1550 | } |
1551 | } else { |
1552 | /* no invalid pages left */ |
1553 | l_did_read = l_uplSize; |
1554 | if (!file_tail_page_valid) { |
1555 | file_tail_page_valid = true; |
1556 | } |
1557 | } |
1558 | |
1559 | if (err) { |
1560 | break; |
1561 | } |
1562 | |
1563 | did_read += l_did_read; |
1564 | l_pl_offset += l_did_read; |
1565 | l_uplPos += l_did_read; |
1566 | l_uplSize -= l_did_read; |
1567 | } |
1568 | |
1569 | /* Zero out the region after EOF in the last page (if needed) */ |
1570 | if (!err && !file_tail_page_valid && (uplSize < rounded_uplSize)) { |
1571 | memset(s: (char*)vec.buf + uplSize, c: 0, n: (size_t)(rounded_uplSize - uplSize)); |
1572 | } |
1573 | } |
1574 | } |
1575 | if (err) { |
1576 | decmpfs_ktriage_record(KTRIAGE_DECMPFS_FETCH_UNCOMPRESSED_DATA_FAILED, err) |
1577 | DebugLogWithPath("decmpfs_fetch_uncompressed_data err %d\n" , err); |
1578 | int cmp_state = decmpfs_fast_get_state(cp); |
1579 | if (cmp_state == FILE_IS_CONVERTING) { |
1580 | DebugLogWithPath("cmp_state == FILE_IS_CONVERTING\n" ); |
1581 | cmp_state = wait_for_decompress(cp); |
1582 | if (cmp_state == FILE_IS_COMPRESSED) { |
1583 | DebugLogWithPath("cmp_state == FILE_IS_COMPRESSED\n" ); |
1584 | /* a decompress was attempted but it failed, let's try calling fetch again */ |
1585 | goto decompress; |
1586 | } |
1587 | } |
1588 | if (cmp_state == FILE_IS_NOT_COMPRESSED) { |
1589 | DebugLogWithPath("cmp_state == FILE_IS_NOT_COMPRESSED\n" ); |
1590 | /* the file was decompressed after we started reading it */ |
1591 | *is_compressed = 0; /* instruct caller to fall back to its normal path */ |
1592 | } |
1593 | } |
1594 | |
1595 | if (!err && verify_block_size) { |
1596 | size_t cur_verify_block_size = verify_block_size; |
1597 | |
1598 | if ((err = VNOP_VERIFY(vp, uplPos, vec.buf, rounded_uplSize, &cur_verify_block_size, NULL, 0, NULL))) { |
1599 | ErrorLogWithPath("Verification failed with error %d, uplPos = %lld, uplSize = %d, did_read = %d, valid_pages = %d, invalid_pages = %d, tail_page_valid = %d\n" , |
1600 | err, (long long)uplPos, (int)rounded_uplSize, (int)did_read, num_valid_pages, num_invalid_pages, file_tail_page_valid); |
1601 | } |
1602 | /* XXX : If the verify block size changes, redo the read */ |
1603 | } |
1604 | |
1605 | #if CONFIG_IOSCHED |
1606 | upl_unmark_decmp(upl: pl); |
1607 | #endif /* CONFIG_IOSCHED */ |
1608 | |
1609 | kr = ubc_upl_unmap(pl); data = NULL; /* make sure to set data to NULL so we don't try to unmap again below */ |
1610 | if (kr != KERN_SUCCESS) { |
1611 | ErrorLogWithPath("ubc_upl_unmap error %d\n" , (int)kr); |
1612 | } else { |
1613 | if (!err) { |
1614 | /* commit our pages */ |
1615 | kr = commit_upl(upl: pl, pl_offset, uplSize: (size_t)rounded_uplSize, UPL_COMMIT_FREE_ON_EMPTY, abort: 0 /* commit */); |
1616 | /* If there were any pages after the page containing EOF, abort them. */ |
1617 | if (rounded_uplSize < size) { |
1618 | kr = commit_upl(upl: pl, pl_offset: (upl_offset_t)(pl_offset + rounded_uplSize), uplSize: (size_t)(size - rounded_uplSize), |
1619 | UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR, abort: 1 /* abort */); |
1620 | } |
1621 | } |
1622 | } |
1623 | |
1624 | out: |
1625 | if (data) { |
1626 | ubc_upl_unmap(pl); |
1627 | } |
1628 | if (hdr != NULL) { |
1629 | kfree_data(hdr, alloc_size); |
1630 | } |
1631 | if (cmpdata_locked) { |
1632 | decmpfs_unlock_compressed_data(cp, exclusive: 0); |
1633 | } |
1634 | if (err) { |
1635 | #if 0 |
1636 | if (err != ENXIO && err != ENOSPC) { |
1637 | char *path = zalloc(ZV_NAMEI); |
1638 | panic("%s: decmpfs_pagein_compressed: err %d" , vnpath(vp, path, PATH_MAX), err); |
1639 | zfree(ZV_NAMEI, path); |
1640 | } |
1641 | #endif /* 0 */ |
1642 | ErrorLogWithPath("err %d\n" , err); |
1643 | } |
1644 | return err; |
1645 | } |
1646 | |
1647 | errno_t |
1648 | decmpfs_read_compressed(struct vnop_read_args *ap, int *is_compressed, decmpfs_cnode *cp) |
1649 | { |
1650 | /* handles a read request from vfs for a compressed file */ |
1651 | |
1652 | uio_t uio = ap->a_uio; |
1653 | vnode_t vp = ap->a_vp; |
1654 | int err = 0; |
1655 | int countInt = 0; |
1656 | off_t uplPos = 0; |
1657 | user_ssize_t uplSize = 0; |
1658 | user_ssize_t uplRemaining = 0; |
1659 | off_t curUplPos = 0; |
1660 | user_ssize_t curUplSize = 0; |
1661 | kern_return_t kr = KERN_SUCCESS; |
1662 | int abort_read = 0; |
1663 | void *data = NULL; |
1664 | uint64_t did_read = 0; |
1665 | upl_t upl = NULL; |
1666 | upl_page_info_t *pli = NULL; |
1667 | decmpfs_header *hdr = NULL; |
1668 | size_t alloc_size = 0; |
1669 | uint64_t cachedSize = 0; |
1670 | off_t uioPos = 0; |
1671 | user_ssize_t uioRemaining = 0; |
1672 | size_t verify_block_size = 0; |
1673 | size_t alignment_size = PAGE_SIZE; |
1674 | int cmpdata_locked = 0; |
1675 | |
1676 | decmpfs_lock_compressed_data(cp, exclusive: 0); cmpdata_locked = 1; |
1677 | |
1678 | uplPos = uio_offset(a_uio: uio); |
1679 | uplSize = uio_resid(a_uio: uio); |
1680 | VerboseLogWithPath("uplPos %lld uplSize %lld\n" , uplPos, uplSize); |
1681 | |
1682 | cachedSize = decmpfs_cnode_get_vnode_cached_size(cp); |
1683 | |
1684 | if ((uint64_t)uplPos + uplSize > cachedSize) { |
1685 | /* truncate the read to the size of the file */ |
1686 | uplSize = (user_ssize_t)(cachedSize - uplPos); |
1687 | } |
1688 | |
1689 | /* give the cluster layer a chance to fill in whatever it already has */ |
1690 | countInt = (uplSize > INT_MAX) ? INT_MAX : (int)uplSize; |
1691 | err = cluster_copy_ubc_data(vp, uio, &countInt, 0); |
1692 | if (err != 0) { |
1693 | goto out; |
1694 | } |
1695 | |
1696 | /* figure out what's left */ |
1697 | uioPos = uio_offset(a_uio: uio); |
1698 | uioRemaining = uio_resid(a_uio: uio); |
1699 | if ((uint64_t)uioPos + uioRemaining > cachedSize) { |
1700 | /* truncate the read to the size of the file */ |
1701 | uioRemaining = (user_ssize_t)(cachedSize - uioPos); |
1702 | } |
1703 | |
1704 | if (uioRemaining <= 0) { |
1705 | /* nothing left */ |
1706 | goto out; |
1707 | } |
1708 | |
1709 | err = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 0, hdr_size: &alloc_size); |
1710 | if (err != 0) { |
1711 | goto out; |
1712 | } |
1713 | if (!compression_type_valid(vp, hdr)) { |
1714 | err = ENOTSUP; |
1715 | goto out; |
1716 | } |
1717 | |
1718 | uplPos = uioPos; |
1719 | uplSize = uioRemaining; |
1720 | #if COMPRESSION_DEBUG |
1721 | DebugLogWithPath("uplPos %lld uplSize %lld\n" , (uint64_t)uplPos, (uint64_t)uplSize); |
1722 | #endif |
1723 | |
1724 | lck_rw_lock_shared(lck: &decompressorsLock); |
1725 | decmpfs_adjust_fetch_region_func adjust_fetch = decmp_get_func(vp, hdr->compression_type, adjust_fetch); |
1726 | if (adjust_fetch) { |
1727 | /* give the compressor a chance to adjust the portion of the file that we read */ |
1728 | adjust_fetch(vp, decmpfs_ctx, hdr, &uplPos, &uplSize); |
1729 | VerboseLogWithPath("adjusted uplPos %lld uplSize %lld\n" , (uint64_t)uplPos, (uint64_t)uplSize); |
1730 | } |
1731 | lck_rw_unlock_shared(lck: &decompressorsLock); |
1732 | |
1733 | /* clip the adjusted size to the size of the file */ |
1734 | if ((uint64_t)uplPos + uplSize > cachedSize) { |
1735 | /* truncate the read to the size of the file */ |
1736 | uplSize = (user_ssize_t)(cachedSize - uplPos); |
1737 | } |
1738 | |
1739 | if (uplSize <= 0) { |
1740 | /* nothing left */ |
1741 | goto out; |
1742 | } |
1743 | |
1744 | /* |
1745 | * since we're going to create a upl for the given region of the file, |
1746 | * make sure we're on page boundaries |
1747 | */ |
1748 | |
1749 | /* If the verify block size is larger than the page size, the UPL needs to aligned to it */ |
1750 | err = VNOP_VERIFY(vp, uplPos, NULL, 0, &verify_block_size, NULL, VNODE_VERIFY_DEFAULT, NULL); |
1751 | if (err) { |
1752 | goto out; |
1753 | } else if (verify_block_size) { |
1754 | if (verify_block_size & (verify_block_size - 1)) { |
1755 | ErrorLogWithPath("verify block size is not power of 2, no verification will be done\n" ); |
1756 | verify_block_size = 0; |
1757 | } else if (verify_block_size > PAGE_SIZE) { |
1758 | alignment_size = verify_block_size; |
1759 | } |
1760 | } |
1761 | |
1762 | if (uplPos & (alignment_size - 1)) { |
1763 | /* round position down to page boundary */ |
1764 | uplSize += (uplPos & (alignment_size - 1)); |
1765 | uplPos &= ~(alignment_size - 1); |
1766 | } |
1767 | |
1768 | /* round size up to alignement_size multiple */ |
1769 | uplSize = (uplSize + (alignment_size - 1)) & ~(alignment_size - 1); |
1770 | |
1771 | VerboseLogWithPath("new uplPos %lld uplSize %lld\n" , (uint64_t)uplPos, (uint64_t)uplSize); |
1772 | |
1773 | uplRemaining = uplSize; |
1774 | curUplPos = uplPos; |
1775 | curUplSize = 0; |
1776 | |
1777 | while (uplRemaining > 0) { |
1778 | /* start after the last upl */ |
1779 | curUplPos += curUplSize; |
1780 | |
1781 | /* clip to max upl size */ |
1782 | curUplSize = uplRemaining; |
1783 | if (curUplSize > MAX_UPL_SIZE_BYTES) { |
1784 | curUplSize = MAX_UPL_SIZE_BYTES; |
1785 | } |
1786 | |
1787 | /* create the upl */ |
1788 | kr = ubc_create_upl_kernel(vp, curUplPos, (int)curUplSize, &upl, &pli, UPL_SET_LITE, VM_KERN_MEMORY_FILE); |
1789 | if (kr != KERN_SUCCESS) { |
1790 | ErrorLogWithPath("ubc_create_upl error %d\n" , (int)kr); |
1791 | err = EINVAL; |
1792 | goto out; |
1793 | } |
1794 | VerboseLogWithPath("curUplPos %lld curUplSize %lld\n" , (uint64_t)curUplPos, (uint64_t)curUplSize); |
1795 | |
1796 | #if CONFIG_IOSCHED |
1797 | /* Mark the UPL as the requesting UPL for decompression */ |
1798 | upl_mark_decmp(upl); |
1799 | #endif /* CONFIG_IOSCHED */ |
1800 | |
1801 | /* map the upl */ |
1802 | kr = ubc_upl_map(upl, (vm_offset_t*)&data); |
1803 | if (kr != KERN_SUCCESS) { |
1804 | commit_upl(upl, pl_offset: 0, uplSize: curUplSize, UPL_ABORT_FREE_ON_EMPTY, abort: 1); |
1805 | #if 0 |
1806 | char *path = zalloc(ZV_NAMEI); |
1807 | panic("%s: decmpfs_read_compressed: ubc_upl_map error %d" , vnpath(vp, path, PATH_MAX), (int)kr); |
1808 | zfree(ZV_NAMEI, path); |
1809 | #else /* 0 */ |
1810 | ErrorLogWithPath("ubc_upl_map kr=0x%x\n" , (int)kr); |
1811 | #endif /* 0 */ |
1812 | err = EINVAL; |
1813 | goto out; |
1814 | } |
1815 | |
1816 | /* make sure the map succeeded */ |
1817 | if (!data) { |
1818 | commit_upl(upl, pl_offset: 0, uplSize: curUplSize, UPL_ABORT_FREE_ON_EMPTY, abort: 1); |
1819 | |
1820 | ErrorLogWithPath("ubc_upl_map mapped null\n" ); |
1821 | err = EINVAL; |
1822 | goto out; |
1823 | } |
1824 | |
1825 | /* fetch uncompressed data into the mapped upl */ |
1826 | decmpfs_vector vec; |
1827 | decompress: |
1828 | vec = (decmpfs_vector){ .buf = data, .size = curUplSize }; |
1829 | err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset: curUplPos, size: curUplSize, nvec: 1, vec: &vec, bytes_read: &did_read); |
1830 | if (err) { |
1831 | ErrorLogWithPath("decmpfs_fetch_uncompressed_data err %d\n" , err); |
1832 | |
1833 | /* maybe the file is converting to decompressed */ |
1834 | int cmp_state = decmpfs_fast_get_state(cp); |
1835 | if (cmp_state == FILE_IS_CONVERTING) { |
1836 | ErrorLogWithPath("cmp_state == FILE_IS_CONVERTING\n" ); |
1837 | cmp_state = wait_for_decompress(cp); |
1838 | if (cmp_state == FILE_IS_COMPRESSED) { |
1839 | ErrorLogWithPath("cmp_state == FILE_IS_COMPRESSED\n" ); |
1840 | /* a decompress was attempted but it failed, let's try fetching again */ |
1841 | goto decompress; |
1842 | } |
1843 | } |
1844 | if (cmp_state == FILE_IS_NOT_COMPRESSED) { |
1845 | ErrorLogWithPath("cmp_state == FILE_IS_NOT_COMPRESSED\n" ); |
1846 | /* the file was decompressed after we started reading it */ |
1847 | abort_read = 1; /* we're not going to commit our data */ |
1848 | *is_compressed = 0; /* instruct caller to fall back to its normal path */ |
1849 | } |
1850 | kr = KERN_FAILURE; |
1851 | did_read = 0; |
1852 | } |
1853 | |
1854 | /* zero out the remainder of the last page */ |
1855 | memset(s: (char*)data + did_read, c: 0, n: (size_t)(curUplSize - did_read)); |
1856 | if (!err && verify_block_size) { |
1857 | size_t cur_verify_block_size = verify_block_size; |
1858 | |
1859 | if ((err = VNOP_VERIFY(vp, curUplPos, data, curUplSize, &cur_verify_block_size, NULL, 0, NULL))) { |
1860 | ErrorLogWithPath("Verification failed with error %d\n" , err); |
1861 | abort_read = 1; |
1862 | } |
1863 | /* XXX : If the verify block size changes, redo the read */ |
1864 | } |
1865 | |
1866 | kr = ubc_upl_unmap(upl); |
1867 | if (kr == KERN_SUCCESS) { |
1868 | if (abort_read) { |
1869 | kr = commit_upl(upl, pl_offset: 0, uplSize: curUplSize, UPL_ABORT_FREE_ON_EMPTY, abort: 1); |
1870 | } else { |
1871 | VerboseLogWithPath("uioPos %lld uioRemaining %lld\n" , (uint64_t)uioPos, (uint64_t)uioRemaining); |
1872 | if (uioRemaining) { |
1873 | off_t uplOff = uioPos - curUplPos; |
1874 | if (uplOff < 0) { |
1875 | ErrorLogWithPath("uplOff %lld should never be negative\n" , (int64_t)uplOff); |
1876 | err = EINVAL; |
1877 | } else if (uplOff > INT_MAX) { |
1878 | ErrorLogWithPath("uplOff %lld too large\n" , (int64_t)uplOff); |
1879 | err = EINVAL; |
1880 | } else { |
1881 | off_t count = curUplPos + curUplSize - uioPos; |
1882 | if (count < 0) { |
1883 | /* this upl is entirely before the uio */ |
1884 | } else { |
1885 | if (count > uioRemaining) { |
1886 | count = uioRemaining; |
1887 | } |
1888 | int icount = (count > INT_MAX) ? INT_MAX : (int)count; |
1889 | int io_resid = icount; |
1890 | err = cluster_copy_upl_data(uio, upl, (int)uplOff, &io_resid); |
1891 | int copied = icount - io_resid; |
1892 | VerboseLogWithPath("uplOff %lld count %lld copied %lld\n" , (uint64_t)uplOff, (uint64_t)count, (uint64_t)copied); |
1893 | if (err) { |
1894 | ErrorLogWithPath("cluster_copy_upl_data err %d\n" , err); |
1895 | } |
1896 | uioPos += copied; |
1897 | uioRemaining -= copied; |
1898 | } |
1899 | } |
1900 | } |
1901 | kr = commit_upl(upl, pl_offset: 0, uplSize: curUplSize, UPL_COMMIT_FREE_ON_EMPTY | UPL_COMMIT_INACTIVATE, abort: 0); |
1902 | if (err) { |
1903 | goto out; |
1904 | } |
1905 | } |
1906 | } else { |
1907 | ErrorLogWithPath("ubc_upl_unmap error %d\n" , (int)kr); |
1908 | } |
1909 | |
1910 | uplRemaining -= curUplSize; |
1911 | } |
1912 | |
1913 | out: |
1914 | |
1915 | if (hdr != NULL) { |
1916 | kfree_data(hdr, alloc_size); |
1917 | } |
1918 | if (cmpdata_locked) { |
1919 | decmpfs_unlock_compressed_data(cp, exclusive: 0); |
1920 | } |
1921 | if (err) {/* something went wrong */ |
1922 | ErrorLogWithPath("err %d\n" , err); |
1923 | return err; |
1924 | } |
1925 | |
1926 | #if COMPRESSION_DEBUG |
1927 | uplSize = uio_resid(uio); |
1928 | if (uplSize) { |
1929 | VerboseLogWithPath("still %lld bytes to copy\n" , uplSize); |
1930 | } |
1931 | #endif |
1932 | return 0; |
1933 | } |
1934 | |
1935 | int |
1936 | decmpfs_free_compressed_data(vnode_t vp, decmpfs_cnode *cp) |
1937 | { |
1938 | /* |
1939 | * call out to the decompressor to free remove any data associated with this compressed file |
1940 | * then delete the file's compression xattr |
1941 | */ |
1942 | decmpfs_header *hdr = NULL; |
1943 | size_t alloc_size = 0; |
1944 | |
1945 | /* |
1946 | * Trace the following parameters on entry with event-id 0x03120010. |
1947 | * |
1948 | * @vp->v_id: vnode-id of the file for which to free compressed data. |
1949 | */ |
1950 | DECMPFS_EMIT_TRACE_ENTRY(DECMPDBG_FREE_COMPRESSED_DATA, vp->v_id); |
1951 | |
1952 | int err = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 0, hdr_size: &alloc_size); |
1953 | if (err) { |
1954 | ErrorLogWithPath("decmpfs_fetch_compressed_header err %d\n" , err); |
1955 | } else { |
1956 | lck_rw_lock_shared(lck: &decompressorsLock); |
1957 | decmpfs_free_compressed_data_func free_data = decmp_get_func(vp, hdr->compression_type, free_data); |
1958 | if (free_data) { |
1959 | err = free_data(vp, decmpfs_ctx, hdr); |
1960 | } else { |
1961 | /* nothing to do, so no error */ |
1962 | err = 0; |
1963 | } |
1964 | lck_rw_unlock_shared(lck: &decompressorsLock); |
1965 | |
1966 | if (err != 0) { |
1967 | ErrorLogWithPath("decompressor err %d\n" , err); |
1968 | } |
1969 | } |
1970 | /* |
1971 | * Trace the following parameters on return with event-id 0x03120010. |
1972 | * |
1973 | * @vp->v_id: vnode-id of the file for which to free compressed data. |
1974 | * @err: value returned from this function. |
1975 | */ |
1976 | DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_FREE_COMPRESSED_DATA, vp->v_id, err); |
1977 | |
1978 | /* delete the xattr */ |
1979 | err = vn_removexattr(vp, DECMPFS_XATTR_NAME, 0, decmpfs_ctx); |
1980 | |
1981 | if (hdr != NULL) { |
1982 | kfree_data(hdr, alloc_size); |
1983 | } |
1984 | return err; |
1985 | } |
1986 | |
1987 | #pragma mark --- file conversion routines --- |
1988 | |
1989 | static int |
1990 | unset_compressed_flag(vnode_t vp) |
1991 | { |
1992 | int err = 0; |
1993 | struct vnode_attr va; |
1994 | struct fsioc_cas_bsdflags cas; |
1995 | int i; |
1996 | |
1997 | # define MAX_CAS_BSDFLAGS_LOOPS 4 |
1998 | /* UF_COMPRESSED should be manipulated only with FSIOC_CAS_BSDFLAGS */ |
1999 | for (i = 0; i < MAX_CAS_BSDFLAGS_LOOPS; i++) { |
2000 | VATTR_INIT(&va); |
2001 | VATTR_WANTED(&va, va_flags); |
2002 | err = vnode_getattr(vp, vap: &va, ctx: decmpfs_ctx); |
2003 | if (err != 0) { |
2004 | ErrorLogWithPath("vnode_getattr err %d, num retries %d\n" , err, i); |
2005 | goto out; |
2006 | } |
2007 | |
2008 | cas.expected_flags = va.va_flags; |
2009 | cas.new_flags = va.va_flags & ~UF_COMPRESSED; |
2010 | err = VNOP_IOCTL(vp, FSIOC_CAS_BSDFLAGS, data: (caddr_t)&cas, FWRITE, ctx: decmpfs_ctx); |
2011 | |
2012 | if ((err == 0) && (va.va_flags == cas.actual_flags)) { |
2013 | goto out; |
2014 | } |
2015 | |
2016 | if ((err != 0) && (err != EAGAIN)) { |
2017 | break; |
2018 | } |
2019 | } |
2020 | |
2021 | /* fallback to regular chflags if FSIOC_CAS_BSDFLAGS is not supported */ |
2022 | if (err == ENOTTY) { |
2023 | VATTR_INIT(&va); |
2024 | VATTR_SET(&va, va_flags, cas.new_flags); |
2025 | err = vnode_setattr(vp, vap: &va, ctx: decmpfs_ctx); |
2026 | if (err != 0) { |
2027 | ErrorLogWithPath("vnode_setattr err %d\n" , err); |
2028 | } |
2029 | } else if (va.va_flags != cas.actual_flags) { |
2030 | ErrorLogWithPath("FSIOC_CAS_BSDFLAGS err: flags mismatc. actual (%x) expected (%x), num retries %d\n" , cas.actual_flags, va.va_flags, i); |
2031 | } else if (err != 0) { |
2032 | ErrorLogWithPath("FSIOC_CAS_BSDFLAGS err %d, num retries %d\n" , err, i); |
2033 | } |
2034 | |
2035 | out: |
2036 | return err; |
2037 | } |
2038 | |
2039 | int |
2040 | decmpfs_decompress_file(vnode_t vp, decmpfs_cnode *cp, off_t toSize, int truncate_okay, int skiplock) |
2041 | { |
2042 | /* convert a compressed file to an uncompressed file */ |
2043 | |
2044 | int err = 0; |
2045 | char *data = NULL; |
2046 | uio_t uio_w = 0; |
2047 | off_t offset = 0; |
2048 | uint32_t old_state = 0; |
2049 | uint32_t new_state = 0; |
2050 | int update_file_state = 0; |
2051 | size_t allocSize = 0; |
2052 | decmpfs_header *hdr = NULL; |
2053 | size_t hdr_size = 0; |
2054 | int cmpdata_locked = 0; |
2055 | off_t remaining = 0; |
2056 | uint64_t uncompressed_size = 0; |
2057 | |
2058 | /* |
2059 | * Trace the following parameters on entry with event-id 0x03120000. |
2060 | * |
2061 | * @vp->v_id: vnode-id of the file being decompressed. |
2062 | * @toSize: uncompress given bytes of the file. |
2063 | * @truncate_okay: on error it is OK to truncate. |
2064 | * @skiplock: compressed data is locked, skip locking again. |
2065 | * |
2066 | * Please NOTE: @toSize can overflow in theory but here it is safe. |
2067 | */ |
2068 | DECMPFS_EMIT_TRACE_ENTRY(DECMPDBG_DECOMPRESS_FILE, vp->v_id, |
2069 | (int)toSize, truncate_okay, skiplock); |
2070 | |
2071 | if (!skiplock) { |
2072 | decmpfs_lock_compressed_data(cp, exclusive: 1); cmpdata_locked = 1; |
2073 | } |
2074 | |
2075 | decompress: |
2076 | old_state = decmpfs_fast_get_state(cp); |
2077 | |
2078 | switch (old_state) { |
2079 | case FILE_IS_NOT_COMPRESSED: |
2080 | { |
2081 | /* someone else decompressed the file */ |
2082 | err = 0; |
2083 | goto out; |
2084 | } |
2085 | |
2086 | case FILE_TYPE_UNKNOWN: |
2087 | { |
2088 | /* the file is in an unknown state, so update the state and retry */ |
2089 | (void)decmpfs_file_is_compressed(vp, cp); |
2090 | |
2091 | /* try again */ |
2092 | goto decompress; |
2093 | } |
2094 | |
2095 | case FILE_IS_COMPRESSED: |
2096 | { |
2097 | /* the file is compressed, so decompress it */ |
2098 | break; |
2099 | } |
2100 | |
2101 | default: |
2102 | { |
2103 | /* |
2104 | * this shouldn't happen since multiple calls to decmpfs_decompress_file lock each other out, |
2105 | * and when decmpfs_decompress_file returns, the state should be always be set back to |
2106 | * FILE_IS_NOT_COMPRESSED or FILE_IS_UNKNOWN |
2107 | */ |
2108 | err = EINVAL; |
2109 | goto out; |
2110 | } |
2111 | } |
2112 | |
2113 | err = decmpfs_fetch_compressed_header(vp, cp, hdrOut: &hdr, returnInvalid: 0, hdr_size: &hdr_size); |
2114 | if (err != 0) { |
2115 | goto out; |
2116 | } |
2117 | |
2118 | uncompressed_size = hdr->uncompressed_size; |
2119 | if (toSize == -1) { |
2120 | toSize = hdr->uncompressed_size; |
2121 | } |
2122 | |
2123 | if (toSize == 0) { |
2124 | /* special case truncating the file to zero bytes */ |
2125 | goto nodecmp; |
2126 | } else if ((uint64_t)toSize > hdr->uncompressed_size) { |
2127 | /* the caller is trying to grow the file, so we should decompress all the data */ |
2128 | toSize = hdr->uncompressed_size; |
2129 | } |
2130 | |
2131 | allocSize = MIN(64 * 1024, (size_t)toSize); |
2132 | data = (char *)kalloc_data(allocSize, Z_WAITOK); |
2133 | if (!data) { |
2134 | err = ENOMEM; |
2135 | goto out; |
2136 | } |
2137 | |
2138 | uio_w = uio_create(a_iovcount: 1, a_offset: 0LL, a_spacetype: UIO_SYSSPACE, a_iodirection: UIO_WRITE); |
2139 | if (!uio_w) { |
2140 | err = ENOMEM; |
2141 | goto out; |
2142 | } |
2143 | uio_w->uio_flags |= UIO_FLAGS_IS_COMPRESSED_FILE; |
2144 | |
2145 | remaining = toSize; |
2146 | |
2147 | /* tell the buffer cache that this is an empty file */ |
2148 | ubc_setsize(vp, 0); |
2149 | |
2150 | /* if we got here, we need to decompress the file */ |
2151 | decmpfs_cnode_set_vnode_state(cp, state: FILE_IS_CONVERTING, skiplock: 1); |
2152 | |
2153 | while (remaining > 0) { |
2154 | /* loop decompressing data from the file and writing it into the data fork */ |
2155 | |
2156 | uint64_t bytes_read = 0; |
2157 | decmpfs_vector vec = { .buf = data, .size = (user_ssize_t)MIN(allocSize, remaining) }; |
2158 | err = decmpfs_fetch_uncompressed_data(vp, cp, hdr, offset, size: vec.size, nvec: 1, vec: &vec, bytes_read: &bytes_read); |
2159 | if (err != 0) { |
2160 | ErrorLogWithPath("decmpfs_fetch_uncompressed_data err %d\n" , err); |
2161 | goto out; |
2162 | } |
2163 | |
2164 | if (bytes_read == 0) { |
2165 | /* we're done reading data */ |
2166 | break; |
2167 | } |
2168 | |
2169 | uio_reset(a_uio: uio_w, a_offset: offset, a_spacetype: UIO_SYSSPACE, a_iodirection: UIO_WRITE); |
2170 | err = uio_addiov(a_uio: uio_w, CAST_USER_ADDR_T(data), a_length: (user_size_t)bytes_read); |
2171 | if (err != 0) { |
2172 | ErrorLogWithPath("uio_addiov err %d\n" , err); |
2173 | err = ENOMEM; |
2174 | goto out; |
2175 | } |
2176 | |
2177 | err = VNOP_WRITE(vp, uio: uio_w, ioflag: 0, ctx: decmpfs_ctx); |
2178 | if (err != 0) { |
2179 | /* if the write failed, truncate the file to zero bytes */ |
2180 | ErrorLogWithPath("VNOP_WRITE err %d\n" , err); |
2181 | break; |
2182 | } |
2183 | offset += bytes_read; |
2184 | remaining -= bytes_read; |
2185 | } |
2186 | |
2187 | if (err == 0) { |
2188 | if (offset != toSize) { |
2189 | ErrorLogWithPath("file decompressed to %lld instead of %lld\n" , offset, toSize); |
2190 | err = EINVAL; |
2191 | goto out; |
2192 | } |
2193 | } |
2194 | |
2195 | if (err == 0) { |
2196 | /* sync the data and metadata */ |
2197 | err = VNOP_FSYNC(vp, MNT_WAIT, ctx: decmpfs_ctx); |
2198 | if (err != 0) { |
2199 | ErrorLogWithPath("VNOP_FSYNC err %d\n" , err); |
2200 | goto out; |
2201 | } |
2202 | } |
2203 | |
2204 | if (err != 0) { |
2205 | /* write, setattr, or fsync failed */ |
2206 | ErrorLogWithPath("aborting decompress, err %d\n" , err); |
2207 | if (truncate_okay) { |
2208 | /* truncate anything we might have written */ |
2209 | int error = vnode_setsize(vp, 0, ioflag: 0, decmpfs_ctx); |
2210 | ErrorLogWithPath("vnode_setsize err %d\n" , error); |
2211 | } |
2212 | goto out; |
2213 | } |
2214 | |
2215 | nodecmp: |
2216 | /* if we're truncating the file to zero bytes, we'll skip ahead to here */ |
2217 | |
2218 | /* unset the compressed flag */ |
2219 | unset_compressed_flag(vp); |
2220 | |
2221 | /* free the compressed data associated with this file */ |
2222 | err = decmpfs_free_compressed_data(vp, cp); |
2223 | if (err != 0) { |
2224 | ErrorLogWithPath("decmpfs_free_compressed_data err %d\n" , err); |
2225 | } |
2226 | |
2227 | /* |
2228 | * even if free_compressed_data or vnode_getattr/vnode_setattr failed, return success |
2229 | * since we succeeded in writing all of the file data to the data fork |
2230 | */ |
2231 | err = 0; |
2232 | |
2233 | /* if we got this far, the file was successfully decompressed */ |
2234 | update_file_state = 1; |
2235 | new_state = FILE_IS_NOT_COMPRESSED; |
2236 | |
2237 | #if COMPRESSION_DEBUG |
2238 | { |
2239 | uint64_t filesize = 0; |
2240 | vnsize(vp, &filesize); |
2241 | DebugLogWithPath("new file size %lld\n" , filesize); |
2242 | } |
2243 | #endif |
2244 | |
2245 | out: |
2246 | if (hdr != NULL) { |
2247 | kfree_data(hdr, hdr_size); |
2248 | } |
2249 | kfree_data(data, allocSize); |
2250 | |
2251 | if (uio_w) { |
2252 | uio_free(a_uio: uio_w); |
2253 | } |
2254 | |
2255 | if (err != 0) { |
2256 | /* if there was a failure, reset compression flags to unknown and clear the buffer cache data */ |
2257 | update_file_state = 1; |
2258 | new_state = FILE_TYPE_UNKNOWN; |
2259 | if (uncompressed_size) { |
2260 | ubc_setsize(vp, 0); |
2261 | ubc_setsize(vp, uncompressed_size); |
2262 | } |
2263 | } |
2264 | |
2265 | if (update_file_state) { |
2266 | lck_mtx_lock(lck: &decompress_channel_mtx); |
2267 | decmpfs_cnode_set_vnode_state(cp, state: new_state, skiplock: 1); |
2268 | wakeup(chan: (caddr_t)&decompress_channel); /* wake up anyone who might have been waiting for decompression */ |
2269 | lck_mtx_unlock(lck: &decompress_channel_mtx); |
2270 | } |
2271 | |
2272 | if (cmpdata_locked) { |
2273 | decmpfs_unlock_compressed_data(cp, exclusive: 1); |
2274 | } |
2275 | /* |
2276 | * Trace the following parameters on return with event-id 0x03120000. |
2277 | * |
2278 | * @vp->v_id: vnode-id of the file being decompressed. |
2279 | * @err: value returned from this function. |
2280 | */ |
2281 | DECMPFS_EMIT_TRACE_RETURN(DECMPDBG_DECOMPRESS_FILE, vp->v_id, err); |
2282 | return err; |
2283 | } |
2284 | |
2285 | #pragma mark --- Type1 compressor --- |
2286 | |
2287 | /* |
2288 | * The "Type1" compressor stores the data fork directly in the compression xattr |
2289 | */ |
2290 | |
2291 | static int |
2292 | decmpfs_validate_compressed_file_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr) |
2293 | { |
2294 | int err = 0; |
2295 | |
2296 | if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) { |
2297 | err = EINVAL; |
2298 | goto out; |
2299 | } |
2300 | out: |
2301 | return err; |
2302 | } |
2303 | |
2304 | static int |
2305 | decmpfs_fetch_uncompressed_data_Type1(__unused vnode_t vp, __unused vfs_context_t ctx, decmpfs_header *hdr, off_t offset, user_ssize_t size, int nvec, decmpfs_vector *vec, uint64_t *bytes_read) |
2306 | { |
2307 | int err = 0; |
2308 | int i; |
2309 | user_ssize_t remaining; |
2310 | |
2311 | if (hdr->uncompressed_size + sizeof(decmpfs_disk_header) != (uint64_t)hdr->attr_size) { |
2312 | err = EINVAL; |
2313 | goto out; |
2314 | } |
2315 | |
2316 | #if COMPRESSION_DEBUG |
2317 | static int dummy = 0; // prevent syslog from coalescing printfs |
2318 | DebugLogWithPath("%d memcpy %lld at %lld\n" , dummy++, size, (uint64_t)offset); |
2319 | #endif |
2320 | |
2321 | remaining = size; |
2322 | for (i = 0; (i < nvec) && (remaining > 0); i++) { |
2323 | user_ssize_t curCopy = vec[i].size; |
2324 | if (curCopy > remaining) { |
2325 | curCopy = remaining; |
2326 | } |
2327 | memcpy(dst: vec[i].buf, src: hdr->attr_bytes + offset, n: curCopy); |
2328 | offset += curCopy; |
2329 | remaining -= curCopy; |
2330 | } |
2331 | |
2332 | if ((bytes_read) && (err == 0)) { |
2333 | *bytes_read = (size - remaining); |
2334 | } |
2335 | |
2336 | out: |
2337 | return err; |
2338 | } |
2339 | |
2340 | SECURITY_READ_ONLY_EARLY(static decmpfs_registration) Type1Reg = |
2341 | { |
2342 | .decmpfs_registration = DECMPFS_REGISTRATION_VERSION, |
2343 | .validate = decmpfs_validate_compressed_file_Type1, |
2344 | .adjust_fetch = NULL,/* no adjust necessary */ |
2345 | .fetch = decmpfs_fetch_uncompressed_data_Type1, |
2346 | .free_data = NULL,/* no free necessary */ |
2347 | .get_flags = NULL/* no flags */ |
2348 | }; |
2349 | |
2350 | #pragma mark --- decmpfs initialization --- |
2351 | |
2352 | void |
2353 | decmpfs_init(void) |
2354 | { |
2355 | static int done = 0; |
2356 | if (done) { |
2357 | return; |
2358 | } |
2359 | |
2360 | decmpfs_ctx = vfs_context_create(ctx: vfs_context_kernel()); |
2361 | |
2362 | register_decmpfs_decompressor(compression_type: CMP_Type1, registration: &Type1Reg); |
2363 | |
2364 | ktriage_register_subsystem_strings(KDBG_TRIAGE_SUBSYS_DECMPFS, subsystem_strings: &ktriage_decmpfs_subsystem_strings); |
2365 | |
2366 | done = 1; |
2367 | } |
2368 | #endif /* FS_COMPRESSION */ |
2369 | |