1 | /* |
2 | * Copyright (c) 2016-2016 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #pragma once |
30 | #include <stdint.h> |
31 | #include <stddef.h> |
32 | #include <kern/assert.h> |
33 | #include <machine/machlimits.h> |
34 | #include "lz4_assembly_select.h" |
35 | #include "lz4_constants.h" |
36 | |
37 | #define memcpy __builtin_memcpy |
38 | |
39 | #pragma mark - Building blocks |
40 | |
41 | // Represents a position in the input stream |
42 | typedef struct { uint32_t offset; uint32_t word; } lz4_hash_entry_t; |
43 | static const size_t lz4_hash_table_size = LZ4_COMPRESS_HASH_ENTRIES*sizeof(lz4_hash_entry_t); |
44 | |
45 | // Worker function for lz4 encode. Underlies both the buffer and stream encode operations. |
46 | // Performs lz4 encoding of up to 2gb of data, updates dst_ptr and src_ptr to point to the |
47 | // first byte of output and input that couldn't be completely processed, respectively. |
48 | // |
49 | // If skip_final_literals is 0, the entire src buffer is encoded, by emitting a final sequence of literals |
50 | // at the end of the compressed payload. |
51 | // |
52 | // If skip_final_literals is not 0, this final literal sequence is not emitted, and the src buffer is |
53 | // partially encoded (the length of this literal sequence varies). |
54 | extern void lz4_encode_2gb(uint8_t **dst_ptr, size_t dst_size, |
55 | const uint8_t **src_ptr, const uint8_t *src_begin, size_t src_size, |
56 | lz4_hash_entry_t hash_table[LZ4_COMPRESS_HASH_ENTRIES],int skip_final_literals); |
57 | |
58 | extern int lz4_decode(uint8_t **dst_ptr, uint8_t *dst_begin, uint8_t *dst_end, |
59 | const uint8_t **src_ptr, const uint8_t *src_end); |
60 | |
61 | #if LZ4_ENABLE_ASSEMBLY_DECODE |
62 | extern int lz4_decode_asm(uint8_t **dst_ptr, uint8_t *dst_begin, uint8_t *dst_end, |
63 | const uint8_t **src_ptr, const uint8_t *src_end); |
64 | #endif |
65 | |
66 | #pragma mark - Buffer interfaces |
67 | |
68 | static const size_t lz4_encode_scratch_size = lz4_hash_table_size; |
69 | static const size_t lz4_decode_scratch_size = 0; |
70 | |
71 | #pragma mark - Buffer interfaces (LZ4 RAW) |
72 | |
73 | size_t lz4raw_encode_buffer(uint8_t * __restrict dst_buffer, size_t dst_size, |
74 | const uint8_t * __restrict src_buffer, size_t src_size, |
75 | lz4_hash_entry_t hash_table[LZ4_COMPRESS_HASH_ENTRIES]); |
76 | |
77 | size_t lz4raw_decode_buffer(uint8_t * __restrict dst_buffer, size_t dst_size, |
78 | const uint8_t * __restrict src_buffer, size_t src_size, |
79 | void * __restrict work __attribute__((unused))); |
80 | |
81 | typedef __attribute__((__ext_vector_type__(8))) uint8_t vector_uchar8; |
82 | typedef __attribute__((__ext_vector_type__(16))) uint8_t vector_uchar16; |
83 | typedef __attribute__((__ext_vector_type__(32))) uint8_t vector_uchar32; |
84 | typedef __attribute__((__ext_vector_type__(64))) uint8_t vector_uchar64; |
85 | typedef __attribute__((__ext_vector_type__(16),__aligned__(1))) uint8_t packed_uchar16; |
86 | typedef __attribute__((__ext_vector_type__(32),__aligned__(1))) uint8_t packed_uchar32; |
87 | typedef __attribute__((__ext_vector_type__(64),__aligned__(1))) uint8_t packed_uchar64; |
88 | |
89 | typedef __attribute__((__ext_vector_type__(4))) uint16_t vector_ushort4; |
90 | typedef __attribute__((__ext_vector_type__(4),__aligned__(2))) uint16_t packed_ushort4; |
91 | |
92 | typedef __attribute__((__ext_vector_type__(2))) int32_t vector_int2; |
93 | typedef __attribute__((__ext_vector_type__(4))) int32_t vector_int4; |
94 | typedef __attribute__((__ext_vector_type__(8))) int32_t vector_int8; |
95 | |
96 | typedef __attribute__((__ext_vector_type__(4))) uint32_t vector_uint4; |
97 | |
98 | #define UTIL_FUNCTION static inline __attribute__((__always_inline__)) __attribute__((__overloadable__)) |
99 | |
100 | // Load N bytes from unaligned location PTR |
101 | UTIL_FUNCTION uint16_t load2(const void * ptr) { uint16_t data; memcpy(&data,ptr,sizeof data); return data; } |
102 | UTIL_FUNCTION uint32_t load4(const void * ptr) { uint32_t data; memcpy(&data,ptr,sizeof data); return data; } |
103 | UTIL_FUNCTION uint64_t load8(const void * ptr) { uint64_t data; memcpy(&data,ptr,sizeof data); return data; } |
104 | UTIL_FUNCTION vector_uchar16 load16(const void * ptr) { return (const vector_uchar16)*(const packed_uchar16 *)ptr; } |
105 | UTIL_FUNCTION vector_uchar32 load32(const void * ptr) { return (const vector_uchar32)*(const packed_uchar32 *)ptr; } |
106 | UTIL_FUNCTION vector_uchar64 load64(const void * ptr) { return (const vector_uchar64)*(const packed_uchar64 *)ptr; } |
107 | |
108 | // Store N bytes to unaligned location PTR |
109 | UTIL_FUNCTION void store2(void * ptr,uint16_t data) { memcpy(ptr,&data,sizeof data); } |
110 | UTIL_FUNCTION void store4(void * ptr,uint32_t data) { memcpy(ptr,&data,sizeof data); } |
111 | UTIL_FUNCTION void store8(void * ptr,uint64_t data) { memcpy(ptr,&data,sizeof data); } |
112 | UTIL_FUNCTION void store16(void * ptr,vector_uchar16 data) { *(packed_uchar16 *)ptr = (packed_uchar16)data; } |
113 | UTIL_FUNCTION void store32(void * ptr,vector_uchar32 data) { *(packed_uchar32 *)ptr = (packed_uchar32)data; } |
114 | UTIL_FUNCTION void store64(void * ptr,vector_uchar64 data) { *(packed_uchar64 *)ptr = (packed_uchar64)data; } |
115 | |
116 | // Load+Store N bytes from unaligned locations SRC to DST. No overlap allowed. |
117 | UTIL_FUNCTION void copy8(void * dst,const void * src) { store8(dst,load8(src)); } |
118 | UTIL_FUNCTION void copy16(void * dst,const void * src) { *(packed_uchar16 *)dst = *(const packed_uchar16 *)src; } |
119 | UTIL_FUNCTION void copy32(void * dst,const void * src) { *(packed_uchar32 *)dst = *(const packed_uchar32 *)src; } |
120 | |
121 | #undef memcpy |
122 | |