1 | /* |
2 | * Copyright (c) 2016 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_APACHE_LICENSE_HEADER_START@ |
5 | * |
6 | * Licensed under the Apache License, Version 2.0 (the "License"); |
7 | * you may not use this file except in compliance with the License. |
8 | * You may obtain a copy of the License at |
9 | * |
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
11 | * |
12 | * Unless required by applicable law or agreed to in writing, software |
13 | * distributed under the License is distributed on an "AS IS" BASIS, |
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
15 | * See the License for the specific language governing permissions and |
16 | * limitations under the License. |
17 | * |
18 | * @APPLE_APACHE_LICENSE_HEADER_END@ |
19 | */ |
20 | |
21 | #ifndef __FIREHOSE_CHUNK_PRIVATE__ |
22 | #define __FIREHOSE_CHUNK_PRIVATE__ |
23 | |
24 | #if KERNEL |
25 | #include <machine/atomic.h> |
26 | #endif |
27 | #include <stdatomic.h> |
28 | #include <sys/param.h> |
29 | #include <os/base.h> |
30 | #include "firehose_types_private.h" |
31 | #include "tracepoint_private.h" |
32 | |
33 | __BEGIN_DECLS |
34 | |
35 | #define FIREHOSE_CHUNK_SIZE 4096ul |
36 | |
37 | #define FIREHOSE_CHUNK_POS_ENTRY_OFFS_INC (1ULL << 0) |
38 | #define FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC (1ULL << 16) |
39 | #define FIREHOSE_CHUNK_POS_REFCNT_INC (1ULL << 32) |
40 | #define FIREHOSE_CHUNK_POS_FULL_BIT (1ULL << 56) |
41 | #define FIREHOSE_CHUNK_POS_USABLE_FOR_STREAM(pos, stream) \ |
42 | ((((pos).fcp_pos >> 48) & 0x1ff) == (uint16_t)stream) |
43 | |
44 | typedef union { |
45 | _Atomic(uint64_t) fcp_atomic_pos; |
46 | uint64_t fcp_pos; |
47 | struct { |
48 | uint16_t fcp_next_entry_offs; |
49 | uint16_t fcp_private_offs; |
50 | uint8_t fcp_refcnt; |
51 | uint8_t fcp_qos; |
52 | uint8_t fcp_stream; |
53 | uint8_t fcp_flag_full : 1; |
54 | uint8_t fcp_flag_io : 1; |
55 | uint8_t fcp_quarantined : 1; |
56 | uint8_t _fcp_flag_unused : 5; |
57 | }; |
58 | } firehose_chunk_pos_u; |
59 | |
60 | typedef struct firehose_chunk_s { |
61 | uint8_t fc_start[0]; |
62 | firehose_chunk_pos_u fc_pos; |
63 | uint64_t fc_timestamp; |
64 | uint8_t fc_data[FIREHOSE_CHUNK_SIZE - 8 - 8]; |
65 | } *firehose_chunk_t; |
66 | |
67 | typedef struct firehose_chunk_range_s { |
68 | uint16_t fcr_offset; // offset from the start of the chunk |
69 | uint16_t fcr_length; |
70 | } *firehose_chunk_range_t; |
71 | |
72 | #if defined(KERNEL) || defined(OS_FIREHOSE_SPI) |
73 | |
74 | OS_ALWAYS_INLINE |
75 | static inline bool |
76 | firehose_chunk_pos_fits(firehose_chunk_pos_u pos, uint16_t size) |
77 | { |
78 | return pos.fcp_next_entry_offs + size <= pos.fcp_private_offs; |
79 | } |
80 | |
81 | #define FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE (-1) |
82 | #define FIREHOSE_CHUNK_TRY_RESERVE_FAIL ( 0) |
83 | |
84 | OS_ALWAYS_INLINE |
85 | static inline long |
86 | firehose_chunk_tracepoint_try_reserve(firehose_chunk_t fc, uint64_t stamp, |
87 | firehose_stream_t stream, uint8_t qos, uint16_t pubsize, |
88 | uint16_t privsize, uint8_t **privptr) |
89 | { |
90 | const uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data); |
91 | firehose_chunk_pos_u orig, pos; |
92 | bool reservation_failed, stamp_delta_fits; |
93 | |
94 | stamp_delta_fits = ((stamp - fc->fc_timestamp) >> 48) == 0; |
95 | |
96 | // no acquire barrier because the returned space is written to only |
97 | os_atomic_rmw_loop(&fc->fc_pos.fcp_atomic_pos, |
98 | orig.fcp_pos, pos.fcp_pos, relaxed, { |
99 | if (orig.fcp_pos == 0) { |
100 | // we acquired a really really old reference, and we probably |
101 | // just faulted in a new page |
102 | os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL); |
103 | } |
104 | if (!FIREHOSE_CHUNK_POS_USABLE_FOR_STREAM(orig, stream)) { |
105 | // nothing to do if the chunk is full, or the stream doesn't match, |
106 | // in which case the thread probably: |
107 | // - loaded the chunk ref |
108 | // - been suspended a long while |
109 | // - read the chunk to find a very old thing |
110 | os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL); |
111 | } |
112 | pos = orig; |
113 | if (!firehose_chunk_pos_fits(orig, |
114 | ft_size + pubsize + privsize) || !stamp_delta_fits) { |
115 | pos.fcp_flag_full = true; |
116 | reservation_failed = true; |
117 | } else { |
118 | if (qos > pos.fcp_qos) pos.fcp_qos = qos; |
119 | // using these *_INC macros is so that the compiler generates better |
120 | // assembly: using the struct individual fields forces the compiler |
121 | // to handle carry propagations, and we know it won't happen |
122 | pos.fcp_pos += roundup(ft_size + pubsize, 8) * |
123 | FIREHOSE_CHUNK_POS_ENTRY_OFFS_INC; |
124 | pos.fcp_pos -= privsize * FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC; |
125 | pos.fcp_pos += FIREHOSE_CHUNK_POS_REFCNT_INC; |
126 | const uint16_t minimum_payload_size = 16; |
127 | if (!firehose_chunk_pos_fits(pos, |
128 | roundup(ft_size + minimum_payload_size , 8))) { |
129 | // if we can't even have minimum_payload_size bytes of payload |
130 | // for the next tracepoint, just flush right away |
131 | pos.fcp_flag_full = true; |
132 | } |
133 | reservation_failed = false; |
134 | } |
135 | }); |
136 | |
137 | if (reservation_failed) { |
138 | if (pos.fcp_refcnt) { |
139 | // nothing to do, there is a thread writing that will pick up |
140 | // the "FULL" flag on flush and push as a consequence |
141 | return FIREHOSE_CHUNK_TRY_RESERVE_FAIL; |
142 | } |
143 | // caller must enqueue chunk |
144 | return FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE; |
145 | } |
146 | if (privptr) { |
147 | *privptr = fc->fc_start + pos.fcp_private_offs; |
148 | } |
149 | return orig.fcp_next_entry_offs; |
150 | } |
151 | |
152 | OS_ALWAYS_INLINE |
153 | static inline firehose_tracepoint_t |
154 | firehose_chunk_tracepoint_begin(firehose_chunk_t fc, uint64_t stamp, |
155 | uint16_t pubsize, uint64_t thread_id, long offset) |
156 | { |
157 | firehose_tracepoint_t ft = (firehose_tracepoint_t) |
158 | __builtin_assume_aligned(fc->fc_start + offset, 8); |
159 | stamp -= fc->fc_timestamp; |
160 | stamp |= (uint64_t)pubsize << 48; |
161 | // The compiler barrier is needed for userland process death handling, see |
162 | // (tracepoint-begin) in libdispatch's firehose_buffer_stream_chunk_install. |
163 | atomic_store_explicit(&ft->ft_atomic_stamp_and_length, stamp, |
164 | memory_order_relaxed); |
165 | __asm__ __volatile__("" ::: "memory" ); |
166 | ft->ft_thread = thread_id; |
167 | return ft; |
168 | } |
169 | |
170 | OS_ALWAYS_INLINE |
171 | static inline bool |
172 | firehose_chunk_tracepoint_end(firehose_chunk_t fc, |
173 | firehose_tracepoint_t ft, firehose_tracepoint_id_u ftid) |
174 | { |
175 | firehose_chunk_pos_u pos; |
176 | |
177 | atomic_store_explicit(&ft->ft_id.ftid_atomic_value, |
178 | ftid.ftid_value, memory_order_release); |
179 | pos.fcp_pos = atomic_fetch_sub_explicit(&fc->fc_pos.fcp_atomic_pos, |
180 | FIREHOSE_CHUNK_POS_REFCNT_INC, memory_order_relaxed); |
181 | return pos.fcp_refcnt == 1 && pos.fcp_flag_full; |
182 | } |
183 | |
184 | #endif // defined(KERNEL) || defined(OS_FIREHOSE_SPI) |
185 | |
186 | __END_DECLS |
187 | |
188 | #endif // __FIREHOSE_CHUNK_PRIVATE__ |
189 | |