1 | #include <kern/assert.h> |
2 | #include <kern/debug.h> |
3 | #include <pexpert/pexpert.h> |
4 | #include <kern/btlog.h> |
5 | #include <kern/backtrace.h> |
6 | #include <libkern/libkern.h> |
7 | #include "refcnt.h" |
8 | |
9 | #define OS_REFCNT_MAX_COUNT ((os_ref_count_t)0x0FFFFFFFUL) |
10 | |
11 | #if OS_REFCNT_DEBUG |
12 | os_refgrp_decl(static, global_ref_group, "all" , NULL); |
13 | static bool ref_debug_enable = false; |
14 | static const size_t ref_log_nrecords = 1000000; |
15 | |
16 | #define REFLOG_BTDEPTH 10 |
17 | #define REFLOG_RETAIN 1 |
18 | #define REFLOG_RELEASE 2 |
19 | |
20 | #define __debug_only |
21 | #else |
22 | # define __debug_only __unused |
23 | #endif /* OS_REFCNT_DEBUG */ |
24 | |
25 | static const char * |
26 | ref_grp_name(struct os_refcnt __debug_only *rc) |
27 | { |
28 | #if OS_REFCNT_DEBUG |
29 | if (rc && rc->ref_group && rc->ref_group->grp_name) { |
30 | return rc->ref_group->grp_name; |
31 | } |
32 | #endif |
33 | return "<null>" ; |
34 | } |
35 | |
36 | static void |
37 | os_ref_check_underflow(struct os_refcnt *rc, os_ref_count_t count) |
38 | { |
39 | if (__improbable(count == 0)) { |
40 | panic("os_refcnt: underflow (rc=%p, grp=%s)\n" , rc, ref_grp_name(rc)); |
41 | __builtin_unreachable(); |
42 | } |
43 | } |
44 | |
45 | static void |
46 | os_ref_assert_referenced(struct os_refcnt *rc, os_ref_count_t count) |
47 | { |
48 | if (__improbable(count == 0)) { |
49 | panic("os_refcnt: used unsafely when zero (rc=%p, grp=%s)\n" , rc, ref_grp_name(rc)); |
50 | __builtin_unreachable(); |
51 | } |
52 | } |
53 | |
54 | static void |
55 | os_ref_check_overflow(struct os_refcnt *rc, os_ref_count_t count) |
56 | { |
57 | if (__improbable(count >= OS_REFCNT_MAX_COUNT)) { |
58 | panic("os_refcnt: overflow (rc=%p, grp=%s)\n" , rc, ref_grp_name(rc)); |
59 | __builtin_unreachable(); |
60 | } |
61 | } |
62 | |
63 | static void |
64 | os_ref_check_retain(struct os_refcnt *rc, os_ref_count_t count) |
65 | { |
66 | os_ref_assert_referenced(rc, count); |
67 | os_ref_check_overflow(rc, count); |
68 | } |
69 | |
70 | #if OS_REFCNT_DEBUG |
71 | static void |
72 | ref_log_op(struct os_refgrp *grp, void *elem, int op) |
73 | { |
74 | if (!ref_debug_enable || grp == NULL) { |
75 | return; |
76 | } |
77 | |
78 | if (grp->grp_log == NULL) { |
79 | ref_log_op(grp->grp_parent, elem, op); |
80 | return; |
81 | } |
82 | |
83 | uintptr_t bt[REFLOG_BTDEPTH]; |
84 | uint32_t nframes = backtrace(bt, REFLOG_BTDEPTH); |
85 | btlog_add_entry((btlog_t *)grp->grp_log, elem, op, (void **)bt, nframes); |
86 | } |
87 | |
88 | static void |
89 | ref_log_drop(struct os_refgrp *grp, void *elem) |
90 | { |
91 | if (!ref_debug_enable || grp == NULL) { |
92 | return; |
93 | } |
94 | |
95 | if (grp->grp_log == NULL) { |
96 | ref_log_drop(grp->grp_parent, elem); |
97 | return; |
98 | } |
99 | |
100 | btlog_remove_entries_for_element(grp->grp_log, elem); |
101 | } |
102 | |
103 | static void |
104 | ref_log_init(struct os_refgrp *grp) |
105 | { |
106 | if (grp->grp_log != NULL) { |
107 | return; |
108 | } |
109 | |
110 | char grpbuf[128]; |
111 | char *refgrp = grpbuf; |
112 | if (!PE_parse_boot_argn("rlog" , refgrp, sizeof(grpbuf))) { |
113 | return; |
114 | } |
115 | |
116 | const char *g; |
117 | while ((g = strsep(&refgrp, "," )) != NULL) { |
118 | if (strcmp(g, grp->grp_name) == 0) { |
119 | /* enable logging on this refgrp */ |
120 | grp->grp_log = btlog_create(ref_log_nrecords, REFLOG_BTDEPTH, true); |
121 | assert(grp->grp_log); |
122 | ref_debug_enable = true; |
123 | return; |
124 | } |
125 | } |
126 | |
127 | } |
128 | |
129 | /* |
130 | * attach a new refcnt to a group |
131 | */ |
132 | static void |
133 | ref_attach_to_group(struct os_refcnt *rc, struct os_refgrp *grp, os_ref_count_t init_count) |
134 | { |
135 | if (grp == NULL) { |
136 | return; |
137 | } |
138 | |
139 | if (atomic_fetch_add_explicit(&grp->grp_children, 1, memory_order_relaxed) == 0) { |
140 | /* First reference count object in this group. Check if we should enable |
141 | * refcount logging. */ |
142 | ref_log_init(grp); |
143 | } |
144 | |
145 | atomic_fetch_add_explicit(&grp->grp_count, init_count, memory_order_relaxed); |
146 | atomic_fetch_add_explicit(&grp->grp_retain_total, init_count, memory_order_relaxed); |
147 | |
148 | if (grp == &global_ref_group) { |
149 | return; |
150 | } |
151 | |
152 | if (grp->grp_parent == NULL) { |
153 | grp->grp_parent = &global_ref_group; |
154 | } |
155 | |
156 | ref_attach_to_group(rc, grp->grp_parent, init_count); |
157 | } |
158 | |
159 | static inline void |
160 | ref_retain_group(struct os_refgrp *grp) |
161 | { |
162 | if (grp) { |
163 | atomic_fetch_add_explicit(&grp->grp_count, 1, memory_order_relaxed); |
164 | atomic_fetch_add_explicit(&grp->grp_retain_total, 1, memory_order_relaxed); |
165 | ref_retain_group(grp->grp_parent); |
166 | } |
167 | } |
168 | |
169 | static inline void |
170 | ref_release_group(struct os_refgrp *grp, bool final) |
171 | { |
172 | if (grp) { |
173 | atomic_fetch_sub_explicit(&grp->grp_count, 1, memory_order_relaxed); |
174 | atomic_fetch_add_explicit(&grp->grp_release_total, 1, memory_order_relaxed); |
175 | if (final) { |
176 | atomic_fetch_sub_explicit(&grp->grp_children, 1, memory_order_relaxed); |
177 | } |
178 | |
179 | ref_release_group(grp->grp_parent, final); |
180 | } |
181 | } |
182 | #endif |
183 | |
184 | #undef os_ref_init_count |
185 | void |
186 | os_ref_init_count(struct os_refcnt *rc, struct os_refgrp __debug_only *grp, os_ref_count_t count) |
187 | { |
188 | atomic_init(&rc->ref_count, count); |
189 | |
190 | #if OS_REFCNT_DEBUG |
191 | assert(count > 0); |
192 | if (grp) { |
193 | rc->ref_group = grp; |
194 | } else { |
195 | rc->ref_group = &global_ref_group; |
196 | } |
197 | |
198 | ref_attach_to_group(rc, rc->ref_group, count); |
199 | |
200 | for (os_ref_count_t i = 0; i < count; i++) { |
201 | ref_log_op(rc->ref_group, (void *)rc, REFLOG_RETAIN); |
202 | } |
203 | #endif |
204 | } |
205 | |
206 | void |
207 | os_ref_retain(struct os_refcnt *rc) |
208 | { |
209 | os_ref_count_t old = atomic_fetch_add_explicit(&rc->ref_count, 1, memory_order_relaxed); |
210 | os_ref_check_retain(rc, old); |
211 | |
212 | #if OS_REFCNT_DEBUG |
213 | ref_retain_group(rc->ref_group); |
214 | ref_log_op(rc->ref_group, (void *)rc, REFLOG_RETAIN); |
215 | #endif |
216 | } |
217 | |
218 | bool |
219 | os_ref_retain_try(struct os_refcnt *rc) |
220 | { |
221 | os_ref_count_t cur = os_ref_get_count(rc); |
222 | |
223 | while (1) { |
224 | if (__improbable(cur == 0)) { |
225 | return false; |
226 | } |
227 | |
228 | os_ref_check_retain(rc, cur); |
229 | |
230 | if (atomic_compare_exchange_weak_explicit(&rc->ref_count, &cur, cur+1, |
231 | memory_order_relaxed, memory_order_relaxed)) { |
232 | #if OS_REFCNT_DEBUG |
233 | ref_retain_group(rc->ref_group); |
234 | ref_log_op(rc->ref_group, (void *)rc, REFLOG_RETAIN); |
235 | #endif |
236 | return true; |
237 | } |
238 | } |
239 | } |
240 | |
241 | os_ref_count_t |
242 | os_ref_release_explicit(struct os_refcnt *rc, memory_order release_order, memory_order dealloc_order) |
243 | { |
244 | #if OS_REFCNT_DEBUG |
245 | /* |
246 | * Care not to use 'rc' after the decrement because it might be deallocated |
247 | * under us. |
248 | */ |
249 | struct os_refgrp *grp = rc->ref_group; |
250 | ref_log_op(grp, (void *)rc, REFLOG_RELEASE); |
251 | #endif |
252 | |
253 | os_ref_count_t val = atomic_fetch_sub_explicit(&rc->ref_count, 1, release_order); |
254 | os_ref_check_underflow(rc, val); |
255 | if (__improbable(--val == 0)) { |
256 | atomic_load_explicit(&rc->ref_count, dealloc_order); |
257 | #if OS_REFCNT_DEBUG |
258 | ref_log_drop(grp, (void *)rc); /* rc is only used as an identifier */ |
259 | #endif |
260 | } |
261 | |
262 | #if OS_REFCNT_DEBUG |
263 | ref_release_group(grp, !val); |
264 | #endif |
265 | |
266 | return val; |
267 | } |
268 | |
269 | void |
270 | os_ref_retain_locked(struct os_refcnt *rc) |
271 | { |
272 | os_ref_count_t val = rc->ref_count; |
273 | os_ref_check_retain(rc, val); |
274 | rc->ref_count = ++val; |
275 | |
276 | #if OS_REFCNT_DEBUG |
277 | ref_retain_group(rc->ref_group); |
278 | ref_log_op(rc->ref_group, (void *)rc, REFLOG_RETAIN); |
279 | #endif |
280 | } |
281 | |
282 | os_ref_count_t |
283 | os_ref_release_locked(struct os_refcnt *rc) |
284 | { |
285 | os_ref_count_t val = rc->ref_count; |
286 | os_ref_check_underflow(rc, val); |
287 | rc->ref_count = --val; |
288 | |
289 | #if OS_REFCNT_DEBUG |
290 | ref_release_group(rc->ref_group, !val); |
291 | ref_log_op(rc->ref_group, (void *)rc, REFLOG_RELEASE); |
292 | if (val == 0) { |
293 | ref_log_drop(rc->ref_group, (void *)rc); |
294 | } |
295 | #endif |
296 | return val; |
297 | } |
298 | |
299 | |