1 | /* |
2 | * Copyright (c) 2017 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _OS_REFCNT_H_ |
30 | #define _OS_REFCNT_H_ |
31 | |
32 | /* |
33 | * os_refcnt reference counting API |
34 | * |
35 | * Two flavors are provided: atomic and locked. Atomic internally uses C11 atomic |
36 | * operations and requires no external synchronization, whereas the locked flavor |
37 | * assumes the refcnt object is locked by the caller. It is NOT safe to |
38 | * mix-and-match locked and atomic calls. |
39 | * |
40 | * 'refgrp's are a way to (hierarchically) group like refcount objects for |
41 | * debugging purposes. The group keeps track of the total number and aggregate |
42 | * reference count of member refcounts, and the "rlog=" boot-arg is used to enable |
43 | * refcount logging by group name. Named groups can be created explicitly with |
44 | * os_refgrp_decl(), or implicitly by passing NULL for the refgrp when |
45 | * initializing a refcnt object. In the latter case, the group name is the same as |
46 | * the function enclosing the init call. Groups are only available on DEV or DEBUG |
47 | * builds, and are otherwise compiled out. |
48 | */ |
49 | |
50 | #include <stdatomic.h> |
51 | #include <stdbool.h> |
52 | #include <os/base.h> |
53 | |
54 | struct os_refcnt; |
55 | struct os_refgrp; |
56 | typedef struct os_refcnt os_refcnt_t; |
57 | |
58 | /* type of the internal counter */ |
59 | typedef uint32_t os_ref_count_t; |
60 | typedef _Atomic(os_ref_count_t) os_ref_atomic_t; |
61 | |
62 | /* |
63 | * OS_REF_INITIALIZER |
64 | * OS_REF_ATOMIC_INITIALIZER |
65 | * |
66 | * Static initializers that create refcnt objects with safe initial values for use |
67 | * between declaration and initialization (os_ref*_init()). Equivalent to zeroing. |
68 | */ |
69 | |
70 | #ifndef KERNEL |
71 | # include <stdlib.h> |
72 | # include <stdio.h> |
73 | # ifndef __improbable |
74 | # define __improbable(x) x |
75 | # endif |
76 | # ifndef panic |
77 | # define panic(x, ...) do { fprintf(stderr, x, __VA_ARGS__); abort(); } while (0) |
78 | # endif |
79 | #endif |
80 | |
81 | #ifndef OS_REFCNT_DEBUG |
82 | # if DEVELOPMENT || DEBUG |
83 | # define OS_REFCNT_DEBUG 1 |
84 | # else |
85 | # define OS_REFCNT_DEBUG 0 |
86 | # endif |
87 | #endif |
88 | |
89 | #if __has_attribute(diagnose_if) |
90 | # define os_error_if(cond, msg) __attribute__((diagnose_if((cond), (msg), "error"))) |
91 | #else |
92 | # define os_error_if(...) |
93 | #endif |
94 | |
95 | __BEGIN_DECLS |
96 | |
97 | /* |
98 | * os_ref_init: initialize an os_refcnt with a count of 1 |
99 | * os_ref_init_count: initialize an os_refcnt with a specific count >= 1 |
100 | */ |
101 | #define os_ref_init(rc, grp) os_ref_init_count((rc), (grp), 1) |
102 | static void os_ref_init_count(struct os_refcnt *, struct os_refgrp *, os_ref_count_t count) |
103 | os_error_if(count == 0, "Reference count must be non-zero initialized" ); |
104 | |
105 | /* |
106 | * os_refgrp_decl(qual, var, name, parent): declare a refgroup object 'var' with |
107 | * given name string and parent group. |
108 | */ |
109 | |
110 | /* |
111 | * |
112 | * os_ref_retain: acquire a reference (increment reference count by 1) atomically. |
113 | * |
114 | * os_ref_release: release a reference (decrement reference count) atomically and |
115 | * return the new count. Memory is synchronized such that the dealloc block |
116 | * (i.e. code handling the final release() == 0 call) sees up-to-date memory |
117 | * with respect to all prior release()s on the same refcnt object. This |
118 | * memory ordering is sufficient for most use cases. |
119 | * |
120 | * os_ref_release_relaxed: same as release() but with weaker relaxed memory ordering. |
121 | * This can be used when the dealloc block is already synchronized with other |
122 | * accesses to the object (for example, with a lock). |
123 | * |
124 | * os_ref_release_live: release a reference that is guaranteed not to be the last one. |
125 | */ |
126 | static void os_ref_retain(struct os_refcnt *); |
127 | static os_ref_count_t os_ref_release(struct os_refcnt *) OS_WARN_RESULT; |
128 | static os_ref_count_t os_ref_release_relaxed(struct os_refcnt *) OS_WARN_RESULT; |
129 | static void os_ref_release_live(struct os_refcnt *); |
130 | |
131 | /* |
132 | * os_ref_retain_try: a variant of atomic retain that fails for objects with a |
133 | * zero reference count. The caller must therefore ensure that the object |
134 | * remains alive for any possible retain_try() caller, usually by using a |
135 | * lock protecting both the retain and dealloc paths. This variant is useful |
136 | * for objects stored in a collection, because no lock is required on the |
137 | * release() side until the object is deallocated. |
138 | */ |
139 | static bool os_ref_retain_try(struct os_refcnt *) OS_WARN_RESULT; |
140 | |
141 | /* |
142 | * os_ref_retain_locked: acquire a reference on an object protected by a held |
143 | * lock. The caller must ensure mutual exclusivity of retain_locked() and |
144 | * release_locked() calls on the same object. |
145 | * |
146 | * os_ref_release_locked: release a reference on an object protected by a held |
147 | * lock. |
148 | */ |
149 | static void os_ref_retain_locked(struct os_refcnt *); |
150 | static os_ref_count_t os_ref_release_locked(struct os_refcnt *) OS_WARN_RESULT; |
151 | |
152 | /* |
153 | * os_ref_get_count: return the current reference count. This is unsafe for |
154 | * synchronization. |
155 | */ |
156 | static os_ref_count_t os_ref_get_count(struct os_refcnt *rc); |
157 | |
158 | |
159 | #if XNU_KERNEL_PRIVATE |
160 | #pragma GCC visibility push(hidden) |
161 | |
162 | /* |
163 | * Raw API that uses a plain atomic counter (os_ref_atomic_t) and a separate |
164 | * refgroup. This can be used in situations where the refcount object must be |
165 | * fixed size, for example for embedding in structures with ABI stability |
166 | * requirements. |
167 | */ |
168 | |
169 | #define os_ref_init_raw(rc, grp) os_ref_init_count_raw((rc), (grp), 1) |
170 | static void os_ref_init_count_raw(os_ref_atomic_t *, struct os_refgrp *, os_ref_count_t count) |
171 | os_error_if(count == 0, "Reference count must be non-zero initialized" ); |
172 | static void os_ref_retain_floor(struct os_refcnt *, os_ref_count_t f) |
173 | os_error_if(!__builtin_constant_p(f) || f == 0, "refcount floor must be >= 1" ); |
174 | static void os_ref_retain_raw(os_ref_atomic_t *, struct os_refgrp *); |
175 | static void os_ref_retain_floor_raw(os_ref_atomic_t *, os_ref_count_t f, struct os_refgrp *) |
176 | os_error_if(!__builtin_constant_p(f) || f == 0, "refcount floor must be >= 1" ); |
177 | static os_ref_count_t os_ref_release_raw(os_ref_atomic_t *, struct os_refgrp *) OS_WARN_RESULT; |
178 | static os_ref_count_t os_ref_release_raw_relaxed(os_ref_atomic_t *, struct os_refgrp *) OS_WARN_RESULT; |
179 | static void os_ref_release_live_raw(os_ref_atomic_t *, struct os_refgrp *); |
180 | static bool os_ref_retain_try_raw(os_ref_atomic_t *, struct os_refgrp *) OS_WARN_RESULT; |
181 | static bool os_ref_retain_floor_try_raw(os_ref_atomic_t *, os_ref_count_t f, struct os_refgrp *) OS_WARN_RESULT |
182 | os_error_if(!__builtin_constant_p(f) || f == 0, "refcount floor must be >= 1" ); |
183 | static void os_ref_retain_locked_raw(os_ref_atomic_t *, struct os_refgrp *); |
184 | static void os_ref_retain_floor_locked_raw(os_ref_atomic_t *, os_ref_count_t f, struct os_refgrp *) |
185 | os_error_if(!__builtin_constant_p(f) || f == 0, "refcount floor must be >= 1" ); |
186 | static os_ref_count_t os_ref_release_locked_raw(os_ref_atomic_t *, struct os_refgrp *) OS_WARN_RESULT; |
187 | static os_ref_count_t os_ref_get_count_raw(os_ref_atomic_t *rc); |
188 | |
189 | |
190 | /* |
191 | * Bitwise API: like the raw API, but allows some bits in the refcount value to be |
192 | * reserved for other purposes. 'b' defines the number of trailing (LSB) reserved |
193 | * bits, which the refcnt_raw API will never modify (except at init()). |
194 | * |
195 | * It is assumed that users of this API always use atomic ops on the |
196 | * os_ref_atomic_t (or hold a lock for the locked variants), and never modify the |
197 | * top (32 - 'b') bits. |
198 | * |
199 | * Due to guard bits, the maximum reference count is 2^(28 - 'b') - 1, and the |
200 | * maximum 'b' is 26 bits. This API can also be used just to limit the max |
201 | * refcount. |
202 | * |
203 | * The "*_raw_mask" APIs return the raw bit pattern of the refcount (with a type |
204 | * of uint32_t), that the caller is supposed to decode. Other APIs that return |
205 | * os_ref_count_t return a normalized refcount where the trailing bits have been |
206 | * removed. |
207 | * |
208 | * "locked" variants aren't provided as the point of these interfaces |
209 | * is to combine flags into a refcount and be able to manipulate both |
210 | * atomically with respect to each other. |
211 | */ |
212 | |
213 | /* Initialize the reference count and reserved bits */ |
214 | #define os_ref_init_mask(rc, b, grp, bits) os_ref_init_count_mask((rc), (b), (grp), 1, bits) |
215 | void os_ref_init_count_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp, |
216 | os_ref_count_t init_count, uint32_t init_bits) |
217 | os_error_if(init_count == 0, "Reference count must be non-zero initialized" ) |
218 | os_error_if(b > 26, "Bitwise reference count limited to 26 bits" ) |
219 | os_error_if(init_bits >= (1U << b), "Bits out of range" ); |
220 | |
221 | static uint32_t os_ref_get_raw_mask(os_ref_atomic_t *rc); |
222 | static uint32_t os_ref_get_bits_mask(os_ref_atomic_t *rc, uint32_t b); |
223 | static os_ref_count_t os_ref_get_count_mask(os_ref_atomic_t *rc, uint32_t b); |
224 | |
225 | static void |
226 | os_ref_retain_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp); |
227 | static void |
228 | os_ref_retain_acquire_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp); |
229 | static uint32_t |
230 | os_ref_retain_try_mask(os_ref_atomic_t *, uint32_t b, uint32_t reject_mask, |
231 | struct os_refgrp *grp) OS_WARN_RESULT; |
232 | static bool |
233 | os_ref_retain_try_acquire_mask(os_ref_atomic_t *, uint32_t b, uint32_t reject_mask, |
234 | struct os_refgrp *grp) OS_WARN_RESULT; |
235 | |
236 | static uint32_t |
237 | os_ref_release_raw_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp) OS_WARN_RESULT; |
238 | static uint32_t |
239 | os_ref_release_raw_relaxed_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp) OS_WARN_RESULT; |
240 | static os_ref_count_t |
241 | os_ref_release_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp) OS_WARN_RESULT; |
242 | static os_ref_count_t |
243 | os_ref_release_relaxed_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp) OS_WARN_RESULT; |
244 | static uint32_t |
245 | os_ref_release_live_raw_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp); |
246 | static void |
247 | os_ref_release_live_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp); |
248 | |
249 | #pragma GCC visibility pop |
250 | #endif /* XNU_KERNEL_PRIVATE */ |
251 | |
252 | __END_DECLS |
253 | |
254 | #include <os/refcnt_internal.h> |
255 | #endif |
256 | |