1 | /* |
2 | * Copyright (c) 2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _KERN_APFS_REFLOCK_H_ |
30 | #define _KERN_APFS_REFLOCK_H_ |
31 | |
32 | #include <sys/cdefs.h> |
33 | #include <kern/kern_types.h> |
34 | #include <kern/locks.h> |
35 | |
36 | /* |
37 | * kern_apfs_reflock_t is an object that provides a refcount protected by an embedded lock. |
38 | * Manipulating the refcount is expected to be the most common operation on this object; |
39 | * the refcount can be changed (incremented or decremented) when the lock is not held. |
40 | * Some users might require to halt the refcount manipulation while some operations |
41 | * are in progress. To express this, kern_apfs_reflock_t allow to lock the object providing |
42 | * mutual exclusion between those operations and the refcount. |
43 | * When the object is locked all new lock requests, increments and decrements of the kern_apfs_reflock_t |
44 | * will fail, and the user can choose to wait for the object to be unlocked. |
45 | * The thread that locked the object will inherit the priority push of all the |
46 | * threads waiting for it to be unlocked. |
47 | * Further, refcount transitions 0->1 and 1->0, allow to atomically lock the reflock |
48 | * providing the possibility to cleanup/initialize the state. |
49 | */ |
50 | |
51 | #ifdef KERNEL_PRIVATE |
52 | |
53 | #if XNU_KERNEL_PRIVATE |
54 | #define KERN_APFS_REFLOCK_WAITERS_BIT 16 |
55 | #define KERN_APFS_REFLOCK_REFCOUNT_BIT (64 - (SWI_COND_OWNER_BITS + KERN_APFS_REFLOCK_WAITERS_BIT + 4)) |
56 | #define KERN_APFS_REFLOCK_MAXREFCOUNT ((1ull << KERN_APFS_REFLOCK_REFCOUNT_BIT) - 1) |
57 | #define KERN_APFS_REFLOCK_MAXWAITERS ((1ull << KERN_APFS_REFLOCK_WAITERS_BIT) - 1) |
58 | #define KERN_APFS_REFLOCK_DESTROYED (~(0ull)) |
59 | |
60 | /* |
61 | * Mask to debounce the sleep. Needs to be kept up to date with kern_apfs_reflock. |
62 | * Equivalent to: |
63 | * mask = {.kern_apfs_rl_owner = ((1 << SWI_COND_OWNER_BITS) - 1), |
64 | * .kern_apfs_rl_delayed_free = 1, |
65 | * .kern_apfs_rl_wake = 1, |
66 | * .kern_apfs_rl_allocated = 1, |
67 | * .kern_apfs_rl_allow_force = 1}; |
68 | */ |
69 | |
70 | #define KERN_APFS_SLEEP_DEBOUNCE_MASK ((uint64_t)0xf0000fffff) |
71 | |
72 | typedef struct kern_apfs_reflock { |
73 | union { |
74 | cond_swi_var64_s kern_apfs_rl_data; |
75 | struct { |
76 | uint64_t kern_apfs_rl_owner: SWI_COND_OWNER_BITS, |
77 | kern_apfs_rl_waiters: KERN_APFS_REFLOCK_WAITERS_BIT, |
78 | kern_apfs_rl_delayed_free: 1, |
79 | kern_apfs_rl_wake: 1, |
80 | kern_apfs_rl_allocated: 1, |
81 | kern_apfs_rl_allow_force: 1, |
82 | kern_apfs_rl_count: KERN_APFS_REFLOCK_REFCOUNT_BIT; |
83 | }; |
84 | }; |
85 | } *kern_apfs_reflock_t; |
86 | |
87 | |
88 | #else /* XNU_KERNEL_PRIVATE */ |
89 | typedef struct kern_apfs_reflock { |
90 | uint64_t opaque; |
91 | } *kern_apfs_reflock_t; |
92 | #endif /* XNU_KERNEL_PRIVATE */ |
93 | |
94 | __options_decl(kern_apfs_reflock_in_flags_t, uint32_t, { |
95 | KERN_APFS_REFLOCK_IN_DEFAULT = 0x0, |
96 | KERN_APFS_REFLOCK_IN_LOCK_IF_LAST = 0x1, |
97 | KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST = 0x2, |
98 | KERN_APFS_REFLOCK_IN_WILL_WAIT = 0x4, |
99 | KERN_APFS_REFLOCK_IN_FORCE = 0x8, |
100 | KERN_APFS_REFLOCK_IN_ALLOW_FORCE = 0x10, |
101 | }); |
102 | |
103 | __options_decl(kern_apfs_reflock_out_flags_t, uint32_t, { |
104 | KERN_APFS_REFLOCK_OUT_DEFAULT = 0x0, |
105 | KERN_APFS_REFLOCK_OUT_LOCKED = 0x1, |
106 | }); |
107 | |
108 | __BEGIN_DECLS |
109 | |
110 | /* |
111 | * Name: kern_apfs_reflock_data |
112 | * |
113 | * Description: declares a kern_apfs_reflock variable with specified storage class. |
114 | * The reflock will be stored in this variable and it is the caller's responsibility |
115 | * to ensure that this variable's memory is going to be accessible by all threads that will use |
116 | * the kern_apfs_reflock. |
117 | * Every kern_apfs_reflock function will require a pointer to this variable as parameter. |
118 | * |
119 | * The variable needs to be initialized once with kern_apfs_reflock_init() and destroyed once with |
120 | * kern_apfs_reflock_destroy() when not needed anymore. |
121 | * |
122 | * Args: |
123 | * Arg1: storage class. |
124 | * Arg2: variable name. |
125 | */ |
126 | #define kern_apfs_reflock_data(class, name) class struct kern_apfs_reflock name |
127 | |
128 | /* |
129 | * Name: kern_apfs_reflock_init |
130 | * |
131 | * Description: initializes a kern_apfs_reflock_t. |
132 | * |
133 | * Args: |
134 | * Arg1: kern_apfs_reflock object. |
135 | * |
136 | * Conditions: the memory pointed by kern_apfs_reflock_t needs to be available |
137 | * while any of the other functions are executed. |
138 | * It is the caller responsibility to guarantee that all functions call |
139 | * executed on the kern_apfs_reflock have terminated before freeing it, |
140 | * including possible kern_apfs_reflock_wait_for_unlock(). If it is not possible |
141 | * to safely synchronize kern_apfs_reflock_wait_for_unlock() calls |
142 | * kern_apfs_reflock_alloc_init() should be used instead. |
143 | */ |
144 | void kern_apfs_reflock_init(kern_apfs_reflock_t reflock); |
145 | |
146 | /* |
147 | * Name: kern_apfs_reflock_destroy |
148 | * |
149 | * Description: destroys a kern_apfs_reflock_t. |
150 | * |
151 | * Args: |
152 | * Arg1: kern_apfs_reflock object. |
153 | * |
154 | * Conditions: the object must have been previously initialized with kern_apfs_reflock_init. |
155 | * Any access past this point to the kern_apfs_reflock will be considered invalid. |
156 | */ |
157 | void kern_apfs_reflock_destroy(kern_apfs_reflock_t reflock); |
158 | |
159 | /* |
160 | * Name: kern_apfs_reflock_alloc_init |
161 | * |
162 | * Description: allocates a kern_apfs_reflock_t. |
163 | * |
164 | * Returns: allocated kern_apfs_reflock_t. |
165 | * |
166 | * Conditions: It is the caller responsibility to guarantee that all functions call |
167 | * executed on the kern_apfs_reflock_t returned by this function |
168 | * (except for kern_apfs_reflock_wait_for_unlock()) have terminated before freeing it. |
169 | * It is safe to execute concurrent or later kern_apfs_reflock_wait_for_unlock() |
170 | * calls as long as the matching kern_apfs_reflock_try_get_ref(), |
171 | * kern_apfs_reflock_try_put_ref() or kern_apfs_reflock_try_lock() was executed before |
172 | * the call to kern_apfs_reflock_free(). In this case the free of the object will be delegated |
173 | * to the last concurrent kern_apfs_reflock_wait_for_unlock() executed. |
174 | */ |
175 | kern_apfs_reflock_t kern_apfs_reflock_alloc_init(void); |
176 | |
177 | /* |
178 | * Name: kern_apfs_reflock_free |
179 | * |
180 | * Description: frees and destroys a kern_apfs_reflock_t. |
181 | * |
182 | * Args: |
183 | * Arg1: kern_apfs_reflock object. |
184 | * |
185 | * Conditions: It is the caller responsability to guarantee that all functions call |
186 | * executed on the kern_apfs_reflock_t (except kern_apfs_reflock_wait_for_unlock()) |
187 | * have terminated before freeing it. |
188 | * It is safe to execute concurrent or later kern_apfs_reflock_wait_for_unlock() |
189 | * calls as long as the matching kern_apfs_reflock_try_get_ref(), |
190 | * kern_apfs_reflock_try_put_ref() or kern_apfs_reflock_try_lock() was executed before |
191 | * the call to kern_apfs_reflock_free(). In this case the free of the object will be delegated |
192 | * to the last concurrent kern_apfs_reflock_wait_for_unlock() executed. |
193 | */ |
194 | void kern_apfs_reflock_free(kern_apfs_reflock_t reflock); |
195 | |
196 | /* |
197 | * Name: kern_apfs_reflock_try_get_ref |
198 | * |
199 | * Description: tries to get a reference on the kern_apfs_reflock. |
200 | * The operation will succeed if the lock on the object is not held. |
201 | * In case of failure the caller can choose to wait for the lock to unlock |
202 | * with a subsequent call to kern_apfs_reflock_wait_for_unlock(). |
203 | * |
204 | * Args: |
205 | * Arg1: kern_apfs_reflock object. |
206 | * Arg2: in flags can be a combination of: |
207 | * - KERN_APFS_REFLOCK_IN_DEFAULT for default behaviour. |
208 | * - KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST will lock the reflock if the refcount was incremented |
209 | * in the "init" transition, so from 0->1. |
210 | * - KERN_APFS_REFLOCK_IN_WILL_WAIT if the try_get() fails, then the thread will call kern_apfs_reflock_wait_for_unlock(). |
211 | * kern_apfs_reflock_wait_for_unlock() cannot be called after this function fails if this |
212 | * flag was not set. |
213 | * - KERN_APFS_REFLOCK_IN_FORCE if the reflock was locked from a kern_apfs_reflock_try_lock() with KERN_APFS_REFLOCK_IN_ALLOW_FORCE |
214 | * this flag will allow to get the reference even if the object is locked. |
215 | * Even with this flag set the function might fail if the reflock was locked from a |
216 | * kern_apfs_reflock_try_get_ref() or kern_apfs_reflock_try_put_ref(). |
217 | * NOTE: KERN_APFS_REFLOCK_IN_FORCE and KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST cannot be used together. |
218 | * Arg3: out flags: |
219 | * - KERN_APFS_REFLOCK_OUT_DEFAULT if the lock was not acquired. |
220 | * - KERN_APFS_REFLOCK_OUT_LOCKED if the lock was acquired. |
221 | * |
222 | * Returns: true if the reference was acquired, false otherwise. |
223 | * If KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST was set and the reference was successfully acquired, out_flags will indicate if the |
224 | * lock was acquired. |
225 | * |
226 | * |
227 | * Conditions: If KERN_APFS_REFLOCK_IN_WILL_WAIT was set, a kern_apfs_reflock_wait_for_unlock() |
228 | * needs to be called in case of failure. |
229 | * If KERN_APFS_REFLOCK_OUT_LOCKED is returned on the out_flags a corresponding kern_apfs_reflock_wait_for_unlock() needs to be called |
230 | * by the same thread and the thread cannot execute in userspace until the unlock is called. |
231 | */ |
232 | bool kern_apfs_reflock_try_get_ref(kern_apfs_reflock_t reflock, kern_apfs_reflock_in_flags_t in_flags, kern_apfs_reflock_out_flags_t *out_flags); |
233 | |
234 | /* |
235 | * Name: kern_apfs_reflock_try_put_ref |
236 | * |
237 | * Description: tries to put a reference on the kern_apfs_reflock. |
238 | * The operation will succeed if the lock on the object is not held. |
239 | * In case of failure the caller can choose to wait for the lock to unlock |
240 | * with a subsequent call to kern_apfs_reflock_wait_for_unlock(). |
241 | * |
242 | * Args: |
243 | * Arg1: kern_apfs_reflock object. |
244 | * Arg2: in flags can be a combination of: |
245 | * - KERN_APFS_REFLOCK_IN_DEFAULT for default behaviour. |
246 | * - KERN_APFS_REFLOCK_IN_LOCK_IF_LAST will lock the reflock if the refcount was decremented |
247 | * in the "cleanup" transition, so from 1->0. |
248 | * - KERN_APFS_REFLOCK_IN_WILL_WAIT if the try_put() fails, then the thread will call kern_apfs_reflock_wait_for_unlock(). |
249 | * kern_apfs_reflock_wait_for_unlock() cannot be called after this function fails if this |
250 | * flag was not set. |
251 | * - KERN_APFS_REFLOCK_IN_FORCE if the reflock was locked from a kern_apfs_reflock_try_lock() with KERN_APFS_REFLOCK_IN_ALLOW_FORCE |
252 | * this flag will allow to put the reference even if the object is locked. |
253 | * Even with this flag set the function might fail if the reflock was locked from a |
254 | * kern_apfs_reflock_try_get_ref() or kern_apfs_reflock_try_put_ref(). |
255 | * NOTE: KERN_APFS_REFLOCK_IN_FORCE and KERN_APFS_REFLOCK_IN_LOCK_IF_LAST cannot be used together. |
256 | * Arg3: out flags: |
257 | * - KERN_APFS_REFLOCK_OUT_DEFAULT if the lock was not acquired. |
258 | * - KERN_APFS_REFLOCK_OUT_LOCKED if the lock was acquired. |
259 | * |
260 | * Returns: true if the reference was successfully decremented, false otherwise. |
261 | * If KERN_APFS_REFLOCK_IN_LOCK_IF_LAST was set and the reference was successfully decremented, out_flags will indicate if the |
262 | * lock was acquired. |
263 | * |
264 | * |
265 | * Conditions: If KERN_APFS_REFLOCK_IN_WILL_WAIT was set, a kern_apfs_reflock_wait_for_unlock() |
266 | * needs to be called in case of failure. |
267 | * If KERN_APFS_REFLOCK_OUT_LOCKED is returned on the out_flags a corresponding kern_apfs_reflock_wait_for_unlock() needs to be called |
268 | * by the same theread and the thread cannot execute in userspace until the unlock is called. |
269 | * |
270 | */ |
271 | bool kern_apfs_reflock_try_put_ref(kern_apfs_reflock_t reflock, kern_apfs_reflock_in_flags_t in_flags, kern_apfs_reflock_out_flags_t *out_flags); |
272 | |
273 | /* |
274 | * Name: kern_apfs_reflock_try_lock |
275 | * |
276 | * Description: tries to acquire the lock on the kern_apfs_reflock. |
277 | * The operation will succeed if the lock on the object is not held. |
278 | * In case of failure the caller can choose to wait for the lock to unlock |
279 | * with a subsequent call to kern_apfs_reflock_wait_for_unlock(). |
280 | * |
281 | * Args: |
282 | * Arg1: kern_apfs_reflock object. |
283 | * Arg2: in flags can be a combination of: |
284 | * - KERN_APFS_REFLOCK_IN_DEFAULT for default behaviour. |
285 | * - KERN_APFS_REFLOCK_IN_WILL_WAIT if the try_lock() fails, then the thread will call kern_apfs_reflock_wait_for_unlock(). |
286 | * kern_apfs_reflock_wait_for_unlock() cannot be called after this function fails if this |
287 | * flag was not set. |
288 | * - KERN_APFS_REFLOCK_IN_ALLOW_FORCE if this flag is set, kern_apfs_reflock_try_put_ref() and kern_apfs_reflock_try_get_ref() with |
289 | * flag KERN_APFS_REFLOCK_IN_FORCE set will succed even after this call locked the reflock. |
290 | * |
291 | * Arg3: refcount_when_lock pointer into which return the value of the refcount at the moment of lock. |
292 | * |
293 | * Returns: true if the lock was acquired, false otherwise. |
294 | * |
295 | * Conditions: If KERN_APFS_REFLOCK_IN_WILL_WAIT was set, a kern_apfs_reflock_wait_for_unlock() |
296 | * needs to be called in case of failure. |
297 | * If the lock was acquired a subsequent kern_apfs_reflock_unlock() by the same theread and |
298 | * the thread cannot execute in userspace until the unlock is called. |
299 | * Recursive locking is not allowed. |
300 | */ |
301 | bool kern_apfs_reflock_try_lock(kern_apfs_reflock_t reflock, kern_apfs_reflock_in_flags_t in_flags, uint32_t *refcount_when_lock); |
302 | |
303 | /* |
304 | * Name: kern_apfs_reflock_wait_for_unlock |
305 | * |
306 | * Description: waits for the lock to be unlocked. |
307 | * While waiting the priority of this thread will contribute |
308 | * to the priority push of the owner of the reflock. |
309 | * NOTE: it is not guaranteed that by the time this calls |
310 | * returns the reflock is unlocked, as it might have been re-locked |
311 | * after the current thread has been woken up. |
312 | * If needed, the matching kern_apfs_reflock_try_get_ref(), kern_apfs_reflock_try_put_ref() or |
313 | * kern_apfs_reflock_try_lock() should be re-driven after this function. |
314 | * |
315 | * Args: |
316 | * Arg1: reflock object. |
317 | * Arg2: interruptible flag for wait. |
318 | * Arg3: deadline for wait. |
319 | * |
320 | * Returns: result of the wait. |
321 | * THREAD_AWAKENED - normal wakeup |
322 | * THREAD_TIMED_OUT - timeout expired |
323 | * THREAD_INTERRUPTED - aborted/interrupted |
324 | * THREAD_NOT_WAITING - thread didn't need to wait |
325 | */ |
326 | wait_result_t kern_apfs_reflock_wait_for_unlock(kern_apfs_reflock_t reflock, wait_interrupt_t interruptible, uint64_t deadline); |
327 | |
328 | /* |
329 | * Name: kern_apfs_reflock_unlock |
330 | * |
331 | * Description: unlocks a reflock obj. |
332 | * |
333 | * Args: |
334 | * Arg1: reflock object. |
335 | * |
336 | * Conditions: the same thread that locked the object needs to unlock it. |
337 | */ |
338 | void kern_apfs_reflock_unlock(kern_apfs_reflock_t reflock); |
339 | |
340 | /* |
341 | * Name: kern_apfs_reflock_read_ref |
342 | * |
343 | * Description: reads the refcount counter. |
344 | * Note: using this function is racy, as the refcount can change |
345 | * after this function reads it. Its usage is discouraged. |
346 | * |
347 | * Args: |
348 | * Arg1: reflock object. |
349 | * |
350 | * Returns: refcount value. |
351 | */ |
352 | uint64_t kern_apfs_reflock_read_ref(kern_apfs_reflock_t reflock); |
353 | __END_DECLS |
354 | |
355 | #endif /* KERNEL_PRIVATE */ |
356 | #endif /* _KERN_APFS_REFLOCK_H_ */ |
357 | |