1/*
2 * Copyright (c) 2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _KERN_APFS_REFLOCK_H_
30#define _KERN_APFS_REFLOCK_H_
31
32#include <sys/cdefs.h>
33#include <kern/kern_types.h>
34#include <kern/locks.h>
35
36/*
37 * kern_apfs_reflock_t is an object that provides a refcount protected by an embedded lock.
38 * Manipulating the refcount is expected to be the most common operation on this object;
39 * the refcount can be changed (incremented or decremented) when the lock is not held.
40 * Some users might require to halt the refcount manipulation while some operations
41 * are in progress. To express this, kern_apfs_reflock_t allow to lock the object providing
42 * mutual exclusion between those operations and the refcount.
43 * When the object is locked all new lock requests, increments and decrements of the kern_apfs_reflock_t
44 * will fail, and the user can choose to wait for the object to be unlocked.
45 * The thread that locked the object will inherit the priority push of all the
46 * threads waiting for it to be unlocked.
47 * Further, refcount transitions 0->1 and 1->0, allow to atomically lock the reflock
48 * providing the possibility to cleanup/initialize the state.
49 */
50
51#ifdef KERNEL_PRIVATE
52
53#if XNU_KERNEL_PRIVATE
54#define KERN_APFS_REFLOCK_WAITERS_BIT 16
55#define KERN_APFS_REFLOCK_REFCOUNT_BIT (64 - (SWI_COND_OWNER_BITS + KERN_APFS_REFLOCK_WAITERS_BIT + 4))
56#define KERN_APFS_REFLOCK_MAXREFCOUNT ((1ull << KERN_APFS_REFLOCK_REFCOUNT_BIT) - 1)
57#define KERN_APFS_REFLOCK_MAXWAITERS ((1ull << KERN_APFS_REFLOCK_WAITERS_BIT) - 1)
58#define KERN_APFS_REFLOCK_DESTROYED (~(0ull))
59
60/*
61 * Mask to debounce the sleep. Needs to be kept up to date with kern_apfs_reflock.
62 * Equivalent to:
63 * mask = {.kern_apfs_rl_owner = ((1 << SWI_COND_OWNER_BITS) - 1),
64 * .kern_apfs_rl_delayed_free = 1,
65 * .kern_apfs_rl_wake = 1,
66 * .kern_apfs_rl_allocated = 1,
67 * .kern_apfs_rl_allow_force = 1};
68 */
69
70#define KERN_APFS_SLEEP_DEBOUNCE_MASK ((uint64_t)0xf0000fffff)
71
72typedef struct kern_apfs_reflock {
73 union {
74 cond_swi_var64_s kern_apfs_rl_data;
75 struct {
76 uint64_t kern_apfs_rl_owner: SWI_COND_OWNER_BITS,
77 kern_apfs_rl_waiters: KERN_APFS_REFLOCK_WAITERS_BIT,
78 kern_apfs_rl_delayed_free: 1,
79 kern_apfs_rl_wake: 1,
80 kern_apfs_rl_allocated: 1,
81 kern_apfs_rl_allow_force: 1,
82 kern_apfs_rl_count: KERN_APFS_REFLOCK_REFCOUNT_BIT;
83 };
84 };
85} *kern_apfs_reflock_t;
86
87
88#else /* XNU_KERNEL_PRIVATE */
89typedef struct kern_apfs_reflock {
90 uint64_t opaque;
91} *kern_apfs_reflock_t;
92#endif /* XNU_KERNEL_PRIVATE */
93
94__options_decl(kern_apfs_reflock_in_flags_t, uint32_t, {
95 KERN_APFS_REFLOCK_IN_DEFAULT = 0x0,
96 KERN_APFS_REFLOCK_IN_LOCK_IF_LAST = 0x1,
97 KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST = 0x2,
98 KERN_APFS_REFLOCK_IN_WILL_WAIT = 0x4,
99 KERN_APFS_REFLOCK_IN_FORCE = 0x8,
100 KERN_APFS_REFLOCK_IN_ALLOW_FORCE = 0x10,
101});
102
103__options_decl(kern_apfs_reflock_out_flags_t, uint32_t, {
104 KERN_APFS_REFLOCK_OUT_DEFAULT = 0x0,
105 KERN_APFS_REFLOCK_OUT_LOCKED = 0x1,
106});
107
108__BEGIN_DECLS
109
110/*
111 * Name: kern_apfs_reflock_data
112 *
113 * Description: declares a kern_apfs_reflock variable with specified storage class.
114 * The reflock will be stored in this variable and it is the caller's responsibility
115 * to ensure that this variable's memory is going to be accessible by all threads that will use
116 * the kern_apfs_reflock.
117 * Every kern_apfs_reflock function will require a pointer to this variable as parameter.
118 *
119 * The variable needs to be initialized once with kern_apfs_reflock_init() and destroyed once with
120 * kern_apfs_reflock_destroy() when not needed anymore.
121 *
122 * Args:
123 * Arg1: storage class.
124 * Arg2: variable name.
125 */
126#define kern_apfs_reflock_data(class, name) class struct kern_apfs_reflock name
127
128/*
129 * Name: kern_apfs_reflock_init
130 *
131 * Description: initializes a kern_apfs_reflock_t.
132 *
133 * Args:
134 * Arg1: kern_apfs_reflock object.
135 *
136 * Conditions: the memory pointed by kern_apfs_reflock_t needs to be available
137 * while any of the other functions are executed.
138 * It is the caller responsibility to guarantee that all functions call
139 * executed on the kern_apfs_reflock have terminated before freeing it,
140 * including possible kern_apfs_reflock_wait_for_unlock(). If it is not possible
141 * to safely synchronize kern_apfs_reflock_wait_for_unlock() calls
142 * kern_apfs_reflock_alloc_init() should be used instead.
143 */
144void kern_apfs_reflock_init(kern_apfs_reflock_t reflock);
145
146/*
147 * Name: kern_apfs_reflock_destroy
148 *
149 * Description: destroys a kern_apfs_reflock_t.
150 *
151 * Args:
152 * Arg1: kern_apfs_reflock object.
153 *
154 * Conditions: the object must have been previously initialized with kern_apfs_reflock_init.
155 * Any access past this point to the kern_apfs_reflock will be considered invalid.
156 */
157void kern_apfs_reflock_destroy(kern_apfs_reflock_t reflock);
158
159/*
160 * Name: kern_apfs_reflock_alloc_init
161 *
162 * Description: allocates a kern_apfs_reflock_t.
163 *
164 * Returns: allocated kern_apfs_reflock_t.
165 *
166 * Conditions: It is the caller responsibility to guarantee that all functions call
167 * executed on the kern_apfs_reflock_t returned by this function
168 * (except for kern_apfs_reflock_wait_for_unlock()) have terminated before freeing it.
169 * It is safe to execute concurrent or later kern_apfs_reflock_wait_for_unlock()
170 * calls as long as the matching kern_apfs_reflock_try_get_ref(),
171 * kern_apfs_reflock_try_put_ref() or kern_apfs_reflock_try_lock() was executed before
172 * the call to kern_apfs_reflock_free(). In this case the free of the object will be delegated
173 * to the last concurrent kern_apfs_reflock_wait_for_unlock() executed.
174 */
175kern_apfs_reflock_t kern_apfs_reflock_alloc_init(void);
176
177/*
178 * Name: kern_apfs_reflock_free
179 *
180 * Description: frees and destroys a kern_apfs_reflock_t.
181 *
182 * Args:
183 * Arg1: kern_apfs_reflock object.
184 *
185 * Conditions: It is the caller responsability to guarantee that all functions call
186 * executed on the kern_apfs_reflock_t (except kern_apfs_reflock_wait_for_unlock())
187 * have terminated before freeing it.
188 * It is safe to execute concurrent or later kern_apfs_reflock_wait_for_unlock()
189 * calls as long as the matching kern_apfs_reflock_try_get_ref(),
190 * kern_apfs_reflock_try_put_ref() or kern_apfs_reflock_try_lock() was executed before
191 * the call to kern_apfs_reflock_free(). In this case the free of the object will be delegated
192 * to the last concurrent kern_apfs_reflock_wait_for_unlock() executed.
193 */
194void kern_apfs_reflock_free(kern_apfs_reflock_t reflock);
195
196/*
197 * Name: kern_apfs_reflock_try_get_ref
198 *
199 * Description: tries to get a reference on the kern_apfs_reflock.
200 * The operation will succeed if the lock on the object is not held.
201 * In case of failure the caller can choose to wait for the lock to unlock
202 * with a subsequent call to kern_apfs_reflock_wait_for_unlock().
203 *
204 * Args:
205 * Arg1: kern_apfs_reflock object.
206 * Arg2: in flags can be a combination of:
207 * - KERN_APFS_REFLOCK_IN_DEFAULT for default behaviour.
208 * - KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST will lock the reflock if the refcount was incremented
209 * in the "init" transition, so from 0->1.
210 * - KERN_APFS_REFLOCK_IN_WILL_WAIT if the try_get() fails, then the thread will call kern_apfs_reflock_wait_for_unlock().
211 * kern_apfs_reflock_wait_for_unlock() cannot be called after this function fails if this
212 * flag was not set.
213 * - KERN_APFS_REFLOCK_IN_FORCE if the reflock was locked from a kern_apfs_reflock_try_lock() with KERN_APFS_REFLOCK_IN_ALLOW_FORCE
214 * this flag will allow to get the reference even if the object is locked.
215 * Even with this flag set the function might fail if the reflock was locked from a
216 * kern_apfs_reflock_try_get_ref() or kern_apfs_reflock_try_put_ref().
217 * NOTE: KERN_APFS_REFLOCK_IN_FORCE and KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST cannot be used together.
218 * Arg3: out flags:
219 * - KERN_APFS_REFLOCK_OUT_DEFAULT if the lock was not acquired.
220 * - KERN_APFS_REFLOCK_OUT_LOCKED if the lock was acquired.
221 *
222 * Returns: true if the reference was acquired, false otherwise.
223 * If KERN_APFS_REFLOCK_IN_LOCK_IF_FIRST was set and the reference was successfully acquired, out_flags will indicate if the
224 * lock was acquired.
225 *
226 *
227 * Conditions: If KERN_APFS_REFLOCK_IN_WILL_WAIT was set, a kern_apfs_reflock_wait_for_unlock()
228 * needs to be called in case of failure.
229 * If KERN_APFS_REFLOCK_OUT_LOCKED is returned on the out_flags a corresponding kern_apfs_reflock_wait_for_unlock() needs to be called
230 * by the same thread and the thread cannot execute in userspace until the unlock is called.
231 */
232bool kern_apfs_reflock_try_get_ref(kern_apfs_reflock_t reflock, kern_apfs_reflock_in_flags_t in_flags, kern_apfs_reflock_out_flags_t *out_flags);
233
234/*
235 * Name: kern_apfs_reflock_try_put_ref
236 *
237 * Description: tries to put a reference on the kern_apfs_reflock.
238 * The operation will succeed if the lock on the object is not held.
239 * In case of failure the caller can choose to wait for the lock to unlock
240 * with a subsequent call to kern_apfs_reflock_wait_for_unlock().
241 *
242 * Args:
243 * Arg1: kern_apfs_reflock object.
244 * Arg2: in flags can be a combination of:
245 * - KERN_APFS_REFLOCK_IN_DEFAULT for default behaviour.
246 * - KERN_APFS_REFLOCK_IN_LOCK_IF_LAST will lock the reflock if the refcount was decremented
247 * in the "cleanup" transition, so from 1->0.
248 * - KERN_APFS_REFLOCK_IN_WILL_WAIT if the try_put() fails, then the thread will call kern_apfs_reflock_wait_for_unlock().
249 * kern_apfs_reflock_wait_for_unlock() cannot be called after this function fails if this
250 * flag was not set.
251 * - KERN_APFS_REFLOCK_IN_FORCE if the reflock was locked from a kern_apfs_reflock_try_lock() with KERN_APFS_REFLOCK_IN_ALLOW_FORCE
252 * this flag will allow to put the reference even if the object is locked.
253 * Even with this flag set the function might fail if the reflock was locked from a
254 * kern_apfs_reflock_try_get_ref() or kern_apfs_reflock_try_put_ref().
255 * NOTE: KERN_APFS_REFLOCK_IN_FORCE and KERN_APFS_REFLOCK_IN_LOCK_IF_LAST cannot be used together.
256 * Arg3: out flags:
257 * - KERN_APFS_REFLOCK_OUT_DEFAULT if the lock was not acquired.
258 * - KERN_APFS_REFLOCK_OUT_LOCKED if the lock was acquired.
259 *
260 * Returns: true if the reference was successfully decremented, false otherwise.
261 * If KERN_APFS_REFLOCK_IN_LOCK_IF_LAST was set and the reference was successfully decremented, out_flags will indicate if the
262 * lock was acquired.
263 *
264 *
265 * Conditions: If KERN_APFS_REFLOCK_IN_WILL_WAIT was set, a kern_apfs_reflock_wait_for_unlock()
266 * needs to be called in case of failure.
267 * If KERN_APFS_REFLOCK_OUT_LOCKED is returned on the out_flags a corresponding kern_apfs_reflock_wait_for_unlock() needs to be called
268 * by the same theread and the thread cannot execute in userspace until the unlock is called.
269 *
270 */
271bool kern_apfs_reflock_try_put_ref(kern_apfs_reflock_t reflock, kern_apfs_reflock_in_flags_t in_flags, kern_apfs_reflock_out_flags_t *out_flags);
272
273/*
274 * Name: kern_apfs_reflock_try_lock
275 *
276 * Description: tries to acquire the lock on the kern_apfs_reflock.
277 * The operation will succeed if the lock on the object is not held.
278 * In case of failure the caller can choose to wait for the lock to unlock
279 * with a subsequent call to kern_apfs_reflock_wait_for_unlock().
280 *
281 * Args:
282 * Arg1: kern_apfs_reflock object.
283 * Arg2: in flags can be a combination of:
284 * - KERN_APFS_REFLOCK_IN_DEFAULT for default behaviour.
285 * - KERN_APFS_REFLOCK_IN_WILL_WAIT if the try_lock() fails, then the thread will call kern_apfs_reflock_wait_for_unlock().
286 * kern_apfs_reflock_wait_for_unlock() cannot be called after this function fails if this
287 * flag was not set.
288 * - KERN_APFS_REFLOCK_IN_ALLOW_FORCE if this flag is set, kern_apfs_reflock_try_put_ref() and kern_apfs_reflock_try_get_ref() with
289 * flag KERN_APFS_REFLOCK_IN_FORCE set will succed even after this call locked the reflock.
290 *
291 * Arg3: refcount_when_lock pointer into which return the value of the refcount at the moment of lock.
292 *
293 * Returns: true if the lock was acquired, false otherwise.
294 *
295 * Conditions: If KERN_APFS_REFLOCK_IN_WILL_WAIT was set, a kern_apfs_reflock_wait_for_unlock()
296 * needs to be called in case of failure.
297 * If the lock was acquired a subsequent kern_apfs_reflock_unlock() by the same theread and
298 * the thread cannot execute in userspace until the unlock is called.
299 * Recursive locking is not allowed.
300 */
301bool kern_apfs_reflock_try_lock(kern_apfs_reflock_t reflock, kern_apfs_reflock_in_flags_t in_flags, uint32_t *refcount_when_lock);
302
303/*
304 * Name: kern_apfs_reflock_wait_for_unlock
305 *
306 * Description: waits for the lock to be unlocked.
307 * While waiting the priority of this thread will contribute
308 * to the priority push of the owner of the reflock.
309 * NOTE: it is not guaranteed that by the time this calls
310 * returns the reflock is unlocked, as it might have been re-locked
311 * after the current thread has been woken up.
312 * If needed, the matching kern_apfs_reflock_try_get_ref(), kern_apfs_reflock_try_put_ref() or
313 * kern_apfs_reflock_try_lock() should be re-driven after this function.
314 *
315 * Args:
316 * Arg1: reflock object.
317 * Arg2: interruptible flag for wait.
318 * Arg3: deadline for wait.
319 *
320 * Returns: result of the wait.
321 * THREAD_AWAKENED - normal wakeup
322 * THREAD_TIMED_OUT - timeout expired
323 * THREAD_INTERRUPTED - aborted/interrupted
324 * THREAD_NOT_WAITING - thread didn't need to wait
325 */
326wait_result_t kern_apfs_reflock_wait_for_unlock(kern_apfs_reflock_t reflock, wait_interrupt_t interruptible, uint64_t deadline);
327
328/*
329 * Name: kern_apfs_reflock_unlock
330 *
331 * Description: unlocks a reflock obj.
332 *
333 * Args:
334 * Arg1: reflock object.
335 *
336 * Conditions: the same thread that locked the object needs to unlock it.
337 */
338void kern_apfs_reflock_unlock(kern_apfs_reflock_t reflock);
339
340/*
341 * Name: kern_apfs_reflock_read_ref
342 *
343 * Description: reads the refcount counter.
344 * Note: using this function is racy, as the refcount can change
345 * after this function reads it. Its usage is discouraged.
346 *
347 * Args:
348 * Arg1: reflock object.
349 *
350 * Returns: refcount value.
351 */
352uint64_t kern_apfs_reflock_read_ref(kern_apfs_reflock_t reflock);
353__END_DECLS
354
355#endif /* KERNEL_PRIVATE */
356#endif /* _KERN_APFS_REFLOCK_H_ */
357