1 | /* |
2 | * Copyright (c) 2022 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | |
30 | #ifndef _KERN_LOCK_PTR_H_ |
31 | #define _KERN_LOCK_PTR_H_ |
32 | |
33 | #include <kern/lock_types.h> |
34 | #include <kern/lock_group.h> |
35 | #include <kern/lock_attr.h> |
36 | #include <vm/vm_memtag.h> |
37 | |
38 | __BEGIN_DECLS |
39 | #pragma GCC visibility push(hidden) |
40 | |
41 | #if CONFIG_KERNEL_TAGGING |
42 | #define HW_LCK_PTR_BITS (64 - 4 - 1 - 1 - 16) |
43 | #else /* !CONFIG_KERNEL_TAGGING */ |
44 | #define HW_LCK_PTR_BITS (64 - 1 - 1 - 16) |
45 | #endif /* !CONFIG_KERNEL_TAGGING */ |
46 | |
47 | /*! |
48 | * @typedef hw_lck_ptr_t |
49 | * |
50 | * @brief |
51 | * Low level lock that can share the slot of a pointer value. |
52 | * |
53 | * @discussion |
54 | * An @c hw_lck_ptr_t is a fair spinlock fitting in a single |
55 | * pointer sized word, and allows for that word to retain |
56 | * its original pointer value without any loss. |
57 | * |
58 | * It uses the top bits of the pointer which in the kernel |
59 | * aren't significant to store an MCS queue as well as various |
60 | * state bits. |
61 | * |
62 | * The pointer can't be PAC signed, but the MTESAN tag |
63 | * is preserved (when the feature is on). |
64 | * |
65 | * No assumption is made on the alignment of the pointer, |
66 | * and clients may use the low bits of the pointer to encode |
67 | * state as they see fit. |
68 | * |
69 | * It is intended to be used for fine grained locking |
70 | * in hash table bucket heads or similar structures. |
71 | * |
72 | * The pointer value can be changed by taking the lock, |
73 | * (which returns the current value of the pointer), |
74 | * and then unlock with the new pointer value. |
75 | */ |
76 | typedef union hw_lck_ptr { |
77 | struct { |
78 | intptr_t lck_ptr_bits : HW_LCK_PTR_BITS; |
79 | #if CONFIG_KERNEL_TAGGING |
80 | uintptr_t lck_ptr_tag : 4; |
81 | #endif /* CONFIG_KERNEL_TAGGING */ |
82 | uintptr_t lck_ptr_locked : 1; |
83 | uintptr_t lck_ptr_stats : 1; |
84 | uint16_t lck_ptr_mcs_tail __kernel_ptr_semantics; |
85 | }; |
86 | uintptr_t lck_ptr_value; |
87 | } hw_lck_ptr_t; |
88 | |
89 | |
90 | /* init/destroy */ |
91 | |
92 | #if !LCK_GRP_USE_ARG |
93 | #define hw_lck_ptr_init(lck, val, grp) hw_lck_ptr_init(lck, val) |
94 | #define hw_lck_ptr_destroy(lck, grp) hw_lck_ptr_destroy(lck) |
95 | #endif /* !LCK_GRP_USE_ARG */ |
96 | |
97 | /*! |
98 | * @function hw_lck_ptr_init() |
99 | * |
100 | * @brief |
101 | * Initializes an hw_lck_ptr_t with the specified pointer value. |
102 | * |
103 | * @discussion |
104 | * hw_lck_ptr_destroy() must be called to destroy this lock. |
105 | * |
106 | * @param lck the lock to initialize |
107 | * @param val the pointer value to store in the lock |
108 | * @param grp the lock group associated with this lock |
109 | */ |
110 | extern void hw_lck_ptr_init( |
111 | hw_lck_ptr_t *lck, |
112 | void *val, |
113 | lck_grp_t *grp); |
114 | |
115 | /*! |
116 | * @function hw_lck_ptr_destroy() |
117 | * |
118 | * @brief |
119 | * Detroys an hw_lck_ptr_t initialized with hw_lck_ptr_init(). |
120 | * |
121 | * @param lck the lock to destroy |
122 | * @param grp the lock group associated with this lock |
123 | */ |
124 | extern void hw_lck_ptr_destroy( |
125 | hw_lck_ptr_t *lck, |
126 | lck_grp_t *grp); |
127 | |
128 | /*! |
129 | * @function hw_lck_ptr_held() |
130 | * |
131 | * @brief |
132 | * Returns whether the pointer lock is currently held by anyone. |
133 | * |
134 | * @param lck the lock to check. |
135 | */ |
136 | extern bool hw_lck_ptr_held( |
137 | hw_lck_ptr_t *lck) __result_use_check; |
138 | |
139 | |
140 | /* lock */ |
141 | |
142 | #if !LCK_GRP_USE_ARG |
143 | #define hw_lck_ptr_lock(lck, grp) hw_lck_ptr_lock(lck) |
144 | #define hw_lck_ptr_lock_nopreempt(lck, grp) hw_lck_ptr_lock_nopreempt(lck) |
145 | #endif /* !LCK_GRP_USE_ARG */ |
146 | |
147 | |
148 | /*! |
149 | * @function hw_lck_ptr_lock() |
150 | * |
151 | * @brief |
152 | * Locks a pointer lock, and returns its current value. |
153 | * |
154 | * @discussion |
155 | * This call will disable preemption. |
156 | * |
157 | * @param lck the lock to lock |
158 | * @param grp the lock group associated with this lock |
159 | */ |
160 | extern void *hw_lck_ptr_lock( |
161 | hw_lck_ptr_t *lck, |
162 | lck_grp_t *grp) __result_use_check; |
163 | |
164 | /*! |
165 | * @function hw_lck_ptr_lock_nopreempt() |
166 | * |
167 | * @brief |
168 | * Locks a pointer lock, and returns its current value. |
169 | * |
170 | * @discussion |
171 | * Preemption must be disabled to make this call, |
172 | * and must stay disabled until @c hw_lck_ptr_unlock() |
173 | * or @c hw_lck_ptr_unlock_nopreempt() is called |
174 | * |
175 | * @param lck the lock to lock |
176 | * @param grp the lock group associated with this lock |
177 | */ |
178 | extern void *hw_lck_ptr_lock_nopreempt( |
179 | hw_lck_ptr_t *lck, |
180 | lck_grp_t *grp) __result_use_check; |
181 | |
182 | |
183 | /* unlock */ |
184 | |
185 | #if !LCK_GRP_USE_ARG |
186 | #define hw_lck_ptr_unlock(lck, val, grp) \ |
187 | hw_lck_ptr_unlock(lck, val) |
188 | #define hw_lck_ptr_unlock_nopreempt(lck, val, grp) \ |
189 | hw_lck_ptr_unlock_nopreempt(lck, val) |
190 | #endif /* !LCK_GRP_USE_ARG */ |
191 | |
192 | |
193 | /*! |
194 | * @function hw_lck_ptr_unlock() |
195 | * |
196 | * @brief |
197 | * Unlocks a pointer lock, and update its currently held value. |
198 | * |
199 | * @discussion |
200 | * This call will reenable preemption. |
201 | * |
202 | * @param lck the lock to unlock |
203 | * @param val the value to update the pointer to |
204 | * @param grp the lock group associated with this lock |
205 | */ |
206 | extern void hw_lck_ptr_unlock( |
207 | hw_lck_ptr_t *lck, |
208 | void *val, |
209 | lck_grp_t *grp); |
210 | |
211 | /*! |
212 | * @function hw_lck_ptr_unlock_nopreempt() |
213 | * |
214 | * @brief |
215 | * Unlocks a pointer lock, and update its currently held value. |
216 | * |
217 | * @discussion |
218 | * This call will not reenable preemption. |
219 | * |
220 | * @param lck the lock to unlock |
221 | * @param val the value to update the pointer to |
222 | * @param grp the lock group associated with this lock |
223 | */ |
224 | extern void hw_lck_ptr_unlock_nopreempt( |
225 | hw_lck_ptr_t *lck, |
226 | void *val, |
227 | lck_grp_t *grp); |
228 | |
229 | |
230 | /* wait_for_value */ |
231 | |
232 | #if !LCK_GRP_USE_ARG |
233 | #define hw_lck_ptr_wait_for_value(lck, val, grp) \ |
234 | hw_lck_ptr_wait_for_value(lck, val) |
235 | #endif /* !LCK_GRP_USE_ARG */ |
236 | |
237 | static inline void * |
238 | __hw_lck_ptr_value(hw_lck_ptr_t val) |
239 | { |
240 | vm_offset_t ptr = val.lck_ptr_bits; |
241 | |
242 | #if CONFIG_KERNEL_TAGGING |
243 | if (ptr) { |
244 | ptr = vm_memtag_add_ptr_tag(ptr, val.lck_ptr_tag); |
245 | } |
246 | #endif /* CONFIG_KERNEL_TAGGING */ |
247 | return (void *)ptr; |
248 | } |
249 | |
250 | |
251 | /*! |
252 | * @function hw_lck_ptr_value() |
253 | * |
254 | * @brief |
255 | * Returns the pointer value currently held by this lock. |
256 | * |
257 | * @param lck the lock to get the pointer value of. |
258 | */ |
259 | static inline void * |
260 | hw_lck_ptr_value(hw_lck_ptr_t *lck) |
261 | { |
262 | hw_lck_ptr_t tmp; |
263 | |
264 | tmp = atomic_load_explicit((hw_lck_ptr_t _Atomic *)lck, |
265 | memory_order_relaxed); |
266 | |
267 | return __hw_lck_ptr_value(val: tmp); |
268 | } |
269 | |
270 | /*! |
271 | * @function hw_lck_ptr_wait_for_value() |
272 | * |
273 | * @brief |
274 | * Spins until the pointer in the lock has the specified value. |
275 | * |
276 | * @discussion |
277 | * This function has an implicit acquire barrier pairing |
278 | * with the hw_lck_ptr_unlock() which sets the observed value. |
279 | * |
280 | * @param lck the lock to spin on |
281 | * @param val the value to wait for |
282 | * @param grp the lock group associated with this lock |
283 | */ |
284 | extern void hw_lck_ptr_wait_for_value( |
285 | hw_lck_ptr_t *lck, |
286 | void *val, |
287 | lck_grp_t *grp); |
288 | |
289 | |
290 | #pragma GCC visibility pop |
291 | __END_DECLS |
292 | |
293 | #endif /* _KERN_LOCK_PTR_H_ */ |
294 | |