1/*
2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * Copyright (C) 1998 Apple Computer
30 * All Rights Reserved
31 */
32/*
33 * @OSF_COPYRIGHT@
34 */
35/*
36 * Mach Operating System
37 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
38 * All Rights Reserved.
39 *
40 * Permission to use, copy, modify and distribute this software and its
41 * documentation is hereby granted, provided that both the copyright
42 * notice and this permission notice appear in all copies of the
43 * software, derivative works or modified versions, and any portions
44 * thereof, and that both notices appear in supporting documentation.
45 *
46 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
47 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
48 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
49 *
50 * Carnegie Mellon requests users of this software to return to
51 *
52 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
53 * School of Computer Science
54 * Carnegie Mellon University
55 * Pittsburgh PA 15213-3890
56 *
57 * any improvements or extensions that they make and grant Carnegie Mellon
58 * the rights to redistribute these changes.
59 */
60/*
61 * File: kern/simple_lock.h (derived from kern/lock.h)
62 * Author: Avadis Tevanian, Jr., Michael Wayne Young
63 * Date: 1985
64 *
65 * Atomic primitives and Simple Locking primitives definitions
66 */
67
68#ifdef KERNEL_PRIVATE
69
70#ifndef _KERN_SIMPLE_LOCK_H_
71#define _KERN_SIMPLE_LOCK_H_
72
73#include <mach/boolean.h>
74#include <kern/lock_types.h>
75#include <kern/lock_group.h>
76#include <machine/simple_lock.h>
77
78#ifdef XNU_KERNEL_PRIVATE
79
80#if MACH_KERNEL_PRIVATE
81#include <machine/atomic.h>
82#include <mach_ldebug.h>
83#endif
84
85__BEGIN_DECLS
86
87#pragma GCC visibility push(hidden)
88
89#ifdef MACH_KERNEL_PRIVATE
90
91#define HW_LOCK_STATE_TO_THREAD(state) ((thread_t)(state))
92#define HW_LOCK_THREAD_TO_STATE(thread) ((uintptr_t)(thread))
93
94extern void hw_lock_init(
95 hw_lock_t);
96
97extern void hw_lock_lock(
98 hw_lock_t
99 LCK_GRP_ARG(lck_grp_t*));
100
101extern void hw_lock_lock_nopreempt(
102 hw_lock_t
103 LCK_GRP_ARG(lck_grp_t*));
104
105extern unsigned int hw_lock_to(
106 hw_lock_t,
107 hw_spin_policy_t
108 LCK_GRP_ARG(lck_grp_t*)) __result_use_check;
109
110extern unsigned int hw_lock_to_nopreempt(
111 hw_lock_t,
112 hw_spin_policy_t
113 LCK_GRP_ARG(lck_grp_t*)) __result_use_check;
114
115extern unsigned int hw_lock_try(
116 hw_lock_t
117 LCK_GRP_ARG(lck_grp_t*)) __result_use_check;
118
119extern unsigned int hw_lock_try_nopreempt(
120 hw_lock_t
121 LCK_GRP_ARG(lck_grp_t*)) __result_use_check;
122
123#if !LCK_GRP_USE_ARG
124#define hw_lock_lock(lck, grp) \
125 hw_lock_lock(lck)
126
127#define hw_lock_lock_nopreempt(lck, grp) \
128 hw_lock_lock_nopreempt(lck)
129
130#define hw_lock_to(lck, spec, grp) \
131 hw_lock_to(lck, spec)
132
133#define hw_lock_to_nopreempt(lck, spec, grp) \
134 hw_lock_to_nopreempt(lck, spec)
135
136#define hw_lock_try(lck, grp) \
137 hw_lock_try(lck)
138
139#define hw_lock_try_nopreempt(lck, grp) \
140 hw_lock_try_nopreempt(lck)
141#endif /* !LCK_GRP_USE_ARG */
142
143extern void hw_lock_unlock(
144 hw_lock_t);
145
146extern void hw_lock_unlock_nopreempt(
147 hw_lock_t);
148
149extern void hw_lock_assert(
150 hw_lock_t lock,
151 unsigned int type);
152
153extern unsigned int hw_lock_held(
154 hw_lock_t) __result_use_check;
155
156extern boolean_t hw_atomic_test_and_set32(
157 uint32_t *target,
158 uint32_t test_mask,
159 uint32_t set_mask,
160 enum memory_order ord,
161 boolean_t wait);
162
163extern boolean_t atomic_test_and_set32(
164 uint32_t *target,
165 uint32_t test_mask,
166 uint32_t set_mask,
167 enum memory_order ord,
168 boolean_t wait);
169
170extern void atomic_exchange_abort(
171 void);
172
173extern boolean_t atomic_exchange_complete32(
174 uint32_t *target,
175 uint32_t previous,
176 uint32_t newval,
177 enum memory_order ord);
178
179extern uint32_t atomic_exchange_begin32(
180 uint32_t *target,
181 uint32_t *previous,
182 enum memory_order ord);
183
184#if defined(__arm__) || defined(__arm64__)
185uint32_t load_exclusive32(
186 uint32_t *target,
187 enum memory_order ord);
188boolean_t store_exclusive32(
189 uint32_t *target,
190 uint32_t value,
191 enum memory_order ord);
192#endif /* defined(__arm__)||defined(__arm64__) */
193
194extern void usimple_unlock_nopreempt(
195 usimple_lock_t);
196
197extern hw_spin_timeout_t hw_spin_compute_timeout(
198 hw_spin_policy_t policy);
199
200extern bool hw_spin_in_ppl(
201 hw_spin_timeout_t to) __pure2;
202
203extern bool hw_spin_should_keep_spinning(
204 void *lock,
205 hw_spin_policy_t policy,
206 hw_spin_timeout_t to,
207 hw_spin_state_t *state);
208
209#endif /* MACH_KERNEL_PRIVATE */
210
211struct usimple_lock_startup_spec {
212 usimple_lock_t lck;
213 unsigned short lck_init_arg;
214};
215
216extern void usimple_lock_startup_init(
217 struct usimple_lock_startup_spec *spec);
218
219#define SIMPLE_LOCK_DECLARE(var, arg) \
220 decl_simple_lock_data(, var); \
221 static __startup_data struct usimple_lock_startup_spec \
222 __startup_usimple_lock_spec_ ## var = { &var, arg }; \
223 STARTUP_ARG(LOCKS, STARTUP_RANK_FOURTH, usimple_lock_startup_init, \
224 &__startup_usimple_lock_spec_ ## var)
225
226extern uint32_t hw_wait_while_equals32(
227 uint32_t *address,
228 uint32_t current);
229
230extern uint64_t hw_wait_while_equals64(
231 uint64_t *address,
232 uint64_t current);
233
234#if __LP64__
235#define hw_wait_while_equals_long(ptr, cur) ({ \
236 static_assert(sizeof(*(ptr)) == sizeof(long)); \
237 (typeof(cur))hw_wait_while_equals64(__DEVOLATILE(uint64_t *, ptr), (uint64_t)(cur)); \
238})
239#else
240#define hw_wait_while_equals_long(ptr, cur) ({ \
241 static_assert(sizeof(*(ptr)) == sizeof(long)); \
242 (typeof(cur))hw_wait_while_equals32(__DEVOLATILE(uint32_t *, ptr), (uint32_t)(cur)); \
243})
244#endif
245
246
247extern void usimple_lock_init(
248 usimple_lock_t,
249 unsigned short);
250
251extern void usimple_lock(
252 usimple_lock_t
253 LCK_GRP_ARG(lck_grp_t*));
254
255extern unsigned int usimple_lock_try(
256 usimple_lock_t
257 LCK_GRP_ARG(lck_grp_t*)) __result_use_check;
258
259extern void usimple_lock_assert(
260 usimple_lock_t lock,
261 unsigned int type);
262
263extern void usimple_lock_try_lock_loop(
264 usimple_lock_t
265 LCK_GRP_ARG(lck_grp_t*));
266
267#if defined(__x86_64__)
268extern unsigned int usimple_lock_try_lock_mp_signal_safe_loop_deadline(
269 usimple_lock_t,
270 uint64_t
271 LCK_GRP_ARG(lck_grp_t*)) /* __result_use_check */;
272
273extern unsigned int usimple_lock_try_lock_mp_signal_safe_loop_duration(
274 usimple_lock_t,
275 uint64_t
276 LCK_GRP_ARG(lck_grp_t*)) __result_use_check;
277#endif
278
279extern void usimple_unlock(
280 usimple_lock_t);
281
282#if !LCK_GRP_USE_ARG
283#define usimple_lock(lck, grp) \
284 usimple_lock(lck)
285
286#define usimple_lock_try(lck, grp) \
287 usimple_lock_try(lck)
288
289#define usimple_lock_try_lock_loop(lck, grp) \
290 usimple_lock_try_lock_loop(lck)
291
292#if defined(__x86_64__)
293#define usimple_lock_try_lock_mp_signal_safe_loop_deadline(lck, ddl, grp) \
294 usimple_lock_try_lock_mp_signal_safe_loop_deadline(lck, ddl)
295#define usimple_lock_try_lock_mp_signal_safe_loop_duration(lck, dur, grp) \
296 usimple_lock_try_lock_mp_signal_safe_loop_duration(lck, dur)
297#endif
298#endif /* !LCK_GRP_USE_ARG */
299
300
301/*
302 * If we got to here and we still don't have simple_lock_init
303 * defined, then we must either be outside the osfmk component,
304 * running on a true SMP, or need debug.
305 */
306#if !defined(simple_lock_init)
307#define simple_lock_init(l, t) usimple_lock_init(l,t)
308#define simple_lock(l, grp) usimple_lock(l, grp)
309#define simple_unlock(l) usimple_unlock(l)
310#define simple_lock_assert(l, x) usimple_lock_assert((l), (x))
311#define simple_lock_try(l, grp) usimple_lock_try(l, grp)
312#define simple_lock_try_lock_loop(l, grp) usimple_lock_try_lock_loop(l, grp)
313#define simple_lock_try_lock_mp_signal_safe_loop_deadline(l, ddl, grp) \
314 usimple_lock_try_lock_mp_signal_safe_loop_deadline(l, ddl, grp)
315#define simple_lock_try_lock_mp_signal_safe_loop_duration(l, dur, grp) \
316 usimple_lock_try_lock_mp_signal_safe_loop_duration(l, dur, grp)
317#define simple_lock_addr(l) (&(l))
318#endif /* !defined(simple_lock_init) */
319
320#ifdef MACH_KERNEL_PRIVATE
321
322typedef uint32_t hw_lock_bit_t;
323
324extern const struct hw_spin_policy hw_lock_bit_policy;
325#if __arm64__
326extern const struct hw_spin_policy hw_lock_bit_policy_2s;
327#endif
328extern const struct hw_spin_policy hw_lock_spin_policy;
329extern const struct hw_spin_policy hw_lock_spin_panic_policy;
330#if DEBUG || DEVELOPMENT
331extern const struct hw_spin_policy hw_lock_test_give_up_policy;
332#endif /* DEBUG || DEVELOPMENT */
333
334extern void hw_lock_bit(
335 hw_lock_bit_t *,
336 unsigned int
337 LCK_GRP_ARG(lck_grp_t*));
338
339extern void hw_lock_bit_nopreempt(
340 hw_lock_bit_t *,
341 unsigned int
342 LCK_GRP_ARG(lck_grp_t*));
343
344
345extern bool hw_lock_bit_try(
346 hw_lock_bit_t *,
347 unsigned int
348 LCK_GRP_ARG(lck_grp_t*)) __result_use_check;
349
350extern unsigned int hw_lock_bit_to(
351 hw_lock_bit_t *,
352 unsigned int,
353 hw_spin_policy_t
354 LCK_GRP_ARG(lck_grp_t*)) __result_use_check;
355
356extern hw_lock_status_t hw_lock_bit_to_b(
357 hw_lock_bit_t *,
358 unsigned int,
359 hw_spin_policy_t,
360 bool (^lock_pause)(void)
361 LCK_GRP_ARG(lck_grp_t*));
362
363extern void hw_unlock_bit(
364 hw_lock_bit_t *,
365 unsigned int);
366
367extern void hw_unlock_bit_nopreempt(
368 hw_lock_bit_t *,
369 unsigned int);
370
371#define hw_lock_bit_held(l, b) \
372 (((*(l)) & (1 << (b))) != 0)
373
374#if !LCK_GRP_USE_ARG
375#define hw_lock_bit(lck, bit, grp) \
376 hw_lock_bit(lck, bit)
377
378#define hw_lock_bit_nopreempt(lck, bit, grp) \
379 hw_lock_bit_nopreempt(lck, bit)
380
381
382#define hw_lock_bit_try(lck, bit, grp) \
383 hw_lock_bit_try(lck, bit)
384
385#define hw_lock_bit_to(lck, bit, spec, grp) \
386 hw_lock_bit_to(lck, bit, spec)
387
388#define hw_lock_bit_to_b(lck, bit, spec, pause, grp) \
389 hw_lock_bit_to_b(lck, bit, spec, pause)
390
391#endif /* !LCK_GRP_USE_ARG */
392#endif /* MACH_KERNEL_PRIVATE */
393
394__END_DECLS
395
396#pragma GCC visibility pop
397
398#endif /* XNU_KERNEL_PRIVATE */
399#endif /*!_KERN_SIMPLE_LOCK_H_*/
400
401#endif /* KERNEL_PRIVATE */
402