1/*
2 * Copyright (c) 2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _ARM_ATOMIC_H_
30#define _ARM_ATOMIC_H_
31
32#include <mach/boolean.h>
33#include <arm/smp.h>
34
35// Parameter for __builtin_arm_dmb
36#define DMB_NSH 0x7
37#define DMB_ISHLD 0x9
38#define DMB_ISHST 0xa
39#define DMB_ISH 0xb
40#define DMB_SY 0xf
41
42// Parameter for __builtin_arm_dsb
43#define DSB_NSH 0x7
44#define DSB_ISHLD 0x9
45#define DSB_ISHST 0xa
46#define DSB_ISH 0xb
47#define DSB_SY 0xf
48
49// Parameter for __builtin_arm_isb
50#define ISB_SY 0xf
51
52#if __SMP__
53
54#define memory_order_consume_smp memory_order_consume
55#define memory_order_acquire_smp memory_order_acquire
56#define memory_order_release_smp memory_order_release
57#define memory_order_acq_rel_smp memory_order_acq_rel
58#define memory_order_seq_cst_smp memory_order_seq_cst
59
60#else
61
62#define memory_order_consume_smp memory_order_relaxed
63#define memory_order_acquire_smp memory_order_relaxed
64#define memory_order_release_smp memory_order_relaxed
65#define memory_order_acq_rel_smp memory_order_relaxed
66#define memory_order_seq_cst_smp memory_order_relaxed
67
68#endif
69
70/*
71 * Atomic operations functions
72 *
73 * These static functions are designed for inlining
74 * It is expected that the memory_order arguments are
75 * known at compile time. This collapses these
76 * functions into a simple atomic operation
77 */
78
79static inline boolean_t
80memory_order_has_acquire(enum memory_order ord)
81{
82 switch (ord) {
83 case memory_order_consume:
84 case memory_order_acquire:
85 case memory_order_acq_rel:
86 case memory_order_seq_cst:
87 return TRUE;
88 default:
89 return FALSE;
90 }
91}
92
93static inline boolean_t
94memory_order_has_release(enum memory_order ord)
95{
96 switch (ord) {
97 case memory_order_release:
98 case memory_order_acq_rel:
99 case memory_order_seq_cst:
100 return TRUE;
101 default:
102 return FALSE;
103 }
104}
105
106#ifdef ATOMIC_PRIVATE
107
108#define clear_exclusive() __builtin_arm_clrex()
109
110__unused static uint32_t
111load_exclusive32(uint32_t *target, enum memory_order ord)
112{
113 uint32_t value;
114
115#if __arm__
116 if (memory_order_has_release(ord)) {
117 // Pre-load release barrier
118 atomic_thread_fence(memory_order_release);
119 }
120 value = __builtin_arm_ldrex(target);
121#else
122 if (memory_order_has_acquire(ord))
123 value = __builtin_arm_ldaex(target); // ldaxr
124 else
125 value = __builtin_arm_ldrex(target); // ldxr
126#endif // __arm__
127 return value;
128}
129
130__unused static boolean_t
131store_exclusive32(uint32_t *target, uint32_t value, enum memory_order ord)
132{
133 boolean_t err;
134
135#if __arm__
136 err = __builtin_arm_strex(value, target);
137 if (memory_order_has_acquire(ord)) {
138 // Post-store acquire barrier
139 atomic_thread_fence(memory_order_acquire);
140 }
141#else
142 if (memory_order_has_release(ord))
143 err = __builtin_arm_stlex(value, target); // stlxr
144 else
145 err = __builtin_arm_strex(value, target); // stxr
146#endif // __arm__
147 return !err;
148}
149
150__unused static uintptr_t
151load_exclusive(uintptr_t *target, enum memory_order ord)
152{
153#if !__LP64__
154 return load_exclusive32((uint32_t *)target, ord);
155#else
156 uintptr_t value;
157
158 if (memory_order_has_acquire(ord))
159 value = __builtin_arm_ldaex(target); // ldaxr
160 else
161 value = __builtin_arm_ldrex(target); // ldxr
162 return value;
163#endif // __arm__
164}
165
166__unused static boolean_t
167store_exclusive(uintptr_t *target, uintptr_t value, enum memory_order ord)
168{
169#if !__LP64__
170 return store_exclusive32((uint32_t *)target, value, ord);
171#else
172 boolean_t err;
173
174 if (memory_order_has_release(ord))
175 err = __builtin_arm_stlex(value, target); // stlxr
176 else
177 err = __builtin_arm_strex(value, target); // stxr
178 return !err;
179#endif
180}
181
182__unused static boolean_t
183atomic_compare_exchange(uintptr_t *target, uintptr_t oldval, uintptr_t newval,
184 enum memory_order orig_ord, boolean_t wait)
185{
186 enum memory_order ord = orig_ord;
187 uintptr_t value;
188
189
190#if __arm__
191 ord = memory_order_relaxed;
192 if (memory_order_has_release(orig_ord)) {
193 atomic_thread_fence(memory_order_release);
194 }
195#endif
196 do {
197 value = load_exclusive(target, ord);
198 if (value != oldval) {
199 if (wait)
200 wait_for_event(); // Wait with monitor held
201 else
202 clear_exclusive(); // Clear exclusive monitor
203 return FALSE;
204 }
205 } while (!store_exclusive(target, newval, ord));
206#if __arm__
207 if (memory_order_has_acquire(orig_ord)) {
208 atomic_thread_fence(memory_order_acquire);
209 }
210#endif
211 return TRUE;
212}
213
214#endif // ATOMIC_PRIVATE
215
216#if __arm__
217#undef os_atomic_rmw_loop
218#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \
219 boolean_t _result = FALSE; uint32_t _err = 0; \
220 typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \
221 for (;;) { \
222 ov = __builtin_arm_ldrex(_p); \
223 __VA_ARGS__; \
224 if (!_err && memory_order_has_release(memory_order_##m)) { \
225 /* only done for the first loop iteration */ \
226 atomic_thread_fence(memory_order_release); \
227 } \
228 _err = __builtin_arm_strex(nv, _p); \
229 if (__builtin_expect(!_err, 1)) { \
230 if (memory_order_has_acquire(memory_order_##m)) { \
231 atomic_thread_fence(memory_order_acquire); \
232 } \
233 _result = TRUE; \
234 break; \
235 } \
236 } \
237 _result; \
238 })
239
240#undef os_atomic_rmw_loop_give_up
241#define os_atomic_rmw_loop_give_up(expr) \
242 ({ __builtin_arm_clrex(); expr; __builtin_trap(); })
243
244#else
245
246#undef os_atomic_rmw_loop
247#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \
248 boolean_t _result = FALSE; \
249 typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \
250 do { \
251 if (memory_order_has_acquire(memory_order_##m)) { \
252 ov = __builtin_arm_ldaex(_p); \
253 } else { \
254 ov = __builtin_arm_ldrex(_p); \
255 } \
256 __VA_ARGS__; \
257 if (memory_order_has_release(memory_order_##m)) { \
258 _result = !__builtin_arm_stlex(nv, _p); \
259 } else { \
260 _result = !__builtin_arm_strex(nv, _p); \
261 } \
262 } while (__builtin_expect(!_result, 0)); \
263 _result; \
264 })
265
266#undef os_atomic_rmw_loop_give_up
267#define os_atomic_rmw_loop_give_up(expr) \
268 ({ __builtin_arm_clrex(); expr; __builtin_trap(); })
269#endif
270
271#undef os_atomic_force_dependency_on
272#if defined(__arm64__)
273#define os_atomic_force_dependency_on(p, e) ({ \
274 unsigned long _v; \
275 __asm__("and %x[_v], %x[_e], xzr" : [_v] "=r" (_v) : [_e] "r" (e)); \
276 (typeof(*(p)) *)((char *)(p) + _v); \
277 })
278#else
279#define os_atomic_force_dependency_on(p, e) ({ \
280 unsigned long _v; \
281 __asm__("and %[_v], %[_e], #0" : [_v] "=r" (_v) : [_e] "r" (e)); \
282 (typeof(*(p)) *)((char *)(p) + _v); \
283 })
284#endif // defined(__arm64__)
285
286#endif // _ARM_ATOMIC_H_
287