1 | /* |
2 | * Copyright (c) 2015 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _ARM_ATOMIC_H_ |
30 | #define _ARM_ATOMIC_H_ |
31 | |
32 | #include <mach/boolean.h> |
33 | #include <arm/smp.h> |
34 | |
35 | // Parameter for __builtin_arm_dmb |
36 | #define DMB_NSH 0x7 |
37 | #define DMB_ISHLD 0x9 |
38 | #define DMB_ISHST 0xa |
39 | #define DMB_ISH 0xb |
40 | #define DMB_SY 0xf |
41 | |
42 | // Parameter for __builtin_arm_dsb |
43 | #define DSB_NSH 0x7 |
44 | #define DSB_ISHLD 0x9 |
45 | #define DSB_ISHST 0xa |
46 | #define DSB_ISH 0xb |
47 | #define DSB_SY 0xf |
48 | |
49 | // Parameter for __builtin_arm_isb |
50 | #define ISB_SY 0xf |
51 | |
52 | #if __SMP__ |
53 | |
54 | #define memory_order_consume_smp memory_order_consume |
55 | #define memory_order_acquire_smp memory_order_acquire |
56 | #define memory_order_release_smp memory_order_release |
57 | #define memory_order_acq_rel_smp memory_order_acq_rel |
58 | #define memory_order_seq_cst_smp memory_order_seq_cst |
59 | |
60 | #else |
61 | |
62 | #define memory_order_consume_smp memory_order_relaxed |
63 | #define memory_order_acquire_smp memory_order_relaxed |
64 | #define memory_order_release_smp memory_order_relaxed |
65 | #define memory_order_acq_rel_smp memory_order_relaxed |
66 | #define memory_order_seq_cst_smp memory_order_relaxed |
67 | |
68 | #endif |
69 | |
70 | /* |
71 | * Atomic operations functions |
72 | * |
73 | * These static functions are designed for inlining |
74 | * It is expected that the memory_order arguments are |
75 | * known at compile time. This collapses these |
76 | * functions into a simple atomic operation |
77 | */ |
78 | |
79 | static inline boolean_t |
80 | memory_order_has_acquire(enum memory_order ord) |
81 | { |
82 | switch (ord) { |
83 | case memory_order_consume: |
84 | case memory_order_acquire: |
85 | case memory_order_acq_rel: |
86 | case memory_order_seq_cst: |
87 | return TRUE; |
88 | default: |
89 | return FALSE; |
90 | } |
91 | } |
92 | |
93 | static inline boolean_t |
94 | memory_order_has_release(enum memory_order ord) |
95 | { |
96 | switch (ord) { |
97 | case memory_order_release: |
98 | case memory_order_acq_rel: |
99 | case memory_order_seq_cst: |
100 | return TRUE; |
101 | default: |
102 | return FALSE; |
103 | } |
104 | } |
105 | |
106 | #ifdef ATOMIC_PRIVATE |
107 | |
108 | #define clear_exclusive() __builtin_arm_clrex() |
109 | |
110 | __unused static uint32_t |
111 | load_exclusive32(uint32_t *target, enum memory_order ord) |
112 | { |
113 | uint32_t value; |
114 | |
115 | #if __arm__ |
116 | if (memory_order_has_release(ord)) { |
117 | // Pre-load release barrier |
118 | atomic_thread_fence(memory_order_release); |
119 | } |
120 | value = __builtin_arm_ldrex(target); |
121 | #else |
122 | if (memory_order_has_acquire(ord)) |
123 | value = __builtin_arm_ldaex(target); // ldaxr |
124 | else |
125 | value = __builtin_arm_ldrex(target); // ldxr |
126 | #endif // __arm__ |
127 | return value; |
128 | } |
129 | |
130 | __unused static boolean_t |
131 | store_exclusive32(uint32_t *target, uint32_t value, enum memory_order ord) |
132 | { |
133 | boolean_t err; |
134 | |
135 | #if __arm__ |
136 | err = __builtin_arm_strex(value, target); |
137 | if (memory_order_has_acquire(ord)) { |
138 | // Post-store acquire barrier |
139 | atomic_thread_fence(memory_order_acquire); |
140 | } |
141 | #else |
142 | if (memory_order_has_release(ord)) |
143 | err = __builtin_arm_stlex(value, target); // stlxr |
144 | else |
145 | err = __builtin_arm_strex(value, target); // stxr |
146 | #endif // __arm__ |
147 | return !err; |
148 | } |
149 | |
150 | __unused static uintptr_t |
151 | load_exclusive(uintptr_t *target, enum memory_order ord) |
152 | { |
153 | #if !__LP64__ |
154 | return load_exclusive32((uint32_t *)target, ord); |
155 | #else |
156 | uintptr_t value; |
157 | |
158 | if (memory_order_has_acquire(ord)) |
159 | value = __builtin_arm_ldaex(target); // ldaxr |
160 | else |
161 | value = __builtin_arm_ldrex(target); // ldxr |
162 | return value; |
163 | #endif // __arm__ |
164 | } |
165 | |
166 | __unused static boolean_t |
167 | store_exclusive(uintptr_t *target, uintptr_t value, enum memory_order ord) |
168 | { |
169 | #if !__LP64__ |
170 | return store_exclusive32((uint32_t *)target, value, ord); |
171 | #else |
172 | boolean_t err; |
173 | |
174 | if (memory_order_has_release(ord)) |
175 | err = __builtin_arm_stlex(value, target); // stlxr |
176 | else |
177 | err = __builtin_arm_strex(value, target); // stxr |
178 | return !err; |
179 | #endif |
180 | } |
181 | |
182 | __unused static boolean_t |
183 | atomic_compare_exchange(uintptr_t *target, uintptr_t oldval, uintptr_t newval, |
184 | enum memory_order orig_ord, boolean_t wait) |
185 | { |
186 | enum memory_order ord = orig_ord; |
187 | uintptr_t value; |
188 | |
189 | |
190 | #if __arm__ |
191 | ord = memory_order_relaxed; |
192 | if (memory_order_has_release(orig_ord)) { |
193 | atomic_thread_fence(memory_order_release); |
194 | } |
195 | #endif |
196 | do { |
197 | value = load_exclusive(target, ord); |
198 | if (value != oldval) { |
199 | if (wait) |
200 | wait_for_event(); // Wait with monitor held |
201 | else |
202 | clear_exclusive(); // Clear exclusive monitor |
203 | return FALSE; |
204 | } |
205 | } while (!store_exclusive(target, newval, ord)); |
206 | #if __arm__ |
207 | if (memory_order_has_acquire(orig_ord)) { |
208 | atomic_thread_fence(memory_order_acquire); |
209 | } |
210 | #endif |
211 | return TRUE; |
212 | } |
213 | |
214 | #endif // ATOMIC_PRIVATE |
215 | |
216 | #if __arm__ |
217 | #undef os_atomic_rmw_loop |
218 | #define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ |
219 | boolean_t _result = FALSE; uint32_t _err = 0; \ |
220 | typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \ |
221 | for (;;) { \ |
222 | ov = __builtin_arm_ldrex(_p); \ |
223 | __VA_ARGS__; \ |
224 | if (!_err && memory_order_has_release(memory_order_##m)) { \ |
225 | /* only done for the first loop iteration */ \ |
226 | atomic_thread_fence(memory_order_release); \ |
227 | } \ |
228 | _err = __builtin_arm_strex(nv, _p); \ |
229 | if (__builtin_expect(!_err, 1)) { \ |
230 | if (memory_order_has_acquire(memory_order_##m)) { \ |
231 | atomic_thread_fence(memory_order_acquire); \ |
232 | } \ |
233 | _result = TRUE; \ |
234 | break; \ |
235 | } \ |
236 | } \ |
237 | _result; \ |
238 | }) |
239 | |
240 | #undef os_atomic_rmw_loop_give_up |
241 | #define os_atomic_rmw_loop_give_up(expr) \ |
242 | ({ __builtin_arm_clrex(); expr; __builtin_trap(); }) |
243 | |
244 | #else |
245 | |
246 | #undef os_atomic_rmw_loop |
247 | #define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ |
248 | boolean_t _result = FALSE; \ |
249 | typeof(atomic_load(p)) *_p = (typeof(atomic_load(p)) *)(p); \ |
250 | do { \ |
251 | if (memory_order_has_acquire(memory_order_##m)) { \ |
252 | ov = __builtin_arm_ldaex(_p); \ |
253 | } else { \ |
254 | ov = __builtin_arm_ldrex(_p); \ |
255 | } \ |
256 | __VA_ARGS__; \ |
257 | if (memory_order_has_release(memory_order_##m)) { \ |
258 | _result = !__builtin_arm_stlex(nv, _p); \ |
259 | } else { \ |
260 | _result = !__builtin_arm_strex(nv, _p); \ |
261 | } \ |
262 | } while (__builtin_expect(!_result, 0)); \ |
263 | _result; \ |
264 | }) |
265 | |
266 | #undef os_atomic_rmw_loop_give_up |
267 | #define os_atomic_rmw_loop_give_up(expr) \ |
268 | ({ __builtin_arm_clrex(); expr; __builtin_trap(); }) |
269 | #endif |
270 | |
271 | #undef os_atomic_force_dependency_on |
272 | #if defined(__arm64__) |
273 | #define os_atomic_force_dependency_on(p, e) ({ \ |
274 | unsigned long _v; \ |
275 | __asm__("and %x[_v], %x[_e], xzr" : [_v] "=r" (_v) : [_e] "r" (e)); \ |
276 | (typeof(*(p)) *)((char *)(p) + _v); \ |
277 | }) |
278 | #else |
279 | #define os_atomic_force_dependency_on(p, e) ({ \ |
280 | unsigned long _v; \ |
281 | __asm__("and %[_v], %[_e], #0" : [_v] "=r" (_v) : [_e] "r" (e)); \ |
282 | (typeof(*(p)) *)((char *)(p) + _v); \ |
283 | }) |
284 | #endif // defined(__arm64__) |
285 | |
286 | #endif // _ARM_ATOMIC_H_ |
287 | |