1/*
2 * Copyright (c) 2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/*
30 * This header provides some gory details to implement the <os/atomic_private.h>
31 * interfaces. Nothing in this header should be called directly, no promise is
32 * made to keep this interface stable.
33 *
34 * Architecture overrides.
35 */
36
37#ifndef __OS_ATOMIC_PRIVATE_H__
38#error "Do not include <os/atomic_private_arch.h> directly, use <os/atomic_private.h>"
39#endif
40
41#ifndef __OS_ATOMIC_PRIVATE_ARCH_H__
42#define __OS_ATOMIC_PRIVATE_ARCH_H__
43
44#pragma mark - arm v7
45
46#if defined(__arm__)
47
48#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
49/*
50 * On armv7, we do provide fine grained dependency injection, so
51 * memory_order_dependency maps to relaxed as far as thread fences are concerned
52 */
53#undef _os_atomic_mo_dependency
54#define _os_atomic_mo_dependency memory_order_relaxed
55
56#undef os_atomic_make_dependency
57#define os_atomic_make_dependency(v) ({ \
58 os_atomic_dependency_t _dep; \
59 __asm__ __volatile__("and %[_dep], %[_v], #0" \
60 : [_dep] "=r" (_dep.__opaque_zero) \
61 : [_v] "r" (v)); \
62 os_compiler_barrier(acquire); \
63 _dep; \
64})
65#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
66
67#define os_atomic_clear_exclusive() __builtin_arm_clrex()
68
69#define os_atomic_load_exclusive(p, m) ({ \
70 os_atomic_basetypeof(p) _r = __builtin_arm_ldrex(os_cast_to_nonatomic_pointer(p)); \
71 _os_memory_fence_after_atomic(m); \
72 _os_compiler_barrier_after_atomic(m); \
73 _r; \
74})
75
76#define os_atomic_store_exclusive(p, v, m) ({ \
77 _os_compiler_barrier_before_atomic(m); \
78 _os_memory_fence_before_atomic(m); \
79 !__builtin_arm_strex(v, os_cast_to_nonatomic_pointer(p)); \
80})
81
82/*
83 * armv7 override of os_atomic_rmw_loop
84 * documentation for os_atomic_rmw_loop is in <os/atomic_private.h>
85 */
86#undef os_atomic_rmw_loop
87#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \
88 int _result = 0; uint32_t _err = 0; \
89 __auto_type *_p = os_cast_to_nonatomic_pointer(p); \
90 for (;;) { \
91 ov = __builtin_arm_ldrex(_p); \
92 __VA_ARGS__; \
93 if (!_err) { \
94 /* release barrier only done for the first loop iteration */ \
95 _os_memory_fence_before_atomic(m); \
96 } \
97 _err = __builtin_arm_strex(nv, _p); \
98 if (__builtin_expect(!_err, 1)) { \
99 _os_memory_fence_after_atomic(m); \
100 _result = 1; \
101 break; \
102 } \
103 } \
104 _os_compiler_barrier_after_atomic(m); \
105 _result; \
106})
107
108/*
109 * armv7 override of os_atomic_rmw_loop_give_up
110 * documentation for os_atomic_rmw_loop_give_up is in <os/atomic_private.h>
111 */
112#undef os_atomic_rmw_loop_give_up
113#define os_atomic_rmw_loop_give_up(...) \
114 ({ os_atomic_clear_exclusive(); __VA_ARGS__; break; })
115
116#endif // __arm__
117
118#pragma mark - arm64
119
120#if defined(__arm64__)
121
122#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
123/*
124 * On arm64, we do provide fine grained dependency injection, so
125 * memory_order_dependency maps to relaxed as far as thread fences are concerned
126 */
127#undef _os_atomic_mo_dependency
128#define _os_atomic_mo_dependency memory_order_relaxed
129
130#undef os_atomic_make_dependency
131#if __ARM64_ARCH_8_32__
132#define os_atomic_make_dependency(v) ({ \
133 os_atomic_dependency_t _dep; \
134 __asm__ __volatile__("and %w[_dep], %w[_v], wzr" \
135 : [_dep] "=r" (_dep.__opaque_zero) \
136 : [_v] "r" (v)); \
137 os_compiler_barrier(acquire); \
138 _dep; \
139})
140#else
141#define os_atomic_make_dependency(v) ({ \
142 os_atomic_dependency_t _dep; \
143 __asm__ __volatile__("and %[_dep], %[_v], xzr" \
144 : [_dep] "=r" (_dep.__opaque_zero) \
145 : [_v] "r" (v)); \
146 os_compiler_barrier(acquire); \
147 _dep; \
148})
149#endif
150#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
151
152#if defined(__ARM_ARCH_8_4__)
153/* on armv8.4 16-byte aligned load/store pair is atomic */
154#undef os_atomic_load_is_plain
155#define os_atomic_load_is_plain(p) (sizeof(*(p)) <= 16)
156#endif
157
158#define os_atomic_clear_exclusive() __builtin_arm_clrex()
159
160#define os_atomic_load_exclusive(p, m) ({ \
161 os_atomic_basetypeof(p) _r = _os_atomic_mo_has_acquire(_os_atomic_mo_##m##_smp) \
162 ? __builtin_arm_ldaex(os_cast_to_nonatomic_pointer(p)) \
163 : __builtin_arm_ldrex(os_cast_to_nonatomic_pointer(p)); \
164 _os_compiler_barrier_after_atomic(m); \
165 _r; \
166})
167
168#define os_atomic_store_exclusive(p, v, m) ({ \
169 _os_compiler_barrier_before_atomic(m); \
170 (_os_atomic_mo_has_release(_os_atomic_mo_##m##_smp) \
171 ? !__builtin_arm_stlex(v, os_cast_to_nonatomic_pointer(p)) \
172 : !__builtin_arm_strex(v, os_cast_to_nonatomic_pointer(p))); \
173})
174
175#if OS_ATOMIC_USE_LLSC
176
177/*
178 * arm64 (without armv81 atomics) override of os_atomic_rmw_loop
179 * documentation for os_atomic_rmw_loop is in <os/atomic_private.h>
180 */
181#undef os_atomic_rmw_loop
182#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \
183 int _result = 0; \
184 __auto_type *_p = os_cast_to_nonatomic_pointer(p); \
185 _os_compiler_barrier_before_atomic(m); \
186 do { \
187 if (_os_atomic_mo_has_acquire(_os_atomic_mo_##m##_smp)) { \
188 ov = __builtin_arm_ldaex(_p); \
189 } else { \
190 ov = __builtin_arm_ldrex(_p); \
191 } \
192 __VA_ARGS__; \
193 if (_os_atomic_mo_has_release(_os_atomic_mo_##m##_smp)) { \
194 _result = !__builtin_arm_stlex(nv, _p); \
195 } else { \
196 _result = !__builtin_arm_strex(nv, _p); \
197 } \
198 } while (__builtin_expect(!_result, 0)); \
199 _os_compiler_barrier_after_atomic(m); \
200 _result; \
201})
202
203/*
204 * arm64 override of os_atomic_rmw_loop_give_up
205 * documentation for os_atomic_rmw_loop_give_up is in <os/atomic_private.h>
206 */
207#undef os_atomic_rmw_loop_give_up
208#define os_atomic_rmw_loop_give_up(...) \
209 ({ os_atomic_clear_exclusive(); __VA_ARGS__; break; })
210
211#endif // OS_ATOMIC_USE_LLSC
212
213#endif // __arm64__
214
215#endif /* __OS_ATOMIC_PRIVATE_ARCH_H__ */
216