1/*
2 * Copyright (c) 2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/*
30 * This header provides some gory details to implement the <os/atomic_private.h>
31 * interfaces. Nothing in this header should be called directly, no promise is
32 * made to keep this interface stable.
33 */
34
35#ifndef __OS_ATOMIC_PRIVATE_H__
36#error "Do not include <os/atomic_private_impl.h> directly, use <os/atomic_private.h>"
37#endif
38
39#ifndef __OS_ATOMIC_PRIVATE_IMPL_H__
40#define __OS_ATOMIC_PRIVATE_IMPL_H__
41
42#pragma mark - implementation details
43
44static inline int
45_os_atomic_mo_has_acquire(OS_ATOMIC_STD memory_order ord)
46{
47 switch (ord) {
48 case os_atomic_std(memory_order_consume):
49 case os_atomic_std(memory_order_acquire):
50 case os_atomic_std(memory_order_acq_rel):
51 case os_atomic_std(memory_order_seq_cst):
52 return 1;
53 default:
54 return 0;
55 }
56}
57
58static inline int
59_os_atomic_mo_has_release(OS_ATOMIC_STD memory_order ord)
60{
61 switch (ord) {
62 case os_atomic_std(memory_order_release):
63 case os_atomic_std(memory_order_acq_rel):
64 case os_atomic_std(memory_order_seq_cst):
65 return 1;
66 default:
67 return 0;
68 }
69}
70
71#define _os_atomic_mo_relaxed os_atomic_std(memory_order_relaxed)
72#define _os_atomic_mo_compiler_acquire os_atomic_std(memory_order_relaxed)
73#define _os_atomic_mo_compiler_release os_atomic_std(memory_order_relaxed)
74#define _os_atomic_mo_compiler_acq_rel os_atomic_std(memory_order_relaxed)
75#define _os_atomic_mo_consume os_atomic_std(memory_order_consume)
76#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
77#define _os_atomic_mo_dependency os_atomic_std(memory_order_acquire)
78#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
79#define _os_atomic_mo_acquire os_atomic_std(memory_order_acquire)
80#define _os_atomic_mo_release os_atomic_std(memory_order_release)
81#define _os_atomic_mo_acq_rel os_atomic_std(memory_order_acq_rel)
82#define _os_atomic_mo_seq_cst os_atomic_std(memory_order_seq_cst)
83
84/*
85 * Mapping between symbolic memory orderings and actual ones
86 * to take SMP into account.
87 */
88#if OS_ATOMIC_CONFIG_SMP
89#define _os_atomic_mo_relaxed_smp _os_atomic_mo_relaxed
90#define _os_atomic_mo_compiler_acquire_smp _os_atomic_mo_relaxed
91#define _os_atomic_mo_compiler_release_smp _os_atomic_mo_relaxed
92#define _os_atomic_mo_compiler_acq_rel_smp _os_atomic_mo_relaxed
93#define _os_atomic_mo_consume_smp _os_atomic_mo_consume
94#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
95#define _os_atomic_mo_dependency_smp _os_atomic_mo_dependency
96#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
97#define _os_atomic_mo_acquire_smp _os_atomic_mo_acquire
98#define _os_atomic_mo_release_smp _os_atomic_mo_release
99#define _os_atomic_mo_acq_rel_smp _os_atomic_mo_acq_rel
100#define _os_atomic_mo_seq_cst_smp _os_atomic_mo_seq_cst
101#else
102#define _os_atomic_mo_relaxed_smp _os_atomic_mo_relaxed
103#define _os_atomic_mo_compiler_acquire_smp _os_atomic_mo_relaxed
104#define _os_atomic_mo_compiler_release_smp _os_atomic_mo_relaxed
105#define _os_atomic_mo_compiler_acq_rel_smp _os_atomic_mo_relaxed
106#define _os_atomic_mo_consume_smp _os_atomic_mo_relaxed
107#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
108#define _os_atomic_mo_dependency_smp _os_atomic_mo_relaxed
109#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
110#define _os_atomic_mo_acquire_smp _os_atomic_mo_relaxed
111#define _os_atomic_mo_release_smp _os_atomic_mo_relaxed
112#define _os_atomic_mo_acq_rel_smp _os_atomic_mo_relaxed
113#define _os_atomic_mo_seq_cst_smp _os_atomic_mo_relaxed
114#endif
115
116#if KERNEL_PRIVATE
117#define memory_order_relaxed_smp _os_atomic_mo_relaxed_smp
118#define memory_order_compiler_acquire_smp _os_atomic_mo_compiler_acquire_smp
119#define memory_order_compiler_release_smp _os_atomic_mo_compiler_release_smp
120#define memory_order_compiler_acq_rel_smp _os_atomic_mo_compiler_acq_rel_smp
121#define memory_order_consume_smp _os_atomic_mo_consume_smp
122#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
123#define memory_order_dependency _os_atomic_mo_dependency
124#define memory_order_dependency_smp _os_atomic_mo_dependency_smp
125#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
126#define memory_order_acquire_smp _os_atomic_mo_acquire_smp
127#define memory_order_release_smp _os_atomic_mo_release_smp
128#define memory_order_acq_rel_smp _os_atomic_mo_acq_rel_smp
129#define memory_order_seq_cst_smp _os_atomic_mo_seq_cst_smp
130#endif
131
132/*
133 * Hack needed for os_compiler_barrier() to work (including with empty argument)
134 */
135#define _os_compiler_barrier_relaxed _os_atomic_mo_relaxed
136#define _os_compiler_barrier_acquire _os_atomic_mo_acquire
137#define _os_compiler_barrier_release _os_atomic_mo_release
138#define _os_compiler_barrier_acq_rel _os_atomic_mo_acq_rel
139#define _os_compiler_barrier_ _os_atomic_mo_acq_rel
140
141/*
142 * Mapping between compiler barrier/memory orders and:
143 * - compiler barriers before atomics ("rel_barrier")
144 * - compiler barriers after atomics ("acq_barrier")
145 */
146#define _os_rel_barrier_relaxed _os_atomic_mo_relaxed
147#define _os_rel_barrier_compiler_acquire _os_atomic_mo_relaxed
148#define _os_rel_barrier_compiler_release _os_atomic_mo_release
149#define _os_rel_barrier_compiler_acq_rel _os_atomic_mo_release
150#define _os_rel_barrier_consume _os_atomic_mo_relaxed
151#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
152#define _os_rel_barrier_dependency _os_atomic_mo_relaxed
153#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
154#define _os_rel_barrier_acquire _os_atomic_mo_relaxed
155#define _os_rel_barrier_release _os_atomic_mo_release
156#define _os_rel_barrier_acq_rel _os_atomic_mo_release
157#define _os_rel_barrier_seq_cst _os_atomic_mo_release
158
159#define _os_acq_barrier_relaxed _os_atomic_mo_relaxed
160#define _os_acq_barrier_compiler_acquire _os_atomic_mo_acquire
161#define _os_acq_barrier_compiler_release _os_atomic_mo_relaxed
162#define _os_acq_barrier_compiler_acq_rel _os_atomic_mo_acquire
163#define _os_acq_barrier_consume _os_atomic_mo_acquire
164#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
165#define _os_acq_barrier_dependency _os_atomic_mo_acquire
166#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
167#define _os_acq_barrier_acquire _os_atomic_mo_acquire
168#define _os_acq_barrier_release _os_atomic_mo_relaxed
169#define _os_acq_barrier_acq_rel _os_atomic_mo_acquire
170#define _os_acq_barrier_seq_cst _os_atomic_mo_acquire
171
172#define _os_compiler_barrier_before_atomic(m) \
173 os_atomic_std(atomic_signal_fence)(_os_rel_barrier_##m)
174#define _os_compiler_barrier_after_atomic(m) \
175 os_atomic_std(atomic_signal_fence)(_os_acq_barrier_##m)
176
177/*
178 * Mapping between compiler barrier/memmory orders and:
179 * - memory fences before atomics ("rel_fence")
180 * - memory fences after atomics ("acq_fence")
181 */
182#define _os_rel_fence_relaxed _os_atomic_mo_relaxed
183#define _os_rel_fence_compiler_acquire _os_atomic_mo_relaxed
184#define _os_rel_fence_compiler_release _os_atomic_mo_release
185#define _os_rel_fence_compiler_acq_rel _os_atomic_mo_release
186#define _os_rel_fence_consume _os_atomic_mo_relaxed_smp
187#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
188#define _os_rel_fence_dependency _os_atomic_mo_relaxed_smp
189#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
190#define _os_rel_fence_acquire _os_atomic_mo_relaxed_smp
191#define _os_rel_fence_release _os_atomic_mo_release_smp
192#define _os_rel_fence_acq_rel _os_atomic_mo_release_smp
193#define _os_rel_fence_seq_cst _os_atomic_mo_release_smp
194
195#define _os_acq_fence_relaxed _os_atomic_mo_relaxed
196#define _os_acq_fence_compiler_acquire _os_atomic_mo_relaxed
197#define _os_acq_fence_compiler_release _os_atomic_mo_relaxed
198#define _os_acq_fence_compiler_acq_rel _os_atomic_mo_relaxed
199#define _os_acq_fence_consume _os_atomic_mo_acquire_smp
200#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
201#define _os_acq_fence_dependency _os_atomic_mo_dependency_smp
202#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
203#define _os_acq_fence_acquire _os_atomic_mo_acquire_smp
204#define _os_acq_fence_release _os_atomic_mo_relaxed_smp
205#define _os_acq_fence_acq_rel _os_atomic_mo_acquire_smp
206#define _os_acq_fence_seq_cst _os_atomic_mo_acquire_smp
207
208#define _os_memory_fence_before_atomic(m) \
209 os_atomic_std(atomic_thread_fence)(_os_rel_fence_##m)
210#define _os_memory_fence_after_atomic(m) \
211 os_atomic_std(atomic_thread_fence)(_os_acq_fence_##m)
212
213/*
214 * Misc. helpers
215 */
216
217#define _os_atomic_value_cast(p, v) \
218 ({ __typeof__(*os_cast_to_nonatomic_pointer(p)) ___v = (v); ___v; })
219
220#define _os_atomic_c11_op_orig(p, v, m, o) ({ \
221 _os_compiler_barrier_before_atomic(m); \
222 __auto_type _r = os_atomic_std(atomic_##o##_explicit)(\
223 os_cast_to_atomic_pointer(p), \
224 _os_atomic_value_cast(p, v), \
225 _os_atomic_mo_##m##_smp); \
226 _os_compiler_barrier_after_atomic(m); \
227 _r; \
228})
229
230#define _os_atomic_c11_op(p, v, m, o, op) ({ \
231 __auto_type _v = _os_atomic_value_cast(p, v); \
232 _os_atomic_c11_op_orig(p, _v, m, o) op _v; \
233})
234
235#define _os_atomic_clang_op_orig(p, v, m, o) ({ \
236 _os_compiler_barrier_before_atomic(m); \
237 __auto_type _r = __atomic_##o(os_cast_to_nonatomic_pointer(p), \
238 _os_atomic_value_cast(p, v), \
239 _os_atomic_mo_##m##_smp); \
240 _os_compiler_barrier_after_atomic(m); \
241 _r; \
242})
243
244#define _os_atomic_clang_op(p, v, m, o, op) ({ \
245 __auto_type _v = _os_atomic_value_cast(p, v); \
246 __auto_type _s = _os_atomic_clang_op_orig(p, _v, m, o); \
247 op(_s, _v); \
248})
249
250#if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
251#define _os_atomic_auto_dependency(e) \
252 _Generic(e, \
253 os_atomic_dependency_t: (e), \
254 default: os_atomic_make_dependency(e))
255#endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY
256
257#endif /* __OS_ATOMIC_PRIVATE_IMPL_H__ */
258