1 | /* |
2 | * Copyright (c) 2015 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _MACHINE_ATOMIC_H |
30 | #define _MACHINE_ATOMIC_H |
31 | |
32 | #include <stdatomic.h> |
33 | |
34 | #define _os_atomic_c11_atomic(p) \ |
35 | ((typeof(*(p)) _Atomic *)(p)) |
36 | |
37 | #define _os_atomic_basetypeof(p) \ |
38 | typeof(atomic_load(((typeof(*(p)) _Atomic *)(p)))) |
39 | |
40 | #define _os_atomic_c11_op_orig(p, v, m, o) \ |
41 | atomic_##o##_explicit(_os_atomic_c11_atomic(p), v, \ |
42 | memory_order_##m) |
43 | |
44 | #define _os_atomic_c11_op(p, v, m, o, op) \ |
45 | ({ typeof(v) _v = (v); _os_atomic_c11_op_orig(p, v, m, o) op _v; }) |
46 | |
47 | #define os_atomic_thread_fence(m) atomic_thread_fence(memory_order_##m) |
48 | |
49 | #define os_atomic_load(p, m) \ |
50 | atomic_load_explicit(_os_atomic_c11_atomic(p), memory_order_##m) |
51 | #define os_atomic_store(p, v, m) _os_atomic_c11_op_orig(p, v, m, store) |
52 | |
53 | #define os_atomic_add_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_add) |
54 | #define os_atomic_add(p, v, m) _os_atomic_c11_op(p, v, m, fetch_add, +) |
55 | |
56 | #define os_atomic_inc_orig(p, m) _os_atomic_c11_op_orig(p, 1, m, fetch_add) |
57 | #define os_atomic_inc(p, m) _os_atomic_c11_op(p, 1, m, fetch_add, +) |
58 | |
59 | #define os_atomic_sub_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_sub) |
60 | #define os_atomic_sub(p, v, m) _os_atomic_c11_op(p, v, m, fetch_sub, -) |
61 | |
62 | #define os_atomic_dec_orig(p, m) _os_atomic_c11_op_orig(p, 1, m, fetch_sub) |
63 | #define os_atomic_dec(p, m) _os_atomic_c11_op(p, 1, m, fetch_sub, -) |
64 | |
65 | #define os_atomic_and_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_and) |
66 | #define os_atomic_and(p, v, m) _os_atomic_c11_op(p, v, m, fetch_and, &) |
67 | |
68 | #define os_atomic_or_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_or) |
69 | #define os_atomic_or(p, v, m) _os_atomic_c11_op(p, v, m, fetch_or, |) |
70 | |
71 | #define os_atomic_xor_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_xor) |
72 | #define os_atomic_xor(p, v, m) _os_atomic_c11_op(p, v, m, fetch_xor, ^) |
73 | |
74 | #define os_atomic_xchg(p, v, m) _os_atomic_c11_op_orig(p, v, m, exchange) |
75 | |
76 | #define os_atomic_cmpxchg(p, e, v, m) \ |
77 | ({ _os_atomic_basetypeof(p) _r = (e); \ |
78 | atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \ |
79 | &_r, v, memory_order_##m, memory_order_relaxed); }) |
80 | #define os_atomic_cmpxchgv(p, e, v, g, m) \ |
81 | ({ _os_atomic_basetypeof(p) _r = (e); int _b = \ |
82 | atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \ |
83 | &_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r; _b; }) |
84 | #define os_atomic_cmpxchgvw(p, e, v, g, m) \ |
85 | ({ _os_atomic_basetypeof(p) _r = (e); int _b = \ |
86 | atomic_compare_exchange_weak_explicit(_os_atomic_c11_atomic(p), \ |
87 | &_r, v, memory_order_##m, memory_order_relaxed); *(g) = _r; _b; }) |
88 | |
89 | #define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ |
90 | bool _result = false; \ |
91 | typeof(p) _p = (p); \ |
92 | ov = os_atomic_load(_p, relaxed); \ |
93 | do { \ |
94 | __VA_ARGS__; \ |
95 | _result = os_atomic_cmpxchgvw(_p, ov, nv, &ov, m); \ |
96 | } while (!_result); \ |
97 | _result; \ |
98 | }) |
99 | |
100 | #define os_atomic_rmw_loop_give_up_with_fence(m, expr) \ |
101 | ({ os_atomic_thread_fence(m); expr; __builtin_unreachable(); }) |
102 | #define os_atomic_rmw_loop_give_up(expr) \ |
103 | os_atomic_rmw_loop_give_up_with_fence(relaxed, expr) |
104 | |
105 | #define os_atomic_force_dependency_on(p, e) (p) |
106 | #define os_atomic_load_with_dependency_on(p, e) \ |
107 | os_atomic_load(os_atomic_force_dependency_on(p, e), relaxed) |
108 | |
109 | #if defined (__x86_64__) |
110 | #include "i386/atomic.h" |
111 | #elif defined (__arm__) || defined (__arm64__) |
112 | #include "arm/atomic.h" |
113 | #else |
114 | #error architecture not supported |
115 | #endif |
116 | |
117 | #endif /* _MACHINE_ATOMIC_H */ |
118 | |