1 | /* |
2 | * Copyright (c) 2020 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _KERN_PERCPU_H_ |
30 | #define _KERN_PERCPU_H_ |
31 | |
32 | #include <mach/vm_types.h> |
33 | #include <mach/machine/vm_param.h> /* For PAGE_MASK */ |
34 | |
35 | __BEGIN_DECLS |
36 | |
37 | #if XNU_KERNEL_PRIVATE |
38 | #include <libkern/section_keywords.h> |
39 | |
40 | #pragma GCC visibility push(hidden) |
41 | |
42 | /*! |
43 | * @macro PERCPU_DECL |
44 | * |
45 | * @abstract |
46 | * Declares a per-CPU variable in a header. |
47 | * |
48 | * @param type_t the per-CPU variable type |
49 | * @param name the per-CPU variable name |
50 | */ |
51 | #define PERCPU_DECL(type_t, name) \ |
52 | extern type_t __PERCPU_NAME(name) |
53 | |
54 | /*! |
55 | * @macro PERCPU_DATA |
56 | * |
57 | * @abstract |
58 | * Defines a per-CPU variable in a translation unit. |
59 | * |
60 | * @discussion |
61 | * @c PERCPU_DECL can be used in headers to export the variable to clients. |
62 | * |
63 | * By default, per-cpu data is 0-initialized. Per-CPU data is allocated during |
64 | * the STARTUP_SUB_KMEM phase and can be initialized with a STARTUP |
65 | * callback in any later phase. |
66 | * |
67 | * Usage is: |
68 | * <code> |
69 | * [ static ] type PERCPU_DATA(name); |
70 | * </code> |
71 | * |
72 | * @param name the per-CPU variable name |
73 | */ |
74 | #define PERCPU_DATA(name) \ |
75 | __percpu __PERCPU_NAME(name) = {0} |
76 | |
77 | /* |
78 | * Same as before, but as a temporary hack with a 0 initializer |
79 | * instead of {0}, because clang has a bug where it does not accept |
80 | * the latter for _Atomic types. (And we want to keep the initializer |
81 | * to prevent people to think they can initialize it to anything else |
82 | * but 0.) |
83 | */ |
84 | #define PERCPU_DATA_HACK_78750602(name) \ |
85 | __percpu __PERCPU_NAME(name) = 0 |
86 | |
87 | /*! |
88 | * @macro PERCPU_GET |
89 | * |
90 | * @abstract |
91 | * Gets a pointer to the per-CPU instance of the variable for the processor the |
92 | * code is currently running on. |
93 | * |
94 | * @discussion |
95 | * It is expected that preemption or interrupts are disabled when this is used, |
96 | * as a context-switch might move the current thread to another CPU. |
97 | * |
98 | * It is also valid in code that wasn't already disabling preemption and cares |
99 | * about code-gen size a lot to use this outside of a preemption-disabled |
100 | * section provided that the data is modified using atomics. |
101 | * |
102 | * Note that if several per-CPU pointers are acquired in short succession, |
103 | * @c PERCPU_GET_WITH_BASE can be used to avoid the repeated calls to |
104 | * @c current_percpu_base() which the compiler wont't elide. |
105 | * |
106 | * @param name the per-CPU variable name |
107 | */ |
108 | #define PERCPU_GET(name) \ |
109 | __PERCPU_CAST(name, current_percpu_base() + __PERCPU_ADDR(name)) |
110 | |
111 | /*! |
112 | * @function current_percpu_base() |
113 | * |
114 | * @abstract |
115 | * Returns an offset that can be passed to @c PERCPU_GET_WITH_BASE(). |
116 | * |
117 | * @see PERCPU_GET() for conditions of use. |
118 | */ |
119 | extern vm_offset_t current_percpu_base(void); |
120 | |
121 | /*! |
122 | * @function other_percpu_base() |
123 | * |
124 | * @abstract |
125 | * Returns an offset that can be passed to @c PERCPU_GET_WITH_BASE(), |
126 | * for the speficied cpu number. |
127 | * |
128 | * @param cpu_number the cpu number for which we want a base. |
129 | */ |
130 | extern vm_offset_t other_percpu_base(int cpu_number); |
131 | |
132 | /*! |
133 | * @macro PERCPU_GET_MASTER |
134 | * |
135 | * @abstract |
136 | * Gets a pointer to the master per-CPU instance of the variable. |
137 | * |
138 | * @param base the per-CPU base to use |
139 | * @param name the per-CPU variable name |
140 | */ |
141 | #define PERCPU_GET_MASTER(name) \ |
142 | (&__PERCPU_NAME(name)) |
143 | |
144 | /*! |
145 | * @macro PERCPU_GET_WITH_BASE |
146 | * |
147 | * @abstract |
148 | * Gets a pointer to the per-CPU instance of the variable for the specified |
149 | * base. |
150 | * |
151 | * @param base the per-CPU base to use |
152 | * @param name the per-CPU variable name |
153 | */ |
154 | #define PERCPU_GET_WITH_BASE(base, name) \ |
155 | __PERCPU_CAST(name, base + __PERCPU_ADDR(name)) |
156 | |
157 | /*! |
158 | * @macro PERCPU_GET_RELATIVE |
159 | * |
160 | * @abstract |
161 | * Gets a pointer to the per-CPU instance of a variable relative to another |
162 | * known one. |
163 | * |
164 | * @description |
165 | * When a per-CPU slot address is known, but the caller doesn't know the base |
166 | * from which it was derived, then this allows to compute another per-CPU slot |
167 | * address for a different variable but for the same CPU, without any loads. |
168 | * |
169 | * @param name the per-CPU variable name |
170 | * @param other the other per-CPU variable name |
171 | * @param ptr a pointer to the other variable slot |
172 | */ |
173 | #define PERCPU_GET_RELATIVE(name, other, ptr) ({ \ |
174 | __PERCPU_TYPE(other) __other_ptr = (ptr); /* type check */ \ |
175 | vm_offset_t __offs = __PERCPU_ADDR(name) - __PERCPU_ADDR(other); \ |
176 | __PERCPU_CAST(name, (vm_address_t)__other_ptr + __offs); \ |
177 | }) |
178 | |
179 | /*! |
180 | * @macro percpu_foreach_base() |
181 | * |
182 | * @abstract |
183 | * Enumerates all Per-CPU variable bases. |
184 | * |
185 | * @param it the name of the iterator |
186 | */ |
187 | #define percpu_foreach_base(it) \ |
188 | for (vm_offset_t it = 0, \ |
189 | __next_ ## it = percpu_base.start, \ |
190 | __end_ ## it = percpu_base.end; \ |
191 | \ |
192 | it <= __end_ ## it; \ |
193 | \ |
194 | it = __next_ ## it, \ |
195 | __next_ ## it += percpu_section_size()) |
196 | |
197 | /*! |
198 | * @macro percpu_foreach() |
199 | * |
200 | * @abstract |
201 | * Enumerates all Per-CPU variable instances. |
202 | * |
203 | * @param it the name of the iterator |
204 | * @param name the per-CPU variable name |
205 | */ |
206 | #define percpu_foreach(it, name) \ |
207 | for (__PERCPU_TYPE(name) it, \ |
208 | __unsafe_indexable __base_ ## it = NULL, \ |
209 | __unsafe_indexable __next_ ## it = __PERCPU_CAST(name, percpu_base.start), \ |
210 | __unsafe_indexable __end_ ## it = __PERCPU_CAST(name, percpu_base.end); \ |
211 | \ |
212 | (it = __PERCPU_CAST(name, __PERCPU_ADDR(name) + (vm_address_t)__base_ ## it), \ |
213 | __base_ ## it <= __end_ ## it); \ |
214 | \ |
215 | __base_ ## it = __next_ ## it, \ |
216 | __next_ ## it = __PERCPU_CAST(name, (vm_address_t)__base_ ## it + percpu_section_size())) |
217 | |
218 | /*! |
219 | * @macro percpu_foreach_secondary_base() |
220 | * |
221 | * @abstract |
222 | * Enumerates all Per-CPU variable bases, skipping the master slot. |
223 | * |
224 | * @param it the name of the iterator |
225 | */ |
226 | #define percpu_foreach_secondary_base(it) \ |
227 | for (vm_offset_t it = percpu_base.start, __end_ ## it = percpu_base.end; \ |
228 | it <= __end_ ## it; it += percpu_section_size()) |
229 | |
230 | /*! |
231 | * @macro percpu_foreach_secondary() |
232 | * |
233 | * @abstract |
234 | * Enumerates all Per-CPU variable instances, skipping the master slot. |
235 | * |
236 | * @param it the name of the iterator |
237 | * @param name the per-CPU variable name |
238 | */ |
239 | #define percpu_foreach_secondary(it, name) \ |
240 | for (__PERCPU_TYPE(name) it, \ |
241 | __unsafe_indexable __base_ ## it = __PERCPU_CAST(name, percpu_base.start), \ |
242 | __unsafe_indexable __end_ ## it = __PERCPU_CAST(name, percpu_base.end); \ |
243 | \ |
244 | (it = __PERCPU_CAST(name, __PERCPU_ADDR(name) + (vm_address_t)__base_ ## it), \ |
245 | __base_ ## it <= __end_ ## it); \ |
246 | \ |
247 | __base_ ## it = __PERCPU_CAST(name, (vm_address_t)__base_ ## it + percpu_section_size())) |
248 | |
249 | #pragma mark - implementation details |
250 | |
251 | /* |
252 | * Below this point are implementation details that should not be used directly, |
253 | * except by the macros above, or architecture specific code. |
254 | */ |
255 | |
256 | #define __percpu __attribute__((section("__DATA, __percpu"))) |
257 | #define __PERCPU_NAME(name) percpu_slot_ ## name |
258 | #define __PERCPU_ADDR(name) ((caddr_t)&__PERCPU_NAME(name)) |
259 | #define __PERCPU_TYPE(name) typeof(&__PERCPU_NAME(name)) |
260 | #define __PERCPU_CAST(name, expr) __unsafe_forge_bidi_indexable(__PERCPU_TYPE(name), (vm_address_t)(expr), sizeof(__PERCPU_NAME(name))) |
261 | |
262 | /* |
263 | * Note for implementors: |
264 | * |
265 | * A `base` represents a pointer in the percpu allocation offset by |
266 | * `percpu_section_start()` so that PERCPU_GET() is a single addition. |
267 | * |
268 | * percpu_base.end is inclusive, so that percpu_foreach() and |
269 | * percpu_foreach_base() can do a `<=` comparison. |
270 | * |
271 | * Because the first base is `0` (because the master CPU is using the static |
272 | * percpu section), it allows for the compiler to know that for the first |
273 | * iteration the comparison is always true. |
274 | */ |
275 | extern struct percpu_base { |
276 | vm_address_t start; |
277 | vm_address_t end; |
278 | vm_offset_t size; |
279 | } percpu_base; |
280 | |
281 | static __pure2 inline vm_offset_t |
282 | percpu_section_start(void) |
283 | { |
284 | extern char __percpu_section_start[0] __SECTION_START_SYM("__DATA" , "__percpu" ); |
285 | return (vm_offset_t)__percpu_section_start; |
286 | } |
287 | |
288 | static __pure2 inline vm_offset_t |
289 | percpu_section_end(void) |
290 | { |
291 | extern char __percpu_section_end[0] __SECTION_END_SYM("__DATA" , "__percpu" ); |
292 | return (vm_offset_t)__percpu_section_end; |
293 | } |
294 | |
295 | static __pure2 inline vm_size_t |
296 | percpu_section_size(void) |
297 | { |
298 | /** |
299 | * TODO: remove page rounding once we have a linker construct that gives us the correct page-padded size |
300 | * See rdar://problem/97665399. |
301 | */ |
302 | return ((percpu_section_end() - percpu_section_start()) + PAGE_MASK) & ~((vm_size_t)PAGE_MASK); |
303 | } |
304 | |
305 | #pragma GCC visibility pop |
306 | #endif /* XNU_KERNEL_PRIVATE */ |
307 | |
308 | __END_DECLS |
309 | |
310 | #endif /* _KERN_PERCPU_H_ */ |
311 | |