1 | /* |
2 | * Copyright (c) 2020 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | #ifdef XNU_KERNEL_PRIVATE |
29 | |
30 | #ifndef _KERN_COUNTER_H |
31 | #define _KERN_COUNTER_H |
32 | |
33 | /*! |
34 | * @file <kern/counter.h> |
35 | * |
36 | * @brief |
37 | * Module for working with 64bit relaxed atomic counters. |
38 | * |
39 | * @discussion |
40 | * Different counter types have different speed-memory tradeoffs, but |
41 | * they all share a common interface. |
42 | * |
43 | * Counters can be statically allocated or dynamically allocated. |
44 | * |
45 | * Statically allocated counters are always backed by per-cpu storage which means |
46 | * writes take place on the current CPUs value and reads sum all of the per-cpu values. |
47 | * |
48 | * Dynamically allocated counters can be either per-cpu or use a single 64bit value. |
49 | * To create a per-cpu counter, use the scalable_counter_t type. Note that this |
50 | * trades of additional memory for better scalability. |
51 | * To create a single 64bit counter, use the atomic_counter_t type. |
52 | * |
53 | * For most counters you can just use the counter_t type and the choice of |
54 | * scalable or atomic will be made at compile time based on the target. |
55 | * |
56 | * The counter types are opaque handles. They ARE NOT COPYABLE. If you need |
57 | * to make a copy of a counter, you should do so like this: |
58 | * <code> |
59 | * counter_t original; |
60 | * ... |
61 | * counter_t copy; |
62 | * counter_alloc(©); |
63 | * counter_add(©, counter_load(&original)); |
64 | * ... |
65 | * // Make sure to free them at some point. |
66 | * counter_free(&original); |
67 | * counter_free(©); |
68 | * </code> |
69 | * |
70 | * Static counter example: |
71 | * <code> |
72 | * SCALABLE_COUNTER_DEFINE(my_counter); |
73 | * ... |
74 | * counter_inc(&my_counter); |
75 | * assert(counter_load(&my_counter) == 1); |
76 | * </code> |
77 | * |
78 | * Dynamic Counter Example: |
79 | * <code> |
80 | * scalable_counter_t my_percpu_counter; |
81 | * atomic_counter_t my_atomic_counter; |
82 | * counter_t my_counter; |
83 | * |
84 | * // All three counters share the same interface. So to change the speed-memory |
85 | * // tradeoff just change the type. |
86 | * counter_init(&my_scalable_counter); |
87 | * counter_init(&my_atomic_counter); |
88 | * counter_init(&my_counter); |
89 | * |
90 | * counter_inc(&my_scalable_counter); |
91 | * counter_inc(&my_atomic_counter); |
92 | * counter_inc(&my_counter); |
93 | * |
94 | * assert(counter_load(&my_scalable_counter) == 1); |
95 | * assert(counter_load(&my_atomic_counter) == 1); |
96 | * assert(counter_load(&my_counter) == 1); |
97 | * </code> |
98 | */ |
99 | |
100 | #include <mach/mach_types.h> |
101 | #include <kern/macro_help.h> |
102 | #include <kern/startup.h> |
103 | #include <kern/zalloc.h> |
104 | |
105 | typedef uint64_t *__zpercpu scalable_counter_t; |
106 | typedef uint64_t atomic_counter_t; |
107 | /* Generic counter base type. Does not have an implementation. */ |
108 | struct generic_counter_t; |
109 | |
110 | /*! |
111 | * @macro SCALABLE_COUNTER_DECLARE |
112 | * |
113 | * @abstract |
114 | * (optionally) declares a static per-cpu counter (in a header). |
115 | * |
116 | * @param var the name of the counter. |
117 | */ |
118 | #define SCALABLE_COUNTER_DECLARE(name) \ |
119 | extern scalable_counter_t name; |
120 | |
121 | /*! |
122 | * @macro SCALABLE_COUNTER_DEFINE |
123 | * |
124 | * @abstract |
125 | * Defines a static per-cpu counter. |
126 | * Counter can only be accessed after the TUNABLES phase of startup. |
127 | * |
128 | * @param var the name of the counter. |
129 | */ |
130 | #define SCALABLE_COUNTER_DEFINE(name) \ |
131 | __startup_data uint64_t __ ##name##_early_storage = 0; \ |
132 | scalable_counter_t name = {&__##name##_early_storage}; \ |
133 | STARTUP_ARG(TUNABLES, STARTUP_RANK_MIDDLE, scalable_counter_static_boot_mangle, &name); \ |
134 | STARTUP_ARG(PERCPU, STARTUP_RANK_SECOND, scalable_counter_static_init, &name); |
135 | |
136 | /* |
137 | * Initialize a per-cpu counter. |
138 | * May block and will never fail. |
139 | * This counter must be freed with counter_free. |
140 | */ |
141 | OS_OVERLOADABLE |
142 | extern void counter_alloc(struct generic_counter_t *); |
143 | |
144 | OS_OVERLOADABLE |
145 | extern void counter_free(struct generic_counter_t *); |
146 | /* |
147 | * Add amount to counter. |
148 | * @param amount The amount to add. |
149 | */ |
150 | OS_OVERLOADABLE |
151 | extern void counter_add(struct generic_counter_t *, uint64_t amount); |
152 | |
153 | /* |
154 | * Add 1 to this counter. |
155 | */ |
156 | OS_OVERLOADABLE |
157 | extern void counter_inc(struct generic_counter_t *); |
158 | |
159 | /* |
160 | * Subtract 1 from this counter. |
161 | */ |
162 | OS_OVERLOADABLE |
163 | extern void counter_dec(struct generic_counter_t *); |
164 | |
165 | /* Variants of the above operations where the caller takes responsibility for disabling preemption. */ |
166 | OS_OVERLOADABLE |
167 | extern void counter_add_preemption_disabled(struct generic_counter_t *, uint64_t amount); |
168 | OS_OVERLOADABLE |
169 | extern void counter_inc_preemption_disabled(struct generic_counter_t *); |
170 | OS_OVERLOADABLE |
171 | extern void counter_dec_preemption_disabled(struct generic_counter_t *); |
172 | |
173 | /* |
174 | * Read the value of the percpu counter. |
175 | * Note that this will cause synchronization of all the sharded values. |
176 | */ |
177 | OS_OVERLOADABLE |
178 | extern uint64_t counter_load(struct generic_counter_t *); |
179 | |
180 | #pragma mark implementation details |
181 | /* NB: Nothing below here should be used directly. */ |
182 | |
183 | __startup_func void scalable_counter_static_boot_mangle(scalable_counter_t *counter); |
184 | __startup_func void scalable_counter_static_init(scalable_counter_t *counter); |
185 | |
186 | #if XNU_TARGET_OS_WATCH || XNU_TARGET_OS_TV |
187 | #define ATOMIC_COUNTER_USE_PERCPU 0 |
188 | #else |
189 | #define ATOMIC_COUNTER_USE_PERCPU 1 |
190 | #endif /* XNU_TARGET_OS_OSX */ |
191 | |
192 | #if ATOMIC_COUNTER_USE_PERCPU |
193 | typedef scalable_counter_t counter_t; |
194 | #else |
195 | typedef atomic_counter_t counter_t; |
196 | #endif /* ATOMIC_COUNTER_USE_PERCPU */ |
197 | |
198 | #define COUNTER_MAKE_PROTOTYPES(counter_t) \ |
199 | OS_OVERLOADABLE \ |
200 | extern void counter_alloc(counter_t *); \ |
201 | \ |
202 | OS_OVERLOADABLE \ |
203 | extern void counter_free(counter_t *); \ |
204 | \ |
205 | OS_OVERLOADABLE \ |
206 | extern void counter_add(counter_t *, uint64_t amount); \ |
207 | \ |
208 | OS_OVERLOADABLE \ |
209 | extern void counter_inc(counter_t *); \ |
210 | \ |
211 | OS_OVERLOADABLE \ |
212 | extern void counter_dec(counter_t *); \ |
213 | \ |
214 | OS_OVERLOADABLE \ |
215 | extern void counter_add_preemption_disabled(counter_t *, uint64_t amount); \ |
216 | \ |
217 | OS_OVERLOADABLE \ |
218 | extern void counter_inc_preemption_disabled(counter_t *); \ |
219 | \ |
220 | OS_OVERLOADABLE \ |
221 | extern void counter_dec_preemption_disabled(counter_t *); \ |
222 | \ |
223 | OS_OVERLOADABLE \ |
224 | extern uint64_t counter_load(counter_t *); |
225 | |
226 | COUNTER_MAKE_PROTOTYPES(scalable_counter_t); |
227 | COUNTER_MAKE_PROTOTYPES(atomic_counter_t); |
228 | |
229 | #endif /* _KERN_COUNTER_H */ |
230 | |
231 | #endif /* XNU_KERNEL_PRIVATE */ |
232 | |