1 | /* |
2 | * Copyright (c) 2021 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | #ifndef _KERN_LOCK_TYPES_H |
29 | #define _KERN_LOCK_TYPES_H |
30 | |
31 | #include <kern/kern_types.h> |
32 | |
33 | __BEGIN_DECLS |
34 | |
35 | #define LCK_SLEEP_MASK 0x3f /* Valid actions */ |
36 | |
37 | /*! |
38 | * @enum lck_sleep_action_t |
39 | * |
40 | * @abstract |
41 | * An action to pass to the @c lck_*_sleep* family of functions. |
42 | */ |
43 | __options_decl(lck_sleep_action_t, unsigned int, { |
44 | LCK_SLEEP_DEFAULT = 0x00, /**< Release the lock while waiting for the event, then reclaim */ |
45 | LCK_SLEEP_UNLOCK = 0x01, /**< Release the lock and return unheld */ |
46 | LCK_SLEEP_SHARED = 0x02, /**< Reclaim the lock in shared mode (RW only) */ |
47 | LCK_SLEEP_EXCLUSIVE = 0x04, /**< Reclaim the lock in exclusive mode (RW only) */ |
48 | LCK_SLEEP_SPIN = 0x08, /**< Reclaim the lock in spin mode (mutex only) */ |
49 | LCK_SLEEP_PROMOTED_PRI = 0x10, /**< Sleep at a promoted priority */ |
50 | LCK_SLEEP_SPIN_ALWAYS = 0x20, /**< Reclaim the lock in spin-always mode (mutex only) */ |
51 | }); |
52 | |
53 | __options_decl(lck_wake_action_t, unsigned int, { |
54 | LCK_WAKE_DEFAULT = 0x00, /* If waiters are present, transfer their push to the wokenup thread */ |
55 | LCK_WAKE_DO_NOT_TRANSFER_PUSH = 0x01, /* Do not transfer waiters push when waking up */ |
56 | }); |
57 | |
58 | typedef const struct hw_spin_policy *hw_spin_policy_t; |
59 | |
60 | #if XNU_KERNEL_PRIVATE |
61 | |
62 | /*! |
63 | * @enum lck_option_t |
64 | * |
65 | * @abstract |
66 | * Lock options to pass to "lcks=" boot-arg |
67 | */ |
68 | __options_decl(lck_option_t, unsigned int, { |
69 | LCK_OPTION_ENABLE_DEBUG = 0x01, /**< Request debug in default attribute */ |
70 | LCK_OPTION_ENABLE_STAT = 0x02, /**< Request lock group statistics in default attribute */ |
71 | LCK_OPTION_DISABLE_RW_PRIO = 0x04, /**< Disable RW lock priority promotion */ |
72 | LCK_OPTION_ENABLE_TIME_STAT = 0x08, /**< Request time lock group statistics in default attribute */ |
73 | LCK_OPTION_DISABLE_RW_DEBUG = 0x10, /**< Disable RW lock best-effort debugging. */ |
74 | }); |
75 | |
76 | #endif // XNU_KERNEL_PRIVATE |
77 | |
78 | #if MACH_KERNEL_PRIVATE |
79 | |
80 | /* |
81 | * The "hardware lock". Low-level locking primitives that |
82 | * MUST be exported by machine-dependent code; this abstraction |
83 | * must provide atomic, non-blocking mutual exclusion that |
84 | * is invulnerable to uniprocessor or SMP races, interrupts, |
85 | * traps or any other events. |
86 | * |
87 | * hw_lock_data_t machine-specific lock data structure |
88 | * hw_lock_t pointer to hw_lock_data_t |
89 | * |
90 | * An implementation must export these data types and must |
91 | * also provide routines to manipulate them (see prototypes, |
92 | * below). These routines may be external, inlined, optimized, |
93 | * or whatever, based on the kernel configuration. In the event |
94 | * that the implementation wishes to define its own prototypes, |
95 | * macros, or inline functions, it may define LOCK_HW_PROTOS |
96 | * to disable the definitions below. |
97 | * |
98 | * Mach does not expect these locks to support statistics, |
99 | * debugging, tracing or any other complexity. In certain |
100 | * configurations, Mach will build other locking constructs |
101 | * on top of this one. A correctly functioning Mach port need |
102 | * only implement these locks to be successful. However, |
103 | * greater efficiency may be gained with additional machine- |
104 | * dependent optimizations for the locking constructs defined |
105 | * later in kern/lock.h.. |
106 | */ |
107 | struct hslock { |
108 | uintptr_t lock_data __kernel_data_semantics; |
109 | }; |
110 | typedef struct hslock hw_lock_data_t, *hw_lock_t; |
111 | |
112 | /*! |
113 | * @enum hw_lock_status_t |
114 | * |
115 | * @abstract |
116 | * Used to pass information about very low level locking primitives. |
117 | * |
118 | */ |
119 | __enum_closed_decl(hw_lock_status_t, int, { |
120 | /** |
121 | * The lock was not taken because it is in an invalid state, |
122 | * or the memory was unmapped. |
123 | * |
124 | * This is only valid for @c *_allow_invalid() variants. |
125 | * |
126 | * Preemption is preserved to the caller level for all variants. |
127 | */ |
128 | HW_LOCK_INVALID = -1, |
129 | |
130 | /** |
131 | * the lock wasn't acquired and is contended / timed out. |
132 | * |
133 | * - @c *_nopreempt() variants: preemption level preserved |
134 | * - @c *_trylock() variants: preemption level preserved |
135 | * - other variants: preemption is disabled |
136 | */ |
137 | HW_LOCK_CONTENDED = 0, |
138 | |
139 | /** |
140 | * the lock was acquired successfully |
141 | * |
142 | * - @c *_nopreempt() variants: preemption level preserved |
143 | * - other variants: preemption is disabled |
144 | */ |
145 | HW_LOCK_ACQUIRED = 1, |
146 | }); |
147 | |
148 | /*! |
149 | * @enum hw_spin_timeout_status_t |
150 | * |
151 | * @abstract |
152 | * Used by spinlock timeout handlers. |
153 | * |
154 | * @const HW_LOCK_TIMEOUT_RETURN |
155 | * Tell the @c hw_lock*_to* caller to break out of the wait |
156 | * and return HW_LOCK_CONTENDED. |
157 | * |
158 | * @const HW_LOCK_TIMEOUT_CONTINUE |
159 | * Keep spinning for another "timeout". |
160 | */ |
161 | __enum_closed_decl(hw_spin_timeout_status_t, _Bool, { |
162 | HW_LOCK_TIMEOUT_RETURN, /**< return without taking the lock */ |
163 | HW_LOCK_TIMEOUT_CONTINUE, /**< keep spinning */ |
164 | }); |
165 | |
166 | |
167 | /*! |
168 | * @typedef hw_spin_timeout_t |
169 | * |
170 | * @abstract |
171 | * Describes the timeout used for a given spinning session. |
172 | */ |
173 | typedef struct { |
174 | uint64_t hwst_timeout; |
175 | #if SCHED_HYGIENE_DEBUG |
176 | bool hwst_in_ppl; |
177 | bool hwst_interruptible; |
178 | #endif /* SCHED_HYGIENE_DEBUG */ |
179 | } hw_spin_timeout_t; |
180 | |
181 | |
182 | /*! |
183 | * @typedef hw_spin_state_t |
184 | * |
185 | * @abstract |
186 | * Keeps track of the various timings used for spinning |
187 | */ |
188 | typedef struct { |
189 | uint64_t hwss_start; |
190 | uint64_t hwss_now; |
191 | uint64_t hwss_deadline; |
192 | #if SCHED_HYGIENE_DEBUG |
193 | uint64_t hwss_irq_start; |
194 | uint64_t hwss_irq_end; |
195 | #endif /* SCHED_HYGIENE_DEBUG */ |
196 | } hw_spin_state_t; |
197 | |
198 | |
199 | /*! |
200 | * @typedef hw_spin_timeout_fn_t |
201 | * |
202 | * @abstract |
203 | * The type of the timeout handlers for low level locking primitives. |
204 | * |
205 | * @discussion |
206 | * Typical handlers are written to just panic and not return |
207 | * unless some very specific conditions are met (debugging, ...). |
208 | * |
209 | * For formatting purposes, we provide HW_SPIN_TIMEOUT{,_DETAILS}{_FMT,_ARG} |
210 | * |
211 | * Those are meant to be used inside an hw_spin_timeout_fn_t function |
212 | * to form informative panic strings, like this: |
213 | * |
214 | * panic("MyLock[%p] " HW_SPIN_TIMEOUT_FMT "; " |
215 | * "<lock specific things> " HW_SPIN_TIMEOUT_DETAILS_FMT, |
216 | * lock_address, HW_SPIN_TIMEOUT_ARG(to, st), |
217 | * <lock specific args>, HW_SPIN_TIMEOUT_DETAILS_ARG(to, st)); |
218 | * |
219 | * This ensures consistent panic string style, and transparent adoption |
220 | * for any new diagnostic/debugging features at all call-sites. |
221 | */ |
222 | typedef hw_spin_timeout_status_t (hw_spin_timeout_fn_t)(void *lock, |
223 | hw_spin_timeout_t to, hw_spin_state_t st); |
224 | |
225 | #define HW_SPIN_TIMEOUT_FMT \ |
226 | "timeout after %llu ticks" |
227 | #define HW_SPIN_TIMEOUT_ARG(to, st) \ |
228 | ((st).hwss_now - (st).hwss_start) |
229 | |
230 | #if SCHED_HYGIENE_DEBUG |
231 | #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT \ |
232 | ", irq time: %llu" |
233 | #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) \ |
234 | , ((st).hwss_irq_end - (st).hwss_irq_start) |
235 | #else |
236 | #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT |
237 | #define HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) |
238 | #endif |
239 | |
240 | #define HW_SPIN_TIMEOUT_DETAILS_FMT \ |
241 | "start time: %llu, now: %llu, timeout: %llu" \ |
242 | HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT |
243 | #define HW_SPIN_TIMEOUT_DETAILS_ARG(to, st) \ |
244 | (st).hwss_start, (st).hwss_now, (to).hwst_timeout \ |
245 | HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) |
246 | |
247 | /*! |
248 | * @struct hw_spin_policy |
249 | * |
250 | * @abstract |
251 | * Describes the spinning policy for a given lock. |
252 | */ |
253 | struct hw_spin_policy { |
254 | const char *hwsp_name; |
255 | union { |
256 | const uint64_t *hwsp_timeout; |
257 | const _Atomic uint64_t *hwsp_timeout_atomic; |
258 | }; |
259 | uint16_t hwsp_timeout_shift; |
260 | uint16_t hwsp_lock_offset; |
261 | |
262 | hw_spin_timeout_fn_t *hwsp_op_timeout; |
263 | }; |
264 | |
265 | #if __x86_64__ |
266 | #define LCK_MTX_USE_ARCH 1 |
267 | #else |
268 | #define LCK_MTX_USE_ARCH 0 |
269 | #endif |
270 | #endif /* MACH_KERNEL_PRIVATE */ |
271 | |
272 | __END_DECLS |
273 | |
274 | #endif /* _KERN_LOCK_TYPES_H */ |
275 | |