1/*
2 * Copyright (c) 2021 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#ifndef _KERN_LOCK_TYPES_H
29#define _KERN_LOCK_TYPES_H
30
31#include <kern/kern_types.h>
32
33__BEGIN_DECLS
34
35#define LCK_SLEEP_MASK 0x3f /* Valid actions */
36
37/*!
38 * @enum lck_sleep_action_t
39 *
40 * @abstract
41 * An action to pass to the @c lck_*_sleep* family of functions.
42 */
43__options_decl(lck_sleep_action_t, unsigned int, {
44 LCK_SLEEP_DEFAULT = 0x00, /**< Release the lock while waiting for the event, then reclaim */
45 LCK_SLEEP_UNLOCK = 0x01, /**< Release the lock and return unheld */
46 LCK_SLEEP_SHARED = 0x02, /**< Reclaim the lock in shared mode (RW only) */
47 LCK_SLEEP_EXCLUSIVE = 0x04, /**< Reclaim the lock in exclusive mode (RW only) */
48 LCK_SLEEP_SPIN = 0x08, /**< Reclaim the lock in spin mode (mutex only) */
49 LCK_SLEEP_PROMOTED_PRI = 0x10, /**< Sleep at a promoted priority */
50 LCK_SLEEP_SPIN_ALWAYS = 0x20, /**< Reclaim the lock in spin-always mode (mutex only) */
51});
52
53__options_decl(lck_wake_action_t, unsigned int, {
54 LCK_WAKE_DEFAULT = 0x00, /* If waiters are present, transfer their push to the wokenup thread */
55 LCK_WAKE_DO_NOT_TRANSFER_PUSH = 0x01, /* Do not transfer waiters push when waking up */
56});
57
58typedef const struct hw_spin_policy *hw_spin_policy_t;
59
60#if XNU_KERNEL_PRIVATE
61
62/*!
63 * @enum lck_option_t
64 *
65 * @abstract
66 * Lock options to pass to "lcks=" boot-arg
67 */
68__options_decl(lck_option_t, unsigned int, {
69 LCK_OPTION_ENABLE_DEBUG = 0x01, /**< Request debug in default attribute */
70 LCK_OPTION_ENABLE_STAT = 0x02, /**< Request lock group statistics in default attribute */
71 LCK_OPTION_DISABLE_RW_PRIO = 0x04, /**< Disable RW lock priority promotion */
72 LCK_OPTION_ENABLE_TIME_STAT = 0x08, /**< Request time lock group statistics in default attribute */
73 LCK_OPTION_DISABLE_RW_DEBUG = 0x10, /**< Disable RW lock best-effort debugging. */
74});
75
76#endif // XNU_KERNEL_PRIVATE
77
78#if MACH_KERNEL_PRIVATE
79
80/*
81 * The "hardware lock". Low-level locking primitives that
82 * MUST be exported by machine-dependent code; this abstraction
83 * must provide atomic, non-blocking mutual exclusion that
84 * is invulnerable to uniprocessor or SMP races, interrupts,
85 * traps or any other events.
86 *
87 * hw_lock_data_t machine-specific lock data structure
88 * hw_lock_t pointer to hw_lock_data_t
89 *
90 * An implementation must export these data types and must
91 * also provide routines to manipulate them (see prototypes,
92 * below). These routines may be external, inlined, optimized,
93 * or whatever, based on the kernel configuration. In the event
94 * that the implementation wishes to define its own prototypes,
95 * macros, or inline functions, it may define LOCK_HW_PROTOS
96 * to disable the definitions below.
97 *
98 * Mach does not expect these locks to support statistics,
99 * debugging, tracing or any other complexity. In certain
100 * configurations, Mach will build other locking constructs
101 * on top of this one. A correctly functioning Mach port need
102 * only implement these locks to be successful. However,
103 * greater efficiency may be gained with additional machine-
104 * dependent optimizations for the locking constructs defined
105 * later in kern/lock.h..
106 */
107struct hslock {
108 uintptr_t lock_data __kernel_data_semantics;
109};
110typedef struct hslock hw_lock_data_t, *hw_lock_t;
111
112/*!
113 * @enum hw_lock_status_t
114 *
115 * @abstract
116 * Used to pass information about very low level locking primitives.
117 *
118 */
119__enum_closed_decl(hw_lock_status_t, int, {
120 /**
121 * The lock was not taken because it is in an invalid state,
122 * or the memory was unmapped.
123 *
124 * This is only valid for @c *_allow_invalid() variants.
125 *
126 * Preemption is preserved to the caller level for all variants.
127 */
128 HW_LOCK_INVALID = -1,
129
130 /**
131 * the lock wasn't acquired and is contended / timed out.
132 *
133 * - @c *_nopreempt() variants: preemption level preserved
134 * - @c *_trylock() variants: preemption level preserved
135 * - other variants: preemption is disabled
136 */
137 HW_LOCK_CONTENDED = 0,
138
139 /**
140 * the lock was acquired successfully
141 *
142 * - @c *_nopreempt() variants: preemption level preserved
143 * - other variants: preemption is disabled
144 */
145 HW_LOCK_ACQUIRED = 1,
146});
147
148/*!
149 * @enum hw_spin_timeout_status_t
150 *
151 * @abstract
152 * Used by spinlock timeout handlers.
153 *
154 * @const HW_LOCK_TIMEOUT_RETURN
155 * Tell the @c hw_lock*_to* caller to break out of the wait
156 * and return HW_LOCK_CONTENDED.
157 *
158 * @const HW_LOCK_TIMEOUT_CONTINUE
159 * Keep spinning for another "timeout".
160 */
161__enum_closed_decl(hw_spin_timeout_status_t, _Bool, {
162 HW_LOCK_TIMEOUT_RETURN, /**< return without taking the lock */
163 HW_LOCK_TIMEOUT_CONTINUE, /**< keep spinning */
164});
165
166
167/*!
168 * @typedef hw_spin_timeout_t
169 *
170 * @abstract
171 * Describes the timeout used for a given spinning session.
172 */
173typedef struct {
174 uint64_t hwst_timeout;
175#if SCHED_HYGIENE_DEBUG
176 bool hwst_in_ppl;
177 bool hwst_interruptible;
178#endif /* SCHED_HYGIENE_DEBUG */
179} hw_spin_timeout_t;
180
181
182/*!
183 * @typedef hw_spin_state_t
184 *
185 * @abstract
186 * Keeps track of the various timings used for spinning
187 */
188typedef struct {
189 uint64_t hwss_start;
190 uint64_t hwss_now;
191 uint64_t hwss_deadline;
192#if SCHED_HYGIENE_DEBUG
193 uint64_t hwss_irq_start;
194 uint64_t hwss_irq_end;
195#endif /* SCHED_HYGIENE_DEBUG */
196} hw_spin_state_t;
197
198
199/*!
200 * @typedef hw_spin_timeout_fn_t
201 *
202 * @abstract
203 * The type of the timeout handlers for low level locking primitives.
204 *
205 * @discussion
206 * Typical handlers are written to just panic and not return
207 * unless some very specific conditions are met (debugging, ...).
208 *
209 * For formatting purposes, we provide HW_SPIN_TIMEOUT{,_DETAILS}{_FMT,_ARG}
210 *
211 * Those are meant to be used inside an hw_spin_timeout_fn_t function
212 * to form informative panic strings, like this:
213 *
214 * panic("MyLock[%p] " HW_SPIN_TIMEOUT_FMT "; "
215 * "<lock specific things> " HW_SPIN_TIMEOUT_DETAILS_FMT,
216 * lock_address, HW_SPIN_TIMEOUT_ARG(to, st),
217 * <lock specific args>, HW_SPIN_TIMEOUT_DETAILS_ARG(to, st));
218 *
219 * This ensures consistent panic string style, and transparent adoption
220 * for any new diagnostic/debugging features at all call-sites.
221 */
222typedef hw_spin_timeout_status_t (hw_spin_timeout_fn_t)(void *lock,
223 hw_spin_timeout_t to, hw_spin_state_t st);
224
225#define HW_SPIN_TIMEOUT_FMT \
226 "timeout after %llu ticks"
227#define HW_SPIN_TIMEOUT_ARG(to, st) \
228 ((st).hwss_now - (st).hwss_start)
229
230#if SCHED_HYGIENE_DEBUG
231#define HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT \
232 ", irq time: %llu"
233#define HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st) \
234 , ((st).hwss_irq_end - (st).hwss_irq_start)
235#else
236#define HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT
237#define HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st)
238#endif
239
240#define HW_SPIN_TIMEOUT_DETAILS_FMT \
241 "start time: %llu, now: %llu, timeout: %llu" \
242 HW_SPIN_TIMEOUT_SCHED_HYGIENE_FMT
243#define HW_SPIN_TIMEOUT_DETAILS_ARG(to, st) \
244 (st).hwss_start, (st).hwss_now, (to).hwst_timeout \
245 HW_SPIN_TIMEOUT_SCHED_HYGIENE_ARG(to, st)
246
247/*!
248 * @struct hw_spin_policy
249 *
250 * @abstract
251 * Describes the spinning policy for a given lock.
252 */
253struct hw_spin_policy {
254 const char *hwsp_name;
255 union {
256 const uint64_t *hwsp_timeout;
257 const _Atomic uint64_t *hwsp_timeout_atomic;
258 };
259 uint16_t hwsp_timeout_shift;
260 uint16_t hwsp_lock_offset;
261
262 hw_spin_timeout_fn_t *hwsp_op_timeout;
263};
264
265#if __x86_64__
266#define LCK_MTX_USE_ARCH 1
267#else
268#define LCK_MTX_USE_ARCH 0
269#endif
270#endif /* MACH_KERNEL_PRIVATE */
271
272__END_DECLS
273
274#endif /* _KERN_LOCK_TYPES_H */
275