1/*
2 * Copyright (c) 2018-2021 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#ifndef _KERN_LOCK_GROUP_H
29#define _KERN_LOCK_GROUP_H
30
31#include <kern/queue.h>
32#include <kern/lock_types.h>
33#if XNU_KERNEL_PRIVATE
34#include <kern/startup.h>
35#include <os/refcnt.h>
36#endif /* XNU_KERNEL_PRIVATE */
37
38__BEGIN_DECLS
39
40/*!
41 * @typedef lck_grp_t
42 *
43 * @abstract
44 * The opaque type of a lock group.
45 *
46 * @discussion
47 * A lock group is used to denote a set of locks that serve
48 * a similar purpose, and hold an equivalent "rank" in the lock hierarchy.
49 *
50 * This structure can then provide some statistics and anchor checks
51 * in development kernels for an entire family of locks.
52 */
53typedef struct _lck_grp_ lck_grp_t;
54#define LCK_GRP_NULL ((lck_grp_t *)NULL)
55
56/*!
57 * @typedef lck_grp_attr_t
58 *
59 * @abstract
60 * The opaque type for attributes to a group.
61 *
62 * @discussion
63 * A lock group attribute is meant to configure
64 * a group, as a group configuration becomes
65 * immutable once made.
66 */
67typedef struct _lck_grp_attr_ lck_grp_attr_t;
68#define LCK_GRP_ATTR_NULL ((lck_grp_attr_t *)NULL)
69
70extern lck_grp_attr_t *lck_grp_attr_alloc_init(
71 void);
72
73extern void lck_grp_attr_setdefault(
74 lck_grp_attr_t *attr);
75
76extern void lck_grp_attr_setstat(
77 lck_grp_attr_t *attr);
78
79extern void lck_grp_attr_free(
80 lck_grp_attr_t *attr);
81
82extern lck_grp_t *lck_grp_alloc_init(
83 const char *grp_name,
84 lck_grp_attr_t *attr);
85
86extern void lck_grp_free(
87 lck_grp_t *grp);
88
89#if XNU_KERNEL_PRIVATE
90#pragma GCC visibility push(hidden)
91
92/*
93 * Arguments wrapped in LCK_GRP_ARG() will be elided
94 * when LOCK_STATS is not set.
95 *
96 * Arguments wrapped with LCK_GRP_PROBEARG() will be
97 * NULL when LOCK_STATS is not set
98 */
99#if LOCK_STATS
100#if !CONFIG_DTRACE
101#error invalid configuration: LOCK_STATS needs dtrace
102#endif
103#define LCK_GRP_ARG(expr) , expr
104#define LCK_GRP_PROBEARG(grp) grp
105#define LCK_GRP_USE_ARG 1
106#else
107#define LCK_GRP_ARG(expr)
108#define LCK_GRP_PROBEARG(grp) LCK_GRP_NULL
109#define LCK_GRP_USE_ARG 0
110#endif /* LOCK_STATS */
111
112__enum_decl(lck_debug_feature_t, uint32_t, {
113 LCK_DEBUG_LOCKSTAT,
114 LCK_DEBUG_LOCKPROF,
115
116 LCK_DEBUG_MAX,
117});
118
119extern uint32_t LcksOpts;
120
121extern struct lck_debug_state {
122 uint32_t lds_value;
123 long lds_counts[LCK_DEBUG_MAX];
124} lck_debug_state;
125
126__options_decl(lck_grp_options_t, uint32_t, {
127 LCK_GRP_ATTR_NONE = 0x00000000,
128
129#if MACH_KERNEL_PRIVATE
130 LCK_GRP_ATTR_ID_MASK = 0x0000ffff,
131 LCK_GRP_ATTR_STAT = 0x00010000, /* enable non time stats */
132 LCK_GRP_ATTR_TIME_STAT = 0x00020000, /* enable time stats */
133 LCK_GRP_ATTR_DEBUG = 0x00040000, /* profile locks of this group */
134 LCK_GRP_ATTR_ALLOCATED = 0x80000000,
135#endif
136});
137
138#if CONFIG_DTRACE
139typedef struct _lck_grp_stat_ {
140 uint64_t lgs_count;
141 uint32_t lgs_enablings;
142 /*
143 * Protected by dtrace_lock
144 */
145 uint32_t lgs_probeid;
146 uint64_t lgs_limit;
147} lck_grp_stat_t;
148
149typedef struct _lck_grp_stats_ {
150 lck_grp_stat_t lgss_spin_held;
151 lck_grp_stat_t lgss_spin_miss;
152 lck_grp_stat_t lgss_spin_spin;
153
154 lck_grp_stat_t lgss_ticket_held;
155 lck_grp_stat_t lgss_ticket_miss;
156 lck_grp_stat_t lgss_ticket_spin;
157
158 lck_grp_stat_t lgss_mtx_held;
159 lck_grp_stat_t lgss_mtx_direct_wait;
160 lck_grp_stat_t lgss_mtx_miss;
161 lck_grp_stat_t lgss_mtx_wait;
162} lck_grp_stats_t;
163#endif /* CONFIG_DTRACE */
164
165#define LCK_GRP_MAX_NAME 64
166
167struct _lck_grp_ {
168 os_ref_atomic_t lck_grp_refcnt;
169 uint32_t lck_grp_attr_id;
170 uint32_t lck_grp_spincnt;
171 uint32_t lck_grp_ticketcnt;
172 uint32_t lck_grp_mtxcnt;
173 uint32_t lck_grp_rwcnt;
174 char lck_grp_name[LCK_GRP_MAX_NAME];
175#if CONFIG_DTRACE
176 lck_grp_stats_t lck_grp_stats;
177#endif /* CONFIG_DTRACE */
178};
179
180struct _lck_grp_attr_ {
181 lck_grp_options_t grp_attr_val;
182};
183
184struct lck_grp_spec {
185 lck_grp_t *grp;
186 char grp_name[LCK_GRP_MAX_NAME];
187 lck_grp_options_t grp_flags;
188};
189
190/*
191 * Auto-initializing lock group declarations
192 * -----------------------------------------
193 *
194 * Use LCK_GRP_DECLARE to declare an automatically initialized group.
195 */
196#define LCK_GRP_DECLARE_ATTR(var, name, flags) \
197 __PLACE_IN_SECTION("__DATA,__lock_grp") lck_grp_t var; \
198 static __startup_data struct lck_grp_spec \
199 __startup_lck_grp_spec_ ## var = { &var, name, flags }; \
200 STARTUP_ARG(LOCKS, STARTUP_RANK_SECOND, lck_grp_startup_init, \
201 &__startup_lck_grp_spec_ ## var)
202
203#define LCK_GRP_DECLARE(var, name) \
204 LCK_GRP_DECLARE_ATTR(var, name, LCK_GRP_ATTR_NONE);
205
206extern bool lck_grp_has_stats(
207 lck_grp_t *grp);
208
209extern void lck_grp_startup_init(
210 struct lck_grp_spec *spec);
211
212extern void lck_grp_init(
213 lck_grp_t *grp,
214 const char* grp_name,
215 lck_grp_attr_t *attr);
216
217extern lck_grp_t *lck_grp_init_flags(
218 lck_grp_t *grp,
219 const char* grp_name,
220 lck_grp_options_t grp_flags);
221
222extern lck_grp_t *lck_grp_resolve(
223 uint32_t grp_attr_id) __pure2;
224
225extern void lck_grp_assert_id(
226 lck_grp_t *grp,
227 uint32_t grp_attr_id);
228#if DEBUG || DEVELOPMENT
229#define LCK_GRP_ASSERT_ID(...) lck_grp_assert_id(__VA_ARGS__)
230#else
231#define LCK_GRP_ASSERT_ID(...) ((void)0)
232#endif
233
234extern void lck_grp_reference(
235 lck_grp_t *grp,
236 uint32_t *cnt);
237
238extern void lck_grp_deallocate(
239 lck_grp_t *grp,
240 uint32_t *cnt);
241
242extern void lck_grp_foreach(
243 bool (^block)(lck_grp_t *));
244
245
246extern void lck_grp_enable_feature(
247 lck_debug_feature_t feat);
248
249extern void lck_grp_disable_feature(
250 lck_debug_feature_t feat);
251
252__pure2
253static inline uint32_t
254lck_opts_get(void)
255{
256 return LcksOpts;
257}
258
259#pragma GCC visibility pop
260
261
262#endif /* XNU_KERNEL_PRIVATE */
263
264__END_DECLS
265
266#endif /* _KERN_LOCK_GROUP_H */
267