1/*
2 * Copyright (c) 2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28#include <sys/ioctl.h>
29
30#include <sys/stat.h>
31#include <miscfs/devfs/devfs.h>
32#include <sys/conf.h>
33#include <sys/systm.h>
34#include <sys/dtrace.h>
35#include <sys/dtrace_impl.h>
36#include <kern/lock_group.h>
37#include <kern/lock_stat.h>
38
39#if LOCK_STATS
40
41#define LP_NODE "lockprof"
42
43#define LOCKPROF_AFRAMES 3
44#define LOCKPROF_LEN 64
45
46static dtrace_provider_id_t lockprof_id;
47
48#define LOCKPROF_MAX 10000 /* maximum number of lockprof probes */
49static uint32_t lockprof_count; /* current number of lockprof probes */
50
51enum probe_flags {
52 /*
53 * Counts time spent spinning/blocking
54 */
55 TIME_EVENT = 0x01,
56 /*
57 * Requires LCK_GRP_ATTR_STAT to be set on the lock
58 * group, either via lck_grp_attr_setsta on the lock group,
59 * or globally via the lcks=3 boot-arg
60 */
61 STAT_NEEDED = 0x02
62};
63
64static const struct {
65 const char *prefix;
66 int flags;
67 size_t count_offset;
68 size_t stat_offset;
69} probes[] = {
70 {"spin-held-", 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_held)},
71 {"spin-miss-", 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_miss)},
72 {"spin-spin-", TIME_EVENT, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_spin)},
73 {"ticket-held-", 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_held)},
74 {"ticket-miss-", 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_miss)},
75 {"ticket-spin-", TIME_EVENT, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_spin)},
76 {"adaptive-held-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_held)},
77 {"adaptive-miss-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_miss)},
78 {"adaptive-wait-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_wait)},
79 {"adaptive-direct-wait-", STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_direct_wait)},
80 {NULL, false, 0, 0}
81};
82
83/*
84 * Default defined probes for counting events
85 */
86const static int hold_defaults[] = {
87 10000 /* 10000 events */
88};
89
90/*
91 * Default defined probes for time events
92 */
93const static struct {
94 unsigned int time;
95 const char *suffix;
96 uint64_t mult;
97} cont_defaults[] = {
98 {100, "ms", NANOSEC / MILLISEC} /* 100 ms */
99};
100
101typedef struct lockprof_probe {
102 int lockprof_kind;
103 dtrace_id_t lockprof_id;
104 uint64_t lockprof_limit;
105 lck_grp_t *lockprof_grp;
106} lockprof_probe_t;
107
108static int
109lockprof_lock_count(lck_grp_t *grp, int kind)
110{
111 return *(int*)((uintptr_t)(grp) + probes[kind].count_offset);
112}
113
114static void
115probe_create(int kind, const char *suffix, const char *grp_name, uint64_t count, uint64_t mult)
116{
117 uint64_t limit = count * mult;
118
119 if (probes[kind].flags & TIME_EVENT) {
120 nanoseconds_to_absolutetime(limit, &limit);
121 }
122
123 lck_grp_foreach(^bool (lck_grp_t *grp) {
124 char name[LOCKPROF_LEN];
125
126 if (!grp_name || grp_name[0] == '\0' || strcmp(grp_name, grp->lck_grp_name) == 0) {
127 snprintf(name, sizeof(name), "%s%llu%s", probes[kind].prefix, count, suffix ?: "");
128
129 if (dtrace_probe_lookup(lockprof_id, grp->lck_grp_name, NULL, name) != 0) {
130 return true;
131 }
132 if (lockprof_lock_count(grp, kind) == 0) {
133 return true;
134 }
135 if ((probes[kind].flags & STAT_NEEDED) && !lck_grp_has_stats(grp)) {
136 return true;
137 }
138 if (lockprof_count >= LOCKPROF_MAX) {
139 return false;
140 }
141
142 lockprof_probe_t *probe = kmem_zalloc(sizeof(lockprof_probe_t), KM_SLEEP);
143 probe->lockprof_kind = kind;
144 probe->lockprof_limit = limit;
145 probe->lockprof_grp = grp;
146
147 lck_grp_reference(grp, NULL);
148
149 probe->lockprof_id = dtrace_probe_create(lockprof_id, grp->lck_grp_name, NULL, name,
150 LOCKPROF_AFRAMES, probe);
151
152 lockprof_count++;
153 }
154
155 return true;
156 });
157}
158
159static void
160lockprof_provide(void *arg, const dtrace_probedesc_t *desc)
161{
162#pragma unused(arg)
163 size_t event_id, i, j, len;
164
165 if (desc == NULL) {
166 for (i = 0; i < sizeof(hold_defaults) / sizeof(hold_defaults[0]); i++) {
167 for (j = 0; probes[j].prefix != NULL; j++) {
168 if (!(probes[j].flags & TIME_EVENT)) {
169 probe_create(j, NULL, NULL, hold_defaults[i], 1);
170 }
171 }
172 }
173 for (i = 0; i < sizeof(cont_defaults) / sizeof(cont_defaults[0]); i++) {
174 for (j = 0; probes[j].prefix != NULL; j++) {
175 if (probes[j].flags & TIME_EVENT) {
176 probe_create(j, cont_defaults[i].suffix, NULL, cont_defaults[i].time, cont_defaults[i].mult);
177 }
178 }
179 }
180 return;
181 }
182
183 const char *name, *suffix = NULL;
184 hrtime_t val = 0, mult = 1;
185
186 const struct {
187 const char *name;
188 hrtime_t mult;
189 } suffixes[] = {
190 { "us", NANOSEC / MICROSEC },
191 { "usec", NANOSEC / MICROSEC },
192 { "ms", NANOSEC / MILLISEC },
193 { "msec", NANOSEC / MILLISEC },
194 { "s", NANOSEC / SEC },
195 { "sec", NANOSEC / SEC },
196 { NULL, 0 }
197 };
198
199 name = desc->dtpd_name;
200
201 for (event_id = 0; probes[event_id].prefix != NULL; event_id++) {
202 len = strlen(probes[event_id].prefix);
203
204 if (strncmp(name, probes[event_id].prefix, len) != 0) {
205 continue;
206 }
207 break;
208 }
209
210 if (probes[event_id].prefix == NULL) {
211 return;
212 }
213
214
215 /*
216 * We need to start before any time suffix.
217 */
218 for (i = strlen(name); i >= len; i--) {
219 if (name[i] >= '0' && name[i] <= '9') {
220 break;
221 }
222 suffix = &name[i];
223 }
224
225 /*
226 * Now determine the numerical value present in the probe name.
227 */
228 for (uint64_t m = 1; i >= len; i--) {
229 if (name[i] < '0' || name[i] > '9') {
230 return;
231 }
232
233 val += (name[i] - '0') * m;
234 m *= (hrtime_t)10;
235 }
236
237 if (val == 0) {
238 return;
239 }
240
241 if (probes[event_id].flags & TIME_EVENT) {
242 for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
243 if (suffix && (strncasecmp(suffixes[i].name, suffix, strlen(suffixes[i].name) + 1) == 0)) {
244 mult = suffixes[i].mult;
245 break;
246 }
247 }
248 if (suffixes[i].name == NULL) {
249 return;
250 }
251 } else if (suffix && (*suffix != '\0')) {
252 return;
253 }
254
255 probe_create(event_id, suffix, desc->dtpd_mod, val, mult);
256}
257
258
259static lck_grp_stat_t*
260lockprof_stat(lck_grp_t *grp, int kind)
261{
262 return (lck_grp_stat_t*)((uintptr_t)&grp->lck_grp_stats + probes[kind].stat_offset);
263}
264
265static int
266lockprof_enable(void *arg, dtrace_id_t id, void *parg)
267{
268#pragma unused(arg, id, parg)
269 lockprof_probe_t *probe = (lockprof_probe_t*)parg;
270 lck_grp_t *grp = probe->lockprof_grp;
271 lck_grp_stat_t *stat;
272
273 if (grp == NULL) {
274 return -1;
275 }
276
277 if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
278 return -1;
279 }
280
281 /*
282 * lockprof_enable/disable are called with
283 * dtrace_lock held
284 */
285 if (stat->lgs_limit != 0) {
286 return -1;
287 }
288
289 stat->lgs_limit = probe->lockprof_limit;
290 stat->lgs_probeid = probe->lockprof_id;
291 lck_grp_stat_enable(stat);
292 lck_grp_enable_feature(LCK_DEBUG_LOCKPROF);
293
294 return 0;
295}
296
297static void
298lockprof_disable(void *arg, dtrace_id_t id, void *parg)
299{
300#pragma unused(arg, id)
301 lockprof_probe_t *probe = (lockprof_probe_t*)parg;
302 lck_grp_t *grp = probe->lockprof_grp;
303 lck_grp_stat_t *stat;
304
305 if (grp == NULL) {
306 return;
307 }
308
309 if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) {
310 return;
311 }
312
313 if (stat->lgs_limit == 0 || !lck_grp_stat_enabled(stat)) {
314 return;
315 }
316
317 stat->lgs_limit = 0;
318 stat->lgs_probeid = 0;
319 lck_grp_stat_disable(stat);
320 lck_grp_disable_feature(LCK_DEBUG_LOCKPROF);
321}
322
323static void
324lockprof_destroy(void *arg, dtrace_id_t id, void *parg)
325{
326#pragma unused(arg, id)
327 lockprof_probe_t *probe = (lockprof_probe_t*)parg;
328 lck_grp_deallocate(probe->lockprof_grp, NULL);
329 kmem_free(probe, sizeof(lockprof_probe_t));
330 lockprof_count--;
331}
332
333static void
334lockprof_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
335{
336#pragma unused(arg, id, parg)
337 const char *argdesc = NULL;
338 switch (desc->dtargd_ndx) {
339 case 0:
340 argdesc = "lck_grp_t*";
341 break;
342 case 1:
343 argdesc = "uint64_t";
344 break;
345 }
346
347 if (argdesc) {
348 strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN);
349 } else {
350 desc->dtargd_ndx = DTRACE_ARGNONE;
351 }
352}
353static dtrace_pattr_t lockprof_attr = {
354 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
355 { DTRACE_STABILITY_UNSTABLE, DTRACE_STABILITY_UNSTABLE, DTRACE_CLASS_UNKNOWN },
356 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
357 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
358 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
359};
360
361static dtrace_pops_t lockprof_pops = {
362 .dtps_provide = lockprof_provide,
363 .dtps_provide_module = NULL,
364 .dtps_enable = lockprof_enable,
365 .dtps_disable = lockprof_disable,
366 .dtps_suspend = NULL,
367 .dtps_resume = NULL,
368 .dtps_getargdesc = lockprof_getargdesc,
369 .dtps_getargval = NULL,
370 .dtps_usermode = NULL,
371 .dtps_destroy = lockprof_destroy
372};
373
374static int
375_lockprof_open(dev_t dev, int flags, int devtype, struct proc *p)
376{
377#pragma unused(dev,flags,devtype,p)
378 return 0;
379}
380
381static const struct cdevsw lockprof_cdevsw =
382{
383 .d_open = _lockprof_open,
384 .d_close = eno_opcl,
385 .d_read = eno_rdwrt,
386 .d_write = eno_rdwrt,
387 .d_ioctl = eno_ioctl,
388 .d_stop = eno_stop,
389 .d_reset = eno_reset,
390 .d_select = eno_select,
391 .d_mmap = eno_mmap,
392 .d_strategy = eno_strat,
393 .d_reserved_1 = eno_getc,
394 .d_reserved_2 = eno_putc,
395};
396
397
398#endif /* LOCK_STATS */
399void lockprof_init(void);
400void
401lockprof_init(void)
402{
403#if LOCK_STATS
404 int majorno = cdevsw_add(-1, &lockprof_cdevsw);
405
406 if (majorno < 0) {
407 panic("dtrace: failed to allocate a major number");
408 return;
409 }
410
411 if (dtrace_register(LP_NODE, &lockprof_attr, DTRACE_PRIV_KERNEL,
412 NULL, &lockprof_pops, NULL, &lockprof_id) != 0) {
413 panic("dtrace: failed to register lockprof provider");
414 }
415
416 dev_t dev = makedev(majorno, 0);
417
418 if (devfs_make_node( dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666,
419 LP_NODE ) == NULL) {
420 panic("dtrace: devfs_make_node failed for lockprof");
421 }
422
423#endif /* LOCK_STATS */
424}
425