1 | /* |
2 | * Copyright (c) 2019 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | #include <sys/ioctl.h> |
29 | |
30 | #include <sys/stat.h> |
31 | #include <miscfs/devfs/devfs.h> |
32 | #include <sys/conf.h> |
33 | #include <sys/systm.h> |
34 | #include <sys/dtrace.h> |
35 | #include <sys/dtrace_impl.h> |
36 | #include <kern/lock_group.h> |
37 | #include <kern/lock_stat.h> |
38 | |
39 | #if LOCK_STATS |
40 | |
41 | #define LP_NODE "lockprof" |
42 | |
43 | #define LOCKPROF_AFRAMES 3 |
44 | #define LOCKPROF_LEN 64 |
45 | |
46 | static dtrace_provider_id_t lockprof_id; |
47 | |
48 | #define LOCKPROF_MAX 10000 /* maximum number of lockprof probes */ |
49 | static uint32_t lockprof_count; /* current number of lockprof probes */ |
50 | |
51 | enum probe_flags { |
52 | /* |
53 | * Counts time spent spinning/blocking |
54 | */ |
55 | TIME_EVENT = 0x01, |
56 | /* |
57 | * Requires LCK_GRP_ATTR_STAT to be set on the lock |
58 | * group, either via lck_grp_attr_setsta on the lock group, |
59 | * or globally via the lcks=3 boot-arg |
60 | */ |
61 | STAT_NEEDED = 0x02 |
62 | }; |
63 | |
64 | static const struct { |
65 | const char *prefix; |
66 | int flags; |
67 | size_t count_offset; |
68 | size_t stat_offset; |
69 | } probes[] = { |
70 | {"spin-held-" , 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_held)}, |
71 | {"spin-miss-" , 0, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_miss)}, |
72 | {"spin-spin-" , TIME_EVENT, offsetof(lck_grp_t, lck_grp_spincnt), offsetof(lck_grp_stats_t, lgss_spin_spin)}, |
73 | {"ticket-held-" , 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_held)}, |
74 | {"ticket-miss-" , 0, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_miss)}, |
75 | {"ticket-spin-" , TIME_EVENT, offsetof(lck_grp_t, lck_grp_ticketcnt), offsetof(lck_grp_stats_t, lgss_ticket_spin)}, |
76 | {"adaptive-held-" , STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_held)}, |
77 | {"adaptive-miss-" , STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_miss)}, |
78 | {"adaptive-wait-" , STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_wait)}, |
79 | {"adaptive-direct-wait-" , STAT_NEEDED, offsetof(lck_grp_t, lck_grp_mtxcnt), offsetof(lck_grp_stats_t, lgss_mtx_direct_wait)}, |
80 | {NULL, false, 0, 0} |
81 | }; |
82 | |
83 | /* |
84 | * Default defined probes for counting events |
85 | */ |
86 | const static int hold_defaults[] = { |
87 | 10000 /* 10000 events */ |
88 | }; |
89 | |
90 | /* |
91 | * Default defined probes for time events |
92 | */ |
93 | const static struct { |
94 | unsigned int time; |
95 | const char *suffix; |
96 | uint64_t mult; |
97 | } cont_defaults[] = { |
98 | {100, "ms" , NANOSEC / MILLISEC} /* 100 ms */ |
99 | }; |
100 | |
101 | typedef struct lockprof_probe { |
102 | int lockprof_kind; |
103 | dtrace_id_t lockprof_id; |
104 | uint64_t lockprof_limit; |
105 | lck_grp_t *lockprof_grp; |
106 | } lockprof_probe_t; |
107 | |
108 | static int |
109 | lockprof_lock_count(lck_grp_t *grp, int kind) |
110 | { |
111 | return *(int*)((uintptr_t)(grp) + probes[kind].count_offset); |
112 | } |
113 | |
114 | static void |
115 | probe_create(int kind, const char *suffix, const char *grp_name, uint64_t count, uint64_t mult) |
116 | { |
117 | uint64_t limit = count * mult; |
118 | |
119 | if (probes[kind].flags & TIME_EVENT) { |
120 | nanoseconds_to_absolutetime(limit, &limit); |
121 | } |
122 | |
123 | lck_grp_foreach(^bool (lck_grp_t *grp) { |
124 | char name[LOCKPROF_LEN]; |
125 | |
126 | if (!grp_name || grp_name[0] == '\0' || strcmp(grp_name, grp->lck_grp_name) == 0) { |
127 | snprintf(name, sizeof(name), "%s%llu%s" , probes[kind].prefix, count, suffix ?: "" ); |
128 | |
129 | if (dtrace_probe_lookup(lockprof_id, grp->lck_grp_name, NULL, name) != 0) { |
130 | return true; |
131 | } |
132 | if (lockprof_lock_count(grp, kind) == 0) { |
133 | return true; |
134 | } |
135 | if ((probes[kind].flags & STAT_NEEDED) && !lck_grp_has_stats(grp)) { |
136 | return true; |
137 | } |
138 | if (lockprof_count >= LOCKPROF_MAX) { |
139 | return false; |
140 | } |
141 | |
142 | lockprof_probe_t *probe = kmem_zalloc(sizeof(lockprof_probe_t), KM_SLEEP); |
143 | probe->lockprof_kind = kind; |
144 | probe->lockprof_limit = limit; |
145 | probe->lockprof_grp = grp; |
146 | |
147 | lck_grp_reference(grp, NULL); |
148 | |
149 | probe->lockprof_id = dtrace_probe_create(lockprof_id, grp->lck_grp_name, NULL, name, |
150 | LOCKPROF_AFRAMES, probe); |
151 | |
152 | lockprof_count++; |
153 | } |
154 | |
155 | return true; |
156 | }); |
157 | } |
158 | |
159 | static void |
160 | lockprof_provide(void *arg, const dtrace_probedesc_t *desc) |
161 | { |
162 | #pragma unused(arg) |
163 | size_t event_id, i, j, len; |
164 | |
165 | if (desc == NULL) { |
166 | for (i = 0; i < sizeof(hold_defaults) / sizeof(hold_defaults[0]); i++) { |
167 | for (j = 0; probes[j].prefix != NULL; j++) { |
168 | if (!(probes[j].flags & TIME_EVENT)) { |
169 | probe_create(j, NULL, NULL, hold_defaults[i], 1); |
170 | } |
171 | } |
172 | } |
173 | for (i = 0; i < sizeof(cont_defaults) / sizeof(cont_defaults[0]); i++) { |
174 | for (j = 0; probes[j].prefix != NULL; j++) { |
175 | if (probes[j].flags & TIME_EVENT) { |
176 | probe_create(j, cont_defaults[i].suffix, NULL, cont_defaults[i].time, cont_defaults[i].mult); |
177 | } |
178 | } |
179 | } |
180 | return; |
181 | } |
182 | |
183 | const char *name, *suffix = NULL; |
184 | hrtime_t val = 0, mult = 1; |
185 | |
186 | const struct { |
187 | const char *name; |
188 | hrtime_t mult; |
189 | } suffixes[] = { |
190 | { "us" , NANOSEC / MICROSEC }, |
191 | { "usec" , NANOSEC / MICROSEC }, |
192 | { "ms" , NANOSEC / MILLISEC }, |
193 | { "msec" , NANOSEC / MILLISEC }, |
194 | { "s" , NANOSEC / SEC }, |
195 | { "sec" , NANOSEC / SEC }, |
196 | { NULL, 0 } |
197 | }; |
198 | |
199 | name = desc->dtpd_name; |
200 | |
201 | for (event_id = 0; probes[event_id].prefix != NULL; event_id++) { |
202 | len = strlen(probes[event_id].prefix); |
203 | |
204 | if (strncmp(name, probes[event_id].prefix, len) != 0) { |
205 | continue; |
206 | } |
207 | break; |
208 | } |
209 | |
210 | if (probes[event_id].prefix == NULL) { |
211 | return; |
212 | } |
213 | |
214 | |
215 | /* |
216 | * We need to start before any time suffix. |
217 | */ |
218 | for (i = strlen(name); i >= len; i--) { |
219 | if (name[i] >= '0' && name[i] <= '9') { |
220 | break; |
221 | } |
222 | suffix = &name[i]; |
223 | } |
224 | |
225 | /* |
226 | * Now determine the numerical value present in the probe name. |
227 | */ |
228 | for (uint64_t m = 1; i >= len; i--) { |
229 | if (name[i] < '0' || name[i] > '9') { |
230 | return; |
231 | } |
232 | |
233 | val += (name[i] - '0') * m; |
234 | m *= (hrtime_t)10; |
235 | } |
236 | |
237 | if (val == 0) { |
238 | return; |
239 | } |
240 | |
241 | if (probes[event_id].flags & TIME_EVENT) { |
242 | for (i = 0, mult = 0; suffixes[i].name != NULL; i++) { |
243 | if (suffix && (strncasecmp(suffixes[i].name, suffix, strlen(suffixes[i].name) + 1) == 0)) { |
244 | mult = suffixes[i].mult; |
245 | break; |
246 | } |
247 | } |
248 | if (suffixes[i].name == NULL) { |
249 | return; |
250 | } |
251 | } else if (suffix && (*suffix != '\0')) { |
252 | return; |
253 | } |
254 | |
255 | probe_create(event_id, suffix, desc->dtpd_mod, val, mult); |
256 | } |
257 | |
258 | |
259 | static lck_grp_stat_t* |
260 | lockprof_stat(lck_grp_t *grp, int kind) |
261 | { |
262 | return (lck_grp_stat_t*)((uintptr_t)&grp->lck_grp_stats + probes[kind].stat_offset); |
263 | } |
264 | |
265 | static int |
266 | lockprof_enable(void *arg, dtrace_id_t id, void *parg) |
267 | { |
268 | #pragma unused(arg, id, parg) |
269 | lockprof_probe_t *probe = (lockprof_probe_t*)parg; |
270 | lck_grp_t *grp = probe->lockprof_grp; |
271 | lck_grp_stat_t *stat; |
272 | |
273 | if (grp == NULL) { |
274 | return -1; |
275 | } |
276 | |
277 | if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) { |
278 | return -1; |
279 | } |
280 | |
281 | /* |
282 | * lockprof_enable/disable are called with |
283 | * dtrace_lock held |
284 | */ |
285 | if (stat->lgs_limit != 0) { |
286 | return -1; |
287 | } |
288 | |
289 | stat->lgs_limit = probe->lockprof_limit; |
290 | stat->lgs_probeid = probe->lockprof_id; |
291 | lck_grp_stat_enable(stat); |
292 | lck_grp_enable_feature(LCK_DEBUG_LOCKPROF); |
293 | |
294 | return 0; |
295 | } |
296 | |
297 | static void |
298 | lockprof_disable(void *arg, dtrace_id_t id, void *parg) |
299 | { |
300 | #pragma unused(arg, id) |
301 | lockprof_probe_t *probe = (lockprof_probe_t*)parg; |
302 | lck_grp_t *grp = probe->lockprof_grp; |
303 | lck_grp_stat_t *stat; |
304 | |
305 | if (grp == NULL) { |
306 | return; |
307 | } |
308 | |
309 | if ((stat = lockprof_stat(grp, probe->lockprof_kind)) == NULL) { |
310 | return; |
311 | } |
312 | |
313 | if (stat->lgs_limit == 0 || !lck_grp_stat_enabled(stat)) { |
314 | return; |
315 | } |
316 | |
317 | stat->lgs_limit = 0; |
318 | stat->lgs_probeid = 0; |
319 | lck_grp_stat_disable(stat); |
320 | lck_grp_disable_feature(LCK_DEBUG_LOCKPROF); |
321 | } |
322 | |
323 | static void |
324 | lockprof_destroy(void *arg, dtrace_id_t id, void *parg) |
325 | { |
326 | #pragma unused(arg, id) |
327 | lockprof_probe_t *probe = (lockprof_probe_t*)parg; |
328 | lck_grp_deallocate(probe->lockprof_grp, NULL); |
329 | kmem_free(probe, sizeof(lockprof_probe_t)); |
330 | lockprof_count--; |
331 | } |
332 | |
333 | static void |
334 | lockprof_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) |
335 | { |
336 | #pragma unused(arg, id, parg) |
337 | const char *argdesc = NULL; |
338 | switch (desc->dtargd_ndx) { |
339 | case 0: |
340 | argdesc = "lck_grp_t*" ; |
341 | break; |
342 | case 1: |
343 | argdesc = "uint64_t" ; |
344 | break; |
345 | } |
346 | |
347 | if (argdesc) { |
348 | strlcpy(desc->dtargd_native, argdesc, DTRACE_ARGTYPELEN); |
349 | } else { |
350 | desc->dtargd_ndx = DTRACE_ARGNONE; |
351 | } |
352 | } |
353 | static dtrace_pattr_t lockprof_attr = { |
354 | { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, |
355 | { DTRACE_STABILITY_UNSTABLE, DTRACE_STABILITY_UNSTABLE, DTRACE_CLASS_UNKNOWN }, |
356 | { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, |
357 | { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, |
358 | { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, |
359 | }; |
360 | |
361 | static dtrace_pops_t lockprof_pops = { |
362 | .dtps_provide = lockprof_provide, |
363 | .dtps_provide_module = NULL, |
364 | .dtps_enable = lockprof_enable, |
365 | .dtps_disable = lockprof_disable, |
366 | .dtps_suspend = NULL, |
367 | .dtps_resume = NULL, |
368 | .dtps_getargdesc = lockprof_getargdesc, |
369 | .dtps_getargval = NULL, |
370 | .dtps_usermode = NULL, |
371 | .dtps_destroy = lockprof_destroy |
372 | }; |
373 | |
374 | static int |
375 | _lockprof_open(dev_t dev, int flags, int devtype, struct proc *p) |
376 | { |
377 | #pragma unused(dev,flags,devtype,p) |
378 | return 0; |
379 | } |
380 | |
381 | static const struct cdevsw lockprof_cdevsw = |
382 | { |
383 | .d_open = _lockprof_open, |
384 | .d_close = eno_opcl, |
385 | .d_read = eno_rdwrt, |
386 | .d_write = eno_rdwrt, |
387 | .d_ioctl = eno_ioctl, |
388 | .d_stop = eno_stop, |
389 | .d_reset = eno_reset, |
390 | .d_select = eno_select, |
391 | .d_mmap = eno_mmap, |
392 | .d_strategy = eno_strat, |
393 | .d_reserved_1 = eno_getc, |
394 | .d_reserved_2 = eno_putc, |
395 | }; |
396 | |
397 | |
398 | #endif /* LOCK_STATS */ |
399 | void lockprof_init(void); |
400 | void |
401 | lockprof_init(void) |
402 | { |
403 | #if LOCK_STATS |
404 | int majorno = cdevsw_add(-1, &lockprof_cdevsw); |
405 | |
406 | if (majorno < 0) { |
407 | panic("dtrace: failed to allocate a major number" ); |
408 | return; |
409 | } |
410 | |
411 | if (dtrace_register(LP_NODE, &lockprof_attr, DTRACE_PRIV_KERNEL, |
412 | NULL, &lockprof_pops, NULL, &lockprof_id) != 0) { |
413 | panic("dtrace: failed to register lockprof provider" ); |
414 | } |
415 | |
416 | dev_t dev = makedev(majorno, 0); |
417 | |
418 | if (devfs_make_node( dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, |
419 | LP_NODE ) == NULL) { |
420 | panic("dtrace: devfs_make_node failed for lockprof" ); |
421 | } |
422 | |
423 | #endif /* LOCK_STATS */ |
424 | } |
425 | |