1 | /* |
2 | * CDDL HEADER START |
3 | * |
4 | * The contents of this file are subject to the terms of the |
5 | * Common Development and Distribution License (the "License"). |
6 | * You may not use this file except in compliance with the License. |
7 | * |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
9 | * or http://www.opensolaris.org/os/licensing. |
10 | * See the License for the specific language governing permissions |
11 | * and limitations under the License. |
12 | * |
13 | * When distributing Covered Code, include this CDDL HEADER in each |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
15 | * If applicable, add the following below this CDDL HEADER, with the |
16 | * fields enclosed by brackets "[]" replaced with your own identifying |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] |
18 | * |
19 | * CDDL HEADER END |
20 | */ |
21 | /* |
22 | * Copyright 2009 Sun Microsystems, Inc. All rights reserved. |
23 | * Use is subject to license terms. |
24 | */ |
25 | |
26 | #include <sys/param.h> |
27 | #include <sys/systm.h> |
28 | #include <sys/errno.h> |
29 | #include <sys/stat.h> |
30 | #include <sys/ioctl.h> |
31 | #include <sys/conf.h> |
32 | #include <sys/fcntl.h> |
33 | #include <miscfs/devfs/devfs.h> |
34 | |
35 | #include <sys/dtrace.h> |
36 | #include <sys/dtrace_impl.h> |
37 | |
38 | #include <sys/dtrace_glue.h> |
39 | |
40 | #include <sys/lockstat.h> |
41 | |
42 | #include <kern/lock_stat.h> |
43 | |
44 | #define PROBE_ARGS0(a, b, c, d, e) "\000" |
45 | #define PROBE_ARGS1(a, b, c, d, e) a "\000" |
46 | #define PROBE_ARGS2(a, b, c, d, e) a "\000" b "\000" |
47 | #define PROBE_ARGS3(a, b, c, d, e) a "\000" b "\000" c "\000" |
48 | #define PROBE_ARGS4(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" |
49 | #define PROBE_ARGS5(a, b, c, d, e) a "\000" b "\000" c "\000" d "\000" e "\000" |
50 | #define PROBE_ARGS_(a, b, c, d, e, n, ...) PROBE_ARGS##n(a, b, c, d, e) |
51 | #define PROBE_ARGS(...) PROBE_ARGS_(__VA_ARGS__, 5, 4, 3, 2, 1, 0) |
52 | |
53 | #define LOCKSTAT_PROBE(func, name, probe, ...) \ |
54 | {func, name, probe, DTRACE_IDNONE, PROBE_ARGS(__VA_ARGS__)} |
55 | |
56 | #if defined(__x86_64__) |
57 | #define LOCKSTAT_AFRAMES 1 |
58 | #elif defined(__arm64__) |
59 | #define LOCKSTAT_AFRAMES 2 |
60 | #else |
61 | #error "architecture not supported" |
62 | #endif |
63 | |
64 | typedef struct lockstat_probe { |
65 | const char *lsp_func; |
66 | const char *lsp_name; |
67 | int lsp_probe; |
68 | dtrace_id_t lsp_id; |
69 | const char *lsp_args; |
70 | } lockstat_probe_t; |
71 | |
72 | lockstat_probe_t lockstat_probes[] = |
73 | { |
74 | // Mutex probes |
75 | // .. acquire/release |
76 | LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_ACQUIRE, LS_LCK_MTX_LOCK_ACQUIRE, "lck_mtx_t" , "lck_grp_t" ), |
77 | LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN, LSA_ACQUIRE, LS_LCK_MTX_LOCK_SPIN_ACQUIRE, "lck_mtx_t" , "lck_grp_t" ), |
78 | LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_ACQUIRE, "lck_mtx_t" , "lck_grp_t" ), |
79 | LOCKSTAT_PROBE(LS_LCK_MTX_TRY_LOCK_SPIN, LSA_ACQUIRE, LS_LCK_MTX_TRY_LOCK_SPIN_ACQUIRE, "lck_mtx_t" , "lck_grp_t" ), |
80 | LOCKSTAT_PROBE(LS_LCK_MTX_UNLOCK, LSA_RELEASE, LS_LCK_MTX_UNLOCK_RELEASE, "lck_mtx_t" , "lck_grp_t" ), |
81 | |
82 | // .. blocking and spinning |
83 | LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_BLOCK, LS_LCK_MTX_LOCK_BLOCK, "lck_mtx_t" , "uint64_t" , "lck_grp_t" ), |
84 | LOCKSTAT_PROBE(LS_LCK_MTX_LOCK, LSA_SPIN, LS_LCK_MTX_LOCK_ADAPTIVE_SPIN, "lck_mtx_t" , "uint64_t" , "lck_grp_t" ), |
85 | LOCKSTAT_PROBE(LS_LCK_MTX_LOCK_SPIN, LSA_SPIN, LS_LCK_MTX_LOCK_SPIN_SPIN, "lck_mtx_t" , "uint64_t" , "lck_grp_t" ), |
86 | |
87 | // RW lock probes |
88 | // TODO: This should not be a uint64_t ! |
89 | LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_LOCK_SHARED_ACQUIRE, "lck_rw_t" , "uint64_t" ), |
90 | LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_BLOCK, "lck_rw_t" , "uint64_t" , "_Bool" , "_Bool" , "int" ), |
91 | LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_SPIN, "lck_rw_t" , "uint64_t" , "_Bool" , "_Bool" , "int" ), |
92 | LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_LOCK_EXCL_ACQUIRE, "lck_rw_t" , "uint64_t" ), |
93 | // TODO: This should NOT be a uint64_t |
94 | LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_EXCL_BLOCK, "lck_rw_t" , "uint64_t" , "_Bool" , "_Bool" , "int" ), |
95 | LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_EXCL_SPIN, "lck_rw_t" , "uint64_t" , "int" ), |
96 | LOCKSTAT_PROBE(LS_LCK_RW_DONE, LSR_RELEASE, LS_LCK_RW_DONE_RELEASE, "lck_rw_t" , "_Bool" ), |
97 | // TODO : This should NOT be a uint64_t |
98 | LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_SHARED, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, "lck_rw_t" , "uint64_t" ), |
99 | LOCKSTAT_PROBE(LS_LCK_RW_TRY_LOCK_EXCL, LSR_ACQUIRE, LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, "lck_rw_t" , "uint64_t" ), |
100 | LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE, LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, "lck_rw_t" , "_Bool" ), |
101 | LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_SPIN, LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, "lck_rw_t" , "uint64_t" ), |
102 | LOCKSTAT_PROBE(LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_BLOCK, LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, "lck_rw_t" , "uint64_t" , "_Bool" , "_Bool" , "int" ), |
103 | |
104 | // Spin lock probes |
105 | //TODO : Separate the probes for the hw_bit from the probe for the normal hw locks |
106 | LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_ACQUIRE, LS_LCK_SPIN_LOCK_ACQUIRE, "hw_lock_t" ), |
107 | LOCKSTAT_PROBE(LS_LCK_SPIN_LOCK, LSS_SPIN, LS_LCK_SPIN_LOCK_SPIN, "hw_lock_t" , "uint64_t" , "uint64_t" ), |
108 | LOCKSTAT_PROBE(LS_LCK_SPIN_UNLOCK, LSS_RELEASE, LS_LCK_SPIN_UNLOCK_RELEASE, "hw_lock_t" ), |
109 | LOCKSTAT_PROBE(LS_LCK_RW_LOCK_EXCL_TO_SHARED, LSR_DOWNGRADE, LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, "lck_rw_t" ), |
110 | // Ticket lock probes |
111 | LOCKSTAT_PROBE(LS_LCK_TICKET_LOCK, LST_ACQUIRE, LS_LCK_TICKET_LOCK_ACQUIRE, "lck_ticket_t" ), |
112 | LOCKSTAT_PROBE(LS_LCK_TICKET_LOCK, LST_RELEASE, LS_LCK_TICKET_LOCK_RELEASE, "lck_ticket_t" ), |
113 | LOCKSTAT_PROBE(LS_LCK_TICKET_LOCK, LST_SPIN, LS_LCK_TICKET_LOCK_SPIN, "lck_ticket_t" ), |
114 | { |
115 | NULL, NULL, .lsp_probe: 0, .lsp_id: 0, NULL |
116 | } |
117 | }; |
118 | |
119 | dtrace_id_t lockstat_probemap[LS_NPROBES]; |
120 | |
121 | static dtrace_provider_id_t lockstat_id; |
122 | |
123 | /*ARGSUSED*/ |
124 | static int |
125 | lockstat_enable(void *arg, dtrace_id_t id, void *parg) |
126 | { |
127 | #pragma unused(arg) /* __APPLE__ */ |
128 | |
129 | lockstat_probe_t *probe = parg; |
130 | |
131 | ASSERT(!lockstat_probemap[probe->lsp_probe]); |
132 | |
133 | lockstat_probemap[probe->lsp_probe] = id; |
134 | lck_grp_enable_feature(feat: LCK_DEBUG_LOCKSTAT); |
135 | |
136 | return 0; |
137 | } |
138 | |
139 | /*ARGSUSED*/ |
140 | static void |
141 | lockstat_disable(void *arg, dtrace_id_t id, void *parg) |
142 | { |
143 | #pragma unused(arg, id) /* __APPLE__ */ |
144 | |
145 | lockstat_probe_t *probe = parg; |
146 | |
147 | ASSERT(lockstat_probemap[probe->lsp_probe]); |
148 | |
149 | lockstat_probemap[probe->lsp_probe] = 0; |
150 | lck_grp_disable_feature(feat: LCK_DEBUG_LOCKSTAT); |
151 | } |
152 | |
153 | /*ARGSUSED*/ |
154 | static void |
155 | lockstat_provide(void *arg, const dtrace_probedesc_t *desc) |
156 | { |
157 | #pragma unused(arg, desc) /* __APPLE__ */ |
158 | |
159 | int i = 0; |
160 | |
161 | for (i = 0; lockstat_probes[i].lsp_func != NULL; i++) { |
162 | lockstat_probe_t *probe = &lockstat_probes[i]; |
163 | |
164 | if (dtrace_probe_lookup(lockstat_id, "mach_kernel" , |
165 | probe->lsp_func, probe->lsp_name) != 0) { |
166 | continue; |
167 | } |
168 | |
169 | ASSERT(!probe->lsp_id); |
170 | probe->lsp_id = dtrace_probe_create(lockstat_id, |
171 | "mach_kernel" , probe->lsp_func, probe->lsp_name, |
172 | LOCKSTAT_AFRAMES, probe); |
173 | } |
174 | } |
175 | |
176 | |
177 | /*ARGSUSED*/ |
178 | static void |
179 | lockstat_destroy(void *arg, dtrace_id_t id, void *parg) |
180 | { |
181 | #pragma unused(arg, id) |
182 | |
183 | lockstat_probe_t *probe = parg; |
184 | |
185 | ASSERT(!lockstat_probemap[probe->lsp_probe]); |
186 | probe->lsp_id = 0; |
187 | } |
188 | |
189 | static void |
190 | lockstat_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) |
191 | { |
192 | #pragma unused(arg, id) |
193 | lockstat_probe_t *probe = parg; |
194 | const char* argdesc = probe->lsp_args; |
195 | int narg = 0; |
196 | |
197 | desc->dtargd_native[0] = '\0'; |
198 | desc->dtargd_xlate[0] = '\0'; |
199 | |
200 | while (argdesc[0] != '\0') { |
201 | if (narg == desc->dtargd_ndx) { |
202 | strlcpy(dst: desc->dtargd_native, src: argdesc, DTRACE_ARGTYPELEN); |
203 | return; |
204 | } |
205 | argdesc += strlen(s: argdesc) + 1; |
206 | narg++; |
207 | } |
208 | |
209 | desc->dtargd_ndx = DTRACE_ARGNONE; |
210 | } |
211 | |
212 | static dtrace_pattr_t lockstat_attr = { |
213 | { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, |
214 | { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, |
215 | { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, |
216 | { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, |
217 | { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, |
218 | }; |
219 | |
220 | static dtrace_pops_t lockstat_pops = { |
221 | .dtps_provide = lockstat_provide, |
222 | .dtps_provide_module = NULL, |
223 | .dtps_enable = lockstat_enable, |
224 | .dtps_disable = lockstat_disable, |
225 | .dtps_suspend = NULL, |
226 | .dtps_resume = NULL, |
227 | .dtps_getargdesc = lockstat_getargdesc, |
228 | .dtps_getargval = NULL, |
229 | .dtps_usermode = NULL, |
230 | .dtps_destroy = lockstat_destroy |
231 | }; |
232 | |
233 | static int |
234 | lockstat_attach(dev_info_t *devi) |
235 | { |
236 | if (ddi_create_minor_node(devi, "lockstat" , S_IFCHR, 0, |
237 | DDI_PSEUDO, 0) == DDI_FAILURE || |
238 | dtrace_register("lockstat" , &lockstat_attr, DTRACE_PRIV_KERNEL, |
239 | NULL, &lockstat_pops, NULL, &lockstat_id) != 0) { |
240 | ddi_remove_minor_node(devi, NULL); |
241 | return DDI_FAILURE; |
242 | } |
243 | |
244 | return DDI_SUCCESS; |
245 | } |
246 | |
247 | static int |
248 | _lockstat_open(dev_t dev, int flags, int devtype, struct proc *p) |
249 | { |
250 | #pragma unused(dev,flags,devtype,p) |
251 | return 0; |
252 | } |
253 | |
254 | |
255 | static const struct cdevsw lockstat_cdevsw = |
256 | { |
257 | .d_open = _lockstat_open, |
258 | .d_close = eno_opcl, |
259 | .d_read = eno_rdwrt, |
260 | .d_write = eno_rdwrt, |
261 | .d_ioctl = eno_ioctl, |
262 | .d_stop = eno_stop, |
263 | .d_reset = eno_reset, |
264 | .d_select = eno_select, |
265 | .d_mmap = eno_mmap, |
266 | .d_strategy = eno_strat, |
267 | .d_reserved_1 = eno_getc, |
268 | .d_reserved_2 = eno_putc, |
269 | }; |
270 | |
271 | void lockstat_init( void ); |
272 | |
273 | void |
274 | lockstat_init( void ) |
275 | { |
276 | int majdevno = cdevsw_add(-1, &lockstat_cdevsw); |
277 | |
278 | if (majdevno < 0) { |
279 | printf("lockstat_init: failed to allocate a major number!\n" ); |
280 | return; |
281 | } |
282 | |
283 | lockstat_attach(devi: (dev_info_t*)(uintptr_t)majdevno); |
284 | } |
285 | |