1 | /* |
2 | * CDDL HEADER START |
3 | * |
4 | * The contents of this file are subject to the terms of the |
5 | * Common Development and Distribution License (the "License"). |
6 | * You may not use this file except in compliance with the License. |
7 | * |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
9 | * or http://www.opensolaris.org/os/licensing. |
10 | * See the License for the specific language governing permissions |
11 | * and limitations under the License. |
12 | * |
13 | * When distributing Covered Code, include this CDDL HEADER in each |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
15 | * If applicable, add the following below this CDDL HEADER, with the |
16 | * fields enclosed by brackets "[]" replaced with your own identifying |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] |
18 | * |
19 | * CDDL HEADER END |
20 | */ |
21 | /* |
22 | * Copyright 2009 Sun Microsystems, Inc. All rights reserved. |
23 | * Use is subject to license terms. |
24 | */ |
25 | |
26 | #include <sys/param.h> |
27 | #include <sys/systm.h> |
28 | #include <sys/errno.h> |
29 | #include <sys/stat.h> |
30 | #include <sys/ioctl.h> |
31 | #include <sys/conf.h> |
32 | #include <sys/fcntl.h> |
33 | #include <miscfs/devfs/devfs.h> |
34 | |
35 | #if defined(__arm64__) |
36 | #include <arm/caches_internal.h> |
37 | #endif /* defined(__arm64__) */ |
38 | |
39 | #include <sys/dtrace.h> |
40 | #include <sys/dtrace_impl.h> |
41 | |
42 | #include <sys/dtrace_glue.h> |
43 | |
44 | #include <sys/sdt_impl.h> |
45 | extern int dtrace_kernel_symbol_mode; |
46 | |
47 | #include <ptrauth.h> |
48 | |
49 | /* #include <machine/trap.h */ |
50 | struct savearea_t; /* Used anonymously */ |
51 | |
52 | #if defined(__arm64__) |
53 | typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int); |
54 | extern perfCallback tempDTraceTrapHook; |
55 | extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int); |
56 | #define SDT_PATCHVAL 0xe7eeee7e |
57 | #define SDT_AFRAMES 7 |
58 | #elif defined(__x86_64__) |
59 | typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, int); |
60 | extern perfCallback tempDTraceTrapHook; |
61 | extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, int); |
62 | #define SDT_PATCHVAL 0xf0 |
63 | #define SDT_AFRAMES 6 |
64 | #else |
65 | #error Unknown architecture |
66 | #endif |
67 | |
68 | #define SDT_PROBETAB_SIZE 0x1000 /* 4k entries -- 16K total */ |
69 | |
70 | #define SDT_UNKNOWN_FUNCNAME "." /* function symbol name when not found in symbol table */ |
71 | |
72 | |
73 | static int sdt_verbose = 0; |
74 | sdt_probe_t **sdt_probetab; |
75 | int sdt_probetab_size; |
76 | int sdt_probetab_mask; |
77 | |
78 | /*ARGSUSED*/ |
79 | static void |
80 | __sdt_provide_module(void *arg, struct modctl *ctl) |
81 | { |
82 | #pragma unused(arg) |
83 | char *modname = ctl->mod_modname; |
84 | sdt_probedesc_t *sdpd; |
85 | sdt_probe_t *sdp, *old; |
86 | sdt_provider_t *prov; |
87 | |
88 | /* |
89 | * One for all, and all for one: if we haven't yet registered all of |
90 | * our providers, we'll refuse to provide anything. |
91 | */ |
92 | for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) { |
93 | if (prov->sdtp_id == DTRACE_PROVNONE) { |
94 | return; |
95 | } |
96 | } |
97 | |
98 | /* Nothing to do. Module is either invalid or we haven't found any SDT probe descriptions. */ |
99 | if (!ctl || ctl->mod_sdtprobecnt != 0 || (sdpd = ctl->mod_sdtdesc) == NULL) { |
100 | return; |
101 | } |
102 | |
103 | for (sdpd = ctl->mod_sdtdesc; sdpd != NULL; sdpd = sdpd->sdpd_next) { |
104 | dtrace_id_t id; |
105 | |
106 | /* Validate probe's provider name. Do not provide probes for unknown providers. */ |
107 | for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) { |
108 | if (strcmp(s1: prov->sdtp_prefix, s2: sdpd->sdpd_prov) == 0) { |
109 | break; |
110 | } |
111 | } |
112 | |
113 | if (prov->sdtp_name == NULL) { |
114 | printf("Ignoring probes from unsupported provider %s\n" , sdpd->sdpd_prov); |
115 | continue; |
116 | } |
117 | |
118 | if (sdpd->sdpd_func == NULL) { |
119 | /* |
120 | * Ignore probes for which we don't have any symbol. That's likely some problem with |
121 | * __sdt section processing. |
122 | */ |
123 | printf("Ignoring probe %s (no symbol name)\n" , sdpd->sdpd_name); |
124 | continue; |
125 | } |
126 | |
127 | sdp = kmem_zalloc(sizeof(sdt_probe_t), KM_SLEEP); |
128 | sdp->sdp_loadcnt = ctl->mod_loadcnt; |
129 | sdp->sdp_ctl = ctl; |
130 | sdp->sdp_name = kmem_alloc(strlen(sdpd->sdpd_name) + 1, KM_SLEEP); |
131 | (void) strlcpy(dst: sdp->sdp_name, src: sdpd->sdpd_name, n: strlen(s: sdpd->sdpd_name) + 1); |
132 | sdp->sdp_namelen = strlen(s: sdpd->sdpd_name) + 1; |
133 | sdp->sdp_provider = prov; |
134 | |
135 | /* |
136 | * We have our provider. Now create the probe. |
137 | */ |
138 | if ((id = dtrace_probe_lookup(prov->sdtp_id, modname, |
139 | sdpd->sdpd_func, sdp->sdp_name)) != DTRACE_IDNONE) { |
140 | old = dtrace_probe_arg(prov->sdtp_id, id); |
141 | ASSERT(old != NULL); |
142 | |
143 | sdp->sdp_next = old->sdp_next; |
144 | sdp->sdp_id = id; |
145 | old->sdp_next = sdp; |
146 | } else { |
147 | sdp->sdp_id = dtrace_probe_create(prov->sdtp_id, |
148 | modname, sdpd->sdpd_func, sdp->sdp_name, SDT_AFRAMES, sdp); |
149 | |
150 | ctl->mod_sdtprobecnt++; |
151 | } |
152 | |
153 | #if 0 |
154 | printf("__sdt_provide_module: sdpd=0x%p sdp=0x%p name=%s, id=%d\n" , sdpd, sdp, |
155 | sdp->sdp_name, sdp->sdp_id); |
156 | #endif |
157 | |
158 | sdp->sdp_hashnext = |
159 | sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)]; |
160 | sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp; |
161 | |
162 | sdp->sdp_patchval = SDT_PATCHVAL; |
163 | sdp->sdp_patchpoint = (sdt_instr_t *)sdpd->sdpd_offset; |
164 | sdp->sdp_savedval = *sdp->sdp_patchpoint; |
165 | } |
166 | } |
167 | |
168 | /*ARGSUSED*/ |
169 | static void |
170 | sdt_destroy(void *arg, dtrace_id_t id, void *parg) |
171 | { |
172 | #pragma unused(arg,id) |
173 | sdt_probe_t *sdp = parg, *old, *last, *hash; |
174 | int ndx; |
175 | |
176 | struct modctl *ctl = sdp->sdp_ctl; |
177 | |
178 | /* |
179 | * Decrement SDT probe counts only when a probe being destroyed belongs to the |
180 | * currently loaded version of a module and not the stale one. |
181 | */ |
182 | if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt && ctl->mod_loaded) { |
183 | ctl->mod_sdtprobecnt--; |
184 | } |
185 | |
186 | while (sdp != NULL) { |
187 | old = sdp; |
188 | |
189 | /* |
190 | * Now we need to remove this probe from the sdt_probetab. |
191 | */ |
192 | ndx = SDT_ADDR2NDX(sdp->sdp_patchpoint); |
193 | last = NULL; |
194 | hash = sdt_probetab[ndx]; |
195 | |
196 | while (hash != sdp) { |
197 | ASSERT(hash != NULL); |
198 | last = hash; |
199 | hash = hash->sdp_hashnext; |
200 | } |
201 | |
202 | if (last != NULL) { |
203 | last->sdp_hashnext = sdp->sdp_hashnext; |
204 | } else { |
205 | sdt_probetab[ndx] = sdp->sdp_hashnext; |
206 | } |
207 | |
208 | kmem_free(sdp->sdp_name, sdp->sdp_namelen); |
209 | sdp = sdp->sdp_next; |
210 | kmem_free(old, sizeof(sdt_probe_t)); |
211 | } |
212 | } |
213 | |
214 | /*ARGSUSED*/ |
215 | static int |
216 | sdt_enable(void *arg, dtrace_id_t id, void *parg) |
217 | { |
218 | #pragma unused(arg,id) |
219 | sdt_probe_t *sdp = parg; |
220 | struct modctl *ctl = sdp->sdp_ctl; |
221 | |
222 | ctl->mod_nenabled++; |
223 | |
224 | /* |
225 | * If this module has disappeared since we discovered its probes, |
226 | * refuse to enable it. |
227 | */ |
228 | if (!ctl->mod_loaded) { |
229 | if (sdt_verbose) { |
230 | cmn_err(CE_NOTE, "sdt is failing for probe %s " |
231 | "(module %s unloaded)" , |
232 | sdp->sdp_name, ctl->mod_modname); |
233 | } |
234 | goto err; |
235 | } |
236 | |
237 | /* |
238 | * Now check that our modctl has the expected load count. If it |
239 | * doesn't, this module must have been unloaded and reloaded -- and |
240 | * we're not going to touch it. |
241 | */ |
242 | if (ctl->mod_loadcnt != sdp->sdp_loadcnt) { |
243 | if (sdt_verbose) { |
244 | cmn_err(CE_NOTE, "sdt is failing for probe %s " |
245 | "(module %s reloaded)" , |
246 | sdp->sdp_name, ctl->mod_modname); |
247 | } |
248 | goto err; |
249 | } |
250 | |
251 | dtrace_casptr(&tempDTraceTrapHook, NULL, ptrauth_nop_cast(void *, &fbt_perfCallback)); |
252 | if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) { |
253 | if (sdt_verbose) { |
254 | cmn_err(CE_NOTE, "sdt_enable is failing for probe %s " |
255 | "in module %s: tempDTraceTrapHook already occupied." , |
256 | sdp->sdp_name, ctl->mod_modname); |
257 | } |
258 | return 0; |
259 | } |
260 | |
261 | while (sdp != NULL) { |
262 | (void)ml_nofault_copy(virtsrc: (vm_offset_t)&sdp->sdp_patchval, virtdst: (vm_offset_t)sdp->sdp_patchpoint, |
263 | size: (vm_size_t)sizeof(sdp->sdp_patchval)); |
264 | |
265 | /* |
266 | * Make the patched instruction visible via a data + instruction |
267 | * cache fush on platforms that need it |
268 | */ |
269 | flush_dcache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_patchval), 0); |
270 | invalidate_icache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_patchval), 0); |
271 | |
272 | sdp = sdp->sdp_next; |
273 | } |
274 | |
275 | err: |
276 | return 0; |
277 | } |
278 | |
279 | /*ARGSUSED*/ |
280 | static void |
281 | sdt_disable(void *arg, dtrace_id_t id, void *parg) |
282 | { |
283 | #pragma unused(arg,id) |
284 | sdt_probe_t *sdp = parg; |
285 | struct modctl *ctl = sdp->sdp_ctl; |
286 | |
287 | ctl->mod_nenabled--; |
288 | |
289 | if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt) { |
290 | goto err; |
291 | } |
292 | |
293 | while (sdp != NULL) { |
294 | (void)ml_nofault_copy(virtsrc: (vm_offset_t)&sdp->sdp_savedval, virtdst: (vm_offset_t)sdp->sdp_patchpoint, |
295 | size: (vm_size_t)sizeof(sdp->sdp_savedval)); |
296 | /* |
297 | * Make the patched instruction visible via a data + instruction |
298 | * cache flush on platforms that need it |
299 | */ |
300 | flush_dcache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_savedval), 0); |
301 | invalidate_icache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_savedval), 0); |
302 | sdp = sdp->sdp_next; |
303 | } |
304 | |
305 | err: |
306 | ; |
307 | } |
308 | |
309 | static dtrace_pops_t sdt_pops = { |
310 | .dtps_provide = NULL, |
311 | .dtps_provide_module = sdt_provide_module, |
312 | .dtps_enable = sdt_enable, |
313 | .dtps_disable = sdt_disable, |
314 | .dtps_suspend = NULL, |
315 | .dtps_resume = NULL, |
316 | .dtps_getargdesc = sdt_getargdesc, |
317 | .dtps_getargval = sdt_getarg, |
318 | .dtps_usermode = NULL, |
319 | .dtps_destroy = sdt_destroy, |
320 | }; |
321 | |
322 | /*ARGSUSED*/ |
323 | static int |
324 | sdt_attach(dev_info_t *devi) |
325 | { |
326 | sdt_provider_t *prov; |
327 | |
328 | if (ddi_create_minor_node(devi, "sdt" , S_IFCHR, |
329 | 0, DDI_PSEUDO, 0) == DDI_FAILURE) { |
330 | cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node" ); |
331 | ddi_remove_minor_node(devi, NULL); |
332 | return DDI_FAILURE; |
333 | } |
334 | |
335 | if (sdt_probetab_size == 0) { |
336 | sdt_probetab_size = SDT_PROBETAB_SIZE; |
337 | } |
338 | |
339 | sdt_probetab_mask = sdt_probetab_size - 1; |
340 | sdt_probetab = |
341 | kmem_zalloc(sdt_probetab_size * sizeof(sdt_probe_t *), KM_SLEEP); |
342 | dtrace_invop_add(sdt_invop); |
343 | |
344 | for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) { |
345 | if (dtrace_register(prov->sdtp_name, prov->sdtp_attr, |
346 | DTRACE_PRIV_KERNEL, NULL, |
347 | &sdt_pops, prov, &prov->sdtp_id) != 0) { |
348 | cmn_err(CE_WARN, "failed to register sdt provider %s" , |
349 | prov->sdtp_name); |
350 | } |
351 | } |
352 | |
353 | return DDI_SUCCESS; |
354 | } |
355 | |
356 | /* |
357 | * APPLE NOTE: sdt_detach not implemented |
358 | */ |
359 | #if !defined(__APPLE__) |
360 | /*ARGSUSED*/ |
361 | static int |
362 | sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) |
363 | { |
364 | sdt_provider_t *prov; |
365 | |
366 | switch (cmd) { |
367 | case DDI_DETACH: |
368 | break; |
369 | |
370 | case DDI_SUSPEND: |
371 | return DDI_SUCCESS; |
372 | |
373 | default: |
374 | return DDI_FAILURE; |
375 | } |
376 | |
377 | for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) { |
378 | if (prov->sdtp_id != DTRACE_PROVNONE) { |
379 | if (dtrace_unregister(prov->sdtp_id) != 0) { |
380 | return DDI_FAILURE; |
381 | } |
382 | |
383 | prov->sdtp_id = DTRACE_PROVNONE; |
384 | } |
385 | } |
386 | |
387 | dtrace_invop_remove(sdt_invop); |
388 | kmem_free(sdt_probetab, sdt_probetab_size * sizeof(sdt_probe_t *)); |
389 | |
390 | return DDI_SUCCESS; |
391 | } |
392 | #endif /* __APPLE__ */ |
393 | |
394 | d_open_t _sdt_open; |
395 | |
396 | int |
397 | _sdt_open(dev_t dev, int flags, int devtype, struct proc *p) |
398 | { |
399 | #pragma unused(dev,flags,devtype,p) |
400 | return 0; |
401 | } |
402 | |
403 | #define SDT_MAJOR -24 /* let the kernel pick the device number */ |
404 | |
405 | static const struct cdevsw sdt_cdevsw = |
406 | { |
407 | .d_open = _sdt_open, |
408 | .d_close = eno_opcl, |
409 | .d_read = eno_rdwrt, |
410 | .d_write = eno_rdwrt, |
411 | .d_ioctl = eno_ioctl, |
412 | .d_stop = eno_stop, |
413 | .d_reset = eno_reset, |
414 | .d_select = eno_select, |
415 | .d_mmap = eno_mmap, |
416 | .d_strategy = eno_strat, |
417 | .d_reserved_1 = eno_getc, |
418 | .d_reserved_2 = eno_putc, |
419 | }; |
420 | |
421 | |
422 | #include <mach-o/nlist.h> |
423 | #include <libkern/kernel_mach_header.h> |
424 | |
425 | /* |
426 | * Represents single record in __DATA_CONST,__sdt section. |
427 | */ |
428 | typedef struct dtrace_sdt_def { |
429 | uintptr_t dsd_addr; /* probe site location */ |
430 | const char *dsd_prov; /* provider's name */ |
431 | const char *dsd_name; /* probe's name */ |
432 | } __attribute__((__packed__)) dtrace_sdt_def_t; |
433 | |
434 | /* |
435 | * Creates a copy of name and unescapes '-' characters. |
436 | */ |
437 | static char * |
438 | sdt_strdup_name(const char *name) |
439 | { |
440 | size_t len = strlen(s: name) + 1; |
441 | size_t i, j; |
442 | char *nname = kmem_alloc(len, KM_SLEEP); |
443 | |
444 | for (i = 0, j = 0; name[j] != '\0'; i++) { |
445 | if (name[j] == '_' && name[j + 1] == '_') { |
446 | nname[i] = '-'; |
447 | j += 2; |
448 | } else { |
449 | nname[i] = name[j++]; |
450 | } |
451 | } |
452 | |
453 | nname[i] = '\0'; |
454 | return nname; |
455 | } |
456 | |
457 | /* |
458 | * Returns Mach-O header that should be used for given modctl. |
459 | */ |
460 | static kernel_mach_header_t * |
461 | sdt_get_module_mh(struct modctl *ctl) |
462 | { |
463 | kernel_mach_header_t *mh = (kernel_mach_header_t *)ctl->mod_address; |
464 | |
465 | /* Static KEXTs have their __sdt section merged into kernel's __sdt. */ |
466 | if (MOD_IS_STATIC_KEXT(ctl)) { |
467 | mh = &_mh_execute_header; |
468 | } |
469 | |
470 | if (mh->magic != MH_MAGIC_KERNEL) { |
471 | return NULL; |
472 | } |
473 | |
474 | return mh; |
475 | } |
476 | |
477 | /* |
478 | * Finds symbol table for given kernel module. |
479 | */ |
480 | static uint32_t |
481 | sdt_find_symbol_table(struct modctl *ctl, kernel_nlist_t **sym, char **strings) |
482 | { |
483 | kernel_mach_header_t *mh = sdt_get_module_mh(ctl); |
484 | struct load_command *cmd = (struct load_command *)&mh[1]; |
485 | kernel_segment_command_t *orig_le = NULL; |
486 | struct symtab_command *orig_st = NULL; |
487 | |
488 | for (int i = 0; i < mh->ncmds; i++) { |
489 | if (cmd->cmd == LC_SEGMENT_KERNEL) { |
490 | kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd; |
491 | |
492 | if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT)) { |
493 | orig_le = orig_sg; |
494 | } |
495 | } else if (cmd->cmd == LC_SYMTAB) { |
496 | orig_st = (struct symtab_command *) cmd; |
497 | } |
498 | |
499 | cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize); |
500 | } |
501 | |
502 | if ((orig_st == NULL) || (orig_le == NULL)) { |
503 | return 0; |
504 | } |
505 | |
506 | *sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff); |
507 | *strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff); |
508 | |
509 | return orig_st->nsyms; |
510 | } |
511 | |
512 | /* Last kernel address. */ |
513 | static SECURITY_READ_ONLY_LATE(vm_address_t) kern_end = (vm_address_t)-1; |
514 | |
515 | void |
516 | sdt_early_init(void) |
517 | { |
518 | kernel_mach_header_t *mh = &_mh_execute_header; |
519 | kernel_section_t *sec_ks = NULL; |
520 | kc_format_t kc_format; |
521 | |
522 | if (!PE_get_primary_kc_format(type: &kc_format)) { |
523 | kc_format = KCFormatUnknown; |
524 | } |
525 | |
526 | /* |
527 | * Detects end of kernel's text in static kernel cache. It is the last text address before |
528 | * the first kext text section start. |
529 | */ |
530 | if (kc_format == KCFormatStatic) { |
531 | if ((sec_ks = getsectbynamefromheader(header: mh, seg_name: "__PRELINK_INFO" , sect_name: "__kmod_start" )) == NULL) { |
532 | printf("SDT: unable to find prelink info\n" ); |
533 | return; |
534 | } |
535 | |
536 | /* find the MIN(start_address) of all kexts in this image. */ |
537 | const uint64_t *start_addr = (const uint64_t *)sec_ks->addr; |
538 | for (int i = 0; i < sec_ks->size / sizeof(uint64_t); i++) { |
539 | if (kern_end > start_addr[i]) { |
540 | kern_end = start_addr[i]; |
541 | } |
542 | } |
543 | } |
544 | } |
545 | |
546 | /* |
547 | * Finds TEXT range that belongs to given module. |
548 | */ |
549 | static int |
550 | sdt_find_module_text_range(struct modctl *ctl, vm_address_t *start, vm_address_t *end) |
551 | { |
552 | kc_format_t kc_format; |
553 | |
554 | if (!PE_get_primary_kc_format(type: &kc_format)) { |
555 | kc_format = KCFormatUnknown; |
556 | } |
557 | |
558 | /* Adjust kernel region for static kernel cache. */ |
559 | *start = ctl->mod_address; |
560 | |
561 | if (MOD_IS_MACH_KERNEL(ctl) && kc_format == KCFormatStatic) { |
562 | *end = kern_end; |
563 | } else { |
564 | *end = ctl->mod_address + ctl->mod_size; |
565 | } |
566 | |
567 | return 1; |
568 | } |
569 | |
570 | /* |
571 | * Processes SDT section in given Mach-O header |
572 | */ |
573 | void |
574 | sdt_load_machsect(struct modctl *ctl) |
575 | { |
576 | kernel_mach_header_t *mh = sdt_get_module_mh(ctl); |
577 | kernel_section_t *sec_sdt = NULL; |
578 | char *strings = NULL; |
579 | kernel_nlist_t *sym = NULL; |
580 | vm_address_t text_start, text_end; |
581 | unsigned int len; |
582 | uint32_t nsyms = 0; |
583 | |
584 | if (mh == NULL) { |
585 | return; |
586 | } |
587 | |
588 | /* Ignore SDT definitions if we don't know where they belong. */ |
589 | if (!sdt_find_module_text_range(ctl, start: &text_start, end: &text_end)) { |
590 | printf("SDT: Unable to determine text range for %s\n" , ctl->mod_modname); |
591 | return; |
592 | } |
593 | |
594 | /* Do not load SDTs when asked to use kernel symbols but symbol table is not available. */ |
595 | if (MOD_HAS_KERNEL_SYMBOLS(ctl) && (nsyms = sdt_find_symbol_table(ctl, sym: &sym, strings: &strings)) == 0) { |
596 | printf("SDT: No kernel symbols for %s\n" , ctl->mod_modname); |
597 | return; |
598 | } |
599 | |
600 | /* Locate DTrace SDT section in the object. */ |
601 | if ((sec_sdt = getsectbynamefromheader(header: mh, seg_name: "__DATA_CONST" , sect_name: "__sdt" )) == NULL) { |
602 | return; |
603 | } |
604 | |
605 | /* |
606 | * Iterate over SDT section and establish all SDT probe descriptions. |
607 | */ |
608 | dtrace_sdt_def_t *sdtdef = (dtrace_sdt_def_t *)(sec_sdt->addr); |
609 | for (size_t k = 0; k < sec_sdt->size / sizeof(dtrace_sdt_def_t); k++, sdtdef++) { |
610 | unsigned long best = 0; |
611 | |
612 | /* |
613 | * Static KEXTs share __sdt section with kernel after linking. It is required |
614 | * to filter out description and pick only those that belong to requested |
615 | * module or kernel itself. |
616 | */ |
617 | if (MOD_IS_STATIC_KEXT(ctl) || MOD_IS_MACH_KERNEL(ctl)) { |
618 | if ((sdtdef->dsd_addr < text_start) || (sdtdef->dsd_addr > text_end)) { |
619 | continue; |
620 | } |
621 | } else { |
622 | /* Skip over probe descripton that do not belong to current module. */ |
623 | if (!dtrace_addr_in_module((void *)sdtdef->dsd_addr, ctl)) { |
624 | continue; |
625 | } |
626 | } |
627 | |
628 | sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP); |
629 | |
630 | /* Unescape probe name and keep a note of the size of original memory allocation. */ |
631 | sdpd->sdpd_name = sdt_strdup_name(name: sdtdef->dsd_name); |
632 | sdpd->sdpd_namelen = strlen(s: sdtdef->dsd_name) + 1; |
633 | |
634 | /* Used only for provider structure lookup so there is no need to make dynamic copy. */ |
635 | sdpd->sdpd_prov = sdtdef->dsd_prov; |
636 | |
637 | /* |
638 | * Find the symbol immediately preceding the sdt probe site just discovered, |
639 | * that symbol names the function containing the sdt probe. |
640 | */ |
641 | sdpd->sdpd_func = NULL; |
642 | |
643 | if (MOD_HAS_KERNEL_SYMBOLS(ctl)) { |
644 | const char *funcname = SDT_UNKNOWN_FUNCNAME; |
645 | |
646 | for (int i = 0; i < nsyms; i++) { |
647 | uint8_t jn_type = sym[i].n_type & N_TYPE; |
648 | char *jname = strings + sym[i].n_un.n_strx; |
649 | |
650 | if ((N_SECT != jn_type && N_ABS != jn_type)) { |
651 | continue; |
652 | } |
653 | |
654 | if (0 == sym[i].n_un.n_strx) { /* iff a null, "", name. */ |
655 | continue; |
656 | } |
657 | |
658 | if (*jname == '_') { |
659 | jname += 1; |
660 | } |
661 | |
662 | if (sdtdef->dsd_addr <= (unsigned long)sym[i].n_value) { |
663 | continue; |
664 | } |
665 | |
666 | if ((unsigned long)sym[i].n_value > best) { |
667 | best = (unsigned long)sym[i].n_value; |
668 | funcname = jname; |
669 | } |
670 | } |
671 | |
672 | len = strlen(s: funcname) + 1; |
673 | sdpd->sdpd_func = kmem_alloc(len, KM_SLEEP); |
674 | (void) strlcpy(dst: sdpd->sdpd_func, src: funcname, n: len); |
675 | } |
676 | |
677 | #if defined(__arm64__) |
678 | sdpd->sdpd_offset = sdtdef->dsd_addr & ~0x1LU; |
679 | #else |
680 | sdpd->sdpd_offset = sdtdef->dsd_addr; |
681 | #endif /* __arm64__ */ |
682 | |
683 | sdpd->sdpd_next = (sdt_probedesc_t *)ctl->mod_sdtdesc; |
684 | ctl->mod_sdtdesc = sdpd; |
685 | } |
686 | } |
687 | |
688 | void |
689 | sdt_init( void ) |
690 | { |
691 | int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw); |
692 | |
693 | if (majdevno < 0) { |
694 | printf("sdt_init: failed to allocate a major number!\n" ); |
695 | return; |
696 | } |
697 | |
698 | if (dtrace_sdt_probes_restricted()) { |
699 | return; |
700 | } |
701 | |
702 | sdt_attach(devi: (dev_info_t*)(uintptr_t)majdevno); |
703 | } |
704 | |
705 | #undef SDT_MAJOR |
706 | |
707 | /* |
708 | * Provide SDT modules with userspace symbols. |
709 | * |
710 | * A module contains only partially filled in SDT probe descriptions because symbols were |
711 | * not available at the time when __sdt section was loaded. Fixup descriptons before providing |
712 | * the probes. |
713 | */ |
714 | static void |
715 | sdt_provide_module_user_syms(void *arg, struct modctl *ctl) |
716 | { |
717 | sdt_probedesc_t *sdpd; |
718 | dtrace_module_symbols_t *mod_sym = ctl->mod_user_symbols; |
719 | |
720 | if (mod_sym == NULL) { |
721 | printf("DTrace missing userspace symbols for module %s\n" , ctl->mod_modname); |
722 | return; |
723 | } |
724 | |
725 | /* Fixup missing probe description parts. */ |
726 | for (sdpd = ctl->mod_sdtdesc; sdpd != NULL; sdpd = sdpd->sdpd_next) { |
727 | ASSERT(sdpd->sdpd_func == NULL); |
728 | const char *funcname = SDT_UNKNOWN_FUNCNAME; |
729 | |
730 | /* Look for symbol that contains SDT probe offset. */ |
731 | for (int i = 0; i < mod_sym->dtmodsyms_count; i++) { |
732 | dtrace_symbol_t *symbol = &mod_sym->dtmodsyms_symbols[i]; |
733 | char *name = symbol->dtsym_name; |
734 | |
735 | /* |
736 | * Every function symbol gets extra '_' prepended in the Mach-O symbol table. |
737 | * Strip it away to make a probe's function name match source code. |
738 | */ |
739 | if (*name == '_') { |
740 | name += 1; |
741 | } |
742 | |
743 | if (!symbol->dtsym_addr) { |
744 | continue; |
745 | } |
746 | |
747 | /* Ignore symbols that do not belong to this module. */ |
748 | if (!dtrace_addr_in_module((void *)symbol->dtsym_addr, ctl)) { |
749 | continue; |
750 | } |
751 | |
752 | /* Pick symbol name when we found match. */ |
753 | if ((symbol->dtsym_addr <= sdpd->sdpd_offset) && |
754 | (sdpd->sdpd_offset < symbol->dtsym_addr + symbol->dtsym_size)) { |
755 | funcname = name; |
756 | break; |
757 | } |
758 | } |
759 | |
760 | size_t len = strlen(s: funcname) + 1; |
761 | sdpd->sdpd_func = kmem_alloc(len, KM_SLEEP); |
762 | (void) strlcpy(dst: sdpd->sdpd_func, src: funcname, n: len); |
763 | } |
764 | |
765 | /* Probe descriptionds are now fixed up. Provide them as usual. */ |
766 | __sdt_provide_module(arg, ctl); |
767 | } |
768 | |
769 | /*ARGSUSED*/ |
770 | void |
771 | sdt_provide_module(void *arg, struct modctl *ctl) |
772 | { |
773 | ASSERT(ctl != NULL); |
774 | ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER); |
775 | LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED); |
776 | |
777 | if (MOD_SDT_DONE(ctl)) { |
778 | return; |
779 | } |
780 | |
781 | if (MOD_HAS_KERNEL_SYMBOLS(ctl)) { |
782 | __sdt_provide_module(arg, ctl); |
783 | ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED; |
784 | return; |
785 | } |
786 | |
787 | if (MOD_HAS_USERSPACE_SYMBOLS(ctl)) { |
788 | sdt_provide_module_user_syms(arg, ctl); |
789 | ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED; |
790 | return; |
791 | } |
792 | |
793 | /* |
794 | * The SDT provider's module is not detachable so we don't have to re-provide SDT |
795 | * probes if that happens. After succesfull providing, the probe descriptions are |
796 | * no longer required. If module gets re-loaded it will get a new set of probe |
797 | * descriptions from its __sdt section. |
798 | */ |
799 | if (MOD_SDT_PROBES_PROVIDED(ctl)) { |
800 | sdt_probedesc_t *sdpd = ctl->mod_sdtdesc; |
801 | while (sdpd) { |
802 | sdt_probedesc_t *this_sdpd = sdpd; |
803 | kmem_free((void *)sdpd->sdpd_name, sdpd->sdpd_namelen); |
804 | if (sdpd->sdpd_func) { |
805 | kmem_free((void *)sdpd->sdpd_func, strlen(s: sdpd->sdpd_func) + 1); |
806 | } |
807 | sdpd = sdpd->sdpd_next; |
808 | kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t)); |
809 | } |
810 | ctl->mod_sdtdesc = NULL; |
811 | } |
812 | } |
813 | |