1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#include <sys/param.h>
27#include <sys/systm.h>
28#include <sys/errno.h>
29#include <sys/stat.h>
30#include <sys/ioctl.h>
31#include <sys/conf.h>
32#include <sys/fcntl.h>
33#include <miscfs/devfs/devfs.h>
34
35#if defined(__arm64__)
36#include <arm/caches_internal.h>
37#endif /* defined(__arm64__) */
38
39#include <sys/dtrace.h>
40#include <sys/dtrace_impl.h>
41
42#include <sys/dtrace_glue.h>
43
44#include <sys/sdt_impl.h>
45extern int dtrace_kernel_symbol_mode;
46
47#include <ptrauth.h>
48
49/* #include <machine/trap.h */
50struct savearea_t; /* Used anonymously */
51
52#if defined(__arm64__)
53typedef kern_return_t (*perfCallback)(int, struct savearea_t *, __unused int, __unused int);
54extern perfCallback tempDTraceTrapHook;
55extern kern_return_t fbt_perfCallback(int, struct savearea_t *, __unused int, __unused int);
56#define SDT_PATCHVAL 0xe7eeee7e
57#define SDT_AFRAMES 7
58#elif defined(__x86_64__)
59typedef kern_return_t (*perfCallback)(int, struct savearea_t *, uintptr_t *, int);
60extern perfCallback tempDTraceTrapHook;
61extern kern_return_t fbt_perfCallback(int, struct savearea_t *, uintptr_t *, int);
62#define SDT_PATCHVAL 0xf0
63#define SDT_AFRAMES 6
64#else
65#error Unknown architecture
66#endif
67
68#define SDT_PROBETAB_SIZE 0x1000 /* 4k entries -- 16K total */
69
70#define SDT_UNKNOWN_FUNCNAME "." /* function symbol name when not found in symbol table */
71
72
73static int sdt_verbose = 0;
74sdt_probe_t **sdt_probetab;
75int sdt_probetab_size;
76int sdt_probetab_mask;
77
78/*ARGSUSED*/
79static void
80__sdt_provide_module(void *arg, struct modctl *ctl)
81{
82#pragma unused(arg)
83 char *modname = ctl->mod_modname;
84 sdt_probedesc_t *sdpd;
85 sdt_probe_t *sdp, *old;
86 sdt_provider_t *prov;
87
88 /*
89 * One for all, and all for one: if we haven't yet registered all of
90 * our providers, we'll refuse to provide anything.
91 */
92 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
93 if (prov->sdtp_id == DTRACE_PROVNONE) {
94 return;
95 }
96 }
97
98 /* Nothing to do. Module is either invalid or we haven't found any SDT probe descriptions. */
99 if (!ctl || ctl->mod_sdtprobecnt != 0 || (sdpd = ctl->mod_sdtdesc) == NULL) {
100 return;
101 }
102
103 for (sdpd = ctl->mod_sdtdesc; sdpd != NULL; sdpd = sdpd->sdpd_next) {
104 dtrace_id_t id;
105
106 /* Validate probe's provider name. Do not provide probes for unknown providers. */
107 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
108 if (strcmp(s1: prov->sdtp_prefix, s2: sdpd->sdpd_prov) == 0) {
109 break;
110 }
111 }
112
113 if (prov->sdtp_name == NULL) {
114 printf("Ignoring probes from unsupported provider %s\n", sdpd->sdpd_prov);
115 continue;
116 }
117
118 if (sdpd->sdpd_func == NULL) {
119 /*
120 * Ignore probes for which we don't have any symbol. That's likely some problem with
121 * __sdt section processing.
122 */
123 printf("Ignoring probe %s (no symbol name)\n", sdpd->sdpd_name);
124 continue;
125 }
126
127 sdp = kmem_zalloc(sizeof(sdt_probe_t), KM_SLEEP);
128 sdp->sdp_loadcnt = ctl->mod_loadcnt;
129 sdp->sdp_ctl = ctl;
130 sdp->sdp_name = kmem_alloc(strlen(sdpd->sdpd_name) + 1, KM_SLEEP);
131 (void) strlcpy(dst: sdp->sdp_name, src: sdpd->sdpd_name, n: strlen(s: sdpd->sdpd_name) + 1);
132 sdp->sdp_namelen = strlen(s: sdpd->sdpd_name) + 1;
133 sdp->sdp_provider = prov;
134
135 /*
136 * We have our provider. Now create the probe.
137 */
138 if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,
139 sdpd->sdpd_func, sdp->sdp_name)) != DTRACE_IDNONE) {
140 old = dtrace_probe_arg(prov->sdtp_id, id);
141 ASSERT(old != NULL);
142
143 sdp->sdp_next = old->sdp_next;
144 sdp->sdp_id = id;
145 old->sdp_next = sdp;
146 } else {
147 sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,
148 modname, sdpd->sdpd_func, sdp->sdp_name, SDT_AFRAMES, sdp);
149
150 ctl->mod_sdtprobecnt++;
151 }
152
153#if 0
154 printf("__sdt_provide_module: sdpd=0x%p sdp=0x%p name=%s, id=%d\n", sdpd, sdp,
155 sdp->sdp_name, sdp->sdp_id);
156#endif
157
158 sdp->sdp_hashnext =
159 sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];
160 sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp;
161
162 sdp->sdp_patchval = SDT_PATCHVAL;
163 sdp->sdp_patchpoint = (sdt_instr_t *)sdpd->sdpd_offset;
164 sdp->sdp_savedval = *sdp->sdp_patchpoint;
165 }
166}
167
168/*ARGSUSED*/
169static void
170sdt_destroy(void *arg, dtrace_id_t id, void *parg)
171{
172#pragma unused(arg,id)
173 sdt_probe_t *sdp = parg, *old, *last, *hash;
174 int ndx;
175
176 struct modctl *ctl = sdp->sdp_ctl;
177
178 /*
179 * Decrement SDT probe counts only when a probe being destroyed belongs to the
180 * currently loaded version of a module and not the stale one.
181 */
182 if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt && ctl->mod_loaded) {
183 ctl->mod_sdtprobecnt--;
184 }
185
186 while (sdp != NULL) {
187 old = sdp;
188
189 /*
190 * Now we need to remove this probe from the sdt_probetab.
191 */
192 ndx = SDT_ADDR2NDX(sdp->sdp_patchpoint);
193 last = NULL;
194 hash = sdt_probetab[ndx];
195
196 while (hash != sdp) {
197 ASSERT(hash != NULL);
198 last = hash;
199 hash = hash->sdp_hashnext;
200 }
201
202 if (last != NULL) {
203 last->sdp_hashnext = sdp->sdp_hashnext;
204 } else {
205 sdt_probetab[ndx] = sdp->sdp_hashnext;
206 }
207
208 kmem_free(sdp->sdp_name, sdp->sdp_namelen);
209 sdp = sdp->sdp_next;
210 kmem_free(old, sizeof(sdt_probe_t));
211 }
212}
213
214/*ARGSUSED*/
215static int
216sdt_enable(void *arg, dtrace_id_t id, void *parg)
217{
218#pragma unused(arg,id)
219 sdt_probe_t *sdp = parg;
220 struct modctl *ctl = sdp->sdp_ctl;
221
222 ctl->mod_nenabled++;
223
224 /*
225 * If this module has disappeared since we discovered its probes,
226 * refuse to enable it.
227 */
228 if (!ctl->mod_loaded) {
229 if (sdt_verbose) {
230 cmn_err(CE_NOTE, "sdt is failing for probe %s "
231 "(module %s unloaded)",
232 sdp->sdp_name, ctl->mod_modname);
233 }
234 goto err;
235 }
236
237 /*
238 * Now check that our modctl has the expected load count. If it
239 * doesn't, this module must have been unloaded and reloaded -- and
240 * we're not going to touch it.
241 */
242 if (ctl->mod_loadcnt != sdp->sdp_loadcnt) {
243 if (sdt_verbose) {
244 cmn_err(CE_NOTE, "sdt is failing for probe %s "
245 "(module %s reloaded)",
246 sdp->sdp_name, ctl->mod_modname);
247 }
248 goto err;
249 }
250
251 dtrace_casptr(&tempDTraceTrapHook, NULL, ptrauth_nop_cast(void *, &fbt_perfCallback));
252 if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
253 if (sdt_verbose) {
254 cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
255 "in module %s: tempDTraceTrapHook already occupied.",
256 sdp->sdp_name, ctl->mod_modname);
257 }
258 return 0;
259 }
260
261 while (sdp != NULL) {
262 (void)ml_nofault_copy(virtsrc: (vm_offset_t)&sdp->sdp_patchval, virtdst: (vm_offset_t)sdp->sdp_patchpoint,
263 size: (vm_size_t)sizeof(sdp->sdp_patchval));
264
265 /*
266 * Make the patched instruction visible via a data + instruction
267 * cache fush on platforms that need it
268 */
269 flush_dcache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_patchval), 0);
270 invalidate_icache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_patchval), 0);
271
272 sdp = sdp->sdp_next;
273 }
274
275err:
276 return 0;
277}
278
279/*ARGSUSED*/
280static void
281sdt_disable(void *arg, dtrace_id_t id, void *parg)
282{
283#pragma unused(arg,id)
284 sdt_probe_t *sdp = parg;
285 struct modctl *ctl = sdp->sdp_ctl;
286
287 ctl->mod_nenabled--;
288
289 if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt) {
290 goto err;
291 }
292
293 while (sdp != NULL) {
294 (void)ml_nofault_copy(virtsrc: (vm_offset_t)&sdp->sdp_savedval, virtdst: (vm_offset_t)sdp->sdp_patchpoint,
295 size: (vm_size_t)sizeof(sdp->sdp_savedval));
296 /*
297 * Make the patched instruction visible via a data + instruction
298 * cache flush on platforms that need it
299 */
300 flush_dcache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_savedval), 0);
301 invalidate_icache((vm_offset_t)sdp->sdp_patchpoint, (vm_size_t)sizeof(sdp->sdp_savedval), 0);
302 sdp = sdp->sdp_next;
303 }
304
305err:
306 ;
307}
308
309static dtrace_pops_t sdt_pops = {
310 .dtps_provide = NULL,
311 .dtps_provide_module = sdt_provide_module,
312 .dtps_enable = sdt_enable,
313 .dtps_disable = sdt_disable,
314 .dtps_suspend = NULL,
315 .dtps_resume = NULL,
316 .dtps_getargdesc = sdt_getargdesc,
317 .dtps_getargval = sdt_getarg,
318 .dtps_usermode = NULL,
319 .dtps_destroy = sdt_destroy,
320};
321
322/*ARGSUSED*/
323static int
324sdt_attach(dev_info_t *devi)
325{
326 sdt_provider_t *prov;
327
328 if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
329 0, DDI_PSEUDO, 0) == DDI_FAILURE) {
330 cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
331 ddi_remove_minor_node(devi, NULL);
332 return DDI_FAILURE;
333 }
334
335 if (sdt_probetab_size == 0) {
336 sdt_probetab_size = SDT_PROBETAB_SIZE;
337 }
338
339 sdt_probetab_mask = sdt_probetab_size - 1;
340 sdt_probetab =
341 kmem_zalloc(sdt_probetab_size * sizeof(sdt_probe_t *), KM_SLEEP);
342 dtrace_invop_add(sdt_invop);
343
344 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
345 if (dtrace_register(prov->sdtp_name, prov->sdtp_attr,
346 DTRACE_PRIV_KERNEL, NULL,
347 &sdt_pops, prov, &prov->sdtp_id) != 0) {
348 cmn_err(CE_WARN, "failed to register sdt provider %s",
349 prov->sdtp_name);
350 }
351 }
352
353 return DDI_SUCCESS;
354}
355
356/*
357 * APPLE NOTE: sdt_detach not implemented
358 */
359#if !defined(__APPLE__)
360/*ARGSUSED*/
361static int
362sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
363{
364 sdt_provider_t *prov;
365
366 switch (cmd) {
367 case DDI_DETACH:
368 break;
369
370 case DDI_SUSPEND:
371 return DDI_SUCCESS;
372
373 default:
374 return DDI_FAILURE;
375 }
376
377 for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
378 if (prov->sdtp_id != DTRACE_PROVNONE) {
379 if (dtrace_unregister(prov->sdtp_id) != 0) {
380 return DDI_FAILURE;
381 }
382
383 prov->sdtp_id = DTRACE_PROVNONE;
384 }
385 }
386
387 dtrace_invop_remove(sdt_invop);
388 kmem_free(sdt_probetab, sdt_probetab_size * sizeof(sdt_probe_t *));
389
390 return DDI_SUCCESS;
391}
392#endif /* __APPLE__ */
393
394d_open_t _sdt_open;
395
396int
397_sdt_open(dev_t dev, int flags, int devtype, struct proc *p)
398{
399#pragma unused(dev,flags,devtype,p)
400 return 0;
401}
402
403#define SDT_MAJOR -24 /* let the kernel pick the device number */
404
405static const struct cdevsw sdt_cdevsw =
406{
407 .d_open = _sdt_open,
408 .d_close = eno_opcl,
409 .d_read = eno_rdwrt,
410 .d_write = eno_rdwrt,
411 .d_ioctl = eno_ioctl,
412 .d_stop = eno_stop,
413 .d_reset = eno_reset,
414 .d_select = eno_select,
415 .d_mmap = eno_mmap,
416 .d_strategy = eno_strat,
417 .d_reserved_1 = eno_getc,
418 .d_reserved_2 = eno_putc,
419};
420
421
422#include <mach-o/nlist.h>
423#include <libkern/kernel_mach_header.h>
424
425/*
426 * Represents single record in __DATA_CONST,__sdt section.
427 */
428typedef struct dtrace_sdt_def {
429 uintptr_t dsd_addr; /* probe site location */
430 const char *dsd_prov; /* provider's name */
431 const char *dsd_name; /* probe's name */
432} __attribute__((__packed__)) dtrace_sdt_def_t;
433
434/*
435 * Creates a copy of name and unescapes '-' characters.
436 */
437static char *
438sdt_strdup_name(const char *name)
439{
440 size_t len = strlen(s: name) + 1;
441 size_t i, j;
442 char *nname = kmem_alloc(len, KM_SLEEP);
443
444 for (i = 0, j = 0; name[j] != '\0'; i++) {
445 if (name[j] == '_' && name[j + 1] == '_') {
446 nname[i] = '-';
447 j += 2;
448 } else {
449 nname[i] = name[j++];
450 }
451 }
452
453 nname[i] = '\0';
454 return nname;
455}
456
457/*
458 * Returns Mach-O header that should be used for given modctl.
459 */
460static kernel_mach_header_t *
461sdt_get_module_mh(struct modctl *ctl)
462{
463 kernel_mach_header_t *mh = (kernel_mach_header_t *)ctl->mod_address;
464
465 /* Static KEXTs have their __sdt section merged into kernel's __sdt. */
466 if (MOD_IS_STATIC_KEXT(ctl)) {
467 mh = &_mh_execute_header;
468 }
469
470 if (mh->magic != MH_MAGIC_KERNEL) {
471 return NULL;
472 }
473
474 return mh;
475}
476
477/*
478 * Finds symbol table for given kernel module.
479 */
480static uint32_t
481sdt_find_symbol_table(struct modctl *ctl, kernel_nlist_t **sym, char **strings)
482{
483 kernel_mach_header_t *mh = sdt_get_module_mh(ctl);
484 struct load_command *cmd = (struct load_command *)&mh[1];
485 kernel_segment_command_t *orig_le = NULL;
486 struct symtab_command *orig_st = NULL;
487
488 for (int i = 0; i < mh->ncmds; i++) {
489 if (cmd->cmd == LC_SEGMENT_KERNEL) {
490 kernel_segment_command_t *orig_sg = (kernel_segment_command_t *) cmd;
491
492 if (LIT_STRNEQL(orig_sg->segname, SEG_LINKEDIT)) {
493 orig_le = orig_sg;
494 }
495 } else if (cmd->cmd == LC_SYMTAB) {
496 orig_st = (struct symtab_command *) cmd;
497 }
498
499 cmd = (struct load_command *) ((uintptr_t) cmd + cmd->cmdsize);
500 }
501
502 if ((orig_st == NULL) || (orig_le == NULL)) {
503 return 0;
504 }
505
506 *sym = (kernel_nlist_t *)(orig_le->vmaddr + orig_st->symoff - orig_le->fileoff);
507 *strings = (char *)(orig_le->vmaddr + orig_st->stroff - orig_le->fileoff);
508
509 return orig_st->nsyms;
510}
511
512/* Last kernel address. */
513static SECURITY_READ_ONLY_LATE(vm_address_t) kern_end = (vm_address_t)-1;
514
515void
516sdt_early_init(void)
517{
518 kernel_mach_header_t *mh = &_mh_execute_header;
519 kernel_section_t *sec_ks = NULL;
520 kc_format_t kc_format;
521
522 if (!PE_get_primary_kc_format(type: &kc_format)) {
523 kc_format = KCFormatUnknown;
524 }
525
526 /*
527 * Detects end of kernel's text in static kernel cache. It is the last text address before
528 * the first kext text section start.
529 */
530 if (kc_format == KCFormatStatic) {
531 if ((sec_ks = getsectbynamefromheader(header: mh, seg_name: "__PRELINK_INFO", sect_name: "__kmod_start")) == NULL) {
532 printf("SDT: unable to find prelink info\n");
533 return;
534 }
535
536 /* find the MIN(start_address) of all kexts in this image. */
537 const uint64_t *start_addr = (const uint64_t *)sec_ks->addr;
538 for (int i = 0; i < sec_ks->size / sizeof(uint64_t); i++) {
539 if (kern_end > start_addr[i]) {
540 kern_end = start_addr[i];
541 }
542 }
543 }
544}
545
546/*
547 * Finds TEXT range that belongs to given module.
548 */
549static int
550sdt_find_module_text_range(struct modctl *ctl, vm_address_t *start, vm_address_t *end)
551{
552 kc_format_t kc_format;
553
554 if (!PE_get_primary_kc_format(type: &kc_format)) {
555 kc_format = KCFormatUnknown;
556 }
557
558 /* Adjust kernel region for static kernel cache. */
559 *start = ctl->mod_address;
560
561 if (MOD_IS_MACH_KERNEL(ctl) && kc_format == KCFormatStatic) {
562 *end = kern_end;
563 } else {
564 *end = ctl->mod_address + ctl->mod_size;
565 }
566
567 return 1;
568}
569
570/*
571 * Processes SDT section in given Mach-O header
572 */
573void
574sdt_load_machsect(struct modctl *ctl)
575{
576 kernel_mach_header_t *mh = sdt_get_module_mh(ctl);
577 kernel_section_t *sec_sdt = NULL;
578 char *strings = NULL;
579 kernel_nlist_t *sym = NULL;
580 vm_address_t text_start, text_end;
581 unsigned int len;
582 uint32_t nsyms = 0;
583
584 if (mh == NULL) {
585 return;
586 }
587
588 /* Ignore SDT definitions if we don't know where they belong. */
589 if (!sdt_find_module_text_range(ctl, start: &text_start, end: &text_end)) {
590 printf("SDT: Unable to determine text range for %s\n", ctl->mod_modname);
591 return;
592 }
593
594 /* Do not load SDTs when asked to use kernel symbols but symbol table is not available. */
595 if (MOD_HAS_KERNEL_SYMBOLS(ctl) && (nsyms = sdt_find_symbol_table(ctl, sym: &sym, strings: &strings)) == 0) {
596 printf("SDT: No kernel symbols for %s\n", ctl->mod_modname);
597 return;
598 }
599
600 /* Locate DTrace SDT section in the object. */
601 if ((sec_sdt = getsectbynamefromheader(header: mh, seg_name: "__DATA_CONST", sect_name: "__sdt")) == NULL) {
602 return;
603 }
604
605 /*
606 * Iterate over SDT section and establish all SDT probe descriptions.
607 */
608 dtrace_sdt_def_t *sdtdef = (dtrace_sdt_def_t *)(sec_sdt->addr);
609 for (size_t k = 0; k < sec_sdt->size / sizeof(dtrace_sdt_def_t); k++, sdtdef++) {
610 unsigned long best = 0;
611
612 /*
613 * Static KEXTs share __sdt section with kernel after linking. It is required
614 * to filter out description and pick only those that belong to requested
615 * module or kernel itself.
616 */
617 if (MOD_IS_STATIC_KEXT(ctl) || MOD_IS_MACH_KERNEL(ctl)) {
618 if ((sdtdef->dsd_addr < text_start) || (sdtdef->dsd_addr > text_end)) {
619 continue;
620 }
621 } else {
622 /* Skip over probe descripton that do not belong to current module. */
623 if (!dtrace_addr_in_module((void *)sdtdef->dsd_addr, ctl)) {
624 continue;
625 }
626 }
627
628 sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP);
629
630 /* Unescape probe name and keep a note of the size of original memory allocation. */
631 sdpd->sdpd_name = sdt_strdup_name(name: sdtdef->dsd_name);
632 sdpd->sdpd_namelen = strlen(s: sdtdef->dsd_name) + 1;
633
634 /* Used only for provider structure lookup so there is no need to make dynamic copy. */
635 sdpd->sdpd_prov = sdtdef->dsd_prov;
636
637 /*
638 * Find the symbol immediately preceding the sdt probe site just discovered,
639 * that symbol names the function containing the sdt probe.
640 */
641 sdpd->sdpd_func = NULL;
642
643 if (MOD_HAS_KERNEL_SYMBOLS(ctl)) {
644 const char *funcname = SDT_UNKNOWN_FUNCNAME;
645
646 for (int i = 0; i < nsyms; i++) {
647 uint8_t jn_type = sym[i].n_type & N_TYPE;
648 char *jname = strings + sym[i].n_un.n_strx;
649
650 if ((N_SECT != jn_type && N_ABS != jn_type)) {
651 continue;
652 }
653
654 if (0 == sym[i].n_un.n_strx) { /* iff a null, "", name. */
655 continue;
656 }
657
658 if (*jname == '_') {
659 jname += 1;
660 }
661
662 if (sdtdef->dsd_addr <= (unsigned long)sym[i].n_value) {
663 continue;
664 }
665
666 if ((unsigned long)sym[i].n_value > best) {
667 best = (unsigned long)sym[i].n_value;
668 funcname = jname;
669 }
670 }
671
672 len = strlen(s: funcname) + 1;
673 sdpd->sdpd_func = kmem_alloc(len, KM_SLEEP);
674 (void) strlcpy(dst: sdpd->sdpd_func, src: funcname, n: len);
675 }
676
677#if defined(__arm64__)
678 sdpd->sdpd_offset = sdtdef->dsd_addr & ~0x1LU;
679#else
680 sdpd->sdpd_offset = sdtdef->dsd_addr;
681#endif /* __arm64__ */
682
683 sdpd->sdpd_next = (sdt_probedesc_t *)ctl->mod_sdtdesc;
684 ctl->mod_sdtdesc = sdpd;
685 }
686}
687
688void
689sdt_init( void )
690{
691 int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw);
692
693 if (majdevno < 0) {
694 printf("sdt_init: failed to allocate a major number!\n");
695 return;
696 }
697
698 if (dtrace_sdt_probes_restricted()) {
699 return;
700 }
701
702 sdt_attach(devi: (dev_info_t*)(uintptr_t)majdevno);
703}
704
705#undef SDT_MAJOR
706
707/*
708 * Provide SDT modules with userspace symbols.
709 *
710 * A module contains only partially filled in SDT probe descriptions because symbols were
711 * not available at the time when __sdt section was loaded. Fixup descriptons before providing
712 * the probes.
713 */
714static void
715sdt_provide_module_user_syms(void *arg, struct modctl *ctl)
716{
717 sdt_probedesc_t *sdpd;
718 dtrace_module_symbols_t *mod_sym = ctl->mod_user_symbols;
719
720 if (mod_sym == NULL) {
721 printf("DTrace missing userspace symbols for module %s\n", ctl->mod_modname);
722 return;
723 }
724
725 /* Fixup missing probe description parts. */
726 for (sdpd = ctl->mod_sdtdesc; sdpd != NULL; sdpd = sdpd->sdpd_next) {
727 ASSERT(sdpd->sdpd_func == NULL);
728 const char *funcname = SDT_UNKNOWN_FUNCNAME;
729
730 /* Look for symbol that contains SDT probe offset. */
731 for (int i = 0; i < mod_sym->dtmodsyms_count; i++) {
732 dtrace_symbol_t *symbol = &mod_sym->dtmodsyms_symbols[i];
733 char *name = symbol->dtsym_name;
734
735 /*
736 * Every function symbol gets extra '_' prepended in the Mach-O symbol table.
737 * Strip it away to make a probe's function name match source code.
738 */
739 if (*name == '_') {
740 name += 1;
741 }
742
743 if (!symbol->dtsym_addr) {
744 continue;
745 }
746
747 /* Ignore symbols that do not belong to this module. */
748 if (!dtrace_addr_in_module((void *)symbol->dtsym_addr, ctl)) {
749 continue;
750 }
751
752 /* Pick symbol name when we found match. */
753 if ((symbol->dtsym_addr <= sdpd->sdpd_offset) &&
754 (sdpd->sdpd_offset < symbol->dtsym_addr + symbol->dtsym_size)) {
755 funcname = name;
756 break;
757 }
758 }
759
760 size_t len = strlen(s: funcname) + 1;
761 sdpd->sdpd_func = kmem_alloc(len, KM_SLEEP);
762 (void) strlcpy(dst: sdpd->sdpd_func, src: funcname, n: len);
763 }
764
765 /* Probe descriptionds are now fixed up. Provide them as usual. */
766 __sdt_provide_module(arg, ctl);
767}
768
769/*ARGSUSED*/
770void
771sdt_provide_module(void *arg, struct modctl *ctl)
772{
773 ASSERT(ctl != NULL);
774 ASSERT(dtrace_kernel_symbol_mode != DTRACE_KERNEL_SYMBOLS_NEVER);
775 LCK_MTX_ASSERT(&mod_lock, LCK_MTX_ASSERT_OWNED);
776
777 if (MOD_SDT_DONE(ctl)) {
778 return;
779 }
780
781 if (MOD_HAS_KERNEL_SYMBOLS(ctl)) {
782 __sdt_provide_module(arg, ctl);
783 ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED;
784 return;
785 }
786
787 if (MOD_HAS_USERSPACE_SYMBOLS(ctl)) {
788 sdt_provide_module_user_syms(arg, ctl);
789 ctl->mod_flags |= MODCTL_SDT_PROBES_PROVIDED;
790 return;
791 }
792
793 /*
794 * The SDT provider's module is not detachable so we don't have to re-provide SDT
795 * probes if that happens. After succesfull providing, the probe descriptions are
796 * no longer required. If module gets re-loaded it will get a new set of probe
797 * descriptions from its __sdt section.
798 */
799 if (MOD_SDT_PROBES_PROVIDED(ctl)) {
800 sdt_probedesc_t *sdpd = ctl->mod_sdtdesc;
801 while (sdpd) {
802 sdt_probedesc_t *this_sdpd = sdpd;
803 kmem_free((void *)sdpd->sdpd_name, sdpd->sdpd_namelen);
804 if (sdpd->sdpd_func) {
805 kmem_free((void *)sdpd->sdpd_func, strlen(s: sdpd->sdpd_func) + 1);
806 }
807 sdpd = sdpd->sdpd_next;
808 kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t));
809 }
810 ctl->mod_sdtdesc = NULL;
811 }
812}
813