1 | /* |
2 | * Copyright (c) 2007-2018 Apple Inc. All rights reserved. |
3 | */ |
4 | /* |
5 | * CDDL HEADER START |
6 | * |
7 | * The contents of this file are subject to the terms of the |
8 | * Common Development and Distribution License, Version 1.0 only |
9 | * (the "License"). You may not use this file except in compliance |
10 | * with the License. |
11 | * |
12 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
13 | * or http://www.opensolaris.org/os/licensing. |
14 | * See the License for the specific language governing permissions |
15 | * and limitations under the License. |
16 | * |
17 | * When distributing Covered Code, include this CDDL HEADER in each |
18 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
19 | * If applicable, add the following below this CDDL HEADER, with the |
20 | * fields enclosed by brackets "[]" replaced with your own identifying |
21 | * information: Portions Copyright [yyyy] [name of copyright owner] |
22 | * |
23 | * CDDL HEADER END |
24 | */ |
25 | /* |
26 | * Copyright 2005 Sun Microsystems, Inc. All rights reserved. |
27 | * Use is subject to license terms. |
28 | */ |
29 | |
30 | #include <kern/thread.h> |
31 | #include <mach/thread_status.h> |
32 | #include <arm64/proc_reg.h> |
33 | #include <arm/caches_internal.h> |
34 | |
35 | #include <mach-o/loader.h> |
36 | #include <mach-o/nlist.h> |
37 | #include <libkern/kernel_mach_header.h> |
38 | |
39 | #include <sys/param.h> |
40 | #include <sys/systm.h> |
41 | #include <sys/errno.h> |
42 | #include <sys/stat.h> |
43 | #include <sys/ioctl.h> |
44 | #include <sys/conf.h> |
45 | #include <sys/fcntl.h> |
46 | #include <miscfs/devfs/devfs.h> |
47 | |
48 | #include <sys/dtrace.h> |
49 | #include <sys/dtrace_impl.h> |
50 | #include <sys/fbt.h> |
51 | |
52 | #include <sys/dtrace_glue.h> |
53 | |
54 | #if __has_include(<ptrauth.h>) |
55 | #include <ptrauth.h> |
56 | #endif |
57 | |
58 | #define DTRACE_INVOP_PUSH_FRAME 11 |
59 | |
60 | #define DTRACE_INVOP_NOP_SKIP 4 |
61 | #define DTRACE_INVOP_ADD_FP_SP_SKIP 4 |
62 | |
63 | #define DTRACE_INVOP_POP_PC_SKIP 2 |
64 | |
65 | /* |
66 | * stp fp, lr, [sp, #val] |
67 | * stp fp, lr, [sp, #val]! |
68 | */ |
69 | #define FBT_IS_ARM64_FRAME_PUSH(x) \ |
70 | (((x) & 0xffc07fff) == 0xa9007bfd || ((x) & 0xffc07fff) == 0xa9807bfd) |
71 | |
72 | /* |
73 | * stp Xt1, Xt2, [sp, #val] |
74 | * stp Xt1, Xt2, [sp, #val]! |
75 | */ |
76 | #define FBT_IS_ARM64_PUSH(x) \ |
77 | (((x) & 0xffc003e0) == 0xa90003e0 || ((x) & 0xffc003e0) == 0xa98003e0) |
78 | |
79 | /* |
80 | * ldp fp, lr, [sp, #val] |
81 | * ldp fp, lr, [sp], #val |
82 | */ |
83 | #define FBT_IS_ARM64_FRAME_POP(x) \ |
84 | (((x) & 0xffc07fff) == 0xa9407bfd || ((x) & 0xffc07fff) == 0xa8c07bfd) |
85 | |
86 | #define FBT_IS_ARM64_ADD_FP_SP(x) (((x) & 0xffc003ff) == 0x910003fd) /* add fp, sp, #val (add fp, sp, #0 == mov fp, sp) */ |
87 | #define FBT_IS_ARM64_RET(x) (((x) == 0xd65f03c0) || ((x) == 0xd65f0fff)) /* ret, retab */ |
88 | |
89 | |
90 | #define FBT_B_MASK 0xff000000 |
91 | #define FBT_B_IMM_MASK 0x00ffffff |
92 | #define FBT_B_INSTR 0x14000000 |
93 | |
94 | #define FBT_IS_ARM64_B_INSTR(x) ((x & FBT_B_MASK) == FBT_B_INSTR) |
95 | #define FBT_GET_ARM64_B_IMM(x) ((x & FBT_B_IMM_MASK) << 2) |
96 | |
97 | #define FBT_PATCHVAL 0xe7eeee7e |
98 | #define FBT_AFRAMES_ENTRY 7 |
99 | #define FBT_AFRAMES_RETURN 7 |
100 | |
101 | #define FBT_ENTRY "entry" |
102 | #define FBT_RETURN "return" |
103 | #define FBT_ADDR2NDX(addr) ((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask) |
104 | |
105 | extern dtrace_provider_id_t fbt_id; |
106 | extern fbt_probe_t **fbt_probetab; |
107 | extern int fbt_probetab_mask; |
108 | |
109 | kern_return_t fbt_perfCallback(int, struct arm_saved_state *, __unused int, __unused int); |
110 | |
111 | int |
112 | fbt_invop(uintptr_t addr, uintptr_t * stack, uintptr_t rval) |
113 | { |
114 | fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; |
115 | |
116 | for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { |
117 | if ((uintptr_t) fbt->fbtp_patchpoint == addr) { |
118 | if (0 == CPU->cpu_dtrace_invop_underway) { |
119 | CPU->cpu_dtrace_invop_underway = 1; /* Race not possible on |
120 | * this per-cpu state */ |
121 | |
122 | /* |
123 | * Stack looks like this: |
124 | * |
125 | * [Higher addresses] |
126 | * |
127 | * Frame of caller |
128 | * Extra args for callee |
129 | * ------------------------ |
130 | * fbt entry probe: |
131 | * Frame from traced function: <previous sp (e.g. 0x1000), return address> |
132 | * fbt return probe: |
133 | * Missing as the return probe has already popped the frame in the callee and |
134 | * traps with LR set to the return address in caller. |
135 | * ------------------------ |
136 | * arm_context_t |
137 | * ------------------------ |
138 | * Frame from trap handler: <previous sp (e.g. 0x1000) , traced PC > |
139 | * The traced function has either never pushed the frame |
140 | * or already popped it. So there is no frame in the |
141 | * backtrace pointing to the frame on the stack containing |
142 | * the LR in the caller. |
143 | * ------------------------ |
144 | * | |
145 | * | |
146 | * | stack grows this way |
147 | * | |
148 | * | |
149 | * v |
150 | * [Lower addresses] |
151 | * |
152 | * cpu_dtrace_caller compensates for fact that the LR is not stored on stack as explained |
153 | * above. When walking the stack, when we reach the frame where we extract a PC in the |
154 | * patched function, we put the cpu_dtrace_caller in the backtrace instead. The next |
155 | * frame we extract will be in the caller's caller, so we output a backtrace starting |
156 | * at the caller and going sequentially up the stack. |
157 | */ |
158 | arm_saved_state_t *regs = (arm_saved_state_t *)(&((arm_context_t *)stack)->ss); |
159 | |
160 | CPU->cpu_dtrace_caller = get_saved_state_lr(regs); |
161 | |
162 | /* When fbt_roffset is non-zero, we know we are handling a return probe point. */ |
163 | if (fbt->fbtp_roffset == 0) { |
164 | dtrace_probe(fbt->fbtp_id, arg0: get_saved_state_reg(regs, reg: 0), arg1: get_saved_state_reg(regs, reg: 1), |
165 | arg2: get_saved_state_reg(regs, reg: 2), arg3: get_saved_state_reg(regs, reg: 3), arg4: get_saved_state_reg(regs, reg: 4)); |
166 | } else { |
167 | dtrace_probe(fbt->fbtp_id, arg0: fbt->fbtp_roffset, arg1: rval, arg2: 0, arg3: 0, arg4: 0); |
168 | } |
169 | |
170 | CPU->cpu_dtrace_caller = 0; |
171 | CPU->cpu_dtrace_invop_underway = 0; |
172 | } |
173 | |
174 | /* |
175 | * On other architectures, we return a DTRACE constant to let the callback function |
176 | * know what was replaced. On the ARM, since the function prologue/epilogue machine code |
177 | * can vary, we need the actual bytes of the instruction, so return the savedval instead. |
178 | */ |
179 | return fbt->fbtp_savedval; |
180 | } |
181 | } |
182 | |
183 | return 0; |
184 | } |
185 | |
186 | #define IS_USER_TRAP(regs) (PSR64_IS_USER(get_saved_state_cpsr(regs))) |
187 | #define T_INVALID_OPCODE EXC_BAD_INSTRUCTION |
188 | #define FBT_EXCEPTION_CODE T_INVALID_OPCODE |
189 | |
190 | kern_return_t |
191 | fbt_perfCallback( |
192 | int trapno, |
193 | struct arm_saved_state * regs, |
194 | __unused int unused1, |
195 | __unused int unused2) |
196 | { |
197 | kern_return_t retval = KERN_FAILURE; |
198 | |
199 | if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(regs)) { |
200 | boolean_t oldlevel = 0; |
201 | machine_inst_t emul = 0; |
202 | uint64_t sp, lr; |
203 | uint32_t imm; |
204 | |
205 | oldlevel = ml_set_interrupts_enabled(FALSE); |
206 | |
207 | __asm__ volatile ( |
208 | "Ldtrace_invop_callsite_pre_label:\n" |
209 | ".data\n" |
210 | ".private_extern _dtrace_invop_callsite_pre\n" |
211 | "_dtrace_invop_callsite_pre:\n" |
212 | " .quad Ldtrace_invop_callsite_pre_label\n" |
213 | ".text\n" |
214 | ); |
215 | |
216 | emul = dtrace_invop(get_saved_state_pc(regs), (uintptr_t*) regs, get_saved_state_reg(regs, reg: 0)); |
217 | |
218 | __asm__ volatile ( |
219 | "Ldtrace_invop_callsite_post_label:\n" |
220 | ".data\n" |
221 | ".private_extern _dtrace_invop_callsite_post\n" |
222 | "_dtrace_invop_callsite_post:\n" |
223 | " .quad Ldtrace_invop_callsite_post_label\n" |
224 | ".text\n" |
225 | ); |
226 | |
227 | if (emul == DTRACE_INVOP_NOP) { |
228 | /* |
229 | * Skip over the patched NOP planted by sdt |
230 | */ |
231 | add_saved_state_pc(regs, DTRACE_INVOP_NOP_SKIP); |
232 | retval = KERN_SUCCESS; |
233 | } else if (FBT_IS_ARM64_ADD_FP_SP(emul)) { |
234 | /* retrieve the value to add */ |
235 | uint64_t val = (emul >> 10) & 0xfff; |
236 | assert(val < 4096); |
237 | |
238 | /* retrieve sp */ |
239 | sp = get_saved_state_sp(regs); |
240 | |
241 | /* |
242 | * emulate the instruction: |
243 | * add fp, sp, #val |
244 | */ |
245 | assert(sp < (UINT64_MAX - val)); |
246 | set_saved_state_fp(regs, fp: sp + val); |
247 | |
248 | /* skip over the bytes of the patched instruction */ |
249 | add_saved_state_pc(regs, DTRACE_INVOP_ADD_FP_SP_SKIP); |
250 | |
251 | retval = KERN_SUCCESS; |
252 | } else if (FBT_IS_ARM64_RET(emul)) { |
253 | lr = get_saved_state_lr(regs); |
254 | #if __has_feature(ptrauth_calls) |
255 | lr = (user_addr_t) ptrauth_strip((void *)lr, ptrauth_key_return_address); |
256 | #endif |
257 | set_saved_state_pc(regs, pc: lr); |
258 | retval = KERN_SUCCESS; |
259 | } else if (FBT_IS_ARM64_B_INSTR(emul)) { |
260 | imm = FBT_GET_ARM64_B_IMM(emul); |
261 | add_saved_state_pc(regs, diff: imm); |
262 | retval = KERN_SUCCESS; |
263 | } else if (emul == FBT_PATCHVAL) { |
264 | /* Means we encountered an error but handled it, try same inst again */ |
265 | retval = KERN_SUCCESS; |
266 | } else { |
267 | retval = KERN_FAILURE; |
268 | } |
269 | |
270 | ml_set_interrupts_enabled(enable: oldlevel); |
271 | } |
272 | |
273 | return retval; |
274 | } |
275 | |
276 | void |
277 | fbt_provide_probe(struct modctl *ctl, const char *modname, const char* symbolName, machine_inst_t* symbolStart, machine_inst_t *instrHigh) |
278 | { |
279 | int doenable = 0; |
280 | dtrace_id_t thisid; |
281 | |
282 | fbt_probe_t *newfbt, *retfbt, *entryfbt; |
283 | machine_inst_t *instr, *pushinstr = NULL, *limit, theInstr; |
284 | int foundPushLR, savedRegs; |
285 | |
286 | /* |
287 | * Guard against null and invalid symbols |
288 | */ |
289 | if (!symbolStart || !instrHigh || instrHigh < symbolStart) { |
290 | kprintf(fmt: "dtrace: %s has an invalid address\n" , symbolName); |
291 | return; |
292 | } |
293 | |
294 | /* |
295 | * Assume the compiler doesn't schedule instructions in the prologue. |
296 | */ |
297 | foundPushLR = 0; |
298 | savedRegs = -1; |
299 | limit = (machine_inst_t *)instrHigh; |
300 | |
301 | assert(sizeof(*instr) == 4); |
302 | |
303 | for (instr = symbolStart, theInstr = 0; instr < instrHigh; instr++) { |
304 | /* |
305 | * Count the number of time we pushed something onto the stack |
306 | * before hitting a frame push. That will give us an estimation |
307 | * of how many stack pops we should expect when looking for the |
308 | * RET instruction. |
309 | */ |
310 | theInstr = *instr; |
311 | if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) { |
312 | foundPushLR = 1; |
313 | pushinstr = instr; |
314 | } |
315 | |
316 | if (foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr))) { |
317 | /* Guard against a random setting of fp from sp, we make sure we found the push first */ |
318 | break; |
319 | } |
320 | if (FBT_IS_ARM64_RET(theInstr)) { /* We've gone too far, bail. */ |
321 | break; |
322 | } |
323 | if (FBT_IS_ARM64_FRAME_POP(theInstr)) { /* We've gone too far, bail. */ |
324 | break; |
325 | } |
326 | } |
327 | |
328 | if (!(foundPushLR && (FBT_IS_ARM64_ADD_FP_SP(theInstr)))) { |
329 | return; |
330 | } |
331 | |
332 | thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_ENTRY); |
333 | newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); |
334 | newfbt->fbtp_next = NULL; |
335 | strlcpy(dst: (char *)&(newfbt->fbtp_name), src: symbolName, MAX_FBTP_NAME_CHARS ); |
336 | |
337 | if (thisid != 0) { |
338 | /* |
339 | * The dtrace_probe previously existed, so we have to hook |
340 | * the newfbt entry onto the end of the existing fbt's |
341 | * chain. |
342 | * If we find an fbt entry that was previously patched to |
343 | * fire, (as indicated by the current patched value), then |
344 | * we want to enable this newfbt on the spot. |
345 | */ |
346 | entryfbt = dtrace_probe_arg(fbt_id, thisid); |
347 | ASSERT(entryfbt != NULL); |
348 | for (; entryfbt != NULL; entryfbt = entryfbt->fbtp_next) { |
349 | if (entryfbt->fbtp_currentval == entryfbt->fbtp_patchval) { |
350 | doenable++; |
351 | } |
352 | |
353 | if (entryfbt->fbtp_next == NULL) { |
354 | entryfbt->fbtp_next = newfbt; |
355 | newfbt->fbtp_id = entryfbt->fbtp_id; |
356 | break; |
357 | } |
358 | } |
359 | } else { |
360 | /* |
361 | * The dtrace_probe did not previously exist, so we |
362 | * create it and hook in the newfbt. Since the probe is |
363 | * new, we obviously do not need to enable it on the spot. |
364 | */ |
365 | newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, symbolName, FBT_ENTRY, FBT_AFRAMES_ENTRY, newfbt); |
366 | doenable = 0; |
367 | } |
368 | |
369 | newfbt->fbtp_patchpoint = instr; |
370 | newfbt->fbtp_ctl = ctl; |
371 | newfbt->fbtp_loadcnt = ctl->mod_loadcnt; |
372 | newfbt->fbtp_rval = DTRACE_INVOP_PUSH_FRAME; |
373 | newfbt->fbtp_savedval = theInstr; |
374 | newfbt->fbtp_patchval = FBT_PATCHVAL; |
375 | newfbt->fbtp_currentval = 0; |
376 | newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; |
377 | fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; |
378 | |
379 | if (doenable) { |
380 | fbt_enable(NULL, id: newfbt->fbtp_id, parg: newfbt); |
381 | } |
382 | |
383 | /* |
384 | * The fbt entry chain is in place, one entry point per symbol. |
385 | * The fbt return chain can have multiple return points per |
386 | * symbol. |
387 | * Here we find the end of the fbt return chain. |
388 | */ |
389 | |
390 | doenable = 0; |
391 | |
392 | thisid = dtrace_probe_lookup(fbt_id, modname, symbolName, FBT_RETURN); |
393 | |
394 | if (thisid != 0) { |
395 | /* The dtrace_probe previously existed, so we have to |
396 | * find the end of the existing fbt chain. If we find |
397 | * an fbt return that was previously patched to fire, |
398 | * (as indicated by the currrent patched value), then |
399 | * we want to enable any new fbts on the spot. |
400 | */ |
401 | retfbt = dtrace_probe_arg(fbt_id, thisid); |
402 | ASSERT(retfbt != NULL); |
403 | for (; retfbt != NULL; retfbt = retfbt->fbtp_next) { |
404 | if (retfbt->fbtp_currentval == retfbt->fbtp_patchval) { |
405 | doenable++; |
406 | } |
407 | if (retfbt->fbtp_next == NULL) { |
408 | break; |
409 | } |
410 | } |
411 | } else { |
412 | doenable = 0; |
413 | retfbt = NULL; |
414 | } |
415 | |
416 | /* |
417 | * Go back to the start of the function, in case |
418 | * the compiler emitted pcrel data loads |
419 | * before FP was adjusted. |
420 | */ |
421 | instr = pushinstr + 1; |
422 | again: |
423 | if (instr >= limit) { |
424 | return; |
425 | } |
426 | |
427 | /* XXX FIXME ... extra jump table detection? */ |
428 | |
429 | /* |
430 | * OK, it's an instruction. |
431 | */ |
432 | theInstr = *instr; |
433 | |
434 | /* Walked onto the start of the next routine? If so, bail out from this function */ |
435 | if (FBT_IS_ARM64_FRAME_PUSH(theInstr)) { |
436 | if (!retfbt) { |
437 | kprintf(fmt: "dtrace: fbt: No return probe for %s, walked to next routine at 0x%016llx\n" , symbolName, (uint64_t)instr); |
438 | } |
439 | return; |
440 | } |
441 | |
442 | /* XXX fancy detection of end of function using PC-relative loads */ |
443 | |
444 | /* |
445 | * Look for: |
446 | * ldp fp, lr, [sp], #val |
447 | * ldp fp, lr, [sp, #val] |
448 | */ |
449 | if (!FBT_IS_ARM64_FRAME_POP(theInstr)) { |
450 | instr++; |
451 | goto again; |
452 | } |
453 | |
454 | /* go to the next instruction */ |
455 | instr++; |
456 | |
457 | /* Scan ahead for a ret or a branch outside the function */ |
458 | for (; instr < limit; instr++) { |
459 | theInstr = *instr; |
460 | if (FBT_IS_ARM64_RET(theInstr)) { |
461 | break; |
462 | } |
463 | if (FBT_IS_ARM64_B_INSTR(theInstr)) { |
464 | machine_inst_t *dest = instr + FBT_GET_ARM64_B_IMM(theInstr); |
465 | /* |
466 | * Check whether the destination of the branch |
467 | * is outside of the function |
468 | */ |
469 | if (dest >= limit || dest < symbolStart) { |
470 | break; |
471 | } |
472 | } |
473 | } |
474 | |
475 | if (!FBT_IS_ARM64_RET(theInstr) && !FBT_IS_ARM64_B_INSTR(theInstr)) { |
476 | return; |
477 | } |
478 | |
479 | newfbt = kmem_zalloc(sizeof(fbt_probe_t), KM_SLEEP); |
480 | newfbt->fbtp_next = NULL; |
481 | strlcpy(dst: (char *)&(newfbt->fbtp_name), src: symbolName, MAX_FBTP_NAME_CHARS ); |
482 | |
483 | if (retfbt == NULL) { |
484 | newfbt->fbtp_id = dtrace_probe_create(fbt_id, modname, |
485 | symbolName, FBT_RETURN, FBT_AFRAMES_RETURN, newfbt); |
486 | } else { |
487 | retfbt->fbtp_next = newfbt; |
488 | newfbt->fbtp_id = retfbt->fbtp_id; |
489 | } |
490 | |
491 | retfbt = newfbt; |
492 | newfbt->fbtp_patchpoint = instr; |
493 | newfbt->fbtp_ctl = ctl; |
494 | newfbt->fbtp_loadcnt = ctl->mod_loadcnt; |
495 | |
496 | ASSERT(FBT_IS_ARM64_RET(theInstr) || FBT_IS_ARM64_B_INSTR(theInstr)); |
497 | newfbt->fbtp_rval = DTRACE_INVOP_RET; |
498 | newfbt->fbtp_roffset = (uintptr_t) ((uint8_t*) instr - (uint8_t *)symbolStart); |
499 | newfbt->fbtp_savedval = theInstr; |
500 | newfbt->fbtp_patchval = FBT_PATCHVAL; |
501 | newfbt->fbtp_currentval = 0; |
502 | newfbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; |
503 | fbt_probetab[FBT_ADDR2NDX(instr)] = newfbt; |
504 | |
505 | if (doenable) { |
506 | fbt_enable(NULL, id: newfbt->fbtp_id, parg: newfbt); |
507 | } |
508 | |
509 | instr++; |
510 | goto again; |
511 | } |
512 | |