1/*
2 * Copyright (c) 2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#if CONFIG_PGTRACE
30#include <mach/mach_types.h>
31#include <IOKit/IOLib.h>
32#include <sys/msgbuf.h>
33#include <sys/errno.h>
34#include <arm64/pgtrace.h>
35#include <libkern/OSDebug.h>
36
37typedef struct {
38 queue_chain_t chain;
39
40 pmap_t pmap;
41 vm_offset_t start;
42 vm_offset_t end;
43} probe_t;
44
45#if CONFIG_PGTRACE_NONKEXT
46#include "pgtrace_decoder.h"
47
48//--------------------------------------------
49// Macros
50//
51#define RBUF_DEFAULT_SIZE 1024
52#define RBUF_IDX(idx, mask) ((idx) & (mask))
53#define MSG_MAX 130
54
55//--------------------------------------------
56// Types
57//
58typedef uint8_t RWLOCK;
59
60typedef struct {
61 uint64_t id;
62 pgtrace_run_result_t res;
63 void *stack[PGTRACE_STACK_DEPTH];
64} log_t;
65
66//--------------------------------------------
67// Statics
68//
69static struct {
70 log_t *logs; // Protect
71 uint32_t size; // Protect
72 uint64_t rdidx, wridx; // Protect
73 decl_simple_lock_data(, loglock);
74
75 uint64_t id;
76 uint32_t option;
77 uint32_t enabled;
78 uint32_t bytes;
79
80 queue_head_t probes; // Protect
81
82 lck_grp_t *lock_grp;
83 lck_grp_attr_t *lock_grp_attr;
84 lck_attr_t *lock_attr;
85 lck_mtx_t probelock;
86} pgtrace = {};
87
88//--------------------------------------------
89// Globals
90//
91void pgtrace_init(void)
92{
93 simple_lock_init(&pgtrace.loglock, 0);
94
95 pgtrace.lock_attr = lck_attr_alloc_init();
96 pgtrace.lock_grp_attr = lck_grp_attr_alloc_init();
97 pgtrace.lock_grp = lck_grp_alloc_init("pgtrace_lock", pgtrace.lock_grp_attr);
98
99 lck_mtx_init(&pgtrace.probelock, pgtrace.lock_grp, pgtrace.lock_attr);
100
101 queue_init(&pgtrace.probes);
102
103 pgtrace.size = RBUF_DEFAULT_SIZE;
104 pgtrace.logs = kalloc(RBUF_DEFAULT_SIZE * sizeof(log_t));
105}
106
107void pgtrace_clear_probe(void)
108{
109 probe_t *p, *next;
110 queue_head_t *q = &pgtrace.probes;
111
112 lck_mtx_lock(&pgtrace.probelock);
113
114 p = (probe_t *)queue_first(q);
115 while (!queue_end(q, (queue_entry_t)p)) {
116 next = (probe_t *)queue_next(&(p->chain));
117
118 queue_remove(q, p, probe_t *, chain);
119 kfree(p, sizeof(probe_t));
120
121 p = next;
122 }
123
124 lck_mtx_unlock(&pgtrace.probelock);
125
126 return;
127}
128
129int pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end)
130{
131 probe_t *p;
132 queue_head_t *q = &pgtrace.probes;
133
134 if (start > end) {
135 kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end);
136 return -1;
137 }
138
139 p = kalloc(sizeof(probe_t));
140 p->start = start;
141 p->end = end;
142 if (thread == NULL) {
143 p->pmap = NULL;
144 } else {
145 p->pmap = vm_map_pmap(thread->map);
146 }
147
148 lck_mtx_lock(&pgtrace.probelock);
149 queue_enter(q, p, probe_t *, chain);
150 lck_mtx_unlock(&pgtrace.probelock);
151
152 return 0;
153}
154
155void pgtrace_start(void)
156{
157 probe_t *p;
158 queue_head_t *q = &pgtrace.probes;
159
160 kprintf("%s\n", __func__);
161
162 if (pgtrace.enabled) {
163 return;
164 }
165
166 pgtrace.enabled = 1;
167
168 lck_mtx_lock(&pgtrace.probelock);
169
170 queue_iterate(q, p, probe_t *, chain) {
171 pmap_pgtrace_add_page(p->pmap, p->start, p->end);
172 }
173
174 lck_mtx_unlock(&pgtrace.probelock);
175
176 return;
177}
178
179void pgtrace_stop(void)
180{
181 probe_t *p;
182 queue_head_t *q = &pgtrace.probes;
183
184 kprintf("%s\n", __func__);
185
186 lck_mtx_lock(&pgtrace.probelock);
187
188 queue_iterate(q, p, probe_t *, chain) {
189 pmap_pgtrace_delete_page(p->pmap, p->start, p->end);
190 }
191
192 lck_mtx_unlock(&pgtrace.probelock);
193
194 pgtrace.enabled = 0;
195}
196
197uint32_t pgtrace_get_size(void)
198{
199 return pgtrace.size;
200}
201
202bool pgtrace_set_size(uint32_t size)
203{
204 log_t *old_buf, *new_buf;
205 uint32_t old_size, new_size = 1;
206
207 // round up to next power of 2
208 while (size > new_size) {
209 new_size <<= 1;
210 if (new_size > 0x100000) {
211 // over million entries
212 kprintf("%s: size=%x new_size=%x is too big\n", __func__, size, new_size);
213 return false;
214 }
215 }
216
217 new_buf = kalloc(new_size * sizeof(log_t));
218 if (new_buf == NULL) {
219 kprintf("%s: can't allocate new_size=%x\n entries", __func__, new_size);
220 return false;
221 }
222
223 pgtrace_stop();
224
225 simple_lock(&pgtrace.loglock);
226 old_buf = pgtrace.logs;
227 old_size = pgtrace.size;
228 pgtrace.logs = new_buf;
229 pgtrace.size = new_size;
230 pgtrace.rdidx = pgtrace.wridx = 0;
231 simple_unlock(&pgtrace.loglock);
232
233 if (old_buf) {
234 kfree(old_buf, old_size * sizeof(log_t));
235 }
236
237 return true;
238}
239
240void pgtrace_clear_trace(void)
241{
242 simple_lock(&pgtrace.loglock);
243 pgtrace.rdidx = pgtrace.wridx = 0;
244 simple_unlock(&pgtrace.loglock);
245}
246
247boolean_t pgtrace_active(void)
248{
249 return (pgtrace.enabled > 0);
250}
251
252uint32_t pgtrace_get_option(void)
253{
254 return pgtrace.option;
255}
256
257void pgtrace_set_option(uint32_t option)
258{
259 pgtrace.option = option;
260}
261
262// pgtrace_write_log() is in interrupt disabled context
263void pgtrace_write_log(pgtrace_run_result_t res)
264{
265 uint8_t i;
266 log_t log = {};
267 const char *rwmap[] = { "R", "W", "PREFETCH" };
268
269 log.id = pgtrace.id++;
270 log.res = res;
271
272 if (pgtrace.option & PGTRACE_OPTION_KPRINTF) {
273 char msg[MSG_MAX];
274 char *p;
275
276 p = msg;
277
278 snprintf(p, MSG_MAX, "%llu %s ", res.rr_time, rwmap[res.rr_rw]);
279 p += strlen(p);
280
281 for (i = 0; i < res.rr_num; i++) {
282 snprintf(p, MSG_MAX-(p-msg), "%lx=%llx ", res.rr_addrdata[i].ad_addr, res.rr_addrdata[i].ad_data);
283 p += strlen(p);
284 }
285
286 kprintf("%s %s\n", __func__, msg);
287 }
288
289 if (pgtrace.option & PGTRACE_OPTION_STACK) {
290 OSBacktrace(log.stack, PGTRACE_STACK_DEPTH);
291 }
292
293 pgtrace.bytes += sizeof(log);
294
295 simple_lock(&pgtrace.loglock);
296
297 pgtrace.logs[RBUF_IDX(pgtrace.wridx, pgtrace.size-1)] = log;
298
299 // Advance rdidx if ring is full
300 if (RBUF_IDX(pgtrace.wridx, pgtrace.size-1) == RBUF_IDX(pgtrace.rdidx, pgtrace.size-1) &&
301 (pgtrace.wridx != pgtrace.rdidx)) {
302 pgtrace.rdidx++;
303 }
304 pgtrace.wridx++;
305
306 // Signal if ring was empty
307 if (pgtrace.wridx == (pgtrace.rdidx + 1)) {
308 thread_wakeup(pgtrace.logs);
309 }
310
311 simple_unlock(&pgtrace.loglock);
312
313 return;
314}
315
316// pgtrace_read_log() is in user thread
317int64_t pgtrace_read_log(uint8_t *buf, uint32_t size)
318{
319 int total, front, back;
320 boolean_t ints;
321 wait_result_t wr;
322
323 if (pgtrace.enabled == FALSE) {
324 return -EINVAL;
325 }
326
327 total = size / sizeof(log_t);
328
329 // Check if buf is too small
330 if (buf && total == 0) {
331 return -EINVAL;
332 }
333
334 ints = ml_set_interrupts_enabled(FALSE);
335 simple_lock(&pgtrace.loglock);
336
337 // Wait if ring is empty
338 if (pgtrace.rdidx == pgtrace.wridx) {
339 assert_wait(pgtrace.logs, THREAD_ABORTSAFE);
340
341 simple_unlock(&pgtrace.loglock);
342 ml_set_interrupts_enabled(ints);
343
344 wr = thread_block(NULL);
345 if (wr != THREAD_AWAKENED) {
346 return -EINTR;
347 }
348
349 ints = ml_set_interrupts_enabled(FALSE);
350 simple_lock(&pgtrace.loglock);
351 }
352
353 // Trim the size
354 if ((pgtrace.rdidx + total) > pgtrace.wridx) {
355 total = (int)(pgtrace.wridx - pgtrace.rdidx);
356 }
357
358 // Copy front
359 if ((RBUF_IDX(pgtrace.rdidx, pgtrace.size-1) + total) >= pgtrace.size) {
360 front = pgtrace.size - RBUF_IDX(pgtrace.rdidx, pgtrace.size-1);
361 } else {
362 front = total;
363 }
364
365 memcpy(buf, &(pgtrace.logs[RBUF_IDX(pgtrace.rdidx, pgtrace.size-1)]), front*sizeof(log_t));
366
367 // Copy back if any
368 back = total-front;
369 if (back) {
370 buf += front * sizeof(log_t);
371 memcpy(buf, pgtrace.logs, back*sizeof(log_t));
372 }
373
374 pgtrace.rdidx += total;
375
376 simple_unlock(&pgtrace.loglock);
377 ml_set_interrupts_enabled(ints);
378
379 return total*sizeof(log_t);
380}
381
382int pgtrace_get_stats(pgtrace_stats_t *stats)
383{
384 if (!stats) {
385 return -1;
386 }
387
388 stats->stat_logger.sl_bytes = pgtrace.bytes;
389 pgtrace_decoder_get_stats(stats);
390
391 return 0;
392}
393
394#else // CONFIG_PGTRACE_NONKEXT
395
396static struct {
397 bool active;
398 decoder_t *decoder;
399 logger_t *logger;
400 queue_head_t probes;
401
402 lck_grp_t *lock_grp;
403 lck_grp_attr_t *lock_grp_attr;
404 lck_attr_t *lock_attr;
405 lck_mtx_t probelock;
406} pgtrace = {};
407
408//------------------------------------
409// functions for pmap fault handler
410// - pgtrace_decode_and_run
411// - pgtrace_write_log
412//------------------------------------
413int pgtrace_decode_and_run(uint32_t inst, vm_offset_t fva, vm_map_offset_t *cva_page, arm_saved_state_t *ss, pgtrace_run_result_t *res)
414{
415 vm_offset_t pa, cva;
416 pgtrace_instruction_info_t info;
417 vm_offset_t cva_front_page = cva_page[0];
418 vm_offset_t cva_cur_page = cva_page[1];
419
420 pgtrace.decoder->decode(inst, ss, &info);
421
422 if (info.addr == fva) {
423 cva = cva_cur_page + (fva & ARM_PGMASK);
424 } else {
425 // which means a front page is not a tracing page
426 cva = cva_front_page + (fva & ARM_PGMASK);
427 }
428
429 pa = mmu_kvtop(cva);
430 if (!pa) {
431 panic("%s: invalid address cva=%lx fva=%lx info.addr=%lx inst=%x", __func__, cva, fva, info.addr, inst);
432 }
433
434 absolutetime_to_nanoseconds(mach_absolute_time(), &res->rr_time);
435
436 pgtrace.decoder->run(inst, pa, cva, ss, res);
437
438 return 0;
439}
440
441int pgtrace_write_log(pgtrace_run_result_t res)
442{
443 pgtrace.logger->write(res);
444 return 0;
445}
446
447//------------------------------------
448// functions for kext
449// - pgtrace_init
450// - pgtrace_add_probe
451// - pgtrace_clear_probe
452// - pgtrace_start
453// - pgtrace_stop
454// - pgtrace_active
455//------------------------------------
456int pgtrace_init(decoder_t *decoder, logger_t *logger)
457{
458 kprintf("%s decoder=%p logger=%p\n", __func__, decoder, logger);
459
460 assert(decoder && logger);
461
462 if (decoder->magic != 0xfeedface || logger->magic != 0xfeedface ||
463 strcmp(decoder->arch, "arm64") != 0 || strcmp(logger->arch, "arm64") != 0) {
464 kprintf("%s:wrong decoder/logger magic=%llx/%llx arch=%s/%s", __func__, decoder->magic, logger->magic, decoder->arch, logger->arch);
465 return EINVAL;
466 }
467
468 pgtrace.lock_attr = lck_attr_alloc_init();
469 pgtrace.lock_grp_attr = lck_grp_attr_alloc_init();
470 pgtrace.lock_grp = lck_grp_alloc_init("pgtrace_lock", pgtrace.lock_grp_attr);
471
472 lck_mtx_init(&pgtrace.probelock, pgtrace.lock_grp, pgtrace.lock_attr);
473
474 queue_init(&pgtrace.probes);
475 pgtrace.decoder = decoder;
476 pgtrace.logger = logger;
477
478 return 0;
479}
480
481int pgtrace_add_probe(thread_t thread, vm_offset_t start, vm_offset_t end)
482{
483 probe_t *p;
484 queue_head_t *q = &pgtrace.probes;
485
486 kprintf("%s start=%lx end=%lx\n", __func__, start, end);
487
488 if (start > end) {
489 kprintf("%s Invalid start=%lx end=%lx\n", __func__, start, end);
490 return -1;
491 }
492
493 p = kalloc(sizeof(probe_t));
494 p->start = start;
495 p->end = end;
496 if (thread == NULL) {
497 p->pmap = NULL;
498 } else {
499 p->pmap = vm_map_pmap(thread->map);
500 }
501
502 lck_mtx_lock(&pgtrace.probelock);
503 queue_enter(q, p, probe_t *, chain);
504 lck_mtx_unlock(&pgtrace.probelock);
505
506 return 0;
507}
508
509void pgtrace_clear_probe(void)
510{
511 probe_t *p, *next;
512 queue_head_t *q = &pgtrace.probes;
513
514 kprintf("%s\n", __func__);
515
516 lck_mtx_lock(&pgtrace.probelock);
517
518 p = (probe_t *)queue_first(q);
519 while (!queue_end(q, (queue_entry_t)p)) {
520 next = (probe_t *)queue_next(&(p->chain));
521
522 queue_remove(q, p, probe_t *, chain);
523 kfree(p, sizeof(probe_t));
524
525 p = next;
526 }
527
528 lck_mtx_unlock(&pgtrace.probelock);
529
530 return;
531}
532
533void pgtrace_start(void)
534{
535 probe_t *p;
536 queue_head_t *q = &pgtrace.probes;
537
538 kprintf("%s\n", __func__);
539
540 if (pgtrace.active == true) {
541 return;
542 }
543
544 pgtrace.active = true;
545
546 lck_mtx_lock(&pgtrace.probelock);
547
548 queue_iterate(q, p, probe_t *, chain) {
549 pmap_pgtrace_add_page(p->pmap, p->start, p->end);
550 }
551
552 lck_mtx_unlock(&pgtrace.probelock);
553
554 return;
555}
556
557void pgtrace_stop(void)
558{
559 probe_t *p;
560 queue_head_t *q = &pgtrace.probes;
561
562 kprintf("%s\n", __func__);
563
564 lck_mtx_lock(&pgtrace.probelock);
565
566 queue_iterate(q, p, probe_t *, chain) {
567 pmap_pgtrace_delete_page(p->pmap, p->start, p->end);
568 }
569
570 lck_mtx_unlock(&pgtrace.probelock);
571
572 pgtrace.active = false;
573}
574
575bool pgtrace_active(void)
576{
577 return pgtrace.active;
578}
579#endif // CONFIG_PGTRACE_NONKEXT
580#else
581// empty funcs for release kernel
582extern void pgtrace_stop(void);
583extern void pgtrace_start(void);
584extern void pgtrace_clear_probe(void);
585extern void pgtrace_add_probe(void);
586extern void pgtrace_init(void);
587extern void pgtrace_active(void);
588void pgtrace_stop(void) {}
589void pgtrace_start(void) {}
590void pgtrace_clear_probe(void) {}
591void pgtrace_add_probe(void) {}
592void pgtrace_init(void) {}
593void pgtrace_active(void) {}
594#endif
595