1/*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * FILE_ID: thread_status.h
30 */
31
32
33#ifndef _ARM_THREAD_STATUS_H_
34#define _ARM_THREAD_STATUS_H_
35
36#if defined (__arm__) || defined (__arm64__)
37
38#include <mach/machine/_structs.h>
39#include <mach/machine/thread_state.h>
40#include <mach/message.h>
41#include <mach/vm_types.h>
42
43#ifdef XNU_KERNEL_PRIVATE
44#include <os/refcnt.h>
45#endif
46
47/*
48 * Support for determining the state of a thread
49 */
50
51
52/*
53 * Flavors
54 */
55
56#define ARM_THREAD_STATE 1
57#define ARM_UNIFIED_THREAD_STATE ARM_THREAD_STATE
58#define ARM_VFP_STATE 2
59#define ARM_EXCEPTION_STATE 3
60#define ARM_DEBUG_STATE 4 /* pre-armv8 */
61#define THREAD_STATE_NONE 5
62#define ARM_THREAD_STATE64 6
63#define ARM_EXCEPTION_STATE64 7
64// ARM_THREAD_STATE_LAST 8 /* legacy */
65#define ARM_THREAD_STATE32 9
66
67#ifdef XNU_KERNEL_PRIVATE
68#define X86_THREAD_STATE_NONE 13 /* i386/thread_status.h THREAD_STATE_NONE */
69#endif /* XNU_KERNEL_PRIVATE */
70
71/* API */
72#define ARM_DEBUG_STATE32 14
73#define ARM_DEBUG_STATE64 15
74#define ARM_NEON_STATE 16
75#define ARM_NEON_STATE64 17
76#define ARM_CPMU_STATE64 18
77
78#ifdef XNU_KERNEL_PRIVATE
79/* For kernel use */
80#define ARM_SAVED_STATE32 20
81#define ARM_SAVED_STATE64 21
82#define ARM_NEON_SAVED_STATE32 22
83#define ARM_NEON_SAVED_STATE64 23
84#endif /* XNU_KERNEL_PRIVATE */
85
86#define ARM_PAGEIN_STATE 27
87
88
89#ifndef ARM_STATE_FLAVOR_IS_OTHER_VALID
90#define ARM_STATE_FLAVOR_IS_OTHER_VALID(_flavor_) 0
91#endif
92
93#define FLAVOR_MODIFIES_CORE_CPU_REGISTERS(x) \
94((x == ARM_THREAD_STATE) || \
95 (x == ARM_THREAD_STATE32) || \
96 (x == ARM_THREAD_STATE64))
97
98#define VALID_THREAD_STATE_FLAVOR(x) \
99 ((x == ARM_THREAD_STATE) || \
100 (x == ARM_VFP_STATE) || \
101 (x == ARM_EXCEPTION_STATE) || \
102 (x == ARM_DEBUG_STATE) || \
103 (x == THREAD_STATE_NONE) || \
104 (x == ARM_THREAD_STATE32) || \
105 (x == ARM_THREAD_STATE64) || \
106 (x == ARM_EXCEPTION_STATE64) || \
107 (x == ARM_NEON_STATE) || \
108 (x == ARM_NEON_STATE64) || \
109 (x == ARM_DEBUG_STATE32) || \
110 (x == ARM_DEBUG_STATE64) || \
111 (x == ARM_PAGEIN_STATE) || \
112 (ARM_STATE_FLAVOR_IS_OTHER_VALID(x)))
113
114struct arm_state_hdr {
115 uint32_t flavor;
116 uint32_t count;
117};
118typedef struct arm_state_hdr arm_state_hdr_t;
119
120typedef _STRUCT_ARM_THREAD_STATE arm_thread_state_t;
121typedef _STRUCT_ARM_THREAD_STATE arm_thread_state32_t;
122typedef _STRUCT_ARM_THREAD_STATE64 arm_thread_state64_t;
123
124#if !defined(KERNEL)
125#if __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__)
126
127/* Accessor macros for arm_thread_state64_t pointer fields */
128
129/* Return pc field of arm_thread_state64_t as a data pointer value */
130#define arm_thread_state64_get_pc(ts) \
131 __darwin_arm_thread_state64_get_pc(ts)
132/* Return pc field of arm_thread_state64_t as a function pointer. May return
133 * NULL if a valid function pointer cannot be constructed, the caller should
134 * fall back to the arm_thread_state64_get_pc() macro in that case. */
135#define arm_thread_state64_get_pc_fptr(ts) \
136 __darwin_arm_thread_state64_get_pc_fptr(ts)
137/* Set pc field of arm_thread_state64_t to a function pointer */
138#define arm_thread_state64_set_pc_fptr(ts, fptr) \
139 __darwin_arm_thread_state64_set_pc_fptr(ts, fptr)
140/* Return lr field of arm_thread_state64_t as a data pointer value */
141#define arm_thread_state64_get_lr(ts) \
142 __darwin_arm_thread_state64_get_lr(ts)
143/* Return lr field of arm_thread_state64_t as a function pointer. May return
144 * NULL if a valid function pointer cannot be constructed, the caller should
145 * fall back to the arm_thread_state64_get_lr() macro in that case. */
146#define arm_thread_state64_get_lr_fptr(ts) \
147 __darwin_arm_thread_state64_get_lr_fptr(ts)
148/* Set lr field of arm_thread_state64_t to a function pointer */
149#define arm_thread_state64_set_lr_fptr(ts, fptr) \
150 __darwin_arm_thread_state64_set_lr_fptr(ts, fptr)
151/* Return sp field of arm_thread_state64_t as a data pointer value */
152#define arm_thread_state64_get_sp(ts) \
153 __darwin_arm_thread_state64_get_sp(ts)
154/* Set sp field of arm_thread_state64_t to a data pointer value */
155#define arm_thread_state64_set_sp(ts, ptr) \
156 __darwin_arm_thread_state64_set_sp(ts, ptr)
157/* Return fp field of arm_thread_state64_t as a data pointer value */
158#define arm_thread_state64_get_fp(ts) \
159 __darwin_arm_thread_state64_get_fp(ts)
160/* Set fp field of arm_thread_state64_t to a data pointer value */
161#define arm_thread_state64_set_fp(ts, ptr) \
162 __darwin_arm_thread_state64_set_fp(ts, ptr)
163/* Strip ptr auth bits from pc, lr, sp and fp field of arm_thread_state64_t */
164#define arm_thread_state64_ptrauth_strip(ts) \
165 __darwin_arm_thread_state64_ptrauth_strip(ts)
166
167#endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__) */
168#endif /* !defined(KERNEL) */
169
170struct arm_unified_thread_state {
171 arm_state_hdr_t ash;
172 union {
173 arm_thread_state32_t ts_32;
174 arm_thread_state64_t ts_64;
175 } uts;
176};
177#define ts_32 uts.ts_32
178#define ts_64 uts.ts_64
179typedef struct arm_unified_thread_state arm_unified_thread_state_t;
180
181#define ARM_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
182 (sizeof (arm_thread_state_t)/sizeof(uint32_t)))
183#define ARM_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
184 (sizeof (arm_thread_state32_t)/sizeof(uint32_t)))
185#define ARM_THREAD_STATE64_COUNT ((mach_msg_type_number_t) \
186 (sizeof (arm_thread_state64_t)/sizeof(uint32_t)))
187#define ARM_UNIFIED_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
188 (sizeof (arm_unified_thread_state_t)/sizeof(uint32_t)))
189
190
191typedef _STRUCT_ARM_VFP_STATE arm_vfp_state_t;
192typedef _STRUCT_ARM_NEON_STATE arm_neon_state_t;
193typedef _STRUCT_ARM_NEON_STATE arm_neon_state32_t;
194typedef _STRUCT_ARM_NEON_STATE64 arm_neon_state64_t;
195
196
197typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state_t;
198typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state32_t;
199typedef _STRUCT_ARM_EXCEPTION_STATE64 arm_exception_state64_t;
200
201typedef _STRUCT_ARM_DEBUG_STATE32 arm_debug_state32_t;
202typedef _STRUCT_ARM_DEBUG_STATE64 arm_debug_state64_t;
203
204typedef _STRUCT_ARM_PAGEIN_STATE arm_pagein_state_t;
205
206#if defined(XNU_KERNEL_PRIVATE) && defined(__arm64__)
207/* See below for ARM64 kernel structure definition for arm_debug_state. */
208#else /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
209/*
210 * Otherwise not ARM64 kernel and we must preserve legacy ARM definitions of
211 * arm_debug_state for binary compatability of userland consumers of this file.
212 */
213#if defined(__arm__)
214typedef _STRUCT_ARM_DEBUG_STATE arm_debug_state_t;
215#elif defined(__arm64__)
216typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_debug_state_t;
217#else /* defined(__arm__) */
218#error Undefined architecture
219#endif /* defined(__arm__) */
220#endif /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
221
222#define ARM_VFP_STATE_COUNT ((mach_msg_type_number_t) \
223 (sizeof (arm_vfp_state_t)/sizeof(uint32_t)))
224
225#define ARM_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \
226 (sizeof (arm_exception_state_t)/sizeof(uint32_t)))
227
228#define ARM_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \
229 (sizeof (arm_exception_state64_t)/sizeof(uint32_t)))
230
231#define ARM_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
232 (sizeof (arm_debug_state_t)/sizeof(uint32_t)))
233
234#define ARM_DEBUG_STATE32_COUNT ((mach_msg_type_number_t) \
235 (sizeof (arm_debug_state32_t)/sizeof(uint32_t)))
236
237#define ARM_PAGEIN_STATE_COUNT ((mach_msg_type_number_t) \
238 (sizeof (arm_pagein_state_t)/sizeof(uint32_t)))
239
240#define ARM_DEBUG_STATE64_COUNT ((mach_msg_type_number_t) \
241 (sizeof (arm_debug_state64_t)/sizeof(uint32_t)))
242
243#define ARM_NEON_STATE_COUNT ((mach_msg_type_number_t) \
244 (sizeof (arm_neon_state_t)/sizeof(uint32_t)))
245
246#define ARM_NEON_STATE64_COUNT ((mach_msg_type_number_t) \
247 (sizeof (arm_neon_state64_t)/sizeof(uint32_t)))
248
249#define MACHINE_THREAD_STATE ARM_THREAD_STATE
250#define MACHINE_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
251
252
253/*
254 * Largest state on this machine:
255 */
256#define THREAD_MACHINE_STATE_MAX THREAD_STATE_MAX
257
258#ifdef XNU_KERNEL_PRIVATE
259
260#if CONFIG_DTRACE
261#define HAS_ADD_SAVED_STATE_PC 1
262#define HAS_SET_SAVED_STATE_PC 1
263#define HAS_SET_SAVED_STATE_LR 1
264#define HAS_SET_SAVED_STATE_REG 1
265#define HAS_MASK_SAVED_STATE_CPSR 1
266#endif /* CONFIG_DTRACE */
267
268#if CONFIG_KDP_INTERACTIVE_DEBUGGING
269#define HAS_SET_SAVED_STATE_CPSR 1
270#endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
271
272#if CONFIG_XNUPOST
273#define HAS_ADD_SAVED_STATE_PC 1
274#define HAS_SET_SAVED_STATE_PC 1
275#endif /* CONFIG_DTRACE */
276
277#if DEBUG || DEVELOPMENT
278#define HAS_ADD_SAVED_STATE_PC 1
279#endif
280
281
282static inline boolean_t
283is_thread_state32(const arm_unified_thread_state_t *its)
284{
285 return its->ash.flavor == ARM_THREAD_STATE32;
286}
287
288static inline boolean_t
289is_thread_state64(const arm_unified_thread_state_t *its)
290{
291 return its->ash.flavor == ARM_THREAD_STATE64;
292}
293
294static inline arm_thread_state32_t*
295thread_state32(arm_unified_thread_state_t *its)
296{
297 return &its->ts_32;
298}
299
300static inline arm_thread_state64_t*
301thread_state64(arm_unified_thread_state_t *its)
302{
303 return &its->ts_64;
304}
305
306static inline const arm_thread_state32_t*
307const_thread_state32(const arm_unified_thread_state_t *its)
308{
309 return &its->ts_32;
310}
311
312static inline const arm_thread_state64_t*
313const_thread_state64(const arm_unified_thread_state_t *its)
314{
315 return &its->ts_64;
316}
317
318#if defined(__arm64__)
319
320#include <kern/assert.h>
321#include <arm64/proc_reg.h>
322#define CAST_ASSERT_SAFE(type, val) (assert((val) == ((type)(val))), (type)(val))
323
324/*
325 * GPR context
326 */
327
328struct arm_saved_state32 {
329 uint32_t r[13]; /* General purpose register r0-r12 */
330 uint32_t sp; /* Stack pointer r13 */
331 uint32_t lr; /* Link register r14 */
332 uint32_t pc; /* Program counter r15 */
333 uint32_t cpsr; /* Current program status register */
334 uint32_t far; /* Virtual fault address */
335 uint32_t esr; /* Exception syndrome register */
336 uint32_t exception; /* Exception number */
337};
338typedef struct arm_saved_state32 arm_saved_state32_t;
339
340struct arm_saved_state32_tagged {
341 uint32_t tag;
342 struct arm_saved_state32 state;
343};
344typedef struct arm_saved_state32_tagged arm_saved_state32_tagged_t;
345
346#define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
347 (sizeof(arm_saved_state32_t)/sizeof(unsigned int)))
348
349struct arm_saved_state64 {
350 uint64_t x[29]; /* General purpose registers x0-x28 */
351 uint64_t fp; /* Frame pointer x29 */
352 uint64_t lr; /* Link register x30 */
353 uint64_t sp; /* Stack pointer x31 */
354 uint64_t pc; /* Program counter */
355 uint32_t cpsr; /* Current program status register */
356 uint32_t reserved; /* Reserved padding */
357 uint64_t far; /* Virtual fault address */
358 uint32_t esr; /* Exception syndrome register */
359 uint32_t exception; /* Exception number */
360#if HAS_APPLE_PAC
361 uint64_t jophash;
362#endif /* HAS_APPLE_PAC */
363};
364typedef struct arm_saved_state64 arm_saved_state64_t;
365
366#define ARM_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
367 (sizeof(arm_saved_state64_t)/sizeof(unsigned int)))
368
369struct arm_saved_state64_tagged {
370 uint32_t tag;
371 struct arm_saved_state64 state;
372};
373typedef struct arm_saved_state64_tagged arm_saved_state64_tagged_t;
374
375struct arm_saved_state {
376 arm_state_hdr_t ash;
377 union {
378 struct arm_saved_state32 ss_32;
379 struct arm_saved_state64 ss_64;
380 } uss;
381} __attribute__((aligned(16)));
382#define ss_32 uss.ss_32
383#define ss_64 uss.ss_64
384
385typedef struct arm_saved_state arm_saved_state_t;
386
387struct arm_kernel_saved_state {
388 uint64_t x[10]; /* General purpose registers x19-x28 */
389 uint64_t fp; /* Frame pointer x29 */
390 uint64_t lr; /* Link register x30 */
391 uint64_t sp; /* Stack pointer x31 */
392 /* Some things here we DO need to preserve */
393 uint8_t pc_was_in_userspace;
394 uint8_t ssbs;
395 uint8_t dit;
396 uint8_t uao;
397} __attribute__((aligned(16)));
398
399typedef struct arm_kernel_saved_state arm_kernel_saved_state_t;
400
401extern void ml_panic_on_invalid_old_cpsr(const arm_saved_state_t *) __attribute__((noreturn));
402
403extern void ml_panic_on_invalid_new_cpsr(const arm_saved_state_t *, uint32_t) __attribute__((noreturn));
404
405#if HAS_APPLE_PAC
406
407#include <sys/cdefs.h>
408
409/*
410 * Used by MANIPULATE_SIGNED_THREAD_STATE(), potentially from C++ (IOKit) code.
411 * Open-coded to prevent a circular dependency between mach/arm/thread_status.h
412 * and osfmk/arm/machine_routines.h.
413 */
414__BEGIN_DECLS
415extern uint64_t ml_pac_safe_interrupts_disable(void);
416extern void ml_pac_safe_interrupts_restore(uint64_t);
417__END_DECLS
418
419/*
420 * Methods used to sign and check thread state to detect corruptions of saved
421 * thread state across exceptions and context switches.
422 */
423extern void ml_sign_thread_state(arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
424
425extern void ml_check_signed_state(const arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
426
427/* XXX: including stddef.f here breaks ctfmerge on some builds, so use __builtin_offsetof() instead of offsetof() */
428#define ss64_offsetof(x) __builtin_offsetof(struct arm_saved_state, ss_64.x)
429
430/**
431 * Verify the signed thread state in _iss, execute the assembly instructions
432 * _instr, and re-sign the modified thread state. Varargs specify additional
433 * inputs.
434 *
435 * _instr may read or modify the thread state in the following registers:
436 *
437 * x0: _iss
438 * x1: authed _iss->ss_64.pc
439 * w2: authed _iss->ss_64.cpsr
440 * x3: authed _iss->ss_64.lr
441 * x4: authed _iss->ss_64.x16
442 * x5: authed _iss->ss_64.x17
443 * x6: scratch register
444 * x7: scratch register
445 *
446 * If _instr makes no changes to the thread state, it may skip re-signing by
447 * branching to label 0.
448 */
449#define MANIPULATE_SIGNED_THREAD_STATE(_iss, _instr, ...) \
450 do { \
451 uint64_t _intr = ml_pac_safe_interrupts_disable(); \
452 asm volatile ( \
453 "mov x9, lr" "\n" \
454 "mov x0, %[iss]" "\n" \
455 "msr SPSel, #1" "\n" \
456 "ldp x4, x5, [x0, %[SS64_X16]]" "\n" \
457 "ldr x7, [x0, %[SS64_PC]]" "\n" \
458 "ldr w8, [x0, %[SS64_CPSR]]" "\n" \
459 "ldr x3, [x0, %[SS64_LR]]" "\n" \
460 "mov x1, x7" "\n" \
461 "mov w2, w8" "\n" \
462 "bl _ml_check_signed_state" "\n" \
463 "mov x1, x7" "\n" \
464 "mov w2, w8" "\n" \
465 _instr "\n" \
466 "bl _ml_sign_thread_state" "\n" \
467 "0:" "\n" \
468 "msr SPSel, #0" "\n" \
469 "mov lr, x9" "\n" \
470 : \
471 : [iss] "r"(_iss), \
472 [SS64_X16] "i"(ss64_offsetof(x[16])), \
473 [SS64_PC] "i"(ss64_offsetof(pc)), \
474 [SS64_CPSR] "i"(ss64_offsetof(cpsr)), \
475 [SS64_LR] "i"(ss64_offsetof(lr)),##__VA_ARGS__ \
476 : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", \
477 "x9", "x16" \
478 ); \
479 ml_pac_safe_interrupts_restore(_intr); \
480 } while (0)
481
482#define VERIFY_USER_THREAD_STATE_INSTR \
483 "and w6, w2, %[CPSR_EL_MASK]" "\n" \
484 "cmp w6, %[CPSR_EL0]" "\n" \
485 "b.eq 1f" "\n" \
486 "bl _ml_panic_on_invalid_old_cpsr" "\n" \
487 "brk #0" "\n" \
488 "1:" "\n"
489
490#define VERIFY_USER_THREAD_STATE_INPUTS \
491 [CPSR_EL_MASK] "i"(PSR64_MODE_EL_MASK), \
492 [CPSR_EL0] "i"(PSR64_MODE_EL0)
493
494#define MANIPULATE_SIGNED_USER_THREAD_STATE(_iss, _instr, ...) \
495 MANIPULATE_SIGNED_THREAD_STATE(_iss, \
496 VERIFY_USER_THREAD_STATE_INSTR \
497 _instr, \
498 VERIFY_USER_THREAD_STATE_INPUTS, ##__VA_ARGS__)
499
500static inline void
501check_and_sign_copied_user_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
502{
503 MANIPULATE_SIGNED_USER_THREAD_STATE(src,
504 "mov x0, %[dst]",
505 [dst] "r"(dst)
506 );
507}
508#endif /* HAS_APPLE_PAC */
509
510static inline boolean_t
511is_saved_state32(const arm_saved_state_t *iss)
512{
513 return iss->ash.flavor == ARM_SAVED_STATE32;
514}
515
516static inline boolean_t
517is_saved_state64(const arm_saved_state_t *iss)
518{
519 return iss->ash.flavor == ARM_SAVED_STATE64;
520}
521
522static inline arm_saved_state32_t*
523saved_state32(arm_saved_state_t *iss)
524{
525 return &iss->ss_32;
526}
527
528static inline const arm_saved_state32_t*
529const_saved_state32(const arm_saved_state_t *iss)
530{
531 return &iss->ss_32;
532}
533
534static inline arm_saved_state64_t*
535saved_state64(arm_saved_state_t *iss)
536{
537 return &iss->ss_64;
538}
539
540static inline const arm_saved_state64_t*
541const_saved_state64(const arm_saved_state_t *iss)
542{
543 return &iss->ss_64;
544}
545
546static inline register_t
547get_saved_state_pc(const arm_saved_state_t *iss)
548{
549 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->pc : const_saved_state64(iss)->pc);
550}
551
552#if HAS_ADD_SAVED_STATE_PC
553static inline void
554add_saved_state_pc(arm_saved_state_t *iss, int diff)
555{
556 if (is_saved_state32(iss)) {
557 uint64_t pc = saved_state32(iss)->pc + (uint32_t)diff;
558 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
559 } else {
560#if HAS_APPLE_PAC
561 MANIPULATE_SIGNED_THREAD_STATE(iss,
562 "mov w6, %w[diff] \n"
563 "add x1, x1, w6, sxtw \n"
564 "str x1, [x0, %[SS64_PC]] \n",
565 [diff] "r"(diff)
566 );
567#else
568 saved_state64(iss)->pc += (unsigned long)diff;
569#endif /* HAS_APPLE_PAC */
570 }
571}
572#endif /* HAS_ADD_SAVED_STATE_PC */
573
574static inline void
575add_user_saved_state_pc(arm_saved_state_t *iss, int diff)
576{
577 if (is_saved_state32(iss)) {
578 uint64_t pc = saved_state32(iss)->pc + (uint32_t)diff;
579 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
580 } else {
581#if HAS_APPLE_PAC
582 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
583 "mov w6, %w[diff] \n"
584 "add x1, x1, w6, sxtw \n"
585 "str x1, [x0, %[SS64_PC]] \n",
586 [diff] "r"(diff)
587 );
588#else
589 saved_state64(iss)->pc += (unsigned long)diff;
590#endif /* HAS_APPLE_PAC */
591 }
592}
593
594#if HAS_SET_SAVED_STATE_PC
595static inline void
596set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
597{
598 if (is_saved_state32(iss)) {
599 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
600 } else {
601#if HAS_APPLE_PAC
602 MANIPULATE_SIGNED_THREAD_STATE(iss,
603 "mov x1, %[pc] \n"
604 "str x1, [x0, %[SS64_PC]] \n",
605 [pc] "r"(pc)
606 );
607#else
608 saved_state64(iss)->pc = (unsigned long)pc;
609#endif /* HAS_APPLE_PAC */
610 }
611}
612#endif /* HAS_SET_SAVED_STATE_PC */
613
614static inline void
615set_user_saved_state_pc(arm_saved_state_t *iss, register_t pc)
616{
617 if (is_saved_state32(iss)) {
618 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
619 } else {
620#if HAS_APPLE_PAC
621 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
622 "mov x1, %[pc] \n"
623 "str x1, [x0, %[SS64_PC]] \n",
624 [pc] "r"(pc)
625 );
626#else
627 saved_state64(iss)->pc = (unsigned long)pc;
628#endif /* HAS_APPLE_PAC */
629 }
630}
631
632static inline register_t
633get_saved_state_sp(const arm_saved_state_t *iss)
634{
635 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->sp : const_saved_state64(iss)->sp);
636}
637
638static inline void
639set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
640{
641 if (is_saved_state32(iss)) {
642 saved_state32(iss)->sp = CAST_ASSERT_SAFE(uint32_t, sp);
643 } else {
644 saved_state64(iss)->sp = (uint64_t)sp;
645 }
646}
647
648static inline register_t
649get_saved_state_lr(const arm_saved_state_t *iss)
650{
651 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->lr : const_saved_state64(iss)->lr);
652}
653
654#if HAS_SET_SAVED_STATE_LR
655static inline void
656set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
657{
658 if (is_saved_state32(iss)) {
659 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
660 } else {
661#if HAS_APPLE_PAC
662 MANIPULATE_SIGNED_THREAD_STATE(iss,
663 "mov x3, %[lr] \n"
664 "str x3, [x0, %[SS64_LR]] \n",
665 [lr] "r"(lr)
666 );
667#else
668 saved_state64(iss)->lr = (unsigned long)lr;
669#endif /* HAS_APPLE_PAC */
670 }
671}
672#endif /* HAS_SET_SAVED_STATE_PC */
673
674static inline void
675set_user_saved_state_lr(arm_saved_state_t *iss, register_t lr)
676{
677 if (is_saved_state32(iss)) {
678 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
679 } else {
680#if HAS_APPLE_PAC
681 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
682 "mov x3, %[lr] \n"
683 "str x3, [x0, %[SS64_LR]] \n",
684 [lr] "r"(lr)
685 );
686#else
687 saved_state64(iss)->lr = (unsigned long)lr;
688#endif /* HAS_APPLE_PAC */
689 }
690}
691
692static inline register_t
693get_saved_state_fp(const arm_saved_state_t *iss)
694{
695 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->r[7] : const_saved_state64(iss)->fp);
696}
697
698static inline void
699set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
700{
701 if (is_saved_state32(iss)) {
702 saved_state32(iss)->r[7] = CAST_ASSERT_SAFE(uint32_t, fp);
703 } else {
704 saved_state64(iss)->fp = (uint64_t)fp;
705 }
706}
707
708static inline int
709check_saved_state_reglimit(const arm_saved_state_t *iss, unsigned reg)
710{
711 return is_saved_state32(iss) ? (reg < ARM_SAVED_STATE32_COUNT) : (reg < ARM_SAVED_STATE64_COUNT);
712}
713
714static inline register_t
715get_saved_state_reg(const arm_saved_state_t *iss, unsigned reg)
716{
717 if (!check_saved_state_reglimit(iss, reg)) {
718 return 0;
719 }
720
721 return (register_t)(is_saved_state32(iss) ? (const_saved_state32(iss)->r[reg]) : (const_saved_state64(iss)->x[reg]));
722}
723
724#if HAS_SET_SAVED_STATE_REG
725static inline void
726set_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
727{
728 if (!check_saved_state_reglimit(iss, reg)) {
729 return;
730 }
731
732 if (is_saved_state32(iss)) {
733 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
734 } else {
735#if HAS_APPLE_PAC
736 /* x16 and x17 are part of the jophash */
737 if (reg == 16) {
738 MANIPULATE_SIGNED_THREAD_STATE(iss,
739 "mov x4, %[value] \n"
740 "str x4, [x0, %[SS64_X16]] \n",
741 [value] "r"(value)
742 );
743 return;
744 } else if (reg == 17) {
745 MANIPULATE_SIGNED_THREAD_STATE(iss,
746 "mov x5, %[value] \n"
747 "str x5, [x0, %[SS64_X17]] \n",
748 [value] "r"(value),
749 [SS64_X17] "i"(ss64_offsetof(x[17]))
750 );
751 return;
752 }
753#endif
754 saved_state64(iss)->x[reg] = (uint64_t)value;
755 }
756}
757#endif /* HAS_SET_SAVED_STATE_REG */
758
759static inline void
760set_user_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
761{
762 if (!check_saved_state_reglimit(iss, reg)) {
763 return;
764 }
765
766 if (is_saved_state32(iss)) {
767 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
768 } else {
769#if HAS_APPLE_PAC
770 /* x16 and x17 are part of the jophash */
771 if (reg == 16) {
772 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
773 "mov x4, %[value] \n"
774 "str x4, [x0, %[SS64_X16]] \n",
775 [value] "r"(value)
776 );
777 return;
778 } else if (reg == 17) {
779 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
780 "mov x5, %[value] \n"
781 "str x5, [x0, %[SS64_X17]] \n",
782 [value] "r"(value),
783 [SS64_X17] "i"(ss64_offsetof(x[17]))
784 );
785 return;
786 }
787#endif
788 saved_state64(iss)->x[reg] = (uint64_t)value;
789 }
790}
791
792
793static inline uint32_t
794get_saved_state_cpsr(const arm_saved_state_t *iss)
795{
796 return is_saved_state32(iss) ? const_saved_state32(iss)->cpsr : const_saved_state64(iss)->cpsr;
797}
798
799#if HAS_MASK_SAVED_STATE_CPSR
800static inline void
801mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
802{
803 if (is_saved_state32(iss)) {
804 saved_state32(iss)->cpsr |= set_bits;
805 saved_state32(iss)->cpsr &= ~clear_bits;
806 } else {
807#if HAS_APPLE_PAC
808 MANIPULATE_SIGNED_THREAD_STATE(iss,
809 "mov w6, %w[set_bits] \n"
810 "orr w2, w2, w6, lsl #0 \n"
811 "mov w6, %w[clear_bits] \n"
812 "bic w2, w2, w6, lsl #0 \n"
813 "str w2, [x0, %[SS64_CPSR]] \n",
814 [set_bits] "r"(set_bits),
815 [clear_bits] "r"(clear_bits)
816 );
817#else
818 saved_state64(iss)->cpsr |= set_bits;
819 saved_state64(iss)->cpsr &= ~clear_bits;
820#endif /* HAS_APPLE_PAC */
821 }
822}
823#endif /* HAS_MASK_SAVED_STATE_CPSR */
824
825static inline void
826mask_user_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
827{
828 if (is_saved_state32(iss)) {
829 uint32_t new_cpsr = saved_state32(iss)->cpsr;
830 new_cpsr |= set_bits;
831 new_cpsr &= ~clear_bits;
832 if (!PSR_IS_USER(new_cpsr)) {
833 ml_panic_on_invalid_new_cpsr(iss, new_cpsr);
834 }
835 saved_state32(iss)->cpsr = new_cpsr;
836 } else {
837#if HAS_APPLE_PAC
838 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
839 "mov w6, %w[set_bits] \n"
840 "orr w2, w2, w6, lsl #0 \n"
841 "mov w6, %w[clear_bits] \n"
842 "bic w2, w2, w6, lsl #0 \n"
843 "and w6, w2, %[CPSR_EL_MASK] \n"
844 "cmp w6, %[CPSR_EL0] \n"
845 "b.eq 1f \n"
846 "mov w1, w2 \n"
847 "bl _ml_panic_on_invalid_new_cpsr \n"
848 "brk #0 \n"
849 "1: \n"
850 "str w2, [x0, %[SS64_CPSR]] \n",
851 [set_bits] "r"(set_bits),
852 [clear_bits] "r"(clear_bits)
853 );
854#else
855 uint32_t new_cpsr = saved_state64(iss)->cpsr;
856 new_cpsr |= set_bits;
857 new_cpsr &= ~clear_bits;
858 if (!PSR64_IS_USER(new_cpsr)) {
859 ml_panic_on_invalid_new_cpsr(iss, new_cpsr);
860 }
861 saved_state64(iss)->cpsr = new_cpsr;
862#endif /* HAS_APPLE_PAC */
863 }
864}
865
866#if HAS_SET_SAVED_STATE_CPSR
867static inline void
868set_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
869{
870 if (is_saved_state32(iss)) {
871 saved_state32(iss)->cpsr = cpsr;
872 } else {
873#if HAS_APPLE_PAC
874 MANIPULATE_SIGNED_THREAD_STATE(iss,
875 "mov w2, %w[cpsr] \n"
876 "str w2, [x0, %[SS64_CPSR]] \n",
877 [cpsr] "r"(cpsr)
878 );
879#else
880 saved_state64(iss)->cpsr = cpsr;
881#endif /* HAS_APPLE_PAC */
882 }
883}
884#endif /* HAS_SET_SAVED_STATE_CPSR */
885
886static inline void
887set_user_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
888{
889 if (is_saved_state32(iss)) {
890 if (!PSR_IS_USER(cpsr)) {
891 ml_panic_on_invalid_new_cpsr(iss, cpsr);
892 }
893 saved_state32(iss)->cpsr = cpsr;
894 } else {
895#if HAS_APPLE_PAC
896 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
897 "mov w2, %w[cpsr] \n"
898 "and w6, w2, %[CPSR_EL_MASK] \n"
899 "cmp w6, %[CPSR_EL0] \n"
900 "b.eq 1f \n"
901 "mov w1, w2 \n"
902 "bl _ml_panic_on_invalid_new_cpsr \n"
903 "brk #0 \n"
904 "1: \n"
905 "str w2, [x0, %[SS64_CPSR]] \n",
906 [cpsr] "r"(cpsr)
907 );
908#else
909 if (!PSR64_IS_USER(cpsr)) {
910 ml_panic_on_invalid_new_cpsr(iss, cpsr);
911 }
912 saved_state64(iss)->cpsr = cpsr;
913#endif /* HAS_APPLE_PAC */
914 }
915}
916
917static inline register_t
918get_saved_state_far(const arm_saved_state_t *iss)
919{
920 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->far : const_saved_state64(iss)->far);
921}
922
923static inline void
924set_saved_state_far(arm_saved_state_t *iss, register_t far)
925{
926 if (is_saved_state32(iss)) {
927 saved_state32(iss)->far = CAST_ASSERT_SAFE(uint32_t, far);
928 } else {
929 saved_state64(iss)->far = (uint64_t)far;
930 }
931}
932
933static inline uint32_t
934get_saved_state_esr(const arm_saved_state_t *iss)
935{
936 return is_saved_state32(iss) ? const_saved_state32(iss)->esr : const_saved_state64(iss)->esr;
937}
938
939static inline void
940set_saved_state_esr(arm_saved_state_t *iss, uint32_t esr)
941{
942 if (is_saved_state32(iss)) {
943 saved_state32(iss)->esr = esr;
944 } else {
945 saved_state64(iss)->esr = esr;
946 }
947}
948
949static inline uint32_t
950get_saved_state_exc(const arm_saved_state_t *iss)
951{
952 return is_saved_state32(iss) ? const_saved_state32(iss)->exception : const_saved_state64(iss)->exception;
953}
954
955static inline void
956set_saved_state_exc(arm_saved_state_t *iss, uint32_t exc)
957{
958 if (is_saved_state32(iss)) {
959 saved_state32(iss)->exception = exc;
960 } else {
961 saved_state64(iss)->exception = exc;
962 }
963}
964
965extern void panic_unimplemented(void);
966
967/**
968 * Extracts the SVC (Supervisor Call) number from the appropriate GPR (General
969 * Purpose Register).
970 *
971 * @param[in] iss the 32-bit or 64-bit ARM saved state (i.e. trap frame).
972 *
973 * @return The SVC number.
974 */
975static inline int
976get_saved_state_svc_number(const arm_saved_state_t *iss)
977{
978 return is_saved_state32(iss) ? (int)const_saved_state32(iss)->r[12] : (int)const_saved_state64(iss)->x[ARM64_SYSCALL_CODE_REG_NUM]; /* Only first word counts here */
979}
980
981typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_legacy_debug_state_t;
982
983struct arm_debug_aggregate_state {
984 arm_state_hdr_t dsh;
985 union {
986 arm_debug_state32_t ds32;
987 arm_debug_state64_t ds64;
988 } uds;
989 os_refcnt_t ref;
990} __attribute__((aligned(16)));
991
992typedef struct arm_debug_aggregate_state arm_debug_state_t;
993
994#define ARM_LEGACY_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
995 (sizeof (arm_legacy_debug_state_t)/sizeof(uint32_t)))
996
997/*
998 * NEON context
999 */
1000typedef __uint128_t uint128_t;
1001typedef uint64_t uint64x2_t __attribute__((ext_vector_type(2)));
1002typedef uint32_t uint32x4_t __attribute__((ext_vector_type(4)));
1003
1004struct arm_neon_saved_state32 {
1005 union {
1006 uint128_t q[16];
1007 uint64_t d[32];
1008 uint32_t s[32];
1009 } v;
1010 uint32_t fpsr;
1011 uint32_t fpcr;
1012};
1013typedef struct arm_neon_saved_state32 arm_neon_saved_state32_t;
1014
1015#define ARM_NEON_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
1016 (sizeof (arm_neon_saved_state32_t)/sizeof(unsigned int)))
1017
1018struct arm_neon_saved_state64 {
1019 union {
1020 uint128_t q[32];
1021 uint64x2_t d[32];
1022 uint32x4_t s[32];
1023 } v;
1024 uint32_t fpsr;
1025 uint32_t fpcr;
1026};
1027typedef struct arm_neon_saved_state64 arm_neon_saved_state64_t;
1028
1029#define ARM_NEON_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
1030 (sizeof (arm_neon_saved_state64_t)/sizeof(unsigned int)))
1031
1032struct arm_neon_saved_state {
1033 arm_state_hdr_t nsh;
1034 union {
1035 struct arm_neon_saved_state32 ns_32;
1036 struct arm_neon_saved_state64 ns_64;
1037 } uns;
1038};
1039typedef struct arm_neon_saved_state arm_neon_saved_state_t;
1040#define ns_32 uns.ns_32
1041#define ns_64 uns.ns_64
1042
1043struct arm_kernel_neon_saved_state {
1044 uint64_t d[8];
1045 uint32_t fpcr;
1046};
1047typedef struct arm_kernel_neon_saved_state arm_kernel_neon_saved_state_t;
1048
1049static inline boolean_t
1050is_neon_saved_state32(const arm_neon_saved_state_t *state)
1051{
1052 return state->nsh.flavor == ARM_NEON_SAVED_STATE32;
1053}
1054
1055static inline boolean_t
1056is_neon_saved_state64(const arm_neon_saved_state_t *state)
1057{
1058 return state->nsh.flavor == ARM_NEON_SAVED_STATE64;
1059}
1060
1061static inline arm_neon_saved_state32_t *
1062neon_state32(arm_neon_saved_state_t *state)
1063{
1064 return &state->ns_32;
1065}
1066
1067static inline arm_neon_saved_state64_t *
1068neon_state64(arm_neon_saved_state_t *state)
1069{
1070 return &state->ns_64;
1071}
1072
1073
1074
1075/*
1076 * Aggregated context
1077 */
1078
1079struct arm_context {
1080 struct arm_saved_state ss;
1081 struct arm_neon_saved_state ns;
1082};
1083typedef struct arm_context arm_context_t;
1084
1085struct arm_kernel_context {
1086 struct arm_kernel_saved_state ss;
1087 struct arm_kernel_neon_saved_state ns;
1088};
1089typedef struct arm_kernel_context arm_kernel_context_t;
1090
1091extern void saved_state_to_thread_state64(const arm_saved_state_t*, arm_thread_state64_t*);
1092extern void thread_state64_to_saved_state(const arm_thread_state64_t*, arm_saved_state_t*);
1093
1094#else /* defined(__arm64__) */
1095#error Unknown arch
1096#endif /* defined(__arm64__) */
1097
1098extern void saved_state_to_thread_state32(const arm_saved_state_t*, arm_thread_state32_t*);
1099extern void thread_state32_to_saved_state(const arm_thread_state32_t*, arm_saved_state_t*);
1100
1101#endif /* XNU_KERNEL_PRIVATE */
1102
1103#endif /* defined (__arm__) || defined (__arm64__) */
1104
1105#endif /* _ARM_THREAD_STATUS_H_ */
1106