1/*
2 * Copyright (c) 2007-2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _ARM_LOCKS_H_
30#define _ARM_LOCKS_H_
31
32#include <kern/kern_types.h>
33#ifdef MACH_KERNEL_PRIVATE
34#include <arm/hw_lock_types.h>
35#endif
36
37
38#ifdef MACH_KERNEL_PRIVATE
39
40extern unsigned int LcksOpts;
41
42#define enaLkDeb 0x00000001 /* Request debug in default attribute */
43#define enaLkStat 0x00000002 /* Request statistic in default attribute */
44#define disLkRWPrio 0x00000004 /* Disable RW lock priority promotion */
45
46#define disLkType 0x80000000 /* Disable type checking */
47#define disLktypeb 0
48#define disLkThread 0x40000000 /* Disable ownership checking */
49#define disLkThreadb 1
50#define enaLkExtStck 0x20000000 /* Enable extended backtrace */
51#define enaLkExtStckb 2
52#define disLkMyLck 0x10000000 /* Disable recursive lock dectection */
53#define disLkMyLckb 3
54
55#endif
56
57#ifdef MACH_KERNEL_PRIVATE
58typedef struct {
59 struct hslock hwlock;
60 uintptr_t type;
61} lck_spin_t;
62
63#define lck_spin_data hwlock.lock_data
64
65#define LCK_SPIN_TAG_DESTROYED 0xdead /* lock marked as Destroyed */
66
67#define LCK_SPIN_TYPE 0x00000011
68
69#else
70#ifdef KERNEL_PRIVATE
71
72typedef struct {
73 uintptr_t opaque[2];
74} lck_spin_t;
75
76#else
77typedef struct __lck_spin_t__ lck_spin_t;
78#endif // KERNEL_PRIVATE
79#endif // MACH_KERNEL_PRIVATE
80
81#ifdef MACH_KERNEL_PRIVATE
82typedef struct _lck_mtx_ {
83 union {
84
85 uintptr_t lck_mtx_data; /* Thread pointer plus lock bits */
86 uintptr_t lck_mtx_tag; /* Tag for type */
87 }; /* arm: 4 arm64: 8 */
88 union {
89 struct {
90 uint16_t lck_mtx_waiters;/* Number of waiters */
91 uint8_t lck_mtx_pri; /* Priority to inherit */
92 uint8_t lck_mtx_type; /* Type */
93 };
94 struct {
95 struct _lck_mtx_ext_ *lck_mtx_ptr; /* Indirect pointer */
96 };
97 }; /* arm: 4 arm64: 8 */
98} lck_mtx_t; /* arm: 8 arm64: 16 */
99
100/* Shared between mutex and read-write locks */
101#define LCK_ILOCK_BIT 0
102#define ARM_LCK_WAITERS_BIT 1
103#define LCK_ILOCK (1 << LCK_ILOCK_BIT)
104#define ARM_LCK_WAITERS (1 << ARM_LCK_WAITERS_BIT)
105
106#define LCK_MTX_TYPE 0x22 /* lock type */
107
108#define LCK_MTX_TAG_INDIRECT 0x00001007 /* lock marked as Indirect */
109#define LCK_MTX_TAG_DESTROYED 0x00002007 /* lock marked as Destroyed */
110
111#define LCK_FRAMES_MAX 8
112
113extern uint64_t MutexSpin;
114
115typedef struct {
116 unsigned int type;
117 vm_offset_t stack[LCK_FRAMES_MAX];
118 vm_offset_t thread;
119} lck_mtx_deb_t;
120
121#define MUTEX_TAG 0x4d4d
122
123typedef struct {
124 unsigned int lck_mtx_stat_data;
125} lck_mtx_stat_t;
126
127typedef struct _lck_mtx_ext_ {
128 lck_mtx_t lck_mtx; /* arm: 12 arm64: 24 */
129 struct _lck_grp_ *lck_mtx_grp; /* arm: 4 arm64: 8 */
130 unsigned int lck_mtx_attr; /* arm: 4 arm64: 4 */
131 lck_mtx_stat_t lck_mtx_stat; /* arm: 4 arm64: 4 */
132 lck_mtx_deb_t lck_mtx_deb; /* arm: 40 arm64: 80 */
133} lck_mtx_ext_t; /* arm: 64 arm64: 120 */
134
135#define LCK_MTX_ATTR_DEBUG 0x1
136#define LCK_MTX_ATTR_DEBUGb 31
137#define LCK_MTX_ATTR_STAT 0x2
138#define LCK_MTX_ATTR_STATb 30
139
140#define LCK_MTX_EVENT(lck) ((event_t)(((unsigned int*)(lck))+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))
141#define LCK_EVENT_TO_MUTEX(event) ((lck_mtx_t *)(uintptr_t)(((unsigned int *)(event)) - ((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))
142
143#else
144#ifdef KERNEL_PRIVATE
145typedef struct {
146 uintptr_t opaque[2];
147} lck_mtx_t;
148
149typedef struct {
150#if defined(__arm64__)
151 unsigned long opaque[16];
152#else /* __arm__ */
153 unsigned int opaque[16];
154#endif
155} lck_mtx_ext_t;
156
157#else
158typedef struct __lck_mtx_t__ lck_mtx_t;
159#endif
160#endif
161
162#ifdef MACH_KERNEL_PRIVATE
163
164typedef union {
165 struct {
166 uint16_t shared_count; /* No. of shared granted request */
167 uint16_t interlock: 1, /* Interlock */
168 priv_excl: 1, /* priority for Writer */
169 want_upgrade: 1, /* Read-to-write upgrade waiting */
170 want_excl: 1, /* Writer is waiting, or locked for write */
171 r_waiting: 1, /* Someone is sleeping on lock */
172 w_waiting: 1, /* Writer is sleeping on lock */
173 can_sleep: 1, /* Can attempts to lock go to sleep? */
174 _pad2: 8, /* padding */
175 tag_valid: 1; /* Field is actually a tag, not a bitfield */
176#if __arm64__
177 uint32_t _pad4;
178#endif
179 };
180 struct {
181 uint32_t data; /* Single word version of bitfields and shared count */
182#if __arm64__
183 uint32_t lck_rw_pad4;
184#endif
185 };
186} lck_rw_word_t;
187
188typedef struct {
189 lck_rw_word_t word;
190 thread_t lck_rw_owner;
191} lck_rw_t; /* arm: 8 arm64: 16 */
192
193#define lck_rw_shared_count word.shared_count
194#define lck_rw_interlock word.interlock
195#define lck_rw_priv_excl word.priv_excl
196#define lck_rw_want_upgrade word.want_upgrade
197#define lck_rw_want_excl word.want_excl
198#define lck_r_waiting word.r_waiting
199#define lck_w_waiting word.w_waiting
200#define lck_rw_can_sleep word.can_sleep
201#define lck_rw_data word.data
202// tag and data reference the same memory. When the tag_valid bit is set,
203// the data word should be treated as a tag instead of a bitfield.
204#define lck_rw_tag_valid word.tag_valid
205#define lck_rw_tag word.data
206
207#define LCK_RW_SHARED_READER_OFFSET 0
208#define LCK_RW_INTERLOCK_BIT 16
209#define LCK_RW_PRIV_EXCL_BIT 17
210#define LCK_RW_WANT_UPGRADE_BIT 18
211#define LCK_RW_WANT_EXCL_BIT 19
212#define LCK_RW_R_WAITING_BIT 20
213#define LCK_RW_W_WAITING_BIT 21
214#define LCK_RW_CAN_SLEEP_BIT 22
215// 23-30
216#define LCK_RW_TAG_VALID_BIT 31
217
218#define LCK_RW_INTERLOCK (1 << LCK_RW_INTERLOCK_BIT)
219#define LCK_RW_R_WAITING (1 << LCK_RW_R_WAITING_BIT)
220#define LCK_RW_W_WAITING (1 << LCK_RW_W_WAITING_BIT)
221#define LCK_RW_WANT_UPGRADE (1 << LCK_RW_WANT_UPGRADE_BIT)
222#define LCK_RW_WANT_EXCL (1 << LCK_RW_WANT_EXCL_BIT)
223#define LCK_RW_TAG_VALID (1 << LCK_RW_TAG_VALID_BIT)
224#define LCK_RW_PRIV_EXCL (1 << LCK_RW_PRIV_EXCL_BIT)
225#define LCK_RW_SHARED_MASK (0xffff << LCK_RW_SHARED_READER_OFFSET)
226#define LCK_RW_SHARED_READER (0x1 << LCK_RW_SHARED_READER_OFFSET)
227
228#define LCK_RW_TAG_DESTROYED ((LCK_RW_TAG_VALID | 0xdddddeadu)) /* lock marked as Destroyed */
229
230#define LCK_RW_WRITER_EVENT(lck) (event_t)((uintptr_t)(lck)+1)
231#define LCK_RW_READER_EVENT(lck) (event_t)((uintptr_t)(lck)+2)
232#define WRITE_EVENT_TO_RWLOCK(event) ((lck_rw_t *)((uintptr_t)(event)-1))
233#define READ_EVENT_TO_RWLOCK(event) ((lck_rw_t *)((uintptr_t)(event)-2))
234
235#if __ARM_ENABLE_WFE_
236
237#define wait_for_event() __builtin_arm_wfe()
238#if __arm__
239#define set_event() do{__builtin_arm_dsb(DSB_ISHST);__builtin_arm_sev();}while(0)
240#define LOCK_SNOOP_SPINS 4
241#else
242#define set_event() do{}while(0) // arm64 sev is implicit in stlxr
243#define LOCK_SNOOP_SPINS 0x300
244#endif
245
246#else
247
248#define wait_for_event() __builtin_arm_clrex()
249#define set_event() do{}while(0)
250#define LOCK_SNOOP_SPINS 0x300
251
252#endif // __ARM_ENABLE_WFE_
253
254#if LOCK_PRIVATE
255
256#define LOCK_PANIC_TIMEOUT 0xc00000 // 12.5 m ticks = 250ms with 24MHz OSC
257
258#define PLATFORM_LCK_ILOCK LCK_ILOCK
259
260
261/*
262 * Lock state to thread pointer
263 * Clear the bottom bits
264 */
265#define LCK_MTX_STATE_TO_THREAD(s) (thread_t)(s & ~(LCK_ILOCK | ARM_LCK_WAITERS))
266/*
267 * Thread pointer to lock state
268 * arm thread pointers are aligned such that the bottom two bits are clear
269 */
270#define LCK_MTX_THREAD_TO_STATE(t) ((uintptr_t)t)
271/*
272 * Thread pointer mask
273 */
274#define LCK_MTX_THREAD_MASK (~(uintptr_t)(LCK_ILOCK | ARM_LCK_WAITERS))
275
276#define disable_preemption_for_thread(t) ((volatile thread_t)t)->machine.preemption_count++
277#define preemption_disabled_for_thread(t) (((volatile thread_t)t)->machine.preemption_count > 0)
278
279
280__unused static void disable_interrupts_noread(void)
281{
282#if __arm__
283 __asm__ volatile ("cpsid if" ::: "memory"); // Mask IRQ FIQ
284#else
285 __builtin_arm_wsr64("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF)); // Mask IRQ FIQ
286#endif
287}
288
289__unused static inline long get_interrupts(void)
290{
291 long state;
292
293#if __arm__
294 __asm__ volatile ("mrs %[state], cpsr" :[state] "=r" (state)); // Read cpsr
295#else
296 state = __builtin_arm_rsr64("DAIF"); // Read interrupt state
297#endif
298 return state;
299}
300
301__unused static inline long disable_interrupts(void)
302{
303 long state;
304
305 state = get_interrupts(); // Get previous state
306 disable_interrupts_noread(); // Disable
307 return state;
308}
309
310__unused static inline void restore_interrupts(long state)
311{
312#if __arm__
313 __asm__ volatile ("msr cpsr, %[state]" :: [state] "r" (state) : "cc", "memory"); // Restore CPSR
314#elif __arm64__
315 __builtin_arm_wsr64("DAIF", state); // Restore masks
316#endif
317}
318
319#endif // LOCK_PRIVATE
320
321#else
322#ifdef KERNEL_PRIVATE
323typedef struct {
324 uintptr_t opaque[2];
325} lck_rw_t;
326#else
327typedef struct __lck_rw_t__ lck_rw_t;
328#endif
329#endif
330
331#endif /* _ARM_LOCKS_H_ */
332