1/*
2 * Copyright (c) 2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifdef KERNEL_PRIVATE
30
31#ifndef _PTHREAD_SHIMS_H_
32#define _PTHREAD_SHIMS_H_
33
34#ifndef ASSEMBLER
35
36#include <kern/block_hint.h>
37#include <kern/clock.h>
38#include <kern/kern_types.h>
39#include <kern/kcdata.h>
40#include <kern/locks.h>
41#include <kern/turnstile.h>
42#include <pthread/priority_private.h>
43#include <sys/user.h>
44#include <sys/_types.h>
45#include <sys/_types/_sigset_t.h>
46#include <sys/kernel_types.h>
47
48#ifndef PTHREAD_INTERNAL
49struct uthread;
50struct ksyn_waitq_element;
51#define M_PROC 41
52#endif
53
54#if !defined(_SCHED_CALL_T_DEFINED)
55#define _SCHED_CALL_T_DEFINED
56typedef void (*sched_call_t)(int type, thread_t thread);
57#endif
58
59/*
60 * Increment each time new reserved slots are used. When the pthread
61 * kext registers this table, it will include the version of the xnu
62 * headers that it was built against.
63 */
64#define PTHREAD_FUNCTIONS_TABLE_VERSION 1
65
66typedef const struct pthread_functions_s {
67 int version;
68
69 /* internal calls, kernel core -> kext */
70 void (*pthread_init)(void);
71
72 void *__unused_was_fill_procworkqueue;
73 void *__unused1;
74 void *__unused2;
75 void *__unused_was_workqueue_exit;
76 void *__unused_was_workqueue_mark_exiting;
77 void *__unused_was_workqueue_thread_yielded;
78
79 void (*pth_proc_hashinit)(proc_t p);
80 void (*pth_proc_hashdelete)(proc_t p);
81
82 /* syscall stubs */
83 int (*bsdthread_create)(struct proc *p, user_addr_t user_func, user_addr_t user_funcarg, user_addr_t user_stack, user_addr_t user_pthread, uint32_t flags, user_addr_t *retval);
84 int (*bsdthread_register)(struct proc *p, user_addr_t threadstart, user_addr_t wqthread, int pthsize, user_addr_t dummy_value, user_addr_t targetconc_ptr, uint64_t dispatchqueue_offset, int32_t *retval);
85 int (*bsdthread_terminate)(struct proc *p, user_addr_t stackaddr, size_t size, uint32_t kthport, uint32_t sem, int32_t *retval);
86 int (*thread_selfid)(struct proc *p, uint64_t *retval);
87 void *__unused_was_workq_kernreturn;
88 void *__unused_was_workq_open;
89
90 /* psynch syscalls */
91 int (*psynch_mutexwait)(proc_t p, user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags, uint32_t *retval);
92 int (*psynch_mutexdrop)(proc_t p, user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags, uint32_t *retval);
93 int (*psynch_cvbroad)(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, user_addr_t mutex, uint64_t mugen, uint64_t tid, uint32_t *retval);
94 int (*psynch_cvsignal)(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, user_addr_t mutex, uint64_t mugen, uint64_t tid, uint32_t flags, uint32_t *retval);
95 int (*psynch_cvwait)(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, user_addr_t mutex, uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec, uint32_t * retval);
96 int (*psynch_cvclrprepost)(proc_t p, user_addr_t cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags, int *retval);
97 int (*psynch_rw_longrdlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
98 int (*psynch_rw_rdlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
99 int (*psynch_rw_unlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
100 int (*psynch_rw_wrlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
101 int (*psynch_rw_yieldwrlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
102
103 void *__unused_was_workqueue_get_sched_callback;
104
105 /* New register function with TSD offset */
106 int (*bsdthread_register2)(struct proc *p, user_addr_t threadstart, user_addr_t wqthread, uint32_t flags, user_addr_t stack_addr_hint, user_addr_t targetconc_ptr, uint32_t dispatchqueue_offset, uint32_t tsd_offset, int32_t *retval);
107
108 void *__unused_was_bsdthread_ctl;
109 void *__unused_was_workq_reqthreads;
110
111 void *__unused_was_thread_qos_from_pthread_priority;
112 void *__unused_was_get_pwq_state_kdp;
113 void *__unused3;
114 void *__unused_was_pthread_priority_canonicalize2;
115 void *__unused_was_workq_thread_has_been_unbound;
116
117 void (*pthread_find_owner)(thread_t thread, struct stackshot_thread_waitinfo *waitinfo);
118 void *(*pthread_get_thread_kwq)(thread_t thread);
119
120 void *__unused_was_workq_threadreq;
121
122 int (*workq_handle_stack_events)(proc_t p, thread_t th, vm_map_t map,
123 user_addr_t stackaddr, mach_port_name_t kport,
124 user_addr_t events, int nevents, int upcall_flags);
125
126 int (*workq_create_threadstack)(proc_t p, vm_map_t vmap,
127 mach_vm_offset_t *out_addr);
128
129 int (*workq_destroy_threadstack)(proc_t p, vm_map_t vmap,
130 mach_vm_offset_t stackaddr);
131
132 void (*workq_setup_thread)(proc_t p, thread_t th, vm_map_t map,
133 user_addr_t stackaddr, mach_port_name_t kport, int th_qos,
134 int setup_flags, int upcall_flags);
135
136 void (*workq_markfree_threadstack)(proc_t p, thread_t, vm_map_t map,
137 user_addr_t stackaddr);
138
139 /* padding for future */
140 void * _pad[83];
141} * pthread_functions_t;
142
143typedef const struct pthread_callbacks_s {
144 int version;
145
146 /* config information */
147 uint32_t config_thread_max;
148 uint32_t (*get_task_threadmax)(void);
149
150 /* proc.h accessors */
151 uint64_t (*proc_get_register)(struct proc *t);
152 void (*proc_set_register)(struct proc *t);
153
154 user_addr_t (*proc_get_threadstart)(struct proc *t);
155 void (*proc_set_threadstart)(struct proc *t, user_addr_t addr);
156 user_addr_t (*proc_get_wqthread)(struct proc *t);
157 void (*proc_set_wqthread)(struct proc *t, user_addr_t addr);
158 int (*proc_get_pthsize)(struct proc *t);
159 void (*proc_set_pthsize)(struct proc *t, int size);
160
161 thread_t (*task_findtid)(task_t t, uint64_t tid);
162 void (*thread_deallocate_safe)(thread_t);
163 bool (*proc_get_jit_entitled)(struct proc *t);
164 void (*proc_set_dispatchqueue_offset)(struct proc *t, uint64_t offset);
165 bool (*proc_get_pthread_jit_allowlist2)(struct proc *t, bool *late_out);
166 void *__unused_was_proc_get_wqinitingptr;
167 void *__unused_was_proc_get_wqptr;
168
169 wait_result_t (*psynch_wait_prepare)(uintptr_t kwq,
170 struct turnstile **tstore, thread_t owner, block_hint_t block_hint,
171 uint64_t deadline);
172
173 void (*psynch_wait_update_complete)(struct turnstile *turnstile);
174
175 void (*psynch_wait_complete)(uintptr_t kwq, struct turnstile **tstore);
176
177 void (*psynch_wait_cleanup)(void);
178
179 kern_return_t (*psynch_wait_wakeup)(uintptr_t kwq,
180 struct ksyn_waitq_element *kwe, struct turnstile **tstore);
181
182 void (*psynch_wait_update_owner)(uintptr_t kwq, thread_t owner,
183 struct turnstile **tstore);
184
185 void* (*proc_get_pthhash)(struct proc *t);
186 void (*proc_set_pthhash)(struct proc *t, void* ptr);
187
188 /* bsd/sys/user.h */
189 void *__unused_was_uthread_get_threadlist;
190 void *__unused_was_uthread_set_threadlist;
191 void *__unused_was_uthread_get_sigmask;
192 void *__unused_was_uthread_set_sigmask;
193
194 void* (*uthread_get_uukwe)(struct uthread *t);
195 void *__unused_was_uthread_get_returnval;
196 void (*uthread_set_returnval)(struct uthread *t, int val);
197 int (*uthread_is_cancelled)(struct uthread *t);
198
199 /* vm_protos.h calls */
200 ipc_space_t (*task_get_ipcspace)(task_t t);
201 void *__unused_was_ipc_port_copyout_send;
202
203 /* osfmk/vm/vm_map.h */
204 kern_return_t (*vm_map_page_info)(vm_map_t map, vm_map_offset_t offset, vm_page_info_flavor_t flavor, vm_page_info_t info, mach_msg_type_number_t *count);
205 mach_port_name_t (*ipc_port_copyout_send_pinned)(ipc_port_t sright, ipc_space_t space);
206
207 /* wq functions */
208 kern_return_t (*thread_set_wq_state32)(thread_t thread, thread_state_t state);
209#if !defined(__arm__)
210 kern_return_t (*thread_set_wq_state64)(thread_t thread, thread_state_t state);
211#endif
212
213 /* sched_prim.h */
214 void (*thread_exception_return)(void);
215 void (*thread_bootstrap_return)(void);
216
217 void *__unused_was_absolutetime_to_microtime;
218 void *__unused_was_thread_set_workq_pri;
219 void *__unused_was_thread_set_workq_qos;
220
221 /* osfmk/kern/thread.h */
222 struct uthread* (*get_bsdthread_info)(thread_t th);
223 void *__unused_was_thread_sched_call;
224 void *__unused_was_thread_static_param;
225 void *__unused_was_thread_create_workq_waiting_parameter;
226 kern_return_t (*thread_policy_set_internal)(thread_t t, thread_policy_flavor_t flavour, thread_policy_t info, mach_msg_type_number_t count);
227
228 void *__unused_was_thread_affinity_set;
229
230 /* bsd/sys/systm.h */
231 void (*unix_syscall_return)(int error);
232
233 void *__unused_was_zalloc;
234 void *__unused_was_zfree;
235 void *__unused_was_zinit;
236
237 /* bsd/kern/kern_sig.c */
238 void (*__pthread_testcancel)(int);
239
240 /* calls without portfolio */
241 kern_return_t (*mach_port_deallocate)(ipc_space_t space, mach_port_name_t name);
242 kern_return_t (*semaphore_signal_internal_trap)(mach_port_name_t sema_name);
243 vm_map_t (*current_map)(void);
244
245 void *__unused_was_convert_thread_to_port;
246 void *__unused_was_thread_create;
247
248 /* mach/thread_act.h */
249 kern_return_t (*thread_resume)(thread_act_t target_act);
250
251 /* bsd/sys/event.h */
252 int (*kevent_workq_internal)(struct proc *p,
253 user_addr_t changelist, int nchanges,
254 user_addr_t eventlist, int nevents,
255 user_addr_t data_out, user_size_t *data_available,
256 unsigned int flags, int32_t *retval);
257
258#if defined(__arm__)
259 void *__unused_was_map_is_1gb;
260#endif
261
262 void *__unused_was_proc_get_dispatchqueue_serialno_offset;
263 void *__unused_was_proc_set_dispatchqueue_serialno_offset;
264
265 void *__unused_was_proc_usynch_thread_qos_add_override_for_resource_check_owner;
266 void (*proc_set_workqueue_quantum_offset)(struct proc *p, uint64_t offset);
267
268 uint32_t (*proc_get_pthread_tsd_offset)(struct proc *p);
269 void (*proc_set_pthread_tsd_offset)(struct proc *p, uint32_t pthread_tsd_offset);
270
271 kern_return_t (*thread_set_tsd_base)(thread_t thread, mach_vm_offset_t tsd_base);
272
273 int (*proc_usynch_get_requested_thread_qos)(struct uthread *);
274 uint64_t (*proc_get_mach_thread_self_tsd_offset)(struct proc *p);
275 void (*proc_set_mach_thread_self_tsd_offset)(struct proc *p, uint64_t mach_thread_self_tsd_offset);
276
277 kern_return_t (*thread_policy_get)(thread_t t, thread_policy_flavor_t flavor, thread_policy_t info, mach_msg_type_number_t *count, boolean_t *get_default);
278 boolean_t (*qos_main_thread_active)(void);
279
280 kern_return_t (*thread_set_voucher_name)(mach_port_name_t name);
281
282 boolean_t (*proc_usynch_thread_qos_add_override_for_resource)(task_t task, struct uthread *, uint64_t tid, int override_qos, boolean_t first_override_for_resource, user_addr_t resource, int resource_type);
283 boolean_t (*proc_usynch_thread_qos_remove_override_for_resource)(task_t task, struct uthread *, uint64_t tid, user_addr_t resource, int resource_type);
284 void *__unused_was_proc_usynch_thread_qos_reset_override_for_resource;
285
286 void *__unused_was_proc_init_wqptr_or_wait;
287
288 uint16_t (*thread_set_tag)(thread_t thread, uint16_t tag);
289 uint16_t (*thread_get_tag)(thread_t thread);
290
291 void *__unused_was_thread_create_pinned;
292
293 kern_return_t (*thread_terminate_pinned)(thread_t thread);
294 ipc_port_t (*convert_thread_to_port_pinned)(thread_t th);
295
296 user_addr_t (*proc_get_stack_addr_hint)(struct proc *p);
297 void (*proc_set_stack_addr_hint)(struct proc *p, user_addr_t stack_addr_hint);
298
299 kern_return_t (*thread_create_immovable)(task_t parent_task, thread_t *new_thread);
300 void (*proc_set_return_to_kernel_offset)(struct proc *t, uint64_t offset);
301
302 void *__unused_was_workloop_fulfill_threadreq;
303 void (*thread_will_park_or_terminate)(thread_t thread);
304
305 void *__unused_was_qos_max_parallelism;
306
307 /* proc_internal.h: struct proc user_stack accessor */
308 user_addr_t (*proc_get_user_stack)(struct proc *p);
309 void *__unused_was_proc_set_user_stack;
310
311 /* padding for future */
312 void* _pad[69];
313} *pthread_callbacks_t;
314
315void
316pthread_kext_register(pthread_functions_t fns, pthread_callbacks_t *callbacks);
317
318#ifdef BSD_KERNEL_PRIVATE
319void thread_will_park_or_terminate(thread_t thread);
320void pthread_init(void);
321extern pthread_callbacks_t pthread_kern;
322extern pthread_functions_t pthread_functions;
323#endif
324
325#endif /* ASSEMBLER */
326#endif /* _PTHREAD_SHIMS_H_ */
327#endif /* KERNEL_PRIVATE */
328