1/*
2 * Copyright (c) 2000-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_FREE_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: thread.h
60 * Author: Avadis Tevanian, Jr.
61 *
62 * This file contains the structure definitions for threads.
63 *
64 */
65/*
66 * Copyright (c) 1993 The University of Utah and
67 * the Computer Systems Laboratory (CSL). All rights reserved.
68 *
69 * Permission to use, copy, modify and distribute this software and its
70 * documentation is hereby granted, provided that both the copyright
71 * notice and this permission notice appear in all copies of the
72 * software, derivative works or modified versions, and any portions
73 * thereof, and that both notices appear in supporting documentation.
74 *
75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
78 *
79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any
80 * improvements that they make and grant CSL redistribution rights.
81 *
82 */
83
84#ifndef _KERN_THREAD_H_
85#define _KERN_THREAD_H_
86
87#include <mach/kern_return.h>
88#include <mach/mach_types.h>
89#include <mach/message.h>
90#include <mach/boolean.h>
91#include <mach/vm_param.h>
92#include <mach/thread_info.h>
93#include <mach/thread_status.h>
94#include <mach/exception_types.h>
95
96#include <kern/kern_types.h>
97#include <vm/vm_kern.h>
98
99#include <sys/cdefs.h>
100
101#ifdef MACH_KERNEL_PRIVATE
102
103#include <mach_assert.h>
104#include <mach_ldebug.h>
105
106#include <ipc/ipc_types.h>
107
108#include <mach/port.h>
109#include <kern/cpu_number.h>
110#include <kern/smp.h>
111#include <kern/queue.h>
112
113#include <kern/timer.h>
114#include <kern/simple_lock.h>
115#include <kern/locks.h>
116#include <kern/sched.h>
117#include <kern/sched_prim.h>
118#include <mach/sfi_class.h>
119#include <kern/thread_call.h>
120#include <kern/thread_group.h>
121#include <kern/timer_call.h>
122#include <kern/task.h>
123#include <kern/exception.h>
124#include <kern/affinity.h>
125#include <kern/debug.h>
126#include <kern/block_hint.h>
127#include <kern/turnstile.h>
128
129#include <kern/waitq.h>
130#include <san/kasan.h>
131
132#include <ipc/ipc_kmsg.h>
133
134#include <machine/cpu_data.h>
135#include <machine/thread.h>
136
137#ifdef XNU_KERNEL_PRIVATE
138/* priority queue static asserts fail for __ARM64_ARCH_8_32__ kext builds */
139#include <kern/priority_queue.h>
140#endif /* XNU_KERNEL_PRIVATE */
141
142#if MONOTONIC
143#include <stdatomic.h>
144#include <machine/monotonic.h>
145#endif /* MONOTONIC */
146
147#if CONFIG_EMBEDDED
148/* Taskwatch related. TODO: find this a better home */
149typedef struct task_watcher task_watch_t;
150#endif /* CONFIG_EMBEDDED */
151
152struct thread {
153
154#if MACH_ASSERT
155#define THREAD_MAGIC 0x1234ABCDDCBA4321ULL
156 /* Ensure nothing uses &thread as a queue entry */
157 uint64_t thread_magic;
158#endif /* MACH_ASSERT */
159
160 /*
161 * NOTE: The runq field in the thread structure has an unusual
162 * locking protocol. If its value is PROCESSOR_NULL, then it is
163 * locked by the thread_lock, but if its value is something else
164 * then it is locked by the associated run queue lock. It is
165 * set to PROCESSOR_NULL without holding the thread lock, but the
166 * transition from PROCESSOR_NULL to non-null must be done
167 * under the thread lock and the run queue lock.
168 *
169 * New waitq APIs allow the 'links' and 'runq' fields to be
170 * anywhere in the thread structure.
171 */
172 union {
173 queue_chain_t runq_links; /* run queue links */
174 queue_chain_t wait_links; /* wait queue links */
175 struct priority_queue_entry wait_prioq_links; /* priority ordered waitq links */
176 };
177
178 processor_t runq; /* run queue assignment */
179
180 event64_t wait_event; /* wait queue event */
181 struct waitq *waitq; /* wait queue this thread is enqueued on */
182 struct turnstile *turnstile; /* thread's turnstile, protected by primitives interlock */
183 void *inheritor; /* inheritor of the primitive the thread will block on */
184 struct priority_queue inheritor_queue; /* Inheritor queue */
185
186 /* Data updated during assert_wait/thread_wakeup */
187#if __SMP__
188 decl_simple_lock_data(,sched_lock) /* scheduling lock (thread_lock()) */
189 decl_simple_lock_data(,wake_lock) /* for thread stop / wait (wake_lock()) */
190#endif
191 integer_t options; /* options set by thread itself */
192#define TH_OPT_INTMASK 0x0003 /* interrupt / abort level */
193#define TH_OPT_VMPRIV 0x0004 /* may allocate reserved memory */
194#define TH_OPT_DTRACE 0x0008 /* executing under dtrace_probe */
195#define TH_OPT_SYSTEM_CRITICAL 0x0010 /* Thread must always be allowed to run - even under heavy load */
196#define TH_OPT_PROC_CPULIMIT 0x0020 /* Thread has a task-wide CPU limit applied to it */
197#define TH_OPT_PRVT_CPULIMIT 0x0040 /* Thread has a thread-private CPU limit applied to it */
198#define TH_OPT_IDLE_THREAD 0x0080 /* Thread is a per-processor idle thread */
199#define TH_OPT_GLOBAL_FORCED_IDLE 0x0100 /* Thread performs forced idle for thermal control */
200#define TH_OPT_SCHED_VM_GROUP 0x0200 /* Thread belongs to special scheduler VM group */
201#define TH_OPT_HONOR_QLIMIT 0x0400 /* Thread will honor qlimit while sending mach_msg, regardless of MACH_SEND_ALWAYS */
202#define TH_OPT_SEND_IMPORTANCE 0x0800 /* Thread will allow importance donation from kernel rpc */
203#define TH_OPT_ZONE_GC 0x1000 /* zone_gc() called on this thread */
204
205 boolean_t wake_active; /* wake event on stop */
206 int at_safe_point; /* thread_abort_safely allowed */
207 ast_t reason; /* why we blocked */
208 uint32_t quantum_remaining;
209 wait_result_t wait_result; /* outcome of wait -
210 * may be examined by this thread
211 * WITHOUT locking */
212 thread_continue_t continuation; /* continue here next dispatch */
213 void *parameter; /* continuation parameter */
214
215 /* Data updated/used in thread_invoke */
216 vm_offset_t kernel_stack; /* current kernel stack */
217 vm_offset_t reserved_stack; /* reserved kernel stack */
218
219#if KASAN
220 struct kasan_thread_data kasan_data;
221#endif
222
223 /* Thread state: */
224 int state;
225/*
226 * Thread states [bits or'ed]
227 */
228#define TH_WAIT 0x01 /* queued for waiting */
229#define TH_SUSP 0x02 /* stopped or requested to stop */
230#define TH_RUN 0x04 /* running or on runq */
231#define TH_UNINT 0x08 /* waiting uninteruptibly */
232#define TH_TERMINATE 0x10 /* halted at termination */
233#define TH_TERMINATE2 0x20 /* added to termination queue */
234#define TH_WAIT_REPORT 0x40 /* the wait is using the sched_call,
235 only set if TH_WAIT is also set */
236#define TH_IDLE 0x80 /* idling processor */
237
238 /* Scheduling information */
239 sched_mode_t sched_mode; /* scheduling mode */
240 sched_mode_t saved_mode; /* saved mode during forced mode demotion */
241
242 /* This thread's contribution to global sched counters */
243 sched_bucket_t th_sched_bucket;
244
245 sfi_class_id_t sfi_class; /* SFI class (XXX Updated on CSW/QE/AST) */
246 sfi_class_id_t sfi_wait_class; /* Currently in SFI wait for this class, protected by sfi_lock */
247
248
249 uint32_t sched_flags; /* current flag bits */
250/* TH_SFLAG_FAIRSHARE_TRIPPED (unused) 0x0001 */
251#define TH_SFLAG_FAILSAFE 0x0002 /* fail-safe has tripped */
252#define TH_SFLAG_THROTTLED 0x0004 /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
253#define TH_SFLAG_DEMOTED_MASK (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE) /* saved_mode contains previous sched_mode */
254
255#define TH_SFLAG_PROMOTED 0x0008 /* sched pri has been promoted by kernel mutex priority promotion */
256#define TH_SFLAG_ABORT 0x0010 /* abort interruptible waits */
257#define TH_SFLAG_ABORTSAFELY 0x0020 /* ... but only those at safe point */
258#define TH_SFLAG_ABORTED_MASK (TH_SFLAG_ABORT | TH_SFLAG_ABORTSAFELY)
259#define TH_SFLAG_DEPRESS 0x0040 /* normal depress yield */
260#define TH_SFLAG_POLLDEPRESS 0x0080 /* polled depress yield */
261#define TH_SFLAG_DEPRESSED_MASK (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
262/* unused TH_SFLAG_PRI_UPDATE 0x0100 */
263#define TH_SFLAG_EAGERPREEMPT 0x0200 /* Any preemption of this thread should be treated as if AST_URGENT applied */
264#define TH_SFLAG_RW_PROMOTED 0x0400 /* promote reason: blocking with RW lock held */
265/* unused TH_SFLAG_THROTTLE_DEMOTED 0x0800 */
266#define TH_SFLAG_WAITQ_PROMOTED 0x1000 /* promote reason: waitq wakeup (generally for IPC receive) */
267
268
269#define TH_SFLAG_EXEC_PROMOTED 0x8000 /* promote reason: thread is in an exec */
270
271/* 'promote reasons' that request a priority floor only, not a custom priority */
272#define TH_SFLAG_PROMOTE_REASON_MASK (TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED | TH_SFLAG_EXEC_PROMOTED)
273
274#define TH_SFLAG_RW_PROMOTED_BIT (10) /* 0x400 */
275
276 int16_t sched_pri; /* scheduled (current) priority */
277 int16_t base_pri; /* base priority */
278 int16_t max_priority; /* copy of max base priority */
279 int16_t task_priority; /* copy of task base priority */
280 int16_t promotion_priority; /* priority thread is currently promoted to */
281
282#if defined(CONFIG_SCHED_GRRR)
283#if 0
284 uint16_t grrr_deficit; /* fixed point (1/1000th quantum) fractional deficit */
285#endif
286#endif
287
288 int16_t promotions; /* level of promotion */
289 int iotier_override; /* atomic operations to set, cleared on ret to user */
290 _Atomic uint32_t ref_count; /* number of references to me */
291
292 lck_mtx_t* waiting_for_mutex; /* points to mutex we're waiting for until we acquire it */
293
294 uint32_t rwlock_count; /* Number of lck_rw_t locks held by thread */
295
296 integer_t importance; /* task-relative importance */
297 uint32_t was_promoted_on_wakeup; /* thread promoted on wakeup to acquire mutex */
298
299 /* Priority depression expiration */
300 integer_t depress_timer_active;
301 timer_call_data_t depress_timer;
302 /* real-time parameters */
303 struct { /* see mach/thread_policy.h */
304 uint32_t period;
305 uint32_t computation;
306 uint32_t constraint;
307 boolean_t preemptible;
308 uint64_t deadline;
309 } realtime;
310
311 uint64_t last_run_time; /* time when thread was switched away from */
312 uint64_t last_made_runnable_time; /* time when thread was unblocked or preempted */
313 uint64_t last_basepri_change_time; /* time when thread was last changed in basepri while runnable */
314 uint64_t same_pri_latency;
315#define THREAD_NOT_RUNNABLE (~0ULL)
316
317
318#if defined(CONFIG_SCHED_MULTIQ)
319 sched_group_t sched_group;
320#endif /* defined(CONFIG_SCHED_MULTIQ) */
321
322 /* Data used during setrun/dispatch */
323 timer_data_t system_timer; /* system mode timer */
324 processor_t bound_processor; /* bound to a processor? */
325 processor_t last_processor; /* processor last dispatched on */
326 processor_t chosen_processor; /* Where we want to run this thread */
327
328 /* Fail-safe computation since last unblock or qualifying yield */
329 uint64_t computation_metered;
330 uint64_t computation_epoch;
331 uint64_t safe_release; /* when to release fail-safe */
332
333 /* Call out from scheduler */
334 void (*sched_call)(
335 int type,
336 thread_t thread);
337#if defined(CONFIG_SCHED_PROTO)
338 uint32_t runqueue_generation; /* last time runqueue was drained */
339#endif
340
341 /* Statistics and timesharing calculations */
342#if defined(CONFIG_SCHED_TIMESHARE_CORE)
343 natural_t sched_stamp; /* last scheduler tick */
344 natural_t sched_usage; /* timesharing cpu usage [sched] */
345 natural_t pri_shift; /* usage -> priority from pset */
346 natural_t cpu_usage; /* instrumented cpu usage [%cpu] */
347 natural_t cpu_delta; /* accumulated cpu_usage delta */
348#endif /* CONFIG_SCHED_TIMESHARE_CORE */
349
350 uint32_t c_switch; /* total context switches */
351 uint32_t p_switch; /* total processor switches */
352 uint32_t ps_switch; /* total pset switches */
353
354 integer_t mutex_count; /* total count of locks held */
355 /* Timing data structures */
356 int precise_user_kernel_time; /* precise user/kernel enabled for this thread */
357 timer_data_t user_timer; /* user mode timer */
358 uint64_t user_timer_save; /* saved user timer value */
359 uint64_t system_timer_save; /* saved system timer value */
360 uint64_t vtimer_user_save; /* saved values for vtimers */
361 uint64_t vtimer_prof_save;
362 uint64_t vtimer_rlim_save;
363 uint64_t vtimer_qos_save;
364
365 timer_data_t ptime; /* time executing in P mode */
366 timer_data_t runnable_timer; /* time the thread is runnable (including running) */
367
368#if CONFIG_SCHED_SFI
369 /* Timing for wait state */
370 uint64_t wait_sfi_begin_time; /* start time for thread waiting in SFI */
371#endif
372
373 /* Timed wait expiration */
374 timer_call_data_t wait_timer;
375 integer_t wait_timer_active;
376 boolean_t wait_timer_is_set;
377
378
379 /*
380 * Processor/cache affinity
381 * - affinity_threads links task threads with the same affinity set
382 */
383 affinity_set_t affinity_set;
384 queue_chain_t affinity_threads;
385
386 /* Various bits of state to stash across a continuation, exclusive to the current thread block point */
387 union {
388 struct {
389 mach_msg_return_t state; /* receive state */
390 mach_port_seqno_t seqno; /* seqno of recvd message */
391 ipc_object_t object; /* object received on */
392 mach_vm_address_t msg_addr; /* receive buffer pointer */
393 mach_msg_size_t rsize; /* max size for recvd msg */
394 mach_msg_size_t msize; /* actual size for recvd msg */
395 mach_msg_option_t option; /* options for receive */
396 mach_port_name_t receiver_name; /* the receive port name */
397 struct knote *knote; /* knote fired for rcv */
398 union {
399 struct ipc_kmsg *kmsg; /* received message */
400 struct ipc_mqueue *peekq; /* mqueue to peek at */
401 struct {
402 mach_msg_priority_t qos; /* received message qos */
403 mach_msg_priority_t oqos; /* override qos for message */
404 } received_qos;
405 };
406 mach_msg_continue_t continuation;
407 } receive;
408 struct {
409 struct semaphore *waitsemaphore; /* semaphore ref */
410 struct semaphore *signalsemaphore; /* semaphore ref */
411 int options; /* semaphore options */
412 kern_return_t result; /* primary result */
413 mach_msg_continue_t continuation;
414 } sema;
415 } saved;
416
417 /* Only user threads can cause guard exceptions, only kernel threads can be thread call threads */
418 union {
419 /* Group and call this thread is working on behalf of */
420 struct {
421 struct thread_call_group * thc_group;
422 struct thread_call * thc_call; /* debug only, may be deallocated */
423 } thc_state;
424
425 /* Structure to save information about guard exception */
426 struct {
427 mach_exception_code_t code;
428 mach_exception_subcode_t subcode;
429 } guard_exc_info;
430 };
431
432 /* Kernel holds on this thread */
433 int16_t suspend_count;
434 /* User level suspensions */
435 int16_t user_stop_count;
436
437 /* IPC data structures */
438#if IMPORTANCE_INHERITANCE
439 natural_t ith_assertions; /* assertions pending drop */
440#endif
441 struct ipc_kmsg_queue ith_messages; /* messages to reap */
442 mach_port_t ith_rpc_reply; /* reply port for kernel RPCs */
443
444 /* Ast/Halt data structures */
445 vm_offset_t recover; /* page fault recover(copyin/out) */
446
447 queue_chain_t threads; /* global list of all threads */
448
449 /* Activation */
450 queue_chain_t task_threads;
451
452 /* Task membership */
453 struct task *task;
454 vm_map_t map;
455#if DEVELOPMENT || DEBUG
456 boolean_t pmap_footprint_suspended;
457#endif /* DEVELOPMENT || DEBUG */
458
459 decl_lck_mtx_data(,mutex)
460
461
462 /* Pending thread ast(s) */
463 ast_t ast;
464
465 /* Miscellaneous bits guarded by mutex */
466 uint32_t
467 active:1, /* Thread is active and has not been terminated */
468 started:1, /* Thread has been started after creation */
469 static_param:1, /* Disallow policy parameter changes */
470 inspection:1, /* TRUE when task is being inspected by crash reporter */
471 policy_reset:1, /* Disallow policy parameter changes on terminating threads */
472 suspend_parked:1, /* thread parked in thread_suspended */
473 corpse_dup:1, /* TRUE when thread is an inactive duplicate in a corpse */
474 :0;
475
476 /* Ports associated with this thread */
477 struct ipc_port *ith_self; /* not a right, doesn't hold ref */
478 struct ipc_port *ith_sself; /* a send right */
479 struct ipc_port *ith_special_reply_port; /* ref to special reply port */
480 struct exception_action *exc_actions;
481
482#ifdef MACH_BSD
483 void *uthread;
484#endif
485
486#if CONFIG_DTRACE
487 uint32_t t_dtrace_flags; /* DTrace thread states */
488#define TH_DTRACE_EXECSUCCESS 0x01
489 uint32_t t_dtrace_predcache;/* DTrace per thread predicate value hint */
490 int64_t t_dtrace_tracing; /* Thread time under dtrace_probe() */
491 int64_t t_dtrace_vtime;
492#endif
493
494 clock_sec_t t_page_creation_time;
495 uint32_t t_page_creation_count;
496 uint32_t t_page_creation_throttled;
497#if (DEVELOPMENT || DEBUG)
498 uint64_t t_page_creation_throttled_hard;
499 uint64_t t_page_creation_throttled_soft;
500#endif /* DEVELOPMENT || DEBUG */
501
502#ifdef KPERF
503/* The high 7 bits are the number of frames to sample of a user callstack. */
504#define T_KPERF_CALLSTACK_DEPTH_OFFSET (25)
505#define T_KPERF_SET_CALLSTACK_DEPTH(DEPTH) (((uint32_t)(DEPTH)) << T_KPERF_CALLSTACK_DEPTH_OFFSET)
506#define T_KPERF_GET_CALLSTACK_DEPTH(FLAGS) ((FLAGS) >> T_KPERF_CALLSTACK_DEPTH_OFFSET)
507#endif
508
509#define T_KPERF_AST_CALLSTACK (1U << 0) /* dump a callstack on thread's next AST */
510#define T_KPERF_AST_DISPATCH (1U << 1) /* dump a name on thread's next AST */
511#define T_KPC_ALLOC (1U << 2) /* thread needs a kpc_buf allocated */
512/* only go up to T_KPERF_CALLSTACK_DEPTH_OFFSET - 1 */
513
514#ifdef KPERF
515 uint32_t kperf_flags;
516 uint32_t kperf_pet_gen; /* last generation of PET that sampled this thread*/
517 uint32_t kperf_c_switch; /* last dispatch detection */
518 uint32_t kperf_pet_cnt; /* how many times a thread has been sampled by PET */
519#endif
520
521#ifdef KPC
522 /* accumulated performance counters for this thread */
523 uint64_t *kpc_buf;
524#endif
525
526#if HYPERVISOR
527 /* hypervisor virtual CPU object associated with this thread */
528 void *hv_thread_target;
529#endif /* HYPERVISOR */
530
531 uint64_t thread_id; /*system wide unique thread-id*/
532
533 /* Statistics accumulated per-thread and aggregated per-task */
534 uint32_t syscalls_unix;
535 uint32_t syscalls_mach;
536 ledger_t t_ledger;
537 ledger_t t_threadledger; /* per thread ledger */
538 ledger_t t_bankledger; /* ledger to charge someone */
539 uint64_t t_deduct_bank_ledger_time; /* cpu time to be deducted from bank ledger */
540 uint64_t t_deduct_bank_ledger_energy; /* energy to be deducted from bank ledger */
541
542#if MONOTONIC
543 struct mt_thread t_monotonic;
544#endif /* MONOTONIC */
545
546 /*** Machine-dependent state ***/
547 struct machine_thread machine;
548
549 /* policy is protected by the thread mutex */
550 struct thread_requested_policy requested_policy;
551 struct thread_effective_policy effective_policy;
552
553 /* usynch override is protected by the task lock, eventually will be thread mutex */
554 struct thread_qos_override {
555 struct thread_qos_override *override_next;
556 uint32_t override_contended_resource_count;
557 int16_t override_qos;
558 int16_t override_resource_type;
559 user_addr_t override_resource;
560 } *overrides;
561
562 uint32_t ipc_overrides;
563 _Atomic uint32_t kqwl_owning_count;
564 uint32_t sync_ipc_overrides;
565 uint16_t user_promotion_basepri;
566 _Atomic uint16_t kevent_ast_bits;
567
568 io_stat_info_t thread_io_stats; /* per-thread I/O statistics */
569
570#if CONFIG_EMBEDDED
571 task_watch_t * taskwatch; /* task watch */
572#endif /* CONFIG_EMBEDDED */
573
574 uint32_t thread_callout_interrupt_wakeups;
575 uint32_t thread_callout_platform_idle_wakeups;
576 uint32_t thread_timer_wakeups_bin_1;
577 uint32_t thread_timer_wakeups_bin_2;
578 uint16_t thread_tag;
579 uint16_t callout_woken_from_icontext:1,
580 callout_woken_from_platform_idle:1,
581 callout_woke_thread:1,
582 thread_bitfield_unused:13;
583
584 mach_port_name_t ith_voucher_name;
585 ipc_voucher_t ith_voucher;
586#if CONFIG_IOSCHED
587 void *decmp_upl;
588#endif /* CONFIG_IOSCHED */
589
590 /* work interval (if any) associated with the thread. Uses thread mutex */
591 struct work_interval *th_work_interval;
592
593#if SCHED_TRACE_THREAD_WAKEUPS
594 uintptr_t thread_wakeup_bt[64];
595#endif
596 turnstile_update_flags_t inheritor_flags; /* inheritor flags for inheritor field */
597 block_hint_t pending_block_hint;
598 block_hint_t block_hint; /* What type of primitive last caused us to block. */
599};
600
601#define ith_state saved.receive.state
602#define ith_object saved.receive.object
603#define ith_msg_addr saved.receive.msg_addr
604#define ith_rsize saved.receive.rsize
605#define ith_msize saved.receive.msize
606#define ith_option saved.receive.option
607#define ith_receiver_name saved.receive.receiver_name
608#define ith_continuation saved.receive.continuation
609#define ith_kmsg saved.receive.kmsg
610#define ith_peekq saved.receive.peekq
611#define ith_knote saved.receive.knote
612#define ith_qos saved.receive.received_qos.qos
613#define ith_qos_override saved.receive.received_qos.oqos
614#define ith_seqno saved.receive.seqno
615
616#define sth_waitsemaphore saved.sema.waitsemaphore
617#define sth_signalsemaphore saved.sema.signalsemaphore
618#define sth_options saved.sema.options
619#define sth_result saved.sema.result
620#define sth_continuation saved.sema.continuation
621
622#define ITH_KNOTE_NULL ((void *)NULL)
623#define ITH_KNOTE_PSEUDO ((void *)0xdeadbeef)
624/*
625 * The ith_knote is used during message delivery, and can safely be interpreted
626 * only when used for one of these codepaths, which the test for the msgt_name
627 * being RECEIVE or SEND_ONCE is about.
628 */
629#define ITH_KNOTE_VALID(kn, msgt_name) \
630 (((kn) != ITH_KNOTE_NULL && (kn) != ITH_KNOTE_PSEUDO) && \
631 ((msgt_name) == MACH_MSG_TYPE_PORT_RECEIVE || \
632 (msgt_name) == MACH_MSG_TYPE_PORT_SEND_ONCE))
633
634#if MACH_ASSERT
635#define assert_thread_magic(thread) assertf((thread)->thread_magic == THREAD_MAGIC, \
636 "bad thread magic 0x%llx for thread %p, expected 0x%llx", \
637 (thread)->thread_magic, (thread), THREAD_MAGIC)
638#else
639#define assert_thread_magic(thread) do { (void)(thread); } while (0)
640#endif
641
642extern void thread_bootstrap(void);
643
644extern void thread_init(void);
645
646extern void thread_daemon_init(void);
647
648#define thread_reference_internal(thread) \
649 (void)atomic_fetch_add_explicit(&(thread)->ref_count, 1, memory_order_relaxed)
650
651#define thread_reference(thread) \
652MACRO_BEGIN \
653 if ((thread) != THREAD_NULL) \
654 thread_reference_internal(thread); \
655MACRO_END
656
657extern void thread_deallocate(
658 thread_t thread);
659
660extern void thread_deallocate_safe(
661 thread_t thread);
662
663extern void thread_inspect_deallocate(
664 thread_inspect_t thread);
665
666extern void thread_terminate_self(void);
667
668extern kern_return_t thread_terminate_internal(
669 thread_t thread);
670
671extern void thread_start(
672 thread_t thread) __attribute__ ((noinline));
673
674extern void thread_start_in_assert_wait(
675 thread_t thread,
676 event_t event,
677 wait_interrupt_t interruptible) __attribute__ ((noinline));
678
679extern void thread_terminate_enqueue(
680 thread_t thread);
681
682extern void thread_exception_enqueue(
683 task_t task,
684 thread_t thread,
685 exception_type_t etype);
686
687extern void thread_copy_resource_info(
688 thread_t dst_thread,
689 thread_t src_thread);
690
691extern void thread_terminate_crashed_threads(void);
692
693extern void turnstile_deallocate_enqueue(
694 struct turnstile *turnstile);
695
696extern void thread_stack_enqueue(
697 thread_t thread);
698
699extern void thread_hold(
700 thread_t thread);
701
702extern void thread_release(
703 thread_t thread);
704
705extern void thread_corpse_continue(void);
706
707extern boolean_t thread_is_active(thread_t thread);
708
709/* Locking for scheduler state, always acquired with interrupts disabled (splsched()) */
710#if __SMP__
711#define thread_lock_init(th) simple_lock_init(&(th)->sched_lock, 0)
712#define thread_lock(th) simple_lock(&(th)->sched_lock)
713#define thread_unlock(th) simple_unlock(&(th)->sched_lock)
714
715#define wake_lock_init(th) simple_lock_init(&(th)->wake_lock, 0)
716#define wake_lock(th) simple_lock(&(th)->wake_lock)
717#define wake_unlock(th) simple_unlock(&(th)->wake_lock)
718#else
719#define thread_lock_init(th) do { (void)th; } while(0)
720#define thread_lock(th) do { (void)th; } while(0)
721#define thread_unlock(th) do { (void)th; } while(0)
722
723#define wake_lock_init(th) do { (void)th; } while(0)
724#define wake_lock(th) do { (void)th; } while(0)
725#define wake_unlock(th) do { (void)th; } while(0)
726#endif
727
728#define thread_should_halt_fast(thread) (!(thread)->active)
729
730extern void stack_alloc(
731 thread_t thread);
732
733extern void stack_handoff(
734 thread_t from,
735 thread_t to);
736
737extern void stack_free(
738 thread_t thread);
739
740extern void stack_free_reserved(
741 thread_t thread);
742
743extern boolean_t stack_alloc_try(
744 thread_t thread);
745
746extern void stack_collect(void);
747
748extern void stack_init(void);
749
750
751extern kern_return_t thread_info_internal(
752 thread_t thread,
753 thread_flavor_t flavor,
754 thread_info_t thread_info_out,
755 mach_msg_type_number_t *thread_info_count);
756
757
758
759extern kern_return_t kernel_thread_create(
760 thread_continue_t continuation,
761 void *parameter,
762 integer_t priority,
763 thread_t *new_thread);
764
765extern kern_return_t kernel_thread_start_priority(
766 thread_continue_t continuation,
767 void *parameter,
768 integer_t priority,
769 thread_t *new_thread);
770
771extern void machine_stack_attach(
772 thread_t thread,
773 vm_offset_t stack);
774
775extern vm_offset_t machine_stack_detach(
776 thread_t thread);
777
778extern void machine_stack_handoff(
779 thread_t old,
780 thread_t new);
781
782extern thread_t machine_switch_context(
783 thread_t old_thread,
784 thread_continue_t continuation,
785 thread_t new_thread);
786
787extern void machine_load_context(
788 thread_t thread) __attribute__((noreturn));
789
790
791extern kern_return_t machine_thread_state_initialize(
792 thread_t thread);
793
794extern kern_return_t machine_thread_set_state(
795 thread_t thread,
796 thread_flavor_t flavor,
797 thread_state_t state,
798 mach_msg_type_number_t count);
799
800extern kern_return_t machine_thread_get_state(
801 thread_t thread,
802 thread_flavor_t flavor,
803 thread_state_t state,
804 mach_msg_type_number_t *count);
805
806extern kern_return_t machine_thread_state_convert_from_user(
807 thread_t thread,
808 thread_flavor_t flavor,
809 thread_state_t tstate,
810 mach_msg_type_number_t count);
811
812extern kern_return_t machine_thread_state_convert_to_user(
813 thread_t thread,
814 thread_flavor_t flavor,
815 thread_state_t tstate,
816 mach_msg_type_number_t *count);
817
818extern kern_return_t machine_thread_dup(
819 thread_t self,
820 thread_t target,
821 boolean_t is_corpse);
822
823extern void machine_thread_init(void);
824
825extern kern_return_t machine_thread_create(
826 thread_t thread,
827 task_t task);
828extern void machine_thread_switch_addrmode(
829 thread_t thread);
830
831extern void machine_thread_destroy(
832 thread_t thread);
833
834extern void machine_set_current_thread(
835 thread_t thread);
836
837extern kern_return_t machine_thread_get_kern_state(
838 thread_t thread,
839 thread_flavor_t flavor,
840 thread_state_t tstate,
841 mach_msg_type_number_t *count);
842
843extern kern_return_t machine_thread_inherit_taskwide(
844 thread_t thread,
845 task_t parent_task);
846
847extern kern_return_t machine_thread_set_tsd_base(
848 thread_t thread,
849 mach_vm_offset_t tsd_base);
850
851#define thread_mtx_lock(thread) lck_mtx_lock(&(thread)->mutex)
852#define thread_mtx_try(thread) lck_mtx_try_lock(&(thread)->mutex)
853#define thread_mtx_unlock(thread) lck_mtx_unlock(&(thread)->mutex)
854
855extern void thread_apc_ast(thread_t thread);
856
857extern void thread_update_qos_cpu_time(thread_t thread);
858
859void act_machine_sv_free(thread_t, int);
860
861vm_offset_t min_valid_stack_address(void);
862vm_offset_t max_valid_stack_address(void);
863
864static inline uint16_t thread_set_tag_internal(thread_t thread, uint16_t tag) {
865 return __sync_fetch_and_or(&thread->thread_tag, tag);
866}
867
868static inline uint16_t thread_get_tag_internal(thread_t thread) {
869 return thread->thread_tag;
870}
871
872
873extern void thread_set_options(uint32_t thopt);
874
875
876#else /* MACH_KERNEL_PRIVATE */
877
878__BEGIN_DECLS
879
880extern void thread_mtx_lock(thread_t thread);
881
882extern void thread_mtx_unlock(thread_t thread);
883
884extern thread_t current_thread(void);
885
886extern void thread_reference(
887 thread_t thread);
888
889extern void thread_deallocate(
890 thread_t thread);
891
892__END_DECLS
893
894#endif /* MACH_KERNEL_PRIVATE */
895
896#ifdef KERNEL_PRIVATE
897
898__BEGIN_DECLS
899
900extern void thread_starts_owning_workloop(
901 thread_t thread);
902
903extern void thread_ends_owning_workloop(
904 thread_t thread);
905
906extern uint32_t thread_owned_workloops_count(
907 thread_t thread);
908
909
910extern uint64_t thread_dispatchqaddr(
911 thread_t thread);
912
913extern uint64_t thread_rettokern_addr(
914 thread_t thread);
915
916__END_DECLS
917
918#endif /* KERNEL_PRIVATE */
919
920#ifdef KERNEL
921__BEGIN_DECLS
922
923extern uint64_t thread_tid(thread_t thread);
924
925__END_DECLS
926
927#endif /* KERNEL */
928
929__BEGIN_DECLS
930
931#ifdef XNU_KERNEL_PRIVATE
932
933/*
934 * Thread tags; for easy identification.
935 */
936#define THREAD_TAG_MAINTHREAD 0x1
937#define THREAD_TAG_CALLOUT 0x2
938#define THREAD_TAG_IOWORKLOOP 0x4
939
940#define THREAD_TAG_PTHREAD 0x10
941#define THREAD_TAG_WORKQUEUE 0x20
942
943uint16_t thread_set_tag(thread_t, uint16_t);
944uint16_t thread_get_tag(thread_t);
945uint64_t thread_last_run_time(thread_t);
946
947extern kern_return_t thread_state_initialize(
948 thread_t thread);
949
950extern kern_return_t thread_setstatus(
951 thread_t thread,
952 int flavor,
953 thread_state_t tstate,
954 mach_msg_type_number_t count);
955
956extern kern_return_t thread_setstatus_from_user(
957 thread_t thread,
958 int flavor,
959 thread_state_t tstate,
960 mach_msg_type_number_t count);
961
962extern kern_return_t thread_getstatus(
963 thread_t thread,
964 int flavor,
965 thread_state_t tstate,
966 mach_msg_type_number_t *count);
967
968extern kern_return_t thread_getstatus_to_user(
969 thread_t thread,
970 int flavor,
971 thread_state_t tstate,
972 mach_msg_type_number_t *count);
973
974extern kern_return_t thread_create_with_continuation(
975 task_t task,
976 thread_t *new_thread,
977 thread_continue_t continuation);
978
979extern kern_return_t thread_create_waiting(task_t task,
980 thread_continue_t continuation,
981 event_t event,
982 thread_t *new_thread);
983
984extern kern_return_t thread_create_workq_waiting(
985 task_t task,
986 thread_continue_t thread_return,
987 thread_t *new_thread);
988
989extern void thread_yield_internal(
990 mach_msg_timeout_t interval);
991
992extern void thread_yield_to_preemption(void);
993
994/*
995 * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
996 *
997 * 1) Block. Prevent CPU consumption of the thread from exceeding the limit.
998 * 2) Exception. Generate a resource consumption exception when the limit is exceeded.
999 * 3) Disable. Remove any existing CPU limit.
1000 */
1001#define THREAD_CPULIMIT_BLOCK 0x1
1002#define THREAD_CPULIMIT_EXCEPTION 0x2
1003#define THREAD_CPULIMIT_DISABLE 0x3
1004
1005struct _thread_ledger_indices {
1006 int cpu_time;
1007};
1008
1009extern struct _thread_ledger_indices thread_ledgers;
1010
1011extern int thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns);
1012extern int thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns);
1013
1014extern void thread_read_times(
1015 thread_t thread,
1016 time_value_t *user_time,
1017 time_value_t *system_time,
1018 time_value_t *runnable_time);
1019
1020extern uint64_t thread_get_runtime_self(void);
1021
1022extern void thread_setuserstack(
1023 thread_t thread,
1024 mach_vm_offset_t user_stack);
1025
1026extern uint64_t thread_adjuserstack(
1027 thread_t thread,
1028 int adjust);
1029
1030extern void thread_setentrypoint(
1031 thread_t thread,
1032 mach_vm_offset_t entry);
1033
1034extern kern_return_t thread_set_tsd_base(
1035 thread_t thread,
1036 mach_vm_offset_t tsd_base);
1037
1038extern kern_return_t thread_setsinglestep(
1039 thread_t thread,
1040 int on);
1041
1042extern kern_return_t thread_userstack(
1043 thread_t,
1044 int,
1045 thread_state_t,
1046 unsigned int,
1047 mach_vm_offset_t *,
1048 int *,
1049 boolean_t);
1050
1051extern kern_return_t thread_entrypoint(
1052 thread_t,
1053 int,
1054 thread_state_t,
1055 unsigned int,
1056 mach_vm_offset_t *);
1057
1058extern kern_return_t thread_userstackdefault(
1059 mach_vm_offset_t *,
1060 boolean_t);
1061
1062extern kern_return_t thread_wire_internal(
1063 host_priv_t host_priv,
1064 thread_t thread,
1065 boolean_t wired,
1066 boolean_t *prev_state);
1067
1068
1069extern kern_return_t thread_dup(thread_t);
1070
1071extern kern_return_t thread_dup2(thread_t, thread_t);
1072
1073#if !defined(_SCHED_CALL_T_DEFINED)
1074#define _SCHED_CALL_T_DEFINED
1075typedef void (*sched_call_t)(
1076 int type,
1077 thread_t thread);
1078#endif
1079
1080#define SCHED_CALL_BLOCK 0x1
1081#define SCHED_CALL_UNBLOCK 0x2
1082
1083extern void thread_sched_call(
1084 thread_t thread,
1085 sched_call_t call);
1086
1087extern boolean_t thread_is_static_param(
1088 thread_t thread);
1089
1090extern task_t get_threadtask(thread_t);
1091
1092/*
1093 * Thread is running within a 64-bit address space.
1094 */
1095#define thread_is_64bit_addr(thd) \
1096 task_has_64Bit_addr(get_threadtask(thd))
1097
1098/*
1099 * Thread is using 64-bit machine state.
1100 */
1101#define thread_is_64bit_data(thd) \
1102 task_has_64Bit_data(get_threadtask(thd))
1103
1104extern void *get_bsdthread_info(thread_t);
1105extern void set_bsdthread_info(thread_t, void *);
1106extern void *uthread_alloc(task_t, thread_t, int);
1107extern event_t workq_thread_init_and_wq_lock(task_t, thread_t); // bsd/pthread/
1108extern void uthread_cleanup_name(void *uthread);
1109extern void uthread_cleanup(task_t, void *, void *);
1110extern void uthread_zone_free(void *);
1111extern void uthread_cred_free(void *);
1112
1113extern void uthread_reset_proc_refcount(void *);
1114#if PROC_REF_DEBUG
1115extern int uthread_get_proc_refcount(void *);
1116extern int proc_ref_tracking_disabled;
1117#endif
1118
1119extern boolean_t thread_should_halt(
1120 thread_t thread);
1121
1122extern boolean_t thread_should_abort(
1123 thread_t);
1124
1125extern int is_64signalregset(void);
1126
1127extern void act_set_kperf(thread_t);
1128extern void act_set_astledger(thread_t thread);
1129extern void act_set_astledger_async(thread_t thread);
1130extern void act_set_io_telemetry_ast(thread_t);
1131
1132extern uint32_t dtrace_get_thread_predcache(thread_t);
1133extern int64_t dtrace_get_thread_vtime(thread_t);
1134extern int64_t dtrace_get_thread_tracing(thread_t);
1135extern boolean_t dtrace_get_thread_reentering(thread_t);
1136extern int dtrace_get_thread_last_cpu_id(thread_t);
1137extern vm_offset_t dtrace_get_kernel_stack(thread_t);
1138extern void dtrace_set_thread_predcache(thread_t, uint32_t);
1139extern void dtrace_set_thread_vtime(thread_t, int64_t);
1140extern void dtrace_set_thread_tracing(thread_t, int64_t);
1141extern void dtrace_set_thread_reentering(thread_t, boolean_t);
1142extern vm_offset_t dtrace_set_thread_recover(thread_t, vm_offset_t);
1143extern void dtrace_thread_bootstrap(void);
1144extern void dtrace_thread_didexec(thread_t);
1145
1146extern int64_t dtrace_calc_thread_recent_vtime(thread_t);
1147
1148
1149extern kern_return_t thread_set_wq_state32(
1150 thread_t thread,
1151 thread_state_t tstate);
1152
1153extern kern_return_t thread_set_wq_state64(
1154 thread_t thread,
1155 thread_state_t tstate);
1156
1157extern vm_offset_t kernel_stack_mask;
1158extern vm_offset_t kernel_stack_size;
1159extern vm_offset_t kernel_stack_depth_max;
1160
1161extern void guard_ast(thread_t);
1162extern void fd_guard_ast(thread_t,
1163 mach_exception_code_t, mach_exception_subcode_t);
1164#if CONFIG_VNGUARD
1165extern void vn_guard_ast(thread_t,
1166 mach_exception_code_t, mach_exception_subcode_t);
1167#endif
1168extern void mach_port_guard_ast(thread_t,
1169 mach_exception_code_t, mach_exception_subcode_t);
1170extern void virt_memory_guard_ast(thread_t,
1171 mach_exception_code_t, mach_exception_subcode_t);
1172extern void thread_guard_violation(thread_t,
1173 mach_exception_code_t, mach_exception_subcode_t);
1174extern void thread_update_io_stats(thread_t, int size, int io_flags);
1175
1176extern kern_return_t thread_set_voucher_name(mach_port_name_t name);
1177extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid);
1178
1179extern void set_thread_rwlock_boost(void);
1180extern void clear_thread_rwlock_boost(void);
1181
1182/*! @function thread_has_thread_name
1183 @abstract Checks if a thread has a name.
1184 @discussion This function takes one input, a thread, and returns a boolean value indicating if that thread already has a name associated with it.
1185 @param th The thread to inspect.
1186 @result TRUE if the thread has a name, FALSE otherwise.
1187*/
1188extern boolean_t thread_has_thread_name(thread_t th);
1189
1190/*! @function thread_set_thread_name
1191 @abstract Set a thread's name.
1192 @discussion This function takes two input parameters: a thread to name, and the name to apply to the thread. The name will be attached to the thread in order to better identify the thread.
1193 @param th The thread to be named.
1194 @param name The name to apply to the thread.
1195*/
1196extern void thread_set_thread_name(thread_t th, const char* name);
1197
1198extern void thread_enable_send_importance(thread_t thread, boolean_t enable);
1199
1200/*
1201 * Translate signal context data pointer to userspace representation
1202 */
1203
1204extern kern_return_t machine_thread_siguctx_pointer_convert_to_user(
1205 thread_t thread,
1206 user_addr_t *uctxp);
1207
1208/*
1209 * Translate array of function pointer syscall arguments from userspace representation
1210 */
1211
1212extern kern_return_t machine_thread_function_pointers_convert_from_user(
1213 thread_t thread,
1214 user_addr_t *fptrs,
1215 uint32_t count);
1216
1217/* Get a backtrace for a threads kernel or user stack (user_p), with pc and optionally
1218 * frame pointer (getfp). Returns bytes added to buffer, and kThreadTruncatedBT in
1219 * thread_trace_flags if a user page is not present after kdp_lightweight_fault() is
1220 * called.
1221 */
1222
1223extern int machine_trace_thread(
1224 thread_t thread,
1225 char *tracepos,
1226 char *tracebound,
1227 int nframes,
1228 boolean_t user_p,
1229 boolean_t getfp,
1230 uint32_t *thread_trace_flags);
1231
1232extern int machine_trace_thread64(thread_t thread,
1233 char *tracepos,
1234 char *tracebound,
1235 int nframes,
1236 boolean_t user_p,
1237 boolean_t getfp,
1238 uint32_t *thread_trace_flags,
1239 uint64_t *sp);
1240
1241/*
1242 * Get the duration of the given thread's last wait.
1243 */
1244uint64_t thread_get_last_wait_duration(thread_t thread);
1245
1246#endif /* XNU_KERNEL_PRIVATE */
1247
1248
1249/*! @function kernel_thread_start
1250 @abstract Create a kernel thread.
1251 @discussion This function takes three input parameters, namely reference to the function that the thread should execute, caller specified data and a reference which is used to return the newly created kernel thread. The function returns KERN_SUCCESS on success or an appropriate kernel code type indicating the error. It may be noted that the caller is responsible for explicitly releasing the reference to the created thread when no longer needed. This should be done by calling thread_deallocate(new_thread).
1252 @param continuation A C-function pointer where the thread will begin execution.
1253 @param parameter Caller specified data to be passed to the new thread.
1254 @param new_thread Reference to the new thread is returned in this parameter.
1255 @result Returns KERN_SUCCESS on success or an appropriate kernel code type.
1256*/
1257
1258extern kern_return_t kernel_thread_start(
1259 thread_continue_t continuation,
1260 void *parameter,
1261 thread_t *new_thread);
1262
1263#ifdef KERNEL_PRIVATE
1264void thread_set_eager_preempt(thread_t thread);
1265void thread_clear_eager_preempt(thread_t thread);
1266void thread_set_honor_qlimit(thread_t thread);
1267void thread_clear_honor_qlimit(thread_t thread);
1268extern ipc_port_t convert_thread_to_port(thread_t);
1269extern ipc_port_t convert_thread_inspect_to_port(thread_inspect_t);
1270extern boolean_t is_vm_privileged(void);
1271extern boolean_t set_vm_privilege(boolean_t);
1272extern kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name);
1273#endif /* KERNEL_PRIVATE */
1274
1275__END_DECLS
1276
1277#endif /* _KERN_THREAD_H_ */
1278