1 | /* |
2 | * Copyright (c) 2014 Apple Computer, Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | #ifndef _WORKQUEUE_INTERNAL_H_ |
30 | #define _WORKQUEUE_INTERNAL_H_ |
31 | |
32 | // Sometimes something gets passed a bucket number and we need a way to express |
33 | // that it's actually the event manager. Use the (0)th bucket for that. |
34 | #define WORKQ_THREAD_QOS_MIN (THREAD_QOS_MAINTENANCE) |
35 | #define WORKQ_THREAD_QOS_MAX (THREAD_QOS_LAST) |
36 | #define WORKQ_THREAD_QOS_CLEANUP (THREAD_QOS_LEGACY) |
37 | #define WORKQ_THREAD_QOS_ABOVEUI (THREAD_QOS_LAST) |
38 | #define WORKQ_THREAD_QOS_MANAGER (THREAD_QOS_LAST + 1) // outside of MIN/MAX |
39 | |
40 | #define WORKQ_NUM_QOS_BUCKETS (WORKQ_THREAD_QOS_MAX - 1) // MT/BG shared |
41 | #define WORKQ_NUM_BUCKETS (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr |
42 | |
43 | /* These definitions are only available to the kext, to avoid bleeding |
44 | * constants and types across the boundary to the userspace library. |
45 | */ |
46 | #ifdef KERNEL |
47 | #pragma mark wq structs |
48 | |
49 | /* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get |
50 | * exported to kernel extensions. |
51 | */ |
52 | #define SCHED_CALL_BLOCK 0x1 |
53 | #define SCHED_CALL_UNBLOCK 0x2 |
54 | |
55 | /* old workq priority scheme */ |
56 | |
57 | #define WORKQUEUE_HIGH_PRIOQUEUE 0 /* high priority queue */ |
58 | #define WORKQUEUE_DEFAULT_PRIOQUEUE 1 /* default priority queue */ |
59 | #define WORKQUEUE_LOW_PRIOQUEUE 2 /* low priority queue */ |
60 | #define WORKQUEUE_BG_PRIOQUEUE 3 /* background priority queue */ |
61 | |
62 | /* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR) |
63 | * This used to be WORKQ_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding |
64 | * N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should |
65 | * not change. So the factor is now always 5. |
66 | */ |
67 | #define WORKQUEUE_CONSTRAINED_FACTOR 5 |
68 | |
69 | #if BSD_KERNEL_PRIVATE |
70 | #include <kern/priority_queue.h> |
71 | #include <kern/thread_call.h> |
72 | #include <kern/turnstile.h> |
73 | #include <mach/kern_return.h> |
74 | #include <sys/queue.h> |
75 | #include <sys/kernel_types.h> |
76 | |
77 | /* struct uthread::uu_workq_flags */ |
78 | #define UT_WORKQ_NEW 0x01 /* First return to userspace */ |
79 | #define UT_WORKQ_RUNNING 0x02 /* On thrunlist, not parked. */ |
80 | #define UT_WORKQ_DYING 0x04 /* Thread is being killed */ |
81 | #define UT_WORKQ_OVERCOMMIT 0x08 /* Overcommit thread. */ |
82 | #define UT_WORKQ_OUTSIDE_QOS 0x10 /* Thread should avoid send QoS changes to kernel */ |
83 | #define UT_WORKQ_IDLE_CLEANUP 0x20 /* Thread is removing its voucher or stack */ |
84 | #define UT_WORKQ_EARLY_BOUND 0x40 /* Thread has been bound early */ |
85 | #define UT_WORKQ_CPUPERCENT 0x80 /* Thread has CPU percent policy active */ |
86 | |
87 | typedef union workq_threadreq_param_s { |
88 | struct { |
89 | uint16_t trp_flags; |
90 | uint8_t trp_pri; |
91 | uint8_t trp_pol; |
92 | uint32_t trp_cpupercent: 8, |
93 | trp_refillms: 24; |
94 | }; |
95 | uint64_t trp_value; |
96 | } workq_threadreq_param_t; |
97 | |
98 | #define TRP_PRIORITY 0x1 |
99 | #define TRP_POLICY 0x2 |
100 | #define TRP_CPUPERCENT 0x4 |
101 | #define TRP_RELEASED 0x8000 |
102 | |
103 | typedef struct workq_threadreq_s { |
104 | union { |
105 | struct priority_queue_entry tr_entry; |
106 | thread_t tr_binding_thread; |
107 | }; |
108 | uint32_t tr_flags; |
109 | uint8_t tr_state; |
110 | thread_qos_t tr_qos; |
111 | uint16_t tr_count; |
112 | } *workq_threadreq_t; |
113 | |
114 | TAILQ_HEAD(threadreq_head, workq_threadreq_s); |
115 | |
116 | #define TR_STATE_IDLE 0 /* request isn't in flight */ |
117 | #define TR_STATE_NEW 1 /* request is being initiated */ |
118 | #define TR_STATE_QUEUED 2 /* request is being queued */ |
119 | #define TR_STATE_BINDING 4 /* request is preposted for bind */ |
120 | |
121 | #define TR_FLAG_KEVENT 0x01 |
122 | #define TR_FLAG_WORKLOOP 0x02 |
123 | #define TR_FLAG_OVERCOMMIT 0x04 |
124 | #define TR_FLAG_WL_PARAMS 0x08 |
125 | #define TR_FLAG_WL_OUTSIDE_QOS 0x10 |
126 | |
127 | #if defined(__LP64__) |
128 | typedef unsigned __int128 wq_thactive_t; |
129 | #else |
130 | typedef uint64_t wq_thactive_t; |
131 | #endif |
132 | |
133 | typedef enum { |
134 | WQ_EXITING = 0x0001, |
135 | WQ_PROC_SUSPENDED = 0x0002, |
136 | WQ_DEATH_CALL_SCHEDULED = 0x0004, |
137 | |
138 | WQ_DELAYED_CALL_SCHEDULED = 0x0010, |
139 | WQ_DELAYED_CALL_PENDED = 0x0020, |
140 | WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040, |
141 | WQ_IMMEDIATE_CALL_PENDED = 0x0080, |
142 | } workq_state_flags_t; |
143 | |
144 | TAILQ_HEAD(workq_uthread_head, uthread); |
145 | |
146 | struct workqueue { |
147 | thread_call_t wq_delayed_call; |
148 | thread_call_t wq_immediate_call; |
149 | thread_call_t wq_death_call; |
150 | struct turnstile *wq_turnstile; |
151 | |
152 | lck_spin_t wq_lock; |
153 | |
154 | uint64_t wq_thread_call_last_run; |
155 | struct os_refcnt wq_refcnt; |
156 | workq_state_flags_t _Atomic wq_flags; |
157 | uint32_t wq_fulfilled; |
158 | uint32_t wq_creations; |
159 | uint32_t wq_timer_interval; |
160 | uint32_t wq_event_manager_priority; |
161 | uint32_t wq_reqcount; /* number of elements on the wq_*_reqlists */ |
162 | uint16_t wq_thdying_count; |
163 | uint16_t wq_threads_scheduled; |
164 | uint16_t wq_constrained_threads_scheduled; |
165 | uint16_t wq_nthreads; |
166 | uint16_t wq_thidlecount; |
167 | uint16_t wq_thscheduled_count[WORKQ_NUM_BUCKETS]; // incl. manager |
168 | |
169 | _Atomic wq_thactive_t wq_thactive; |
170 | _Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS]; |
171 | |
172 | struct proc *wq_proc; |
173 | struct uthread *wq_creator; |
174 | thread_t wq_turnstile_updater; // thread doing a turnstile_update_ineritor |
175 | struct workq_uthread_head wq_thrunlist; |
176 | struct workq_uthread_head wq_thnewlist; |
177 | struct workq_uthread_head wq_thidlelist; |
178 | |
179 | struct priority_queue wq_overcommit_queue; |
180 | struct priority_queue wq_constrained_queue; |
181 | struct priority_queue wq_special_queue; |
182 | workq_threadreq_t wq_event_manager_threadreq; |
183 | }; |
184 | |
185 | static_assert(offsetof(struct workqueue, wq_lock) >= sizeof(struct queue_entry), |
186 | "Make sure workq_deallocate_enqueue can cast the workqueue" ); |
187 | |
188 | #define WORKQUEUE_MAXTHREADS 512 |
189 | #define WQ_STALLED_WINDOW_USECS 200 |
190 | #define WQ_REDUCE_POOL_WINDOW_USECS 5000000 |
191 | #define WQ_MAX_TIMER_INTERVAL_USECS 50000 |
192 | |
193 | #pragma mark definitions |
194 | |
195 | struct kqrequest; |
196 | uint32_t _get_pwq_state_kdp(proc_t p); |
197 | |
198 | void workq_exit(struct proc *p); |
199 | void workq_mark_exiting(struct proc *p); |
200 | |
201 | bool workq_is_exiting(struct proc *p); |
202 | |
203 | struct turnstile *workq_turnstile(struct proc *p); |
204 | |
205 | void workq_thread_set_max_qos(struct proc *p, struct kqrequest *kqr); |
206 | |
207 | void workq_thread_terminate(struct proc *p, struct uthread *uth); |
208 | |
209 | #define WORKQ_THREADREQ_SET_AST_ON_FAILURE 0x01 |
210 | #define WORKQ_THREADREQ_ATTEMPT_REBIND 0x02 |
211 | #define WORKQ_THREADREQ_CAN_CREATE_THREADS 0x04 |
212 | #define WORKQ_THREADREQ_CREATOR_TRANSFER 0x08 |
213 | #define WORKQ_THREADREQ_CREATOR_SYNC_UPDATE 0x10 |
214 | |
215 | // called with the kq req lock held |
216 | bool workq_kern_threadreq_initiate(struct proc *p, struct kqrequest *kqr, |
217 | struct turnstile *ts, thread_qos_t qos, int flags); |
218 | |
219 | // called with the kq req lock held |
220 | void workq_kern_threadreq_modify(struct proc *p, struct kqrequest *kqr, |
221 | thread_qos_t qos, int flags); |
222 | |
223 | // called with the kq req lock held |
224 | void workq_kern_threadreq_update_inheritor(struct proc *p, struct kqrequest *kqr, |
225 | thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags); |
226 | |
227 | void workq_kern_threadreq_lock(struct proc *p); |
228 | void workq_kern_threadreq_unlock(struct proc *p); |
229 | |
230 | void workq_kern_threadreq_redrive(struct proc *p, int flags); |
231 | |
232 | enum workq_set_self_flags { |
233 | WORKQ_SET_SELF_QOS_FLAG = 0x1, |
234 | WORKQ_SET_SELF_VOUCHER_FLAG = 0x2, |
235 | WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x4, |
236 | WORKQ_SET_SELF_TIMESHARE_FLAG = 0x8, |
237 | WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10, |
238 | }; |
239 | |
240 | void workq_proc_suspended(struct proc *p); |
241 | void workq_proc_resumed(struct proc *p); |
242 | |
243 | #endif // BSD_KERNEL_PRIVATE |
244 | |
245 | void workq_init(void); |
246 | |
247 | #endif // KERNEL |
248 | |
249 | #endif // _WORKQUEUE_INTERNAL_H_ |
250 | |