1 | /* |
2 | * Copyright (c) 2000-2009 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | /* |
57 | */ |
58 | |
59 | /* |
60 | * processor.h: Processor and processor-related definitions. |
61 | */ |
62 | |
63 | #ifndef _KERN_PROCESSOR_H_ |
64 | #define _KERN_PROCESSOR_H_ |
65 | |
66 | #include <mach/boolean.h> |
67 | #include <mach/kern_return.h> |
68 | #include <kern/kern_types.h> |
69 | |
70 | #include <sys/cdefs.h> |
71 | |
72 | #ifdef MACH_KERNEL_PRIVATE |
73 | |
74 | #include <mach/mach_types.h> |
75 | #include <kern/ast.h> |
76 | #include <kern/cpu_number.h> |
77 | #include <kern/smp.h> |
78 | #include <kern/simple_lock.h> |
79 | #include <kern/locks.h> |
80 | #include <kern/queue.h> |
81 | #include <kern/sched.h> |
82 | #include <mach/sfi_class.h> |
83 | #include <kern/processor_data.h> |
84 | #include <kern/cpu_quiesce.h> |
85 | |
86 | /* |
87 | * Processor state is accessed by locking the scheduling lock |
88 | * for the assigned processor set. |
89 | * |
90 | * -------------------- SHUTDOWN |
91 | * / ^ ^ |
92 | * _/ | \ |
93 | * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING |
94 | * \_________________^ ^ ^______/ / |
95 | * \__________________/ |
96 | * |
97 | * Most of these state transitions are externally driven as a |
98 | * a directive (for instance telling an IDLE processor to start |
99 | * coming out of the idle state to run a thread). However these |
100 | * are typically paired with a handshake by the processor itself |
101 | * to indicate that it has completed a transition of indeterminate |
102 | * length (for example, the DISPATCHING->RUNNING or START->RUNNING |
103 | * transitions must occur on the processor itself). |
104 | * |
105 | * The boot processor has some special cases, and skips the START state, |
106 | * since it has already bootstrapped and is ready to context switch threads. |
107 | * |
108 | * When a processor is in DISPATCHING or RUNNING state, the current_pri, |
109 | * current_thmode, and deadline fields should be set, so that other |
110 | * processors can evaluate if it is an appropriate candidate for preemption. |
111 | */ |
112 | #if defined(CONFIG_SCHED_DEFERRED_AST) |
113 | /* |
114 | * -------------------- SHUTDOWN |
115 | * / ^ ^ |
116 | * _/ | \ |
117 | * OFF_LINE ---> START ---> RUNNING ---> IDLE ---> DISPATCHING |
118 | * \_________________^ ^ ^______/ ^_____ / / |
119 | * \__________________/ |
120 | * |
121 | * A DISPATCHING processor may be put back into IDLE, if another |
122 | * processor determines that the target processor will have nothing to do |
123 | * upon reaching the RUNNING state. This is racy, but if the target |
124 | * responds and becomes RUNNING, it will not break the processor state |
125 | * machine. |
126 | * |
127 | * This change allows us to cancel an outstanding signal/AST on a processor |
128 | * (if such an operation is supported through hardware or software), and |
129 | * push the processor back into the IDLE state as a power optimization. |
130 | */ |
131 | #endif |
132 | |
133 | #define PROCESSOR_OFF_LINE 0 /* Not available */ |
134 | #define PROCESSOR_SHUTDOWN 1 /* Going off-line */ |
135 | #define PROCESSOR_START 2 /* Being started */ |
136 | /* 3 Formerly Inactive (unavailable) */ |
137 | #define PROCESSOR_IDLE 4 /* Idle (available) */ |
138 | #define PROCESSOR_DISPATCHING 5 /* Dispatching (idle -> active) */ |
139 | #define PROCESSOR_RUNNING 6 /* Normal execution */ |
140 | #define PROCESSOR_STATE_LEN (PROCESSOR_RUNNING+1) |
141 | |
142 | typedef enum { |
143 | PSET_SMP, |
144 | } pset_cluster_type_t; |
145 | |
146 | typedef bitmap_t cpumap_t; |
147 | |
148 | struct processor_set { |
149 | int online_processor_count; |
150 | int load_average; |
151 | |
152 | int cpu_set_low, cpu_set_hi; |
153 | int cpu_set_count; |
154 | int last_chosen; |
155 | cpumap_t cpu_bitmask; |
156 | cpumap_t recommended_bitmask; |
157 | cpumap_t cpu_state_map[PROCESSOR_STATE_LEN]; |
158 | cpumap_t primary_map; |
159 | |
160 | #if __SMP__ |
161 | decl_simple_lock_data(,sched_lock) /* lock for above */ |
162 | #endif |
163 | |
164 | #if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ) |
165 | struct run_queue pset_runq; /* runq for this processor set */ |
166 | #endif |
167 | struct rt_queue rt_runq; /* realtime runq for this processor set */ |
168 | |
169 | #if defined(CONFIG_SCHED_TRADITIONAL) |
170 | int pset_runq_bound_count; |
171 | /* # of threads in runq bound to any processor in pset */ |
172 | #endif |
173 | |
174 | /* CPUs that have been sent an unacknowledged remote AST for scheduling purposes */ |
175 | cpumap_t pending_AST_cpu_mask; |
176 | #if defined(CONFIG_SCHED_DEFERRED_AST) |
177 | /* |
178 | * A separate mask, for ASTs that we may be able to cancel. This is dependent on |
179 | * some level of support for requesting an AST on a processor, and then quashing |
180 | * that request later. |
181 | * |
182 | * The purpose of this field (and the associated codepaths) is to infer when we |
183 | * no longer need a processor that is DISPATCHING to come up, and to prevent it |
184 | * from coming out of IDLE if possible. This should serve to decrease the number |
185 | * of spurious ASTs in the system, and let processors spend longer periods in |
186 | * IDLE. |
187 | */ |
188 | cpumap_t pending_deferred_AST_cpu_mask; |
189 | #endif |
190 | cpumap_t pending_spill_cpu_mask; |
191 | |
192 | struct ipc_port * pset_self; /* port for operations */ |
193 | struct ipc_port * pset_name_self; /* port for information */ |
194 | |
195 | processor_set_t pset_list; /* chain of associated psets */ |
196 | pset_node_t node; |
197 | uint32_t pset_cluster_id; |
198 | pset_cluster_type_t pset_cluster_type; |
199 | }; |
200 | |
201 | extern struct processor_set pset0; |
202 | |
203 | struct pset_node { |
204 | processor_set_t psets; /* list of associated psets */ |
205 | |
206 | pset_node_t nodes; /* list of associated subnodes */ |
207 | pset_node_t node_list; /* chain of associated nodes */ |
208 | |
209 | pset_node_t parent; |
210 | }; |
211 | |
212 | extern struct pset_node pset_node0; |
213 | |
214 | extern queue_head_t tasks, terminated_tasks, threads, corpse_tasks; /* Terminated tasks are ONLY for stackshot */ |
215 | extern int tasks_count, terminated_tasks_count, threads_count; |
216 | decl_lck_mtx_data(extern,tasks_threads_lock) |
217 | decl_lck_mtx_data(extern,tasks_corpse_lock) |
218 | |
219 | struct processor { |
220 | int state; /* See above */ |
221 | bool is_SMT; |
222 | bool is_recommended; |
223 | struct thread *active_thread; /* thread running on processor */ |
224 | struct thread *next_thread; /* next thread when dispatched */ |
225 | struct thread *idle_thread; /* this processor's idle thread. */ |
226 | |
227 | processor_set_t processor_set; /* assigned set */ |
228 | |
229 | int current_pri; /* priority of current thread */ |
230 | sfi_class_id_t current_sfi_class; /* SFI class of current thread */ |
231 | perfcontrol_class_t current_perfctl_class; /* Perfcontrol class for current thread */ |
232 | int starting_pri; /* priority of current thread as it was when scheduled */ |
233 | pset_cluster_type_t current_recommended_pset_type; /* Cluster type recommended for current thread */ |
234 | int cpu_id; /* platform numeric id */ |
235 | cpu_quiescent_state_t cpu_quiesce_state; |
236 | uint64_t cpu_quiesce_last_checkin; |
237 | |
238 | timer_call_data_t quantum_timer; /* timer for quantum expiration */ |
239 | uint64_t quantum_end; /* time when current quantum ends */ |
240 | uint64_t last_dispatch; /* time of last dispatch */ |
241 | |
242 | uint64_t kperf_last_sample_time; /* time of last kperf sample */ |
243 | |
244 | uint64_t deadline; /* current deadline */ |
245 | bool first_timeslice; /* has the quantum expired since context switch */ |
246 | |
247 | #if defined(CONFIG_SCHED_TRADITIONAL) || defined(CONFIG_SCHED_MULTIQ) |
248 | struct run_queue runq; /* runq for this processor */ |
249 | #endif |
250 | |
251 | #if defined(CONFIG_SCHED_TRADITIONAL) |
252 | int runq_bound_count; /* # of threads bound to this processor */ |
253 | #endif |
254 | #if defined(CONFIG_SCHED_GRRR) |
255 | struct grrr_run_queue grrr_runq; /* Group Ratio Round-Robin runq */ |
256 | #endif |
257 | |
258 | processor_t processor_primary; /* pointer to primary processor for |
259 | * secondary SMT processors, or a pointer |
260 | * to ourselves for primaries or non-SMT */ |
261 | processor_t processor_secondary; |
262 | struct ipc_port * processor_self; /* port for operations */ |
263 | |
264 | processor_t processor_list; /* all existing processors */ |
265 | processor_data_t processor_data; /* per-processor data */ |
266 | }; |
267 | |
268 | extern processor_t processor_list; |
269 | decl_simple_lock_data(extern,processor_list_lock) |
270 | |
271 | #define MAX_SCHED_CPUS 64 /* Maximum number of CPUs supported by the scheduler. bits.h:bitmap_*() macros need to be used to support greater than 64 */ |
272 | extern processor_t processor_array[MAX_SCHED_CPUS]; /* array indexed by cpuid */ |
273 | |
274 | extern uint32_t processor_avail_count; |
275 | |
276 | extern processor_t master_processor; |
277 | |
278 | extern boolean_t sched_stats_active; |
279 | |
280 | extern processor_t current_processor(void); |
281 | |
282 | /* Lock macros, always acquired and released with interrupts disabled (splsched()) */ |
283 | |
284 | #if __SMP__ |
285 | #define pset_lock(p) simple_lock(&(p)->sched_lock) |
286 | #define pset_unlock(p) simple_unlock(&(p)->sched_lock) |
287 | #define pset_lock_init(p) simple_lock_init(&(p)->sched_lock, 0) |
288 | #if defined(__arm__) || defined(__arm64__) |
289 | #define pset_assert_locked(p) LCK_SPIN_ASSERT(&(p)->sched_lock, LCK_ASSERT_OWNED) |
290 | #else |
291 | /* See <rdar://problem/39630910> pset_lock() should be converted to use lck_spin_lock() instead of simple_lock() */ |
292 | #define pset_assert_locked(p) do { (void)p; } while(0) |
293 | #endif |
294 | |
295 | #define rt_lock_lock(p) simple_lock(&SCHED(rt_runq)(p)->rt_lock) |
296 | #define rt_lock_unlock(p) simple_unlock(&SCHED(rt_runq)(p)->rt_lock) |
297 | #define rt_lock_init(p) simple_lock_init(&SCHED(rt_runq)(p)->rt_lock, 0) |
298 | #else |
299 | #define pset_lock(p) do { (void)p; } while(0) |
300 | #define pset_unlock(p) do { (void)p; } while(0) |
301 | #define pset_lock_init(p) do { (void)p; } while(0) |
302 | #define pset_assert_locked(p) do { (void)p; } while(0) |
303 | |
304 | #define rt_lock_lock(p) do { (void)p; } while(0) |
305 | #define rt_lock_unlock(p) do { (void)p; } while(0) |
306 | #define rt_lock_init(p) do { (void)p; } while(0) |
307 | #endif |
308 | |
309 | extern void processor_bootstrap(void); |
310 | |
311 | extern void processor_init( |
312 | processor_t processor, |
313 | int cpu_id, |
314 | processor_set_t processor_set); |
315 | |
316 | extern void processor_set_primary( |
317 | processor_t processor, |
318 | processor_t primary); |
319 | |
320 | extern kern_return_t processor_shutdown( |
321 | processor_t processor); |
322 | |
323 | extern void processor_queue_shutdown( |
324 | processor_t processor); |
325 | |
326 | extern processor_set_t processor_pset( |
327 | processor_t processor); |
328 | |
329 | extern pset_node_t pset_node_root(void); |
330 | |
331 | extern processor_set_t pset_create( |
332 | pset_node_t node); |
333 | |
334 | extern void pset_init( |
335 | processor_set_t pset, |
336 | pset_node_t node); |
337 | |
338 | extern processor_set_t pset_find( |
339 | uint32_t cluster_id, |
340 | processor_set_t default_pset); |
341 | |
342 | extern kern_return_t processor_info_count( |
343 | processor_flavor_t flavor, |
344 | mach_msg_type_number_t *count); |
345 | |
346 | #define pset_deallocate(x) |
347 | #define pset_reference(x) |
348 | |
349 | extern void machine_run_count( |
350 | uint32_t count); |
351 | |
352 | extern processor_t machine_choose_processor( |
353 | processor_set_t pset, |
354 | processor_t processor); |
355 | |
356 | #define next_pset(p) (((p)->pset_list != PROCESSOR_SET_NULL)? (p)->pset_list: (p)->node->psets) |
357 | |
358 | #define PSET_THING_TASK 0 |
359 | #define PSET_THING_THREAD 1 |
360 | |
361 | extern kern_return_t processor_set_things( |
362 | processor_set_t pset, |
363 | void **thing_list, |
364 | mach_msg_type_number_t *count, |
365 | int type); |
366 | |
367 | extern pset_cluster_type_t recommended_pset_type(thread_t thread); |
368 | |
369 | inline static bool |
370 | pset_is_recommended(processor_set_t pset) |
371 | { |
372 | return ((pset->recommended_bitmask & pset->cpu_bitmask) != 0); |
373 | } |
374 | |
375 | extern void processor_state_update_idle(processor_t processor); |
376 | extern void processor_state_update_from_thread(processor_t processor, thread_t thread); |
377 | extern void processor_state_update_explicit(processor_t processor, int pri, |
378 | sfi_class_id_t sfi_class, pset_cluster_type_t pset_type, |
379 | perfcontrol_class_t perfctl_class); |
380 | |
381 | #define PSET_LOAD_NUMERATOR_SHIFT 16 |
382 | #define PSET_LOAD_FRACTIONAL_SHIFT 4 |
383 | |
384 | inline static int |
385 | sched_get_pset_load_average(processor_set_t pset) |
386 | { |
387 | return pset->load_average >> (PSET_LOAD_NUMERATOR_SHIFT - PSET_LOAD_FRACTIONAL_SHIFT); |
388 | } |
389 | extern void sched_update_pset_load_average(processor_set_t pset); |
390 | |
391 | inline static void |
392 | pset_update_processor_state(processor_set_t pset, processor_t processor, uint new_state) |
393 | { |
394 | pset_assert_locked(pset); |
395 | |
396 | uint old_state = processor->state; |
397 | uint cpuid = processor->cpu_id; |
398 | |
399 | assert(processor->processor_set == pset); |
400 | assert(bit_test(pset->cpu_bitmask, cpuid)); |
401 | |
402 | assert(old_state < PROCESSOR_STATE_LEN); |
403 | assert(new_state < PROCESSOR_STATE_LEN); |
404 | |
405 | processor->state = new_state; |
406 | |
407 | bit_clear(pset->cpu_state_map[old_state], cpuid); |
408 | bit_set(pset->cpu_state_map[new_state], cpuid); |
409 | |
410 | if ((old_state == PROCESSOR_RUNNING) || (new_state == PROCESSOR_RUNNING)) { |
411 | sched_update_pset_load_average(pset); |
412 | } |
413 | } |
414 | |
415 | #else /* MACH_KERNEL_PRIVATE */ |
416 | |
417 | __BEGIN_DECLS |
418 | |
419 | extern void pset_deallocate( |
420 | processor_set_t pset); |
421 | |
422 | extern void pset_reference( |
423 | processor_set_t pset); |
424 | |
425 | __END_DECLS |
426 | |
427 | #endif /* MACH_KERNEL_PRIVATE */ |
428 | |
429 | #ifdef KERNEL_PRIVATE |
430 | __BEGIN_DECLS |
431 | extern unsigned int processor_count; |
432 | extern processor_t cpu_to_processor(int cpu); |
433 | __END_DECLS |
434 | |
435 | #endif /* KERNEL_PRIVATE */ |
436 | |
437 | #endif /* _KERN_PROCESSOR_H_ */ |
438 | |