1 | /* |
2 | * Copyright (c) 2000-2009 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | /* |
57 | */ |
58 | |
59 | /* |
60 | * processor.c: processor and processor_set manipulation routines. |
61 | */ |
62 | |
63 | #include <mach/boolean.h> |
64 | #include <mach/policy.h> |
65 | #include <mach/processor.h> |
66 | #include <mach/processor_info.h> |
67 | #include <mach/vm_param.h> |
68 | #include <kern/cpu_number.h> |
69 | #include <kern/host.h> |
70 | #include <kern/machine.h> |
71 | #include <kern/misc_protos.h> |
72 | #include <kern/processor.h> |
73 | #include <kern/sched.h> |
74 | #include <kern/task.h> |
75 | #include <kern/thread.h> |
76 | #include <kern/ipc_host.h> |
77 | #include <kern/ipc_tt.h> |
78 | #include <ipc/ipc_port.h> |
79 | #include <kern/kalloc.h> |
80 | |
81 | #include <security/mac_mach_internal.h> |
82 | |
83 | #if defined(CONFIG_XNUPOST) |
84 | |
85 | #include <tests/xnupost.h> |
86 | |
87 | #endif /* CONFIG_XNUPOST */ |
88 | |
89 | /* |
90 | * Exported interface |
91 | */ |
92 | #include <mach/mach_host_server.h> |
93 | #include <mach/processor_set_server.h> |
94 | |
95 | struct processor_set pset0; |
96 | struct pset_node pset_node0; |
97 | decl_simple_lock_data(static,pset_node_lock) |
98 | |
99 | queue_head_t tasks; |
100 | queue_head_t terminated_tasks; /* To be used ONLY for stackshot. */ |
101 | queue_head_t corpse_tasks; |
102 | int tasks_count; |
103 | int terminated_tasks_count; |
104 | queue_head_t threads; |
105 | int threads_count; |
106 | decl_lck_mtx_data(,tasks_threads_lock) |
107 | decl_lck_mtx_data(,tasks_corpse_lock) |
108 | |
109 | processor_t processor_list; |
110 | unsigned int processor_count; |
111 | static processor_t processor_list_tail; |
112 | decl_simple_lock_data(,processor_list_lock) |
113 | |
114 | uint32_t processor_avail_count; |
115 | |
116 | processor_t master_processor; |
117 | int master_cpu = 0; |
118 | boolean_t sched_stats_active = FALSE; |
119 | |
120 | processor_t processor_array[MAX_SCHED_CPUS] = { 0 }; |
121 | |
122 | #if defined(CONFIG_XNUPOST) |
123 | kern_return_t ipi_test(void); |
124 | extern void arm64_ipi_test(void); |
125 | |
126 | kern_return_t |
127 | ipi_test() |
128 | { |
129 | #if __arm64__ |
130 | processor_t p; |
131 | |
132 | for (p = processor_list; p != NULL; p = p->processor_list) { |
133 | thread_bind(p); |
134 | thread_block(THREAD_CONTINUE_NULL); |
135 | kprintf("Running IPI test on cpu %d\n" , p->cpu_id); |
136 | arm64_ipi_test(); |
137 | } |
138 | |
139 | /* unbind thread from specific cpu */ |
140 | thread_bind(PROCESSOR_NULL); |
141 | thread_block(THREAD_CONTINUE_NULL); |
142 | |
143 | T_PASS("Done running IPI tests" ); |
144 | #else |
145 | T_PASS("Unsupported platform. Not running IPI tests" ); |
146 | |
147 | #endif /* __arm64__ */ |
148 | |
149 | return KERN_SUCCESS; |
150 | } |
151 | #endif /* defined(CONFIG_XNUPOST) */ |
152 | |
153 | |
154 | void |
155 | processor_bootstrap(void) |
156 | { |
157 | pset_init(&pset0, &pset_node0); |
158 | pset_node0.psets = &pset0; |
159 | |
160 | simple_lock_init(&pset_node_lock, 0); |
161 | |
162 | queue_init(&tasks); |
163 | queue_init(&terminated_tasks); |
164 | queue_init(&threads); |
165 | queue_init(&corpse_tasks); |
166 | |
167 | simple_lock_init(&processor_list_lock, 0); |
168 | |
169 | master_processor = cpu_to_processor(master_cpu); |
170 | |
171 | processor_init(master_processor, master_cpu, &pset0); |
172 | } |
173 | |
174 | /* |
175 | * Initialize the given processor for the cpu |
176 | * indicated by cpu_id, and assign to the |
177 | * specified processor set. |
178 | */ |
179 | void |
180 | processor_init( |
181 | processor_t processor, |
182 | int cpu_id, |
183 | processor_set_t pset) |
184 | { |
185 | spl_t s; |
186 | |
187 | if (processor != master_processor) { |
188 | /* Scheduler state for master_processor initialized in sched_init() */ |
189 | SCHED(processor_init)(processor); |
190 | } |
191 | |
192 | assert(cpu_id < MAX_SCHED_CPUS); |
193 | |
194 | processor->state = PROCESSOR_OFF_LINE; |
195 | processor->active_thread = processor->next_thread = processor->idle_thread = THREAD_NULL; |
196 | processor->processor_set = pset; |
197 | processor_state_update_idle(processor); |
198 | processor->starting_pri = MINPRI; |
199 | processor->cpu_id = cpu_id; |
200 | timer_call_setup(&processor->quantum_timer, thread_quantum_expire, processor); |
201 | processor->quantum_end = UINT64_MAX; |
202 | processor->deadline = UINT64_MAX; |
203 | processor->first_timeslice = FALSE; |
204 | processor->processor_primary = processor; /* no SMT relationship known at this point */ |
205 | processor->processor_secondary = NULL; |
206 | processor->is_SMT = FALSE; |
207 | processor->is_recommended = (pset->recommended_bitmask & (1ULL << cpu_id)) ? TRUE : FALSE; |
208 | processor->processor_self = IP_NULL; |
209 | processor_data_init(processor); |
210 | processor->processor_list = NULL; |
211 | processor->cpu_quiesce_state = CPU_QUIESCE_COUNTER_NONE; |
212 | processor->cpu_quiesce_last_checkin = 0; |
213 | |
214 | s = splsched(); |
215 | pset_lock(pset); |
216 | bit_set(pset->cpu_bitmask, cpu_id); |
217 | if (pset->cpu_set_count++ == 0) |
218 | pset->cpu_set_low = pset->cpu_set_hi = cpu_id; |
219 | else { |
220 | pset->cpu_set_low = (cpu_id < pset->cpu_set_low)? cpu_id: pset->cpu_set_low; |
221 | pset->cpu_set_hi = (cpu_id > pset->cpu_set_hi)? cpu_id: pset->cpu_set_hi; |
222 | } |
223 | pset_unlock(pset); |
224 | splx(s); |
225 | |
226 | simple_lock(&processor_list_lock); |
227 | if (processor_list == NULL) |
228 | processor_list = processor; |
229 | else |
230 | processor_list_tail->processor_list = processor; |
231 | processor_list_tail = processor; |
232 | processor_count++; |
233 | processor_array[cpu_id] = processor; |
234 | simple_unlock(&processor_list_lock); |
235 | } |
236 | |
237 | void |
238 | processor_set_primary( |
239 | processor_t processor, |
240 | processor_t primary) |
241 | { |
242 | assert(processor->processor_primary == primary || processor->processor_primary == processor); |
243 | /* Re-adjust primary point for this (possibly) secondary processor */ |
244 | processor->processor_primary = primary; |
245 | |
246 | assert(primary->processor_secondary == NULL || primary->processor_secondary == processor); |
247 | if (primary != processor) { |
248 | /* Link primary to secondary, assumes a 2-way SMT model |
249 | * We'll need to move to a queue if any future architecture |
250 | * requires otherwise. |
251 | */ |
252 | assert(processor->processor_secondary == NULL); |
253 | primary->processor_secondary = processor; |
254 | /* Mark both processors as SMT siblings */ |
255 | primary->is_SMT = TRUE; |
256 | processor->is_SMT = TRUE; |
257 | |
258 | processor_set_t pset = processor->processor_set; |
259 | atomic_bit_clear(&pset->primary_map, processor->cpu_id, memory_order_relaxed); |
260 | } |
261 | } |
262 | |
263 | processor_set_t |
264 | processor_pset( |
265 | processor_t processor) |
266 | { |
267 | return (processor->processor_set); |
268 | } |
269 | |
270 | void |
271 | processor_state_update_idle(processor_t processor) |
272 | { |
273 | processor->current_pri = IDLEPRI; |
274 | processor->current_sfi_class = SFI_CLASS_KERNEL; |
275 | processor->current_recommended_pset_type = PSET_SMP; |
276 | processor->current_perfctl_class = PERFCONTROL_CLASS_IDLE; |
277 | } |
278 | |
279 | void |
280 | processor_state_update_from_thread(processor_t processor, thread_t thread) |
281 | { |
282 | processor->current_pri = thread->sched_pri; |
283 | processor->current_sfi_class = thread->sfi_class; |
284 | processor->current_recommended_pset_type = recommended_pset_type(thread); |
285 | processor->current_perfctl_class = thread_get_perfcontrol_class(thread); |
286 | } |
287 | |
288 | void |
289 | processor_state_update_explicit(processor_t processor, int pri, sfi_class_id_t sfi_class, |
290 | pset_cluster_type_t pset_type, perfcontrol_class_t perfctl_class) |
291 | { |
292 | processor->current_pri = pri; |
293 | processor->current_sfi_class = sfi_class; |
294 | processor->current_recommended_pset_type = pset_type; |
295 | processor->current_perfctl_class = perfctl_class; |
296 | } |
297 | |
298 | pset_node_t |
299 | pset_node_root(void) |
300 | { |
301 | return &pset_node0; |
302 | } |
303 | |
304 | processor_set_t |
305 | pset_create( |
306 | pset_node_t node) |
307 | { |
308 | /* some schedulers do not support multiple psets */ |
309 | if (SCHED(multiple_psets_enabled) == FALSE) |
310 | return processor_pset(master_processor); |
311 | |
312 | processor_set_t *prev, pset = kalloc(sizeof (*pset)); |
313 | |
314 | if (pset != PROCESSOR_SET_NULL) { |
315 | pset_init(pset, node); |
316 | |
317 | simple_lock(&pset_node_lock); |
318 | |
319 | prev = &node->psets; |
320 | while (*prev != PROCESSOR_SET_NULL) |
321 | prev = &(*prev)->pset_list; |
322 | |
323 | *prev = pset; |
324 | |
325 | simple_unlock(&pset_node_lock); |
326 | } |
327 | |
328 | return (pset); |
329 | } |
330 | |
331 | /* |
332 | * Find processor set in specified node with specified cluster_id. |
333 | * Returns default_pset if not found. |
334 | */ |
335 | processor_set_t |
336 | pset_find( |
337 | uint32_t cluster_id, |
338 | processor_set_t default_pset) |
339 | { |
340 | simple_lock(&pset_node_lock); |
341 | pset_node_t node = &pset_node0; |
342 | processor_set_t pset = NULL; |
343 | |
344 | do { |
345 | pset = node->psets; |
346 | while (pset != NULL) { |
347 | if (pset->pset_cluster_id == cluster_id) |
348 | break; |
349 | pset = pset->pset_list; |
350 | } |
351 | } while ((node = node->node_list) != NULL); |
352 | simple_unlock(&pset_node_lock); |
353 | if (pset == NULL) |
354 | return default_pset; |
355 | return (pset); |
356 | } |
357 | |
358 | /* |
359 | * Initialize the given processor_set structure. |
360 | */ |
361 | void |
362 | pset_init( |
363 | processor_set_t pset, |
364 | pset_node_t node) |
365 | { |
366 | if (pset != &pset0) { |
367 | /* Scheduler state for pset0 initialized in sched_init() */ |
368 | SCHED(pset_init)(pset); |
369 | SCHED(rt_init)(pset); |
370 | } |
371 | |
372 | pset->online_processor_count = 0; |
373 | pset->load_average = 0; |
374 | pset->cpu_set_low = pset->cpu_set_hi = 0; |
375 | pset->cpu_set_count = 0; |
376 | pset->last_chosen = -1; |
377 | pset->cpu_bitmask = 0; |
378 | pset->recommended_bitmask = ~0ULL; |
379 | pset->primary_map = ~0ULL; |
380 | pset->cpu_state_map[PROCESSOR_OFF_LINE] = ~0ULL; |
381 | for (uint i = PROCESSOR_SHUTDOWN; i < PROCESSOR_STATE_LEN; i++) { |
382 | pset->cpu_state_map[i] = 0; |
383 | } |
384 | pset->pending_AST_cpu_mask = 0; |
385 | #if defined(CONFIG_SCHED_DEFERRED_AST) |
386 | pset->pending_deferred_AST_cpu_mask = 0; |
387 | #endif |
388 | pset->pending_spill_cpu_mask = 0; |
389 | pset_lock_init(pset); |
390 | pset->pset_self = IP_NULL; |
391 | pset->pset_name_self = IP_NULL; |
392 | pset->pset_list = PROCESSOR_SET_NULL; |
393 | pset->node = node; |
394 | pset->pset_cluster_type = PSET_SMP; |
395 | pset->pset_cluster_id = 0; |
396 | } |
397 | |
398 | kern_return_t |
399 | processor_info_count( |
400 | processor_flavor_t flavor, |
401 | mach_msg_type_number_t *count) |
402 | { |
403 | switch (flavor) { |
404 | |
405 | case PROCESSOR_BASIC_INFO: |
406 | *count = PROCESSOR_BASIC_INFO_COUNT; |
407 | break; |
408 | |
409 | case PROCESSOR_CPU_LOAD_INFO: |
410 | *count = PROCESSOR_CPU_LOAD_INFO_COUNT; |
411 | break; |
412 | |
413 | default: |
414 | return (cpu_info_count(flavor, count)); |
415 | } |
416 | |
417 | return (KERN_SUCCESS); |
418 | } |
419 | |
420 | |
421 | kern_return_t |
422 | processor_info( |
423 | processor_t processor, |
424 | processor_flavor_t flavor, |
425 | host_t *host, |
426 | processor_info_t info, |
427 | mach_msg_type_number_t *count) |
428 | { |
429 | int cpu_id, state; |
430 | kern_return_t result; |
431 | |
432 | if (processor == PROCESSOR_NULL) |
433 | return (KERN_INVALID_ARGUMENT); |
434 | |
435 | cpu_id = processor->cpu_id; |
436 | |
437 | switch (flavor) { |
438 | |
439 | case PROCESSOR_BASIC_INFO: |
440 | { |
441 | processor_basic_info_t basic_info; |
442 | |
443 | if (*count < PROCESSOR_BASIC_INFO_COUNT) |
444 | return (KERN_FAILURE); |
445 | |
446 | basic_info = (processor_basic_info_t) info; |
447 | basic_info->cpu_type = slot_type(cpu_id); |
448 | basic_info->cpu_subtype = slot_subtype(cpu_id); |
449 | state = processor->state; |
450 | if (state == PROCESSOR_OFF_LINE) |
451 | basic_info->running = FALSE; |
452 | else |
453 | basic_info->running = TRUE; |
454 | basic_info->slot_num = cpu_id; |
455 | if (processor == master_processor) |
456 | basic_info->is_master = TRUE; |
457 | else |
458 | basic_info->is_master = FALSE; |
459 | |
460 | *count = PROCESSOR_BASIC_INFO_COUNT; |
461 | *host = &realhost; |
462 | |
463 | return (KERN_SUCCESS); |
464 | } |
465 | |
466 | case PROCESSOR_CPU_LOAD_INFO: |
467 | { |
468 | processor_cpu_load_info_t cpu_load_info; |
469 | timer_t idle_state; |
470 | uint64_t idle_time_snapshot1, idle_time_snapshot2; |
471 | uint64_t idle_time_tstamp1, idle_time_tstamp2; |
472 | |
473 | /* |
474 | * We capture the accumulated idle time twice over |
475 | * the course of this function, as well as the timestamps |
476 | * when each were last updated. Since these are |
477 | * all done using non-atomic racy mechanisms, the |
478 | * most we can infer is whether values are stable. |
479 | * timer_grab() is the only function that can be |
480 | * used reliably on another processor's per-processor |
481 | * data. |
482 | */ |
483 | |
484 | if (*count < PROCESSOR_CPU_LOAD_INFO_COUNT) |
485 | return (KERN_FAILURE); |
486 | |
487 | cpu_load_info = (processor_cpu_load_info_t) info; |
488 | if (precise_user_kernel_time) { |
489 | cpu_load_info->cpu_ticks[CPU_STATE_USER] = |
490 | (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, user_state)) / hz_tick_interval); |
491 | cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] = |
492 | (uint32_t)(timer_grab(&PROCESSOR_DATA(processor, system_state)) / hz_tick_interval); |
493 | } else { |
494 | uint64_t tval = timer_grab(&PROCESSOR_DATA(processor, user_state)) + |
495 | timer_grab(&PROCESSOR_DATA(processor, system_state)); |
496 | |
497 | cpu_load_info->cpu_ticks[CPU_STATE_USER] = (uint32_t)(tval / hz_tick_interval); |
498 | cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] = 0; |
499 | } |
500 | |
501 | idle_state = &PROCESSOR_DATA(processor, idle_state); |
502 | idle_time_snapshot1 = timer_grab(idle_state); |
503 | idle_time_tstamp1 = idle_state->tstamp; |
504 | |
505 | /* |
506 | * Idle processors are not continually updating their |
507 | * per-processor idle timer, so it may be extremely |
508 | * out of date, resulting in an over-representation |
509 | * of non-idle time between two measurement |
510 | * intervals by e.g. top(1). If we are non-idle, or |
511 | * have evidence that the timer is being updated |
512 | * concurrently, we consider its value up-to-date. |
513 | */ |
514 | if (PROCESSOR_DATA(processor, current_state) != idle_state) { |
515 | cpu_load_info->cpu_ticks[CPU_STATE_IDLE] = |
516 | (uint32_t)(idle_time_snapshot1 / hz_tick_interval); |
517 | } else if ((idle_time_snapshot1 != (idle_time_snapshot2 = timer_grab(idle_state))) || |
518 | (idle_time_tstamp1 != (idle_time_tstamp2 = idle_state->tstamp))){ |
519 | /* Idle timer is being updated concurrently, second stamp is good enough */ |
520 | cpu_load_info->cpu_ticks[CPU_STATE_IDLE] = |
521 | (uint32_t)(idle_time_snapshot2 / hz_tick_interval); |
522 | } else { |
523 | /* |
524 | * Idle timer may be very stale. Fortunately we have established |
525 | * that idle_time_snapshot1 and idle_time_tstamp1 are unchanging |
526 | */ |
527 | idle_time_snapshot1 += mach_absolute_time() - idle_time_tstamp1; |
528 | |
529 | cpu_load_info->cpu_ticks[CPU_STATE_IDLE] = |
530 | (uint32_t)(idle_time_snapshot1 / hz_tick_interval); |
531 | } |
532 | |
533 | cpu_load_info->cpu_ticks[CPU_STATE_NICE] = 0; |
534 | |
535 | *count = PROCESSOR_CPU_LOAD_INFO_COUNT; |
536 | *host = &realhost; |
537 | |
538 | return (KERN_SUCCESS); |
539 | } |
540 | |
541 | default: |
542 | result = cpu_info(flavor, cpu_id, info, count); |
543 | if (result == KERN_SUCCESS) |
544 | *host = &realhost; |
545 | |
546 | return (result); |
547 | } |
548 | } |
549 | |
550 | kern_return_t |
551 | processor_start( |
552 | processor_t processor) |
553 | { |
554 | processor_set_t pset; |
555 | thread_t thread; |
556 | kern_return_t result; |
557 | spl_t s; |
558 | |
559 | if (processor == PROCESSOR_NULL || processor->processor_set == PROCESSOR_SET_NULL) |
560 | return (KERN_INVALID_ARGUMENT); |
561 | |
562 | if (processor == master_processor) { |
563 | processor_t prev; |
564 | |
565 | prev = thread_bind(processor); |
566 | thread_block(THREAD_CONTINUE_NULL); |
567 | |
568 | result = cpu_start(processor->cpu_id); |
569 | |
570 | thread_bind(prev); |
571 | |
572 | return (result); |
573 | } |
574 | |
575 | s = splsched(); |
576 | pset = processor->processor_set; |
577 | pset_lock(pset); |
578 | if (processor->state != PROCESSOR_OFF_LINE) { |
579 | pset_unlock(pset); |
580 | splx(s); |
581 | |
582 | return (KERN_FAILURE); |
583 | } |
584 | |
585 | pset_update_processor_state(pset, processor, PROCESSOR_START); |
586 | pset_unlock(pset); |
587 | splx(s); |
588 | |
589 | /* |
590 | * Create the idle processor thread. |
591 | */ |
592 | if (processor->idle_thread == THREAD_NULL) { |
593 | result = idle_thread_create(processor); |
594 | if (result != KERN_SUCCESS) { |
595 | s = splsched(); |
596 | pset_lock(pset); |
597 | pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE); |
598 | pset_unlock(pset); |
599 | splx(s); |
600 | |
601 | return (result); |
602 | } |
603 | } |
604 | |
605 | /* |
606 | * If there is no active thread, the processor |
607 | * has never been started. Create a dedicated |
608 | * start up thread. |
609 | */ |
610 | if ( processor->active_thread == THREAD_NULL && |
611 | processor->next_thread == THREAD_NULL ) { |
612 | result = kernel_thread_create((thread_continue_t)processor_start_thread, NULL, MAXPRI_KERNEL, &thread); |
613 | if (result != KERN_SUCCESS) { |
614 | s = splsched(); |
615 | pset_lock(pset); |
616 | pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE); |
617 | pset_unlock(pset); |
618 | splx(s); |
619 | |
620 | return (result); |
621 | } |
622 | |
623 | s = splsched(); |
624 | thread_lock(thread); |
625 | thread->bound_processor = processor; |
626 | processor->next_thread = thread; |
627 | thread->state = TH_RUN; |
628 | thread->last_made_runnable_time = mach_absolute_time(); |
629 | thread_unlock(thread); |
630 | splx(s); |
631 | |
632 | thread_deallocate(thread); |
633 | } |
634 | |
635 | if (processor->processor_self == IP_NULL) |
636 | ipc_processor_init(processor); |
637 | |
638 | result = cpu_start(processor->cpu_id); |
639 | if (result != KERN_SUCCESS) { |
640 | s = splsched(); |
641 | pset_lock(pset); |
642 | pset_update_processor_state(pset, processor, PROCESSOR_OFF_LINE); |
643 | pset_unlock(pset); |
644 | splx(s); |
645 | |
646 | return (result); |
647 | } |
648 | |
649 | ipc_processor_enable(processor); |
650 | |
651 | return (KERN_SUCCESS); |
652 | } |
653 | |
654 | kern_return_t |
655 | processor_exit( |
656 | processor_t processor) |
657 | { |
658 | if (processor == PROCESSOR_NULL) |
659 | return(KERN_INVALID_ARGUMENT); |
660 | |
661 | return(processor_shutdown(processor)); |
662 | } |
663 | |
664 | kern_return_t |
665 | processor_control( |
666 | processor_t processor, |
667 | processor_info_t info, |
668 | mach_msg_type_number_t count) |
669 | { |
670 | if (processor == PROCESSOR_NULL) |
671 | return(KERN_INVALID_ARGUMENT); |
672 | |
673 | return(cpu_control(processor->cpu_id, info, count)); |
674 | } |
675 | |
676 | kern_return_t |
677 | processor_set_create( |
678 | __unused host_t host, |
679 | __unused processor_set_t *new_set, |
680 | __unused processor_set_t *new_name) |
681 | { |
682 | return(KERN_FAILURE); |
683 | } |
684 | |
685 | kern_return_t |
686 | processor_set_destroy( |
687 | __unused processor_set_t pset) |
688 | { |
689 | return(KERN_FAILURE); |
690 | } |
691 | |
692 | kern_return_t |
693 | processor_get_assignment( |
694 | processor_t processor, |
695 | processor_set_t *pset) |
696 | { |
697 | int state; |
698 | |
699 | if (processor == PROCESSOR_NULL) |
700 | return(KERN_INVALID_ARGUMENT); |
701 | |
702 | state = processor->state; |
703 | if (state == PROCESSOR_SHUTDOWN || state == PROCESSOR_OFF_LINE) |
704 | return(KERN_FAILURE); |
705 | |
706 | *pset = &pset0; |
707 | |
708 | return(KERN_SUCCESS); |
709 | } |
710 | |
711 | kern_return_t |
712 | processor_set_info( |
713 | processor_set_t pset, |
714 | int flavor, |
715 | host_t *host, |
716 | processor_set_info_t info, |
717 | mach_msg_type_number_t *count) |
718 | { |
719 | if (pset == PROCESSOR_SET_NULL) |
720 | return(KERN_INVALID_ARGUMENT); |
721 | |
722 | if (flavor == PROCESSOR_SET_BASIC_INFO) { |
723 | processor_set_basic_info_t basic_info; |
724 | |
725 | if (*count < PROCESSOR_SET_BASIC_INFO_COUNT) |
726 | return(KERN_FAILURE); |
727 | |
728 | basic_info = (processor_set_basic_info_t) info; |
729 | basic_info->processor_count = processor_avail_count; |
730 | basic_info->default_policy = POLICY_TIMESHARE; |
731 | |
732 | *count = PROCESSOR_SET_BASIC_INFO_COUNT; |
733 | *host = &realhost; |
734 | return(KERN_SUCCESS); |
735 | } |
736 | else if (flavor == PROCESSOR_SET_TIMESHARE_DEFAULT) { |
737 | policy_timeshare_base_t ts_base; |
738 | |
739 | if (*count < POLICY_TIMESHARE_BASE_COUNT) |
740 | return(KERN_FAILURE); |
741 | |
742 | ts_base = (policy_timeshare_base_t) info; |
743 | ts_base->base_priority = BASEPRI_DEFAULT; |
744 | |
745 | *count = POLICY_TIMESHARE_BASE_COUNT; |
746 | *host = &realhost; |
747 | return(KERN_SUCCESS); |
748 | } |
749 | else if (flavor == PROCESSOR_SET_FIFO_DEFAULT) { |
750 | policy_fifo_base_t fifo_base; |
751 | |
752 | if (*count < POLICY_FIFO_BASE_COUNT) |
753 | return(KERN_FAILURE); |
754 | |
755 | fifo_base = (policy_fifo_base_t) info; |
756 | fifo_base->base_priority = BASEPRI_DEFAULT; |
757 | |
758 | *count = POLICY_FIFO_BASE_COUNT; |
759 | *host = &realhost; |
760 | return(KERN_SUCCESS); |
761 | } |
762 | else if (flavor == PROCESSOR_SET_RR_DEFAULT) { |
763 | policy_rr_base_t rr_base; |
764 | |
765 | if (*count < POLICY_RR_BASE_COUNT) |
766 | return(KERN_FAILURE); |
767 | |
768 | rr_base = (policy_rr_base_t) info; |
769 | rr_base->base_priority = BASEPRI_DEFAULT; |
770 | rr_base->quantum = 1; |
771 | |
772 | *count = POLICY_RR_BASE_COUNT; |
773 | *host = &realhost; |
774 | return(KERN_SUCCESS); |
775 | } |
776 | else if (flavor == PROCESSOR_SET_TIMESHARE_LIMITS) { |
777 | policy_timeshare_limit_t ts_limit; |
778 | |
779 | if (*count < POLICY_TIMESHARE_LIMIT_COUNT) |
780 | return(KERN_FAILURE); |
781 | |
782 | ts_limit = (policy_timeshare_limit_t) info; |
783 | ts_limit->max_priority = MAXPRI_KERNEL; |
784 | |
785 | *count = POLICY_TIMESHARE_LIMIT_COUNT; |
786 | *host = &realhost; |
787 | return(KERN_SUCCESS); |
788 | } |
789 | else if (flavor == PROCESSOR_SET_FIFO_LIMITS) { |
790 | policy_fifo_limit_t fifo_limit; |
791 | |
792 | if (*count < POLICY_FIFO_LIMIT_COUNT) |
793 | return(KERN_FAILURE); |
794 | |
795 | fifo_limit = (policy_fifo_limit_t) info; |
796 | fifo_limit->max_priority = MAXPRI_KERNEL; |
797 | |
798 | *count = POLICY_FIFO_LIMIT_COUNT; |
799 | *host = &realhost; |
800 | return(KERN_SUCCESS); |
801 | } |
802 | else if (flavor == PROCESSOR_SET_RR_LIMITS) { |
803 | policy_rr_limit_t rr_limit; |
804 | |
805 | if (*count < POLICY_RR_LIMIT_COUNT) |
806 | return(KERN_FAILURE); |
807 | |
808 | rr_limit = (policy_rr_limit_t) info; |
809 | rr_limit->max_priority = MAXPRI_KERNEL; |
810 | |
811 | *count = POLICY_RR_LIMIT_COUNT; |
812 | *host = &realhost; |
813 | return(KERN_SUCCESS); |
814 | } |
815 | else if (flavor == PROCESSOR_SET_ENABLED_POLICIES) { |
816 | int *enabled; |
817 | |
818 | if (*count < (sizeof(*enabled)/sizeof(int))) |
819 | return(KERN_FAILURE); |
820 | |
821 | enabled = (int *) info; |
822 | *enabled = POLICY_TIMESHARE | POLICY_RR | POLICY_FIFO; |
823 | |
824 | *count = sizeof(*enabled)/sizeof(int); |
825 | *host = &realhost; |
826 | return(KERN_SUCCESS); |
827 | } |
828 | |
829 | |
830 | *host = HOST_NULL; |
831 | return(KERN_INVALID_ARGUMENT); |
832 | } |
833 | |
834 | /* |
835 | * processor_set_statistics |
836 | * |
837 | * Returns scheduling statistics for a processor set. |
838 | */ |
839 | kern_return_t |
840 | processor_set_statistics( |
841 | processor_set_t pset, |
842 | int flavor, |
843 | processor_set_info_t info, |
844 | mach_msg_type_number_t *count) |
845 | { |
846 | if (pset == PROCESSOR_SET_NULL || pset != &pset0) |
847 | return (KERN_INVALID_PROCESSOR_SET); |
848 | |
849 | if (flavor == PROCESSOR_SET_LOAD_INFO) { |
850 | processor_set_load_info_t load_info; |
851 | |
852 | if (*count < PROCESSOR_SET_LOAD_INFO_COUNT) |
853 | return(KERN_FAILURE); |
854 | |
855 | load_info = (processor_set_load_info_t) info; |
856 | |
857 | load_info->mach_factor = sched_mach_factor; |
858 | load_info->load_average = sched_load_average; |
859 | |
860 | load_info->task_count = tasks_count; |
861 | load_info->thread_count = threads_count; |
862 | |
863 | *count = PROCESSOR_SET_LOAD_INFO_COUNT; |
864 | return(KERN_SUCCESS); |
865 | } |
866 | |
867 | return(KERN_INVALID_ARGUMENT); |
868 | } |
869 | |
870 | /* |
871 | * processor_set_max_priority: |
872 | * |
873 | * Specify max priority permitted on processor set. This affects |
874 | * newly created and assigned threads. Optionally change existing |
875 | * ones. |
876 | */ |
877 | kern_return_t |
878 | processor_set_max_priority( |
879 | __unused processor_set_t pset, |
880 | __unused int max_priority, |
881 | __unused boolean_t change_threads) |
882 | { |
883 | return (KERN_INVALID_ARGUMENT); |
884 | } |
885 | |
886 | /* |
887 | * processor_set_policy_enable: |
888 | * |
889 | * Allow indicated policy on processor set. |
890 | */ |
891 | |
892 | kern_return_t |
893 | processor_set_policy_enable( |
894 | __unused processor_set_t pset, |
895 | __unused int policy) |
896 | { |
897 | return (KERN_INVALID_ARGUMENT); |
898 | } |
899 | |
900 | /* |
901 | * processor_set_policy_disable: |
902 | * |
903 | * Forbid indicated policy on processor set. Time sharing cannot |
904 | * be forbidden. |
905 | */ |
906 | kern_return_t |
907 | processor_set_policy_disable( |
908 | __unused processor_set_t pset, |
909 | __unused int policy, |
910 | __unused boolean_t change_threads) |
911 | { |
912 | return (KERN_INVALID_ARGUMENT); |
913 | } |
914 | |
915 | /* |
916 | * processor_set_things: |
917 | * |
918 | * Common internals for processor_set_{threads,tasks} |
919 | */ |
920 | kern_return_t |
921 | processor_set_things( |
922 | processor_set_t pset, |
923 | void **thing_list, |
924 | mach_msg_type_number_t *count, |
925 | int type) |
926 | { |
927 | unsigned int i; |
928 | task_t task; |
929 | thread_t thread; |
930 | |
931 | task_t *task_list; |
932 | unsigned int actual_tasks; |
933 | vm_size_t task_size, task_size_needed; |
934 | |
935 | thread_t *thread_list; |
936 | unsigned int actual_threads; |
937 | vm_size_t thread_size, thread_size_needed; |
938 | |
939 | void *addr, *newaddr; |
940 | vm_size_t size, size_needed; |
941 | |
942 | if (pset == PROCESSOR_SET_NULL || pset != &pset0) |
943 | return (KERN_INVALID_ARGUMENT); |
944 | |
945 | task_size = 0; |
946 | task_size_needed = 0; |
947 | task_list = NULL; |
948 | actual_tasks = 0; |
949 | |
950 | thread_size = 0; |
951 | thread_size_needed = 0; |
952 | thread_list = NULL; |
953 | actual_threads = 0; |
954 | |
955 | for (;;) { |
956 | lck_mtx_lock(&tasks_threads_lock); |
957 | |
958 | /* do we have the memory we need? */ |
959 | if (type == PSET_THING_THREAD) |
960 | thread_size_needed = threads_count * sizeof(void *); |
961 | #if !CONFIG_MACF |
962 | else |
963 | #endif |
964 | task_size_needed = tasks_count * sizeof(void *); |
965 | |
966 | if (task_size_needed <= task_size && |
967 | thread_size_needed <= thread_size) |
968 | break; |
969 | |
970 | /* unlock and allocate more memory */ |
971 | lck_mtx_unlock(&tasks_threads_lock); |
972 | |
973 | /* grow task array */ |
974 | if (task_size_needed > task_size) { |
975 | if (task_size != 0) |
976 | kfree(task_list, task_size); |
977 | |
978 | assert(task_size_needed > 0); |
979 | task_size = task_size_needed; |
980 | |
981 | task_list = (task_t *)kalloc(task_size); |
982 | if (task_list == NULL) { |
983 | if (thread_size != 0) |
984 | kfree(thread_list, thread_size); |
985 | return (KERN_RESOURCE_SHORTAGE); |
986 | } |
987 | } |
988 | |
989 | /* grow thread array */ |
990 | if (thread_size_needed > thread_size) { |
991 | if (thread_size != 0) |
992 | kfree(thread_list, thread_size); |
993 | |
994 | assert(thread_size_needed > 0); |
995 | thread_size = thread_size_needed; |
996 | |
997 | thread_list = (thread_t *)kalloc(thread_size); |
998 | if (thread_list == 0) { |
999 | if (task_size != 0) |
1000 | kfree(task_list, task_size); |
1001 | return (KERN_RESOURCE_SHORTAGE); |
1002 | } |
1003 | } |
1004 | } |
1005 | |
1006 | /* OK, have memory and the list locked */ |
1007 | |
1008 | /* If we need it, get the thread list */ |
1009 | if (type == PSET_THING_THREAD) { |
1010 | for (thread = (thread_t)queue_first(&threads); |
1011 | !queue_end(&threads, (queue_entry_t)thread); |
1012 | thread = (thread_t)queue_next(&thread->threads)) { |
1013 | #if defined(SECURE_KERNEL) |
1014 | if (thread->task != kernel_task) { |
1015 | #endif |
1016 | thread_reference_internal(thread); |
1017 | thread_list[actual_threads++] = thread; |
1018 | #if defined(SECURE_KERNEL) |
1019 | } |
1020 | #endif |
1021 | } |
1022 | } |
1023 | #if !CONFIG_MACF |
1024 | else { |
1025 | #endif |
1026 | /* get a list of the tasks */ |
1027 | for (task = (task_t)queue_first(&tasks); |
1028 | !queue_end(&tasks, (queue_entry_t)task); |
1029 | task = (task_t)queue_next(&task->tasks)) { |
1030 | #if defined(SECURE_KERNEL) |
1031 | if (task != kernel_task) { |
1032 | #endif |
1033 | task_reference_internal(task); |
1034 | task_list[actual_tasks++] = task; |
1035 | #if defined(SECURE_KERNEL) |
1036 | } |
1037 | #endif |
1038 | } |
1039 | #if !CONFIG_MACF |
1040 | } |
1041 | #endif |
1042 | |
1043 | lck_mtx_unlock(&tasks_threads_lock); |
1044 | |
1045 | #if CONFIG_MACF |
1046 | unsigned int j, used; |
1047 | |
1048 | /* for each task, make sure we are allowed to examine it */ |
1049 | for (i = used = 0; i < actual_tasks; i++) { |
1050 | if (mac_task_check_expose_task(task_list[i])) { |
1051 | task_deallocate(task_list[i]); |
1052 | continue; |
1053 | } |
1054 | task_list[used++] = task_list[i]; |
1055 | } |
1056 | actual_tasks = used; |
1057 | task_size_needed = actual_tasks * sizeof(void *); |
1058 | |
1059 | if (type == PSET_THING_THREAD) { |
1060 | |
1061 | /* for each thread (if any), make sure it's task is in the allowed list */ |
1062 | for (i = used = 0; i < actual_threads; i++) { |
1063 | boolean_t found_task = FALSE; |
1064 | |
1065 | task = thread_list[i]->task; |
1066 | for (j = 0; j < actual_tasks; j++) { |
1067 | if (task_list[j] == task) { |
1068 | found_task = TRUE; |
1069 | break; |
1070 | } |
1071 | } |
1072 | if (found_task) |
1073 | thread_list[used++] = thread_list[i]; |
1074 | else |
1075 | thread_deallocate(thread_list[i]); |
1076 | } |
1077 | actual_threads = used; |
1078 | thread_size_needed = actual_threads * sizeof(void *); |
1079 | |
1080 | /* done with the task list */ |
1081 | for (i = 0; i < actual_tasks; i++) |
1082 | task_deallocate(task_list[i]); |
1083 | kfree(task_list, task_size); |
1084 | task_size = 0; |
1085 | actual_tasks = 0; |
1086 | task_list = NULL; |
1087 | } |
1088 | #endif |
1089 | |
1090 | if (type == PSET_THING_THREAD) { |
1091 | if (actual_threads == 0) { |
1092 | /* no threads available to return */ |
1093 | assert(task_size == 0); |
1094 | if (thread_size != 0) |
1095 | kfree(thread_list, thread_size); |
1096 | *thing_list = NULL; |
1097 | *count = 0; |
1098 | return KERN_SUCCESS; |
1099 | } |
1100 | size_needed = actual_threads * sizeof(void *); |
1101 | size = thread_size; |
1102 | addr = thread_list; |
1103 | } else { |
1104 | if (actual_tasks == 0) { |
1105 | /* no tasks available to return */ |
1106 | assert(thread_size == 0); |
1107 | if (task_size != 0) |
1108 | kfree(task_list, task_size); |
1109 | *thing_list = NULL; |
1110 | *count = 0; |
1111 | return KERN_SUCCESS; |
1112 | } |
1113 | size_needed = actual_tasks * sizeof(void *); |
1114 | size = task_size; |
1115 | addr = task_list; |
1116 | } |
1117 | |
1118 | /* if we allocated too much, must copy */ |
1119 | if (size_needed < size) { |
1120 | newaddr = kalloc(size_needed); |
1121 | if (newaddr == 0) { |
1122 | for (i = 0; i < actual_tasks; i++) { |
1123 | if (type == PSET_THING_THREAD) |
1124 | thread_deallocate(thread_list[i]); |
1125 | else |
1126 | task_deallocate(task_list[i]); |
1127 | } |
1128 | if (size) |
1129 | kfree(addr, size); |
1130 | return (KERN_RESOURCE_SHORTAGE); |
1131 | } |
1132 | |
1133 | bcopy((void *) addr, (void *) newaddr, size_needed); |
1134 | kfree(addr, size); |
1135 | |
1136 | addr = newaddr; |
1137 | size = size_needed; |
1138 | } |
1139 | |
1140 | *thing_list = (void **)addr; |
1141 | *count = (unsigned int)size / sizeof(void *); |
1142 | |
1143 | return (KERN_SUCCESS); |
1144 | } |
1145 | |
1146 | |
1147 | /* |
1148 | * processor_set_tasks: |
1149 | * |
1150 | * List all tasks in the processor set. |
1151 | */ |
1152 | kern_return_t |
1153 | processor_set_tasks( |
1154 | processor_set_t pset, |
1155 | task_array_t *task_list, |
1156 | mach_msg_type_number_t *count) |
1157 | { |
1158 | kern_return_t ret; |
1159 | mach_msg_type_number_t i; |
1160 | |
1161 | ret = processor_set_things(pset, (void **)task_list, count, PSET_THING_TASK); |
1162 | if (ret != KERN_SUCCESS) |
1163 | return ret; |
1164 | |
1165 | /* do the conversion that Mig should handle */ |
1166 | for (i = 0; i < *count; i++) |
1167 | (*task_list)[i] = (task_t)convert_task_to_port((*task_list)[i]); |
1168 | return KERN_SUCCESS; |
1169 | } |
1170 | |
1171 | /* |
1172 | * processor_set_threads: |
1173 | * |
1174 | * List all threads in the processor set. |
1175 | */ |
1176 | #if defined(SECURE_KERNEL) |
1177 | kern_return_t |
1178 | processor_set_threads( |
1179 | __unused processor_set_t pset, |
1180 | __unused thread_array_t *thread_list, |
1181 | __unused mach_msg_type_number_t *count) |
1182 | { |
1183 | return KERN_FAILURE; |
1184 | } |
1185 | #elif defined(CONFIG_EMBEDDED) |
1186 | kern_return_t |
1187 | processor_set_threads( |
1188 | __unused processor_set_t pset, |
1189 | __unused thread_array_t *thread_list, |
1190 | __unused mach_msg_type_number_t *count) |
1191 | { |
1192 | return KERN_NOT_SUPPORTED; |
1193 | } |
1194 | #else |
1195 | kern_return_t |
1196 | processor_set_threads( |
1197 | processor_set_t pset, |
1198 | thread_array_t *thread_list, |
1199 | mach_msg_type_number_t *count) |
1200 | { |
1201 | kern_return_t ret; |
1202 | mach_msg_type_number_t i; |
1203 | |
1204 | ret = processor_set_things(pset, (void **)thread_list, count, PSET_THING_THREAD); |
1205 | if (ret != KERN_SUCCESS) |
1206 | return ret; |
1207 | |
1208 | /* do the conversion that Mig should handle */ |
1209 | for (i = 0; i < *count; i++) |
1210 | (*thread_list)[i] = (thread_t)convert_thread_to_port((*thread_list)[i]); |
1211 | return KERN_SUCCESS; |
1212 | } |
1213 | #endif |
1214 | |
1215 | /* |
1216 | * processor_set_policy_control |
1217 | * |
1218 | * Controls the scheduling attributes governing the processor set. |
1219 | * Allows control of enabled policies, and per-policy base and limit |
1220 | * priorities. |
1221 | */ |
1222 | kern_return_t |
1223 | processor_set_policy_control( |
1224 | __unused processor_set_t pset, |
1225 | __unused int flavor, |
1226 | __unused processor_set_info_t policy_info, |
1227 | __unused mach_msg_type_number_t count, |
1228 | __unused boolean_t change) |
1229 | { |
1230 | return (KERN_INVALID_ARGUMENT); |
1231 | } |
1232 | |
1233 | #undef pset_deallocate |
1234 | void pset_deallocate(processor_set_t pset); |
1235 | void |
1236 | pset_deallocate( |
1237 | __unused processor_set_t pset) |
1238 | { |
1239 | return; |
1240 | } |
1241 | |
1242 | #undef pset_reference |
1243 | void pset_reference(processor_set_t pset); |
1244 | void |
1245 | pset_reference( |
1246 | __unused processor_set_t pset) |
1247 | { |
1248 | return; |
1249 | } |
1250 | |
1251 | pset_cluster_type_t |
1252 | recommended_pset_type(thread_t thread) |
1253 | { |
1254 | (void)thread; |
1255 | return PSET_SMP; |
1256 | } |
1257 | |