1 | /* |
2 | * Copyright (c) 2000-2010 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | /* |
29 | * @OSF_COPYRIGHT@ |
30 | */ |
31 | /* |
32 | * Mach Operating System |
33 | * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University |
34 | * All Rights Reserved. |
35 | * |
36 | * Permission to use, copy, modify and distribute this software and its |
37 | * documentation is hereby granted, provided that both the copyright |
38 | * notice and this permission notice appear in all copies of the |
39 | * software, derivative works or modified versions, and any portions |
40 | * thereof, and that both notices appear in supporting documentation. |
41 | * |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
45 | * |
46 | * Carnegie Mellon requests users of this software to return to |
47 | * |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
49 | * School of Computer Science |
50 | * Carnegie Mellon University |
51 | * Pittsburgh PA 15213-3890 |
52 | * |
53 | * any improvements or extensions that they make and grant Carnegie Mellon |
54 | * the rights to redistribute these changes. |
55 | */ |
56 | /* |
57 | * NOTICE: This file was modified by McAfee Research in 2004 to introduce |
58 | * support for mandatory and extensible security protections. This notice |
59 | * is included in support of clause 2.2 (b) of the Apple Public License, |
60 | * Version 2.0. |
61 | */ |
62 | /* |
63 | */ |
64 | |
65 | /* |
66 | * Mach kernel startup. |
67 | */ |
68 | |
69 | #include <debug.h> |
70 | #include <xpr_debug.h> |
71 | #include <mach_kdp.h> |
72 | |
73 | #include <mach/boolean.h> |
74 | #include <mach/machine.h> |
75 | #include <mach/thread_act.h> |
76 | #include <mach/task_special_ports.h> |
77 | #include <mach/vm_param.h> |
78 | #include <ipc/ipc_init.h> |
79 | #include <kern/assert.h> |
80 | #include <kern/mach_param.h> |
81 | #include <kern/misc_protos.h> |
82 | #include <kern/clock.h> |
83 | #include <kern/coalition.h> |
84 | #include <kern/cpu_number.h> |
85 | #include <kern/cpu_quiesce.h> |
86 | #include <kern/ledger.h> |
87 | #include <kern/machine.h> |
88 | #include <kern/processor.h> |
89 | #include <kern/sched_prim.h> |
90 | #include <kern/turnstile.h> |
91 | #if CONFIG_SCHED_SFI |
92 | #include <kern/sfi.h> |
93 | #endif |
94 | #include <kern/startup.h> |
95 | #include <kern/task.h> |
96 | #include <kern/thread.h> |
97 | #include <kern/timer.h> |
98 | #if CONFIG_TELEMETRY |
99 | #include <kern/telemetry.h> |
100 | #endif |
101 | #include <kern/xpr.h> |
102 | #include <kern/zalloc.h> |
103 | #include <kern/locks.h> |
104 | #include <kern/debug.h> |
105 | #include <corpses/task_corpse.h> |
106 | #include <prng/random.h> |
107 | #include <console/serial_protos.h> |
108 | #include <vm/vm_kern.h> |
109 | #include <vm/vm_init.h> |
110 | #include <vm/vm_map.h> |
111 | #include <vm/vm_object.h> |
112 | #include <vm/vm_page.h> |
113 | #include <vm/vm_pageout.h> |
114 | #include <vm/vm_shared_region.h> |
115 | #include <machine/pmap.h> |
116 | #include <machine/commpage.h> |
117 | #include <libkern/version.h> |
118 | #include <sys/codesign.h> |
119 | #include <sys/kdebug.h> |
120 | #include <sys/random.h> |
121 | #include <sys/ktrace.h> |
122 | #include <libkern/section_keywords.h> |
123 | |
124 | #include <kern/ltable.h> |
125 | #include <kern/waitq.h> |
126 | #include <ipc/ipc_voucher.h> |
127 | #include <voucher/ipc_pthread_priority_internal.h> |
128 | #include <mach/host_info.h> |
129 | #include <pthread/workqueue_internal.h> |
130 | |
131 | #if CONFIG_XNUPOST |
132 | #include <tests/ktest.h> |
133 | #include <tests/xnupost.h> |
134 | #endif |
135 | |
136 | #if CONFIG_ATM |
137 | #include <atm/atm_internal.h> |
138 | #endif |
139 | |
140 | #if CONFIG_CSR |
141 | #include <sys/csr.h> |
142 | #endif |
143 | |
144 | #include <bank/bank_internal.h> |
145 | |
146 | #if ALTERNATE_DEBUGGER |
147 | #include <arm64/alternate_debugger.h> |
148 | #endif |
149 | |
150 | #if MACH_KDP |
151 | #include <kdp/kdp.h> |
152 | #endif |
153 | |
154 | #if CONFIG_MACF |
155 | #include <security/mac_mach_internal.h> |
156 | #if CONFIG_VNGUARD |
157 | extern void vnguard_policy_init(void); |
158 | #endif |
159 | #endif |
160 | |
161 | #if KPC |
162 | #include <kern/kpc.h> |
163 | #endif |
164 | |
165 | #if HYPERVISOR |
166 | #include <kern/hv_support.h> |
167 | #endif |
168 | |
169 | #include <san/kasan.h> |
170 | |
171 | #if defined(__arm__) || defined(__arm64__) |
172 | #include <arm/misc_protos.h> // for arm_vm_prot_finalize |
173 | #endif |
174 | |
175 | #include <i386/pmCPU.h> |
176 | static void kernel_bootstrap_thread(void); |
177 | |
178 | static void load_context( |
179 | thread_t thread); |
180 | #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 |
181 | extern void cpu_userwindow_init(int); |
182 | extern void cpu_physwindow_init(int); |
183 | #endif |
184 | |
185 | #if CONFIG_ECC_LOGGING |
186 | #include <kern/ecc.h> |
187 | #endif |
188 | |
189 | #if (defined(__i386__) || defined(__x86_64__)) && CONFIG_VMX |
190 | #include <i386/vmx/vmx_cpu.h> |
191 | #endif |
192 | |
193 | #if CONFIG_DTRACE |
194 | extern void dtrace_early_init(void); |
195 | extern void sdt_early_init(void); |
196 | #endif |
197 | |
198 | // libkern/OSKextLib.cpp |
199 | extern void OSKextRemoveKextBootstrap(void); |
200 | |
201 | void scale_setup(void); |
202 | extern void bsd_scale_setup(int); |
203 | extern unsigned int semaphore_max; |
204 | extern void stackshot_init(void); |
205 | extern void ktrace_init(void); |
206 | extern void oslog_init(void); |
207 | |
208 | /* |
209 | * Running in virtual memory, on the interrupt stack. |
210 | */ |
211 | |
212 | extern int serverperfmode; |
213 | |
214 | /* size of kernel trace buffer, disabled by default */ |
215 | unsigned int new_nkdbufs = 0; |
216 | unsigned int wake_nkdbufs = 0; |
217 | unsigned int write_trace_on_panic = 0; |
218 | static char trace_typefilter[64] = { 0 }; |
219 | unsigned int trace_wrap = 0; |
220 | boolean_t trace_serial = FALSE; |
221 | boolean_t early_boot_complete = FALSE; |
222 | |
223 | /* physically contiguous carveouts */ |
224 | SECURITY_READ_ONLY_LATE(uintptr_t) phys_carveout = 0; |
225 | SECURITY_READ_ONLY_LATE(uintptr_t) phys_carveout_pa = 0; |
226 | SECURITY_READ_ONLY_LATE(size_t) phys_carveout_size = 0; |
227 | |
228 | /* mach leak logging */ |
229 | int log_leaks = 0; |
230 | |
231 | static inline void |
232 | kernel_bootstrap_log(const char *message) |
233 | { |
234 | // kprintf("kernel_bootstrap: %s\n", message); |
235 | kernel_debug_string_early(message); |
236 | } |
237 | |
238 | static inline void |
239 | kernel_bootstrap_thread_log(const char *message) |
240 | { |
241 | // kprintf("kernel_bootstrap_thread: %s\n", message); |
242 | kernel_debug_string_early(message); |
243 | } |
244 | |
245 | void |
246 | kernel_early_bootstrap(void) |
247 | { |
248 | /* serverperfmode is needed by timer setup */ |
249 | if (PE_parse_boot_argn("serverperfmode" , &serverperfmode, sizeof (serverperfmode))) { |
250 | serverperfmode = 1; |
251 | } |
252 | |
253 | lck_mod_init(); |
254 | |
255 | /* |
256 | * Initialize the timer callout world |
257 | */ |
258 | timer_call_init(); |
259 | |
260 | #if CONFIG_SCHED_SFI |
261 | /* |
262 | * Configure SFI classes |
263 | */ |
264 | sfi_early_init(); |
265 | #endif |
266 | } |
267 | |
268 | extern boolean_t IORamDiskBSDRoot(void); |
269 | extern kern_return_t cpm_preallocate_early(void); |
270 | |
271 | void |
272 | kernel_bootstrap(void) |
273 | { |
274 | kern_return_t result; |
275 | thread_t thread; |
276 | char namep[16]; |
277 | |
278 | printf("%s\n" , version); /* log kernel version */ |
279 | |
280 | if (PE_parse_boot_argn("-l" , namep, sizeof (namep))) /* leaks logging */ |
281 | log_leaks = 1; |
282 | |
283 | PE_parse_boot_argn("trace" , &new_nkdbufs, sizeof (new_nkdbufs)); |
284 | PE_parse_boot_argn("trace_wake" , &wake_nkdbufs, sizeof (wake_nkdbufs)); |
285 | PE_parse_boot_argn("trace_panic" , &write_trace_on_panic, sizeof(write_trace_on_panic)); |
286 | PE_parse_boot_arg_str("trace_typefilter" , trace_typefilter, sizeof(trace_typefilter)); |
287 | PE_parse_boot_argn("trace_wrap" , &trace_wrap, sizeof(trace_wrap)); |
288 | |
289 | scale_setup(); |
290 | |
291 | kernel_bootstrap_log("vm_mem_bootstrap" ); |
292 | vm_mem_bootstrap(); |
293 | |
294 | kernel_bootstrap_log("cs_init" ); |
295 | cs_init(); |
296 | |
297 | kernel_bootstrap_log("vm_mem_init" ); |
298 | vm_mem_init(); |
299 | |
300 | machine_info.memory_size = (uint32_t)mem_size; |
301 | machine_info.max_mem = max_mem; |
302 | machine_info.major_version = version_major; |
303 | machine_info.minor_version = version_minor; |
304 | |
305 | oslog_init(); |
306 | |
307 | #if KASAN |
308 | kernel_bootstrap_log("kasan_late_init" ); |
309 | kasan_late_init(); |
310 | #endif |
311 | |
312 | #if CONFIG_TELEMETRY |
313 | kernel_bootstrap_log("telemetry_init" ); |
314 | telemetry_init(); |
315 | #endif |
316 | |
317 | #if CONFIG_CSR |
318 | kernel_bootstrap_log("csr_init" ); |
319 | csr_init(); |
320 | #endif |
321 | |
322 | if (PE_i_can_has_debugger(NULL)) { |
323 | if (PE_parse_boot_argn("-show_pointers" , &namep, sizeof(namep))) { |
324 | doprnt_hide_pointers = FALSE; |
325 | } |
326 | if (PE_parse_boot_argn("-no_slto_panic" , &namep, sizeof(namep))) { |
327 | extern boolean_t spinlock_timeout_panic; |
328 | spinlock_timeout_panic = FALSE; |
329 | } |
330 | } |
331 | |
332 | kernel_bootstrap_log("console_init" ); |
333 | console_init(); |
334 | |
335 | kernel_bootstrap_log("stackshot_init" ); |
336 | stackshot_init(); |
337 | |
338 | kernel_bootstrap_log("sched_init" ); |
339 | sched_init(); |
340 | |
341 | kernel_bootstrap_log("ltable_bootstrap" ); |
342 | ltable_bootstrap(); |
343 | |
344 | kernel_bootstrap_log("waitq_bootstrap" ); |
345 | waitq_bootstrap(); |
346 | |
347 | kernel_bootstrap_log("ipc_bootstrap" ); |
348 | ipc_bootstrap(); |
349 | |
350 | #if CONFIG_MACF |
351 | kernel_bootstrap_log("mac_policy_init" ); |
352 | mac_policy_init(); |
353 | #endif |
354 | |
355 | kernel_bootstrap_log("ipc_init" ); |
356 | ipc_init(); |
357 | |
358 | /* |
359 | * As soon as the virtual memory system is up, we record |
360 | * that this CPU is using the kernel pmap. |
361 | */ |
362 | kernel_bootstrap_log("PMAP_ACTIVATE_KERNEL" ); |
363 | PMAP_ACTIVATE_KERNEL(master_cpu); |
364 | |
365 | kernel_bootstrap_log("mapping_free_prime" ); |
366 | mapping_free_prime(); /* Load up with temporary mapping blocks */ |
367 | |
368 | kernel_bootstrap_log("machine_init" ); |
369 | machine_init(); |
370 | |
371 | kernel_bootstrap_log("clock_init" ); |
372 | clock_init(); |
373 | |
374 | ledger_init(); |
375 | |
376 | /* |
377 | * Initialize the IPC, task, and thread subsystems. |
378 | */ |
379 | |
380 | #if CONFIG_COALITIONS |
381 | kernel_bootstrap_log("coalitions_init" ); |
382 | coalitions_init(); |
383 | #endif |
384 | |
385 | kernel_bootstrap_log("task_init" ); |
386 | task_init(); |
387 | |
388 | kernel_bootstrap_log("thread_init" ); |
389 | thread_init(); |
390 | |
391 | kernel_bootstrap_log("workq_init" ); |
392 | workq_init(); |
393 | |
394 | kernel_bootstrap_log("turnstiles_init" ); |
395 | turnstiles_init(); |
396 | |
397 | #if CONFIG_ATM |
398 | /* Initialize the Activity Trace Resource Manager. */ |
399 | kernel_bootstrap_log("atm_init" ); |
400 | atm_init(); |
401 | #endif |
402 | kernel_bootstrap_log("mach_init_activity_id" ); |
403 | mach_init_activity_id(); |
404 | |
405 | /* Initialize the BANK Manager. */ |
406 | kernel_bootstrap_log("bank_init" ); |
407 | bank_init(); |
408 | |
409 | kernel_bootstrap_log("ipc_pthread_priority_init" ); |
410 | ipc_pthread_priority_init(); |
411 | |
412 | /* initialize the corpse config based on boot-args */ |
413 | corpses_init(); |
414 | |
415 | /* initialize host_statistics */ |
416 | host_statistics_init(); |
417 | |
418 | /* |
419 | * Create a kernel thread to execute the kernel bootstrap. |
420 | */ |
421 | kernel_bootstrap_log("kernel_thread_create" ); |
422 | result = kernel_thread_create((thread_continue_t)kernel_bootstrap_thread, NULL, MAXPRI_KERNEL, &thread); |
423 | |
424 | if (result != KERN_SUCCESS) panic("kernel_bootstrap: result = %08X\n" , result); |
425 | |
426 | thread->state = TH_RUN; |
427 | thread->last_made_runnable_time = mach_absolute_time(); |
428 | thread_deallocate(thread); |
429 | |
430 | kernel_bootstrap_log("load_context - done" ); |
431 | load_context(thread); |
432 | /*NOTREACHED*/ |
433 | } |
434 | |
435 | int kth_started = 0; |
436 | |
437 | vm_offset_t vm_kernel_addrperm; |
438 | vm_offset_t buf_kernel_addrperm; |
439 | vm_offset_t vm_kernel_addrperm_ext; |
440 | uint64_t vm_kernel_addrhash_salt; |
441 | uint64_t vm_kernel_addrhash_salt_ext; |
442 | |
443 | /* |
444 | * Now running in a thread. Kick off other services, |
445 | * invoke user bootstrap, enter pageout loop. |
446 | */ |
447 | static void |
448 | kernel_bootstrap_thread(void) |
449 | { |
450 | processor_t processor = current_processor(); |
451 | |
452 | #define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */ |
453 | kernel_bootstrap_thread_log("idle_thread_create" ); |
454 | /* |
455 | * Create the idle processor thread. |
456 | */ |
457 | idle_thread_create(processor); |
458 | |
459 | /* |
460 | * N.B. Do not stick anything else |
461 | * before this point. |
462 | * |
463 | * Start up the scheduler services. |
464 | */ |
465 | kernel_bootstrap_thread_log("sched_startup" ); |
466 | sched_startup(); |
467 | |
468 | /* |
469 | * Thread lifecycle maintenance (teardown, stack allocation) |
470 | */ |
471 | kernel_bootstrap_thread_log("thread_daemon_init" ); |
472 | thread_daemon_init(); |
473 | |
474 | /* Create kernel map entry reserve */ |
475 | vm_kernel_reserved_entry_init(); |
476 | |
477 | /* |
478 | * Thread callout service. |
479 | */ |
480 | kernel_bootstrap_thread_log("thread_call_initialize" ); |
481 | thread_call_initialize(); |
482 | |
483 | /* |
484 | * Remain on current processor as |
485 | * additional processors come online. |
486 | */ |
487 | kernel_bootstrap_thread_log("thread_bind" ); |
488 | thread_bind(processor); |
489 | |
490 | #if __arm64__ |
491 | if (IORamDiskBSDRoot()) { |
492 | cpm_preallocate_early(); |
493 | } |
494 | #endif /* __arm64__ */ |
495 | |
496 | /* |
497 | * Initialize ipc thread call support. |
498 | */ |
499 | kernel_bootstrap_thread_log("ipc_thread_call_init" ); |
500 | ipc_thread_call_init(); |
501 | |
502 | /* |
503 | * Kick off memory mapping adjustments. |
504 | */ |
505 | kernel_bootstrap_thread_log("mapping_adjust" ); |
506 | mapping_adjust(); |
507 | |
508 | /* |
509 | * Create the clock service. |
510 | */ |
511 | kernel_bootstrap_thread_log("clock_service_create" ); |
512 | clock_service_create(); |
513 | |
514 | /* |
515 | * Create the device service. |
516 | */ |
517 | device_service_create(); |
518 | |
519 | kth_started = 1; |
520 | |
521 | #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 |
522 | /* |
523 | * Create and initialize the physical copy window for processor 0 |
524 | * This is required before starting kicking off IOKit. |
525 | */ |
526 | cpu_physwindow_init(0); |
527 | #endif |
528 | |
529 | if (PE_i_can_has_debugger(NULL)) { |
530 | unsigned int phys_carveout_mb = 0; |
531 | if (PE_parse_boot_argn("phys_carveout_mb" , &phys_carveout_mb, |
532 | sizeof(phys_carveout_mb)) && phys_carveout_mb > 0) { |
533 | phys_carveout_size = phys_carveout_mb * 1024 * 1024; |
534 | kern_return_t kr = kmem_alloc_contig(kernel_map, |
535 | (vm_offset_t *)&phys_carveout, phys_carveout_size, |
536 | VM_MAP_PAGE_MASK(kernel_map), 0, 0, KMA_NOPAGEWAIT, |
537 | VM_KERN_MEMORY_DIAG); |
538 | if (kr != KERN_SUCCESS) { |
539 | kprintf("failed to allocate %uMB for phys_carveout_mb: %u\n" , |
540 | phys_carveout_mb, (unsigned int)kr); |
541 | } else { |
542 | phys_carveout_pa = kvtophys((vm_offset_t)phys_carveout); |
543 | } |
544 | } |
545 | } |
546 | |
547 | #if MACH_KDP |
548 | kernel_bootstrap_log("kdp_init" ); |
549 | kdp_init(); |
550 | #endif |
551 | |
552 | #if ALTERNATE_DEBUGGER |
553 | alternate_debugger_init(); |
554 | #endif |
555 | |
556 | #if KPC |
557 | kpc_init(); |
558 | #endif |
559 | |
560 | #if CONFIG_ECC_LOGGING |
561 | ecc_log_init(); |
562 | #endif |
563 | |
564 | #if HYPERVISOR |
565 | hv_support_init(); |
566 | #endif |
567 | |
568 | #if CONFIG_TELEMETRY |
569 | kernel_bootstrap_log("bootprofile_init" ); |
570 | bootprofile_init(); |
571 | #endif |
572 | |
573 | #if (defined(__i386__) || defined(__x86_64__)) && CONFIG_VMX |
574 | vmx_init(); |
575 | #endif |
576 | |
577 | kernel_bootstrap_thread_log("ktrace_init" ); |
578 | ktrace_init(); |
579 | |
580 | kdebug_init(new_nkdbufs, trace_typefilter, trace_wrap); |
581 | |
582 | #ifdef MACH_BSD |
583 | kernel_bootstrap_log("bsd_early_init" ); |
584 | bsd_early_init(); |
585 | #endif |
586 | |
587 | #if defined(__arm64__) |
588 | ml_lockdown_init(); |
589 | #endif |
590 | |
591 | #ifdef IOKIT |
592 | kernel_bootstrap_log("PE_init_iokit" ); |
593 | PE_init_iokit(); |
594 | #endif |
595 | |
596 | assert(ml_get_interrupts_enabled() == FALSE); |
597 | |
598 | /* |
599 | * Past this point, kernel subsystems that expect to operate with |
600 | * interrupts or preemption enabled may begin enforcement. |
601 | */ |
602 | early_boot_complete = TRUE; |
603 | |
604 | #if INTERRUPT_MASKED_DEBUG |
605 | // Reset interrupts masked timeout before we enable interrupts |
606 | ml_spin_debug_clear_self(); |
607 | #endif |
608 | (void) spllo(); /* Allow interruptions */ |
609 | |
610 | #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 |
611 | /* |
612 | * Create and initialize the copy window for processor 0 |
613 | * This also allocates window space for all other processors. |
614 | * However, this is dependent on the number of processors - so this call |
615 | * must be after IOKit has been started because IOKit performs processor |
616 | * discovery. |
617 | */ |
618 | cpu_userwindow_init(0); |
619 | #endif |
620 | |
621 | /* |
622 | * Initialize the shared region module. |
623 | */ |
624 | vm_shared_region_init(); |
625 | vm_commpage_init(); |
626 | vm_commpage_text_init(); |
627 | |
628 | #if CONFIG_MACF |
629 | kernel_bootstrap_log("mac_policy_initmach" ); |
630 | mac_policy_initmach(); |
631 | #if CONFIG_VNGUARD |
632 | vnguard_policy_init(); |
633 | #endif |
634 | #endif |
635 | |
636 | #if CONFIG_DTRACE |
637 | dtrace_early_init(); |
638 | sdt_early_init(); |
639 | #endif |
640 | |
641 | |
642 | /* |
643 | * Get rid of segments used to bootstrap kext loading. This removes |
644 | * the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands. |
645 | * Must be done prior to lockdown so that we can free (and possibly relocate) |
646 | * the static KVA mappings used for the jettisoned bootstrap segments. |
647 | */ |
648 | OSKextRemoveKextBootstrap(); |
649 | #if defined(__arm__) || defined(__arm64__) |
650 | #if CONFIG_KERNEL_INTEGRITY |
651 | machine_lockdown_preflight(); |
652 | #endif |
653 | /* |
654 | * Finalize protections on statically mapped pages now that comm page mapping is established. |
655 | */ |
656 | arm_vm_prot_finalize(PE_state.bootArgs); |
657 | #endif |
658 | |
659 | #if CONFIG_SCHED_SFI |
660 | kernel_bootstrap_log("sfi_init" ); |
661 | sfi_init(); |
662 | #endif |
663 | |
664 | /* |
665 | * Initialize the globals used for permuting kernel |
666 | * addresses that may be exported to userland as tokens |
667 | * using VM_KERNEL_ADDRPERM()/VM_KERNEL_ADDRPERM_EXTERNAL(). |
668 | * Force the random number to be odd to avoid mapping a non-zero |
669 | * word-aligned address to zero via addition. |
670 | * Note: at this stage we can use the cryptographically secure PRNG |
671 | * rather than early_random(). |
672 | */ |
673 | read_random(&vm_kernel_addrperm, sizeof(vm_kernel_addrperm)); |
674 | vm_kernel_addrperm |= 1; |
675 | read_random(&buf_kernel_addrperm, sizeof(buf_kernel_addrperm)); |
676 | buf_kernel_addrperm |= 1; |
677 | read_random(&vm_kernel_addrperm_ext, sizeof(vm_kernel_addrperm_ext)); |
678 | vm_kernel_addrperm_ext |= 1; |
679 | read_random(&vm_kernel_addrhash_salt, sizeof(vm_kernel_addrhash_salt)); |
680 | read_random(&vm_kernel_addrhash_salt_ext, sizeof(vm_kernel_addrhash_salt_ext)); |
681 | |
682 | vm_set_restrictions(); |
683 | |
684 | |
685 | #ifdef CONFIG_XNUPOST |
686 | kern_return_t result = kernel_list_tests(); |
687 | result = kernel_do_post(); |
688 | if (result != KERN_SUCCESS) { |
689 | panic("kernel_do_post: Tests failed with result = 0x%08x\n" , result); |
690 | } |
691 | kernel_bootstrap_log("kernel_do_post - done" ); |
692 | #endif /* CONFIG_XNUPOST */ |
693 | |
694 | |
695 | /* |
696 | * Start the user bootstrap. |
697 | */ |
698 | #ifdef MACH_BSD |
699 | bsd_init(); |
700 | #endif |
701 | |
702 | |
703 | /* |
704 | * Get rid of pages used for early boot tracing. |
705 | */ |
706 | kdebug_free_early_buf(); |
707 | |
708 | serial_keyboard_init(); /* Start serial keyboard if wanted */ |
709 | |
710 | vm_page_init_local_q(); |
711 | |
712 | thread_bind(PROCESSOR_NULL); |
713 | |
714 | /* |
715 | * Become the pageout daemon. |
716 | */ |
717 | vm_pageout(); |
718 | /*NOTREACHED*/ |
719 | } |
720 | |
721 | /* |
722 | * slave_main: |
723 | * |
724 | * Load the first thread to start a processor. |
725 | */ |
726 | void |
727 | slave_main(void *machine_param) |
728 | { |
729 | processor_t processor = current_processor(); |
730 | thread_t thread; |
731 | |
732 | /* |
733 | * Use the idle processor thread if there |
734 | * is no dedicated start up thread. |
735 | */ |
736 | if (processor->next_thread == THREAD_NULL) { |
737 | thread = processor->idle_thread; |
738 | thread->continuation = (thread_continue_t)processor_start_thread; |
739 | thread->parameter = machine_param; |
740 | } |
741 | else { |
742 | thread = processor->next_thread; |
743 | processor->next_thread = THREAD_NULL; |
744 | } |
745 | |
746 | load_context(thread); |
747 | /*NOTREACHED*/ |
748 | } |
749 | |
750 | /* |
751 | * processor_start_thread: |
752 | * |
753 | * First thread to execute on a started processor. |
754 | * |
755 | * Called at splsched. |
756 | */ |
757 | void |
758 | processor_start_thread(void *machine_param) |
759 | { |
760 | processor_t processor = current_processor(); |
761 | thread_t self = current_thread(); |
762 | |
763 | slave_machine_init(machine_param); |
764 | |
765 | /* |
766 | * If running the idle processor thread, |
767 | * reenter the idle loop, else terminate. |
768 | */ |
769 | if (self == processor->idle_thread) |
770 | thread_block((thread_continue_t)idle_thread); |
771 | |
772 | thread_terminate(self); |
773 | /*NOTREACHED*/ |
774 | } |
775 | |
776 | /* |
777 | * load_context: |
778 | * |
779 | * Start the first thread on a processor. |
780 | */ |
781 | static void __attribute__((noreturn)) |
782 | load_context( |
783 | thread_t thread) |
784 | { |
785 | processor_t processor = current_processor(); |
786 | |
787 | |
788 | #define load_context_kprintf(x...) /* kprintf("load_context: " x) */ |
789 | |
790 | load_context_kprintf("machine_set_current_thread\n" ); |
791 | machine_set_current_thread(thread); |
792 | |
793 | load_context_kprintf("processor_up\n" ); |
794 | processor_up(processor); |
795 | |
796 | PMAP_ACTIVATE_KERNEL(processor->cpu_id); |
797 | |
798 | /* |
799 | * Acquire a stack if none attached. The panic |
800 | * should never occur since the thread is expected |
801 | * to have reserved stack. |
802 | */ |
803 | load_context_kprintf("thread %p, stack %lx, stackptr %lx\n" , thread, |
804 | thread->kernel_stack, thread->machine.kstackptr); |
805 | if (!thread->kernel_stack) { |
806 | load_context_kprintf("stack_alloc_try\n" ); |
807 | if (!stack_alloc_try(thread)) |
808 | panic("load_context" ); |
809 | } |
810 | |
811 | /* |
812 | * The idle processor threads are not counted as |
813 | * running for load calculations. |
814 | */ |
815 | if (!(thread->state & TH_IDLE)) |
816 | sched_run_incr(thread); |
817 | |
818 | processor->active_thread = thread; |
819 | processor_state_update_explicit(processor, thread->sched_pri, |
820 | SFI_CLASS_KERNEL, PSET_SMP, thread_get_perfcontrol_class(thread)); |
821 | processor->starting_pri = thread->sched_pri; |
822 | processor->deadline = UINT64_MAX; |
823 | thread->last_processor = processor; |
824 | |
825 | processor->last_dispatch = mach_absolute_time(); |
826 | timer_start(&thread->system_timer, processor->last_dispatch); |
827 | PROCESSOR_DATA(processor, thread_timer) = PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer; |
828 | |
829 | timer_start(&PROCESSOR_DATA(processor, system_state), processor->last_dispatch); |
830 | PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, system_state); |
831 | |
832 | |
833 | cpu_quiescent_counter_join(processor->last_dispatch); |
834 | |
835 | PMAP_ACTIVATE_USER(thread, processor->cpu_id); |
836 | |
837 | load_context_kprintf("machine_load_context\n" ); |
838 | machine_load_context(thread); |
839 | /*NOTREACHED*/ |
840 | } |
841 | |
842 | void |
843 | scale_setup() |
844 | { |
845 | int scale = 0; |
846 | #if defined(__LP64__) |
847 | typeof(task_max) task_max_base = task_max; |
848 | |
849 | /* Raise limits for servers with >= 16G */ |
850 | if ((serverperfmode != 0) && ((uint64_t)sane_size >= (uint64_t)(16 * 1024 * 1024 *1024ULL))) { |
851 | scale = (int)((uint64_t)sane_size / (uint64_t)(8 * 1024 * 1024 *1024ULL)); |
852 | /* limit to 128 G */ |
853 | if (scale > 16) |
854 | scale = 16; |
855 | task_max_base = 2500; |
856 | /* Raise limits for machines with >= 3GB */ |
857 | } else if ((uint64_t)sane_size >= (uint64_t)(3 * 1024 * 1024 *1024ULL)) { |
858 | if ((uint64_t)sane_size < (uint64_t)(8 * 1024 * 1024 *1024ULL)) { |
859 | scale = 2; |
860 | } else { |
861 | /* limit to 64GB */ |
862 | scale = MIN(16, (int)((uint64_t)sane_size / (uint64_t)(4 * 1024 * 1024 *1024ULL))); |
863 | } |
864 | } |
865 | |
866 | task_max = MAX(task_max, task_max_base * scale); |
867 | |
868 | if (scale != 0) { |
869 | task_threadmax = task_max; |
870 | thread_max = task_max * 5; |
871 | } |
872 | |
873 | #endif |
874 | |
875 | bsd_scale_setup(scale); |
876 | |
877 | ipc_space_max = SPACE_MAX; |
878 | ipc_port_max = PORT_MAX; |
879 | ipc_pset_max = SET_MAX; |
880 | semaphore_max = SEMAPHORE_MAX; |
881 | } |
882 | |
883 | |