1/*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31
32#ifdef XNU_KERNEL_PRIVATE
33
34#ifndef _KERN_STARTUP_H_
35#define _KERN_STARTUP_H_
36
37#include <stdbool.h>
38#include <stddef.h>
39#include <stdint.h>
40
41#include <libkern/section_keywords.h>
42
43__BEGIN_DECLS
44
45#pragma GCC visibility push(hidden)
46
47/*!
48 * @enum startup_subsystem_id_t
49 *
50 * @abstract
51 * Represents a stage of kernel intialization, ubnd allows for subsystems
52 * to register initializers for a specific stage.
53 *
54 * @discussion
55 * Documentation of each subsystem initialization sequence exists in
56 * @file doc/startup.md.
57 */
58__enum_decl(startup_subsystem_id_t, uint32_t, {
59 STARTUP_SUB_NONE = 0, /**< reserved for the startup subsystem */
60
61 STARTUP_SUB_TUNABLES, /**< support for the tunables subsystem */
62 STARTUP_SUB_TIMEOUTS, /**< configurable machine timeouts */
63 STARTUP_SUB_LOCKS, /**< various subsystem locks */
64 STARTUP_SUB_KPRINTF, /**< kprintf initialization */
65
66 STARTUP_SUB_PMAP_STEAL, /**< to perform various pmap carveouts */
67 STARTUP_SUB_KMEM, /**< once kmem_alloc is ready */
68 STARTUP_SUB_ZALLOC, /**< initialize zalloc and kalloc */
69 STARTUP_SUB_KTRACE, /**< initialize kernel trace */
70 STARTUP_SUB_PERCPU, /**< initialize the percpu subsystem */
71 STARTUP_SUB_EVENT, /**< initiailze the event subsystem */
72
73 STARTUP_SUB_CODESIGNING, /**< codesigning subsystem */
74 STARTUP_SUB_OSLOG, /**< oslog and kernel logging */
75 STARTUP_SUB_MACH_IPC, /**< Mach IPC */
76 STARTUP_SUB_THREAD_CALL, /**< Thread calls */
77 STARTUP_SUB_SYSCTL, /**< registers sysctls */
78 STARTUP_SUB_EARLY_BOOT, /**< interrupts/preemption are turned on */
79
80 STARTUP_SUB_LOCKDOWN = ~0u, /**< reserved for the startup subsystem */
81});
82
83/*!
84 * Stores the last subsystem to have been fully initialized;
85 */
86extern startup_subsystem_id_t startup_phase;
87
88/*!
89 * @enum startup_debug_t
90 *
91 * @abstract
92 * Flags set in the @c startup_debug global to configure startup debugging.
93 */
94__options_decl(startup_debug_t, uint32_t, {
95 STARTUP_DEBUG_NONE = 0x00000000,
96 STARTUP_DEBUG_VERBOSE = 0x00000001,
97});
98
99extern startup_debug_t startup_debug;
100
101/*!
102 * @enum startup_rank
103 *
104 * @abstract
105 * Specifies in which rank a given initializer runs within a given section
106 * to register initializers for a specific rank within the subsystem.
107 *
108 * @description
109 * A startup function, declared with @c STARTUP or @c STARTUP_ARG, can specify
110 * a rank within the subsystem they initialize.
111 *
112 * @c STARTUP_RANK_NTH(n) will let callbacks be run at stage @c n (0-based).
113 *
114 * @c STARTUP_RANK_FIRST, @c STARTUP_RANK_SECOND, @c STARTUP_RANK_THIRD and
115 * @c STARTUP_RANK_FOURTH are given as convenient names for these.
116 *
117 * @c STARTUP_RANK_MIDDLE is a reserved value that will let startup functions
118 * run after all the @c STARTUP_RANK_NTH(n) ones have.
119 *
120 * @c STARTUP_RANK_NTH_LATE_NTH(n) will let callbacks be run then in @c n rank
121 * after the @c STARTUP_RANK_MIDDLE ones (0-based).
122 *
123 * @c STARTUP_RANK_LAST callbacks will run absolutely last after everything
124 * else did for this subsystem.
125 */
126__enum_decl(startup_rank_t, uint32_t, {
127#define STARTUP_RANK_NTH(n) ((startup_rank_t)(n - 1))
128 STARTUP_RANK_FIRST = 0,
129 STARTUP_RANK_SECOND = 1,
130 STARTUP_RANK_THIRD = 2,
131 STARTUP_RANK_FOURTH = 3,
132
133 STARTUP_RANK_MIDDLE = 0x7fffffff,
134
135#define STARTUP_RANK_LATE_NTH(n) \
136 ((startup_rank_t)(STARTUP_RANK_MIDDLE + 1 + (n)))
137
138 STARTUP_RANK_LAST = 0xffffffff,
139});
140
141#if KASAN
142/*
143 * The use of weird sections that get unmapped confuse the hell out of kasan,
144 * so for KASAN leave things in regular __TEXT/__DATA segments
145 */
146#define STARTUP_CODE_SEGSECT "__TEXT,__text"
147#define STARTUP_CONST_SEGSECT "__DATA_CONST,__init"
148#define STARTUP_DATA_SEGSECT "__DATA,__init"
149#define STARTUP_HOOK_SEGMENT "__DATA"
150#define STARTUP_HOOK_SECTION "__init_entry_set"
151#elif defined(__x86_64__)
152/* Intel doesn't have a __BOOTDATA but doesn't protect __KLD */
153#define STARTUP_CODE_SEGSECT "__TEXT,__text"
154#define STARTUP_CONST_SEGSECT "__KLDDATA,__const"
155#define STARTUP_DATA_SEGSECT "__KLDDATA,__init"
156#define STARTUP_HOOK_SEGMENT "__KLDDATA"
157#define STARTUP_HOOK_SECTION "__init_entry_set"
158#else
159/* arm protects __KLD early, so use __BOOTDATA for data */
160#define STARTUP_CODE_SEGSECT "__TEXT,__text"
161#define STARTUP_CONST_SEGSECT "__KLDDATA,__const"
162#define STARTUP_DATA_SEGSECT "__BOOTDATA,__init"
163#define STARTUP_HOOK_SEGMENT "__BOOTDATA"
164#define STARTUP_HOOK_SECTION "__init_entry_set"
165#endif
166
167/*!
168 * @macro __startup_func
169 *
170 * @abstract
171 * Attribute to place on functions used only during the kernel startup phase.
172 *
173 * @description
174 * Code marked with this attribute will be unmapped after kernel lockdown.
175 */
176#define __startup_func \
177 __PLACE_IN_SECTION(STARTUP_CODE_SEGSECT) \
178 __attribute__((cold, visibility("hidden")))
179
180/*!
181 * @macro __startup_data
182 *
183 * @abstract
184 * Attribute to place on globals used during the kernel startup phase.
185 *
186 * @description
187 * Data marked with this attribute will be unmapped after kernel lockdown.
188 */
189#define __startup_data \
190 __PLACE_IN_SECTION(STARTUP_DATA_SEGSECT)
191
192/*!
193 * @macro __startup_const
194 *
195 * @abstract
196 * Attribute to place on global constants used during the kernel startup phase.
197 *
198 * @description
199 * Data marked with this attribute will be unmapped after kernel lockdown.
200 *
201 * __startup_const implies const. Be mindful that for pointers, the `const`
202 * will end up on the wrong side of the star if you put __startup_const at the
203 * start of the declaration.
204 */
205#define __startup_const \
206 __PLACE_IN_SECTION(STARTUP_CONST_SEGSECT) const
207
208/*!
209 * @macro STARTUP
210 *
211 * @abstract
212 * Declares a kernel startup callback.
213 */
214#define STARTUP(subsystem, rank, func) \
215 __STARTUP(func, __LINE__, subsystem, rank, func)
216
217/*!
218 * @macro STARTUP_ARG
219 *
220 * @abstract
221 * Declares a kernel startup callback that takes an argument.
222 */
223#define STARTUP_ARG(subsystem, rank, func, arg) \
224 __STARTUP_ARG(func, __LINE__, subsystem, rank, func, arg)
225
226/*!
227 * @macro TUNABLE
228 *
229 * @abstract
230 * Declares a read-only kernel tunable that is read from a boot-arg with
231 * a default value, without further processing.
232 *
233 * @param type_t
234 * Should be an integer type or bool.
235 *
236 * @param var
237 * The name of the C variable to use for storage.
238 *
239 * @param boot_arg
240 * The name of the boot-arg to parse for initialization
241 *
242 * @param default_value
243 * The default value for the tunable if the boot-arg is absent.
244 */
245#define TUNABLE(type_t, var, boot_arg, default_value) \
246 SECURITY_READ_ONLY_LATE(type_t) var = default_value; \
247 __TUNABLE(type_t, var, boot_arg)
248
249/*!
250 * @macro TUNABLE_WRITEABLE
251 *
252 * @abstract
253 * Declares a writeable kernel tunable that is read from a boot-arg with
254 * a default value, without further processing.
255 *
256 * @param type_t
257 * Should be an integer type or bool.
258 *
259 * @param var
260 * The name of the C variable to use for storage.
261 *
262 * @param boot_arg
263 * The name of the boot-arg to parse for initialization
264 *
265 * @param default_value
266 * The default value for the tunable if the boot-arg is absent.
267 */
268#define TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value) \
269 type_t var = default_value; \
270 __TUNABLE(type_t, var, boot_arg)
271
272#if DEBUG || DEVELOPMENT
273#define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \
274 TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value)
275#else
276#define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \
277 TUNABLE(type_t, var, boot_arg, default_value)
278#endif
279
280/*!
281 * @macro TUNABLE_STR
282 *
283 * @abstract
284 * Declares a read-only kernel tunable that is read from a boot-arg with
285 * a default value, without further processing.
286 *
287 * @param var
288 * The name of the C variable to use for storage.
289 *
290 * @param count
291 * The number of bytes in the buffer.
292 *
293 * @param boot_arg
294 * The name of the boot-arg to parse for initialization
295 *
296 * @param default_value
297 * The default value for the tunable if the boot-arg is absent.
298 */
299#define TUNABLE_STR(var, count, boot_arg, default_value) \
300 char __security_const_late var[count] = default_value; \
301 __TUNABLE_STR(var, boot_arg)
302
303/*!
304 * @enum tunable_dt_flags_t
305 *
306 * @abstract
307 * Flags used with the @c TUNABLE_DT* macros.
308 *
309 * @description
310 * If TUNABLE_DT_CHECK_CHOSEN is set, a value in
311 * /chosen/<dt_base>/<dt_name> takes precedence over any value in
312 * /<dt_base>/<dt_name>. /chosen is by convention the area where
313 * synthesized values not coming from the serialized device tree are
314 * being added, so this provides a way for e.g. the boot-loader to
315 * set/override tunables.
316 */
317__options_decl(tunable_dt_flags_t, uint32_t, {
318 TUNABLE_DT_NONE = 0x00000000,
319 TUNABLE_DT_CHECK_CHOSEN = 0x00000001,
320});
321
322/*!
323 * @macro TUNABLE_DT
324 *
325 * @abstract
326 * Like TUNABLE, but gets the initial value from both Device Tree and
327 * boot-args. The order in which the initial value is resolved is as
328 * follows, with later steps overriding previous ones (if they are
329 * specified):
330 *
331 * 1. Device Tree Entry "/<dt_base>/<dt_name>",
332 * 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry
333 * "/chosen/<dt_base>/<dt_name>" (see the description for
334 * @c tunable_dt_flags_t),
335 * 3. boot-args.
336 *
337 * @param type_t
338 * Should be an integer type or bool.
339 *
340 * @param var
341 * The name of the C variable to use for storage.
342 *
343 * @param dt_base
344 * The name of the DT node containing the property.
345 *
346 * @param dt_name
347 * The name of the DT property containing the default value.
348 *
349 * @param boot_arg
350 * The name of the boot-arg overriding the initial value from the DT.
351 *
352 * @param default_value
353 * The default value for the tunable if both DT entry and boot-arg are
354 * absent.
355 *
356 * @param flags
357 * See the description for @c tunable_dt_flags_t.
358 */
359#define TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \
360 SECURITY_READ_ONLY_LATE(type_t) var = default_value; \
361 __TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags)
362
363/*!
364 * @macro TUNABLE_DT_WRITEABLE
365 *
366 * @abstract
367 * Like TUNABLE_WRITEABLE, but gets the initial value from both Device
368 * Tree and boot-args. The order in which the initial value is
369 * resolved is as follows, with later steps overriding previous ones
370 * (if they are specified):
371 *
372 * 1. Device Tree Entry "/<dt_base>/<dt_name>",
373 * 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry
374 * "/chosen/<dt_base>/<dt_name>" (see the description for
375 * @c tunable_dt_flags_t),
376 * 3. boot-args.
377 *
378 * @param type_t
379 * Should be an integer type or bool.
380 *
381 * @param var
382 * The name of the C variable to use for storage.
383 *
384 * @param dt_base
385 * The name of the DT node containing the property.
386 *
387 * @param dt_name
388 * The name of the DT property containing the default value.
389 *
390 * @param boot_arg
391 * The name of the boot-arg overriding the initial value from the DT.
392 *
393 * @param default_value
394 * The default value for the tunable if both DT entry and boot-arg are
395 * absent.
396 *
397 * @param flags
398 * See the description for @c tunable_dt_flags_t.
399 */
400#define TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \
401 type_t var = default_value; \
402 __TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags)
403
404#if DEBUG || DEVELOPMENT
405#define TUNABLE_DT_DEV_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \
406 TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags)
407#else
408#define TUNABLE_DT_DEV_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \
409 TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags)
410#endif
411
412/*
413 * Machine Timeouts
414 *
415 * Machine Timeouts are timeouts for low level kernel code manifesting
416 * as _Atomic uint64_t variables, whose default value can be
417 * overridden and scaled via the device tree and boot-args.
418 *
419 * Each timeout has a name, looked up directly as the property name in
420 * the device tree in both the "/machine-timeouts" and
421 * "/chosen/machine-timeouts" nodes. The "chosen" property always
422 * overrides the other one. This allows fixed per-device timeouts in
423 * the device tree to be overridden by iBoot in "chosen".
424 *
425 * Additionally, the same name with "-scale" appended is looked up as
426 * properties for optional scale factors. Scale factors are not
427 * overridden by chosen, instead all scale factors (including global
428 * and/or boot-arg scale factors) combine by multiplication.
429 *
430 * The special name "global-scale" provides a scale that applies to
431 * every timeout.
432 *
433 * All property names can be used as boot-args by prefixing
434 * "ml-timeout-", e.g. th global scale is available as the
435 * "ml-timeout-global-scale" boot-arg.
436 *
437 * By convention, if the timeout value resolves to 0, the timeout
438 * should be disabled.
439 */
440
441/*
442 * Machine Timeouts types. See the next section for what unit
443 * they are in.
444 *
445 * We use _Atomic, but only with relaxed ordering: This is just to
446 * make sure all devices see consistent values all the time. Since
447 * the actual timeout value will be seen as 0 before initializaton,
448 * relaxed ordering means that code that runs concurrently with
449 * initialization only risks to see a disabled timeout during early
450 * boot.
451 */
452typedef _Atomic uint64_t machine_timeout_t;
453
454/*
455 * Units
456 *
457 * Machine Timeouts are ALWAYS in picoseconds in the device tree or
458 * boot-args, to avoid confusion when changing or comparing timeouts
459 * as a user, but the actual storage value might contain the same
460 * duration in another unit, calculated by the initialization code.
461 *
462 * This is done because otherwise we would likely introduce another
463 * multiplication in potentially hot code paths, given that code that
464 * actually uses the timeout storage variable is unlikely to work with
465 * picosecond values when comparing against the timeout deadline.
466 *
467 * This unit scale is *only* applied during initialization at early
468 * boot, and only if the timeout's default value was overridden
469 * through the device tree or a boot-arg.
470 */
471#define MACHINE_TIMEOUT_UNIT_PSEC 1
472#define MACHINE_TIMEOUT_UNIT_NSEC 1000
473#define MACHINE_TIMEOUT_UNIT_USEC (1000*1000)
474#define MACHINE_TIMEOUT_UNIT_MSEC (1000*1000*1000)
475// Special unit for timebase ticks (usually 1/24MHz)
476#define MACHINE_TIMEOUT_UNIT_TIMEBASE 0
477
478// DT property names are limited to 31 chars, minus "-global" suffix
479#define MACHINE_TIMEOUT_MAX_NAME_LEN 25
480struct machine_timeout_spec {
481 void *ptr;
482 uint64_t default_value;
483 uint64_t unit_scale;
484 char name[MACHINE_TIMEOUT_MAX_NAME_LEN + 1];
485 bool (*skip_predicate)(struct machine_timeout_spec const *);
486};
487
488extern void
489machine_timeout_init_with_suffix(const struct machine_timeout_spec *spec, char const *phase_suffix);
490
491extern void
492machine_timeout_init(const struct machine_timeout_spec *spec);
493
494#if DEVELOPMENT || DEBUG
495// Late timeout (re-)initialization, at the end of bsd_init()
496extern void
497machine_timeout_bsd_init(void);
498#endif /* DEVELOPMENT || DEBUG */
499
500/*!
501 * @macro MACHINE_TIMEOUT and MACHINE_TIMEOUT_DEV_WRITEABLE
502 *
503 * @abstract
504 * Defines a Machine Timeout that can be overridden and
505 * scaled through the device tree and boot-args.
506 *
507 * The variant with the _DEV_WRITEABLE suffix does not mark the timeout as
508 * SECURITY_READ_ONLY_LATE on DEVELOPMENT kernels, so that e.g.
509 * machine_timeout_init_with_suffix or sysctls can change it after lockdown.
510 *
511 * @param var
512 * The name of the C variable to use for storage. If the storage value
513 * contains 0, the timeout is considered disabled by convention.
514 *
515 * @param timeout_name
516 * The name of the timeout, used for property and boot-arg names. See
517 * the general description of Machine Timeouts above for how this name
518 * ends up being used.
519 *
520 * @param timeout_default
521 * The default value for the timeout if not specified through device
522 * tree or boot-arg. Will still be scaled if a scale factor exists.
523 *
524 * @param var_unit
525 * The unit that the storage variable is in. Note that timeout values
526 * must always be specified as picoseconds in the device tree and
527 * boot-args, but timeout initialization will convert the value to the
528 * unit specified here before writing it to the storage variable.
529 *
530 * @param skip_predicate
531 * Optionally, a function to call to decide whether the timeout should
532 * be set or not. If NULL, the timeout will always be set (if
533 * specified anywhere). A predicate has the following signature:
534 * bool skip_predicate (struct machine_timeout_spec const *)
535 */
536
537#define _MACHINE_TIMEOUT(var, timeout_name, timeout_default, var_unit, skip_pred) \
538 struct machine_timeout_spec \
539 __machine_timeout_spec_ ## var = { \
540 .ptr = &var, \
541 .default_value = timeout_default, \
542 .unit_scale = var_unit, \
543 .name = timeout_name, \
544 .skip_predicate = skip_pred, \
545 }; \
546 __STARTUP_ARG(var, __LINE__, TIMEOUTS, STARTUP_RANK_FIRST, \
547 machine_timeout_init, &__machine_timeout_spec_ ## var)
548
549#define MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) \
550 SECURITY_READ_ONLY_LATE(machine_timeout_t) var = 0; \
551 _MACHINE_TIMEOUT(var, name, default, unit, skip_predicate)
552
553#if DEVELOPMENT || DEBUG
554#define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \
555 machine_timeout_t var = 0; \
556 _MACHINE_TIMEOUT(var, name, default, unit, skip_predicate)
557#else
558#define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \
559 MACHINE_TIMEOUT(var, name, default, unit, skip_predicate)
560#endif /* DEVELOPMENT || DEBUG */
561
562/*!
563 * @macro MACHINE_TIMEOUT_SPEC_REF
564 *
565 * @abstract
566 * References a previously defined MACHINE_TIMEOUT.
567 *
568 * This is primarily useful for overriding individual timeouts
569 * at arbitrary times (even after boot), by manually calling
570 * machine_timeout_init_with_suffix() with this macro
571 * as first argument, and a suffix to apply to both device tree and
572 * boot-arg as second argument.
573 *
574 * @param var
575 * The name of the C variable used for storage, as it was specified
576 * in MACHINE_TIMEOUT.
577 */
578#define MACHINE_TIMEOUT_SPEC_REF(var) (&__machine_timeout_spec_ ## var)
579
580/*!
581 * @macro MACHINE_TIMEOUT_SPEC_DECL
582 *
583 * @abstract
584 * Declaration of machine timeout spec, mostly useful to make it known
585 * for MACHINE_TIMEOUT_SPEC_REF.
586 *
587 * @param var
588 * The name of the C variable used for storage, as it was specified
589 * in MACHINE_TIMEOUT.
590 */
591#define MACHINE_TIMEOUT_SPEC_DECL(var) extern struct machine_timeout_spec __machine_timeout_spec_ ## var
592
593/*
594 * Event subsystem
595 *
596 * This allows to define a few system-wide events that allow for loose coupling
597 * between subsystems interested in those events and the emitter.
598 */
599
600/*!
601 * @macro EVENT_DECLARE()
602 *
603 * @brief
604 * Declares an event namespace with a given callback type.
605 *
606 * @param name The name for the event (typically in all caps).
607 * @param cb_type_t A function type for the callbacks.
608 */
609#define EVENT_DECLARE(name, cb_type_t) \
610 struct name##_event { \
611 struct event_hdr evt_link; \
612 cb_type_t *evt_cb; \
613 }; \
614 extern struct event_hdr name##_HEAD
615
616/*!
617 * @macro EVENT_DEFINE()
618 *
619 * @brief
620 * Defines the head for the event corresponding to an EVENT_DECLARE()
621 */
622#define EVENT_DEFINE(name) \
623 __security_const_late struct event_hdr name##_HEAD
624
625/*!
626 * @macro EVENT_REGISTER_HANDLER()
627 *
628 * @brief
629 * Registers a handler for a given event.
630 *
631 * @param name The name for the event as declared in EVENT_DECLARE()
632 * @param handler The handler to register for this event.
633 */
634#define EVENT_REGISTER_HANDLER(name, handler) \
635 __EVENT_REGISTER(name, __LINE__, handler)
636
637
638/*!
639 * @macro EVENT_INVOKE()
640 *
641 * @brief
642 * Call all the events handlers with the specified arguments.
643 *
644 * @param name The name for the event as declared in EVENT_DECLARE()
645 * @param handler The handler to register for this event.
646 */
647#define EVENT_INVOKE(name, ...) \
648 for (struct event_hdr *__e = &name##_HEAD; (__e = __e->next);) { \
649 __container_of(__e, struct name##_event, \
650 evt_link)->evt_cb(__VA_ARGS__); \
651 }
652
653
654#if DEBUG || DEVELOPMENT
655
656/*!
657 * @macro SYSCTL_TEST_REGISTER
658 *
659 * @abstract
660 * Declares a test that will appear under @c debug.test.${name}.
661 *
662 * @param name
663 * An identifier that will be stringified to form the sysctl test name.
664 *
665 * @param cb
666 * The callback to run, of type:
667 * <code>
668 * int (callback *)(int64_t value, int64_t *);
669 * </code>
670 */
671#define SYSCTL_TEST_REGISTER(name, cb) \
672 static __startup_data struct sysctl_test_setup_spec \
673 __startup_SYSCTL_TEST_ ## name = { \
674 .st_name = #name, \
675 .st_func = &cb, \
676 }; \
677 STARTUP_ARG(SYSCTL, STARTUP_RANK_MIDDLE, \
678 sysctl_register_test_startup, &__startup_SYSCTL_TEST_ ## name)
679
680#endif /* DEBUG || DEVELOPMENT */
681#pragma mark - internals
682
683__END_DECLS
684
685#ifdef __cplusplus
686template <typename T>
687struct __startup_tunable {
688 static const bool value = false;
689};
690
691template <>
692struct __startup_tunable <bool>{
693 static const bool value = true;
694};
695#define __startup_type_is_bool(type_t) __startup_tunable<type_t>::value
696#else
697#define __startup_type_is_bool(type_t) __builtin_types_compatible_p(bool, type_t)
698#endif
699
700__BEGIN_DECLS
701
702#define __TUNABLE(type_t, var, key) \
703 static __startup_const char __startup_TUNABLES_name_ ## var[] = key; \
704 static __startup_const struct startup_tunable_spec \
705 __startup_TUNABLES_spec_ ## var = { \
706 .name = __startup_TUNABLES_name_ ## var, \
707 .var_addr = (void *)&var, \
708 .var_len = sizeof(type_t), \
709 .var_is_bool = __startup_type_is_bool(type_t), \
710 }; \
711 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \
712 kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var)
713
714#define __TUNABLE_STR(var, key) \
715 static __startup_const char __startup_TUNABLES_name_ ## var[] = key; \
716 static __startup_const struct startup_tunable_spec \
717 __startup_TUNABLES_spec_ ## var = { \
718 .name = __startup_TUNABLES_name_ ## var, \
719 .var_addr = (void *)&var, \
720 .var_len = sizeof(var), \
721 .var_is_str = true, \
722 }; \
723 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \
724 kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var)
725
726#define __TUNABLE_DT(type_t, var, dt_base_key, dt_name_key, boot_arg_key, flags) \
727 static __startup_const char __startup_TUNABLES_dt_base_ ## var[] = dt_base_key; \
728 static __startup_const char __startup_TUNABLES_dt_name_ ## var[] = dt_name_key; \
729 static __startup_const char __startup_TUNABLES_name_ ## var[] = boot_arg_key; \
730 static __startup_const struct startup_tunable_dt_spec \
731 __startup_TUNABLES_DT_spec_ ## var = { \
732 .dt_base = __startup_TUNABLES_dt_base_ ## var, \
733 .dt_name = __startup_TUNABLES_dt_name_ ## var, \
734 .dt_chosen_override = (bool)((flags) & TUNABLE_DT_CHECK_CHOSEN), \
735 .boot_arg_name = __startup_TUNABLES_name_ ## var, \
736 .var_addr = (void *)&var, \
737 .var_len = sizeof(type_t), \
738 .var_is_bool = __startup_type_is_bool(type_t), \
739 }; \
740 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \
741 kernel_startup_tunable_dt_init, &__startup_TUNABLES_DT_spec_ ## var)
742
743#ifdef __cplusplus
744#define __STARTUP_FUNC_CAST(func, a) \
745 (void(*)(const void *))func
746#else
747#define __STARTUP_FUNC_CAST(func, a) \
748 (typeof(func(a))(*)(const void *))func
749#endif
750
751
752#define __STARTUP1(name, line, subsystem, rank, func, a, b) \
753 __PLACE_IN_SECTION(STARTUP_HOOK_SEGMENT "," STARTUP_HOOK_SECTION) \
754 static const struct startup_entry \
755 __startup_ ## subsystem ## _entry_ ## name ## _ ## line = { \
756 STARTUP_SUB_ ## subsystem, \
757 rank, __STARTUP_FUNC_CAST(func, a), b, \
758 }
759
760#define __STARTUP(name, line, subsystem, rank, func) \
761 __STARTUP1(name, line, subsystem, rank, func, , NULL)
762
763#define __STARTUP_ARG(name, line, subsystem, rank, func, arg) \
764 __STARTUP1(name, line, subsystem, rank, func, arg, arg)
765
766struct startup_entry {
767 startup_subsystem_id_t subsystem;
768 startup_rank_t rank;
769 void (*func)(const void *);
770 const void *arg;
771};
772
773struct startup_tunable_spec {
774 const char *name;
775 void *var_addr;
776 int var_len;
777 bool var_is_bool;
778 bool var_is_str;
779};
780
781struct startup_tunable_dt_spec {
782 const char *dt_base;
783 const char *dt_name;
784 bool dt_chosen_override;
785 const char *boot_arg_name;
786 void *var_addr;
787 int var_len;
788 bool var_is_bool;
789};
790
791#if DEBUG || DEVELOPMENT
792struct sysctl_test_setup_spec {
793 const char *st_name;
794 int (*st_func)(int64_t, int64_t *);
795};
796
797extern void sysctl_register_test_startup(
798 struct sysctl_test_setup_spec *spec);
799#endif /* DEBUG || DEVELOPMENT */
800
801/*
802 * Kernel and machine startup declarations
803 */
804
805/* Initialize kernel */
806extern void kernel_startup_bootstrap(void);
807extern void kernel_startup_initialize_upto(startup_subsystem_id_t upto);
808extern void kernel_startup_tunable_init(const struct startup_tunable_spec *);
809extern void kernel_startup_tunable_dt_init(const struct startup_tunable_dt_spec *);
810extern void kernel_bootstrap(void);
811
812/* Initialize machine dependent stuff */
813extern void machine_init(void);
814
815extern void slave_main(void *machine_param);
816
817/*
818 * The following must be implemented in machine dependent code.
819 */
820
821/* Slave cpu initialization */
822extern void slave_machine_init(void *machine_param);
823
824/* Device subsystem initialization */
825extern void device_service_create(void);
826
827struct event_hdr {
828 struct event_hdr *next;
829};
830
831extern void event_register_handler(struct event_hdr *event_hdr);
832
833#define __EVENT_REGISTER(name, lno, handler) \
834 static __security_const_late struct name##_event name##_event_##lno = { \
835 .evt_link.next = &name##_HEAD, \
836 .evt_cb = (handler), \
837 }; \
838 __STARTUP_ARG(name, lno, EVENT, STARTUP_RANK_FIRST, \
839 event_register_handler, &name##_event_##lno.evt_link)
840
841#ifdef MACH_BSD
842
843/* BSD subsystem initialization */
844extern void bsd_init(void);
845
846#endif /* MACH_BSD */
847
848#pragma GCC visibility pop
849
850__END_DECLS
851
852#endif /* _KERN_STARTUP_H_ */
853
854#endif /* XNU_KERNEL_PRIVATE */
855