1/*
2 * Copyright (c) 2000,2008-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * Copyright (c) 1997 Apple Inc.
30 *
31 */
32#include <libkern/c++/OSMetaClass.h>
33#include <libkern/c++/OSKext.h>
34#include <libkern/c++/OSLib.h>
35#include <libkern/c++/OSSymbol.h>
36#include <IOKit/IOKitDebug.h>
37
38#include <sys/cdefs.h>
39#if defined(HAS_APPLE_PAC)
40#include <ptrauth.h>
41#define PTRAUTH_STRIP_STRUCTOR(x) ((uintptr_t) ptrauth_strip(ptrauth_nop_cast(void *, (x)), ptrauth_key_function_pointer))
42#else /* defined(HAS_APPLE_PAC) */
43#define PTRAUTH_STRIP_STRUCTOR(x) ((uintptr_t) (x))
44#endif /* !defined(HAS_APPLE_PAC) */
45
46__BEGIN_DECLS
47
48#include <string.h>
49#include <mach/mach_types.h>
50#include <libkern/kernel_mach_header.h>
51#include <libkern/prelink.h>
52#include <stdarg.h>
53
54#if KASAN
55#include <san/kasan.h>
56#endif
57
58#if CONFIG_SPTM
59#include <arm64/sptm/sptm.h>
60#endif
61
62#if PRAGMA_MARK
63#pragma mark Constants &c.
64#endif /* PRAGMA_MARK */
65OSKextLogSpec kOSRuntimeLogSpec =
66 kOSKextLogErrorLevel |
67 kOSKextLogLoadFlag |
68 kOSKextLogKextBookkeepingFlag;
69
70#if PRAGMA_MARK
71#pragma mark Logging Bootstrap
72#endif /* PRAGMA_MARK */
73/*********************************************************************
74* kern_os Logging Bootstrap
75*
76* We can't call in to OSKext until the kernel's C++ environment is up
77* and running, so let's mask those references with a check variable.
78* We print unconditionally if C++ isn't up, but if that's the case
79* we've generally hit a serious error in kernel init!
80*********************************************************************/
81static bool gKernelCPPInitialized = false;
82
83#define OSRuntimeLog(kext, flags, format, args ...) \
84 do { \
85 if (gKernelCPPInitialized) { \
86 OSKextLog((kext), (flags), (format), ## args); \
87 } else { \
88 printf((format), ## args); \
89 } \
90 } while (0)
91
92#if PRAGMA_MARK
93#pragma mark Libkern Init
94#endif /* PRAGMA_MARK */
95/*********************************************************************
96* Libkern Init
97*********************************************************************/
98
99#if __GNUC__ >= 3
100void __dead2
101__cxa_pure_virtual( void )
102{
103 panic("%s", __FUNCTION__);
104}
105#else
106void __dead2
107__pure_virtual( void )
108{
109 panic("%s", __FUNCTION__);
110}
111#endif
112
113extern lck_grp_t * IOLockGroup;
114extern kmod_info_t g_kernel_kmod_info;
115#if CONFIG_SPTM
116extern kmod_info_t g_sptm_kmod_info, g_txm_kmod_info;
117#endif /* CONFIG_SPTM */
118
119enum {
120 kOSSectionNamesDefault = 0,
121 kOSSectionNamesBuiltinKext = 1,
122 kOSSectionNamesCount = 2,
123};
124enum {
125 kOSSectionNameInitializer = 0,
126 kOSSectionNameFinalizer = 1,
127 kOSSectionNameCount = 2
128};
129
130static const char *
131 gOSStructorSectionNames[kOSSectionNamesCount][kOSSectionNameCount] = {
132 { SECT_MODINITFUNC, SECT_MODTERMFUNC },
133 { kBuiltinInitSection, kBuiltinTermSection }
134};
135
136void
137OSlibkernInit(void)
138{
139 // This must be called before calling OSRuntimeInitializeCPP.
140 OSMetaClassBase::initialize();
141
142 g_kernel_kmod_info.address = (vm_address_t) &_mh_execute_header;
143#if CONFIG_SPTM
144 g_sptm_kmod_info.address = (vm_offset_t)SPTMArgs->debug_header->image[DEBUG_HEADER_ENTRY_SPTM];
145 g_txm_kmod_info.address = (vm_offset_t)SPTMArgs->debug_header->image[DEBUG_HEADER_ENTRY_TXM];
146#endif /* CONFIG_SPTM */
147
148 if (kOSReturnSuccess != OSRuntimeInitializeCPP(NULL)) {
149 // &g_kernel_kmod_info, gOSSectionNamesStandard, 0, 0)) {
150 panic("OSRuntime: C++ runtime failed to initialize.");
151 }
152
153 gKernelCPPInitialized = true;
154
155 return;
156}
157
158__END_DECLS
159
160#if PRAGMA_MARK
161#pragma mark C++ Runtime Load/Unload
162#endif /* PRAGMA_MARK */
163/*********************************************************************
164* kern_os C++ Runtime Load/Unload
165*********************************************************************/
166
167typedef void (*structor_t)(void);
168
169static bool
170OSRuntimeCallStructorsInSection(
171 OSKext * theKext,
172 kmod_info_t * kmodInfo,
173 void * metaHandle,
174 kernel_segment_command_t * segment,
175 const char * sectionName,
176 uintptr_t textStart,
177 uintptr_t textEnd)
178{
179 kernel_section_t * section;
180 bool result = TRUE;
181
182 for (section = firstsect(sgp: segment);
183 section != NULL;
184 section = nextsect(sgp: segment, sp: section)) {
185 if (strncmp(s1: section->sectname, s2: sectionName, n: sizeof(section->sectname) - 1)) {
186 continue;
187 }
188 if (section->size == 0) {
189 continue;
190 }
191
192 structor_t * structors = (structor_t *)section->addr;
193 if (!structors) {
194 continue;
195 }
196
197 structor_t structor;
198 uintptr_t value;
199 unsigned long num_structors = section->size / sizeof(structor_t);
200 unsigned int hit_null_structor = 0;
201 unsigned long firstIndex = 0;
202
203 if (textStart) {
204 // bsearch for any in range
205 unsigned long baseIdx;
206 unsigned long lim;
207 firstIndex = num_structors;
208 for (lim = num_structors, baseIdx = 0; lim; lim >>= 1) {
209 structor = structors[baseIdx + (lim >> 1)];
210 if (!structor) {
211 panic("%s: null structor", kmodInfo->name);
212 }
213 value = PTRAUTH_STRIP_STRUCTOR(structor);
214 if ((value >= textStart) && (value < textEnd)) {
215 firstIndex = (baseIdx + (lim >> 1));
216 // scan back for the first in range
217 for (; firstIndex; firstIndex--) {
218 structor = structors[firstIndex - 1];
219 value = PTRAUTH_STRIP_STRUCTOR(structor);
220 if ((value < textStart) || (value >= textEnd)) {
221 break;
222 }
223 }
224 break;
225 }
226 if (textStart > value) {
227 // move right
228 baseIdx += (lim >> 1) + 1;
229 lim--;
230 }
231 // else move left
232 }
233 baseIdx = (baseIdx + (lim >> 1));
234 }
235 for (;
236 (firstIndex < num_structors)
237 && (!metaHandle || OSMetaClass::checkModLoad(loadHandle: metaHandle));
238 firstIndex++) {
239 if ((structor = structors[firstIndex])) {
240 value = PTRAUTH_STRIP_STRUCTOR(structor);
241 if ((textStart && (value < textStart))
242 || (textEnd && (value >= textEnd))) {
243 break;
244 }
245 (*structor)();
246 } else if (!hit_null_structor) {
247 hit_null_structor = 1;
248 OSRuntimeLog(theKext, kOSRuntimeLogSpec,
249 "Null structor in kext %s segment %s!",
250 kmodInfo->name, section->segname);
251 }
252 }
253 if (metaHandle) {
254 result = OSMetaClass::checkModLoad(loadHandle: metaHandle);
255 }
256 break;
257 } /* for (section...) */
258 return result;
259}
260
261/*********************************************************************
262*********************************************************************/
263kern_return_t
264OSRuntimeFinalizeCPP(
265 OSKext * theKext)
266{
267 kern_return_t result = KMOD_RETURN_FAILURE;
268 void * metaHandle = NULL;// do not free
269 kernel_mach_header_t * header;
270 kernel_segment_command_t * segment;
271 kmod_info_t * kmodInfo;
272 const char ** sectionNames;
273 uintptr_t textStart;
274 uintptr_t textEnd;
275
276 textStart = 0;
277 textEnd = 0;
278 sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault];
279 if (theKext) {
280 if (!theKext->isCPPInitialized()) {
281 result = KMOD_RETURN_SUCCESS;
282 goto finish;
283 }
284 kmodInfo = theKext->kmod_info;
285 if (!kmodInfo || !kmodInfo->address) {
286 result = kOSKextReturnInvalidArgument;
287 goto finish;
288 }
289 header = (kernel_mach_header_t *)kmodInfo->address;
290 if (theKext->flags.builtin) {
291 header = (kernel_mach_header_t *)g_kernel_kmod_info.address;
292 textStart = kmodInfo->address;
293 textEnd = textStart + kmodInfo->size;
294 sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext];
295 }
296 } else {
297 kmodInfo = &g_kernel_kmod_info;
298 header = (kernel_mach_header_t *)kmodInfo->address;
299 }
300
301 /* OSKext checks for this condition now, but somebody might call
302 * this function directly (the symbol is exported....).
303 */
304 if (OSMetaClass::modHasInstance(kextID: kmodInfo->name)) {
305 // xxx - Don't log under errors? this is more of an info thing
306 OSRuntimeLog(theKext, kOSRuntimeLogSpec,
307 "Can't tear down kext %s C++; classes have instances:",
308 kmodInfo->name);
309 OSKext::reportOSMetaClassInstances(kextIdentifier: kmodInfo->name, msgLogSpec: kOSRuntimeLogSpec);
310 result = kOSMetaClassHasInstances;
311 goto finish;
312 }
313
314 /* Tell the meta class system that we are starting to unload.
315 * metaHandle isn't actually needed on the finalize path,
316 * so we don't check it here, even though OSMetaClass::postModLoad() will
317 * return a failure (it only does actual work on the init path anyhow).
318 */
319 metaHandle = OSMetaClass::preModLoad(kextID: kmodInfo->name);
320
321 OSSymbol::checkForPageUnload(startAddr: (void *)kmodInfo->address,
322 endAddr: (void *)(kmodInfo->address + kmodInfo->size));
323
324 header = (kernel_mach_header_t *)kmodInfo->address;
325 segment = firstsegfromheader(header);
326
327 for (segment = firstsegfromheader(header);
328 segment != NULL;
329 segment = nextsegfromheader(header, seg: segment)) {
330 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment,
331 sectionName: sectionNames[kOSSectionNameFinalizer], textStart, textEnd);
332 }
333
334 (void)OSMetaClass::postModLoad(loadHandle: metaHandle);
335
336 if (theKext) {
337 theKext->setCPPInitialized(false);
338 }
339 result = KMOD_RETURN_SUCCESS;
340finish:
341 return result;
342}
343
344#if defined(HAS_APPLE_PAC)
345#if !KASAN
346/*
347 * Place this function in __KLD,__text on non-kasan builds so it gets unmapped
348 * after CTRR lockdown.
349 */
350__attribute__((noinline, section("__KLD,__text")))
351#endif
352static void
353OSRuntimeSignStructorsInSegment(kernel_segment_command_t *segment)
354{
355 kernel_section_t * section;
356 structor_t * structors;
357 volatile structor_t structor;
358 size_t idx, num_structors;
359
360 for (section = firstsect(sgp: segment);
361 section != NULL;
362 section = nextsect(sgp: segment, sp: section)) {
363 if ((S_MOD_INIT_FUNC_POINTERS != (SECTION_TYPE & section->flags))
364 && (S_MOD_TERM_FUNC_POINTERS != (SECTION_TYPE & section->flags))) {
365 continue;
366 }
367 structors = (structor_t *)section->addr;
368 if (!structors) {
369 continue;
370 }
371 num_structors = section->size / sizeof(structor_t);
372 for (idx = 0; idx < num_structors; idx++) {
373 structor = structors[idx];
374 if (NULL == structor) {
375 continue;
376 }
377 structor = ptrauth_strip(structor, ptrauth_key_function_pointer);
378 structor = ptrauth_sign_unauthenticated(structor, ptrauth_key_function_pointer, ptrauth_function_pointer_type_discriminator(void (*)(void)));
379 structors[idx] = structor;
380 }
381 } /* for (section...) */
382}
383#endif
384
385/*********************************************************************
386*********************************************************************/
387void
388OSRuntimeSignStructors(
389 kernel_mach_header_t * header __unused)
390{
391#if defined(HAS_APPLE_PAC)
392
393 kernel_segment_command_t * segment;
394
395 for (segment = firstsegfromheader(header);
396 segment != NULL;
397 segment = nextsegfromheader(header, seg: segment)) {
398 OSRuntimeSignStructorsInSegment(segment);
399 } /* for (segment...) */
400#endif /* !defined(XXX) && defined(HAS_APPLE_PAC) */
401}
402
403/*********************************************************************
404*********************************************************************/
405void
406OSRuntimeSignStructorsInFileset(
407 kernel_mach_header_t * fileset_header __unused)
408{
409#if defined(HAS_APPLE_PAC)
410 struct load_command *lc;
411
412 lc = (struct load_command *)((uintptr_t)fileset_header + sizeof(*fileset_header));
413 for (uint32_t i = 0; i < fileset_header->ncmds; i++,
414 lc = (struct load_command *)((uintptr_t)lc + lc->cmdsize)) {
415 if (lc->cmd == LC_FILESET_ENTRY) {
416 struct fileset_entry_command *fse;
417 kernel_mach_header_t *mh;
418
419 fse = (struct fileset_entry_command *)(uintptr_t)lc;
420 mh = (kernel_mach_header_t *)((uintptr_t)fse->vmaddr);
421 OSRuntimeSignStructors(header: mh);
422 } else if (lc->cmd == LC_SEGMENT_64) {
423 /*
424 * Slide/adjust all LC_SEGMENT_64 commands in the fileset
425 * (and any sections in those segments)
426 */
427 kernel_segment_command_t *seg;
428 seg = (kernel_segment_command_t *)(uintptr_t)lc;
429 OSRuntimeSignStructorsInSegment(segment: seg);
430 }
431 }
432
433#endif /* defined(HAS_APPLE_PAC) */
434}
435
436/*********************************************************************
437*********************************************************************/
438kern_return_t
439OSRuntimeInitializeCPP(
440 OSKext * theKext)
441{
442 kern_return_t result = KMOD_RETURN_FAILURE;
443 kernel_mach_header_t * header = NULL;
444 void * metaHandle = NULL;// do not free
445 bool load_success = true;
446 kernel_segment_command_t * segment = NULL;// do not free
447 kernel_segment_command_t * failure_segment = NULL; // do not free
448 kmod_info_t * kmodInfo;
449 const char ** sectionNames;
450 uintptr_t textStart;
451 uintptr_t textEnd;
452
453 textStart = 0;
454 textEnd = 0;
455 sectionNames = gOSStructorSectionNames[kOSSectionNamesDefault];
456 if (theKext) {
457 if (theKext->isCPPInitialized()) {
458 result = KMOD_RETURN_SUCCESS;
459 goto finish;
460 }
461
462 kmodInfo = theKext->kmod_info;
463 if (!kmodInfo || !kmodInfo->address) {
464 result = kOSKextReturnInvalidArgument;
465 goto finish;
466 }
467 header = (kernel_mach_header_t *)kmodInfo->address;
468
469 if (theKext->flags.builtin) {
470 header = (kernel_mach_header_t *)g_kernel_kmod_info.address;
471 textStart = kmodInfo->address;
472 textEnd = textStart + kmodInfo->size;
473 sectionNames = gOSStructorSectionNames[kOSSectionNamesBuiltinKext];
474 }
475 } else {
476 kmodInfo = &g_kernel_kmod_info;
477 header = (kernel_mach_header_t *)kmodInfo->address;
478 }
479
480 /* Tell the meta class system that we are starting the load
481 */
482 metaHandle = OSMetaClass::preModLoad(kextID: kmodInfo->name);
483 assert(metaHandle);
484 if (!metaHandle) {
485 goto finish;
486 }
487
488 /* NO GOTO PAST HERE. */
489
490 /* Scan the header for all constructor sections, in any
491 * segment, and invoke the constructors within those sections.
492 */
493 for (segment = firstsegfromheader(header);
494 segment != NULL && load_success;
495 segment = nextsegfromheader(header, seg: segment)) {
496 /* Record the current segment in the event of a failure.
497 */
498 failure_segment = segment;
499 load_success = OSRuntimeCallStructorsInSection(
500 theKext, kmodInfo, metaHandle, segment,
501 sectionName: sectionNames[kOSSectionNameInitializer],
502 textStart, textEnd);
503 } /* for (segment...) */
504
505 /* We failed so call all of the destructors. We must do this before
506 * calling OSMetaClass::postModLoad() as the OSMetaClass destructors
507 * will alter state (in the metaHandle) used by that function.
508 */
509 if (!load_success) {
510 /* Scan the header for all destructor sections, in any
511 * segment, and invoke the constructors within those sections.
512 */
513 for (segment = firstsegfromheader(header);
514 segment != failure_segment && segment != NULL;
515 segment = nextsegfromheader(header, seg: segment)) {
516 OSRuntimeCallStructorsInSection(theKext, kmodInfo, NULL, segment,
517 sectionName: sectionNames[kOSSectionNameFinalizer], textStart, textEnd);
518 } /* for (segment...) */
519 }
520
521 /* Now, regardless of success so far, do the post-init registration
522 * and cleanup. If we had to call the unloadCPP function, static
523 * destructors have removed classes from the stalled list so no
524 * metaclasses will actually be registered.
525 */
526 result = OSMetaClass::postModLoad(loadHandle: metaHandle);
527
528 /* If we've otherwise been fine up to now, but OSMetaClass::postModLoad()
529 * fails (typically due to a duplicate class), tear down all the C++
530 * stuff from the kext. This isn't necessary for libkern/OSMetaClass stuff,
531 * but may be necessary for other C++ code. We ignore the return value
532 * because it's only a fail when there are existing instances of libkern
533 * classes, and there had better not be any created on the C++ init path.
534 */
535 if (load_success && result != KMOD_RETURN_SUCCESS) {
536 (void)OSRuntimeFinalizeCPP(theKext); //kmodInfo, sectionNames, textStart, textEnd);
537 }
538
539 if (theKext && load_success && result == KMOD_RETURN_SUCCESS) {
540 theKext->setCPPInitialized(true);
541 }
542finish:
543 return result;
544}
545
546/*********************************************************************
547* Unload a kernel segment.
548*********************************************************************/
549
550void
551OSRuntimeUnloadCPPForSegment(
552 kernel_segment_command_t * segment)
553{
554 OSRuntimeCallStructorsInSection(NULL, kmodInfo: &g_kernel_kmod_info, NULL, segment,
555 sectionName: gOSStructorSectionNames[kOSSectionNamesDefault][kOSSectionNameFinalizer], textStart: 0, textEnd: 0);
556}
557
558#if PRAGMA_MARK
559#pragma mark C++ Allocators & Deallocators
560#endif /* PRAGMA_MARK */
561/*********************************************************************
562* C++ Allocators & Deallocators
563*********************************************************************/
564__typed_allocators_ignore_push
565
566void *
567operator new(size_t size)
568{
569 assert(size);
570 return kheap_alloc(KERN_OS_MALLOC, size,
571 Z_VM_TAG_BT(Z_WAITOK_ZERO, VM_KERN_MEMORY_LIBKERN));
572}
573
574void
575operator delete(void * addr)
576#if __cplusplus >= 201103L
577noexcept
578#endif
579{
580 kheap_free_addr(KERN_OS_MALLOC, addr);
581 return;
582}
583
584void *
585operator new[](unsigned long size)
586{
587 return kheap_alloc(KERN_OS_MALLOC, size,
588 Z_VM_TAG_BT(Z_WAITOK_ZERO, VM_KERN_MEMORY_LIBKERN));
589}
590
591void
592operator delete[](void * ptr)
593#if __cplusplus >= 201103L
594noexcept
595#endif
596{
597 if (ptr) {
598#if KASAN
599 /*
600 * Unpoison the C++ array cookie inserted (but not removed) by the
601 * compiler on new[].
602 */
603 kasan_unpoison_cxx_array_cookie(ptr);
604#endif
605 kheap_free_addr(KERN_OS_MALLOC, ptr);
606 }
607 return;
608}
609
610#if __cplusplus >= 201103L
611
612void
613operator delete(void * addr, size_t sz) noexcept
614{
615 kheap_free(KERN_OS_MALLOC, addr, sz);
616}
617
618void
619operator delete[](void * addr, size_t sz) noexcept
620{
621 if (addr) {
622 kheap_free(KERN_OS_MALLOC, addr, sz);
623 }
624}
625
626__typed_allocators_ignore_pop
627
628#endif /* __cplusplus >= 201103L */
629
630/* PR-6481964 - The compiler is going to check for size overflows in calls to
631 * new[], and if there is an overflow, it will call __throw_length_error.
632 * This is an unrecoverable error by the C++ standard, so we must panic here.
633 *
634 * We have to put the function inside the std namespace because of how the
635 * compiler expects the name to be mangled.
636 */
637namespace std {
638void __dead2
639__throw_length_error(const char *msg __unused)
640{
641 panic("Size of array created by new[] has overflowed");
642}
643};
644