1/*
2 * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <sys/sysctl.h>
30#include <kern/backtrace.h>
31#include <kern/host.h>
32#include <kern/zalloc.h>
33
34#include <IOKit/system.h>
35#include <libkern/c++/OSKext.h>
36#include <libkern/OSAtomic.h>
37
38#include <IOKit/IOStatisticsPrivate.h>
39#include <IOKit/IOUserClient.h>
40#include <IOKit/IOEventSource.h>
41#include <IOKit/IOKitDebug.h>
42
43#if IOKITSTATS
44bool IOStatistics::enabled = false;
45
46uint32_t IOStatistics::sequenceID = 0;
47
48uint32_t IOStatistics::lastClassIndex = 0;
49uint32_t IOStatistics::lastKextIndex = 0;
50
51uint32_t IOStatistics::loadedKexts = 0;
52uint32_t IOStatistics::registeredClasses = 0;
53uint32_t IOStatistics::registeredCounters = 0;
54uint32_t IOStatistics::registeredWorkloops = 0;
55
56uint32_t IOStatistics::attachedEventSources = 0;
57
58IOWorkLoopDependency *IOStatistics::nextWorkLoopDependency = NULL;
59
60/* Logging */
61
62#define LOG_LEVEL 0
63
64#define LOG(level, format, ...) \
65do { \
66 if (level <= LOG_LEVEL) \
67 printf(format, ##__VA_ARGS__); \
68} while (0)
69
70/* Locks */
71
72IORWLock *IOStatistics::lock = NULL;
73
74/* Kext tree */
75
76KextNode *IOStatistics::kextHint = NULL;
77
78IOStatistics::KextTreeHead IOStatistics::kextHead = RB_INITIALIZER(&IOStatistics::kextHead);
79
80int
81IOStatistics::kextNodeCompare(KextNode *e1, KextNode *e2)
82{
83 if (e1->kext < e2->kext) {
84 return -1;
85 } else if (e1->kext > e2->kext) {
86 return 1;
87 } else {
88 return 0;
89 }
90}
91
92RB_GENERATE(IOStatistics::KextTree, KextNode, link, kextNodeCompare);
93
94/* Kext tree ordered by address */
95
96IOStatistics::KextAddressTreeHead IOStatistics::kextAddressHead = RB_INITIALIZER(&IOStatistics::kextAddressHead);
97
98int
99IOStatistics::kextAddressNodeCompare(KextNode *e1, KextNode *e2)
100{
101 if (e1->address < e2->address) {
102 return -1;
103 } else if (e1->address > e2->address) {
104 return 1;
105 } else {
106 return 0;
107 }
108}
109
110RB_GENERATE(IOStatistics::KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
111
112/* Class tree */
113
114IOStatistics::ClassTreeHead IOStatistics::classHead = RB_INITIALIZER(&IOStatistics::classHead);
115
116int
117IOStatistics::classNodeCompare(ClassNode *e1, ClassNode *e2)
118{
119 if (e1->metaClass < e2->metaClass) {
120 return -1;
121 } else if (e1->metaClass > e2->metaClass) {
122 return 1;
123 } else {
124 return 0;
125 }
126}
127
128RB_GENERATE(IOStatistics::ClassTree, ClassNode, tLink, classNodeCompare);
129
130/* Workloop dependencies */
131
132int
133IOWorkLoopCounter::loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2)
134{
135 if (e1->loadTag < e2->loadTag) {
136 return -1;
137 } else if (e1->loadTag > e2->loadTag) {
138 return 1;
139 } else {
140 return 0;
141 }
142}
143
144RB_GENERATE(IOWorkLoopCounter::DependencyTree, IOWorkLoopDependency, link, IOWorkLoopCounter::loadTagCompare);
145
146/* sysctl stuff */
147
148static int
149oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req)
150{
151 int error = EINVAL;
152 uint32_t request = arg2;
153
154 if (!IOStatistics::isEnabled()) {
155 return ENOENT;
156 }
157
158 switch (request) {
159 case kIOStatisticsGeneral:
160 error = IOStatistics::getStatistics(req);
161 break;
162 case kIOStatisticsWorkLoop:
163 error = IOStatistics::getWorkLoopStatistics(req);
164 break;
165 case kIOStatisticsUserClient:
166 error = IOStatistics::getUserClientStatistics(req);
167 break;
168 default:
169 break;
170 }
171
172 return error;
173}
174
175SYSCTL_NODE(_debug, OID_AUTO, iokit_statistics, CTLFLAG_RW | CTLFLAG_LOCKED, NULL, "IOStatistics");
176
177static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, general,
178 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED,
179 NULL, kIOStatisticsGeneral, oid_sysctl, "S", "");
180
181static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, workloop,
182 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED,
183 NULL, kIOStatisticsWorkLoop, oid_sysctl, "S", "");
184
185static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, userclient,
186 CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
187 NULL, kIOStatisticsUserClient, oid_sysctl, "S", "");
188
189
190void
191IOStatistics::initialize()
192{
193 if (enabled) {
194 return;
195 }
196
197 /* Only enabled if the boot argument is set. */
198 if (!(kIOStatistics & gIOKitDebug)) {
199 return;
200 }
201
202 lock = IORWLockAlloc();
203 if (!lock) {
204 return;
205 }
206
207 nextWorkLoopDependency = kalloc_type(IOWorkLoopDependency, Z_WAITOK);
208 if (!nextWorkLoopDependency) {
209 return;
210 }
211
212 enabled = true;
213}
214
215void
216IOStatistics::onKextLoad(OSKext *kext, kmod_info_t *kmod_info)
217{
218 KextNode *ke;
219
220 assert(kext && kmod_info);
221
222 if (!enabled) {
223 return;
224 }
225
226 LOG(1, "IOStatistics::onKextLoad: %s, tag %d, address 0x%llx, address end 0x%llx\n",
227 kext->getIdentifierCString(), kmod_info->id, (uint64_t)kmod_info->address, (uint64_t)(kmod_info->address + kmod_info->size));
228
229 ke = kalloc_type(KextNode, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
230 if (!ke) {
231 return;
232 }
233
234 ke->kext = kext;
235 ke->loadTag = kmod_info->id;
236 ke->address = kmod_info->address;
237 ke->address_end = kmod_info->address + kmod_info->size;
238
239 SLIST_INIT(&ke->classList);
240 TAILQ_INIT(&ke->userClientCallList);
241
242 IORWLockWrite(lock);
243
244 RB_INSERT(KextTree, &kextHead, ke);
245 RB_INSERT(KextAddressTree, &kextAddressHead, ke);
246
247 sequenceID++;
248 loadedKexts++;
249 lastKextIndex++;
250
251 IORWLockUnlock(lock);
252}
253
254void
255IOStatistics::onKextUnload(OSKext *kext)
256{
257 KextNode sought, *found;
258
259 assert(kext);
260
261 if (!enabled) {
262 return;
263 }
264
265 LOG(1, "IOStatistics::onKextUnload: %s\n", kext->getIdentifierCString());
266
267 IORWLockWrite(lock);
268
269 sought.kext = kext;
270 found = RB_FIND(KextTree, &kextHead, &sought);
271 if (found) {
272 IOWorkLoopCounter *wlc;
273 IOUserClientProcessEntry *uce;
274
275 /* Disconnect workloop counters; cleanup takes place in unregisterWorkLoop() */
276 while ((wlc = SLIST_FIRST(&found->workLoopList))) {
277 SLIST_REMOVE_HEAD(&found->workLoopList, link);
278 wlc->parentKext = NULL;
279 }
280
281 /* Free up the user client list */
282 while ((uce = TAILQ_FIRST(&found->userClientCallList))) {
283 TAILQ_REMOVE(&found->userClientCallList, uce, link);
284 kfree_type(IOUserClientProcessEntry, uce);
285 }
286
287 /* Remove from kext trees */
288 RB_REMOVE(KextTree, &kextHead, found);
289 RB_REMOVE(KextAddressTree, &kextAddressHead, found);
290
291 /*
292 * Clear a matching kextHint to avoid use after free in
293 * onClassAdded() for a class add after a KEXT unload.
294 */
295 if (found == kextHint) {
296 kextHint = NULL;
297 }
298
299 /* Finally, free the class node */
300 kfree_type(KextNode, found);
301
302 sequenceID++;
303 loadedKexts--;
304 } else {
305 panic("IOStatistics::onKextUnload: cannot find kext: %s", kext->getIdentifierCString());
306 }
307
308 IORWLockUnlock(lock);
309}
310
311void
312IOStatistics::onClassAdded(OSKext *parentKext, OSMetaClass *metaClass)
313{
314 ClassNode *ce;
315 KextNode soughtKext, *foundKext = NULL;
316
317 assert(parentKext && metaClass);
318
319 if (!enabled) {
320 return;
321 }
322
323 LOG(1, "IOStatistics::onClassAdded: %s\n", metaClass->getClassName());
324
325 ce = kalloc_type(ClassNode, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
326 if (!ce) {
327 return;
328 }
329
330 IORWLockWrite(lock);
331
332 /* Hinted? */
333 if (kextHint && kextHint->kext == parentKext) {
334 foundKext = kextHint;
335 } else {
336 soughtKext.kext = parentKext;
337 foundKext = RB_FIND(KextTree, &kextHead, &soughtKext);
338 }
339
340 if (foundKext) {
341 ClassNode soughtClass, *foundClass = NULL;
342 const OSMetaClass *superClass;
343
344 ce->metaClass = metaClass;
345 ce->classID = lastClassIndex++;
346 ce->parentKext = foundKext;
347
348 /* Has superclass? */
349 superClass = ce->metaClass->getSuperClass();
350 if (superClass) {
351 soughtClass.metaClass = superClass;
352 foundClass = RB_FIND(ClassTree, &classHead, &soughtClass);
353 }
354 ce->superClassID = foundClass ? foundClass->classID : (uint32_t)(-1);
355
356 SLIST_INIT(&ce->counterList);
357 SLIST_INIT(&ce->userClientList);
358
359 RB_INSERT(ClassTree, &classHead, ce);
360 SLIST_INSERT_HEAD(&foundKext->classList, ce, lLink);
361
362 foundKext->classes++;
363
364 kextHint = foundKext;
365
366 sequenceID++;
367 registeredClasses++;
368 } else {
369 panic("IOStatistics::onClassAdded: cannot find parent kext: %s", parentKext->getIdentifierCString());
370 }
371
372 IORWLockUnlock(lock);
373}
374
375void
376IOStatistics::onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass)
377{
378 ClassNode sought, *found;
379
380 assert(parentKext && metaClass);
381
382 if (!enabled) {
383 return;
384 }
385
386 LOG(1, "IOStatistics::onClassRemoved: %s\n", metaClass->getClassName());
387
388 IORWLockWrite(lock);
389
390 sought.metaClass = metaClass;
391 found = RB_FIND(ClassTree, &classHead, &sought);
392 if (found) {
393 IOEventSourceCounter *esc;
394 IOUserClientCounter *ucc;
395
396 /* Free up the list of counters */
397 while ((esc = SLIST_FIRST(&found->counterList))) {
398 SLIST_REMOVE_HEAD(&found->counterList, link);
399 kfree_type(IOEventSourceCounter, esc);
400 }
401
402 /* Free up the user client list */
403 while ((ucc = SLIST_FIRST(&found->userClientList))) {
404 SLIST_REMOVE_HEAD(&found->userClientList, link);
405 kfree_type(IOUserClientCounter, ucc);
406 }
407
408 /* Remove from class tree */
409 RB_REMOVE(ClassTree, &classHead, found);
410
411 /* Remove from parent */
412 SLIST_REMOVE(&found->parentKext->classList, found, ClassNode, lLink);
413
414 /* Finally, free the class node */
415 kfree_type(ClassNode, found);
416
417 sequenceID++;
418 registeredClasses--;
419 } else {
420 panic("IOStatistics::onClassRemoved: cannot find class: %s", metaClass->getClassName());
421 }
422
423 IORWLockUnlock(lock);
424}
425
426IOEventSourceCounter *
427IOStatistics::registerEventSource(OSObject *inOwner)
428{
429 IOEventSourceCounter *counter = NULL;
430 ClassNode sought, *found = NULL;
431 boolean_t createDummyCounter = FALSE;
432
433 assert(inOwner);
434
435 if (!enabled) {
436 return NULL;
437 }
438
439 counter = kalloc_type(IOEventSourceCounter, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
440 if (!counter) {
441 return NULL;
442 }
443
444 IORWLockWrite(lock);
445
446 /* Workaround for <rdar://problem/7158117> - create a dummy counter when inOwner is bad.
447 * We use retainCount here as our best indication that the pointer is awry.
448 */
449 if (inOwner->retainCount > 0xFFFFFF) {
450 kprintf(fmt: "IOStatistics::registerEventSource - bad metaclass %p\n", inOwner);
451 createDummyCounter = TRUE;
452 } else {
453 sought.metaClass = inOwner->getMetaClass();
454 found = RB_FIND(ClassTree, &classHead, &sought);
455 }
456
457 if (found) {
458 counter->parentClass = found;
459 SLIST_INSERT_HEAD(&found->counterList, counter, link);
460 registeredCounters++;
461 }
462
463 if (!(createDummyCounter || found)) {
464 panic("IOStatistics::registerEventSource: cannot find parent class: %s", inOwner->getMetaClass()->getClassName());
465 }
466
467 IORWLockUnlock(lock);
468
469 return counter;
470}
471
472void
473IOStatistics::unregisterEventSource(IOEventSourceCounter *counter)
474{
475 if (!counter) {
476 return;
477 }
478
479 IORWLockWrite(lock);
480
481 if (counter->parentClass) {
482 SLIST_REMOVE(&counter->parentClass->counterList, counter, IOEventSourceCounter, link);
483 registeredCounters--;
484 }
485 kfree_type(IOEventSourceCounter, counter);
486
487 IORWLockUnlock(lock);
488}
489
490IOWorkLoopCounter*
491IOStatistics::registerWorkLoop(IOWorkLoop *workLoop)
492{
493 IOWorkLoopCounter *counter = NULL;
494 KextNode *found;
495
496 assert(workLoop);
497
498 if (!enabled) {
499 return NULL;
500 }
501
502 counter = kalloc_type(IOWorkLoopCounter, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
503 if (!counter) {
504 return NULL;
505 }
506
507 found = getKextNodeFromBacktrace(TRUE);
508 if (!found) {
509 panic("IOStatistics::registerWorkLoop: cannot find parent kext");
510 }
511
512 counter->parentKext = found;
513 counter->workLoop = workLoop;
514 RB_INIT(&counter->dependencyHead);
515 SLIST_INSERT_HEAD(&found->workLoopList, counter, link);
516 registeredWorkloops++;
517
518 releaseKextNode(node: found);
519
520 return counter;
521}
522
523void
524IOStatistics::unregisterWorkLoop(IOWorkLoopCounter *counter)
525{
526 if (!counter) {
527 return;
528 }
529
530 IORWLockWrite(lock);
531 if (counter->parentKext) {
532 SLIST_REMOVE(&counter->parentKext->workLoopList, counter, IOWorkLoopCounter, link);
533 }
534 kfree_type(IOWorkLoopCounter, counter);
535 registeredWorkloops--;
536
537 IORWLockUnlock(lock);
538}
539
540IOUserClientCounter *
541IOStatistics::registerUserClient(IOUserClient *userClient)
542{
543 ClassNode sought, *found;
544 IOUserClientCounter *counter = NULL;
545
546 assert(userClient);
547
548 if (!enabled) {
549 return NULL;
550 }
551
552 counter = kalloc_type(IOUserClientCounter, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
553 if (!counter) {
554 return NULL;
555 }
556
557 IORWLockWrite(lock);
558
559 sought.metaClass = userClient->getMetaClass();
560
561 found = RB_FIND(ClassTree, &classHead, &sought);
562 if (found) {
563 counter->parentClass = found;
564 SLIST_INSERT_HEAD(&found->userClientList, counter, link);
565 } else {
566 panic("IOStatistics::registerUserClient: cannot find parent class: %s", sought.metaClass->getClassName());
567 }
568
569 IORWLockUnlock(lock);
570
571 return counter;
572}
573
574void
575IOStatistics::unregisterUserClient(IOUserClientCounter *counter)
576{
577 if (!counter) {
578 return;
579 }
580
581 IORWLockWrite(lock);
582
583 SLIST_REMOVE(&counter->parentClass->userClientList, counter, IOUserClientCounter, link);
584 kfree_type(IOUserClientCounter, counter);
585
586 IORWLockUnlock(lock);
587}
588
589void
590IOStatistics::attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
591{
592 if (!wlc) {
593 return;
594 }
595
596 IORWLockWrite(lock);
597
598 if (!nextWorkLoopDependency) {
599 return;
600 }
601
602 attachedEventSources++;
603 wlc->attachedEventSources++;
604
605 /* Track the kext dependency */
606 nextWorkLoopDependency->loadTag = esc->parentClass->parentKext->loadTag;
607 if (NULL == RB_INSERT(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, nextWorkLoopDependency)) {
608 nextWorkLoopDependency = kalloc_type(IOWorkLoopDependency, Z_WAITOK);
609 }
610
611 IORWLockUnlock(lock);
612}
613
614void
615IOStatistics::detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
616{
617 IOWorkLoopDependency sought, *found;
618
619 if (!wlc) {
620 return;
621 }
622
623 IORWLockWrite(lock);
624
625 attachedEventSources--;
626 wlc->attachedEventSources--;
627
628 sought.loadTag = esc->parentClass->parentKext->loadTag;
629
630 found = RB_FIND(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, &sought);
631 if (found) {
632 RB_REMOVE(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, found);
633 kfree_type(IOWorkLoopDependency, found);
634 }
635
636 IORWLockUnlock(lock);
637}
638
639int
640IOStatistics::getStatistics(sysctl_req *req)
641{
642 int error;
643 uint32_t calculatedSize, size;
644 char *buffer, *ptr;
645 IOStatisticsHeader *header;
646
647 assert(IOStatistics::enabled && req);
648
649 IORWLockRead(IOStatistics::lock);
650
651 /* Work out how much we need to allocate. IOStatisticsKext is of variable size. */
652 calculatedSize = sizeof(IOStatisticsHeader) +
653 sizeof(IOStatisticsGlobal) +
654 (sizeof(IOStatisticsKext) * loadedKexts) + (sizeof(uint32_t) * registeredClasses) +
655 (sizeof(IOStatisticsMemory) * loadedKexts) +
656 (sizeof(IOStatisticsClass) * registeredClasses) +
657 (sizeof(IOStatisticsCounter) * registeredClasses) +
658 (sizeof(IOStatisticsKextIdentifier) * loadedKexts) +
659 (sizeof(IOStatisticsClassName) * registeredClasses);
660
661 /* Size request? */
662 if (req->oldptr == USER_ADDR_NULL) {
663 error = SYSCTL_OUT(req, NULL, calculatedSize);
664 goto exit;
665 }
666
667 /* Read only */
668 if (req->newptr != USER_ADDR_NULL) {
669 error = EPERM;
670 goto exit;
671 }
672
673 buffer = (char*)kalloc_data(calculatedSize, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
674 if (!buffer) {
675 error = ENOMEM;
676 goto exit;
677 }
678
679 ptr = buffer;
680
681 header = (IOStatisticsHeader*)((void*)ptr);
682
683 header->sig = IOSTATISTICS_SIG;
684 header->ver = IOSTATISTICS_VER;
685
686 header->seq = sequenceID;
687
688 ptr += sizeof(IOStatisticsHeader);
689
690 /* Global data - seq, timers, interrupts, etc) */
691 header->globalStatsOffset = sizeof(IOStatisticsHeader);
692 size = copyGlobalStatistics(stats: (IOStatisticsGlobal*)((void*)ptr));
693 ptr += size;
694
695 /* Kext statistics */
696 header->kextStatsOffset = header->globalStatsOffset + size;
697 size = copyKextStatistics(stats: (IOStatisticsKext*)((void*)ptr));
698 ptr += size;
699
700 /* Memory allocation info */
701 header->memoryStatsOffset = header->kextStatsOffset + size;
702 size = copyMemoryStatistics(stats: (IOStatisticsMemory*)((void*)ptr));
703 ptr += size;
704
705 /* Class statistics */
706 header->classStatsOffset = header->memoryStatsOffset + size;
707 size = copyClassStatistics(stats: (IOStatisticsClass*)((void*)ptr));
708 ptr += size;
709
710 /* Dynamic class counter data */
711 header->counterStatsOffset = header->classStatsOffset + size;
712 size = copyCounterStatistics(stats: (IOStatisticsCounter*)((void*)ptr));
713 ptr += size;
714
715 /* Kext identifiers */
716 header->kextIdentifiersOffset = header->counterStatsOffset + size;
717 size = copyKextIdentifiers(kextIDs: (IOStatisticsKextIdentifier*)((void*)ptr));
718 ptr += size;
719
720 /* Class names */
721 header->classNamesOffset = header->kextIdentifiersOffset + size;
722 size = copyClassNames(classNames: (IOStatisticsClassName*)ptr);
723 ptr += size;
724
725 LOG(2, "IOStatistics::getStatistics - calculatedSize 0x%x, kexts 0x%x, classes 0x%x.\n",
726 calculatedSize, loadedKexts, registeredClasses);
727
728 assert((uint32_t)(ptr - buffer) == calculatedSize );
729
730 error = SYSCTL_OUT(req, buffer, calculatedSize);
731
732 kfree_data(buffer, calculatedSize);
733
734exit:
735 IORWLockUnlock(IOStatistics::lock);
736 return error;
737}
738
739int
740IOStatistics::getWorkLoopStatistics(sysctl_req *req)
741{
742 int error;
743 uint32_t calculatedSize, size;
744 char *buffer;
745 IOStatisticsWorkLoopHeader *header;
746
747 assert(IOStatistics::enabled && req);
748
749 IORWLockRead(IOStatistics::lock);
750
751 /* Approximate how much we need to allocate (worse case estimate) */
752 calculatedSize = sizeof(IOStatisticsWorkLoop) * registeredWorkloops +
753 sizeof(uint32_t) * attachedEventSources;
754
755 /* Size request? */
756 if (req->oldptr == USER_ADDR_NULL) {
757 error = SYSCTL_OUT(req, NULL, calculatedSize);
758 goto exit;
759 }
760
761 /* Read only */
762 if (req->newptr != USER_ADDR_NULL) {
763 error = EPERM;
764 goto exit;
765 }
766
767 buffer = (char*)kalloc_data(calculatedSize, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
768 if (!buffer) {
769 error = ENOMEM;
770 goto exit;
771 }
772 header = (IOStatisticsWorkLoopHeader*)((void*)buffer);
773
774 header->sig = IOSTATISTICS_SIG_WORKLOOP;
775 header->ver = IOSTATISTICS_VER;
776
777 header->seq = sequenceID;
778
779 header->workloopCount = registeredWorkloops;
780
781 size = copyWorkLoopStatistics(workLoopStats: &header->workLoopStats);
782
783 LOG(2, "IOStatistics::getWorkLoopStatistics: calculatedSize %d, size %d\n", calculatedSize, size);
784
785 assert( size <= calculatedSize );
786
787 error = SYSCTL_OUT(req, buffer, size);
788
789 kfree_data(buffer, calculatedSize);
790
791exit:
792 IORWLockUnlock(IOStatistics::lock);
793 return error;
794}
795
796int
797IOStatistics::getUserClientStatistics(sysctl_req *req)
798{
799 int error;
800 uint32_t calculatedSize, size;
801 char *buffer;
802 uint32_t requestedLoadTag = 0;
803 IOStatisticsUserClientHeader *header;
804
805 assert(IOStatistics::enabled && req);
806
807 IORWLockRead(IOStatistics::lock);
808
809 /* Work out how much we need to allocate */
810 calculatedSize = sizeof(IOStatisticsUserClientHeader) +
811 sizeof(IOStatisticsUserClientCall) * IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS * loadedKexts;
812
813 /* Size request? */
814 if (req->oldptr == USER_ADDR_NULL) {
815 error = SYSCTL_OUT(req, NULL, calculatedSize);
816 goto exit;
817 }
818
819 /* Kext request (potentially) valid? */
820 if (!req->newptr || req->newlen < sizeof(requestedLoadTag)) {
821 error = EINVAL;
822 goto exit;
823 }
824
825 error = SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag));
826 if (error) {
827 goto exit;
828 }
829
830 LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag);
831
832 buffer = (char*)kalloc_data(calculatedSize, (zalloc_flags_t)(Z_WAITOK | Z_ZERO));
833 if (!buffer) {
834 error = ENOMEM;
835 goto exit;
836 }
837 header = (IOStatisticsUserClientHeader*)((void*)buffer);
838
839 header->sig = IOSTATISTICS_SIG_USERCLIENT;
840 header->ver = IOSTATISTICS_VER;
841
842 header->seq = sequenceID;
843
844 header->processes = 0;
845
846 size = copyUserClientStatistics(stats: header, loadTag: requestedLoadTag);
847
848 assert((sizeof(IOStatisticsUserClientHeader) + size) <= calculatedSize);
849
850 if (size) {
851 error = SYSCTL_OUT(req, buffer, sizeof(IOStatisticsUserClientHeader) + size);
852 } else {
853 error = EINVAL;
854 }
855
856 kfree_data(buffer, calculatedSize);
857
858exit:
859 IORWLockUnlock(IOStatistics::lock);
860 return error;
861}
862
863uint32_t
864IOStatistics::copyGlobalStatistics(IOStatisticsGlobal *stats)
865{
866 stats->kextCount = loadedKexts;
867 stats->classCount = registeredClasses;
868 stats->workloops = registeredWorkloops;
869
870 return sizeof(IOStatisticsGlobal);
871}
872
873uint32_t
874IOStatistics::copyKextStatistics(IOStatisticsKext *stats)
875{
876 KextNode *ke;
877 ClassNode *ce;
878 uint32_t index = 0;
879
880 RB_FOREACH(ke, KextTree, &kextHead) {
881 stats->loadTag = ke->loadTag;
882 ke->kext->getSizeInfo(loadSize: &stats->loadSize, wiredSize: &stats->wiredSize);
883
884 stats->classes = ke->classes;
885
886 /* Append indices of owned classes */
887 SLIST_FOREACH(ce, &ke->classList, lLink) {
888 stats->classIndexes[index++] = ce->classID;
889 }
890
891 stats = (IOStatisticsKext *)((void*)((char*)stats + sizeof(IOStatisticsKext) + (ke->classes * sizeof(uint32_t))));
892 }
893
894 return sizeof(IOStatisticsKext) * loadedKexts + sizeof(uint32_t) * registeredClasses;
895}
896
897uint32_t
898IOStatistics::copyMemoryStatistics(IOStatisticsMemory *stats)
899{
900 KextNode *ke;
901
902 RB_FOREACH(ke, KextTree, &kextHead) {
903 stats->allocatedSize = ke->memoryCounters[kIOStatisticsMalloc];
904 stats->freedSize = ke->memoryCounters[kIOStatisticsFree];
905 stats->allocatedAlignedSize = ke->memoryCounters[kIOStatisticsMallocAligned];
906 stats->freedAlignedSize = ke->memoryCounters[kIOStatisticsFreeAligned];
907 stats->allocatedContiguousSize = ke->memoryCounters[kIOStatisticsMallocContiguous];
908 stats->freedContiguousSize = ke->memoryCounters[kIOStatisticsFreeContiguous];
909 stats->allocatedPageableSize = ke->memoryCounters[kIOStatisticsMallocPageable];
910 stats->freedPageableSize = ke->memoryCounters[kIOStatisticsFreePageable];
911 stats++;
912 }
913
914 return sizeof(IOStatisticsMemory) * loadedKexts;
915}
916
917uint32_t
918IOStatistics::copyClassStatistics(IOStatisticsClass *stats)
919{
920 KextNode *ke;
921 ClassNode *ce;
922
923 RB_FOREACH(ke, KextTree, &kextHead) {
924 SLIST_FOREACH(ce, &ke->classList, lLink) {
925 stats->classID = ce->classID;
926 stats->superClassID = ce->superClassID;
927 stats->classSize = ce->metaClass->getClassSize();
928
929 stats++;
930 }
931 }
932
933 return sizeof(IOStatisticsClass) * registeredClasses;
934}
935
936uint32_t
937IOStatistics::copyCounterStatistics(IOStatisticsCounter *stats)
938{
939 KextNode *ke;
940 ClassNode *ce;
941
942 RB_FOREACH(ke, KextTree, &kextHead) {
943 SLIST_FOREACH(ce, &ke->classList, lLink) {
944 IOUserClientCounter *userClientCounter;
945 IOEventSourceCounter *counter;
946
947 stats->classID = ce->classID;
948 stats->classInstanceCount = ce->metaClass->getInstanceCount();
949
950 IOStatisticsUserClients *uc = &stats->userClientStatistics;
951
952 /* User client counters */
953 SLIST_FOREACH(userClientCounter, &ce->userClientList, link) {
954 uc->clientCalls += userClientCounter->clientCalls;
955 uc->created++;
956 }
957
958 IOStatisticsInterruptEventSources *iec = &stats->interruptEventSourceStatistics;
959 IOStatisticsInterruptEventSources *fiec = &stats->filterInterruptEventSourceStatistics;
960 IOStatisticsTimerEventSources *tec = &stats->timerEventSourceStatistics;
961 IOStatisticsCommandGates *cgc = &stats->commandGateStatistics;
962 IOStatisticsCommandQueues *cqc = &stats->commandQueueStatistics;
963 IOStatisticsDerivedEventSources *dec = &stats->derivedEventSourceStatistics;
964
965 /* Event source counters */
966 SLIST_FOREACH(counter, &ce->counterList, link) {
967 switch (counter->type) {
968 case kIOStatisticsInterruptEventSourceCounter:
969 iec->created++;
970 iec->produced += counter->u.interrupt.produced;
971 iec->checksForWork += counter->u.interrupt.checksForWork;
972 break;
973 case kIOStatisticsFilterInterruptEventSourceCounter:
974 fiec->created++;
975 fiec->produced += counter->u.filter.produced;
976 fiec->checksForWork += counter->u.filter.checksForWork;
977 break;
978 case kIOStatisticsTimerEventSourceCounter:
979 tec->created++;
980 tec->timeouts += counter->u.timer.timeouts;
981 tec->checksForWork += counter->u.timer.checksForWork;
982 tec->timeOnGate += counter->timeOnGate;
983 tec->closeGateCalls += counter->closeGateCalls;
984 tec->openGateCalls += counter->openGateCalls;
985 break;
986 case kIOStatisticsCommandGateCounter:
987 cgc->created++;
988 cgc->timeOnGate += counter->timeOnGate;
989 cgc->actionCalls += counter->u.commandGate.actionCalls;
990 break;
991 case kIOStatisticsCommandQueueCounter:
992 cqc->created++;
993 cqc->actionCalls += counter->u.commandQueue.actionCalls;
994 break;
995 case kIOStatisticsDerivedEventSourceCounter:
996 dec->created++;
997 dec->timeOnGate += counter->timeOnGate;
998 dec->closeGateCalls += counter->closeGateCalls;
999 dec->openGateCalls += counter->openGateCalls;
1000 break;
1001 default:
1002 break;
1003 }
1004 }
1005
1006 stats++;
1007 }
1008 }
1009
1010 return sizeof(IOStatisticsCounter) * registeredClasses;
1011}
1012
1013uint32_t
1014IOStatistics::copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs)
1015{
1016 KextNode *ke;
1017
1018 RB_FOREACH(ke, KextTree, &kextHead) {
1019 strncpy(kextIDs->identifier, ke->kext->getIdentifierCString(), kIOStatisticsDriverNameLength);
1020 kextIDs++;
1021 }
1022
1023 return sizeof(IOStatisticsKextIdentifier) * loadedKexts;
1024}
1025
1026uint32_t
1027IOStatistics::copyClassNames(IOStatisticsClassName *classNames)
1028{
1029 KextNode *ke;
1030 ClassNode *ce;
1031
1032 RB_FOREACH(ke, KextTree, &kextHead) {
1033 SLIST_FOREACH(ce, &ke->classList, lLink) {
1034 strncpy(classNames->name, ce->metaClass->getClassName(), kIOStatisticsClassNameLength);
1035 classNames++;
1036 }
1037 }
1038
1039 return sizeof(IOStatisticsClassName) * registeredClasses;
1040}
1041
1042uint32_t
1043IOStatistics::copyWorkLoopStatistics(IOStatisticsWorkLoop *stats)
1044{
1045 KextNode *ke;
1046 IOWorkLoopCounter *wlc;
1047 IOWorkLoopDependency *dependentNode;
1048 uint32_t size, accumulatedSize = 0;
1049
1050 RB_FOREACH(ke, KextTree, &kextHead) {
1051 SLIST_FOREACH(wlc, &ke->workLoopList, link) {
1052 stats->kextLoadTag = ke->loadTag;
1053 stats->attachedEventSources = wlc->attachedEventSources;
1054 stats->timeOnGate = wlc->timeOnGate;
1055 stats->dependentKexts = 0;
1056 RB_FOREACH(dependentNode, IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead) {
1057 stats->dependentKextLoadTags[stats->dependentKexts] = dependentNode->loadTag;
1058 stats->dependentKexts++;
1059 }
1060
1061 size = sizeof(IOStatisticsWorkLoop) + (sizeof(uint32_t) * stats->dependentKexts);
1062
1063 accumulatedSize += size;
1064 stats = (IOStatisticsWorkLoop*)((void*)((char*)stats + size));
1065 }
1066 }
1067
1068 return accumulatedSize;
1069}
1070
1071uint32_t
1072IOStatistics::copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag)
1073{
1074 KextNode *sought, *found = NULL;
1075 uint32_t procs = 0;
1076 IOUserClientProcessEntry *processEntry;
1077
1078 RB_FOREACH(sought, KextTree, &kextHead) {
1079 if (sought->loadTag == loadTag) {
1080 found = sought;
1081 break;
1082 }
1083 }
1084
1085 if (!found) {
1086 return 0;
1087 }
1088
1089 TAILQ_FOREACH(processEntry, &found->userClientCallList, link) {
1090 strncpy(stats->userClientCalls[procs].processName, processEntry->processName, kIOStatisticsProcessNameLength);
1091 stats->userClientCalls[procs].pid = processEntry->pid;
1092 stats->userClientCalls[procs].calls = processEntry->calls;
1093 stats->processes++;
1094 procs++;
1095 }
1096
1097 return sizeof(IOStatisticsUserClientCall) * stats->processes;
1098}
1099
1100void
1101IOStatistics::storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter)
1102{
1103 OSString *ossUserClientCreator = NULL;
1104 int32_t pid = -1;
1105 KextNode *parentKext;
1106 IOUserClientProcessEntry *entry, *nextEntry, *prevEntry = NULL;
1107 uint32_t count = 0;
1108 const char *ptr = NULL;
1109 OSObject *obj;
1110
1111 /* TODO: see if this can be more efficient */
1112 obj = userClient->copyProperty(aKey: "IOUserClientCreator",
1113 plane: gIOServicePlane,
1114 options: kIORegistryIterateRecursively | kIORegistryIterateParents);
1115
1116 if (!obj) {
1117 goto err_nounlock;
1118 }
1119
1120 ossUserClientCreator = OSDynamicCast(OSString, obj);
1121
1122 if (ossUserClientCreator) {
1123 uint32_t len, lenIter = 0;
1124
1125 ptr = ossUserClientCreator->getCStringNoCopy();
1126 len = ossUserClientCreator->getLength();
1127
1128 while ((*ptr != ' ') && (lenIter < len)) {
1129 ptr++;
1130 lenIter++;
1131 }
1132
1133 if (lenIter < len) {
1134 ptr++; // Skip the space
1135 lenIter++;
1136 pid = 0;
1137 while ((*ptr != ',') && (lenIter < len)) {
1138 pid = pid * 10 + (*ptr - '0');
1139 ptr++;
1140 lenIter++;
1141 }
1142
1143 if (lenIter == len) {
1144 pid = -1;
1145 } else {
1146 ptr += 2;
1147 }
1148 }
1149 }
1150
1151 if (-1 == pid) {
1152 goto err_nounlock;
1153 }
1154
1155 IORWLockWrite(lock);
1156
1157 parentKext = counter->parentClass->parentKext;
1158
1159 TAILQ_FOREACH(entry, &parentKext->userClientCallList, link) {
1160 if (entry->pid == pid) {
1161 /* Found, so increment count and move to the head */
1162 entry->calls++;
1163 if (count) {
1164 TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1165 break;
1166 } else {
1167 /* At the head already, so increment and return */
1168 goto err_unlock;
1169 }
1170 }
1171
1172 count++;
1173 }
1174
1175 if (!entry) {
1176 if (count == IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS) {
1177 /* Max elements hit, so reuse the last */
1178 entry = TAILQ_LAST(&parentKext->userClientCallList, ProcessEntryList);
1179 TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1180 } else {
1181 /* Otherwise, allocate a new entry */
1182 entry = kalloc_type(IOUserClientProcessEntry, Z_WAITOK);
1183 if (!entry) {
1184 goto err_unlock;
1185 }
1186 }
1187
1188 strncpy(entry->processName, ptr, kIOStatisticsProcessNameLength);
1189 entry->pid = pid;
1190 entry->calls = 1;
1191 }
1192
1193 TAILQ_FOREACH(nextEntry, &parentKext->userClientCallList, link) {
1194 if (nextEntry->calls <= entry->calls) {
1195 break;
1196 }
1197
1198 prevEntry = nextEntry;
1199 }
1200
1201 if (!prevEntry) {
1202 TAILQ_INSERT_HEAD(&parentKext->userClientCallList, entry, link);
1203 } else {
1204 TAILQ_INSERT_AFTER(&parentKext->userClientCallList, prevEntry, entry, link);
1205 }
1206
1207err_unlock:
1208 IORWLockUnlock(lock);
1209
1210err_nounlock:
1211 if (obj) {
1212 obj->release();
1213 }
1214}
1215
1216void
1217IOStatistics::countUserClientCall(IOUserClient *client)
1218{
1219 IOUserClient::ExpansionData *data;
1220 IOUserClientCounter *counter;
1221
1222 /* Guard against an uninitialized client object - <rdar://problem/8577946> */
1223 if (!(data = client->reserved)) {
1224 return;
1225 }
1226
1227 if ((counter = data->counter)) {
1228 storeUserClientCallInfo(userClient: client, counter);
1229 OSIncrementAtomic(&counter->clientCalls);
1230 }
1231}
1232
1233KextNode *
1234IOStatistics::getKextNodeFromBacktrace(boolean_t write)
1235{
1236 const uint32_t btMin = 3;
1237
1238 void *bt[16];
1239 unsigned btCount = sizeof(bt) / sizeof(bt[0]);
1240 vm_offset_t *scanAddr = NULL;
1241 uint32_t i;
1242 KextNode *found = NULL, *ke = NULL;
1243
1244 /*
1245 * Gathering the backtrace is a significant source of
1246 * overhead. OSBacktrace does many safety checks that
1247 * are not needed in this situation.
1248 */
1249 btCount = backtrace(bt: (uintptr_t*)bt, btlen: btCount, NULL, NULL);
1250
1251 if (write) {
1252 IORWLockWrite(lock);
1253 } else {
1254 IORWLockRead(lock);
1255 }
1256
1257 /* Ignore first levels */
1258 scanAddr = (vm_offset_t *)&bt[btMin - 1];
1259
1260 for (i = btMin - 1; i < btCount; i++, scanAddr++) {
1261 ke = RB_ROOT(&kextAddressHead);
1262 while (ke) {
1263 if (*scanAddr < ke->address) {
1264 ke = RB_LEFT(ke, addressLink);
1265 } else {
1266 if ((*scanAddr < ke->address_end) && (*scanAddr >= ke->address)) {
1267 if (!ke->kext->isKernelComponent()) {
1268 return ke;
1269 } else {
1270 found = ke;
1271 }
1272 }
1273 ke = RB_RIGHT(ke, addressLink);
1274 }
1275 }
1276 }
1277
1278 if (!found) {
1279 IORWLockUnlock(lock);
1280 }
1281
1282 return found;
1283}
1284
1285void
1286IOStatistics::releaseKextNode(KextNode *node)
1287{
1288#pragma unused(node)
1289 IORWLockUnlock(lock);
1290}
1291
1292/* IOLib allocations */
1293void
1294IOStatistics::countAlloc(uint32_t index, vm_size_t size)
1295{
1296 KextNode *ke;
1297
1298 if (!enabled) {
1299 return;
1300 }
1301 if (size > INT_MAX) {
1302 return;
1303 }
1304
1305 ke = getKextNodeFromBacktrace(FALSE);
1306 if (ke) {
1307 OSAddAtomic((SInt32) size, &ke->memoryCounters[index]);
1308 releaseKextNode(node: ke);
1309 }
1310}
1311
1312#endif /* IOKITSTATS */
1313