1/*
2 * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <sys/sysctl.h>
30#include <kern/host.h>
31#include <kern/zalloc.h>
32
33#include <IOKit/system.h>
34#include <libkern/c++/OSKext.h>
35#include <libkern/OSAtomic.h>
36
37#include <IOKit/IOStatisticsPrivate.h>
38#include <IOKit/IOUserClient.h>
39#include <IOKit/IOEventSource.h>
40#include <IOKit/IOKitDebug.h>
41
42#if IOKITSTATS
43bool IOStatistics::enabled = false;
44
45uint32_t IOStatistics::sequenceID = 0;
46
47uint32_t IOStatistics::lastClassIndex = 0;
48uint32_t IOStatistics::lastKextIndex = 0;
49
50uint32_t IOStatistics::loadedKexts = 0;
51uint32_t IOStatistics::registeredClasses = 0;
52uint32_t IOStatistics::registeredCounters = 0;
53uint32_t IOStatistics::registeredWorkloops = 0;
54
55uint32_t IOStatistics::attachedEventSources = 0;
56
57IOWorkLoopDependency *IOStatistics::nextWorkLoopDependency = NULL;
58
59/* Logging */
60
61#define LOG_LEVEL 0
62
63#define LOG(level, format, ...) \
64do { \
65	if (level <= LOG_LEVEL) \
66		printf(format, ##__VA_ARGS__); \
67} while (0)
68
69/* Locks */
70
71IORWLock *IOStatistics::lock = NULL;
72
73/* Kext tree */
74
75KextNode *IOStatistics::kextHint = NULL;
76
77IOStatistics::KextTreeHead IOStatistics::kextHead = RB_INITIALIZER(&IOStatistics::kextHead);
78
79int IOStatistics::kextNodeCompare(KextNode *e1, KextNode *e2)
80{
81    if (e1->kext < e2->kext)
82        return -1;
83    else if (e1->kext > e2->kext)
84        return 1;
85    else
86        return 0;
87}
88
89RB_GENERATE(IOStatistics::KextTree, KextNode, link, kextNodeCompare);
90
91/* Kext tree ordered by address */
92
93IOStatistics::KextAddressTreeHead IOStatistics::kextAddressHead = RB_INITIALIZER(&IOStatistics::kextAddressHead);
94
95int IOStatistics::kextAddressNodeCompare(KextNode *e1, KextNode *e2)
96{
97    if (e1->address < e2->address)
98        return -1;
99    else if (e1->address > e2->address)
100        return 1;
101    else
102        return 0;
103}
104
105RB_GENERATE(IOStatistics::KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
106
107/* Class tree */
108
109IOStatistics::ClassTreeHead IOStatistics::classHead = RB_INITIALIZER(&IOStatistics::classHead);
110
111int IOStatistics::classNodeCompare(ClassNode *e1, ClassNode *e2) {
112    if (e1->metaClass < e2->metaClass)
113        return -1;
114    else if (e1->metaClass > e2->metaClass)
115        return 1;
116    else
117        return 0;
118}
119
120RB_GENERATE(IOStatistics::ClassTree, ClassNode, tLink, classNodeCompare);
121
122/* Workloop dependencies */
123
124int IOWorkLoopCounter::loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2) {
125    if (e1->loadTag < e2->loadTag)
126        return -1;
127    else if (e1->loadTag > e2->loadTag)
128        return 1;
129    else
130        return 0;
131}
132
133RB_GENERATE(IOWorkLoopCounter::DependencyTree, IOWorkLoopDependency, link, IOWorkLoopCounter::loadTagCompare);
134
135/* sysctl stuff */
136
137static int
138oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req)
139{
140	int error = EINVAL;
141	uint32_t request = arg2;
142
143	switch (request)
144	{
145		case kIOStatisticsGeneral:
146			error = IOStatistics::getStatistics(req);
147			break;
148		case kIOStatisticsWorkLoop:
149			error = IOStatistics::getWorkLoopStatistics(req);
150			break;
151		case kIOStatisticsUserClient:
152			error = IOStatistics::getUserClientStatistics(req);
153			break;
154		default:
155			break;
156	}
157
158	return error;
159}
160
161SYSCTL_NODE(_debug, OID_AUTO, iokit_statistics, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IOStatistics");
162
163static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, general,
164	    CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
165	    0, kIOStatisticsGeneral, oid_sysctl, "S", "");
166
167static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, workloop,
168	    CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
169	    0, kIOStatisticsWorkLoop, oid_sysctl, "S", "");
170
171static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, userclient,
172	    CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
173	    0, kIOStatisticsUserClient, oid_sysctl, "S", "");
174
175void IOStatistics::initialize()
176{
177	if (enabled) {
178		return;
179	}
180
181	/* Only enabled if the boot argument is set. */
182	if (!(kIOStatistics & gIOKitDebug)) {
183		return;
184	}
185
186	sysctl_register_oid(&sysctl__debug_iokit_statistics_general);
187	sysctl_register_oid(&sysctl__debug_iokit_statistics_workloop);
188	sysctl_register_oid(&sysctl__debug_iokit_statistics_userclient);
189
190	lock = IORWLockAlloc();
191	if (!lock) {
192		return;
193	}
194
195	nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
196	if (!nextWorkLoopDependency) {
197		return;
198	}
199
200	enabled = true;
201}
202
203void IOStatistics::onKextLoad(OSKext *kext, kmod_info_t *kmod_info)
204{
205	KextNode *ke;
206
207	assert(kext && kmod_info);
208
209	if (!enabled) {
210		return;
211	}
212
213	LOG(1, "IOStatistics::onKextLoad: %s, tag %d, address 0x%llx, address end 0x%llx\n",
214		kext->getIdentifierCString(), kmod_info->id, (uint64_t)kmod_info->address, (uint64_t)(kmod_info->address + kmod_info->size));
215
216	ke = (KextNode *)kalloc(sizeof(KextNode));
217	if (!ke) {
218		return;
219	}
220
221	memset(ke, 0, sizeof(KextNode));
222
223	ke->kext = kext;
224	ke->loadTag = kmod_info->id;
225	ke->address = kmod_info->address;
226	ke->address_end = kmod_info->address + kmod_info->size;
227
228	SLIST_INIT(&ke->classList);
229	TAILQ_INIT(&ke->userClientCallList);
230
231	IORWLockWrite(lock);
232
233	RB_INSERT(KextTree, &kextHead, ke);
234	RB_INSERT(KextAddressTree, &kextAddressHead, ke);
235
236	sequenceID++;
237	loadedKexts++;
238	lastKextIndex++;
239
240	IORWLockUnlock(lock);
241}
242
243void IOStatistics::onKextUnload(OSKext *kext)
244{
245	KextNode sought, *found;
246
247	assert(kext);
248
249	if (!enabled) {
250		return;
251	}
252
253	LOG(1, "IOStatistics::onKextUnload: %s\n", kext->getIdentifierCString());
254
255	IORWLockWrite(lock);
256
257	sought.kext = kext;
258	found = RB_FIND(KextTree, &kextHead, &sought);
259	if (found) {
260		IOWorkLoopCounter *wlc;
261		IOUserClientProcessEntry *uce;
262
263		/* Disconnect workloop counters; cleanup takes place in unregisterWorkLoop() */
264		while ((wlc = SLIST_FIRST(&found->workLoopList))) {
265			SLIST_REMOVE_HEAD(&found->workLoopList, link);
266			wlc->parentKext = NULL;
267		}
268
269		/* Free up the user client list */
270		while ((uce = TAILQ_FIRST(&found->userClientCallList))) {
271			TAILQ_REMOVE(&found->userClientCallList, uce, link);
272			kfree(uce, sizeof(IOUserClientProcessEntry));
273		}
274
275		/* Remove from kext trees */
276		RB_REMOVE(KextTree, &kextHead, found);
277		RB_REMOVE(KextAddressTree, &kextAddressHead, found);
278
279		/*
280		 * Clear a matching kextHint to avoid use after free in
281		 * onClassAdded() for a class add after a KEXT unload.
282		 */
283		if (found == kextHint) {
284			kextHint = NULL;
285		}
286
287		/* Finally, free the class node */
288		kfree(found, sizeof(KextNode));
289
290		sequenceID++;
291		loadedKexts--;
292	}
293	else {
294		panic("IOStatistics::onKextUnload: cannot find kext: %s", kext->getIdentifierCString());
295	}
296
297	IORWLockUnlock(lock);
298}
299
300void IOStatistics::onClassAdded(OSKext *parentKext, OSMetaClass *metaClass)
301{
302	ClassNode *ce;
303	KextNode soughtKext, *foundKext = NULL;
304
305	assert(parentKext && metaClass);
306
307	if (!enabled) {
308		return;
309	}
310
311	LOG(1, "IOStatistics::onClassAdded: %s\n", metaClass->getClassName());
312
313	ce = (ClassNode *)kalloc(sizeof(ClassNode));
314	if (!ce) {
315		return;
316	}
317
318	memset(ce, 0, sizeof(ClassNode));
319
320	IORWLockWrite(lock);
321
322	/* Hinted? */
323	if (kextHint && kextHint->kext == parentKext) {
324		foundKext = kextHint;
325	}
326	else {
327		soughtKext.kext = parentKext;
328		foundKext = RB_FIND(KextTree, &kextHead, &soughtKext);
329	}
330
331	if (foundKext) {
332		ClassNode soughtClass, *foundClass = NULL;
333		const OSMetaClass *superClass;
334
335		ce->metaClass = metaClass;
336		ce->classID = lastClassIndex++;
337		ce->parentKext = foundKext;
338
339		/* Has superclass? */
340	 	superClass = ce->metaClass->getSuperClass();
341		if (superClass) {
342			soughtClass.metaClass = superClass;
343			foundClass = RB_FIND(ClassTree, &classHead, &soughtClass);
344		}
345		ce->superClassID = foundClass ? foundClass->classID : (uint32_t)(-1);
346
347		SLIST_INIT(&ce->counterList);
348		SLIST_INIT(&ce->userClientList);
349
350		RB_INSERT(ClassTree, &classHead, ce);
351		SLIST_INSERT_HEAD(&foundKext->classList, ce, lLink);
352
353		foundKext->classes++;
354
355		kextHint = foundKext;
356
357		sequenceID++;
358		registeredClasses++;
359	}
360	else {
361		panic("IOStatistics::onClassAdded: cannot find parent kext: %s", parentKext->getIdentifierCString());
362	}
363
364	IORWLockUnlock(lock);
365}
366
367void IOStatistics::onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass)
368{
369	ClassNode sought, *found;
370
371	assert(parentKext && metaClass);
372
373	if (!enabled) {
374		return;
375	}
376
377	LOG(1, "IOStatistics::onClassRemoved: %s\n", metaClass->getClassName());
378
379	IORWLockWrite(lock);
380
381	sought.metaClass = metaClass;
382	found = RB_FIND(ClassTree, &classHead, &sought);
383	if (found) {
384		IOEventSourceCounter *esc;
385		IOUserClientCounter *ucc;
386
387		/* Free up the list of counters */
388		while ((esc = SLIST_FIRST(&found->counterList))) {
389			SLIST_REMOVE_HEAD(&found->counterList, link);
390			kfree(esc, sizeof(IOEventSourceCounter));
391		}
392
393		/* Free up the user client list */
394		while ((ucc = SLIST_FIRST(&found->userClientList))) {
395			SLIST_REMOVE_HEAD(&found->userClientList, link);
396			kfree(ucc, sizeof(IOUserClientCounter));
397		}
398
399		/* Remove from class tree */
400		RB_REMOVE(ClassTree, &classHead, found);
401
402		/* Remove from parent */
403		SLIST_REMOVE(&found->parentKext->classList, found, ClassNode, lLink);
404
405		/* Finally, free the class node */
406		kfree(found, sizeof(ClassNode));
407
408		sequenceID++;
409		registeredClasses--;
410	}
411	else {
412		panic("IOStatistics::onClassRemoved: cannot find class: %s", metaClass->getClassName());
413	}
414
415	IORWLockUnlock(lock);
416}
417
418IOEventSourceCounter *IOStatistics::registerEventSource(OSObject *inOwner)
419{
420	IOEventSourceCounter *counter = NULL;
421	ClassNode sought, *found = NULL;
422	boolean_t createDummyCounter = FALSE;
423
424	assert(inOwner);
425
426	if (!enabled) {
427		return NULL;
428	}
429
430	counter = (IOEventSourceCounter*)kalloc(sizeof(IOEventSourceCounter));
431	if (!counter) {
432		return NULL;
433	}
434
435	memset(counter, 0, sizeof(IOEventSourceCounter));
436
437	IORWLockWrite(lock);
438
439	/* Workaround for <rdar://problem/7158117> - create a dummy counter when inOwner is bad.
440	 * We use retainCount here as our best indication that the pointer is awry.
441	 */
442	if (inOwner->retainCount > 0xFFFFFF) {
443		kprintf("IOStatistics::registerEventSource - bad metaclass %p\n", inOwner);
444		createDummyCounter = TRUE;
445	}
446	else {
447		sought.metaClass = inOwner->getMetaClass();
448		found = RB_FIND(ClassTree, &classHead, &sought);
449	}
450
451	if (found) {
452		counter->parentClass = found;
453		SLIST_INSERT_HEAD(&found->counterList, counter, link);
454		registeredCounters++;
455	}
456
457	if (!(createDummyCounter || found)) {
458		panic("IOStatistics::registerEventSource: cannot find parent class: %s", inOwner->getMetaClass()->getClassName());
459	}
460
461	IORWLockUnlock(lock);
462
463	return counter;
464}
465
466void IOStatistics::unregisterEventSource(IOEventSourceCounter *counter)
467{
468	if (!counter) {
469		return;
470	}
471
472	IORWLockWrite(lock);
473
474	if (counter->parentClass) {
475		SLIST_REMOVE(&counter->parentClass->counterList, counter, IOEventSourceCounter, link);
476		registeredCounters--;
477	}
478	kfree(counter, sizeof(IOEventSourceCounter));
479
480	IORWLockUnlock(lock);
481}
482
483IOWorkLoopCounter* IOStatistics::registerWorkLoop(IOWorkLoop *workLoop)
484{
485	IOWorkLoopCounter *counter = NULL;
486	KextNode *found;
487
488	assert(workLoop);
489
490	if (!enabled) {
491		return NULL;
492	}
493
494	counter = (IOWorkLoopCounter*)kalloc(sizeof(IOWorkLoopCounter));
495	if (!counter) {
496		return NULL;
497	}
498
499	memset(counter, 0, sizeof(IOWorkLoopCounter));
500
501	found = getKextNodeFromBacktrace(TRUE);
502	if (!found) {
503		panic("IOStatistics::registerWorkLoop: cannot find parent kext");
504	}
505
506	counter->parentKext = found;
507	counter->workLoop = workLoop;
508	RB_INIT(&counter->dependencyHead);
509	SLIST_INSERT_HEAD(&found->workLoopList, counter, link);
510	registeredWorkloops++;
511
512	releaseKextNode(found);
513
514	return counter;
515}
516
517void IOStatistics::unregisterWorkLoop(IOWorkLoopCounter *counter)
518{
519	if (!counter) {
520		return;
521	}
522
523	IORWLockWrite(lock);
524	if (counter->parentKext) {
525		SLIST_REMOVE(&counter->parentKext->workLoopList, counter, IOWorkLoopCounter, link);
526	}
527	kfree(counter, sizeof(IOWorkLoopCounter));
528	registeredWorkloops--;
529
530	IORWLockUnlock(lock);
531}
532
533IOUserClientCounter *IOStatistics::registerUserClient(IOUserClient *userClient)
534{
535	ClassNode sought, *found;
536	IOUserClientCounter *counter = NULL;
537
538	assert(userClient);
539
540	if (!enabled) {
541		return NULL;
542	}
543
544	counter = (IOUserClientCounter*)kalloc(sizeof(IOUserClientCounter));
545	if (!counter) {
546		return NULL;
547	}
548
549	memset(counter, 0, sizeof(IOUserClientCounter));
550
551	IORWLockWrite(lock);
552
553	sought.metaClass = userClient->getMetaClass();
554
555	found = RB_FIND(ClassTree, &classHead, &sought);
556	if (found) {
557		counter->parentClass = found;
558		SLIST_INSERT_HEAD(&found->userClientList, counter, link);
559	}
560	else {
561		panic("IOStatistics::registerUserClient: cannot find parent class: %s", sought.metaClass->getClassName());
562	}
563
564	IORWLockUnlock(lock);
565
566	return counter;
567}
568
569void IOStatistics::unregisterUserClient(IOUserClientCounter *counter)
570{
571	if (!counter) {
572		return;
573	}
574
575	IORWLockWrite(lock);
576
577	SLIST_REMOVE(&counter->parentClass->userClientList, counter, IOUserClientCounter, link);
578	kfree(counter, sizeof(IOUserClientCounter));
579
580	IORWLockUnlock(lock);
581}
582
583void IOStatistics::attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
584{
585	if (!wlc) {
586        return;
587	}
588
589	IORWLockWrite(lock);
590
591	if (!nextWorkLoopDependency) {
592		return;
593	}
594
595	attachedEventSources++;
596	wlc->attachedEventSources++;
597
598	/* Track the kext dependency */
599	nextWorkLoopDependency->loadTag = esc->parentClass->parentKext->loadTag;
600	if (NULL == RB_INSERT(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, nextWorkLoopDependency)) {
601		nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
602	}
603
604	IORWLockUnlock(lock);
605}
606
607void IOStatistics::detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
608{
609	IOWorkLoopDependency sought, *found;
610
611	if (!wlc) {
612		return;
613	}
614
615	IORWLockWrite(lock);
616
617	attachedEventSources--;
618	wlc->attachedEventSources--;
619
620	sought.loadTag = esc->parentClass->parentKext->loadTag;
621
622	found = RB_FIND(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, &sought);
623	if (found) {
624		RB_REMOVE(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, found);
625		kfree(found, sizeof(IOWorkLoopDependency));
626	}
627
628	IORWLockUnlock(lock);
629}
630
631int IOStatistics::getStatistics(sysctl_req *req)
632{
633	int error;
634	uint32_t calculatedSize, size;
635	char *buffer, *ptr;
636	IOStatisticsHeader *header;
637
638	assert(IOStatistics::enabled && req);
639
640	IORWLockRead(IOStatistics::lock);
641
642	/* Work out how much we need to allocate. IOStatisticsKext is of variable size. */
643	calculatedSize = sizeof(IOStatisticsHeader) +
644					 sizeof(IOStatisticsGlobal) +
645					(sizeof(IOStatisticsKext) * loadedKexts) + (sizeof(uint32_t) * registeredClasses) +
646					(sizeof(IOStatisticsMemory) * loadedKexts) +
647					(sizeof(IOStatisticsClass) * registeredClasses) +
648					(sizeof(IOStatisticsCounter) * registeredClasses) +
649					(sizeof(IOStatisticsKextIdentifier) * loadedKexts) +
650					(sizeof(IOStatisticsClassName) * registeredClasses);
651
652	/* Size request? */
653	if (req->oldptr == USER_ADDR_NULL) {
654		error = SYSCTL_OUT(req, NULL, calculatedSize);
655		goto exit;
656	}
657
658	/* Read only */
659	if (req->newptr != USER_ADDR_NULL) {
660		error = EPERM;
661		goto exit;
662	}
663
664	buffer = (char*)kalloc(calculatedSize);
665	if (!buffer) {
666		error = ENOMEM;
667		goto exit;
668	}
669
670	memset(buffer, 0, calculatedSize);
671
672	ptr = buffer;
673
674	header = (IOStatisticsHeader*)((void*)ptr);
675
676	header->sig = IOSTATISTICS_SIG;
677	header->ver = IOSTATISTICS_VER;
678
679	header->seq = sequenceID;
680
681	ptr += sizeof(IOStatisticsHeader);
682
683	/* Global data - seq, timers, interrupts, etc) */
684	header->globalStatsOffset = sizeof(IOStatisticsHeader);
685	size = copyGlobalStatistics((IOStatisticsGlobal*)((void*)ptr));
686	ptr += size;
687
688	/* Kext statistics */
689	header->kextStatsOffset = header->globalStatsOffset + size;
690	size = copyKextStatistics((IOStatisticsKext*)((void*)ptr));
691	ptr += size;
692
693	/* Memory allocation info */
694	header->memoryStatsOffset = header->kextStatsOffset + size;
695	size = copyMemoryStatistics((IOStatisticsMemory*)((void*)ptr));
696	ptr += size;
697
698	/* Class statistics */
699	header->classStatsOffset = header->memoryStatsOffset + size;
700	size = copyClassStatistics((IOStatisticsClass*)((void*)ptr));
701	ptr += size;
702
703	/* Dynamic class counter data */
704	header->counterStatsOffset = header->classStatsOffset + size;
705	size = copyCounterStatistics((IOStatisticsCounter*)((void*)ptr));
706	ptr += size;
707
708	/* Kext identifiers */
709	header->kextIdentifiersOffset = header->counterStatsOffset + size;
710	size = copyKextIdentifiers((IOStatisticsKextIdentifier*)((void*)ptr));
711	ptr += size;
712
713	/* Class names */
714	header->classNamesOffset = header->kextIdentifiersOffset + size;
715	size = copyClassNames((IOStatisticsClassName*)ptr);
716	ptr += size;
717
718	LOG(2, "IOStatistics::getStatistics - calculatedSize 0x%x, kexts 0x%x, classes 0x%x.\n",
719	 	calculatedSize, loadedKexts, registeredClasses);
720
721	assert( (uint32_t)(ptr - buffer) == calculatedSize );
722
723	error = SYSCTL_OUT(req, buffer, calculatedSize);
724
725	kfree(buffer, calculatedSize);
726
727exit:
728	IORWLockUnlock(IOStatistics::lock);
729	return error;
730}
731
732int IOStatistics::getWorkLoopStatistics(sysctl_req *req)
733{
734	int error;
735	uint32_t calculatedSize, size;
736	char *buffer;
737	IOStatisticsWorkLoopHeader *header;
738
739	assert(IOStatistics::enabled && req);
740
741	IORWLockRead(IOStatistics::lock);
742
743	/* Approximate how much we need to allocate (worse case estimate) */
744	calculatedSize = sizeof(IOStatisticsWorkLoop) * registeredWorkloops +
745					 sizeof(uint32_t) * attachedEventSources;
746
747	/* Size request? */
748	if (req->oldptr == USER_ADDR_NULL) {
749		error = SYSCTL_OUT(req, NULL, calculatedSize);
750		goto exit;
751	}
752
753	/* Read only */
754	if (req->newptr != USER_ADDR_NULL) {
755		error = EPERM;
756		goto exit;
757	}
758
759	buffer = (char*)kalloc(calculatedSize);
760	if (!buffer) {
761		error = ENOMEM;
762		goto exit;
763	}
764
765	header = (IOStatisticsWorkLoopHeader*)((void*)buffer);
766
767	header->sig = IOSTATISTICS_SIG_WORKLOOP;
768	header->ver = IOSTATISTICS_VER;
769
770	header->seq = sequenceID;
771
772	header->workloopCount = registeredWorkloops;
773
774	size = copyWorkLoopStatistics(&header->workLoopStats);
775
776	LOG(2, "IOStatistics::getWorkLoopStatistics: calculatedSize %d, size %d\n", calculatedSize, size);
777
778	assert( size <= calculatedSize );
779
780	error = SYSCTL_OUT(req, buffer, size);
781
782	kfree(buffer, calculatedSize);
783
784exit:
785	IORWLockUnlock(IOStatistics::lock);
786	return error;
787}
788
789int IOStatistics::getUserClientStatistics(sysctl_req *req)
790{
791	int error;
792	uint32_t calculatedSize, size;
793	char *buffer;
794	uint32_t requestedLoadTag = 0;
795	IOStatisticsUserClientHeader *header;
796
797	assert(IOStatistics::enabled && req);
798
799	IORWLockRead(IOStatistics::lock);
800
801	/* Work out how much we need to allocate */
802	calculatedSize = sizeof(IOStatisticsUserClientHeader) +
803					 sizeof(IOStatisticsUserClientCall) * IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS * loadedKexts;
804
805	/* Size request? */
806	if (req->oldptr == USER_ADDR_NULL) {
807		error = SYSCTL_OUT(req, NULL, calculatedSize);
808		goto exit;
809	}
810
811	/* Kext request (potentially) valid? */
812	if (!req->newptr || req->newlen < sizeof(requestedLoadTag)) {
813		error = EINVAL;
814		goto exit;
815	}
816
817	SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag));
818
819	LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag);
820
821	buffer = (char*)kalloc(calculatedSize);
822	if (!buffer) {
823		error = ENOMEM;
824		goto exit;
825	}
826
827	header = (IOStatisticsUserClientHeader*)((void*)buffer);
828
829	header->sig = IOSTATISTICS_SIG_USERCLIENT;
830	header->ver = IOSTATISTICS_VER;
831
832	header->seq = sequenceID;
833
834	header->processes = 0;
835
836	size = copyUserClientStatistics(header, requestedLoadTag);
837
838	assert((sizeof(IOStatisticsUserClientHeader) + size) <= calculatedSize);
839
840	if (size) {
841		error = SYSCTL_OUT(req, buffer, sizeof(IOStatisticsUserClientHeader) + size);
842	}
843	else {
844		error = EINVAL;
845	}
846
847	kfree(buffer, calculatedSize);
848
849exit:
850	IORWLockUnlock(IOStatistics::lock);
851	return error;
852}
853
854uint32_t IOStatistics::copyGlobalStatistics(IOStatisticsGlobal *stats)
855{
856	stats->kextCount = loadedKexts;
857	stats->classCount = registeredClasses;
858	stats->workloops = registeredWorkloops;
859
860	return sizeof(IOStatisticsGlobal);
861}
862
863uint32_t IOStatistics::copyKextStatistics(IOStatisticsKext *stats)
864{
865	KextNode *ke;
866	ClassNode *ce;
867	uint32_t index = 0;
868
869	RB_FOREACH(ke, KextTree, &kextHead) {
870		stats->loadTag = ke->loadTag;
871		ke->kext->getSizeInfo(&stats->loadSize, &stats->wiredSize);
872
873		stats->classes = ke->classes;
874
875		/* Append indices of owned classes */
876		SLIST_FOREACH(ce, &ke->classList, lLink) {
877			stats->classIndexes[index++] = ce->classID;
878		}
879
880		stats = (IOStatisticsKext *)((void*)((char*)stats + sizeof(IOStatisticsKext) + (ke->classes * sizeof(uint32_t))));
881	}
882
883	return (sizeof(IOStatisticsKext) * loadedKexts + sizeof(uint32_t) * registeredClasses);
884}
885
886uint32_t IOStatistics::copyMemoryStatistics(IOStatisticsMemory *stats)
887{
888	KextNode *ke;
889
890	RB_FOREACH(ke, KextTree, &kextHead) {
891		stats->allocatedSize = ke->memoryCounters[kIOStatisticsMalloc];
892		stats->freedSize = ke->memoryCounters[kIOStatisticsFree];
893		stats->allocatedAlignedSize = ke->memoryCounters[kIOStatisticsMallocAligned];
894		stats->freedAlignedSize = ke->memoryCounters[kIOStatisticsFreeAligned];
895		stats->allocatedContiguousSize = ke->memoryCounters[kIOStatisticsMallocContiguous];
896		stats->freedContiguousSize = ke->memoryCounters[kIOStatisticsFreeContiguous];
897		stats->allocatedPageableSize = ke->memoryCounters[kIOStatisticsMallocPageable];
898		stats->freedPageableSize = ke->memoryCounters[kIOStatisticsFreePageable];
899		stats++;
900	}
901
902	return (sizeof(IOStatisticsMemory) * loadedKexts);
903}
904
905uint32_t IOStatistics::copyClassStatistics(IOStatisticsClass *stats)
906{
907	KextNode *ke;
908	ClassNode *ce;
909
910	RB_FOREACH(ke, KextTree, &kextHead) {
911		SLIST_FOREACH(ce, &ke->classList, lLink) {
912			stats->classID = ce->classID;
913			stats->superClassID = ce->superClassID;
914			stats->classSize = ce->metaClass->getClassSize();
915
916			stats++;
917		}
918	}
919
920	return sizeof(IOStatisticsClass) * registeredClasses;
921}
922
923uint32_t IOStatistics::copyCounterStatistics(IOStatisticsCounter *stats)
924{
925	KextNode *ke;
926	ClassNode *ce;
927
928	RB_FOREACH(ke, KextTree, &kextHead) {
929		SLIST_FOREACH(ce, &ke->classList, lLink) {
930			IOUserClientCounter *userClientCounter;
931			IOEventSourceCounter *counter;
932
933			stats->classID = ce->classID;
934			stats->classInstanceCount = ce->metaClass->getInstanceCount();
935
936			IOStatisticsUserClients *uc = &stats->userClientStatistics;
937
938			/* User client counters */
939			SLIST_FOREACH(userClientCounter, &ce->userClientList, link) {
940				uc->clientCalls += userClientCounter->clientCalls;
941				uc->created++;
942			}
943
944			IOStatisticsInterruptEventSources *iec = &stats->interruptEventSourceStatistics;
945			IOStatisticsInterruptEventSources *fiec = &stats->filterInterruptEventSourceStatistics;
946			IOStatisticsTimerEventSources *tec = &stats->timerEventSourceStatistics;
947			IOStatisticsCommandGates *cgc = &stats->commandGateStatistics;
948			IOStatisticsCommandQueues *cqc = &stats->commandQueueStatistics;
949			IOStatisticsDerivedEventSources *dec = &stats->derivedEventSourceStatistics;
950
951			/* Event source counters */
952			SLIST_FOREACH(counter, &ce->counterList, link) {
953				switch (counter->type) {
954					case kIOStatisticsInterruptEventSourceCounter:
955						iec->created++;
956						iec->produced += counter->u.interrupt.produced;
957						iec->checksForWork += counter->u.interrupt.checksForWork;
958						break;
959					case kIOStatisticsFilterInterruptEventSourceCounter:
960						fiec->created++;
961						fiec->produced += counter->u.filter.produced;
962						fiec->checksForWork += counter->u.filter.checksForWork;
963						break;
964					case kIOStatisticsTimerEventSourceCounter:
965						tec->created++;
966						tec->timeouts += counter->u.timer.timeouts;
967						tec->checksForWork += counter->u.timer.checksForWork;
968						tec->timeOnGate += counter->timeOnGate;
969						tec->closeGateCalls += counter->closeGateCalls;
970						tec->openGateCalls += counter->openGateCalls;
971						break;
972					case kIOStatisticsCommandGateCounter:
973						cgc->created++;
974						cgc->timeOnGate += counter->timeOnGate;
975						cgc->actionCalls += counter->u.commandGate.actionCalls;
976						break;
977					case kIOStatisticsCommandQueueCounter:
978						cqc->created++;
979						cqc->actionCalls += counter->u.commandQueue.actionCalls;
980						break;
981					case kIOStatisticsDerivedEventSourceCounter:
982						dec->created++;
983						dec->timeOnGate += counter->timeOnGate;
984						dec->closeGateCalls += counter->closeGateCalls;
985						dec->openGateCalls += counter->openGateCalls;
986						break;
987					default:
988						break;
989				}
990			}
991
992			stats++;
993		}
994	}
995
996	return sizeof(IOStatisticsCounter) * registeredClasses;
997}
998
999uint32_t IOStatistics::copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs)
1000{
1001	KextNode *ke;
1002
1003	RB_FOREACH(ke, KextTree, &kextHead) {
1004		strncpy(kextIDs->identifier, ke->kext->getIdentifierCString(), kIOStatisticsDriverNameLength);
1005		kextIDs++;
1006	}
1007
1008	return (sizeof(IOStatisticsKextIdentifier) * loadedKexts);
1009}
1010
1011uint32_t IOStatistics::copyClassNames(IOStatisticsClassName *classNames)
1012{
1013	KextNode *ke;
1014	ClassNode *ce;
1015
1016	RB_FOREACH(ke, KextTree, &kextHead) {
1017		SLIST_FOREACH(ce, &ke->classList, lLink) {
1018			strncpy(classNames->name, ce->metaClass->getClassName(), kIOStatisticsClassNameLength);
1019			classNames++;
1020		}
1021	}
1022
1023	return (sizeof(IOStatisticsClassName) * registeredClasses);
1024}
1025
1026uint32_t IOStatistics::copyWorkLoopStatistics(IOStatisticsWorkLoop *stats)
1027{
1028	KextNode *ke;
1029	IOWorkLoopCounter *wlc;
1030	IOWorkLoopDependency *dependentNode;
1031	uint32_t size, accumulatedSize = 0;
1032
1033	RB_FOREACH(ke, KextTree, &kextHead) {
1034		SLIST_FOREACH(wlc, &ke->workLoopList, link) {
1035			stats->kextLoadTag = ke->loadTag;
1036			stats->attachedEventSources = wlc->attachedEventSources;
1037			stats->timeOnGate = wlc->timeOnGate;
1038			stats->dependentKexts = 0;
1039			RB_FOREACH(dependentNode, IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead) {
1040				stats->dependentKextLoadTags[stats->dependentKexts] = dependentNode->loadTag;
1041				stats->dependentKexts++;
1042			}
1043
1044			size = sizeof(IOStatisticsWorkLoop) + (sizeof(uint32_t) * stats->dependentKexts);
1045
1046			accumulatedSize += size;
1047			stats = (IOStatisticsWorkLoop*)((void*)((char*)stats + size));
1048		}
1049	}
1050
1051	return accumulatedSize;
1052}
1053
1054uint32_t IOStatistics::copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag)
1055{
1056	KextNode *sought, *found = NULL;
1057	uint32_t procs = 0;
1058	IOUserClientProcessEntry *processEntry;
1059
1060	RB_FOREACH(sought, KextTree, &kextHead) {
1061		if (sought->loadTag == loadTag) {
1062			found = sought;
1063			break;
1064		}
1065	}
1066
1067	if (!found) {
1068		return 0;
1069	}
1070
1071	TAILQ_FOREACH(processEntry, &found->userClientCallList, link) {
1072		strncpy(stats->userClientCalls[procs].processName, processEntry->processName, kIOStatisticsProcessNameLength);
1073		stats->userClientCalls[procs].pid = processEntry->pid;
1074		stats->userClientCalls[procs].calls = processEntry->calls;
1075		stats->processes++;
1076		procs++;
1077	}
1078
1079	return sizeof(IOStatisticsUserClientCall) * stats->processes;
1080}
1081
1082void IOStatistics::storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter)
1083{
1084	OSString *ossUserClientCreator = NULL;
1085	int32_t pid = -1;
1086	KextNode *parentKext;
1087	IOUserClientProcessEntry *entry, *nextEntry, *prevEntry = NULL;
1088	uint32_t count = 0;
1089	const char *ptr = NULL;
1090	OSObject *obj;
1091
1092	/* TODO: see if this can be more efficient */
1093	obj = userClient->copyProperty("IOUserClientCreator",
1094					gIOServicePlane,
1095					kIORegistryIterateRecursively | kIORegistryIterateParents);
1096
1097	if (!obj)
1098		goto err_nounlock;
1099
1100	ossUserClientCreator = OSDynamicCast(OSString, obj);
1101
1102	if (ossUserClientCreator) {
1103		uint32_t len, lenIter = 0;
1104
1105		ptr = ossUserClientCreator->getCStringNoCopy();
1106		len = ossUserClientCreator->getLength();
1107
1108		while ((*ptr != ' ') && (lenIter < len)) {
1109			ptr++;
1110			lenIter++;
1111		}
1112
1113		if (lenIter < len) {
1114			ptr++; // Skip the space
1115			lenIter++;
1116			pid = 0;
1117			while ( (*ptr != ',') && (lenIter < len)) {
1118				pid = pid*10 + (*ptr - '0');
1119				ptr++;
1120				lenIter++;
1121			}
1122
1123			if(lenIter == len) {
1124				pid = -1;
1125			} else {
1126				ptr += 2;
1127			}
1128		}
1129	}
1130
1131	if (-1 == pid)
1132		goto err_nounlock;
1133
1134	IORWLockWrite(lock);
1135
1136	parentKext = counter->parentClass->parentKext;
1137
1138	TAILQ_FOREACH(entry, &parentKext->userClientCallList, link) {
1139		if (entry->pid == pid) {
1140			/* Found, so increment count and move to the head */
1141			entry->calls++;
1142			if (count) {
1143				TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1144				break;
1145			}
1146			else {
1147				/* At the head already, so increment and return */
1148				goto err_unlock;
1149			}
1150		}
1151
1152		count++;
1153	}
1154
1155	if (!entry) {
1156		if (count == IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS) {
1157			/* Max elements hit, so reuse the last */
1158			entry = TAILQ_LAST(&parentKext->userClientCallList, ProcessEntryList);
1159			TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1160		}
1161		else {
1162			/* Otherwise, allocate a new entry */
1163			entry = (IOUserClientProcessEntry*)kalloc(sizeof(IOUserClientProcessEntry));
1164			if (!entry) {
1165			    IORWLockUnlock(lock);
1166				return;
1167			}
1168		}
1169
1170		strncpy(entry->processName, ptr, kIOStatisticsProcessNameLength);
1171		entry->pid = pid;
1172		entry->calls = 1;
1173	}
1174
1175	TAILQ_FOREACH(nextEntry, &parentKext->userClientCallList, link) {
1176		if (nextEntry->calls <= entry->calls)
1177			break;
1178
1179		prevEntry = nextEntry;
1180	}
1181
1182	if (!prevEntry)
1183		TAILQ_INSERT_HEAD(&parentKext->userClientCallList, entry, link);
1184	else
1185		TAILQ_INSERT_AFTER(&parentKext->userClientCallList, prevEntry, entry, link);
1186
1187err_unlock:
1188	IORWLockUnlock(lock);
1189
1190err_nounlock:
1191	if (obj)
1192		obj->release();
1193}
1194
1195void IOStatistics::countUserClientCall(IOUserClient *client) {
1196	IOUserClient::ExpansionData *data;
1197	IOUserClientCounter *counter;
1198
1199	/* Guard against an uninitialized client object - <rdar://problem/8577946> */
1200	if (!(data = client->reserved)) {
1201		return;
1202	}
1203
1204	if ((counter = data->counter)) {
1205		storeUserClientCallInfo(client, counter);
1206		OSIncrementAtomic(&counter->clientCalls);
1207	}
1208}
1209
1210KextNode *IOStatistics::getKextNodeFromBacktrace(boolean_t write) {
1211	const uint32_t btMin = 3;
1212
1213	void *bt[16];
1214	unsigned btCount = sizeof(bt) / sizeof(bt[0]);
1215	vm_offset_t *scanAddr = NULL;
1216	uint32_t i;
1217	KextNode *found = NULL, *ke = NULL;
1218
1219	/*
1220	 * Gathering the backtrace is a significant source of
1221	 * overhead. OSBacktrace does many safety checks that
1222	 * are not needed in this situation.
1223	 */
1224	btCount = fastbacktrace((uintptr_t*)bt, btCount);
1225
1226	if (write) {
1227		IORWLockWrite(lock);
1228	} else {
1229		IORWLockRead(lock);
1230	}
1231
1232	/* Ignore first levels */
1233	scanAddr = (vm_offset_t *)&bt[btMin - 1];
1234
1235	for (i = btMin - 1; i < btCount; i++, scanAddr++) {
1236		ke = RB_ROOT(&kextAddressHead);
1237		while (ke) {
1238			if (*scanAddr < ke->address) {
1239				ke = RB_LEFT(ke, addressLink);
1240			}
1241			else {
1242				if ((*scanAddr < ke->address_end) && (*scanAddr >= ke->address)) {
1243 					if (!ke->kext->isKernelComponent()) {
1244 						return ke;
1245					} else {
1246						found = ke;
1247					}
1248				}
1249				ke = RB_RIGHT(ke, addressLink);
1250			}
1251		}
1252	}
1253
1254	if (!found) {
1255		IORWLockUnlock(lock);
1256	}
1257
1258	return found;
1259}
1260
1261void IOStatistics::releaseKextNode(KextNode *node) {
1262#pragma unused(node)
1263	IORWLockUnlock(lock);
1264}
1265
1266/* IOLib allocations */
1267void IOStatistics::countAlloc(uint32_t index, vm_size_t size) {
1268	KextNode *ke;
1269
1270	if (!enabled) {
1271		return;
1272	}
1273
1274	ke = getKextNodeFromBacktrace(FALSE);
1275	if (ke) {
1276		OSAddAtomic(size, &ke->memoryCounters[index]);
1277		releaseKextNode(ke);
1278	}
1279}
1280
1281#endif /* IOKITSTATS */
1282