1/*
2 * Copyright (c) 2010 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <sys/sysctl.h>
30#include <kern/host.h>
31
32#include <IOKit/system.h>
33#include <libkern/c++/OSKext.h>
34#include <libkern/OSAtomic.h>
35
36#include <IOKit/IOStatisticsPrivate.h>
37#include <IOKit/IOUserClient.h>
38#include <IOKit/IOEventSource.h>
39#include <IOKit/IOKitDebug.h>
40
41#if IOKITSTATS
42bool IOStatistics::enabled = false;
43
44uint32_t IOStatistics::sequenceID = 0;
45
46uint32_t IOStatistics::lastClassIndex = 0;
47uint32_t IOStatistics::lastKextIndex = 0;
48
49uint32_t IOStatistics::loadedKexts = 0;
50uint32_t IOStatistics::registeredClasses = 0;
51uint32_t IOStatistics::registeredCounters = 0;
52uint32_t IOStatistics::registeredWorkloops = 0;
53
54uint32_t IOStatistics::attachedEventSources = 0;
55
56IOWorkLoopDependency *IOStatistics::nextWorkLoopDependency = NULL;
57
58/* Logging */
59
60#define LOG_LEVEL 0
61
62#define LOG(level, format, ...) \
63do { \
64	if (level <= LOG_LEVEL) \
65		printf(format, ##__VA_ARGS__); \
66} while (0)
67
68/* Locks */
69
70IORWLock *IOStatistics::lock = NULL;
71
72/* Kext tree */
73
74KextNode *IOStatistics::kextHint = NULL;
75
76IOStatistics::KextTreeHead IOStatistics::kextHead = RB_INITIALIZER(&IOStatistics::kextHead);
77
78int IOStatistics::kextNodeCompare(KextNode *e1, KextNode *e2)
79{
80    if (e1->kext < e2->kext)
81        return -1;
82    else if (e1->kext > e2->kext)
83        return 1;
84    else
85        return 0;
86}
87
88RB_GENERATE(IOStatistics::KextTree, KextNode, link, kextNodeCompare);
89
90/* Kext tree ordered by address */
91
92IOStatistics::KextAddressTreeHead IOStatistics::kextAddressHead = RB_INITIALIZER(&IOStatistics::kextAddressHead);
93
94int IOStatistics::kextAddressNodeCompare(KextNode *e1, KextNode *e2)
95{
96    if (e1->address < e2->address)
97        return -1;
98    else if (e1->address > e2->address)
99        return 1;
100    else
101        return 0;
102}
103
104RB_GENERATE(IOStatistics::KextAddressTree, KextNode, addressLink, kextAddressNodeCompare);
105
106/* Class tree */
107
108IOStatistics::ClassTreeHead IOStatistics::classHead = RB_INITIALIZER(&IOStatistics::classHead);
109
110int IOStatistics::classNodeCompare(ClassNode *e1, ClassNode *e2) {
111    if (e1->metaClass < e2->metaClass)
112        return -1;
113    else if (e1->metaClass > e2->metaClass)
114        return 1;
115    else
116        return 0;
117}
118
119RB_GENERATE(IOStatistics::ClassTree, ClassNode, tLink, classNodeCompare);
120
121/* Workloop dependencies */
122
123int IOWorkLoopCounter::loadTagCompare(IOWorkLoopDependency *e1, IOWorkLoopDependency *e2) {
124    if (e1->loadTag < e2->loadTag)
125        return -1;
126    else if (e1->loadTag > e2->loadTag)
127        return 1;
128    else
129        return 0;
130}
131
132RB_GENERATE(IOWorkLoopCounter::DependencyTree, IOWorkLoopDependency, link, IOWorkLoopCounter::loadTagCompare);
133
134/* sysctl stuff */
135
136static int
137oid_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1, int arg2, struct sysctl_req *req)
138{
139	int error = EINVAL;
140	uint32_t request = arg2;
141
142	switch (request)
143	{
144		case kIOStatisticsGeneral:
145			error = IOStatistics::getStatistics(req);
146			break;
147		case kIOStatisticsWorkLoop:
148			error = IOStatistics::getWorkLoopStatistics(req);
149			break;
150		case kIOStatisticsUserClient:
151			error = IOStatistics::getUserClientStatistics(req);
152			break;
153		default:
154			break;
155	}
156
157	return error;
158}
159
160SYSCTL_NODE(_debug, OID_AUTO, iokit_statistics, CTLFLAG_RW | CTLFLAG_LOCKED, 0, "IOStatistics");
161
162static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, general,
163	    CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
164	    0, kIOStatisticsGeneral, oid_sysctl, "S", "");
165
166static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, workloop,
167	    CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
168	    0, kIOStatisticsWorkLoop, oid_sysctl, "S", "");
169
170static SYSCTL_PROC(_debug_iokit_statistics, OID_AUTO, userclient,
171	    CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
172	    0, kIOStatisticsUserClient, oid_sysctl, "S", "");
173
174void IOStatistics::initialize()
175{
176	if (enabled) {
177		return;
178	}
179
180	/* Only enabled if the boot argument is set. */
181	if (!(kIOStatistics & gIOKitDebug)) {
182		return;
183	}
184
185	sysctl_register_oid(&sysctl__debug_iokit_statistics_general);
186	sysctl_register_oid(&sysctl__debug_iokit_statistics_workloop);
187	sysctl_register_oid(&sysctl__debug_iokit_statistics_userclient);
188
189	lock = IORWLockAlloc();
190	if (!lock) {
191		return;
192	}
193
194	nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
195	if (!nextWorkLoopDependency) {
196		return;
197	}
198
199	enabled = true;
200}
201
202void IOStatistics::onKextLoad(OSKext *kext, kmod_info_t *kmod_info)
203{
204	KextNode *ke;
205
206	assert(kext && kmod_info);
207
208	if (!enabled) {
209		return;
210	}
211
212	LOG(1, "IOStatistics::onKextLoad: %s, tag %d, address 0x%llx, address end 0x%llx\n",
213		kext->getIdentifierCString(), kmod_info->id, (uint64_t)kmod_info->address, (uint64_t)(kmod_info->address + kmod_info->size));
214
215	ke = (KextNode *)kalloc(sizeof(KextNode));
216	if (!ke) {
217		return;
218	}
219
220	memset(ke, 0, sizeof(KextNode));
221
222	ke->kext = kext;
223	ke->loadTag = kmod_info->id;
224	ke->address = kmod_info->address;
225	ke->address_end = kmod_info->address + kmod_info->size;
226
227	SLIST_INIT(&ke->classList);
228	TAILQ_INIT(&ke->userClientCallList);
229
230	IORWLockWrite(lock);
231
232	RB_INSERT(KextTree, &kextHead, ke);
233	RB_INSERT(KextAddressTree, &kextAddressHead, ke);
234
235	sequenceID++;
236	loadedKexts++;
237	lastKextIndex++;
238
239	IORWLockUnlock(lock);
240}
241
242void IOStatistics::onKextUnload(OSKext *kext)
243{
244	KextNode sought, *found;
245
246	assert(kext);
247
248	if (!enabled) {
249		return;
250	}
251
252	LOG(1, "IOStatistics::onKextUnload: %s\n", kext->getIdentifierCString());
253
254	IORWLockWrite(lock);
255
256	sought.kext = kext;
257	found = RB_FIND(KextTree, &kextHead, &sought);
258	if (found) {
259		IOWorkLoopCounter *wlc;
260		IOUserClientProcessEntry *uce;
261
262		/* Free up the list of counters */
263		while ((wlc = SLIST_FIRST(&found->workLoopList))) {
264			SLIST_REMOVE_HEAD(&found->workLoopList, link);
265			kfree(wlc, sizeof(IOWorkLoopCounter));
266		}
267
268		/* Free up the user client list */
269		while ((uce = TAILQ_FIRST(&found->userClientCallList))) {
270			TAILQ_REMOVE(&found->userClientCallList, uce, link);
271			kfree(uce, sizeof(IOUserClientProcessEntry));
272		}
273
274		/* Remove from kext trees */
275		RB_REMOVE(KextTree, &kextHead, found);
276		RB_REMOVE(KextAddressTree, &kextAddressHead, found);
277
278		/*
279		 * Clear a matching kextHint to avoid use after free in
280		 * onClassAdded() for a class add after a KEXT unload.
281		 */
282		if (found == kextHint) {
283			kextHint = NULL;
284		}
285
286		/* Finally, free the class node */
287		kfree(found, sizeof(KextNode));
288
289		sequenceID++;
290		loadedKexts--;
291	}
292	else {
293		panic("IOStatistics::onKextUnload: cannot find kext: %s", kext->getIdentifierCString());
294	}
295
296	IORWLockUnlock(lock);
297}
298
299void IOStatistics::onClassAdded(OSKext *parentKext, OSMetaClass *metaClass)
300{
301	ClassNode *ce;
302	KextNode soughtKext, *foundKext = NULL;
303
304	assert(parentKext && metaClass);
305
306	if (!enabled) {
307		return;
308	}
309
310	LOG(1, "IOStatistics::onClassAdded: %s\n", metaClass->getClassName());
311
312	ce = (ClassNode *)kalloc(sizeof(ClassNode));
313	if (!ce) {
314		return;
315	}
316
317	memset(ce, 0, sizeof(ClassNode));
318
319	IORWLockWrite(lock);
320
321	/* Hinted? */
322	if (kextHint && kextHint->kext == parentKext) {
323		foundKext = kextHint;
324	}
325	else {
326		soughtKext.kext = parentKext;
327		foundKext = RB_FIND(KextTree, &kextHead, &soughtKext);
328	}
329
330	if (foundKext) {
331		ClassNode soughtClass, *foundClass = NULL;
332		const OSMetaClass *superClass;
333
334		ce->metaClass = metaClass;
335		ce->classID = lastClassIndex++;
336		ce->parentKext = foundKext;
337
338		/* Has superclass? */
339	 	superClass = ce->metaClass->getSuperClass();
340		if (superClass) {
341			soughtClass.metaClass = superClass;
342			foundClass = RB_FIND(ClassTree, &classHead, &soughtClass);
343		}
344		ce->superClassID = foundClass ? foundClass->classID : (uint32_t)(-1);
345
346		SLIST_INIT(&ce->counterList);
347		SLIST_INIT(&ce->userClientList);
348
349		RB_INSERT(ClassTree, &classHead, ce);
350		SLIST_INSERT_HEAD(&foundKext->classList, ce, lLink);
351
352		foundKext->classes++;
353
354		kextHint = foundKext;
355
356		sequenceID++;
357		registeredClasses++;
358	}
359	else {
360		panic("IOStatistics::onClassAdded: cannot find parent kext: %s", parentKext->getIdentifierCString());
361	}
362
363	IORWLockUnlock(lock);
364}
365
366void IOStatistics::onClassRemoved(OSKext *parentKext, OSMetaClass *metaClass)
367{
368	ClassNode sought, *found;
369
370	assert(parentKext && metaClass);
371
372	if (!enabled) {
373		return;
374	}
375
376	LOG(1, "IOStatistics::onClassRemoved: %s\n", metaClass->getClassName());
377
378	IORWLockWrite(lock);
379
380	sought.metaClass = metaClass;
381	found = RB_FIND(ClassTree, &classHead, &sought);
382	if (found) {
383		IOEventSourceCounter *esc;
384		IOUserClientCounter *ucc;
385
386		/* Free up the list of counters */
387		while ((esc = SLIST_FIRST(&found->counterList))) {
388			SLIST_REMOVE_HEAD(&found->counterList, link);
389			kfree(esc, sizeof(IOEventSourceCounter));
390		}
391
392		/* Free up the user client list */
393		while ((ucc = SLIST_FIRST(&found->userClientList))) {
394			SLIST_REMOVE_HEAD(&found->userClientList, link);
395			kfree(ucc, sizeof(IOUserClientCounter));
396		}
397
398		/* Remove from class tree */
399		RB_REMOVE(ClassTree, &classHead, found);
400
401		/* Remove from parent */
402		SLIST_REMOVE(&found->parentKext->classList, found, ClassNode, lLink);
403
404		/* Finally, free the class node */
405		kfree(found, sizeof(ClassNode));
406
407		sequenceID++;
408		registeredClasses--;
409	}
410	else {
411		panic("IOStatistics::onClassRemoved: cannot find class: %s", metaClass->getClassName());
412	}
413
414	IORWLockUnlock(lock);
415}
416
417IOEventSourceCounter *IOStatistics::registerEventSource(OSObject *inOwner)
418{
419	IOEventSourceCounter *counter = NULL;
420	ClassNode sought, *found = NULL;
421	boolean_t createDummyCounter = FALSE;
422
423	assert(inOwner);
424
425	if (!enabled) {
426		return NULL;
427	}
428
429	counter = (IOEventSourceCounter*)kalloc(sizeof(IOEventSourceCounter));
430	if (!counter) {
431		return NULL;
432	}
433
434	memset(counter, 0, sizeof(IOEventSourceCounter));
435
436	IORWLockWrite(lock);
437
438	/* Workaround for <rdar://problem/7158117> - create a dummy counter when inOwner is bad.
439	 * We use retainCount here as our best indication that the pointer is awry.
440	 */
441	if (inOwner->retainCount > 0xFFFFFF) {
442		kprintf("IOStatistics::registerEventSource - bad metaclass %p\n", inOwner);
443		createDummyCounter = TRUE;
444	}
445	else {
446		sought.metaClass = inOwner->getMetaClass();
447		found = RB_FIND(ClassTree, &classHead, &sought);
448	}
449
450	if (found) {
451		counter->parentClass = found;
452		SLIST_INSERT_HEAD(&found->counterList, counter, link);
453		registeredCounters++;
454	}
455
456	if (!(createDummyCounter || found)) {
457		panic("IOStatistics::registerEventSource: cannot find parent class: %s", inOwner->getMetaClass()->getClassName());
458	}
459
460	IORWLockUnlock(lock);
461
462	return counter;
463}
464
465void IOStatistics::unregisterEventSource(IOEventSourceCounter *counter)
466{
467	if (!counter) {
468		return;
469	}
470
471	IORWLockWrite(lock);
472
473	if (counter->parentClass) {
474		SLIST_REMOVE(&counter->parentClass->counterList, counter, IOEventSourceCounter, link);
475		registeredCounters--;
476	}
477	kfree(counter, sizeof(IOEventSourceCounter));
478
479	IORWLockUnlock(lock);
480}
481
482IOWorkLoopCounter* IOStatistics::registerWorkLoop(IOWorkLoop *workLoop)
483{
484	IOWorkLoopCounter *counter = NULL;
485	KextNode *found;
486
487	assert(workLoop);
488
489	if (!enabled) {
490		return NULL;
491	}
492
493	counter = (IOWorkLoopCounter*)kalloc(sizeof(IOWorkLoopCounter));
494	if (!counter) {
495		return NULL;
496	}
497
498	memset(counter, 0, sizeof(IOWorkLoopCounter));
499
500	found = getKextNodeFromBacktrace(TRUE);
501	if (!found) {
502		panic("IOStatistics::registerWorkLoop: cannot find parent kext");
503	}
504
505	counter->parentKext = found;
506	counter->workLoop = workLoop;
507	RB_INIT(&counter->dependencyHead);
508	SLIST_INSERT_HEAD(&found->workLoopList, counter, link);
509	registeredWorkloops++;
510
511	releaseKextNode(found);
512
513	return counter;
514}
515
516void IOStatistics::unregisterWorkLoop(IOWorkLoopCounter *counter)
517{
518	if (!counter) {
519		return;
520	}
521
522	IORWLockWrite(lock);
523
524	SLIST_REMOVE(&counter->parentKext->workLoopList, counter, IOWorkLoopCounter, link);
525	kfree(counter, sizeof(IOWorkLoopCounter));
526	registeredWorkloops--;
527
528	IORWLockUnlock(lock);
529}
530
531IOUserClientCounter *IOStatistics::registerUserClient(IOUserClient *userClient)
532{
533	ClassNode sought, *found;
534	IOUserClientCounter *counter = NULL;
535
536	assert(userClient);
537
538	if (!enabled) {
539		return NULL;
540	}
541
542	counter = (IOUserClientCounter*)kalloc(sizeof(IOUserClientCounter));
543	if (!counter) {
544		return NULL;
545	}
546
547	memset(counter, 0, sizeof(IOUserClientCounter));
548
549	IORWLockWrite(lock);
550
551	sought.metaClass = userClient->getMetaClass();
552
553	found = RB_FIND(ClassTree, &classHead, &sought);
554	if (found) {
555		counter->parentClass = found;
556		SLIST_INSERT_HEAD(&found->userClientList, counter, link);
557	}
558	else {
559		panic("IOStatistics::registerUserClient: cannot find parent class: %s", sought.metaClass->getClassName());
560	}
561
562	IORWLockUnlock(lock);
563
564	return counter;
565}
566
567void IOStatistics::unregisterUserClient(IOUserClientCounter *counter)
568{
569	if (!counter) {
570		return;
571	}
572
573	IORWLockWrite(lock);
574
575	SLIST_REMOVE(&counter->parentClass->userClientList, counter, IOUserClientCounter, link);
576	kfree(counter, sizeof(IOUserClientCounter));
577
578	IORWLockUnlock(lock);
579}
580
581void IOStatistics::attachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
582{
583	if (!wlc) {
584        return;
585	}
586
587	IORWLockWrite(lock);
588
589	if (!nextWorkLoopDependency) {
590		return;
591	}
592
593	attachedEventSources++;
594	wlc->attachedEventSources++;
595
596	/* Track the kext dependency */
597	nextWorkLoopDependency->loadTag = esc->parentClass->parentKext->loadTag;
598	if (NULL == RB_INSERT(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, nextWorkLoopDependency)) {
599		nextWorkLoopDependency = (IOWorkLoopDependency*)kalloc(sizeof(IOWorkLoopDependency));
600	}
601
602	IORWLockUnlock(lock);
603}
604
605void IOStatistics::detachWorkLoopEventSource(IOWorkLoopCounter *wlc, IOEventSourceCounter *esc)
606{
607	IOWorkLoopDependency sought, *found;
608
609	if (!wlc) {
610		return;
611	}
612
613	IORWLockWrite(lock);
614
615	attachedEventSources--;
616	wlc->attachedEventSources--;
617
618	sought.loadTag = esc->parentClass->parentKext->loadTag;
619
620	found = RB_FIND(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, &sought);
621	if (found) {
622		RB_REMOVE(IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead, found);
623		kfree(found, sizeof(IOWorkLoopDependency));
624	}
625
626	IORWLockUnlock(lock);
627}
628
629int IOStatistics::getStatistics(sysctl_req *req)
630{
631	int error;
632	uint32_t calculatedSize, size;
633	char *buffer, *ptr;
634	IOStatisticsHeader *header;
635
636	assert(IOStatistics::enabled && req);
637
638	IORWLockRead(IOStatistics::lock);
639
640	/* Work out how much we need to allocate. IOStatisticsKext is of variable size. */
641	calculatedSize = sizeof(IOStatisticsHeader) +
642					 sizeof(IOStatisticsGlobal) +
643					(sizeof(IOStatisticsKext) * loadedKexts) + (sizeof(uint32_t) * registeredClasses) +
644					(sizeof(IOStatisticsMemory) * loadedKexts) +
645					(sizeof(IOStatisticsClass) * registeredClasses) +
646					(sizeof(IOStatisticsCounter) * registeredClasses) +
647					(sizeof(IOStatisticsKextIdentifier) * loadedKexts) +
648					(sizeof(IOStatisticsClassName) * registeredClasses);
649
650	/* Size request? */
651	if (req->oldptr == USER_ADDR_NULL) {
652		error = SYSCTL_OUT(req, NULL, calculatedSize);
653		goto exit;
654	}
655
656	/* Read only */
657	if (req->newptr != USER_ADDR_NULL) {
658		error = EPERM;
659		goto exit;
660	}
661
662	buffer = (char*)kalloc(calculatedSize);
663	if (!buffer) {
664		error = ENOMEM;
665		goto exit;
666	}
667
668	memset(buffer, 0, calculatedSize);
669
670	ptr = buffer;
671
672	header = (IOStatisticsHeader*)((void*)ptr);
673
674	header->sig = IOSTATISTICS_SIG;
675	header->ver = IOSTATISTICS_VER;
676
677	header->seq = sequenceID;
678
679	ptr += sizeof(IOStatisticsHeader);
680
681	/* Global data - seq, timers, interrupts, etc) */
682	header->globalStatsOffset = sizeof(IOStatisticsHeader);
683	size = copyGlobalStatistics((IOStatisticsGlobal*)((void*)ptr));
684	ptr += size;
685
686	/* Kext statistics */
687	header->kextStatsOffset = header->globalStatsOffset + size;
688	size = copyKextStatistics((IOStatisticsKext*)((void*)ptr));
689	ptr += size;
690
691	/* Memory allocation info */
692	header->memoryStatsOffset = header->kextStatsOffset + size;
693	size = copyMemoryStatistics((IOStatisticsMemory*)((void*)ptr));
694	ptr += size;
695
696	/* Class statistics */
697	header->classStatsOffset = header->memoryStatsOffset + size;
698	size = copyClassStatistics((IOStatisticsClass*)((void*)ptr));
699	ptr += size;
700
701	/* Dynamic class counter data */
702	header->counterStatsOffset = header->classStatsOffset + size;
703	size = copyCounterStatistics((IOStatisticsCounter*)((void*)ptr));
704	ptr += size;
705
706	/* Kext identifiers */
707	header->kextIdentifiersOffset = header->counterStatsOffset + size;
708	size = copyKextIdentifiers((IOStatisticsKextIdentifier*)((void*)ptr));
709	ptr += size;
710
711	/* Class names */
712	header->classNamesOffset = header->kextIdentifiersOffset + size;
713	size = copyClassNames((IOStatisticsClassName*)ptr);
714	ptr += size;
715
716	LOG(2, "IOStatistics::getStatistics - calculatedSize 0x%x, kexts 0x%x, classes 0x%x.\n",
717	 	calculatedSize, loadedKexts, registeredClasses);
718
719	assert( (uint32_t)(ptr - buffer) == calculatedSize );
720
721	error = SYSCTL_OUT(req, buffer, calculatedSize);
722
723	kfree(buffer, calculatedSize);
724
725exit:
726	IORWLockUnlock(IOStatistics::lock);
727	return error;
728}
729
730int IOStatistics::getWorkLoopStatistics(sysctl_req *req)
731{
732	int error;
733	uint32_t calculatedSize, size;
734	char *buffer;
735	IOStatisticsWorkLoopHeader *header;
736
737	assert(IOStatistics::enabled && req);
738
739	IORWLockRead(IOStatistics::lock);
740
741	/* Approximate how much we need to allocate (worse case estimate) */
742	calculatedSize = sizeof(IOStatisticsWorkLoop) * registeredWorkloops +
743					 sizeof(uint32_t) * attachedEventSources;
744
745	/* Size request? */
746	if (req->oldptr == USER_ADDR_NULL) {
747		error = SYSCTL_OUT(req, NULL, calculatedSize);
748		goto exit;
749	}
750
751	/* Read only */
752	if (req->newptr != USER_ADDR_NULL) {
753		error = EPERM;
754		goto exit;
755	}
756
757	buffer = (char*)kalloc(calculatedSize);
758	if (!buffer) {
759		error = ENOMEM;
760		goto exit;
761	}
762
763	header = (IOStatisticsWorkLoopHeader*)((void*)buffer);
764
765	header->sig = IOSTATISTICS_SIG_WORKLOOP;
766	header->ver = IOSTATISTICS_VER;
767
768	header->seq = sequenceID;
769
770	header->workloopCount = registeredWorkloops;
771
772	size = copyWorkLoopStatistics(&header->workLoopStats);
773
774	LOG(2, "IOStatistics::getWorkLoopStatistics: calculatedSize %d, size %d\n", calculatedSize, size);
775
776	assert( size <= calculatedSize );
777
778	error = SYSCTL_OUT(req, buffer, size);
779
780	kfree(buffer, calculatedSize);
781
782exit:
783	IORWLockUnlock(IOStatistics::lock);
784	return error;
785}
786
787int IOStatistics::getUserClientStatistics(sysctl_req *req)
788{
789	int error;
790	uint32_t calculatedSize, size;
791	char *buffer;
792	uint32_t requestedLoadTag = 0;
793	IOStatisticsUserClientHeader *header;
794
795	assert(IOStatistics::enabled && req);
796
797	IORWLockRead(IOStatistics::lock);
798
799	/* Work out how much we need to allocate */
800	calculatedSize = sizeof(IOStatisticsUserClientHeader) +
801					 sizeof(IOStatisticsUserClientCall) * IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS * loadedKexts;
802
803	/* Size request? */
804	if (req->oldptr == USER_ADDR_NULL) {
805		error = SYSCTL_OUT(req, NULL, calculatedSize);
806		goto exit;
807	}
808
809	/* Kext request (potentially) valid? */
810	if (!req->newptr || req->newlen < sizeof(requestedLoadTag)) {
811		error = EINVAL;
812		goto exit;
813	}
814
815	SYSCTL_IN(req, &requestedLoadTag, sizeof(requestedLoadTag));
816
817	LOG(2, "IOStatistics::getUserClientStatistics - requesting kext w/load tag: %d\n", requestedLoadTag);
818
819	buffer = (char*)kalloc(calculatedSize);
820	if (!buffer) {
821		error = ENOMEM;
822		goto exit;
823	}
824
825	header = (IOStatisticsUserClientHeader*)((void*)buffer);
826
827	header->sig = IOSTATISTICS_SIG_USERCLIENT;
828	header->ver = IOSTATISTICS_VER;
829
830	header->seq = sequenceID;
831
832	header->processes = 0;
833
834	size = copyUserClientStatistics(header, requestedLoadTag);
835
836	assert((sizeof(IOStatisticsUserClientHeader) + size) <= calculatedSize);
837
838	if (size) {
839		error = SYSCTL_OUT(req, buffer, sizeof(IOStatisticsUserClientHeader) + size);
840	}
841	else {
842		error = EINVAL;
843	}
844
845	kfree(buffer, calculatedSize);
846
847exit:
848	IORWLockUnlock(IOStatistics::lock);
849	return error;
850}
851
852uint32_t IOStatistics::copyGlobalStatistics(IOStatisticsGlobal *stats)
853{
854	stats->kextCount = loadedKexts;
855	stats->classCount = registeredClasses;
856	stats->workloops = registeredWorkloops;
857
858	return sizeof(IOStatisticsGlobal);
859}
860
861uint32_t IOStatistics::copyKextStatistics(IOStatisticsKext *stats)
862{
863	KextNode *ke;
864	ClassNode *ce;
865	uint32_t index = 0;
866
867	RB_FOREACH(ke, KextTree, &kextHead) {
868		stats->loadTag = ke->loadTag;
869		ke->kext->getSizeInfo(&stats->loadSize, &stats->wiredSize);
870
871		stats->classes = ke->classes;
872
873		/* Append indices of owned classes */
874		SLIST_FOREACH(ce, &ke->classList, lLink) {
875			stats->classIndexes[index++] = ce->classID;
876		}
877
878		stats = (IOStatisticsKext *)((void*)((char*)stats + sizeof(IOStatisticsKext) + (ke->classes * sizeof(uint32_t))));
879	}
880
881	return (sizeof(IOStatisticsKext) * loadedKexts + sizeof(uint32_t) * registeredClasses);
882}
883
884uint32_t IOStatistics::copyMemoryStatistics(IOStatisticsMemory *stats)
885{
886	KextNode *ke;
887
888	RB_FOREACH(ke, KextTree, &kextHead) {
889		stats->allocatedSize = ke->memoryCounters[kIOStatisticsMalloc];
890		stats->freedSize = ke->memoryCounters[kIOStatisticsFree];
891		stats->allocatedAlignedSize = ke->memoryCounters[kIOStatisticsMallocAligned];
892		stats->freedAlignedSize = ke->memoryCounters[kIOStatisticsFreeAligned];
893		stats->allocatedContiguousSize = ke->memoryCounters[kIOStatisticsMallocContiguous];
894		stats->freedContiguousSize = ke->memoryCounters[kIOStatisticsFreeContiguous];
895		stats->allocatedPageableSize = ke->memoryCounters[kIOStatisticsMallocPageable];
896		stats->freedPageableSize = ke->memoryCounters[kIOStatisticsFreePageable];
897		stats++;
898	}
899
900	return (sizeof(IOStatisticsMemory) * loadedKexts);
901}
902
903uint32_t IOStatistics::copyClassStatistics(IOStatisticsClass *stats)
904{
905	KextNode *ke;
906	ClassNode *ce;
907
908	RB_FOREACH(ke, KextTree, &kextHead) {
909		SLIST_FOREACH(ce, &ke->classList, lLink) {
910			stats->classID = ce->classID;
911			stats->superClassID = ce->superClassID;
912			stats->classSize = ce->metaClass->getClassSize();
913
914			stats++;
915		}
916	}
917
918	return sizeof(IOStatisticsClass) * registeredClasses;
919}
920
921uint32_t IOStatistics::copyCounterStatistics(IOStatisticsCounter *stats)
922{
923	KextNode *ke;
924	ClassNode *ce;
925
926	RB_FOREACH(ke, KextTree, &kextHead) {
927		SLIST_FOREACH(ce, &ke->classList, lLink) {
928			IOUserClientCounter *userClientCounter;
929			IOEventSourceCounter *counter;
930
931			stats->classID = ce->classID;
932			stats->classInstanceCount = ce->metaClass->getInstanceCount();
933
934			IOStatisticsUserClients *uc = &stats->userClientStatistics;
935
936			/* User client counters */
937			SLIST_FOREACH(userClientCounter, &ce->userClientList, link) {
938				uc->clientCalls += userClientCounter->clientCalls;
939				uc->created++;
940			}
941
942			IOStatisticsInterruptEventSources *iec = &stats->interruptEventSourceStatistics;
943			IOStatisticsInterruptEventSources *fiec = &stats->filterInterruptEventSourceStatistics;
944			IOStatisticsTimerEventSources *tec = &stats->timerEventSourceStatistics;
945			IOStatisticsCommandGates *cgc = &stats->commandGateStatistics;
946			IOStatisticsCommandQueues *cqc = &stats->commandQueueStatistics;
947			IOStatisticsDerivedEventSources *dec = &stats->derivedEventSourceStatistics;
948
949			/* Event source counters */
950			SLIST_FOREACH(counter, &ce->counterList, link) {
951				switch (counter->type) {
952					case kIOStatisticsInterruptEventSourceCounter:
953						iec->created++;
954						iec->produced += counter->u.interrupt.produced;
955						iec->checksForWork += counter->u.interrupt.checksForWork;
956						break;
957					case kIOStatisticsFilterInterruptEventSourceCounter:
958						fiec->created++;
959						fiec->produced += counter->u.filter.produced;
960						fiec->checksForWork += counter->u.filter.checksForWork;
961						break;
962					case kIOStatisticsTimerEventSourceCounter:
963						tec->created++;
964						tec->timeouts += counter->u.timer.timeouts;
965						tec->checksForWork += counter->u.timer.checksForWork;
966						tec->timeOnGate += counter->timeOnGate;
967						tec->closeGateCalls += counter->closeGateCalls;
968						tec->openGateCalls += counter->openGateCalls;
969						break;
970					case kIOStatisticsCommandGateCounter:
971						cgc->created++;
972						cgc->timeOnGate += counter->timeOnGate;
973						cgc->actionCalls += counter->u.commandGate.actionCalls;
974						break;
975					case kIOStatisticsCommandQueueCounter:
976						cqc->created++;
977						cqc->actionCalls += counter->u.commandQueue.actionCalls;
978						break;
979					case kIOStatisticsDerivedEventSourceCounter:
980						dec->created++;
981						dec->timeOnGate += counter->timeOnGate;
982						dec->closeGateCalls += counter->closeGateCalls;
983						dec->openGateCalls += counter->openGateCalls;
984						break;
985					default:
986						break;
987				}
988			}
989
990			stats++;
991		}
992	}
993
994	return sizeof(IOStatisticsCounter) * registeredClasses;
995}
996
997uint32_t IOStatistics::copyKextIdentifiers(IOStatisticsKextIdentifier *kextIDs)
998{
999	KextNode *ke;
1000
1001	RB_FOREACH(ke, KextTree, &kextHead) {
1002		strncpy(kextIDs->identifier, ke->kext->getIdentifierCString(), kIOStatisticsDriverNameLength);
1003		kextIDs++;
1004	}
1005
1006	return (sizeof(IOStatisticsKextIdentifier) * loadedKexts);
1007}
1008
1009uint32_t IOStatistics::copyClassNames(IOStatisticsClassName *classNames)
1010{
1011	KextNode *ke;
1012	ClassNode *ce;
1013
1014	RB_FOREACH(ke, KextTree, &kextHead) {
1015		SLIST_FOREACH(ce, &ke->classList, lLink) {
1016			strncpy(classNames->name, ce->metaClass->getClassName(), kIOStatisticsClassNameLength);
1017			classNames++;
1018		}
1019	}
1020
1021	return (sizeof(IOStatisticsClassName) * registeredClasses);
1022}
1023
1024uint32_t IOStatistics::copyWorkLoopStatistics(IOStatisticsWorkLoop *stats)
1025{
1026	KextNode *ke;
1027	IOWorkLoopCounter *wlc;
1028	IOWorkLoopDependency *dependentNode;
1029	uint32_t size, accumulatedSize = 0;
1030
1031	RB_FOREACH(ke, KextTree, &kextHead) {
1032		SLIST_FOREACH(wlc, &ke->workLoopList, link) {
1033			stats->kextLoadTag = ke->loadTag;
1034			stats->attachedEventSources = wlc->attachedEventSources;
1035			stats->timeOnGate = wlc->timeOnGate;
1036			stats->dependentKexts = 0;
1037			RB_FOREACH(dependentNode, IOWorkLoopCounter::DependencyTree, &wlc->dependencyHead) {
1038				stats->dependentKextLoadTags[stats->dependentKexts] = dependentNode->loadTag;
1039				stats->dependentKexts++;
1040			}
1041
1042			size = sizeof(IOStatisticsWorkLoop) + (sizeof(uint32_t) * stats->dependentKexts);
1043
1044			accumulatedSize += size;
1045			stats = (IOStatisticsWorkLoop*)((void*)((char*)stats + size));
1046		}
1047	}
1048
1049	return accumulatedSize;
1050}
1051
1052uint32_t IOStatistics::copyUserClientStatistics(IOStatisticsUserClientHeader *stats, uint32_t loadTag)
1053{
1054	KextNode *sought, *found = NULL;
1055	uint32_t procs = 0;
1056	IOUserClientProcessEntry *processEntry;
1057
1058	RB_FOREACH(sought, KextTree, &kextHead) {
1059		if (sought->loadTag == loadTag) {
1060			found = sought;
1061			break;
1062		}
1063	}
1064
1065	if (!found) {
1066		return 0;
1067	}
1068
1069	TAILQ_FOREACH(processEntry, &found->userClientCallList, link) {
1070		strncpy(stats->userClientCalls[procs].processName, processEntry->processName, kIOStatisticsProcessNameLength);
1071		stats->userClientCalls[procs].pid = processEntry->pid;
1072		stats->userClientCalls[procs].calls = processEntry->calls;
1073		stats->processes++;
1074		procs++;
1075	}
1076
1077	return sizeof(IOStatisticsUserClientCall) * stats->processes;
1078}
1079
1080void IOStatistics::storeUserClientCallInfo(IOUserClient *userClient, IOUserClientCounter *counter)
1081{
1082	OSString *ossUserClientCreator = NULL;
1083	int32_t pid = -1;
1084	KextNode *parentKext;
1085	IOUserClientProcessEntry *entry, *nextEntry, *prevEntry = NULL;
1086	uint32_t count = 0;
1087	const char *ptr = NULL;
1088	OSObject *obj;
1089
1090	/* TODO: see if this can be more efficient */
1091	obj = userClient->copyProperty("IOUserClientCreator",
1092					gIOServicePlane,
1093					kIORegistryIterateRecursively | kIORegistryIterateParents);
1094
1095	if (!obj)
1096		goto err_nounlock;
1097
1098	ossUserClientCreator = OSDynamicCast(OSString, obj);
1099
1100	if (ossUserClientCreator) {
1101		uint32_t len, lenIter = 0;
1102
1103		ptr = ossUserClientCreator->getCStringNoCopy();
1104		len = ossUserClientCreator->getLength();
1105
1106		while ((*ptr != ' ') && (lenIter < len)) {
1107			ptr++;
1108			lenIter++;
1109		}
1110
1111		if (lenIter < len) {
1112			ptr++; // Skip the space
1113			lenIter++;
1114			pid = 0;
1115			while ( (*ptr != ',') && (lenIter < len)) {
1116				pid = pid*10 + (*ptr - '0');
1117				ptr++;
1118				lenIter++;
1119			}
1120
1121			if(lenIter == len) {
1122				pid = -1;
1123			} else {
1124				ptr += 2;
1125			}
1126		}
1127	}
1128
1129	if (-1 == pid)
1130		goto err_nounlock;
1131
1132	IORWLockWrite(lock);
1133
1134	parentKext = counter->parentClass->parentKext;
1135
1136	TAILQ_FOREACH(entry, &parentKext->userClientCallList, link) {
1137		if (entry->pid == pid) {
1138			/* Found, so increment count and move to the head */
1139			entry->calls++;
1140			if (count) {
1141				TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1142				break;
1143			}
1144			else {
1145				/* At the head already, so increment and return */
1146				goto err_unlock;
1147			}
1148		}
1149
1150		count++;
1151	}
1152
1153	if (!entry) {
1154		if (count == IOKIT_STATISTICS_RECORDED_USERCLIENT_PROCS) {
1155			/* Max elements hit, so reuse the last */
1156			entry = TAILQ_LAST(&parentKext->userClientCallList, ProcessEntryList);
1157			TAILQ_REMOVE(&parentKext->userClientCallList, entry, link);
1158		}
1159		else {
1160			/* Otherwise, allocate a new entry */
1161			entry = (IOUserClientProcessEntry*)kalloc(sizeof(IOUserClientProcessEntry));
1162			if (!entry) {
1163			    IORWLockUnlock(lock);
1164				return;
1165			}
1166		}
1167
1168		strncpy(entry->processName, ptr, kIOStatisticsProcessNameLength);
1169		entry->pid = pid;
1170		entry->calls = 1;
1171	}
1172
1173	TAILQ_FOREACH(nextEntry, &parentKext->userClientCallList, link) {
1174		if (nextEntry->calls <= entry->calls)
1175			break;
1176
1177		prevEntry = nextEntry;
1178	}
1179
1180	if (!prevEntry)
1181		TAILQ_INSERT_HEAD(&parentKext->userClientCallList, entry, link);
1182	else
1183		TAILQ_INSERT_AFTER(&parentKext->userClientCallList, prevEntry, entry, link);
1184
1185err_unlock:
1186	IORWLockUnlock(lock);
1187
1188err_nounlock:
1189	if (obj)
1190		obj->release();
1191}
1192
1193void IOStatistics::countUserClientCall(IOUserClient *client) {
1194	IOUserClient::ExpansionData *data;
1195	IOUserClientCounter *counter;
1196
1197	/* Guard against an uninitialized client object - <rdar://problem/8577946> */
1198	if (!(data = client->reserved)) {
1199		return;
1200	}
1201
1202	if ((counter = data->counter)) {
1203		storeUserClientCallInfo(client, counter);
1204		OSIncrementAtomic(&counter->clientCalls);
1205	}
1206}
1207
1208KextNode *IOStatistics::getKextNodeFromBacktrace(boolean_t write) {
1209	const uint32_t btMin = 3;
1210
1211	void *bt[16];
1212	unsigned btCount = sizeof(bt) / sizeof(bt[0]);
1213	vm_offset_t *scanAddr = NULL;
1214	uint32_t i;
1215	KextNode *found = NULL, *ke = NULL;
1216
1217	btCount = OSBacktrace(bt, btCount);
1218
1219	if (write) {
1220		IORWLockWrite(lock);
1221	} else {
1222		IORWLockRead(lock);
1223	}
1224
1225	/* Ignore first levels */
1226	scanAddr = (vm_offset_t *)&bt[btMin - 1];
1227
1228	for (i = btMin - 1; i < btCount; i++, scanAddr++) {
1229		ke = RB_ROOT(&kextAddressHead);
1230		while (ke) {
1231			if (*scanAddr < ke->address) {
1232				ke = RB_LEFT(ke, addressLink);
1233			}
1234			else {
1235				if ((*scanAddr < ke->address_end) && (*scanAddr >= ke->address)) {
1236 					if (!ke->kext->isKernelComponent()) {
1237 						return ke;
1238					} else {
1239						found = ke;
1240					}
1241				}
1242				ke = RB_RIGHT(ke, addressLink);
1243			}
1244		}
1245	}
1246
1247	if (!found) {
1248		IORWLockUnlock(lock);
1249	}
1250
1251	return found;
1252}
1253
1254void IOStatistics::releaseKextNode(KextNode *node) {
1255#pragma unused(node)
1256	IORWLockUnlock(lock);
1257}
1258
1259/* IOLib allocations */
1260void IOStatistics::countAlloc(uint32_t index, vm_size_t size) {
1261	KextNode *ke;
1262
1263	if (!enabled) {
1264		return;
1265	}
1266
1267	ke = getKextNodeFromBacktrace(FALSE);
1268	if (ke) {
1269		OSAddAtomic(size, &ke->memoryCounters[index]);
1270		releaseKextNode(ke);
1271	}
1272}
1273
1274#endif /* IOKITSTATS */
1275