1/*
2 * Copyright (c) 1998-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <IOKit/IOInterruptEventSource.h>
30#include <IOKit/IOKitDebug.h>
31#include <IOKit/IOLib.h>
32#include <IOKit/IOService.h>
33#include <IOKit/IOInterrupts.h>
34#include <IOKit/IOTimeStamp.h>
35#include <IOKit/IOWorkLoop.h>
36#include <IOKit/IOInterruptAccountingPrivate.h>
37
38#if IOKITSTATS
39
40#define IOStatisticsInitializeCounter() \
41do { \
42	IOStatistics::setCounterType(IOEventSource::reserved->counter, kIOStatisticsInterruptEventSourceCounter); \
43} while (0)
44
45#define IOStatisticsCheckForWork() \
46do { \
47	IOStatistics::countInterruptCheckForWork(IOEventSource::reserved->counter); \
48} while (0)
49
50#define IOStatisticsInterrupt() \
51do { \
52	IOStatistics::countInterrupt(IOEventSource::reserved->counter); \
53} while (0)
54
55#else
56
57#define IOStatisticsInitializeCounter()
58#define IOStatisticsCheckForWork()
59#define IOStatisticsInterrupt()
60
61#endif // IOKITSTATS
62
63#define super IOEventSource
64
65OSDefineMetaClassAndStructors(IOInterruptEventSource, IOEventSource)
66OSMetaClassDefineReservedUnused(IOInterruptEventSource, 0);
67OSMetaClassDefineReservedUnused(IOInterruptEventSource, 1);
68OSMetaClassDefineReservedUnused(IOInterruptEventSource, 2);
69OSMetaClassDefineReservedUnused(IOInterruptEventSource, 3);
70OSMetaClassDefineReservedUnused(IOInterruptEventSource, 4);
71OSMetaClassDefineReservedUnused(IOInterruptEventSource, 5);
72OSMetaClassDefineReservedUnused(IOInterruptEventSource, 6);
73OSMetaClassDefineReservedUnused(IOInterruptEventSource, 7);
74
75bool IOInterruptEventSource::init(OSObject *inOwner,
76				  Action inAction,
77				  IOService *inProvider,
78				  int inIntIndex)
79{
80    bool res = true;
81
82    if ( !super::init(inOwner, (IOEventSourceAction) inAction) )
83        return false;
84
85    reserved = IONew(ExpansionData, 1);
86
87    if (!reserved) {
88        return false;
89    }
90
91    bzero(reserved, sizeof(ExpansionData));
92
93    provider = inProvider;
94    producerCount = consumerCount = 0;
95    autoDisable = explicitDisable = false;
96    intIndex = ~inIntIndex;
97
98    // Assumes inOwner holds a reference(retain) on the provider
99    if (inProvider) {
100        if (IA_ANY_STATISTICS_ENABLED) {
101            /*
102             * We only treat this as an "interrupt" if it has a provider; if it does,
103             * set up the objects necessary to track interrupt statistics.  Interrupt
104             * event sources without providers are most likely being used as simple
105             * event source in order to poke at workloops and kick off work.
106             *
107             * We also avoid try to avoid interrupt accounting overhead if none of
108             * the statistics are enabled.
109             */
110            reserved->statistics = IONew(IOInterruptAccountingData, 1);
111
112            if (!reserved->statistics) {
113                /*
114                 * We rely on the free() routine to clean up after us if init fails
115                 * midway.
116                 */
117                return false;
118            }
119
120            bzero(reserved->statistics, sizeof(IOInterruptAccountingData));
121
122            reserved->statistics->owner = this;
123        }
124
125        res = (kIOReturnSuccess == registerInterruptHandler(inProvider, inIntIndex));
126
127	if (res) {
128	    intIndex = inIntIndex;
129        }
130    }
131
132    IOStatisticsInitializeCounter();
133
134    return res;
135}
136
137IOReturn IOInterruptEventSource::registerInterruptHandler(IOService *inProvider,
138				  int inIntIndex)
139{
140    IOReturn ret;
141    int intType;
142    IOInterruptAction intHandler;
143
144    ret = inProvider->getInterruptType(inIntIndex, &intType);
145    if (kIOReturnSuccess != ret)
146	return (ret);
147
148    autoDisable = (intType == kIOInterruptTypeLevel);
149    if (autoDisable) {
150	intHandler = OSMemberFunctionCast(IOInterruptAction,
151	    this, &IOInterruptEventSource::disableInterruptOccurred);
152    }
153    else
154	intHandler = OSMemberFunctionCast(IOInterruptAction,
155	    this, &IOInterruptEventSource::normalInterruptOccurred);
156
157    ret = provider->registerInterrupt(inIntIndex, this, intHandler);
158
159    /*
160     * Add statistics to the provider.  The setWorkLoop convention should ensure
161     * that we always go down the unregister path before we register (outside of
162     * init()), so we don't have to worry that we will invoke addInterruptStatistics
163     * erroneously.
164     */
165    if ((ret == kIOReturnSuccess) && (reserved->statistics)) {
166        /*
167         * Stash the normal index value, for the sake of debugging.
168         */
169        reserved->statistics->interruptIndex = inIntIndex;
170
171        /*
172         * We need to hook the interrupt information up to the provider so that it
173         * can find the statistics for this interrupt when desired.  The provider is
174         * responsible for maintaining the reporter for a particular interrupt, and
175         * needs a handle on the statistics so that it can request that the reporter
176         * be updated as needed.  Errors are considered "soft" for the moment (it
177         * will either panic, or fail in a way such that we can still service the
178         * interrupt).
179         */
180        provider->addInterruptStatistics(reserved->statistics, inIntIndex);
181
182        /*
183         * Add the statistics object to the global list of statistics objects; this
184         * is an aid to debugging (we can trivially find statistics for all eligible
185         * interrupts, and dump them; potentially helpful if the system is wedged
186         * due to interrupt activity).
187         */
188        interruptAccountingDataAddToList(reserved->statistics);
189    }
190
191    return (ret);
192}
193
194void
195IOInterruptEventSource::unregisterInterruptHandler(IOService *inProvider,
196				  int inIntIndex)
197{
198    if (reserved->statistics) {
199        interruptAccountingDataRemoveFromList(reserved->statistics);
200        provider->removeInterruptStatistics(reserved->statistics->interruptIndex);
201    }
202
203    provider->unregisterInterrupt(inIntIndex);
204}
205
206
207IOInterruptEventSource *
208IOInterruptEventSource::interruptEventSource(OSObject *inOwner,
209					     Action inAction,
210					     IOService *inProvider,
211					     int inIntIndex)
212{
213    IOInterruptEventSource *me = new IOInterruptEventSource;
214
215    if (me && !me->init(inOwner, inAction, inProvider, inIntIndex)) {
216        me->release();
217        return 0;
218    }
219
220    return me;
221}
222
223void IOInterruptEventSource::free()
224{
225    if (provider && intIndex >= 0)
226        unregisterInterruptHandler(provider, intIndex);
227
228    if (reserved) {
229        if (reserved->statistics) {
230            IODelete(reserved->statistics, IOInterruptAccountingData, 1);
231        }
232
233        IODelete(reserved, ExpansionData, 1);
234    }
235
236    super::free();
237}
238
239void IOInterruptEventSource::enable()
240{
241    if (provider && intIndex >= 0) {
242        provider->enableInterrupt(intIndex);
243        explicitDisable = false;
244        enabled = true;
245    }
246}
247
248void IOInterruptEventSource::disable()
249{
250    if (provider && intIndex >= 0) {
251        provider->disableInterrupt(intIndex);
252        explicitDisable = true;
253        enabled = false;
254    }
255}
256
257void IOInterruptEventSource::setWorkLoop(IOWorkLoop *inWorkLoop)
258{
259    if (inWorkLoop) super::setWorkLoop(inWorkLoop);
260
261    if (provider) {
262	if (!inWorkLoop) {
263	    if (intIndex >= 0) {
264                /*
265                 * It isn't necessarily safe to wait until free() to unregister the interrupt;
266                 * our provider may disappear.
267                 */
268                unregisterInterruptHandler(provider, intIndex);
269		intIndex = ~intIndex;
270	    }
271	} else if ((intIndex < 0) && (kIOReturnSuccess == registerInterruptHandler(provider, ~intIndex))) {
272	    intIndex = ~intIndex;
273	}
274    }
275
276    if (!inWorkLoop) super::setWorkLoop(inWorkLoop);
277}
278
279const IOService *IOInterruptEventSource::getProvider() const
280{
281    return provider;
282}
283
284int IOInterruptEventSource::getIntIndex() const
285{
286    return intIndex;
287}
288
289bool IOInterruptEventSource::getAutoDisable() const
290{
291    return autoDisable;
292}
293
294bool IOInterruptEventSource::checkForWork()
295{
296    uint64_t startSystemTime = 0;
297    uint64_t endSystemTime = 0;
298    uint64_t startCPUTime = 0;
299    uint64_t endCPUTime = 0;
300    unsigned int cacheProdCount = producerCount;
301    int numInts = cacheProdCount - consumerCount;
302    IOInterruptEventAction intAction = (IOInterruptEventAction) action;
303	bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
304
305    IOStatisticsCheckForWork();
306
307	if ( numInts > 0 )
308	{
309		if (trace)
310			IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
311						 VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop);
312
313		if (reserved->statistics) {
314			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
315				startSystemTime = mach_absolute_time();
316			}
317
318			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
319				startCPUTime = thread_get_runtime_self();
320			}
321		}
322
323		// Call the handler
324		(*intAction)(owner, this, numInts);
325
326		if (reserved->statistics) {
327			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
328				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
329			}
330
331			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
332				endCPUTime = thread_get_runtime_self();
333				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
334			}
335
336			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
337				endSystemTime = mach_absolute_time();
338				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
339			}
340		}
341
342		if (trace)
343			IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
344					       VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop);
345
346		consumerCount = cacheProdCount;
347		if (autoDisable && !explicitDisable)
348			enable();
349	}
350
351	else if ( numInts < 0 )
352	{
353		if (trace)
354			IOTimeStampStartConstant(IODBG_INTES(IOINTES_ACTION),
355						 VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop);
356
357		if (reserved->statistics) {
358			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
359				startSystemTime = mach_absolute_time();
360			}
361
362			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
363				startCPUTime = thread_get_runtime_self();
364			}
365		}
366
367		// Call the handler
368		(*intAction)(owner, this, -numInts);
369
370		if (reserved->statistics) {
371			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCountIndex)) {
372				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCountIndex], 1);
373			}
374
375			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelCPUTimeIndex)) {
376				endCPUTime = thread_get_runtime_self();
377				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelCPUTimeIndex], endCPUTime - startCPUTime);
378			}
379
380			if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingSecondLevelSystemTimeIndex)) {
381				endSystemTime = mach_absolute_time();
382				IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingSecondLevelSystemTimeIndex], endSystemTime - startSystemTime);
383			}
384		}
385
386		if (trace)
387			IOTimeStampEndConstant(IODBG_INTES(IOINTES_ACTION),
388					       VM_KERNEL_UNSLIDE(intAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop);
389
390		consumerCount = cacheProdCount;
391		if (autoDisable && !explicitDisable)
392			enable();
393	}
394
395    return false;
396}
397
398void IOInterruptEventSource::normalInterruptOccurred
399    (void */*refcon*/, IOService */*prov*/, int /*source*/)
400{
401	bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
402
403    IOStatisticsInterrupt();
404    producerCount++;
405
406	if (trace)
407	    IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner);
408
409    if (reserved->statistics) {
410        if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
411            IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
412        }
413    }
414
415    signalWorkAvailable();
416
417	if (trace)
418	    IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner);
419}
420
421void IOInterruptEventSource::disableInterruptOccurred
422    (void */*refcon*/, IOService *prov, int source)
423{
424	bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
425
426    prov->disableInterrupt(source);	/* disable the interrupt */
427
428    IOStatisticsInterrupt();
429    producerCount++;
430
431	if (trace)
432	    IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner);
433
434    if (reserved->statistics) {
435        if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
436            IA_ADD_VALUE(&reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
437        }
438    }
439
440    signalWorkAvailable();
441
442	if (trace)
443	    IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner);
444}
445
446void IOInterruptEventSource::interruptOccurred
447    (void *refcon, IOService *prov, int source)
448{
449    if (autoDisable && prov)
450        disableInterruptOccurred(refcon, prov, source);
451    else
452        normalInterruptOccurred(refcon, prov, source);
453}
454
455IOReturn IOInterruptEventSource::warmCPU
456    (uint64_t abstime)
457{
458
459	return ml_interrupt_prewarm(abstime);
460}
461