1/*
2 * Copyright (c) 1998-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <pexpert/pexpert.h>
30#include <IOKit/IOWorkLoop.h>
31#include <IOKit/IOEventSource.h>
32#include <IOKit/IOInterruptEventSource.h>
33#include <IOKit/IOCommandGate.h>
34#include <IOKit/IOTimeStamp.h>
35#include <IOKit/IOKitDebug.h>
36#include <libkern/OSDebug.h>
37#include <kern/thread.h>
38
39#define super OSObject
40
41OSDefineMetaClassAndStructors(IOWorkLoop, OSObject);
42
43// Block of unused functions intended for future use
44#if __LP64__
45OSMetaClassDefineReservedUnused(IOWorkLoop, 0);
46OSMetaClassDefineReservedUnused(IOWorkLoop, 1);
47OSMetaClassDefineReservedUnused(IOWorkLoop, 2);
48#else
49OSMetaClassDefineReservedUsed(IOWorkLoop, 0);
50OSMetaClassDefineReservedUsed(IOWorkLoop, 1);
51OSMetaClassDefineReservedUsed(IOWorkLoop, 2);
52#endif
53OSMetaClassDefineReservedUnused(IOWorkLoop, 3);
54OSMetaClassDefineReservedUnused(IOWorkLoop, 4);
55OSMetaClassDefineReservedUnused(IOWorkLoop, 5);
56OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
57OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
58
59enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
60static inline void SETP(void *addr, unsigned int flag)
61    { unsigned char *num = (unsigned char *) addr; *num |= flag; }
62static inline void CLRP(void *addr, unsigned int flag)
63    { unsigned char *num = (unsigned char *) addr; *num &= ~flag; }
64static inline bool ISSETP(void *addr, unsigned int flag)
65    { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; }
66
67#define fFlags loopRestart
68
69#define passiveEventChain	reserved->passiveEventChain
70
71#if IOKITSTATS
72
73#define IOStatisticsRegisterCounter() \
74do { \
75	reserved->counter = IOStatistics::registerWorkLoop(this); \
76} while(0)
77
78#define IOStatisticsUnregisterCounter() \
79do { \
80	if (reserved) \
81		IOStatistics::unregisterWorkLoop(reserved->counter); \
82} while(0)
83
84#define IOStatisticsOpenGate() \
85do { \
86	IOStatistics::countWorkLoopOpenGate(reserved->counter); \
87} while(0)
88
89#define IOStatisticsCloseGate() \
90do { \
91	IOStatistics::countWorkLoopCloseGate(reserved->counter); \
92} while(0)
93
94#define IOStatisticsAttachEventSource() \
95do { \
96	IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
97} while(0)
98
99#define IOStatisticsDetachEventSource() \
100do { \
101	IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \
102} while(0)
103
104#else
105
106#define IOStatisticsRegisterCounter()
107#define IOStatisticsUnregisterCounter()
108#define IOStatisticsOpenGate()
109#define IOStatisticsCloseGate()
110#define IOStatisticsAttachEventSource()
111#define IOStatisticsDetachEventSource()
112
113#endif /* IOKITSTATS */
114
115bool IOWorkLoop::init()
116{
117    // The super init and gateLock allocation MUST be done first.
118    if ( !super::init() )
119        return false;
120
121	// Allocate our ExpansionData if it hasn't been allocated already.
122	if ( !reserved )
123	{
124		reserved = IONew(ExpansionData,1);
125		if ( !reserved )
126			return false;
127
128		bzero(reserved,sizeof(ExpansionData));
129	}
130
131#if DEBUG
132	OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) );
133#endif
134
135    if ( gateLock == NULL ) {
136        if ( !( gateLock = IORecursiveLockAlloc()) )
137            return false;
138    }
139
140    if ( workToDoLock == NULL ) {
141        if ( !(workToDoLock = IOSimpleLockAlloc()) )
142            return false;
143        IOSimpleLockInit(workToDoLock);
144        workToDo = false;
145    }
146
147    IOStatisticsRegisterCounter();
148
149    if ( controlG == NULL ) {
150        controlG = IOCommandGate::commandGate(
151            this,
152            OSMemberFunctionCast(
153                IOCommandGate::Action,
154                this,
155                &IOWorkLoop::_maintRequest));
156
157        if ( !controlG )
158            return false;
159        // Point the controlGate at the workLoop.  Usually addEventSource
160        // does this automatically.  The problem is in this case addEventSource
161        // uses the control gate and it has to be bootstrapped.
162        controlG->setWorkLoop(this);
163        if (addEventSource(controlG) != kIOReturnSuccess)
164            return false;
165    }
166
167    if ( workThread == NULL ) {
168        thread_continue_t cptr = OSMemberFunctionCast(
169            thread_continue_t,
170            this,
171            &IOWorkLoop::threadMain);
172        if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread))
173            return false;
174    }
175
176    (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP);
177    return true;
178}
179
180IOWorkLoop *
181IOWorkLoop::workLoop()
182{
183    return IOWorkLoop::workLoopWithOptions(0);
184}
185
186IOWorkLoop *
187IOWorkLoop::workLoopWithOptions(IOOptionBits options)
188{
189	IOWorkLoop *me = new IOWorkLoop;
190
191	if (me && options) {
192		me->reserved = IONew(ExpansionData,1);
193		if (!me->reserved) {
194			me->release();
195			return 0;
196		}
197		bzero(me->reserved,sizeof(ExpansionData));
198		me->reserved->options = options;
199	}
200
201	if (me && !me->init()) {
202		me->release();
203		return 0;
204	}
205
206	return me;
207}
208
209// Free is called twice:
210// First when the atomic retainCount transitions from 1 -> 0
211// Secondly when the work loop itself is commiting hari kari
212// Hence the each leg of the free must be single threaded.
213void IOWorkLoop::free()
214{
215    if (workThread) {
216	IOInterruptState is;
217
218	// If we are here then we must be trying to shut down this work loop
219	// in this case disable all of the event source, mark the loop
220	// as terminating and wakeup the work thread itself and return
221	// Note: we hold the gate across the entire operation mainly for the
222	// benefit of our event sources so we can disable them cleanly.
223	closeGate();
224
225	disableAllEventSources();
226
227        is = IOSimpleLockLockDisableInterrupt(workToDoLock);
228	SETP(&fFlags, kLoopTerminate);
229        thread_wakeup_one((void *) &workToDo);
230        IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
231
232	openGate();
233    }
234    else /* !workThread */ {
235        IOEventSource *event, *next;
236
237        for (event = eventChain; event; event = next) {
238            next = event->getNext();
239            event->setWorkLoop(0);
240            event->setNext(0);
241            event->release();
242        }
243        eventChain = 0;
244
245        for (event = passiveEventChain; event; event = next) {
246            next = event->getNext();
247            event->setWorkLoop(0);
248            event->setNext(0);
249            event->release();
250        }
251        passiveEventChain = 0;
252
253	// Either we have a partial initialization to clean up
254	// or the workThread itself is performing hari-kari.
255	// Either way clean up all of our resources and return.
256
257	if (controlG) {
258	    controlG->release();
259	    controlG = 0;
260	}
261
262	if (workToDoLock) {
263	    IOSimpleLockFree(workToDoLock);
264	    workToDoLock = 0;
265	}
266
267	if (gateLock) {
268	    IORecursiveLockFree(gateLock);
269	    gateLock = 0;
270	}
271
272	IOStatisticsUnregisterCounter();
273
274	if (reserved) {
275	    IODelete(reserved, ExpansionData, 1);
276	    reserved = 0;
277	}
278
279	super::free();
280    }
281}
282
283IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent)
284{
285    return controlG->runCommand((void *) mAddEvent, (void *) newEvent);
286}
287
288IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove)
289{
290    return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove);
291}
292
293void IOWorkLoop::enableAllEventSources() const
294{
295    IOEventSource *event;
296
297    for (event = eventChain; event; event = event->getNext())
298        event->enable();
299
300    for (event = passiveEventChain; event; event = event->getNext())
301        event->enable();
302}
303
304void IOWorkLoop::disableAllEventSources() const
305{
306    IOEventSource *event;
307
308    for (event = eventChain; event; event = event->getNext())
309		event->disable();
310
311	/* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */
312    for (event = passiveEventChain; event; event = event->getNext())
313        if (event != controlG)	// Don't disable the control gate
314            event->disable();
315}
316
317void IOWorkLoop::enableAllInterrupts() const
318{
319    IOEventSource *event;
320
321    for (event = eventChain; event; event = event->getNext())
322        if (OSDynamicCast(IOInterruptEventSource, event))
323            event->enable();
324}
325
326void IOWorkLoop::disableAllInterrupts() const
327{
328    IOEventSource *event;
329
330    for (event = eventChain; event; event = event->getNext())
331        if (OSDynamicCast(IOInterruptEventSource, event))
332            event->disable();
333}
334
335
336/* virtual */ bool IOWorkLoop::runEventSources()
337{
338    bool res = false;
339    bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
340    bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
341
342    closeGate();
343    if (ISSETP(&fFlags, kLoopTerminate))
344		goto abort;
345
346    if (traceWL)
347    	IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
348
349    bool more;
350    do {
351		CLRP(&fFlags, kLoopRestart);
352		more = false;
353		IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
354		workToDo = false;
355		IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
356		/* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
357		for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
358
359			if (traceES)
360				IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
361
362			more |= evnt->checkForWork();
363
364			if (traceES)
365				IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
366
367			if (ISSETP(&fFlags, kLoopTerminate))
368				goto abort;
369			else if (fFlags & kLoopRestart) {
370				more = true;
371				break;
372			}
373		}
374    } while (more);
375
376    res = true;
377
378    if (traceWL)
379    	IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
380
381abort:
382    openGate();
383    return res;
384}
385
386/* virtual */ void IOWorkLoop::threadMain()
387{
388restartThread:
389    do {
390	if ( !runEventSources() )
391	    goto exitThread;
392
393	IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
394        if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
395	    assert_wait((void *) &workToDo, false);
396	    IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
397	    thread_continue_t cptr = NULL;
398	    if (!reserved || !(kPreciousStack & reserved->options))
399		cptr = OSMemberFunctionCast(
400			thread_continue_t, this, &IOWorkLoop::threadMain);
401	    thread_block_parameter(cptr, this);
402	    goto restartThread;
403	    /* NOTREACHED */
404	}
405
406	// At this point we either have work to do or we need
407	// to commit suicide.  But no matter
408	// Clear the simple lock and retore the interrupt state
409	IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
410    } while(workToDo);
411
412exitThread:
413	thread_t thread = workThread;
414    workThread = 0;	// Say we don't have a loop and free ourselves
415    free();
416
417	thread_deallocate(thread);
418    (void) thread_terminate(thread);
419}
420
421IOThread IOWorkLoop::getThread() const
422{
423    return workThread;
424}
425
426bool IOWorkLoop::onThread() const
427{
428    return (IOThreadSelf() == workThread);
429}
430
431bool IOWorkLoop::inGate() const
432{
433    return IORecursiveLockHaveLock(gateLock);
434}
435
436// Internal APIs used by event sources to control the thread
437void IOWorkLoop::signalWorkAvailable()
438{
439    if (workToDoLock) {
440        IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
441        workToDo = true;
442        thread_wakeup_one((void *) &workToDo);
443        IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
444    }
445}
446
447void IOWorkLoop::openGate()
448{
449    IOStatisticsOpenGate();
450    IORecursiveLockUnlock(gateLock);
451}
452
453void IOWorkLoop::closeGate()
454{
455    IORecursiveLockLock(gateLock);
456    IOStatisticsCloseGate();
457}
458
459bool IOWorkLoop::tryCloseGate()
460{
461    bool res = (IORecursiveLockTryLock(gateLock) != 0);
462    if (res) {
463        IOStatisticsCloseGate();
464    }
465    return res;
466}
467
468int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType)
469{
470    int res;
471    IOStatisticsOpenGate();
472    res = IORecursiveLockSleep(gateLock, event, interuptibleType);
473    IOStatisticsCloseGate();
474    return res;
475}
476
477int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType)
478{
479    int res;
480    IOStatisticsOpenGate();
481    res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType);
482    IOStatisticsCloseGate();
483    return res;
484}
485
486void IOWorkLoop::wakeupGate(void *event, bool oneThread)
487{
488    IORecursiveLockWakeup(gateLock, event, oneThread);
489}
490
491IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target,
492                                  void *arg0, void *arg1,
493                                  void *arg2, void *arg3)
494{
495    IOReturn res;
496
497    // closeGate is recursive so don't worry if we already hold the lock.
498    closeGate();
499    res = (*inAction)(target, arg0, arg1, arg2, arg3);
500    openGate();
501
502    return res;
503}
504
505IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *)
506{
507    maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC;
508    IOEventSource *inEvent = (IOEventSource *) inD;
509    IOReturn res = kIOReturnSuccess;
510
511    switch (command)
512    {
513    case mAddEvent:
514        if (!inEvent->getWorkLoop()) {
515            SETP(&fFlags, kLoopRestart);
516
517            inEvent->retain();
518            inEvent->setWorkLoop(this);
519            inEvent->setNext(0);
520
521    		/* Check if this is a passive or active event source being added */
522    		if (eventSourcePerformsWork(inEvent)) {
523
524	            if (!eventChain)
525    	            eventChain = inEvent;
526        	    else {
527            	    IOEventSource *event, *next;
528
529                	for (event = eventChain; (next = event->getNext()); event = next)
530                    	;
531                	event->setNext(inEvent);
532
533            	}
534
535            }
536            else {
537
538	            if (!passiveEventChain)
539    	            passiveEventChain = inEvent;
540        	    else {
541            	    IOEventSource *event, *next;
542
543                	for (event = passiveEventChain; (next = event->getNext()); event = next)
544                    	;
545                	event->setNext(inEvent);
546
547            	}
548
549            }
550            IOStatisticsAttachEventSource();
551        }
552        break;
553
554    case mRemoveEvent:
555        if (inEvent->getWorkLoop()) {
556        	IOStatisticsDetachEventSource();
557
558        	if (eventSourcePerformsWork(inEvent)) {
559				if (eventChain == inEvent)
560					eventChain = inEvent->getNext();
561				else {
562					IOEventSource *event, *next;
563
564					event = eventChain;
565					while ((next = event->getNext()) && next != inEvent)
566						event = next;
567
568					if (!next) {
569						res = kIOReturnBadArgument;
570						break;
571					}
572					event->setNext(inEvent->getNext());
573				}
574    		}
575    		else {
576				if (passiveEventChain == inEvent)
577					passiveEventChain = inEvent->getNext();
578				else {
579					IOEventSource *event, *next;
580
581					event = passiveEventChain;
582					while ((next = event->getNext()) && next != inEvent)
583						event = next;
584
585					if (!next) {
586						res = kIOReturnBadArgument;
587						break;
588					}
589					event->setNext(inEvent->getNext());
590				}
591    		}
592
593            inEvent->setWorkLoop(0);
594            inEvent->setNext(0);
595            inEvent->release();
596            SETP(&fFlags, kLoopRestart);
597        }
598        break;
599
600    default:
601        return kIOReturnUnsupported;
602    }
603
604    return res;
605}
606
607bool
608IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource)
609{
610	bool	result = true;
611
612	/*
613	 * The idea here is to see if the subclass of IOEventSource has overridden checkForWork().
614	 * The assumption is that if you override checkForWork(), you need to be
615	 * active and not passive.
616	 *
617	 * We picked a known quantity controlG that does not override
618	 * IOEventSource::checkForWork(), namely the IOCommandGate associated with
619	 * the workloop to which this event source is getting attached.
620	 *
621	 * We do a pointer comparison on the offset in the vtable for inNewEvent against
622	 * the offset in the vtable for inReferenceEvent. This works because
623	 * IOCommandGate's slot for checkForWork() has the address of
624	 * IOEventSource::checkForWork() in it.
625	 *
626	 * Think of OSMemberFunctionCast yielding the value at the vtable offset for
627	 * checkForWork() here. We're just testing to see if it's the same or not.
628	 *
629	 */
630	if (controlG) {
631		void *	ptr1;
632		void *	ptr2;
633
634		ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork);
635		ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork);
636
637		if (ptr1 == ptr2)
638			result = false;
639	}
640
641    return result;
642}
643