1/* 2 * Copyright (c) 1998-2010 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <pexpert/pexpert.h> 30#include <IOKit/IOWorkLoop.h> 31#include <IOKit/IOEventSource.h> 32#include <IOKit/IOInterruptEventSource.h> 33#include <IOKit/IOCommandGate.h> 34#include <IOKit/IOTimeStamp.h> 35#include <IOKit/IOKitDebug.h> 36#include <libkern/OSDebug.h> 37#include <kern/thread.h> 38 39#define super OSObject 40 41OSDefineMetaClassAndStructors(IOWorkLoop, OSObject); 42 43// Block of unused functions intended for future use 44#if __LP64__ 45OSMetaClassDefineReservedUnused(IOWorkLoop, 0); 46OSMetaClassDefineReservedUnused(IOWorkLoop, 1); 47OSMetaClassDefineReservedUnused(IOWorkLoop, 2); 48#else 49OSMetaClassDefineReservedUsed(IOWorkLoop, 0); 50OSMetaClassDefineReservedUsed(IOWorkLoop, 1); 51OSMetaClassDefineReservedUsed(IOWorkLoop, 2); 52#endif 53OSMetaClassDefineReservedUnused(IOWorkLoop, 3); 54OSMetaClassDefineReservedUnused(IOWorkLoop, 4); 55OSMetaClassDefineReservedUnused(IOWorkLoop, 5); 56OSMetaClassDefineReservedUnused(IOWorkLoop, 6); 57OSMetaClassDefineReservedUnused(IOWorkLoop, 7); 58 59enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 }; 60static inline void SETP(void *addr, unsigned int flag) 61 { unsigned char *num = (unsigned char *) addr; *num |= flag; } 62static inline void CLRP(void *addr, unsigned int flag) 63 { unsigned char *num = (unsigned char *) addr; *num &= ~flag; } 64static inline bool ISSETP(void *addr, unsigned int flag) 65 { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; } 66 67#define fFlags loopRestart 68 69#define passiveEventChain reserved->passiveEventChain 70 71#if IOKITSTATS 72 73#define IOStatisticsRegisterCounter() \ 74do { \ 75 reserved->counter = IOStatistics::registerWorkLoop(this); \ 76} while(0) 77 78#define IOStatisticsUnregisterCounter() \ 79do { \ 80 if (reserved) \ 81 IOStatistics::unregisterWorkLoop(reserved->counter); \ 82} while(0) 83 84#define IOStatisticsOpenGate() \ 85do { \ 86 IOStatistics::countWorkLoopOpenGate(reserved->counter); \ 87} while(0) 88 89#define IOStatisticsCloseGate() \ 90do { \ 91 IOStatistics::countWorkLoopCloseGate(reserved->counter); \ 92} while(0) 93 94#define IOStatisticsAttachEventSource() \ 95do { \ 96 IOStatistics::attachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ 97} while(0) 98 99#define IOStatisticsDetachEventSource() \ 100do { \ 101 IOStatistics::detachWorkLoopEventSource(reserved->counter, inEvent->reserved->counter); \ 102} while(0) 103 104#else 105 106#define IOStatisticsRegisterCounter() 107#define IOStatisticsUnregisterCounter() 108#define IOStatisticsOpenGate() 109#define IOStatisticsCloseGate() 110#define IOStatisticsAttachEventSource() 111#define IOStatisticsDetachEventSource() 112 113#endif /* IOKITSTATS */ 114 115bool IOWorkLoop::init() 116{ 117 // The super init and gateLock allocation MUST be done first. 118 if ( !super::init() ) 119 return false; 120 121 // Allocate our ExpansionData if it hasn't been allocated already. 122 if ( !reserved ) 123 { 124 reserved = IONew(ExpansionData,1); 125 if ( !reserved ) 126 return false; 127 128 bzero(reserved,sizeof(ExpansionData)); 129 } 130 131#if DEBUG 132 OSBacktrace ( reserved->allocationBacktrace, sizeof ( reserved->allocationBacktrace ) / sizeof ( reserved->allocationBacktrace[0] ) ); 133#endif 134 135 if ( gateLock == NULL ) { 136 if ( !( gateLock = IORecursiveLockAlloc()) ) 137 return false; 138 } 139 140 if ( workToDoLock == NULL ) { 141 if ( !(workToDoLock = IOSimpleLockAlloc()) ) 142 return false; 143 IOSimpleLockInit(workToDoLock); 144 workToDo = false; 145 } 146 147 IOStatisticsRegisterCounter(); 148 149 if ( controlG == NULL ) { 150 controlG = IOCommandGate::commandGate( 151 this, 152 OSMemberFunctionCast( 153 IOCommandGate::Action, 154 this, 155 &IOWorkLoop::_maintRequest)); 156 157 if ( !controlG ) 158 return false; 159 // Point the controlGate at the workLoop. Usually addEventSource 160 // does this automatically. The problem is in this case addEventSource 161 // uses the control gate and it has to be bootstrapped. 162 controlG->setWorkLoop(this); 163 if (addEventSource(controlG) != kIOReturnSuccess) 164 return false; 165 } 166 167 if ( workThread == NULL ) { 168 thread_continue_t cptr = OSMemberFunctionCast( 169 thread_continue_t, 170 this, 171 &IOWorkLoop::threadMain); 172 if (KERN_SUCCESS != kernel_thread_start(cptr, this, &workThread)) 173 return false; 174 } 175 176 (void) thread_set_tag(workThread, THREAD_TAG_IOWORKLOOP); 177 return true; 178} 179 180IOWorkLoop * 181IOWorkLoop::workLoop() 182{ 183 return IOWorkLoop::workLoopWithOptions(0); 184} 185 186IOWorkLoop * 187IOWorkLoop::workLoopWithOptions(IOOptionBits options) 188{ 189 IOWorkLoop *me = new IOWorkLoop; 190 191 if (me && options) { 192 me->reserved = IONew(ExpansionData,1); 193 if (!me->reserved) { 194 me->release(); 195 return 0; 196 } 197 bzero(me->reserved,sizeof(ExpansionData)); 198 me->reserved->options = options; 199 } 200 201 if (me && !me->init()) { 202 me->release(); 203 return 0; 204 } 205 206 return me; 207} 208 209// Free is called twice: 210// First when the atomic retainCount transitions from 1 -> 0 211// Secondly when the work loop itself is commiting hari kari 212// Hence the each leg of the free must be single threaded. 213void IOWorkLoop::free() 214{ 215 if (workThread) { 216 IOInterruptState is; 217 218 // If we are here then we must be trying to shut down this work loop 219 // in this case disable all of the event source, mark the loop 220 // as terminating and wakeup the work thread itself and return 221 // Note: we hold the gate across the entire operation mainly for the 222 // benefit of our event sources so we can disable them cleanly. 223 closeGate(); 224 225 disableAllEventSources(); 226 227 is = IOSimpleLockLockDisableInterrupt(workToDoLock); 228 SETP(&fFlags, kLoopTerminate); 229 thread_wakeup_one((void *) &workToDo); 230 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 231 232 openGate(); 233 } 234 else /* !workThread */ { 235 IOEventSource *event, *next; 236 237 for (event = eventChain; event; event = next) { 238 next = event->getNext(); 239 event->setWorkLoop(0); 240 event->setNext(0); 241 event->release(); 242 } 243 eventChain = 0; 244 245 for (event = passiveEventChain; event; event = next) { 246 next = event->getNext(); 247 event->setWorkLoop(0); 248 event->setNext(0); 249 event->release(); 250 } 251 passiveEventChain = 0; 252 253 // Either we have a partial initialization to clean up 254 // or the workThread itself is performing hari-kari. 255 // Either way clean up all of our resources and return. 256 257 if (controlG) { 258 controlG->release(); 259 controlG = 0; 260 } 261 262 if (workToDoLock) { 263 IOSimpleLockFree(workToDoLock); 264 workToDoLock = 0; 265 } 266 267 if (gateLock) { 268 IORecursiveLockFree(gateLock); 269 gateLock = 0; 270 } 271 272 IOStatisticsUnregisterCounter(); 273 274 if (reserved) { 275 IODelete(reserved, ExpansionData, 1); 276 reserved = 0; 277 } 278 279 super::free(); 280 } 281} 282 283IOReturn IOWorkLoop::addEventSource(IOEventSource *newEvent) 284{ 285 return controlG->runCommand((void *) mAddEvent, (void *) newEvent); 286} 287 288IOReturn IOWorkLoop::removeEventSource(IOEventSource *toRemove) 289{ 290 return controlG->runCommand((void *) mRemoveEvent, (void *) toRemove); 291} 292 293void IOWorkLoop::enableAllEventSources() const 294{ 295 IOEventSource *event; 296 297 for (event = eventChain; event; event = event->getNext()) 298 event->enable(); 299 300 for (event = passiveEventChain; event; event = event->getNext()) 301 event->enable(); 302} 303 304void IOWorkLoop::disableAllEventSources() const 305{ 306 IOEventSource *event; 307 308 for (event = eventChain; event; event = event->getNext()) 309 event->disable(); 310 311 /* NOTE: controlG is in passiveEventChain since it's an IOCommandGate */ 312 for (event = passiveEventChain; event; event = event->getNext()) 313 if (event != controlG) // Don't disable the control gate 314 event->disable(); 315} 316 317void IOWorkLoop::enableAllInterrupts() const 318{ 319 IOEventSource *event; 320 321 for (event = eventChain; event; event = event->getNext()) 322 if (OSDynamicCast(IOInterruptEventSource, event)) 323 event->enable(); 324} 325 326void IOWorkLoop::disableAllInterrupts() const 327{ 328 IOEventSource *event; 329 330 for (event = eventChain; event; event = event->getNext()) 331 if (OSDynamicCast(IOInterruptEventSource, event)) 332 event->disable(); 333} 334 335 336/* virtual */ bool IOWorkLoop::runEventSources() 337{ 338 bool res = false; 339 bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false; 340 bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false; 341 342 closeGate(); 343 if (ISSETP(&fFlags, kLoopTerminate)) 344 goto abort; 345 346 if (traceWL) 347 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); 348 349 bool more; 350 do { 351 CLRP(&fFlags, kLoopRestart); 352 more = false; 353 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 354 workToDo = false; 355 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 356 /* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */ 357 for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) { 358 359 if (traceES) 360 IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); 361 362 more |= evnt->checkForWork(); 363 364 if (traceES) 365 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt); 366 367 if (ISSETP(&fFlags, kLoopTerminate)) 368 goto abort; 369 else if (fFlags & kLoopRestart) { 370 more = true; 371 break; 372 } 373 } 374 } while (more); 375 376 res = true; 377 378 if (traceWL) 379 IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this); 380 381abort: 382 openGate(); 383 return res; 384} 385 386/* virtual */ void IOWorkLoop::threadMain() 387{ 388restartThread: 389 do { 390 if ( !runEventSources() ) 391 goto exitThread; 392 393 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 394 if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) { 395 assert_wait((void *) &workToDo, false); 396 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 397 thread_continue_t cptr = NULL; 398 if (!reserved || !(kPreciousStack & reserved->options)) 399 cptr = OSMemberFunctionCast( 400 thread_continue_t, this, &IOWorkLoop::threadMain); 401 thread_block_parameter(cptr, this); 402 goto restartThread; 403 /* NOTREACHED */ 404 } 405 406 // At this point we either have work to do or we need 407 // to commit suicide. But no matter 408 // Clear the simple lock and retore the interrupt state 409 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 410 } while(workToDo); 411 412exitThread: 413 thread_t thread = workThread; 414 workThread = 0; // Say we don't have a loop and free ourselves 415 free(); 416 417 thread_deallocate(thread); 418 (void) thread_terminate(thread); 419} 420 421IOThread IOWorkLoop::getThread() const 422{ 423 return workThread; 424} 425 426bool IOWorkLoop::onThread() const 427{ 428 return (IOThreadSelf() == workThread); 429} 430 431bool IOWorkLoop::inGate() const 432{ 433 return IORecursiveLockHaveLock(gateLock); 434} 435 436// Internal APIs used by event sources to control the thread 437void IOWorkLoop::signalWorkAvailable() 438{ 439 if (workToDoLock) { 440 IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock); 441 workToDo = true; 442 thread_wakeup_one((void *) &workToDo); 443 IOSimpleLockUnlockEnableInterrupt(workToDoLock, is); 444 } 445} 446 447void IOWorkLoop::openGate() 448{ 449 IOStatisticsOpenGate(); 450 IORecursiveLockUnlock(gateLock); 451} 452 453void IOWorkLoop::closeGate() 454{ 455 IORecursiveLockLock(gateLock); 456 IOStatisticsCloseGate(); 457} 458 459bool IOWorkLoop::tryCloseGate() 460{ 461 bool res = (IORecursiveLockTryLock(gateLock) != 0); 462 if (res) { 463 IOStatisticsCloseGate(); 464 } 465 return res; 466} 467 468int IOWorkLoop::sleepGate(void *event, UInt32 interuptibleType) 469{ 470 int res; 471 IOStatisticsOpenGate(); 472 res = IORecursiveLockSleep(gateLock, event, interuptibleType); 473 IOStatisticsCloseGate(); 474 return res; 475} 476 477int IOWorkLoop::sleepGate(void *event, AbsoluteTime deadline, UInt32 interuptibleType) 478{ 479 int res; 480 IOStatisticsOpenGate(); 481 res = IORecursiveLockSleepDeadline(gateLock, event, deadline, interuptibleType); 482 IOStatisticsCloseGate(); 483 return res; 484} 485 486void IOWorkLoop::wakeupGate(void *event, bool oneThread) 487{ 488 IORecursiveLockWakeup(gateLock, event, oneThread); 489} 490 491IOReturn IOWorkLoop::runAction(Action inAction, OSObject *target, 492 void *arg0, void *arg1, 493 void *arg2, void *arg3) 494{ 495 IOReturn res; 496 497 // closeGate is recursive so don't worry if we already hold the lock. 498 closeGate(); 499 res = (*inAction)(target, arg0, arg1, arg2, arg3); 500 openGate(); 501 502 return res; 503} 504 505IOReturn IOWorkLoop::_maintRequest(void *inC, void *inD, void *, void *) 506{ 507 maintCommandEnum command = (maintCommandEnum) (uintptr_t) inC; 508 IOEventSource *inEvent = (IOEventSource *) inD; 509 IOReturn res = kIOReturnSuccess; 510 511 switch (command) 512 { 513 case mAddEvent: 514 if (!inEvent->getWorkLoop()) { 515 SETP(&fFlags, kLoopRestart); 516 517 inEvent->retain(); 518 inEvent->setWorkLoop(this); 519 inEvent->setNext(0); 520 521 /* Check if this is a passive or active event source being added */ 522 if (eventSourcePerformsWork(inEvent)) { 523 524 if (!eventChain) 525 eventChain = inEvent; 526 else { 527 IOEventSource *event, *next; 528 529 for (event = eventChain; (next = event->getNext()); event = next) 530 ; 531 event->setNext(inEvent); 532 533 } 534 535 } 536 else { 537 538 if (!passiveEventChain) 539 passiveEventChain = inEvent; 540 else { 541 IOEventSource *event, *next; 542 543 for (event = passiveEventChain; (next = event->getNext()); event = next) 544 ; 545 event->setNext(inEvent); 546 547 } 548 549 } 550 IOStatisticsAttachEventSource(); 551 } 552 break; 553 554 case mRemoveEvent: 555 if (inEvent->getWorkLoop()) { 556 IOStatisticsDetachEventSource(); 557 558 if (eventSourcePerformsWork(inEvent)) { 559 if (eventChain == inEvent) 560 eventChain = inEvent->getNext(); 561 else { 562 IOEventSource *event, *next; 563 564 event = eventChain; 565 while ((next = event->getNext()) && next != inEvent) 566 event = next; 567 568 if (!next) { 569 res = kIOReturnBadArgument; 570 break; 571 } 572 event->setNext(inEvent->getNext()); 573 } 574 } 575 else { 576 if (passiveEventChain == inEvent) 577 passiveEventChain = inEvent->getNext(); 578 else { 579 IOEventSource *event, *next; 580 581 event = passiveEventChain; 582 while ((next = event->getNext()) && next != inEvent) 583 event = next; 584 585 if (!next) { 586 res = kIOReturnBadArgument; 587 break; 588 } 589 event->setNext(inEvent->getNext()); 590 } 591 } 592 593 inEvent->setWorkLoop(0); 594 inEvent->setNext(0); 595 inEvent->release(); 596 SETP(&fFlags, kLoopRestart); 597 } 598 break; 599 600 default: 601 return kIOReturnUnsupported; 602 } 603 604 return res; 605} 606 607bool 608IOWorkLoop::eventSourcePerformsWork(IOEventSource *inEventSource) 609{ 610 bool result = true; 611 612 /* 613 * The idea here is to see if the subclass of IOEventSource has overridden checkForWork(). 614 * The assumption is that if you override checkForWork(), you need to be 615 * active and not passive. 616 * 617 * We picked a known quantity controlG that does not override 618 * IOEventSource::checkForWork(), namely the IOCommandGate associated with 619 * the workloop to which this event source is getting attached. 620 * 621 * We do a pointer comparison on the offset in the vtable for inNewEvent against 622 * the offset in the vtable for inReferenceEvent. This works because 623 * IOCommandGate's slot for checkForWork() has the address of 624 * IOEventSource::checkForWork() in it. 625 * 626 * Think of OSMemberFunctionCast yielding the value at the vtable offset for 627 * checkForWork() here. We're just testing to see if it's the same or not. 628 * 629 */ 630 if (controlG) { 631 void * ptr1; 632 void * ptr2; 633 634 ptr1 = OSMemberFunctionCast(void*, inEventSource, &IOEventSource::checkForWork); 635 ptr2 = OSMemberFunctionCast(void*, controlG, &IOEventSource::checkForWork); 636 637 if (ptr1 == ptr2) 638 result = false; 639 } 640 641 return result; 642} 643