1/* 2 * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <IOKit/assert.h> 30 31#include <libkern/OSTypes.h> 32#include <libkern/OSByteOrder.h> 33#include <libkern/OSDebug.h> 34 35#include <IOKit/IOReturn.h> 36#include <IOKit/IOLib.h> 37#include <IOKit/IODMACommand.h> 38#include <IOKit/IOMapper.h> 39#include <IOKit/IOMemoryDescriptor.h> 40#include <IOKit/IOBufferMemoryDescriptor.h> 41 42#include "IOKitKernelInternal.h" 43 44#define MAPTYPE(type) ((UInt) (type) & kTypeMask) 45#define IS_MAPPED(type) (MAPTYPE(type) != kBypassed) 46#define IS_BYPASSED(type) (MAPTYPE(type) == kBypassed) 47#define IS_NONCOHERENT(type) (MAPTYPE(type) == kNonCoherent) 48 49enum 50{ 51 kWalkSyncIn = 0x01, // bounce -> md 52 kWalkSyncOut = 0x02, // bounce <- md 53 kWalkSyncAlways = 0x04, 54 kWalkPreflight = 0x08, 55 kWalkDoubleBuffer = 0x10, 56 kWalkPrepare = 0x20, 57 kWalkComplete = 0x40, 58 kWalkClient = 0x80 59}; 60 61 62#define fInternalState reserved 63#define fState reserved->fState 64#define fMDSummary reserved->fMDSummary 65 66 67#if 1 68// no direction => OutIn 69#define SHOULD_COPY_DIR(op, direction) \ 70 ((kIODirectionNone == (direction)) \ 71 || (kWalkSyncAlways & (op)) \ 72 || (((kWalkSyncIn & (op)) ? kIODirectionIn : kIODirectionOut) \ 73 & (direction))) 74 75#else 76#define SHOULD_COPY_DIR(state, direction) (true) 77#endif 78 79#if 0 80#define DEBG(fmt, args...) { IOLog(fmt, ## args); kprintf(fmt, ## args); } 81#else 82#define DEBG(fmt, args...) {} 83#endif 84 85/**************************** class IODMACommand ***************************/ 86 87#undef super 88#define super IOCommand 89OSDefineMetaClassAndStructors(IODMACommand, IOCommand); 90 91OSMetaClassDefineReservedUsed(IODMACommand, 0); 92OSMetaClassDefineReservedUsed(IODMACommand, 1); 93OSMetaClassDefineReservedUsed(IODMACommand, 2); 94OSMetaClassDefineReservedUnused(IODMACommand, 3); 95OSMetaClassDefineReservedUnused(IODMACommand, 4); 96OSMetaClassDefineReservedUnused(IODMACommand, 5); 97OSMetaClassDefineReservedUnused(IODMACommand, 6); 98OSMetaClassDefineReservedUnused(IODMACommand, 7); 99OSMetaClassDefineReservedUnused(IODMACommand, 8); 100OSMetaClassDefineReservedUnused(IODMACommand, 9); 101OSMetaClassDefineReservedUnused(IODMACommand, 10); 102OSMetaClassDefineReservedUnused(IODMACommand, 11); 103OSMetaClassDefineReservedUnused(IODMACommand, 12); 104OSMetaClassDefineReservedUnused(IODMACommand, 13); 105OSMetaClassDefineReservedUnused(IODMACommand, 14); 106OSMetaClassDefineReservedUnused(IODMACommand, 15); 107 108IODMACommand * 109IODMACommand::withSpecification(SegmentFunction outSegFunc, 110 UInt8 numAddressBits, 111 UInt64 maxSegmentSize, 112 MappingOptions mappingOptions, 113 UInt64 maxTransferSize, 114 UInt32 alignment, 115 IOMapper *mapper, 116 void *refCon) 117{ 118 IODMACommand * me = new IODMACommand; 119 120 if (me && !me->initWithSpecification(outSegFunc, 121 numAddressBits, maxSegmentSize, 122 mappingOptions, maxTransferSize, 123 alignment, mapper, refCon)) 124 { 125 me->release(); 126 return 0; 127 }; 128 129 return me; 130} 131 132IODMACommand * 133IODMACommand::cloneCommand(void *refCon) 134{ 135 return withSpecification(fOutSeg, fNumAddressBits, fMaxSegmentSize, 136 fMappingOptions, fMaxTransferSize, fAlignMask + 1, fMapper, refCon); 137} 138 139#define kLastOutputFunction ((SegmentFunction) kLastOutputFunction) 140 141bool 142IODMACommand::initWithSpecification(SegmentFunction outSegFunc, 143 UInt8 numAddressBits, 144 UInt64 maxSegmentSize, 145 MappingOptions mappingOptions, 146 UInt64 maxTransferSize, 147 UInt32 alignment, 148 IOMapper *mapper, 149 void *refCon) 150{ 151 IOService * device = 0; 152 153 if (!super::init() || !outSegFunc) 154 return false; 155 156 bool is32Bit = (OutputHost32 == outSegFunc || OutputBig32 == outSegFunc 157 || OutputLittle32 == outSegFunc); 158 if (is32Bit) 159 { 160 if (!numAddressBits) 161 numAddressBits = 32; 162 else if (numAddressBits > 32) 163 return false; // Wrong output function for bits 164 } 165 166 if (numAddressBits && (numAddressBits < PAGE_SHIFT)) 167 return false; 168 169 if (!maxSegmentSize) 170 maxSegmentSize--; // Set Max segment to -1 171 if (!maxTransferSize) 172 maxTransferSize--; // Set Max transfer to -1 173 174 175 if (mapper && !OSDynamicCast(IOMapper, mapper)) 176 { 177 device = mapper; 178 mapper = 0; 179 } 180 if (!mapper) 181 { 182 IOMapper::checkForSystemMapper(); 183 mapper = IOMapper::gSystem; 184 } 185 186 fNumSegments = 0; 187 fBypassMask = 0; 188 fOutSeg = outSegFunc; 189 fNumAddressBits = numAddressBits; 190 fMaxSegmentSize = maxSegmentSize; 191 fMappingOptions = mappingOptions; 192 fMaxTransferSize = maxTransferSize; 193 if (!alignment) 194 alignment = 1; 195 fAlignMask = alignment - 1; 196 fMapper = mapper; 197 fRefCon = refCon; 198 199 switch (MAPTYPE(mappingOptions)) 200 { 201 case kMapped: break; 202 case kNonCoherent: /*fMapper = 0;*/ break; 203 case kBypassed: 204 if (mapper && !mapper->getBypassMask(&fBypassMask)) 205 return false; 206 break; 207 default: 208 return false; 209 }; 210 211 if (fMapper) 212 fMapper->retain(); 213 214 reserved = IONew(IODMACommandInternal, 1); 215 if (!reserved) 216 return false; 217 bzero(reserved, sizeof(IODMACommandInternal)); 218 219 fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions)); 220 fInternalState->fDevice = device; 221 222 return true; 223} 224 225void 226IODMACommand::free() 227{ 228 if (reserved) 229 IODelete(reserved, IODMACommandInternal, 1); 230 231 if (fMapper) 232 fMapper->release(); 233 234 super::free(); 235} 236 237IOReturn 238IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare) 239{ 240 IOReturn err = kIOReturnSuccess; 241 242 if (mem == fMemory) 243 { 244 if (!autoPrepare) 245 { 246 while (fActive) 247 complete(); 248 } 249 return kIOReturnSuccess; 250 } 251 252 if (fMemory) { 253 // As we are almost certainly being called from a work loop thread 254 // if fActive is true it is probably not a good time to potentially 255 // block. Just test for it and return an error 256 if (fActive) 257 return kIOReturnBusy; 258 clearMemoryDescriptor(); 259 } 260 261 if (mem) { 262 bzero(&fMDSummary, sizeof(fMDSummary)); 263 err = mem->dmaCommandOperation(kIOMDGetCharacteristics | (kMapped == MAPTYPE(fMappingOptions)), 264 &fMDSummary, sizeof(fMDSummary)); 265 if (err) 266 return err; 267 268 ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage; 269 270 if ((kMapped == MAPTYPE(fMappingOptions)) 271 && fMapper) 272 fInternalState->fCheckAddressing = false; 273 else 274 fInternalState->fCheckAddressing = (fNumAddressBits && (highPage >= (1UL << (fNumAddressBits - PAGE_SHIFT)))); 275 276 fInternalState->fNewMD = true; 277 mem->retain(); 278 fMemory = mem; 279 280 mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0); 281 if (autoPrepare) { 282 err = prepare(); 283 if (err) { 284 clearMemoryDescriptor(); 285 } 286 } 287 } 288 289 return err; 290} 291 292IOReturn 293IODMACommand::clearMemoryDescriptor(bool autoComplete) 294{ 295 if (fActive && !autoComplete) 296 return (kIOReturnNotReady); 297 298 if (fMemory) { 299 while (fActive) 300 complete(); 301 fMemory->dmaCommandOperation(kIOMDSetDMAInactive, this, 0); 302 fMemory->release(); 303 fMemory = 0; 304 } 305 306 return (kIOReturnSuccess); 307} 308 309const IOMemoryDescriptor * 310IODMACommand::getMemoryDescriptor() const 311{ 312 return fMemory; 313} 314 315 316IOReturn 317IODMACommand::segmentOp( 318 void *reference, 319 IODMACommand *target, 320 Segment64 segment, 321 void *segments, 322 UInt32 segmentIndex) 323{ 324 IOOptionBits op = (uintptr_t) reference; 325 addr64_t maxPhys, address; 326 uint64_t length; 327 uint32_t numPages; 328 329 IODMACommandInternal * state = target->reserved; 330 331 if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperPageAlloc || !target->fMapper)) 332 maxPhys = (1ULL << target->fNumAddressBits); 333 else 334 maxPhys = 0; 335 maxPhys--; 336 337 address = segment.fIOVMAddr; 338 length = segment.fLength; 339 340 assert(address); 341 assert(length); 342 343 if (!state->fMisaligned) 344 { 345 state->fMisaligned |= (0 != (state->fSourceAlignMask & address)); 346 if (state->fMisaligned) DEBG("misaligned %qx:%qx, %lx\n", address, length, state->fSourceAlignMask); 347 } 348 349 if (state->fMisaligned && (kWalkPreflight & op)) 350 return (kIOReturnNotAligned); 351 352 if (!state->fDoubleBuffer) 353 { 354 if ((address + length - 1) <= maxPhys) 355 { 356 length = 0; 357 } 358 else if (address <= maxPhys) 359 { 360 DEBG("tail %qx, %qx", address, length); 361 length = (address + length - maxPhys - 1); 362 address = maxPhys + 1; 363 DEBG("-> %qx, %qx\n", address, length); 364 } 365 } 366 367 if (!length) 368 return (kIOReturnSuccess); 369 370 numPages = atop_64(round_page_64((address & PAGE_MASK) + length)); 371 372 if (kWalkPreflight & op) 373 { 374 state->fCopyPageCount += numPages; 375 } 376 else 377 { 378 vm_page_t lastPage; 379 lastPage = NULL; 380 if (kWalkPrepare & op) 381 { 382 lastPage = state->fCopyNext; 383 for (IOItemCount idx = 0; idx < numPages; idx++) 384 { 385 vm_page_set_offset(lastPage, atop_64(address) + idx); 386 lastPage = vm_page_get_next(lastPage); 387 } 388 } 389 390 if (!lastPage || SHOULD_COPY_DIR(op, target->fMDSummary.fDirection)) 391 { 392 lastPage = state->fCopyNext; 393 for (IOItemCount idx = 0; idx < numPages; idx++) 394 { 395 if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection)) 396 { 397 addr64_t cpuAddr = address; 398 addr64_t remapAddr; 399 uint64_t chunk; 400 401 if ((kMapped == MAPTYPE(target->fMappingOptions)) 402 && target->fMapper) 403 { 404 cpuAddr = target->fMapper->mapAddr(address); 405 } 406 407 remapAddr = ptoa_64(vm_page_get_phys_page(lastPage)); 408 if (!state->fDoubleBuffer) 409 { 410 remapAddr += (address & PAGE_MASK); 411 } 412 chunk = PAGE_SIZE - (address & PAGE_MASK); 413 if (chunk > length) 414 chunk = length; 415 416 DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr, 417 (kWalkSyncIn & op) ? "->" : "<-", 418 address, chunk, op); 419 420 if (kWalkSyncIn & op) 421 { // cppvNoModSnk 422 copypv(remapAddr, cpuAddr, chunk, 423 cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc ); 424 } 425 else 426 { 427 copypv(cpuAddr, remapAddr, chunk, 428 cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc ); 429 } 430 address += chunk; 431 length -= chunk; 432 } 433 lastPage = vm_page_get_next(lastPage); 434 } 435 } 436 state->fCopyNext = lastPage; 437 } 438 439 return kIOReturnSuccess; 440} 441 442IOReturn 443IODMACommand::walkAll(UInt8 op) 444{ 445 IODMACommandInternal * state = fInternalState; 446 447 IOReturn ret = kIOReturnSuccess; 448 UInt32 numSegments; 449 UInt64 offset; 450 451 if (kWalkPreflight & op) 452 { 453 state->fMisaligned = false; 454 state->fDoubleBuffer = false; 455 state->fPrepared = false; 456 state->fCopyNext = NULL; 457 state->fCopyPageAlloc = 0; 458 state->fCopyPageCount = 0; 459 state->fNextRemapPage = NULL; 460 state->fCopyMD = 0; 461 462 if (!(kWalkDoubleBuffer & op)) 463 { 464 offset = 0; 465 numSegments = 0-1; 466 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); 467 } 468 469 op &= ~kWalkPreflight; 470 471 state->fDoubleBuffer = (state->fMisaligned || (kWalkDoubleBuffer & op)); 472 if (state->fDoubleBuffer) 473 state->fCopyPageCount = atop_64(round_page(state->fPreparedLength)); 474 475 if (state->fCopyPageCount) 476 { 477 vm_page_t mapBase = NULL; 478 479 DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount); 480 481 if (!state->fDoubleBuffer) 482 { 483 kern_return_t kr; 484 485 if (fMapper) panic("fMapper copying"); 486 487 kr = vm_page_alloc_list(state->fCopyPageCount, 488 KMA_LOMEM | KMA_NOPAGEWAIT, &mapBase); 489 if (KERN_SUCCESS != kr) 490 { 491 DEBG("vm_page_alloc_list(%d) failed (%d)\n", state->fCopyPageCount, kr); 492 mapBase = NULL; 493 } 494 } 495 496 if (mapBase) 497 { 498 state->fCopyPageAlloc = mapBase; 499 state->fCopyNext = state->fCopyPageAlloc; 500 offset = 0; 501 numSegments = 0-1; 502 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); 503 state->fPrepared = true; 504 op &= ~(kWalkSyncIn | kWalkSyncOut); 505 } 506 else 507 { 508 DEBG("alloc IOBMD\n"); 509 mach_vm_address_t mask = 0xFFFFF000; //state->fSourceAlignMask 510 state->fCopyMD = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task, 511 fMDSummary.fDirection, state->fPreparedLength, mask); 512 513 if (state->fCopyMD) 514 { 515 ret = kIOReturnSuccess; 516 state->fPrepared = true; 517 } 518 else 519 { 520 DEBG("IODMACommand !alloc IOBMD"); 521 return (kIOReturnNoResources); 522 } 523 } 524 } 525 } 526 527 if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op)) 528 { 529 if (state->fCopyPageCount) 530 { 531 DEBG("sync fCopyPageCount %d\n", state->fCopyPageCount); 532 533 if (state->fCopyPageAlloc) 534 { 535 state->fCopyNext = state->fCopyPageAlloc; 536 offset = 0; 537 numSegments = 0-1; 538 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); 539 } 540 else if (state->fCopyMD) 541 { 542 DEBG("sync IOBMD\n"); 543 544 if (SHOULD_COPY_DIR(op, fMDSummary.fDirection)) 545 { 546 IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory); 547 548 IOByteCount bytes; 549 550 if (kWalkSyncIn & op) 551 bytes = poMD->writeBytes(state->fPreparedOffset, 552 state->fCopyMD->getBytesNoCopy(), 553 state->fPreparedLength); 554 else 555 bytes = poMD->readBytes(state->fPreparedOffset, 556 state->fCopyMD->getBytesNoCopy(), 557 state->fPreparedLength); 558 DEBG("fCopyMD %s %lx bytes\n", (kWalkSyncIn & op) ? "wrote" : "read", bytes); 559 ret = (bytes == state->fPreparedLength) ? kIOReturnSuccess : kIOReturnUnderrun; 560 } 561 else 562 ret = kIOReturnSuccess; 563 } 564 } 565 } 566 567 if (kWalkComplete & op) 568 { 569 if (state->fCopyPageAlloc) 570 { 571 vm_page_free_list(state->fCopyPageAlloc, FALSE); 572 state->fCopyPageAlloc = 0; 573 state->fCopyPageCount = 0; 574 } 575 if (state->fCopyMD) 576 { 577 state->fCopyMD->release(); 578 state->fCopyMD = 0; 579 } 580 581 state->fPrepared = false; 582 } 583 return (ret); 584} 585 586UInt8 587IODMACommand::getNumAddressBits(void) 588{ 589 return (fNumAddressBits); 590} 591 592UInt32 593IODMACommand::getAlignment(void) 594{ 595 return (fAlignMask + 1); 596} 597 598IOReturn 599IODMACommand::prepareWithSpecification(SegmentFunction outSegFunc, 600 UInt8 numAddressBits, 601 UInt64 maxSegmentSize, 602 MappingOptions mappingOptions, 603 UInt64 maxTransferSize, 604 UInt32 alignment, 605 IOMapper *mapper, 606 UInt64 offset, 607 UInt64 length, 608 bool flushCache, 609 bool synchronize) 610{ 611 if (fActive) 612 return kIOReturnNotPermitted; 613 614 if (!outSegFunc) 615 return kIOReturnBadArgument; 616 617 bool is32Bit = (OutputHost32 == outSegFunc || OutputBig32 == outSegFunc 618 || OutputLittle32 == outSegFunc); 619 if (is32Bit) 620 { 621 if (!numAddressBits) 622 numAddressBits = 32; 623 else if (numAddressBits > 32) 624 return kIOReturnBadArgument; // Wrong output function for bits 625 } 626 627 if (numAddressBits && (numAddressBits < PAGE_SHIFT)) 628 return kIOReturnBadArgument; 629 630 if (!maxSegmentSize) 631 maxSegmentSize--; // Set Max segment to -1 632 if (!maxTransferSize) 633 maxTransferSize--; // Set Max transfer to -1 634 635 if (mapper && !OSDynamicCast(IOMapper, mapper)) 636 { 637 fInternalState->fDevice = mapper; 638 mapper = 0; 639 } 640 if (!mapper) 641 { 642 IOMapper::checkForSystemMapper(); 643 mapper = IOMapper::gSystem; 644 } 645 646 switch (MAPTYPE(mappingOptions)) 647 { 648 case kMapped: break; 649 case kNonCoherent: break; 650 case kBypassed: 651 if (mapper && !mapper->getBypassMask(&fBypassMask)) 652 return kIOReturnBadArgument; 653 break; 654 default: 655 return kIOReturnBadArgument; 656 }; 657 658 fNumSegments = 0; 659 fBypassMask = 0; 660 fOutSeg = outSegFunc; 661 fNumAddressBits = numAddressBits; 662 fMaxSegmentSize = maxSegmentSize; 663 fMappingOptions = mappingOptions; 664 fMaxTransferSize = maxTransferSize; 665 if (!alignment) 666 alignment = 1; 667 fAlignMask = alignment - 1; 668 if (mapper != fMapper) 669 { 670 mapper->retain(); 671 fMapper->release(); 672 fMapper = mapper; 673 } 674 675 fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions)); 676 677 return prepare(offset, length, flushCache, synchronize); 678} 679 680 681IOReturn 682IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchronize) 683{ 684 IODMACommandInternal * state = fInternalState; 685 IOReturn ret = kIOReturnSuccess; 686 MappingOptions mappingOptions = fMappingOptions; 687 688 if (!length) 689 length = fMDSummary.fLength; 690 691 if (length > fMaxTransferSize) 692 return kIOReturnNoSpace; 693 694 if (IS_NONCOHERENT(mappingOptions) && flushCache) { 695 IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory); 696 697 poMD->performOperation(kIOMemoryIncoherentIOStore, offset, length); 698 } 699 if (fActive++) 700 { 701 if ((state->fPreparedOffset != offset) 702 || (state->fPreparedLength != length)) 703 ret = kIOReturnNotReady; 704 } 705 else 706 { 707 state->fPreparedOffset = offset; 708 state->fPreparedLength = length; 709 710 state->fMapContig = false; 711 state->fMisaligned = false; 712 state->fDoubleBuffer = false; 713 state->fPrepared = false; 714 state->fCopyNext = NULL; 715 state->fCopyPageAlloc = 0; 716 state->fCopyPageCount = 0; 717 state->fNextRemapPage = NULL; 718 state->fCopyMD = 0; 719 state->fLocalMapperPageAlloc = 0; 720 state->fLocalMapperPageCount = 0; 721 722 state->fLocalMapper = (fMapper && (fMapper != IOMapper::gSystem)); 723 724 state->fSourceAlignMask = fAlignMask; 725 if (fMapper) 726 state->fSourceAlignMask &= page_mask; 727 728 state->fCursor = state->fIterateOnly 729 || (!state->fCheckAddressing 730 && (!state->fSourceAlignMask 731 || ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask))))); 732 733 if (!state->fCursor) 734 { 735 IOOptionBits op = kWalkPrepare | kWalkPreflight; 736 if (synchronize) 737 op |= kWalkSyncOut; 738 ret = walkAll(op); 739 } 740 741 if (fMapper) 742 { 743 if (state->fLocalMapper) 744 { 745 state->fLocalMapperPageCount = atop_64(round_page( 746 state->fPreparedLength + ((state->fPreparedOffset + fMDSummary.fPageAlign) & page_mask))); 747 state->fLocalMapperPageAlloc = ptoa_64(fMapper->iovmAllocDMACommand(this, state->fLocalMapperPageCount)); 748 if (!state->fLocalMapperPageAlloc) 749 { 750 DEBG("IODMACommand !iovmAlloc"); 751 return (kIOReturnNoResources); 752 } 753 state->fMapContig = true; 754 } 755 else 756 { 757 IOMDDMAMapArgs mapArgs; 758 bzero(&mapArgs, sizeof(mapArgs)); 759 mapArgs.fMapper = fMapper; 760 mapArgs.fMapSpec.device = state->fDevice; 761 mapArgs.fMapSpec.alignment = fAlignMask + 1; 762 mapArgs.fMapSpec.numAddressBits = fNumAddressBits ? fNumAddressBits : 64; 763 mapArgs.fOffset = state->fPreparedOffset; 764 mapArgs.fLength = state->fPreparedLength; 765 const IOMemoryDescriptor * md = state->fCopyMD; 766 if (!md) md = fMemory; 767 ret = md->dmaCommandOperation(kIOMDDMAMap | state->fIterateOnly, &mapArgs, sizeof(mapArgs)); 768 if (kIOReturnSuccess == ret) 769 { 770 state->fLocalMapperPageAlloc = mapArgs.fAlloc; 771 state->fLocalMapperPageCount = mapArgs.fAllocCount; 772 state->fMapContig = mapArgs.fMapContig; 773 } 774 ret = kIOReturnSuccess; 775 } 776 } 777 778 779 if (kIOReturnSuccess == ret) 780 state->fPrepared = true; 781 } 782 return ret; 783} 784 785IOReturn 786IODMACommand::complete(bool invalidateCache, bool synchronize) 787{ 788 IODMACommandInternal * state = fInternalState; 789 IOReturn ret = kIOReturnSuccess; 790 791 if (fActive < 1) 792 return kIOReturnNotReady; 793 794 if (!--fActive) 795 { 796 if (!state->fCursor) 797 { 798 IOOptionBits op = kWalkComplete; 799 if (synchronize) 800 op |= kWalkSyncIn; 801 ret = walkAll(op); 802 } 803 if (state->fLocalMapperPageAlloc) 804 { 805 if (state->fLocalMapper) 806 { 807 fMapper->iovmFreeDMACommand(this, atop_64(state->fLocalMapperPageAlloc), state->fLocalMapperPageCount); 808 } 809 else if (state->fLocalMapperPageCount) 810 { 811 fMapper->iovmFree(atop_64(state->fLocalMapperPageAlloc), state->fLocalMapperPageCount); 812 } 813 state->fLocalMapperPageAlloc = 0; 814 state->fLocalMapperPageCount = 0; 815 } 816 817 state->fPrepared = false; 818 819 if (IS_NONCOHERENT(fMappingOptions) && invalidateCache) 820 { 821 IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory); 822 823 poMD->performOperation(kIOMemoryIncoherentIOFlush, state->fPreparedOffset, state->fPreparedLength); 824 } 825 } 826 827 return ret; 828} 829 830IOReturn 831IODMACommand::getPreparedOffsetAndLength(UInt64 * offset, UInt64 * length) 832{ 833 IODMACommandInternal * state = fInternalState; 834 if (fActive < 1) 835 return (kIOReturnNotReady); 836 837 if (offset) 838 *offset = state->fPreparedOffset; 839 if (length) 840 *length = state->fPreparedLength; 841 842 return (kIOReturnSuccess); 843} 844 845IOReturn 846IODMACommand::synchronize(IOOptionBits options) 847{ 848 IODMACommandInternal * state = fInternalState; 849 IOReturn ret = kIOReturnSuccess; 850 IOOptionBits op; 851 852 if (kIODirectionOutIn == (kIODirectionOutIn & options)) 853 return kIOReturnBadArgument; 854 855 if (fActive < 1) 856 return kIOReturnNotReady; 857 858 op = 0; 859 if (kForceDoubleBuffer & options) 860 { 861 if (state->fDoubleBuffer) 862 return kIOReturnSuccess; 863 if (state->fCursor) 864 state->fCursor = false; 865 else 866 ret = walkAll(kWalkComplete); 867 868 op |= kWalkPrepare | kWalkPreflight | kWalkDoubleBuffer; 869 } 870 else if (state->fCursor) 871 return kIOReturnSuccess; 872 873 if (kIODirectionIn & options) 874 op |= kWalkSyncIn | kWalkSyncAlways; 875 else if (kIODirectionOut & options) 876 op |= kWalkSyncOut | kWalkSyncAlways; 877 878 ret = walkAll(op); 879 880 return ret; 881} 882 883struct IODMACommandTransferContext 884{ 885 void * buffer; 886 UInt64 bufferOffset; 887 UInt64 remaining; 888 UInt32 op; 889}; 890enum 891{ 892 kIODMACommandTransferOpReadBytes = 1, 893 kIODMACommandTransferOpWriteBytes = 2 894}; 895 896IOReturn 897IODMACommand::transferSegment(void *reference, 898 IODMACommand *target, 899 Segment64 segment, 900 void *segments, 901 UInt32 segmentIndex) 902{ 903 IODMACommandTransferContext * context = (IODMACommandTransferContext *) reference; 904 UInt64 length = min(segment.fLength, context->remaining); 905 addr64_t ioAddr = segment.fIOVMAddr; 906 addr64_t cpuAddr = ioAddr; 907 908 context->remaining -= length; 909 910 while (length) 911 { 912 UInt64 copyLen = length; 913 if ((kMapped == MAPTYPE(target->fMappingOptions)) 914 && target->fMapper) 915 { 916 cpuAddr = target->fMapper->mapAddr(ioAddr); 917 copyLen = min(copyLen, page_size - (ioAddr & (page_size - 1))); 918 ioAddr += copyLen; 919 } 920 921 switch (context->op) 922 { 923 case kIODMACommandTransferOpReadBytes: 924 copypv(cpuAddr, context->bufferOffset + (addr64_t) context->buffer, copyLen, 925 cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap); 926 break; 927 case kIODMACommandTransferOpWriteBytes: 928 copypv(context->bufferOffset + (addr64_t) context->buffer, cpuAddr, copyLen, 929 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap); 930 break; 931 } 932 length -= copyLen; 933 context->bufferOffset += copyLen; 934 } 935 936 return (context->remaining ? kIOReturnSuccess : kIOReturnOverrun); 937} 938 939UInt64 940IODMACommand::transfer(IOOptionBits transferOp, UInt64 offset, void * buffer, UInt64 length) 941{ 942 IODMACommandInternal * state = fInternalState; 943 IODMACommandTransferContext context; 944 Segment64 segments[1]; 945 UInt32 numSegments = 0-1; 946 947 if (fActive < 1) 948 return (0); 949 950 if (offset >= state->fPreparedLength) 951 return (0); 952 length = min(length, state->fPreparedLength - offset); 953 954 context.buffer = buffer; 955 context.bufferOffset = 0; 956 context.remaining = length; 957 context.op = transferOp; 958 (void) genIOVMSegments(kWalkClient, transferSegment, &context, &offset, &segments[0], &numSegments); 959 960 return (length - context.remaining); 961} 962 963UInt64 964IODMACommand::readBytes(UInt64 offset, void *bytes, UInt64 length) 965{ 966 return (transfer(kIODMACommandTransferOpReadBytes, offset, bytes, length)); 967} 968 969UInt64 970IODMACommand::writeBytes(UInt64 offset, const void *bytes, UInt64 length) 971{ 972 return (transfer(kIODMACommandTransferOpWriteBytes, offset, const_cast<void *>(bytes), length)); 973} 974 975IOReturn 976IODMACommand::genIOVMSegments(UInt64 *offsetP, 977 void *segmentsP, 978 UInt32 *numSegmentsP) 979{ 980 return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) fOutSeg, 981 offsetP, segmentsP, numSegmentsP)); 982} 983 984IOReturn 985IODMACommand::genIOVMSegments(uint32_t op, 986 InternalSegmentFunction outSegFunc, 987 void *reference, 988 UInt64 *offsetP, 989 void *segmentsP, 990 UInt32 *numSegmentsP) 991{ 992 IODMACommandInternal * internalState = fInternalState; 993 IOOptionBits mdOp = kIOMDWalkSegments; 994 IOReturn ret = kIOReturnSuccess; 995 996 if (!(kWalkComplete & op) && !fActive) 997 return kIOReturnNotReady; 998 999 if (!offsetP || !segmentsP || !numSegmentsP || !*numSegmentsP) 1000 return kIOReturnBadArgument; 1001 1002 IOMDDMAWalkSegmentArgs *state = 1003 (IOMDDMAWalkSegmentArgs *)(void *) fState; 1004 1005 UInt64 offset = *offsetP + internalState->fPreparedOffset; 1006 UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength; 1007 1008 if (offset >= memLength) 1009 return kIOReturnOverrun; 1010 1011 if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) { 1012 state->fOffset = 0; 1013 state->fIOVMAddr = 0; 1014 internalState->fNextRemapPage = NULL; 1015 internalState->fNewMD = false; 1016 state->fMapped = (IS_MAPPED(fMappingOptions) && fMapper); 1017 mdOp = kIOMDFirstSegment; 1018 }; 1019 1020 UInt64 bypassMask = fBypassMask; 1021 UInt32 segIndex = 0; 1022 UInt32 numSegments = *numSegmentsP; 1023 Segment64 curSeg = { 0, 0 }; 1024 addr64_t maxPhys; 1025 1026 if (fNumAddressBits && (fNumAddressBits < 64)) 1027 maxPhys = (1ULL << fNumAddressBits); 1028 else 1029 maxPhys = 0; 1030 maxPhys--; 1031 1032 while (state->fIOVMAddr || (state->fOffset < memLength)) 1033 { 1034 // state = next seg 1035 if (!state->fIOVMAddr) { 1036 1037 IOReturn rtn; 1038 1039 state->fOffset = offset; 1040 state->fLength = memLength - offset; 1041 1042 if (internalState->fMapContig && internalState->fLocalMapperPageAlloc) 1043 { 1044 state->fIOVMAddr = internalState->fLocalMapperPageAlloc + offset; 1045 rtn = kIOReturnSuccess; 1046#if 0 1047 { 1048 uint64_t checkOffset; 1049 IOPhysicalLength segLen; 1050 for (checkOffset = 0; checkOffset < state->fLength; ) 1051 { 1052 addr64_t phys = const_cast<IOMemoryDescriptor *>(fMemory)->getPhysicalSegment(checkOffset + offset, &segLen, kIOMemoryMapperNone); 1053 if (fMapper->mapAddr(state->fIOVMAddr + checkOffset) != phys) 1054 { 1055 panic("%llx != %llx:%llx, %llx phys: %llx %llx\n", offset, 1056 state->fIOVMAddr + checkOffset, fMapper->mapAddr(state->fIOVMAddr + checkOffset), state->fLength, 1057 phys, checkOffset); 1058 } 1059 checkOffset += page_size - (phys & page_mask); 1060 } 1061 } 1062#endif 1063 } 1064 else 1065 { 1066 const IOMemoryDescriptor * memory = 1067 internalState->fCopyMD ? internalState->fCopyMD : fMemory; 1068 rtn = memory->dmaCommandOperation(mdOp, fState, sizeof(fState)); 1069 mdOp = kIOMDWalkSegments; 1070 } 1071 1072 if (rtn == kIOReturnSuccess) 1073 { 1074 assert(state->fIOVMAddr); 1075 assert(state->fLength); 1076 if ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr) { 1077 UInt64 length = state->fLength; 1078 offset += length; 1079 curSeg.fLength += length; 1080 state->fIOVMAddr = 0; 1081 } 1082 } 1083 else if (rtn == kIOReturnOverrun) 1084 state->fIOVMAddr = state->fLength = 0; // At end 1085 else 1086 return rtn; 1087 } 1088 1089 // seg = state, offset = end of seg 1090 if (!curSeg.fIOVMAddr) 1091 { 1092 UInt64 length = state->fLength; 1093 offset += length; 1094 curSeg.fIOVMAddr = state->fIOVMAddr | bypassMask; 1095 curSeg.fLength = length; 1096 state->fIOVMAddr = 0; 1097 } 1098 1099 if (!state->fIOVMAddr) 1100 { 1101 if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys) 1102 { 1103 if (internalState->fCursor) 1104 { 1105 curSeg.fIOVMAddr = 0; 1106 ret = kIOReturnMessageTooLarge; 1107 break; 1108 } 1109 else if (curSeg.fIOVMAddr <= maxPhys) 1110 { 1111 UInt64 remain, newLength; 1112 1113 newLength = (maxPhys + 1 - curSeg.fIOVMAddr); 1114 DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength); 1115 remain = curSeg.fLength - newLength; 1116 state->fIOVMAddr = newLength + curSeg.fIOVMAddr; 1117 curSeg.fLength = newLength; 1118 state->fLength = remain; 1119 offset -= remain; 1120 } 1121 else 1122 { 1123 UInt64 addr = curSeg.fIOVMAddr; 1124 ppnum_t addrPage = atop_64(addr); 1125 vm_page_t remap = NULL; 1126 UInt64 remain, newLength; 1127 1128 DEBG("sparse switch %qx, %qx ", addr, curSeg.fLength); 1129 1130 remap = internalState->fNextRemapPage; 1131 if (remap && (addrPage == vm_page_get_offset(remap))) 1132 { 1133 } 1134 else for (remap = internalState->fCopyPageAlloc; 1135 remap && (addrPage != vm_page_get_offset(remap)); 1136 remap = vm_page_get_next(remap)) 1137 { 1138 } 1139 1140 if (!remap) panic("no remap page found"); 1141 1142 curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap)) 1143 + (addr & PAGE_MASK); 1144 internalState->fNextRemapPage = vm_page_get_next(remap); 1145 1146 newLength = PAGE_SIZE - (addr & PAGE_MASK); 1147 if (newLength < curSeg.fLength) 1148 { 1149 remain = curSeg.fLength - newLength; 1150 state->fIOVMAddr = addr + newLength; 1151 curSeg.fLength = newLength; 1152 state->fLength = remain; 1153 offset -= remain; 1154 } 1155 DEBG("-> %qx, %qx offset %qx\n", curSeg.fIOVMAddr, curSeg.fLength, offset); 1156 } 1157 } 1158 1159 if (curSeg.fLength > fMaxSegmentSize) 1160 { 1161 UInt64 remain = curSeg.fLength - fMaxSegmentSize; 1162 1163 state->fIOVMAddr = fMaxSegmentSize + curSeg.fIOVMAddr; 1164 curSeg.fLength = fMaxSegmentSize; 1165 1166 state->fLength = remain; 1167 offset -= remain; 1168 } 1169 1170 if (internalState->fCursor 1171 && (0 != (internalState->fSourceAlignMask & curSeg.fIOVMAddr))) 1172 { 1173 curSeg.fIOVMAddr = 0; 1174 ret = kIOReturnNotAligned; 1175 break; 1176 } 1177 1178 if (offset >= memLength) 1179 { 1180 curSeg.fLength -= (offset - memLength); 1181 offset = memLength; 1182 state->fIOVMAddr = state->fLength = 0; // At end 1183 break; 1184 } 1185 } 1186 1187 if (state->fIOVMAddr) { 1188 if ((segIndex + 1 == numSegments)) 1189 break; 1190 1191 ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++); 1192 curSeg.fIOVMAddr = 0; 1193 if (kIOReturnSuccess != ret) 1194 break; 1195 } 1196 } 1197 1198 if (curSeg.fIOVMAddr) { 1199 ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++); 1200 } 1201 1202 if (kIOReturnSuccess == ret) 1203 { 1204 state->fOffset = offset; 1205 *offsetP = offset - internalState->fPreparedOffset; 1206 *numSegmentsP = segIndex; 1207 } 1208 return ret; 1209} 1210 1211IOReturn 1212IODMACommand::clientOutputSegment( 1213 void *reference, IODMACommand *target, 1214 Segment64 segment, void *vSegList, UInt32 outSegIndex) 1215{ 1216 SegmentFunction segmentFunction = (SegmentFunction) reference; 1217 IOReturn ret = kIOReturnSuccess; 1218 1219 if (target->fNumAddressBits && (target->fNumAddressBits < 64) 1220 && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits) 1221 && (target->reserved->fLocalMapperPageAlloc || !target->fMapper)) 1222 { 1223 DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength); 1224 ret = kIOReturnMessageTooLarge; 1225 } 1226 1227 if (!(*segmentFunction)(target, segment, vSegList, outSegIndex)) 1228 { 1229 DEBG("kIOReturnMessageTooLarge(fOutSeg) %qx, %qx\n", segment.fIOVMAddr, segment.fLength); 1230 ret = kIOReturnMessageTooLarge; 1231 } 1232 1233 return (ret); 1234} 1235 1236IOReturn 1237IODMACommand::genIOVMSegments(SegmentFunction segmentFunction, 1238 UInt64 *offsetP, 1239 void *segmentsP, 1240 UInt32 *numSegmentsP) 1241{ 1242 return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) segmentFunction, 1243 offsetP, segmentsP, numSegmentsP)); 1244} 1245 1246bool 1247IODMACommand::OutputHost32(IODMACommand *, 1248 Segment64 segment, void *vSegList, UInt32 outSegIndex) 1249{ 1250 Segment32 *base = (Segment32 *) vSegList; 1251 base[outSegIndex].fIOVMAddr = (UInt32) segment.fIOVMAddr; 1252 base[outSegIndex].fLength = (UInt32) segment.fLength; 1253 return true; 1254} 1255 1256bool 1257IODMACommand::OutputBig32(IODMACommand *, 1258 Segment64 segment, void *vSegList, UInt32 outSegIndex) 1259{ 1260 const UInt offAddr = outSegIndex * sizeof(Segment32); 1261 const UInt offLen = offAddr + sizeof(UInt32); 1262 OSWriteBigInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr); 1263 OSWriteBigInt32(vSegList, offLen, (UInt32) segment.fLength); 1264 return true; 1265} 1266 1267bool 1268IODMACommand::OutputLittle32(IODMACommand *, 1269 Segment64 segment, void *vSegList, UInt32 outSegIndex) 1270{ 1271 const UInt offAddr = outSegIndex * sizeof(Segment32); 1272 const UInt offLen = offAddr + sizeof(UInt32); 1273 OSWriteLittleInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr); 1274 OSWriteLittleInt32(vSegList, offLen, (UInt32) segment.fLength); 1275 return true; 1276} 1277 1278bool 1279IODMACommand::OutputHost64(IODMACommand *, 1280 Segment64 segment, void *vSegList, UInt32 outSegIndex) 1281{ 1282 Segment64 *base = (Segment64 *) vSegList; 1283 base[outSegIndex] = segment; 1284 return true; 1285} 1286 1287bool 1288IODMACommand::OutputBig64(IODMACommand *, 1289 Segment64 segment, void *vSegList, UInt32 outSegIndex) 1290{ 1291 const UInt offAddr = outSegIndex * sizeof(Segment64); 1292 const UInt offLen = offAddr + sizeof(UInt64); 1293 OSWriteBigInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr); 1294 OSWriteBigInt64(vSegList, offLen, (UInt64) segment.fLength); 1295 return true; 1296} 1297 1298bool 1299IODMACommand::OutputLittle64(IODMACommand *, 1300 Segment64 segment, void *vSegList, UInt32 outSegIndex) 1301{ 1302 const UInt offAddr = outSegIndex * sizeof(Segment64); 1303 const UInt offLen = offAddr + sizeof(UInt64); 1304 OSWriteLittleInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr); 1305 OSWriteLittleInt64(vSegList, offLen, (UInt64) segment.fLength); 1306 return true; 1307} 1308 1309 1310