iokit/Kernel/IODMACommand.cpp

76 #define SHOULD_COPY_DIR(state, direction) (true)
329     IODMACommandInternal * state = target->reserved;
331     if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperPageAlloc || !target->fMapper))
343     if (!state->fMisaligned)
345 	state->fMisaligned |= (0 != (state->fSourceAlignMask & address));
346 	if (state->fMisaligned) DEBG("misaligned %qx:%qx, %lx\n", address, length, state->fSourceAlignMask);
349     if (state->fMisaligned && (kWalkPreflight & op))
352     if (!state->fDoubleBuffer)
374 	state->fCopyPageCount += numPages;
382 	    lastPage = state->fCopyNext;
392 	    lastPage = state->fCopyNext;
408 		    if (!state->fDoubleBuffer)
436 	state->fCopyNext = lastPage;
445     IODMACommandInternal * state = fInternalState;
453 	state->fMisaligned     = false;
454 	state->fDoubleBuffer   = false;
455 	state->fPrepared       = false;
456 	state->fCopyNext       = NULL;
457 	state->fCopyPageAlloc  = 0;
458 	state->fCopyPageCount  = 0;
459 	state->fNextRemapPage  = NULL;
460 	state->fCopyMD	       = 0;
466 	    ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
471 	state->fDoubleBuffer = (state->fMisaligned || (kWalkDoubleBuffer & op));
472 	if (state->fDoubleBuffer)
473 	    state->fCopyPageCount = atop_64(round_page(state->fPreparedLength));
475 	if (state->fCopyPageCount)
479 	    DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount);
481 	    if (!state->fDoubleBuffer)
487 		kr = vm_page_alloc_list(state->fCopyPageCount,
491 		    DEBG("vm_page_alloc_list(%d) failed (%d)\n", state->fCopyPageCount, kr);
498 		state->fCopyPageAlloc = mapBase;
499 		state->fCopyNext = state->fCopyPageAlloc;
502 		ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
503 		state->fPrepared = true;
509 		mach_vm_address_t mask = 0xFFFFF000; //state->fSourceAlignMask
510 		state->fCopyMD = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task,
511 				    fMDSummary.fDirection, state->fPreparedLength, mask);
513 		if (state->fCopyMD)
516 		    state->fPrepared = true;
527     if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op))
529 	if (state->fCopyPageCount)
531 	    DEBG("sync fCopyPageCount %d\n", state->fCopyPageCount);
533 	    if (state->fCopyPageAlloc)
535 		state->fCopyNext = state->fCopyPageAlloc;
538 		ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
540 	    else if (state->fCopyMD)
551 			bytes = poMD->writeBytes(state->fPreparedOffset,
552 						    state->fCopyMD->getBytesNoCopy(),
553 						    state->fPreparedLength);
555 			bytes = poMD->readBytes(state->fPreparedOffset,
556 						    state->fCopyMD->getBytesNoCopy(),
557 						    state->fPreparedLength);
559 		    ret = (bytes == state->fPreparedLength) ? kIOReturnSuccess : kIOReturnUnderrun;
569 	if (state->fCopyPageAlloc)
571 	    vm_page_free_list(state->fCopyPageAlloc, FALSE);
572 	    state->fCopyPageAlloc = 0;
573 	    state->fCopyPageCount = 0;
575 	if (state->fCopyMD)
577 	    state->fCopyMD->release();
578 	    state->fCopyMD = 0;
581 	state->fPrepared = false;
684     IODMACommandInternal * state = fInternalState;
701 	if ((state->fPreparedOffset != offset)
702 	  || (state->fPreparedLength != length))
707 	state->fPreparedOffset = offset;
708 	state->fPreparedLength = length;
710 	state->fMapContig      = false;
711 	state->fMisaligned     = false;
712 	state->fDoubleBuffer   = false;
713 	state->fPrepared       = false;
714 	state->fCopyNext       = NULL;
715 	state->fCopyPageAlloc  = 0;
716 	state->fCopyPageCount  = 0;
717 	state->fNextRemapPage  = NULL;
718 	state->fCopyMD         = 0;
719 	state->fLocalMapperPageAlloc = 0;
720 	state->fLocalMapperPageCount = 0;
722 	state->fLocalMapper    = (fMapper && (fMapper != IOMapper::gSystem));
724 	state->fSourceAlignMask = fAlignMask;
726 	    state->fSourceAlignMask &= page_mask;
728 	state->fCursor = state->fIterateOnly
729 			|| (!state->fCheckAddressing
730 			    && (!state->fSourceAlignMask
731 				|| ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask)))));
733 	if (!state->fCursor)
743 	    if (state->fLocalMapper)
745 		state->fLocalMapperPageCount = atop_64(round_page(
746 			state->fPreparedLength + ((state->fPreparedOffset + fMDSummary.fPageAlign) & page_mask)));
747 		state->fLocalMapperPageAlloc = ptoa_64(fMapper->iovmAllocDMACommand(this, state->fLocalMapperPageCount));
748 		if (!state->fLocalMapperPageAlloc)
753 		state->fMapContig = true;
760 		mapArgs.fMapSpec.device         = state->fDevice;
763 		mapArgs.fOffset = state->fPreparedOffset;
764 		mapArgs.fLength = state->fPreparedLength;
765 		const IOMemoryDescriptor * md = state->fCopyMD;
767 		ret = md->dmaCommandOperation(kIOMDDMAMap | state->fIterateOnly, &mapArgs, sizeof(mapArgs));
770 		    state->fLocalMapperPageAlloc = mapArgs.fAlloc;
771 		    state->fLocalMapperPageCount = mapArgs.fAllocCount;
772 		    state->fMapContig = true;
780 	    state->fPrepared = true;
788     IODMACommandInternal * state = fInternalState;
796 	if (!state->fCursor)
803     	if (state->fLocalMapperPageAlloc)
805 	    if (state->fLocalMapper)
807 		fMapper->iovmFreeDMACommand(this, atop_64(state->fLocalMapperPageAlloc), state->fLocalMapperPageCount);
809 	    else if (state->fLocalMapperPageCount)
811 		fMapper->iovmFree(atop_64(state->fLocalMapperPageAlloc), state->fLocalMapperPageCount);
813 	    state->fLocalMapperPageAlloc = 0;
814 	    state->fLocalMapperPageCount = 0;
817 	state->fPrepared = false;
823 	    poMD->performOperation(kIOMemoryIncoherentIOFlush, state->fPreparedOffset, state->fPreparedLength);
833     IODMACommandInternal * state = fInternalState;
838 	*offset = state->fPreparedOffset;
840 	*length = state->fPreparedLength;
848     IODMACommandInternal * state = fInternalState;
861 	if (state->fDoubleBuffer)
863 	if (state->fCursor)
864 	    state->fCursor = false;
870     else if (state->fCursor)
942     IODMACommandInternal *      state = fInternalState;
950     if (offset >= state->fPreparedLength)
952     length = min(length, state->fPreparedLength - offset);
1002     IOMDDMAWalkSegmentArgs *state =
1011     if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) {
1012 	state->fOffset                 = 0;
1013 	state->fIOVMAddr               = 0;
1016 	state->fMapped                 = (IS_MAPPED(fMappingOptions) && fMapper);
1032     while (state->fIOVMAddr || (state->fOffset < memLength))
1034 	// state = next seg
1035 	if (!state->fIOVMAddr) {
1039 	    state->fOffset = offset;
1040 	    state->fLength = memLength - offset;
1044 		state->fIOVMAddr = internalState->fLocalMapperPageAlloc + offset;
1050 		    for (checkOffset = 0; checkOffset < state->fLength; )
1053 			if (fMapper->mapAddr(state->fIOVMAddr + checkOffset) != phys)
1056 				    state->fIOVMAddr + checkOffset, fMapper->mapAddr(state->fIOVMAddr + checkOffset), state->fLength,
1074 		assert(state->fIOVMAddr);
1075 		assert(state->fLength);
1076 		if ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr) {
1077 		    UInt64 length = state->fLength;
1080 		    state->fIOVMAddr = 0;
1084 		state->fIOVMAddr = state->fLength = 0;	// At end
1089 	// seg = state, offset = end of seg
1092 	    UInt64 length = state->fLength;
1094 	    curSeg.fIOVMAddr = state->fIOVMAddr | bypassMask;
1096 	    state->fIOVMAddr = 0;
1099         if (!state->fIOVMAddr)
1116 		    state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
1118 		    state->fLength   = remain;
1150 			state->fIOVMAddr = addr + newLength;
1152 			state->fLength	 = remain;
1163 		state->fIOVMAddr = fMaxSegmentSize + curSeg.fIOVMAddr;
1166 		state->fLength   = remain;
1182 		state->fIOVMAddr = state->fLength = 0;	// At end
1187         if (state->fIOVMAddr) {
1204 	state->fOffset = offset;