1/*
2* Copyright (c) 1998-2001 Apple Computer, Inc. All rights reserved.
3*
4* @APPLE_LICENSE_HEADER_START@
5*
6* The contents of this file constitute Original Code as defined in and
7* are subject to the Apple Public Source License Version 1.1 (the
8* "License").  You may not use this file except in compliance with the
9* License.  Please obtain a copy of the License at
10* http://www.apple.com/publicsource and read it before using this file.
11*
12* This Original Code and all software distributed under the License are
13* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
17* License for the specific language governing rights and limitations
18* under the License.
19*
20* @APPLE_LICENSE_HEADER_END@
21*/
22/*
23*  IOFWUserLocalIsochPort.cpp
24*  IOFireWireFamily
25*
26*  Created by NWG on Tue Mar 20 2001.
27*  Copyright (c) 2001 Apple Computer, Inc. All rights reserved.
28*
29*/
30
31// public
32#import <IOKit/firewire/IOFireWireDevice.h>
33#import <IOKit/firewire/IOFireWireController.h>
34#import <IOKit/firewire/IOFWLocalIsochPort.h>
35#import <IOKit/firewire/IOFWDCLTranslator.h>
36#import <IOKit/firewire/IOFWDCLPool.h>
37#import <IOKit/firewire/IOFWDCL.h>
38#import <IOKit/IOBufferMemoryDescriptor.h>
39#include <IOKit/IOKitKeysPrivate.h>
40#include <IOKit/IODMACommand.h>
41
42// protected
43#import <IOKit/firewire/IOFireWireLink.h>
44
45// private
46#import "IOFireWireUserClient.h"
47#import "IOFWUserIsochPort.h"
48
49#if 0
50// DEBUG
51class DebugThing
52{
53	public :
54		io_user_reference_t * asyncRef ;
55		IOFWUserLocalIsochPort * port;
56} ;
57#endif
58
59// ============================================================
60// utility functions
61// ============================================================
62
63// static in IOFWUtils.cpp, shouldn't be included in IOFWUtils.h
64extern bool findOffsetInRanges ( mach_vm_address_t address, unsigned rangeCount, IOAddressRange ranges[], IOByteCount & outOffset ) ;
65
66static bool
67getDCLDataBuffer(
68	const UserExportDCLCommand *dcl,
69	mach_vm_address_t &			outDataBuffer,
70	mach_vm_size_t &			outDataLength )
71{
72	Boolean	result = false ;
73
74	switch ( dcl->opcode & ~kFWDCLOpFlagMask )
75	{
76		case kDCLSendPacketStartOp:
77		//case kDCLSendPacketWithHeaderStartOp:
78		case kDCLSendPacketOp:
79		case kDCLReceivePacketStartOp:
80		case kDCLReceivePacketOp:
81			outDataBuffer		= ( (UserExportDCLTransferPacket*)dcl )->buffer ;
82			outDataLength		= ( (UserExportDCLTransferPacket *)dcl )->size ;
83			result = true ;
84			break ;
85
86		case kDCLPtrTimeStampOp:
87			outDataBuffer		= ( (UserExportDCLPtrTimeStamp *) dcl )->timeStampPtr ;
88			outDataLength		= sizeof ( mach_vm_address_t ) ;
89			result = true ;
90			break ;
91
92		default:
93			break ;
94	}
95
96	return result ;
97}
98
99static void
100setDCLDataBuffer (
101		DCLCommand*					inDCL,
102		mach_vm_address_t			inDataBuffer,
103		mach_vm_size_t				inDataLength )
104{
105	switch(inDCL->opcode & ~kFWDCLOpFlagMask)
106	{
107		case kDCLSendPacketStartOp:
108		//case kDCLSendPacketWithHeaderStartOp:
109		case kDCLSendPacketOp:
110		case kDCLReceivePacketStartOp:
111		case kDCLReceivePacketOp:
112			((DCLTransferPacket*)inDCL)->buffer		= (void*) inDataBuffer ;
113			((DCLTransferPacket*)inDCL)->size 		= inDataLength ;
114			break ;
115
116		case kDCLSendBufferOp:
117		case kDCLReceiveBufferOp:
118			//zzz what should I do here?
119			break ;
120
121		case kDCLPtrTimeStampOp:
122			((DCLPtrTimeStamp*)inDCL)->timeStampPtr = (UInt32*)inDataBuffer ;
123			break ;
124
125		default:
126			break ;
127	}
128
129}
130
131static IOByteCount
132getDCLSize ( UserExportDCLCommand* dcl )
133{
134	IOByteCount result = 0 ;
135
136	switch(dcl->opcode & ~kFWDCLOpFlagMask)
137	{
138		case kDCLSendPacketStartOp:
139		//case kDCLSendPacketWithHeaderStartOp:
140		case kDCLSendPacketOp:
141		case kDCLReceivePacketStartOp:
142		case kDCLReceivePacketOp:
143			result = sizeof(UserExportDCLTransferPacket) ;
144			break ;
145
146		case kDCLSendBufferOp:
147		case kDCLReceiveBufferOp:
148			result = sizeof(UserExportDCLTransferBuffer) ;
149			break ;
150
151		case kDCLCallProcOp:
152			result = sizeof(UserExportDCLCallProc) ;
153			break ;
154
155		case kDCLLabelOp:
156			result = sizeof(UserExportDCLLabel) ;
157			break ;
158
159		case kDCLJumpOp:
160			result = sizeof(UserExportDCLJump) ;
161			break ;
162
163		case kDCLSetTagSyncBitsOp:
164			result = sizeof(UserExportDCLSetTagSyncBits) ;
165			break ;
166
167		case kDCLUpdateDCLListOp:
168			result = sizeof(UserExportDCLUpdateDCLList) ;
169			break ;
170
171		case kDCLPtrTimeStampOp:
172			result = sizeof(UserExportDCLPtrTimeStamp) ;
173			break;
174
175		case kDCLSkipCycleOp:
176			result = sizeof(UserExportDCLCommand) ;
177			break;
178	}
179
180	return result ;
181}
182
183
184#pragma mark -
185
186#undef super
187#define super IOFWLocalIsochPort
188
189OSDefineMetaClassAndStructors ( IOFWUserLocalIsochPort, super )
190#if 0
191{}
192#endif
193
194#if IOFIREWIREDEBUG > 0
195bool
196IOFWUserLocalIsochPort::serialize( OSSerialize * s ) const
197{
198	const OSString * keys[ 1 ] =
199	{
200		OSString::withCString( "program" )
201	} ;
202
203	const OSObject * objects[ 1 ] =
204	{
205		fProgram ? (const OSObject*)fProgram : (const OSObject*)OSString::withCString( "(null)" )
206	} ;
207
208	OSDictionary * dict = OSDictionary::withObjects( objects, keys, sizeof( keys )/sizeof( OSObject* ) ) ;
209
210	if ( !dict )
211		return false ;
212
213	bool result = dict->serialize( s ) ;
214	dict->release() ;
215
216	return result ;
217}
218#endif // IOFIREWIREDEBUG > 0
219
220void
221IOFWUserLocalIsochPort::free()
222{
223	// release DCL pool (if we have one)
224	if ( fDCLPool )
225	{
226		fDCLPool->release() ;
227		fDCLPool = NULL ;
228	}
229
230	// release fProgramBuffer (if we have one)
231	delete [] (UInt8*)fProgramBuffer ;
232	fProgramBuffer = NULL ;
233
234	if ( fLock )
235	{
236		IORecursiveLockFree( fLock ) ;
237	}
238
239	delete[] fDCLTable ;
240	fDCLTable = NULL ;
241
242	super::free() ;
243}
244
245#if 0
246
247IOReturn checkMemoryInRange( IOMemoryDescriptor * memory, UInt64 mask )
248{
249	IOReturn status = kIOReturnSuccess;
250
251	if( memory == NULL )
252	{
253		status = kIOReturnBadArgument;
254	}
255
256	//
257	// setup
258	//
259
260	bool memory_prepared = false;
261	if( status == kIOReturnSuccess )
262	{
263		status = memory->prepare( kIODirectionInOut );
264	}
265
266	if( status == kIOReturnSuccess )
267	{
268		memory_prepared = true;
269	}
270
271	UInt64 length = 0;
272	IODMACommand * dma_command = NULL;
273	if( status == kIOReturnSuccess )
274	{
275		length = memory->getLength();
276		dma_command = IODMACommand::withSpecification(
277												kIODMACommandOutputHost64,		// segment function
278												64,								// max address bits
279												length,							// max segment size
280												IODMACommand::kMapped | IODMACommand::kIterateOnly,		// IO mapped & don't bounce buffer
281												length,							// max transfer size
282												0,								// page alignment
283												NULL,							// mapper
284												NULL );							// refcon
285		if( dma_command == NULL )
286			status = kIOReturnError;
287
288	}
289
290	if( status == kIOReturnSuccess )
291	{
292		// set memory descriptor and don't prepare it
293		status = dma_command->setMemoryDescriptor( memory, false );
294	}
295
296	bool dma_command_prepared = false;
297	if( status == kIOReturnSuccess )
298	{
299		status = dma_command->prepare( 0, length, true );
300	}
301
302	if( status == kIOReturnSuccess )
303	{
304		dma_command_prepared = true;
305	}
306
307	//
308	// check ranges
309	//
310
311	if( status == kIOReturnSuccess )
312	{
313		UInt64 offset = 0;
314		while( (offset < length) && (status == kIOReturnSuccess) )
315		{
316			IODMACommand::Segment64 segments[10];
317			UInt32 num_segments = 10;
318			status = dma_command->gen64IOVMSegments( &offset, segments, &num_segments );
319			if( status == kIOReturnSuccess )
320			{
321				for( UInt32 i = 0; i < num_segments; i++ )
322				{
323				//	IOLog( "checkSegments - segments[%d].fIOVMAddr = 0x%016llx, fLength = %d\n", i, segments[i].fIOVMAddr, segments[i].fLength  );
324
325					if( (segments[i].fIOVMAddr & (~mask)) )
326					{
327						IOLog( "checkSegmentsFailed - 0x%016llx & 0x%016llx\n", segments[i].fIOVMAddr, mask );
328						status = kIOReturnNotPermitted;
329						break;
330					}
331				}
332			}
333		}
334	}
335
336	//
337	// clean up
338	//
339
340	if( dma_command_prepared )
341	{
342		dma_command->complete();
343		dma_command_prepared = false;
344	}
345
346	if( dma_command )
347	{
348		dma_command->clearMemoryDescriptor();
349		dma_command->release();
350		dma_command = NULL;
351	}
352
353	if( memory_prepared )
354	{
355		memory->complete();
356		memory_prepared = false;
357	}
358
359	return status;
360}
361
362#endif
363
364bool
365IOFWUserLocalIsochPort::initWithUserDCLProgram (
366		AllocateParams * 			params,
367		IOFireWireUserClient & 		userclient,
368		IOFireWireController &		controller )
369{
370	// sanity checking
371	if ( params->programExportBytes == 0 )
372	{
373		ErrorLog ( "No program!" ) ;
374		return false ;
375	}
376
377	fLock = IORecursiveLockAlloc () ;
378	if ( ! fLock )
379	{
380		ErrorLog ( "Couldn't allocate recursive lock\n" ) ;
381		return false ;
382	}
383
384// init easy params
385
386	fUserObj = params->userObj ;
387	fUserClient = & userclient ;
388	fDCLPool = NULL ;
389	fProgramCount = 0;
390	fStarted = false ;
391
392	IOReturn error = kIOReturnSuccess ;
393
394// get user program ranges:
395
396	IOAddressRange * bufferRanges = new IOAddressRange[ params->bufferRangeCount ] ;
397	if ( !bufferRanges )
398	{
399		error = kIOReturnNoMemory ;
400	}
401
402	if ( !error )
403	{
404		error = fUserClient->copyUserData(params->bufferRanges,(mach_vm_address_t)bufferRanges, sizeof ( IOAddressRange ) * params->bufferRangeCount ) ;
405	}
406
407// create descriptor for program buffers
408
409	IOMemoryDescriptor * bufferDesc = NULL ;
410	if ( ! error )
411	{
412		IOByteCount length = 0 ;
413		for ( unsigned index = 0; index < params->bufferRangeCount; ++index )
414		{
415			length += bufferRanges[ index ].length ;
416		}
417
418		bufferDesc = IOMemoryDescriptor::withAddressRanges (	bufferRanges, params->bufferRangeCount, kIODirectionOutIn,
419															fUserClient->getOwningTask() ) ;
420		if ( ! bufferDesc )
421		{
422			error = kIOReturnNoMemory ;
423		}
424		else
425		{
426
427			// IOLog( "IOFWUserLocalIsochPort::initWithUserDCLProgram - checkMemoryInRange status 0x%08lx\n", checkMemoryInRange( bufferDesc, 0x000000001FFFFFFF ) );
428
429			error = bufferDesc->prepare( kIODirectionPrepareToPhys32 ) ;
430
431			FWTrace( kFWTIsoch, kTPIsochPortUserInitWithUserDCLProgram, (uintptr_t)(fUserClient->getOwner()->getController()->getLink()), error, length, 0 );
432
433			// IOLog( "IOFWUserLocalIsochPort::initWithUserDCLProgram - prep 32 checkMemoryInRange status 0x%08lx\n", checkMemoryInRange( bufferDesc, 0x000000001FFFFFFF ) );
434
435		}
436	}
437
438// create map for buffers; we will need to get a virtual address for them
439
440	IOMemoryMap * bufferMap = NULL ;
441	if ( !error )
442	{
443		bufferMap = bufferDesc->map() ;
444		if ( !bufferMap )
445		{
446			DebugLog( "Couldn't map program buffers\n" ) ;
447			error = kIOReturnVMError ;
448		}
449
450		bufferDesc->release() ;
451	}
452
453	IOMemoryDescriptor * userProgramExportDesc = NULL ;
454	if ( !error )
455	{
456		userProgramExportDesc = IOMemoryDescriptor::withAddressRange(
457																	 params->programData,
458																	 params->programExportBytes,
459																	 kIODirectionOut,
460																	 fUserClient->getOwningTask() ) ;
461
462	}
463
464	// get map of program export data
465	if ( userProgramExportDesc )
466	{
467		error = userProgramExportDesc->prepare() ;
468	}
469
470	if ( !error )
471	{
472		DCLCommand * opcodes = NULL ;
473		switch ( params->version )
474		{
475			case kDCLExportDataLegacyVersion :
476
477				error = importUserProgram( userProgramExportDesc, params->bufferRangeCount, bufferRanges, bufferMap ) ;
478				ErrorLogCond( error, "importUserProgram returned %x\n", error ) ;
479
480				if ( ! error )
481				{
482					opcodes = (DCLCommand*)fProgramBuffer ;
483				}
484
485				break ;
486
487			case kDCLExportDataNuDCLRosettaVersion :
488
489				fDCLPool = fUserClient->getOwner()->getBus()->createDCLPool() ;
490
491				if ( ! fDCLPool )
492				{
493					error = kIOReturnNoMemory ;
494				}
495
496				if ( !error )
497				{
498					error = fDCLPool->importUserProgram( userProgramExportDesc, params->bufferRangeCount, bufferRanges, bufferMap ) ;
499				}
500
501				fProgramBuffer = new UInt8[ sizeof( DCLNuDCLLeader ) ] ;
502				{
503					DCLNuDCLLeader * leader = (DCLNuDCLLeader*)fProgramBuffer ;
504					{
505						leader->pNextDCLCommand = NULL ;	// unused - always NULL
506						leader->opcode = kDCLNuDCLLeaderOp ;
507						leader->program = fDCLPool ;
508					}
509
510					opcodes = (DCLCommand*)leader ;
511				}
512
513				break ;
514
515			default :
516
517				ErrorLog ( "unsupported DCL program type\n" ) ;
518				error = kIOReturnBadArgument ;
519
520				break ;
521		}
522
523		ErrorLogCond( !opcodes, "Couldn't get opcodes\n" ) ;
524
525		IODCLProgram * program = NULL ;
526
527		if ( opcodes )
528		{
529//			IOFWLocalIsochPort::printDCLProgram( opcodes ) ;
530
531			IOFireWireBus::DCLTaskInfoAux	infoAux ;
532			{
533				infoAux.version = 2 ;
534
535				infoAux.u.v2.bufferMemoryMap = bufferMap ;
536				infoAux.u.v2.workloop = params->options & kFWIsochPortUseSeparateKernelThread ? createRealtimeThread() : NULL ;
537				infoAux.u.v2.options = (IOFWIsochPortOptions)params->options ;
538			}
539
540			IOFireWireBus::DCLTaskInfo info = { 0, 0, 0, 0, 0, 0, & infoAux } ;
541
542			program = fUserClient->getOwner()->getController()->getLink()->createDCLProgram(	params->talking,
543																								opcodes,
544																								& info,
545																								params->startEvent,
546																								params->startState,
547																								params->startMask ) ;
548
549			bufferMap->release() ;		// retained by DCL program
550			bufferMap = NULL ;
551
552			if (  infoAux.u.v2.workloop )
553			{
554				// If we created a custom workloop, it will be retained by the program...
555				// We can release our reference...
556				infoAux.u.v2.workloop->release() ;
557			}
558
559			DebugLogCond( !program, "createDCLProgram returned nil\n" ) ;
560		}
561
562		if ( program )
563		{
564			if ( ! super::init( program, & controller ) )
565			{
566				ErrorLog ( "IOFWUserIsochPort::init failed\n" ) ;
567				error = kIOReturnError ;
568			}
569		}
570		else
571		{
572			DebugLog ( "Couldn't create DCL program\n" ) ;
573			error = kIOReturnNoMemory ;
574		}
575
576		userProgramExportDesc->complete() ;
577		userProgramExportDesc->release() ;
578		userProgramExportDesc = NULL ;
579	}
580
581	delete [] bufferRanges ;
582
583	InfoLog( "-IOFWUserLocalIsochPort::initWithUserDCLProgram error=%x (build date "__TIME__" "__DATE__")\n", error ) ;
584
585	return ( ! error ) ;
586}
587
588
589IOReturn
590IOFWUserLocalIsochPort::importUserProgram (
591		IOMemoryDescriptor *		userExportDesc,
592		unsigned 					userBufferRangeCount,
593		IOAddressRange				userBufferRanges[],
594		IOMemoryMap *				bufferMap )
595{
596	IOReturn error = kIOReturnSuccess ;
597	UInt8 *pUserImportProgramBuffer;
598
599	// Allocate a temporary buffer to hold user-space exported program.
600	if ( ! error )
601	{
602		pUserImportProgramBuffer = new UInt8[ userExportDesc->getLength() ] ;
603		if ( !pUserImportProgramBuffer )
604		{
605			error = kIOReturnNoMemory ;
606		}
607	}
608
609	// copy user program to kernel buffer:
610	if ( !error )
611	{
612		unsigned byteCount = userExportDesc->readBytes( 0, (void*)pUserImportProgramBuffer, userExportDesc->getLength() ) ;
613		if ( byteCount < userExportDesc->getLength() )
614		{
615			error = kIOReturnVMError ;
616		}
617	}
618
619	// Allocate the buffer for the "real" kernel DCL program.
620	if ( ! error )
621	{
622		fProgramBuffer = new UInt8[ userExportDesc->getLength() ] ;
623		if ( !fProgramBuffer )
624		{
625			error = kIOReturnNoMemory ;
626		}
627	}
628
629	DCLCommand *pCurrentDCL;
630	UserExportDCLCommand *pExportDCL;
631	UInt32 nextUserExportDCLOffset = 0;
632	DCLCommand *pLastDCL = NULL;
633	unsigned size;
634	do
635	{
636		pExportDCL = (UserExportDCLCommand*)(pUserImportProgramBuffer + nextUserExportDCLOffset);
637		pCurrentDCL = (DCLCommand*)(fProgramBuffer + nextUserExportDCLOffset);
638
639		UInt32 opcode = pExportDCL->opcode & ~kFWDCLOpFlagMask;
640
641		// Sanity check
642		if ( opcode > 15 && opcode != 20 )
643		{
644			DebugLog("found invalid DCL in export data\n") ;
645			error = kIOFireWireBogusDCLProgram ;
646			break ;
647		}
648
649		size = getDCLSize( pExportDCL ) ;
650
651		// Set the "next" pointer in the previous DCL
652		if (pLastDCL != NULL)
653			pLastDCL->pNextDCLCommand = pCurrentDCL;
654		pLastDCL = pCurrentDCL;
655
656		switch ( opcode )
657		{
658
659			case kDCLSendPacketStartOp:
660			//case kDCLSendPacketWithHeaderStartOp:
661			case kDCLSendPacketOp:
662			case kDCLReceivePacketStartOp:
663			case kDCLReceivePacketOp:
664				{
665					DCLTransferPacket *pDCLTransferPacket = (DCLTransferPacket*) pCurrentDCL;
666					pDCLTransferPacket->opcode = pExportDCL->opcode;
667					pDCLTransferPacket->compilerData = 0;
668					//pDCLTransferPacket->buffer - handled by calls to getDCLDataBuffer/setDCLDataBuffer, below!
669					//pDCLTransferPacket->size - handled by calls to getDCLDataBuffer/setDCLDataBuffer, below!
670				}
671				break ;
672
673			case kDCLSendBufferOp:
674			case kDCLReceiveBufferOp:
675				{
676					DCLTransferBuffer *pDCLTransferBuffer = (DCLTransferBuffer*) pCurrentDCL;
677					pDCLTransferBuffer->opcode = pExportDCL->opcode;
678					pDCLTransferBuffer->compilerData = 0;
679					//pDCLTransferBuffer->buffer - handled by calls to getDCLDataBuffer/setDCLDataBuffer, below!
680					//pDCLTransferBuffer->size - handled by calls to getDCLDataBuffer/setDCLDataBuffer, below!
681					pDCLTransferBuffer->packetSize = ((UserExportDCLTransferBuffer*)pExportDCL)->packetSize;
682					pDCLTransferBuffer->reserved = ((UserExportDCLTransferBuffer*)pExportDCL)->reserved;
683					pDCLTransferBuffer->bufferOffset = ((UserExportDCLTransferBuffer*)pExportDCL)->bufferOffset;
684				}
685				break ;
686
687			case kDCLCallProcOp:
688				{
689					DCLCallProc *pDCLCallProc = (DCLCallProc*) pCurrentDCL;
690					pDCLCallProc->opcode = pExportDCL->opcode;
691					pDCLCallProc->compilerData = 0;
692					//pDCLCallProc->proc - handled by call to convertToKernelDCL, below
693					//pDCLCallProc->procData - handled by call to convertToKernelDCL, below
694					size += sizeof( uint64_t[kOSAsyncRef64Count] ) ;
695					error = convertToKernelDCL( ((UserExportDCLCallProc*)pExportDCL), pDCLCallProc ) ;
696				}
697				break ;
698
699			case kDCLLabelOp:
700				{
701					DCLLabel *pDCLLabel = (DCLLabel*) pCurrentDCL;
702					pDCLLabel->opcode = pExportDCL->opcode;
703					pDCLLabel->compilerData = 0;
704				}
705				break ;
706
707			case kDCLJumpOp:
708				{
709					DCLJump *pDCLJump = (DCLJump*) pCurrentDCL;
710					pDCLJump->opcode = pExportDCL->opcode;
711					pDCLJump->compilerData = 0;
712					//pDCLJump->pJumpDCLLabel - handled by call to convertToKernelDCL, below
713					error = convertToKernelDCL( ((UserExportDCLJump*)pExportDCL), pDCLJump ) ;
714				}
715				break ;
716
717			case kDCLSetTagSyncBitsOp:
718				{
719					DCLSetTagSyncBits *pDCLSetTagSyncBits = (DCLSetTagSyncBits*) pCurrentDCL;
720					pDCLSetTagSyncBits->opcode = pExportDCL->opcode;
721					pDCLSetTagSyncBits->compilerData = 0;
722					pDCLSetTagSyncBits->tagBits = ((UserExportDCLSetTagSyncBits*)pExportDCL)->tagBits;
723					pDCLSetTagSyncBits->syncBits = ((UserExportDCLSetTagSyncBits*)pExportDCL)->syncBits;
724				}
725				break ;
726
727			case kDCLUpdateDCLListOp:
728				{
729					DCLUpdateDCLList *pDCLUpdateDCLList = (DCLUpdateDCLList*) pCurrentDCL;
730					pDCLUpdateDCLList->opcode = pExportDCL->opcode;
731					pDCLUpdateDCLList->compilerData = 0;
732					//pDCLUpdateDCLList->dclCommandList - handled by call to convertToKernelDCL, below
733					pDCLUpdateDCLList->numDCLCommands = ((UserExportDCLUpdateDCLList*)pExportDCL)->numDCLCommands;
734					size += sizeof( mach_vm_address_t ) * ((UserExportDCLUpdateDCLList*)pExportDCL)->numDCLCommands ;
735					error = convertToKernelDCL( ((UserExportDCLUpdateDCLList*)pExportDCL), pDCLUpdateDCLList ) ;
736				}
737				break ;
738
739			case kDCLPtrTimeStampOp:
740				{
741					DCLPtrTimeStamp *pDCLPtrTimeStamp = (DCLPtrTimeStamp*) pCurrentDCL;
742					pDCLPtrTimeStamp->opcode = pExportDCL->opcode;
743					pDCLPtrTimeStamp->compilerData = 0;
744					//pDCLPtrTimeStamp->timeStampPtr - handled by calls to getDCLDataBuffer/setDCLDataBuffer, below!
745				}
746				break ;
747
748			case kDCLSkipCycleOp:
749				{
750					DCLCommand *pDCLCommand = (DCLCommand*) pCurrentDCL;
751					pDCLCommand->opcode = pExportDCL->opcode;
752					pDCLCommand->compilerData = 0;
753					pDCLCommand->operands[0] = ((UserExportDCLCommand*)pExportDCL)->operands[0];
754				}
755				break ;
756		}
757
758		// Break out of the loop if we got an error!
759		if (error)
760			break;
761
762		// Convert the DCL data pointers from user space to kernel space:
763		// (new style programs will have this step performed automatically when the program
764		// is imported to the kernel...)
765		IOAddressRange tempRange ;
766		tempRange.address = 0;		// supress warning
767		tempRange.length = 0;		// supress warning
768		if ( getDCLDataBuffer ( pExportDCL, tempRange.address, tempRange.length ) )
769		{
770			if ( tempRange.address != NULL && tempRange.length > 0 )
771			{
772				IOByteCount offset ;
773				if ( ! findOffsetInRanges (	tempRange.address, userBufferRangeCount, userBufferRanges, offset ) )
774				{
775					DebugLog( "IOFWUserLocalIsochPort::initWithUserDCLProgram: couldn't find DCL data buffer in buffer ranges") ;
776					error = kIOReturnError;
777					break;
778				}
779
780				// set DCL's data pointer to point to same memory in kernel address space
781				setDCLDataBuffer ( pCurrentDCL, bufferMap->getVirtualAddress() + offset, tempRange.length ) ;
782			}
783		}
784
785		// increment the count of DCLs
786		++fProgramCount ;
787
788		// Break out of this loop if we found the end.
789		if (pExportDCL->pNextDCLCommand == NULL)
790			break;
791		else
792			nextUserExportDCLOffset += size;
793
794		// Sanity Check
795		if (nextUserExportDCLOffset >= userExportDesc->getLength())
796		{
797			error = kIOReturnError;
798			break;
799		}
800	}while(1);
801
802	if ( ! error )
803	{
804		// Set the "next" pointer in the last DCL to NULL
805		pLastDCL->pNextDCLCommand = NULL;
806
807		fDCLTable = new DCLCommand*[ fProgramCount ] ;
808
809		InfoLog( "made DCL table, %d entries\n", fProgramCount ) ;
810
811		if ( !fDCLTable )
812			error = kIOReturnNoMemory ;
813
814		if ( !error )
815		{
816			unsigned index = 0 ;
817			for( DCLCommand * dcl = (DCLCommand*)fProgramBuffer; dcl != NULL; dcl = dcl->pNextDCLCommand )
818			{
819				if ( index >= fProgramCount )
820					panic("dcl table out of bounds\n") ;
821
822				fDCLTable[ index++ ] = dcl ;
823			}
824		}
825	}
826
827	// Need to delete the pUserImportProgramBuffer!
828	if (pUserImportProgramBuffer)
829		delete [] pUserImportProgramBuffer;
830
831	return error ;
832}
833
834#if 0
835IOReturn
836IOFWUserLocalIsochPort::releasePort ()
837{
838//	lock() ;
839//
840//	if ( fBufferDescPrepared )
841//	{
842//		fBufferDesc->complete() ;
843//		fBufferDescPrepared = false ;
844//	}
845//
846//	unlock() ;
847
848	return super::releasePort () ;
849}
850#endif
851
852IOReturn
853IOFWUserLocalIsochPort::start()
854{
855	// calling fProgram->start() takes the isoch workloop lock,
856	// so we don't need to call lock() here...
857//	lock() ;
858
859	IOReturn error = super::start() ;
860
861	fStarted = (!error) ;
862
863//	unlock() ;
864
865	return error ;
866}
867
868IOReturn
869IOFWUserLocalIsochPort::stop()
870{
871	// we are sending a stop token, but take isoch workloop lock to make sure all
872	// callbacks coming from FWIM have already been cleared out.
873
874//	IOFireWireLink * link = fUserClient->getOwner()->getController()->getLink() ;
875//	link->closeIsochGate () ;	// nnn need replacement?
876
877//	lock() ;
878
879	IOReturn error ;
880
881	// we ignore any errors from above here because we need to call super::stop() always
882	error = super::stop() ;
883
884	if ( fStarted )
885	{
886		error = IOFireWireUserClient::sendAsyncResult64( fStopTokenAsyncRef, kIOFireWireLastDCLToken, NULL, 0 ) ;
887
888		fStarted = false ;
889	}
890
891//	unlock() ;
892
893//	link->openIsochGate () ;
894
895 	return error ;
896}
897
898void
899IOFWUserLocalIsochPort::s_dclCallProcHandler( DCLCallProc * dcl )
900{
901#if 0
902#if IOFIREWIREUSERCLIENTDEBUG > 0
903	IOFWUserLocalIsochPort * me = (IOFWUserLocalIsochPort *) holder->obj ;
904
905	DebugLog("+IOFWUserLocalIsochPort::s_dclCallProcHandler, holder=%p, (holder->asyncRef)[0]=0x%x\n", holder, (holder->asyncRef)[0]) ;
906
907	me->fUserClient->getStatistics()->getIsochCallbackCounter()->addValue( 1 ) ;
908#endif
909#endif
910
911	if ( dcl->procData )
912	{
913#if 0
914// DEBUG
915		DebugThing * debugThing = (DebugThing*)dcl->procData ;
916		IOFireWireUserClient::sendAsyncResult64( (io_user_reference_t*)debugThing->asyncRef, kIOReturnSuccess, NULL, 0 ) ;
917		DebugLog("send callback port=%p\n", debugThing->port ) ;
918#else
919		IOFireWireUserClient::sendAsyncResult64( (io_user_reference_t *)dcl->procData, kIOReturnSuccess, NULL, 0 ) ;
920#endif
921	}
922}
923
924void
925IOFWUserLocalIsochPort::s_nuDCLCallout( void * refcon )
926{
927	io_user_reference_t * asyncRef = (io_user_reference_t *)refcon ;
928
929	IOFireWireUserClient::sendAsyncResult64( asyncRef, kIOReturnSuccess, NULL, 0 ) ;
930}
931
932IOReturn
933IOFWUserLocalIsochPort::setAsyncRef_DCLCallProc( OSAsyncReference64 asyncRef )
934{
935	// set up stop token async ref
936	bcopy( asyncRef, fStopTokenAsyncRef, sizeof( OSAsyncReference64 ) ) ;
937
938	// walk through DCL program and set mach port
939	// for all callproc DCLs
940	if ( fDCLPool )
941	{
942		const OSArray * program = fDCLPool->getProgramRef() ;
943		for( unsigned index = 0, count = program->getCount(); index < count; ++index )
944		{
945			IOFWDCL * dcl = reinterpret_cast< IOFWDCL * >( program->getObject( index ) ) ;
946			if ( dcl->getCallback() )
947			{
948				io_user_reference_t * dclAsyncRef = (io_user_reference_t*)dcl->getRefcon() ;
949				if ( asyncRef )
950				{
951					bcopy( asyncRef, dclAsyncRef, sizeof( io_user_reference_t ) * kIOAsyncReservedCount ) ;
952				}
953			}
954
955		}
956
957		program->release() ;
958	}
959	else
960	{
961		for( unsigned index=0; index < fProgramCount; ++index )
962		{
963			DCLCommand * dcl = fDCLTable[ index ] ;
964			if ( ( dcl->opcode & ~kFWDCLOpFlagMask ) == kDCLCallProcOp )
965			{
966#if 0
967// DEBUG
968				if ( ((DCLCallProc*)dcl)->proc && ((DCLCallProc*)dcl)->procData )
969				{
970					((DebugThing*)((DCLCallProc*)dcl)->procData)->asyncRef[0] = asyncRef[0] ;
971				}
972#else
973				{
974					((io_user_reference_t*)((DCLCallProc*)dcl)->procData)[ 0 ] = asyncRef[ 0 ] ;
975				}
976#endif
977			}
978
979			dcl = dcl->pNextDCLCommand ;
980		}
981	}
982
983	return kIOReturnSuccess ;
984}
985
986IOReturn
987IOFWUserLocalIsochPort::modifyJumpDCL ( UInt32 inJumpDCLCompilerData, UInt32 inLabelDCLCompilerData)
988{
989	if ( !fProgram )
990	{
991		ErrorLog("no program!\n") ;
992		return kIOReturnError ;
993	}
994
995	--inJumpDCLCompilerData ;
996	--inLabelDCLCompilerData ;
997
998	// be sure opcodes exist
999	if ( inJumpDCLCompilerData > fProgramCount || inLabelDCLCompilerData > fProgramCount )
1000	{
1001		DebugLog( "IOFWUserLocalIsochPort::modifyJumpDCL: DCL index (inJumpDCLCompilerData=%u, inLabelDCLCompilerData=%u) past end of lookup table (length=%u)\n",
1002				(uint32_t)inJumpDCLCompilerData,
1003				(uint32_t)inLabelDCLCompilerData,
1004				fProgramCount ) ;
1005		return kIOReturnBadArgument ;
1006	}
1007
1008	DCLJump * jumpDCL = (DCLJump *) fDCLTable[ inJumpDCLCompilerData ] ;
1009	DCLLabel * labelDCL = (DCLLabel *) fDCLTable[ inLabelDCLCompilerData ] ;
1010
1011	// make sure we're modifying a jump and that it's pointing to a label
1012	if ( ( jumpDCL->opcode & ~kFWDCLOpFlagMask ) != kDCLJumpOp || ( labelDCL->opcode & ~kFWDCLOpFlagMask ) != kDCLLabelOp )
1013	{
1014		DebugLog("IOFWUserLocalIsochPort::modifyJumpDCL: modifying non-jump (%p, %d) or pointing jump to non-label (%p, %d)\n", jumpDCL, (int)inJumpDCLCompilerData, jumpDCL->pJumpDCLLabel, (int)inLabelDCLCompilerData ) ;
1015		return kIOReturnBadArgument ;
1016	}
1017
1018	// point jump to label
1019	jumpDCL->pJumpDCLLabel = labelDCL ;
1020
1021//	lock() ;
1022	fProgram->closeGate() ;
1023
1024	IOReturn error = notify ( kFWDCLModifyNotification, (DCLCommand**) & jumpDCL, 1 ) ;
1025
1026//	unlock() ;
1027	fProgram->openGate() ;
1028
1029	return error ;
1030}
1031
1032IOReturn
1033IOFWUserLocalIsochPort::modifyDCLSize ( UInt32 dclCompilerData, IOByteCount newSize )
1034{
1035	return kIOReturnUnsupported ;
1036
1037// to fix?
1038#if 0
1039	--dclCompilerData ;
1040
1041	// be sure opcodes exist
1042	if ( dclCompilerData > fUserToKernelDCLLookupTableLength )
1043	{
1044		DebugLog("IOFWUserLocalIsochPort::modifyJumpDCLSize: DCL index (dclCompilerData=%lu) past end of lookup table (length=%lu)\n",
1045				dclCompilerData, fUserToKernelDCLLookupTableLength ) ;
1046		return kIOReturnBadArgument ;
1047	}
1048
1049	DCLTransferPacket*	dcl = (DCLTransferPacket*)( fUserToKernelDCLLookupTable[ dclCompilerData ] ) ;
1050	IOReturn			result = kIOReturnSuccess ;
1051	lock() ;
1052
1053	if (fPort)
1054		result = ((IOFWLocalIsochPort*)fPort)->notify(kFWDCLModifyNotification, (DCLCommand**)&dcl, 1) ;
1055
1056	unlock() ;
1057	return result ;
1058#endif
1059}
1060
1061IOReturn
1062IOFWUserLocalIsochPort::convertToKernelDCL (UserExportDCLUpdateDCLList *pUserExportDCL, DCLUpdateDCLList *dcl)
1063{
1064	UInt8 *pListAddress = (UInt8 *)pUserExportDCL;
1065	pListAddress += sizeof(UserExportDCLUpdateDCLList);
1066	mach_vm_address_t *pExportListItem = (mach_vm_address_t*)pListAddress;
1067
1068	// when the program was imported to the kernel, the update list was placed in
1069	// the DCL program export buffer immediately after the DCL
1070	dcl->dclCommandList = (DCLCommand**)(dcl + 1) ;
1071
1072	for( unsigned index = 0 ; index < dcl->numDCLCommands; ++index )
1073	{
1074		dcl->dclCommandList[ index ] = (DCLCommand*)( fProgramBuffer + pExportListItem[index]) ;
1075		{
1076			unsigned opcode = dcl->dclCommandList[ index ]->opcode & ~kFWDCLOpFlagMask ;
1077			if ( opcode > 15 && opcode != 20 )
1078			{
1079				panic("invalid opcode\n") ;
1080			}
1081		}
1082	}
1083
1084	return kIOReturnSuccess ;
1085}
1086
1087IOReturn
1088IOFWUserLocalIsochPort::convertToKernelDCL (UserExportDCLJump *pUserExportDCL, DCLJump *dcl )
1089{
1090	// the label field contains a an offset from the beginning of fProgramBuffer
1091	// where the label can be found
1092	dcl->pJumpDCLLabel = (DCLLabel*)( fProgramBuffer + (UInt32)pUserExportDCL->pJumpDCLLabel ) ;
1093
1094	if ( ( dcl->pJumpDCLLabel->opcode & ~kFWDCLOpFlagMask ) != kDCLLabelOp )
1095	{
1096		dcl->pJumpDCLLabel = NULL ;
1097		DebugLog( "Jump %p pointing to non-label %p\n", dcl, dcl->pJumpDCLLabel ) ;
1098//		return kIOFireWireBogusDCLProgram ;
1099	}
1100
1101	return kIOReturnSuccess ;
1102}
1103
1104IOReturn
1105IOFWUserLocalIsochPort::convertToKernelDCL (UserExportDCLCallProc *pUserExportDCL, DCLCallProc * dcl)
1106{
1107	//if ( !dcl->proc )
1108	//	return NULL ;
1109
1110	io_user_reference_t * asyncRef = (io_user_reference_t *)(dcl + 1 ) ;
1111
1112	asyncRef[0] = 0 ;
1113	asyncRef[ kIOAsyncCalloutFuncIndex ] = (mach_vm_address_t)pUserExportDCL->proc ;
1114	asyncRef[ kIOAsyncCalloutRefconIndex ] = (io_user_reference_t)pUserExportDCL->procData ;
1115
1116	dcl->proc				= (DCLCallCommandProc*) & s_dclCallProcHandler ;
1117	dcl->procData			= (DCLCallProcDataType) asyncRef ;
1118
1119
1120#if 0
1121// DEBUG
1122	DebugThing * debugThing = new DebugThing ;
1123	dcl->procData			= debugThing ;
1124	debugThing->asyncRef = asyncRef ;
1125	debugThing->port = this ;
1126#else
1127	dcl->procData			= (DCLCallProcDataType) asyncRef ;
1128#endif
1129
1130	return kIOReturnSuccess ;
1131}
1132
1133void
1134IOFWUserLocalIsochPort::exporterCleanup( const OSObject * self )
1135{
1136	IOFWUserLocalIsochPort * me = (IOFWUserLocalIsochPort*)self;
1137	me->stop() ;
1138	me->releasePort() ;
1139}
1140
1141IOReturn
1142IOFWUserLocalIsochPort::userNotify (
1143		UInt32			notificationType,
1144		UInt32			numDCLs,
1145		void *			data,
1146		IOByteCount		dataSize )
1147{
1148	InfoLog("+IOFWUserLocalIsochPort::userNotify, numDCLs=%ld\n", numDCLs ) ;
1149	if ( __builtin_expect( numDCLs > 64, false ) )
1150	{
1151		return kIOReturnBadArgument ;
1152	}
1153
1154	IOFWDCL * 			dcls[ numDCLs ] ;
1155	const OSArray *		program 		= fDCLPool->getProgramRef() ;
1156	unsigned 			programLength	= program->getCount() ;
1157	IOReturn			error			= kIOReturnSuccess ;
1158
1159	switch( (IOFWDCLNotificationType)notificationType )
1160	{
1161		case kFWNuDCLModifyNotification :
1162		{
1163			IOMemoryMap * bufferMap = fProgram->getBufferMap() ;
1164			for( unsigned index=0; index < numDCLs; ++index )
1165			{
1166				unsigned dclIndex = *(unsigned*)data - 1 ;
1167				if ( dclIndex >= programLength )
1168				{
1169					DebugLog("out of range DCL dclIndex=%d, programLength=%d\n", dclIndex, programLength ) ;
1170					error = kIOReturnBadArgument ;
1171				}
1172				else
1173				{
1174					dcls[ index ] = (IOFWDCL*)program->getObject( dclIndex ) ;
1175
1176					data = (UInt8*)data + sizeof( unsigned ) ;
1177					IOByteCount dataSize ;
1178
1179					error = dcls[ index ]->importUserDCL( (UInt8*)data, dataSize, bufferMap, program ) ;
1180
1181					// if there is no branch set, make sure the DCL "branches" to the
1182					// dcl that comes next in the program if there is one...
1183					if ( dclIndex + 1 < programLength && !dcls[ index ]->getBranch() )
1184					{
1185						dcls[ index ]->setBranch( (IOFWDCL*)program->getObject( dclIndex + 1 ) ) ;
1186					}
1187
1188					data = (UInt8*)data + dataSize ;
1189				}
1190
1191				if ( error )
1192				{
1193					break ;
1194				}
1195			}
1196
1197			break ;
1198		}
1199
1200		case kFWNuDCLModifyJumpNotification :
1201		{
1202			unsigned * dclIndexTable = (unsigned*)data ;
1203
1204			// subtract 1 from each index in our list.
1205			// when the notification type is kFWNuDCLModifyJumpNotification, the dcl list
1206			// actually contains pairs of DCL indices. The first is the dcl having its branch modified,
1207			// the second is the index of the DCL to branch to.
1208			{
1209				unsigned index = 0 ;
1210				unsigned pairIndex = 0 ;
1211
1212				while( pairIndex < numDCLs )
1213				{
1214					--dclIndexTable[ index ] ;
1215					if ( dclIndexTable[ index ] >= programLength )
1216					{
1217						DebugLog("out of range DCL index=%d, dclIndices[ index ]=%d, programLength=%d\n", index, dclIndexTable[ index ], programLength ) ;
1218						error = kIOReturnBadArgument ;
1219						break ;
1220					}
1221
1222					dcls[ pairIndex ] = (IOFWDCL*)program->getObject( dclIndexTable[ index ] ) ;
1223
1224					++index ;
1225
1226					if (dclIndexTable[ index ])
1227					{
1228						--dclIndexTable[ index ] ;
1229
1230						if ( dclIndexTable[ index ] >= programLength )
1231						{
1232							DebugLog("out of range DCL index=%d, dclIndices[ index ]=%d, programLength=%d\n", index, dclIndexTable[ index ], programLength ) ;
1233							error = kIOReturnBadArgument ;
1234							break ;
1235						}
1236
1237						dcls[ pairIndex ]->setBranch( (IOFWDCL*)program->getObject( dclIndexTable[ index ] ) ) ;
1238					}
1239					else
1240					{
1241						dcls[ pairIndex ]->setBranch( 0 ) ;
1242					}
1243
1244					++index ;
1245					++pairIndex ;
1246				}
1247			}
1248
1249			break ;
1250		}
1251
1252		case kFWNuDCLUpdateNotification :
1253		{
1254			unsigned index = 0 ;
1255			while ( index < numDCLs )
1256			{
1257				unsigned * dclIndices = (unsigned*)data ;
1258
1259				--dclIndices[ index ] ;
1260				if ( __builtin_expect( dclIndices[ index ] >= programLength, false ) )
1261				{
1262					DebugLog("out of range DCL index=%d, dclIndices[ index ]=%d, programLength=%d\n", index, dclIndices[ index ], programLength ) ;
1263					error = kIOReturnBadArgument ;
1264					break ;
1265				}
1266
1267				dcls[ index ] = (IOFWDCL*)program->getObject( dclIndices[ index ] ) ;
1268
1269				++index ;
1270			}
1271
1272			break ;
1273		}
1274
1275		default:
1276		{
1277			error = kIOReturnBadArgument ;
1278			DebugLog("unsupported notification type 0x%08x\n", (uint32_t)notificationType) ;
1279			break ;
1280		}
1281	}
1282
1283	program->release() ;
1284
1285	return error ? error : notify( (IOFWDCLNotificationType)notificationType, (DCLCommand**)dcls, numDCLs ) ;
1286}
1287
1288IOWorkLoop *
1289IOFWUserLocalIsochPort::createRealtimeThread()
1290{
1291	IOWorkLoop * workloop = IOWorkLoop::workLoop() ;
1292	if ( workloop )
1293	{
1294		// Boost isoc workloop into realtime range
1295		thread_time_constraint_policy_data_t	constraints;
1296		AbsoluteTime							time;
1297
1298		nanoseconds_to_absolutetime(625000, &time);
1299		constraints.period = AbsoluteTime_to_scalar(&time);
1300		nanoseconds_to_absolutetime(60000, &time);
1301		constraints.computation = AbsoluteTime_to_scalar(&time);
1302		nanoseconds_to_absolutetime(1250000, &time);
1303		constraints.constraint = AbsoluteTime_to_scalar(&time);
1304
1305		constraints.preemptible = TRUE;
1306
1307		{
1308			IOThread thread;
1309			thread = workloop->getThread();
1310			thread_policy_set( thread, THREAD_TIME_CONSTRAINT_POLICY, (thread_policy_t) & constraints, THREAD_TIME_CONSTRAINT_POLICY_COUNT );
1311		}
1312	}
1313
1314	return workloop ;
1315}
1316