1/*
2 * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de.
3 * Copyright 2008-2017, Axel D��rfler, axeld@pinc-software.de.
4 * Distributed under the terms of the MIT License.
5 */
6
7
8#include "IORequest.h"
9
10#include <string.h>
11
12#include <arch/debug.h>
13#include <debug.h>
14#include <heap.h>
15#include <kernel.h>
16#include <team.h>
17#include <thread.h>
18#include <util/AutoLock.h>
19#include <vm/vm.h>
20#include <vm/VMAddressSpace.h>
21
22#include "dma_resources.h"
23
24
25//#define TRACE_IO_REQUEST
26#ifdef TRACE_IO_REQUEST
27#	define TRACE(x...) dprintf(x)
28#else
29#	define TRACE(x...) ;
30#endif
31
32
33// partial I/O operation phases
34enum {
35	PHASE_READ_BEGIN	= 0,
36	PHASE_READ_END		= 1,
37	PHASE_DO_ALL		= 2
38};
39
40
41struct virtual_vec_cookie {
42	uint32			vec_index;
43	generic_size_t	vec_offset;
44	area_id			mapped_area;
45	void*			physical_page_handle;
46	addr_t			virtual_address;
47
48	virtual_vec_cookie()
49		:
50		vec_index(0),
51		vec_offset(0),
52		mapped_area(-1),
53		physical_page_handle(NULL),
54		virtual_address((addr_t)-1)
55	{
56	}
57
58	void PutPhysicalPageIfNeeded()
59	{
60		if (virtual_address != (addr_t)-1) {
61			vm_put_physical_page(virtual_address, physical_page_handle);
62			virtual_address = (addr_t)-1;
63		}
64	}
65};
66
67
68// #pragma mark -
69
70
71IORequestChunk::IORequestChunk()
72	:
73	fParent(NULL),
74	fStatus(1)
75{
76}
77
78
79IORequestChunk::~IORequestChunk()
80{
81}
82
83
84//	#pragma mark -
85
86
87IOBuffer*
88IOBuffer::Create(uint32 count, bool vip)
89{
90	size_t size = sizeof(IOBuffer) + sizeof(generic_io_vec) * (count - 1);
91	IOBuffer* buffer
92		= (IOBuffer*)(malloc_etc(size, vip ? HEAP_PRIORITY_VIP : 0));
93	if (buffer == NULL)
94		return NULL;
95
96	buffer->fCapacity = count;
97	buffer->fVecCount = 0;
98	buffer->fUser = false;
99	buffer->fPhysical = false;
100	buffer->fVIP = vip;
101	buffer->fMemoryLocked = false;
102
103	return buffer;
104}
105
106
107void
108IOBuffer::Delete()
109{
110	free_etc(this, fVIP ? HEAP_PRIORITY_VIP : 0);
111}
112
113
114void
115IOBuffer::SetVecs(generic_size_t firstVecOffset, generic_size_t lastVecSize,
116	const generic_io_vec* vecs, uint32 count, generic_size_t length, uint32 flags)
117{
118	memcpy(fVecs, vecs, sizeof(generic_io_vec) * count);
119
120	if (count > 0 && firstVecOffset > 0) {
121		fVecs[0].base += firstVecOffset;
122		fVecs[0].length -= firstVecOffset;
123	}
124	if (lastVecSize > 0)
125		fVecs[count - 1].length = lastVecSize;
126
127	fVecCount = count;
128	fLength = length;
129	fPhysical = (flags & B_PHYSICAL_IO_REQUEST) != 0;
130	fUser = !fPhysical && IS_USER_ADDRESS(vecs[0].base);
131
132#if KDEBUG
133	generic_size_t actualLength = 0;
134	for (size_t i = 0; i < fVecCount; i++)
135		actualLength += fVecs[i].length;
136
137	ASSERT(actualLength == fLength);
138#endif
139}
140
141
142status_t
143IOBuffer::GetNextVirtualVec(void*& _cookie, iovec& vector)
144{
145	virtual_vec_cookie* cookie = (virtual_vec_cookie*)_cookie;
146	if (cookie == NULL) {
147		cookie = new(malloc_flags(fVIP ? HEAP_PRIORITY_VIP : 0))
148			virtual_vec_cookie;
149		if (cookie == NULL)
150			return B_NO_MEMORY;
151
152		_cookie = cookie;
153	}
154
155	// recycle a potential previously mapped page
156	cookie->PutPhysicalPageIfNeeded();
157
158	if (cookie->vec_index >= fVecCount)
159		return B_BAD_INDEX;
160
161	if (!fPhysical) {
162		vector.iov_base = (void*)(addr_t)fVecs[cookie->vec_index].base;
163		vector.iov_len = fVecs[cookie->vec_index++].length;
164		return B_OK;
165	}
166
167	if (cookie->vec_index == 0
168		&& (fVecCount > 1 || fVecs[0].length > B_PAGE_SIZE)) {
169		void* mappedAddress;
170		addr_t mappedSize;
171
172// TODO: This is a potential violation of the VIP requirement, since
173// vm_map_physical_memory_vecs() allocates memory without special flags!
174		cookie->mapped_area = vm_map_physical_memory_vecs(
175			VMAddressSpace::KernelID(), "io buffer mapped physical vecs",
176			&mappedAddress, B_ANY_KERNEL_ADDRESS, &mappedSize,
177			B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, fVecs, fVecCount);
178
179		if (cookie->mapped_area >= 0) {
180			vector.iov_base = mappedAddress;
181			vector.iov_len = mappedSize;
182			return B_OK;
183		} else
184			ktrace_printf("failed to map area: %s\n", strerror(cookie->mapped_area));
185	}
186
187	// fallback to page wise mapping
188	generic_io_vec& currentVec = fVecs[cookie->vec_index];
189	generic_addr_t address = currentVec.base + cookie->vec_offset;
190	size_t pageOffset = address % B_PAGE_SIZE;
191
192// TODO: This is a potential violation of the VIP requirement, since
193// vm_get_physical_page() may allocate memory without special flags!
194	status_t result = vm_get_physical_page(address - pageOffset,
195		&cookie->virtual_address, &cookie->physical_page_handle);
196	if (result != B_OK)
197		return result;
198
199	generic_size_t length = min_c(currentVec.length - cookie->vec_offset,
200		B_PAGE_SIZE - pageOffset);
201
202	vector.iov_base = (void*)(cookie->virtual_address + pageOffset);
203	vector.iov_len = length;
204
205	cookie->vec_offset += length;
206	if (cookie->vec_offset >= currentVec.length) {
207		cookie->vec_index++;
208		cookie->vec_offset = 0;
209	}
210
211	return B_OK;
212}
213
214
215void
216IOBuffer::FreeVirtualVecCookie(void* _cookie)
217{
218	virtual_vec_cookie* cookie = (virtual_vec_cookie*)_cookie;
219	if (cookie->mapped_area >= 0)
220		delete_area(cookie->mapped_area);
221
222	cookie->PutPhysicalPageIfNeeded();
223
224	free_etc(cookie, fVIP ? HEAP_PRIORITY_VIP : 0);
225}
226
227
228status_t
229IOBuffer::LockMemory(team_id team, bool isWrite)
230{
231	if (fMemoryLocked) {
232		panic("memory already locked!");
233		return B_BAD_VALUE;
234	}
235
236	for (uint32 i = 0; i < fVecCount; i++) {
237		status_t status = lock_memory_etc(team, (void*)(addr_t)fVecs[i].base,
238			fVecs[i].length, isWrite ? 0 : B_READ_DEVICE);
239		if (status != B_OK) {
240			_UnlockMemory(team, i, isWrite);
241			return status;
242		}
243	}
244
245	fMemoryLocked = true;
246	return B_OK;
247}
248
249
250void
251IOBuffer::_UnlockMemory(team_id team, size_t count, bool isWrite)
252{
253	for (uint32 i = 0; i < count; i++) {
254		unlock_memory_etc(team, (void*)(addr_t)fVecs[i].base, fVecs[i].length,
255			isWrite ? 0 : B_READ_DEVICE);
256	}
257}
258
259
260void
261IOBuffer::UnlockMemory(team_id team, bool isWrite)
262{
263	if (!fMemoryLocked) {
264		panic("memory not locked");
265		return;
266	}
267
268	_UnlockMemory(team, fVecCount, isWrite);
269	fMemoryLocked = false;
270}
271
272
273void
274IOBuffer::Dump() const
275{
276	kprintf("IOBuffer at %p\n", this);
277
278	kprintf("  origin:     %s\n", fUser ? "user" : "kernel");
279	kprintf("  kind:       %s\n", fPhysical ? "physical" : "virtual");
280	kprintf("  length:     %" B_PRIuGENADDR "\n", fLength);
281	kprintf("  capacity:   %" B_PRIuSIZE "\n", fCapacity);
282	kprintf("  vecs:       %" B_PRIuSIZE "\n", fVecCount);
283
284	for (uint32 i = 0; i < fVecCount; i++) {
285		kprintf("    [%" B_PRIu32 "] %#" B_PRIxGENADDR ", %" B_PRIuGENADDR "\n",
286			i, fVecs[i].base, fVecs[i].length);
287	}
288}
289
290
291// #pragma mark -
292
293
294void
295IOOperation::SetStatus(status_t status, generic_size_t completedLength)
296{
297	IORequestChunk::SetStatus(status);
298	if (IsWrite() == fParent->IsWrite()) {
299		// Determine how many bytes we actually read or wrote,
300		// relative to the original range, not the translated range.
301		const generic_size_t partialBegin = (fOriginalOffset - fOffset);
302		generic_size_t originalTransferredBytes = completedLength;
303		if (originalTransferredBytes < partialBegin)
304			originalTransferredBytes = 0;
305		else
306			originalTransferredBytes -= partialBegin;
307
308		if (originalTransferredBytes > fOriginalLength)
309			originalTransferredBytes = fOriginalLength;
310
311		fTransferredBytes += originalTransferredBytes;
312	}
313}
314
315
316bool
317IOOperation::Finish()
318{
319	TRACE("IOOperation::Finish()\n");
320
321	if (fStatus == B_OK) {
322		if (fParent->IsWrite()) {
323			TRACE("  is write\n");
324			if (fPhase == PHASE_READ_BEGIN) {
325				TRACE("  phase read begin\n");
326				// repair phase adjusted vec
327				fDMABuffer->VecAt(fSavedVecIndex).length = fSavedVecLength;
328
329				// partial write: copy partial begin to bounce buffer
330				bool skipReadEndPhase;
331				status_t error = _CopyPartialBegin(true, skipReadEndPhase);
332				if (error == B_OK) {
333					// We're done with the first phase only (read in begin).
334					// Get ready for next phase...
335					fPhase = HasPartialEnd() && !skipReadEndPhase
336						? PHASE_READ_END : PHASE_DO_ALL;
337					_PrepareVecs();
338					ResetStatus();
339						// TODO: Is there a race condition, if the request is
340						// aborted at the same time?
341					return false;
342				}
343
344				IORequestChunk::SetStatus(error);
345			} else if (fPhase == PHASE_READ_END) {
346				TRACE("  phase read end\n");
347				// repair phase adjusted vec
348				generic_io_vec& vec = fDMABuffer->VecAt(fSavedVecIndex);
349				vec.base += vec.length - fSavedVecLength;
350				vec.length = fSavedVecLength;
351
352				// partial write: copy partial end to bounce buffer
353				status_t error = _CopyPartialEnd(true);
354				if (error == B_OK) {
355					// We're done with the second phase only (read in end).
356					// Get ready for next phase...
357					fPhase = PHASE_DO_ALL;
358					ResetStatus();
359						// TODO: Is there a race condition, if the request is
360						// aborted at the same time?
361					return false;
362				}
363
364				IORequestChunk::SetStatus(error);
365			}
366		}
367	}
368
369	if (fParent->IsRead() && UsesBounceBuffer()) {
370		TRACE("  read with bounce buffer\n");
371		// copy the bounce buffer segments to the final location
372		uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress();
373		phys_addr_t bounceBufferStart
374			= fDMABuffer->PhysicalBounceBufferAddress();
375		phys_addr_t bounceBufferEnd = bounceBufferStart
376			+ fDMABuffer->BounceBufferSize();
377
378		const generic_io_vec* vecs = fDMABuffer->Vecs();
379		uint32 vecCount = fDMABuffer->VecCount();
380
381		status_t error = B_OK;
382
383		// We iterate through the vecs we have read, moving offset (the device
384		// offset) as we go. If [offset, offset + vec.length) intersects with
385		// [startOffset, endOffset) we copy to the final location.
386		off_t offset = fOffset;
387		const off_t startOffset = fOriginalOffset;
388		const off_t endOffset = fOriginalOffset + fOriginalLength;
389
390		for (uint32 i = 0; error == B_OK && i < vecCount; i++) {
391			const generic_io_vec& vec = vecs[i];
392			generic_addr_t base = vec.base;
393			generic_size_t length = vec.length;
394
395			if (offset < startOffset) {
396				// If the complete vector is before the start offset, skip it.
397				if (offset + (off_t)length <= startOffset) {
398					offset += length;
399					continue;
400				}
401
402				// The vector starts before the start offset, but intersects
403				// with it. Skip the part we aren't interested in.
404				generic_size_t diff = startOffset - offset;
405				offset += diff;
406				base += diff;
407				length -= diff;
408			}
409
410			if (offset + (off_t)length > endOffset) {
411				// If we're already beyond the end offset, we're done.
412				if (offset >= endOffset)
413					break;
414
415				// The vector extends beyond the end offset -- cut it.
416				length = endOffset - offset;
417			}
418
419			if (base >= bounceBufferStart && base < bounceBufferEnd) {
420				error = fParent->CopyData(
421					bounceBuffer + (base - bounceBufferStart), offset, length);
422			}
423
424			offset += length;
425		}
426
427		if (error != B_OK)
428			IORequestChunk::SetStatus(error);
429	}
430
431	return true;
432}
433
434
435/*!	Note: SetPartial() must be called first!
436*/
437status_t
438IOOperation::Prepare(IORequest* request)
439{
440	if (fParent != NULL)
441		fParent->RemoveOperation(this);
442
443	fParent = request;
444
445	fTransferredBytes = 0;
446
447	// set initial phase
448	fPhase = PHASE_DO_ALL;
449	if (fParent->IsWrite()) {
450		// Copy data to bounce buffer segments, save the partial begin/end vec,
451		// which will be copied after their respective read phase.
452		if (UsesBounceBuffer()) {
453			TRACE("  write with bounce buffer\n");
454			uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress();
455			phys_addr_t bounceBufferStart
456				= fDMABuffer->PhysicalBounceBufferAddress();
457			phys_addr_t bounceBufferEnd = bounceBufferStart
458				+ fDMABuffer->BounceBufferSize();
459
460			const generic_io_vec* vecs = fDMABuffer->Vecs();
461			uint32 vecCount = fDMABuffer->VecCount();
462			generic_size_t vecOffset = 0;
463			uint32 i = 0;
464
465			off_t offset = fOffset;
466			off_t endOffset = fOffset + fLength;
467
468			if (HasPartialBegin()) {
469				// skip first block
470				generic_size_t toSkip = fBlockSize;
471				while (toSkip > 0) {
472					if (vecs[i].length <= toSkip) {
473						toSkip -= vecs[i].length;
474						i++;
475					} else {
476						vecOffset = toSkip;
477						break;
478					}
479				}
480
481				offset += fBlockSize;
482			}
483
484			if (HasPartialEnd()) {
485				// skip last block
486				generic_size_t toSkip = fBlockSize;
487				while (toSkip > 0) {
488					if (vecs[vecCount - 1].length <= toSkip) {
489						toSkip -= vecs[vecCount - 1].length;
490						vecCount--;
491					} else
492						break;
493				}
494
495				endOffset -= fBlockSize;
496			}
497
498			for (; i < vecCount; i++) {
499				const generic_io_vec& vec = vecs[i];
500				generic_addr_t base = vec.base + vecOffset;
501				generic_size_t length = vec.length - vecOffset;
502				vecOffset = 0;
503
504				if (base >= bounceBufferStart && base < bounceBufferEnd) {
505					if (offset + (off_t)length > endOffset)
506						length = endOffset - offset;
507					status_t error = fParent->CopyData(offset,
508						bounceBuffer + (base - bounceBufferStart), length);
509					if (error != B_OK)
510						return error;
511				}
512
513				offset += length;
514			}
515		}
516
517		if (HasPartialBegin())
518			fPhase = PHASE_READ_BEGIN;
519		else if (HasPartialEnd())
520			fPhase = PHASE_READ_END;
521
522		_PrepareVecs();
523	}
524
525	ResetStatus();
526
527	if (fParent != NULL)
528		fParent->AddOperation(this);
529
530	return B_OK;
531}
532
533
534void
535IOOperation::SetOriginalRange(off_t offset, generic_size_t length)
536{
537	fOriginalOffset = fOffset = offset;
538	fOriginalLength = fLength = length;
539}
540
541
542void
543IOOperation::SetRange(off_t offset, generic_size_t length)
544{
545	fOffset = offset;
546	fLength = length;
547}
548
549
550off_t
551IOOperation::Offset() const
552{
553	return fPhase == PHASE_READ_END ? fOffset + fLength - fBlockSize : fOffset;
554}
555
556
557generic_size_t
558IOOperation::Length() const
559{
560	return fPhase == PHASE_DO_ALL ? fLength : fBlockSize;
561}
562
563
564generic_io_vec*
565IOOperation::Vecs() const
566{
567	switch (fPhase) {
568		case PHASE_READ_END:
569			return fDMABuffer->Vecs() + fSavedVecIndex;
570		case PHASE_READ_BEGIN:
571		case PHASE_DO_ALL:
572		default:
573			return fDMABuffer->Vecs();
574	}
575}
576
577
578uint32
579IOOperation::VecCount() const
580{
581	switch (fPhase) {
582		case PHASE_READ_BEGIN:
583			return fSavedVecIndex + 1;
584		case PHASE_READ_END:
585			return fDMABuffer->VecCount() - fSavedVecIndex;
586		case PHASE_DO_ALL:
587		default:
588			return fDMABuffer->VecCount();
589	}
590}
591
592
593void
594IOOperation::SetPartial(bool partialBegin, bool partialEnd)
595{
596	TRACE("partial begin %d, end %d\n", partialBegin, partialEnd);
597	fPartialBegin = partialBegin;
598	fPartialEnd = partialEnd;
599}
600
601
602bool
603IOOperation::IsWrite() const
604{
605	return fParent->IsWrite() && fPhase == PHASE_DO_ALL;
606}
607
608
609bool
610IOOperation::IsRead() const
611{
612	return fParent->IsRead();
613}
614
615
616void
617IOOperation::_PrepareVecs()
618{
619	// we need to prepare the vecs for consumption by the drivers
620	if (fPhase == PHASE_READ_BEGIN) {
621		generic_io_vec* vecs = fDMABuffer->Vecs();
622		uint32 vecCount = fDMABuffer->VecCount();
623		generic_size_t vecLength = fBlockSize;
624		for (uint32 i = 0; i < vecCount; i++) {
625			generic_io_vec& vec = vecs[i];
626			if (vec.length >= vecLength) {
627				fSavedVecIndex = i;
628				fSavedVecLength = vec.length;
629				vec.length = vecLength;
630				break;
631			}
632			vecLength -= vec.length;
633		}
634	} else if (fPhase == PHASE_READ_END) {
635		generic_io_vec* vecs = fDMABuffer->Vecs();
636		uint32 vecCount = fDMABuffer->VecCount();
637		generic_size_t vecLength = fBlockSize;
638		for (int32 i = vecCount - 1; i >= 0; i--) {
639			generic_io_vec& vec = vecs[i];
640			if (vec.length >= vecLength) {
641				fSavedVecIndex = i;
642				fSavedVecLength = vec.length;
643				vec.base += vec.length - vecLength;
644				vec.length = vecLength;
645				break;
646			}
647			vecLength -= vec.length;
648		}
649	}
650}
651
652
653status_t
654IOOperation::_CopyPartialBegin(bool isWrite, bool& singleBlockOnly)
655{
656	generic_size_t relativeOffset = OriginalOffset() - fOffset;
657	generic_size_t length = fBlockSize - relativeOffset;
658
659	singleBlockOnly = length >= OriginalLength();
660	if (singleBlockOnly)
661		length = OriginalLength();
662
663	TRACE("_CopyPartialBegin(%s, single only %d)\n",
664		isWrite ? "write" : "read", singleBlockOnly);
665
666	if (isWrite) {
667		return fParent->CopyData(OriginalOffset(),
668			(uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, length);
669	} else {
670		return fParent->CopyData(
671			(uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset,
672			OriginalOffset(), length);
673	}
674}
675
676
677status_t
678IOOperation::_CopyPartialEnd(bool isWrite)
679{
680	TRACE("_CopyPartialEnd(%s)\n", isWrite ? "write" : "read");
681
682	const generic_io_vec& lastVec
683		= fDMABuffer->VecAt(fDMABuffer->VecCount() - 1);
684	off_t lastVecPos = fOffset + fLength - fBlockSize;
685	uint8* base = (uint8*)fDMABuffer->BounceBufferAddress()
686		+ (lastVec.base + lastVec.length - fBlockSize
687		- fDMABuffer->PhysicalBounceBufferAddress());
688		// NOTE: this won't work if we don't use the bounce buffer contiguously
689		// (because of boundary alignments).
690	generic_size_t length = OriginalOffset() + OriginalLength() - lastVecPos;
691
692	if (isWrite)
693		return fParent->CopyData(lastVecPos, base, length);
694
695	return fParent->CopyData(base, lastVecPos, length);
696}
697
698
699void
700IOOperation::Dump() const
701{
702	kprintf("io_operation at %p\n", this);
703
704	kprintf("  parent:           %p\n", fParent);
705	kprintf("  status:           %s\n", strerror(fStatus));
706	kprintf("  dma buffer:       %p\n", fDMABuffer);
707	kprintf("  offset:           %-8" B_PRIdOFF " (original: %" B_PRIdOFF ")\n",
708		fOffset, fOriginalOffset);
709	kprintf("  length:           %-8" B_PRIuGENADDR " (original: %"
710		B_PRIuGENADDR ")\n", fLength, fOriginalLength);
711	kprintf("  transferred:      %" B_PRIuGENADDR "\n", fTransferredBytes);
712	kprintf("  block size:       %" B_PRIuGENADDR "\n", fBlockSize);
713	kprintf("  saved vec index:  %u\n", fSavedVecIndex);
714	kprintf("  saved vec length: %u\n", fSavedVecLength);
715	kprintf("  r/w:              %s\n", IsWrite() ? "write" : "read");
716	kprintf("  phase:            %s\n", fPhase == PHASE_READ_BEGIN
717		? "read begin" : fPhase == PHASE_READ_END ? "read end"
718		: fPhase == PHASE_DO_ALL ? "do all" : "unknown");
719	kprintf("  partial begin:    %s\n", fPartialBegin ? "yes" : "no");
720	kprintf("  partial end:      %s\n", fPartialEnd ? "yes" : "no");
721	kprintf("  bounce buffer:    %s\n", fUsesBounceBuffer ? "yes" : "no");
722
723	set_debug_variable("_parent", (addr_t)fParent);
724	set_debug_variable("_buffer", (addr_t)fDMABuffer);
725}
726
727
728// #pragma mark -
729
730
731IORequest::IORequest()
732	:
733	fIsNotified(false),
734	fFinishedCallback(NULL),
735	fFinishedCookie(NULL),
736	fIterationCallback(NULL),
737	fIterationCookie(NULL)
738{
739	mutex_init(&fLock, "I/O request lock");
740	fFinishedCondition.Init(this, "I/O request finished");
741}
742
743
744IORequest::~IORequest()
745{
746	mutex_lock(&fLock);
747	DeleteSubRequests();
748	if (fBuffer != NULL)
749		fBuffer->Delete();
750	mutex_destroy(&fLock);
751}
752
753
754/* static */ IORequest*
755IORequest::Create(bool vip)
756{
757	return vip
758		? new(malloc_flags(HEAP_PRIORITY_VIP)) IORequest
759		: new(std::nothrow) IORequest;
760}
761
762
763status_t
764IORequest::Init(off_t offset, generic_addr_t buffer, generic_size_t length,
765	bool write, uint32 flags)
766{
767	ASSERT(offset >= 0);
768
769	generic_io_vec vec;
770	vec.base = buffer;
771	vec.length = length;
772	return Init(offset, &vec, 1, length, write, flags);
773}
774
775
776status_t
777IORequest::Init(off_t offset, generic_size_t firstVecOffset,
778	generic_size_t lastVecSize, const generic_io_vec* vecs, size_t count,
779	generic_size_t length, bool write, uint32 flags)
780{
781	ASSERT(offset >= 0);
782
783	fBuffer = IOBuffer::Create(count, (flags & B_VIP_IO_REQUEST) != 0);
784	if (fBuffer == NULL)
785		return B_NO_MEMORY;
786
787	fBuffer->SetVecs(firstVecOffset, lastVecSize, vecs, count, length, flags);
788
789	fOwner = NULL;
790	fOffset = offset;
791	fLength = length;
792	fRelativeParentOffset = 0;
793	fTransferSize = 0;
794	fFlags = flags;
795	Thread* thread = thread_get_current_thread();
796	fTeam = thread->team->id;
797	fThread = thread->id;
798	fIsWrite = write;
799	fPartialTransfer = false;
800	fSuppressChildNotifications = false;
801
802	// these are for iteration
803	fVecIndex = 0;
804	fVecOffset = 0;
805	fRemainingBytes = length;
806
807	fPendingChildren = 0;
808
809	fStatus = 1;
810
811	return B_OK;
812}
813
814
815status_t
816IORequest::CreateSubRequest(off_t parentOffset, off_t offset,
817	generic_size_t length, IORequest*& _subRequest)
818{
819	ASSERT(parentOffset >= fOffset && length <= fLength
820		&& parentOffset - fOffset <= (off_t)(fLength - length));
821
822	// find start vec
823	generic_size_t vecOffset = parentOffset - fOffset;
824	generic_io_vec* vecs = fBuffer->Vecs();
825	int32 vecCount = fBuffer->VecCount();
826	int32 startVec = 0;
827	for (; startVec < vecCount; startVec++) {
828		const generic_io_vec& vec = vecs[startVec];
829		if (vecOffset < vec.length)
830			break;
831
832		vecOffset -= vec.length;
833	}
834
835	// count vecs
836	generic_size_t currentVecOffset = vecOffset;
837	int32 endVec = startVec;
838	generic_size_t remainingLength = length;
839	for (; endVec < vecCount; endVec++) {
840		const generic_io_vec& vec = vecs[endVec];
841		if (vec.length - currentVecOffset >= remainingLength)
842			break;
843
844		remainingLength -= vec.length - currentVecOffset;
845		currentVecOffset = 0;
846	}
847
848	// create subrequest
849	IORequest* subRequest = Create((fFlags & B_VIP_IO_REQUEST) != 0);
850	if (subRequest == NULL)
851		return B_NO_MEMORY;
852
853	status_t error = subRequest->Init(offset, vecOffset, remainingLength,
854		vecs + startVec, endVec - startVec + 1, length, fIsWrite,
855		fFlags & ~B_DELETE_IO_REQUEST);
856	if (error != B_OK) {
857		delete subRequest;
858		return error;
859	}
860
861	subRequest->fRelativeParentOffset = parentOffset - fOffset;
862	subRequest->fTeam = fTeam;
863	subRequest->fThread = fThread;
864
865	_subRequest = subRequest;
866	subRequest->SetParent(this);
867
868	MutexLocker _(fLock);
869
870	fChildren.Add(subRequest);
871	fPendingChildren++;
872	TRACE("IORequest::CreateSubRequest(): request: %p, subrequest: %p\n", this,
873		subRequest);
874
875	return B_OK;
876}
877
878
879void
880IORequest::DeleteSubRequests()
881{
882	while (IORequestChunk* chunk = fChildren.RemoveHead())
883		delete chunk;
884	fPendingChildren = 0;
885}
886
887
888void
889IORequest::SetFinishedCallback(io_request_finished_callback callback,
890	void* cookie)
891{
892	fFinishedCallback = callback;
893	fFinishedCookie = cookie;
894}
895
896
897void
898IORequest::SetIterationCallback(io_request_iterate_callback callback,
899	void* cookie)
900{
901	fIterationCallback = callback;
902	fIterationCookie = cookie;
903}
904
905
906io_request_finished_callback
907IORequest::FinishedCallback(void** _cookie) const
908{
909	if (_cookie != NULL)
910		*_cookie = fFinishedCookie;
911	return fFinishedCallback;
912}
913
914
915status_t
916IORequest::Wait(uint32 flags, bigtime_t timeout)
917{
918	MutexLocker locker(fLock);
919
920	if (IsFinished() && fIsNotified)
921		return Status();
922
923	ConditionVariableEntry entry;
924	fFinishedCondition.Add(&entry);
925
926	locker.Unlock();
927
928	status_t error = entry.Wait(flags, timeout);
929	if (error != B_OK)
930		return error;
931
932	return Status();
933}
934
935
936void
937IORequest::NotifyFinished()
938{
939	TRACE("IORequest::NotifyFinished(): request: %p\n", this);
940
941	MutexLocker locker(fLock);
942	ASSERT(fStatus != 1);
943
944	if (fStatus == B_OK && !fPartialTransfer && RemainingBytes() > 0) {
945		// The request is not really done yet. If it has an iteration callback,
946		// call it.
947		if (fIterationCallback != NULL) {
948			ResetStatus();
949			locker.Unlock();
950			bool partialTransfer = false;
951			status_t error = fIterationCallback(fIterationCookie, this,
952				&partialTransfer);
953			if (error == B_OK && !partialTransfer)
954				return;
955
956			// Iteration failed, which means we're responsible for notifying the
957			// requests finished.
958			locker.Lock();
959			fStatus = error;
960			fPartialTransfer = true;
961		}
962	}
963
964	ASSERT(!fIsNotified);
965	ASSERT(fPendingChildren == 0);
966	ASSERT(fChildren.IsEmpty()
967		|| dynamic_cast<IOOperation*>(fChildren.Head()) == NULL);
968
969	// unlock the memory
970	if (fBuffer->IsMemoryLocked())
971		fBuffer->UnlockMemory(fTeam, fIsWrite);
972
973	// Cache the callbacks before we unblock waiters and unlock. Any of the
974	// following could delete this request, so we don't want to touch it
975	// once we have started telling others that it is done.
976	IORequest* parent = fParent;
977	io_request_finished_callback finishedCallback = fFinishedCallback;
978	void* finishedCookie = fFinishedCookie;
979	status_t status = fStatus;
980	generic_size_t lastTransferredOffset
981		= fRelativeParentOffset + fTransferSize;
982	bool partialTransfer = status != B_OK || fPartialTransfer;
983	bool deleteRequest = (fFlags & B_DELETE_IO_REQUEST) != 0;
984
985	// unblock waiters
986	fIsNotified = true;
987	fFinishedCondition.NotifyAll();
988
989	locker.Unlock();
990
991	// notify callback
992	if (finishedCallback != NULL) {
993		finishedCallback(finishedCookie, this, status, partialTransfer,
994			lastTransferredOffset);
995	}
996
997	// notify parent
998	if (parent != NULL) {
999		parent->SubRequestFinished(this, status, partialTransfer,
1000			lastTransferredOffset);
1001	}
1002
1003	if (deleteRequest)
1004		delete this;
1005}
1006
1007
1008/*!	Returns whether this request or any of it's ancestors has a finished or
1009	notification callback. Used to decide whether NotifyFinished() can be called
1010	synchronously.
1011*/
1012bool
1013IORequest::HasCallbacks() const
1014{
1015	if (fFinishedCallback != NULL || fIterationCallback != NULL)
1016		return true;
1017
1018	return fParent != NULL && fParent->HasCallbacks();
1019}
1020
1021
1022void
1023IORequest::SetStatusAndNotify(status_t status)
1024{
1025	MutexLocker locker(fLock);
1026
1027	if (fStatus != 1)
1028		return;
1029
1030	fStatus = status;
1031
1032	locker.Unlock();
1033
1034	NotifyFinished();
1035}
1036
1037
1038void
1039IORequest::OperationFinished(IOOperation* operation)
1040{
1041	TRACE("IORequest::OperationFinished(%p, %#" B_PRIx32 "): request: %p\n",
1042		operation, status, this);
1043
1044	MutexLocker locker(fLock);
1045
1046	fChildren.Remove(operation);
1047	operation->SetParent(NULL);
1048
1049	const status_t status = operation->Status();
1050	const bool partialTransfer =
1051		(operation->TransferredBytes() < operation->OriginalLength());
1052	const generic_size_t transferEndOffset =
1053		(operation->OriginalOffset() + operation->TransferredBytes());
1054
1055	if (status != B_OK || partialTransfer) {
1056		if (fTransferSize > transferEndOffset)
1057			fTransferSize = transferEndOffset;
1058		fPartialTransfer = true;
1059	}
1060
1061	if (status != B_OK && fStatus == 1)
1062		fStatus = status;
1063
1064	if (--fPendingChildren > 0)
1065		return;
1066
1067	// last child finished
1068
1069	// set status, if not done yet
1070	if (fStatus == 1)
1071		fStatus = B_OK;
1072}
1073
1074
1075void
1076IORequest::SubRequestFinished(IORequest* request, status_t status,
1077	bool partialTransfer, generic_size_t transferEndOffset)
1078{
1079	TRACE("IORequest::SubrequestFinished(%p, %#" B_PRIx32 ", %d, %"
1080		B_PRIuGENADDR "): request: %p\n", request, status, partialTransfer, transferEndOffset, this);
1081
1082	MutexLocker locker(fLock);
1083
1084	if (status != B_OK || partialTransfer) {
1085		if (fTransferSize > transferEndOffset)
1086			fTransferSize = transferEndOffset;
1087		fPartialTransfer = true;
1088	}
1089
1090	if (status != B_OK && fStatus == 1)
1091		fStatus = status;
1092
1093	if (--fPendingChildren > 0 || fSuppressChildNotifications)
1094		return;
1095
1096	// last child finished
1097
1098	// set status, if not done yet
1099	if (fStatus == 1)
1100		fStatus = B_OK;
1101
1102	locker.Unlock();
1103
1104	NotifyFinished();
1105}
1106
1107
1108void
1109IORequest::SetUnfinished()
1110{
1111	MutexLocker _(fLock);
1112	ResetStatus();
1113}
1114
1115
1116void
1117IORequest::SetTransferredBytes(bool partialTransfer,
1118	generic_size_t transferredBytes)
1119{
1120	TRACE("%p->IORequest::SetTransferredBytes(%d, %" B_PRIuGENADDR ")\n", this,
1121		partialTransfer, transferredBytes);
1122
1123	MutexLocker _(fLock);
1124
1125	ASSERT(transferredBytes <= fLength);
1126
1127	fPartialTransfer = partialTransfer;
1128	fTransferSize = transferredBytes;
1129}
1130
1131
1132void
1133IORequest::SetSuppressChildNotifications(bool suppress)
1134{
1135	fSuppressChildNotifications = suppress;
1136}
1137
1138
1139void
1140IORequest::Advance(generic_size_t bySize)
1141{
1142	TRACE("IORequest::Advance(%" B_PRIuGENADDR "): remaining: %" B_PRIuGENADDR
1143		" -> %" B_PRIuGENADDR "\n", bySize, fRemainingBytes,
1144		fRemainingBytes - bySize);
1145	fRemainingBytes -= bySize;
1146	fTransferSize += bySize;
1147
1148	generic_io_vec* vecs = fBuffer->Vecs();
1149	uint32 vecCount = fBuffer->VecCount();
1150	while (fVecIndex < vecCount
1151			&& vecs[fVecIndex].length - fVecOffset <= bySize) {
1152		bySize -= vecs[fVecIndex].length - fVecOffset;
1153		fVecOffset = 0;
1154		fVecIndex++;
1155	}
1156
1157	fVecOffset += bySize;
1158}
1159
1160
1161IORequest*
1162IORequest::FirstSubRequest()
1163{
1164	return dynamic_cast<IORequest*>(fChildren.Head());
1165}
1166
1167
1168IORequest*
1169IORequest::NextSubRequest(IORequest* previous)
1170{
1171	if (previous == NULL)
1172		return NULL;
1173	return dynamic_cast<IORequest*>(fChildren.GetNext(previous));
1174}
1175
1176
1177void
1178IORequest::AddOperation(IOOperation* operation)
1179{
1180	MutexLocker locker(fLock);
1181	TRACE("IORequest::AddOperation(%p): request: %p\n", operation, this);
1182	fChildren.Add(operation);
1183	fPendingChildren++;
1184}
1185
1186
1187void
1188IORequest::RemoveOperation(IOOperation* operation)
1189{
1190	MutexLocker locker(fLock);
1191	fChildren.Remove(operation);
1192	operation->SetParent(NULL);
1193}
1194
1195
1196status_t
1197IORequest::CopyData(off_t offset, void* buffer, size_t size)
1198{
1199	return _CopyData(buffer, offset, size, true);
1200}
1201
1202
1203status_t
1204IORequest::CopyData(const void* buffer, off_t offset, size_t size)
1205{
1206	return _CopyData((void*)buffer, offset, size, false);
1207}
1208
1209
1210status_t
1211IORequest::ClearData(off_t offset, generic_size_t size)
1212{
1213	if (size == 0)
1214		return B_OK;
1215
1216	if (offset < fOffset || offset + (off_t)size > fOffset + (off_t)fLength) {
1217		panic("IORequest::ClearData(): invalid range: (%" B_PRIdOFF
1218			", %" B_PRIuGENADDR ")", offset, size);
1219		return B_BAD_VALUE;
1220	}
1221
1222	// If we can, we directly copy from/to the virtual buffer. The memory is
1223	// locked in this case.
1224	status_t (*clearFunction)(generic_addr_t, generic_size_t, team_id);
1225	if (fBuffer->IsPhysical()) {
1226		clearFunction = &IORequest::_ClearDataPhysical;
1227	} else {
1228		clearFunction = fBuffer->IsUser() && fTeam != team_get_current_team_id()
1229			? &IORequest::_ClearDataUser : &IORequest::_ClearDataSimple;
1230	}
1231
1232	// skip bytes if requested
1233	generic_io_vec* vecs = fBuffer->Vecs();
1234	generic_size_t skipBytes = offset - fOffset;
1235	generic_size_t vecOffset = 0;
1236	while (skipBytes > 0) {
1237		if (vecs[0].length > skipBytes) {
1238			vecOffset = skipBytes;
1239			break;
1240		}
1241
1242		skipBytes -= vecs[0].length;
1243		vecs++;
1244	}
1245
1246	// clear vector-wise
1247	while (size > 0) {
1248		generic_size_t toClear = min_c(size, vecs[0].length - vecOffset);
1249		status_t error = clearFunction(vecs[0].base + vecOffset, toClear,
1250			fTeam);
1251		if (error != B_OK)
1252			return error;
1253
1254		size -= toClear;
1255		vecs++;
1256		vecOffset = 0;
1257	}
1258
1259	return B_OK;
1260
1261}
1262
1263
1264status_t
1265IORequest::_CopyData(void* _buffer, off_t offset, size_t size, bool copyIn)
1266{
1267	if (size == 0)
1268		return B_OK;
1269
1270	uint8* buffer = (uint8*)_buffer;
1271
1272	if (offset < fOffset || offset + (off_t)size > fOffset + (off_t)fLength) {
1273		panic("IORequest::_CopyData(): invalid range: (%" B_PRIdOFF ", %lu)",
1274			offset, size);
1275		return B_BAD_VALUE;
1276	}
1277
1278	// If we can, we directly copy from/to the virtual buffer. The memory is
1279	// locked in this case.
1280	status_t (*copyFunction)(void*, generic_addr_t, size_t, team_id, bool);
1281	if (fBuffer->IsPhysical()) {
1282		copyFunction = &IORequest::_CopyPhysical;
1283	} else {
1284		copyFunction = fBuffer->IsUser() && fTeam != team_get_current_team_id()
1285			? &IORequest::_CopyUser : &IORequest::_CopySimple;
1286	}
1287
1288	// skip bytes if requested
1289	generic_io_vec* vecs = fBuffer->Vecs();
1290	generic_size_t skipBytes = offset - fOffset;
1291	generic_size_t vecOffset = 0;
1292	while (skipBytes > 0) {
1293		if (vecs[0].length > skipBytes) {
1294			vecOffset = skipBytes;
1295			break;
1296		}
1297
1298		skipBytes -= vecs[0].length;
1299		vecs++;
1300	}
1301
1302	// copy vector-wise
1303	while (size > 0) {
1304		generic_size_t toCopy = min_c(size, vecs[0].length - vecOffset);
1305		status_t error = copyFunction(buffer, vecs[0].base + vecOffset, toCopy,
1306			fTeam, copyIn);
1307		if (error != B_OK)
1308			return error;
1309
1310		buffer += toCopy;
1311		size -= toCopy;
1312		vecs++;
1313		vecOffset = 0;
1314	}
1315
1316	return B_OK;
1317}
1318
1319
1320/* static */ status_t
1321IORequest::_CopySimple(void* bounceBuffer, generic_addr_t external, size_t size,
1322	team_id team, bool copyIn)
1323{
1324	TRACE("  IORequest::_CopySimple(%p, %#" B_PRIxGENADDR ", %lu, %d)\n",
1325		bounceBuffer, external, size, copyIn);
1326	if (IS_USER_ADDRESS(external)) {
1327		status_t status = B_OK;
1328		if (copyIn)
1329			status = user_memcpy(bounceBuffer, (void*)(addr_t)external, size);
1330		else
1331			status = user_memcpy((void*)(addr_t)external, bounceBuffer, size);
1332		if (status < B_OK)
1333			return status;
1334		return B_OK;
1335	}
1336	if (copyIn)
1337		memcpy(bounceBuffer, (void*)(addr_t)external, size);
1338	else
1339		memcpy((void*)(addr_t)external, bounceBuffer, size);
1340	return B_OK;
1341}
1342
1343
1344/* static */ status_t
1345IORequest::_CopyPhysical(void* bounceBuffer, generic_addr_t external,
1346	size_t size, team_id team, bool copyIn)
1347{
1348	if (copyIn)
1349		return vm_memcpy_from_physical(bounceBuffer, external, size, false);
1350
1351	return vm_memcpy_to_physical(external, bounceBuffer, size, false);
1352}
1353
1354
1355/* static */ status_t
1356IORequest::_CopyUser(void* _bounceBuffer, generic_addr_t _external, size_t size,
1357	team_id team, bool copyIn)
1358{
1359	uint8* bounceBuffer = (uint8*)_bounceBuffer;
1360	uint8* external = (uint8*)(addr_t)_external;
1361
1362	while (size > 0) {
1363		static const int32 kEntryCount = 8;
1364		physical_entry entries[kEntryCount];
1365
1366		uint32 count = kEntryCount;
1367		status_t error = get_memory_map_etc(team, external, size, entries,
1368			&count);
1369		if (error != B_OK && error != B_BUFFER_OVERFLOW) {
1370			panic("IORequest::_CopyUser(): Failed to get physical memory for "
1371				"user memory %p\n", external);
1372			return B_BAD_ADDRESS;
1373		}
1374
1375		for (uint32 i = 0; i < count; i++) {
1376			const physical_entry& entry = entries[i];
1377			error = _CopyPhysical(bounceBuffer, entry.address, entry.size, team,
1378				copyIn);
1379			if (error != B_OK)
1380				return error;
1381
1382			size -= entry.size;
1383			bounceBuffer += entry.size;
1384			external += entry.size;
1385		}
1386	}
1387
1388	return B_OK;
1389}
1390
1391
1392/*static*/ status_t
1393IORequest::_ClearDataSimple(generic_addr_t external, generic_size_t size,
1394	team_id team)
1395{
1396	memset((void*)(addr_t)external, 0, (size_t)size);
1397	return B_OK;
1398}
1399
1400
1401/*static*/ status_t
1402IORequest::_ClearDataPhysical(generic_addr_t external, generic_size_t size,
1403	team_id team)
1404{
1405	return vm_memset_physical((phys_addr_t)external, 0, (phys_size_t)size);
1406}
1407
1408
1409/*static*/ status_t
1410IORequest::_ClearDataUser(generic_addr_t _external, generic_size_t size,
1411	team_id team)
1412{
1413	uint8* external = (uint8*)(addr_t)_external;
1414
1415	while (size > 0) {
1416		static const int32 kEntryCount = 8;
1417		physical_entry entries[kEntryCount];
1418
1419		uint32 count = kEntryCount;
1420		status_t error = get_memory_map_etc(team, external, size, entries,
1421			&count);
1422		if (error != B_OK && error != B_BUFFER_OVERFLOW) {
1423			panic("IORequest::_ClearDataUser(): Failed to get physical memory "
1424				"for user memory %p\n", external);
1425			return B_BAD_ADDRESS;
1426		}
1427
1428		for (uint32 i = 0; i < count; i++) {
1429			const physical_entry& entry = entries[i];
1430			error = _ClearDataPhysical(entry.address, entry.size, team);
1431			if (error != B_OK)
1432				return error;
1433
1434			size -= entry.size;
1435			external += entry.size;
1436		}
1437	}
1438
1439	return B_OK;
1440}
1441
1442
1443void
1444IORequest::Dump() const
1445{
1446	kprintf("io_request at %p\n", this);
1447
1448	kprintf("  owner:             %p\n", fOwner);
1449	kprintf("  parent:            %p\n", fParent);
1450	kprintf("  status:            %s\n", strerror(fStatus));
1451	kprintf("  mutex:             %p\n", &fLock);
1452	kprintf("  IOBuffer:          %p\n", fBuffer);
1453	kprintf("  offset:            %" B_PRIdOFF "\n", fOffset);
1454	kprintf("  length:            %" B_PRIuGENADDR "\n", fLength);
1455	kprintf("  transfer size:     %" B_PRIuGENADDR "\n", fTransferSize);
1456	kprintf("  relative offset:   %" B_PRIuGENADDR "\n", fRelativeParentOffset);
1457	kprintf("  pending children:  %" B_PRId32 "\n", fPendingChildren);
1458	kprintf("  flags:             %#" B_PRIx32 "\n", fFlags);
1459	kprintf("  team:              %" B_PRId32 "\n", fTeam);
1460	kprintf("  thread:            %" B_PRId32 "\n", fThread);
1461	kprintf("  r/w:               %s\n", fIsWrite ? "write" : "read");
1462	kprintf("  partial transfer:  %s\n", fPartialTransfer ? "yes" : "no");
1463	kprintf("  finished cvar:     %p\n", &fFinishedCondition);
1464	kprintf("  iteration:\n");
1465	kprintf("    vec index:       %" B_PRIu32 "\n", fVecIndex);
1466	kprintf("    vec offset:      %" B_PRIuGENADDR "\n", fVecOffset);
1467	kprintf("    remaining bytes: %" B_PRIuGENADDR "\n", fRemainingBytes);
1468	kprintf("  callbacks:\n");
1469	kprintf("    finished %p, cookie %p\n", fFinishedCallback, fFinishedCookie);
1470	kprintf("    iteration %p, cookie %p\n", fIterationCallback,
1471		fIterationCookie);
1472	kprintf("  children:\n");
1473
1474	IORequestChunkList::ConstIterator iterator = fChildren.GetIterator();
1475	while (iterator.HasNext()) {
1476		kprintf("    %p\n", iterator.Next());
1477	}
1478
1479	set_debug_variable("_parent", (addr_t)fParent);
1480	set_debug_variable("_mutex", (addr_t)&fLock);
1481	set_debug_variable("_buffer", (addr_t)fBuffer);
1482	set_debug_variable("_cvar", (addr_t)&fFinishedCondition);
1483}
1484