1/*
2 * Copyright 2008-2009, Stephan A��mus <superstippi@gmx.de>
3 * Copyright 2014, Axel D��rfler, axeld@pinc-software.de
4 * All rights reserved. Distributed under the terms of the MIT License.
5 */
6
7
8#include "CopyEngine.h"
9
10#include <new>
11
12#include <math.h>
13#include <stdio.h>
14#include <string.h>
15#include <sys/resource.h>
16
17#include <Directory.h>
18#include <fs_attr.h>
19#include <NodeInfo.h>
20#include <Path.h>
21#include <SymLink.h>
22
23#include "SemaphoreLocker.h"
24#include "ProgressReporter.h"
25
26
27using std::nothrow;
28
29
30// #pragma mark - EntryFilter
31
32
33CopyEngine::EntryFilter::~EntryFilter()
34{
35}
36
37
38// #pragma mark - CopyEngine
39
40
41CopyEngine::CopyEngine(ProgressReporter* reporter, EntryFilter* entryFilter)
42	:
43	fBufferQueue(),
44	fWriterThread(-1),
45	fQuitting(false),
46
47	fAbsoluteSourcePath(),
48
49	fBytesRead(0),
50	fLastBytesRead(0),
51	fItemsCopied(0),
52	fLastItemsCopied(0),
53	fTimeRead(0),
54
55	fBytesWritten(0),
56	fTimeWritten(0),
57
58	fCurrentTargetFolder(NULL),
59	fCurrentItem(NULL),
60
61	fProgressReporter(reporter),
62	fEntryFilter(entryFilter)
63{
64	fWriterThread = spawn_thread(_WriteThreadEntry, "buffer writer",
65		B_NORMAL_PRIORITY, this);
66
67	if (fWriterThread >= B_OK)
68		resume_thread(fWriterThread);
69
70	// ask for a bunch more file descriptors so that nested copying works well
71	struct rlimit rl;
72	rl.rlim_cur = 512;
73	rl.rlim_max = RLIM_SAVED_MAX;
74	setrlimit(RLIMIT_NOFILE, &rl);
75}
76
77
78CopyEngine::~CopyEngine()
79{
80	while (fBufferQueue.Size() > 0)
81		snooze(10000);
82
83	fQuitting = true;
84	if (fWriterThread >= B_OK) {
85		int32 exitValue;
86		wait_for_thread(fWriterThread, &exitValue);
87	}
88}
89
90
91void
92CopyEngine::ResetTargets(const char* source)
93{
94	// TODO: One could subtract the bytes/items which were added to the
95	// ProgressReporter before resetting them...
96
97	fAbsoluteSourcePath = source;
98
99	fBytesRead = 0;
100	fLastBytesRead = 0;
101	fItemsCopied = 0;
102	fLastItemsCopied = 0;
103	fTimeRead = 0;
104
105	fBytesWritten = 0;
106	fTimeWritten = 0;
107
108	fCurrentTargetFolder = NULL;
109	fCurrentItem = NULL;
110}
111
112
113status_t
114CopyEngine::CollectTargets(const char* source, sem_id cancelSemaphore)
115{
116	off_t bytesToCopy = 0;
117	uint64 itemsToCopy = 0;
118	status_t ret = _CollectCopyInfo(source, cancelSemaphore, bytesToCopy,
119			itemsToCopy);
120	if (ret == B_OK && fProgressReporter != NULL)
121		fProgressReporter->AddItems(itemsToCopy, bytesToCopy);
122	return ret;
123}
124
125
126status_t
127CopyEngine::Copy(const char* _source, const char* _destination,
128	sem_id cancelSemaphore, bool copyAttributes)
129{
130	status_t ret;
131
132	BEntry source(_source);
133	ret = source.InitCheck();
134	if (ret != B_OK)
135		return ret;
136
137	BEntry destination(_destination);
138	ret = destination.InitCheck();
139	if (ret != B_OK)
140		return ret;
141
142	return _Copy(source, destination, cancelSemaphore, copyAttributes);
143}
144
145
146status_t
147CopyEngine::RemoveFolder(BEntry& entry)
148{
149	BDirectory directory(&entry);
150	status_t ret = directory.InitCheck();
151	if (ret != B_OK)
152		return ret;
153
154	BEntry subEntry;
155	while (directory.GetNextEntry(&subEntry) == B_OK) {
156		if (subEntry.IsDirectory()) {
157			ret = CopyEngine::RemoveFolder(subEntry);
158			if (ret != B_OK)
159				return ret;
160		} else {
161			ret = subEntry.Remove();
162			if (ret != B_OK)
163				return ret;
164		}
165	}
166	return entry.Remove();
167}
168
169
170status_t
171CopyEngine::_CopyData(const BEntry& _source, const BEntry& _destination,
172	sem_id cancelSemaphore)
173{
174	SemaphoreLocker lock(cancelSemaphore);
175	if (cancelSemaphore >= 0 && !lock.IsLocked()) {
176		// We are supposed to quit
177		return B_CANCELED;
178	}
179
180	BFile source(&_source, B_READ_ONLY);
181	status_t ret = source.InitCheck();
182	if (ret < B_OK)
183		return ret;
184
185	BFile* destination = new (nothrow) BFile(&_destination,
186		B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE);
187	ret = destination->InitCheck();
188	if (ret < B_OK) {
189		delete destination;
190		return ret;
191	}
192
193	int32 loopIteration = 0;
194
195	while (true) {
196		if (fBufferQueue.Size() >= BUFFER_COUNT) {
197			// the queue is "full", just wait a bit, the
198			// write thread will empty it
199			snooze(1000);
200			continue;
201		}
202
203		// allocate buffer
204		Buffer* buffer = new (nothrow) Buffer(destination);
205		if (!buffer || !buffer->buffer) {
206			delete destination;
207			delete buffer;
208			fprintf(stderr, "reading loop: out of memory\n");
209			return B_NO_MEMORY;
210		}
211
212		// fill buffer
213		ssize_t read = source.Read(buffer->buffer, buffer->size);
214		if (read < 0) {
215			ret = (status_t)read;
216			fprintf(stderr, "Failed to read data: %s\n", strerror(ret));
217			delete buffer;
218			delete destination;
219			break;
220		}
221
222		fBytesRead += read;
223		loopIteration += 1;
224		if (loopIteration % 2 == 0)
225			_UpdateProgress();
226
227		buffer->deleteFile = read == 0;
228		if (read > 0)
229			buffer->validBytes = (size_t)read;
230		else
231			buffer->validBytes = 0;
232
233		// enqueue the buffer
234		ret = fBufferQueue.Push(buffer);
235		if (ret < B_OK) {
236			buffer->deleteFile = false;
237			delete buffer;
238			delete destination;
239			return ret;
240		}
241
242		// quit if done
243		if (read == 0)
244			break;
245	}
246
247	return ret;
248}
249
250
251// #pragma mark -
252
253
254status_t
255CopyEngine::_CollectCopyInfo(const char* _source, sem_id cancelSemaphore,
256	off_t& bytesToCopy, uint64& itemsToCopy)
257{
258	BEntry source(_source);
259	status_t ret = source.InitCheck();
260	if (ret < B_OK)
261		return ret;
262
263	struct stat statInfo;
264	ret = source.GetStat(&statInfo);
265	if (ret < B_OK)
266		return ret;
267
268	SemaphoreLocker lock(cancelSemaphore);
269	if (cancelSemaphore >= 0 && !lock.IsLocked()) {
270		// We are supposed to quit
271		return B_CANCELED;
272	}
273
274	if (fEntryFilter != NULL
275		&& !fEntryFilter->ShouldCopyEntry(source,
276			_RelativeEntryPath(_source), statInfo)) {
277		// Skip this entry
278		return B_OK;
279	}
280
281	if (cancelSemaphore >= 0)
282		lock.Unlock();
283
284	if (S_ISDIR(statInfo.st_mode)) {
285		BDirectory srcFolder(&source);
286		ret = srcFolder.InitCheck();
287		if (ret < B_OK)
288			return ret;
289
290		BEntry entry;
291		while (srcFolder.GetNextEntry(&entry) == B_OK) {
292			BPath entryPath;
293			ret = entry.GetPath(&entryPath);
294			if (ret < B_OK)
295				return ret;
296
297			ret = _CollectCopyInfo(entryPath.Path(), cancelSemaphore,
298					bytesToCopy, itemsToCopy);
299			if (ret < B_OK)
300				return ret;
301		}
302	} else if (S_ISLNK(statInfo.st_mode)) {
303		// link, ignore size
304	} else {
305		bytesToCopy += statInfo.st_size;
306	}
307
308	itemsToCopy++;
309	return B_OK;
310}
311
312
313status_t
314CopyEngine::_Copy(BEntry &source, BEntry &destination,
315	sem_id cancelSemaphore, bool copyAttributes)
316{
317	struct stat sourceInfo;
318	status_t ret = source.GetStat(&sourceInfo);
319	if (ret != B_OK)
320		return ret;
321
322	SemaphoreLocker lock(cancelSemaphore);
323	if (cancelSemaphore >= 0 && !lock.IsLocked()) {
324		// We are supposed to quit
325		return B_CANCELED;
326	}
327
328	BPath sourcePath(&source);
329	ret = sourcePath.InitCheck();
330	if (ret != B_OK)
331		return ret;
332
333	BPath destPath(&destination);
334	ret = destPath.InitCheck();
335	if (ret != B_OK)
336		return ret;
337
338	const char *relativeSourcePath = _RelativeEntryPath(sourcePath.Path());
339	if (fEntryFilter != NULL
340		&& !fEntryFilter->ShouldCopyEntry(source, relativeSourcePath,
341			sourceInfo)) {
342		// Silently skip the filtered entry.
343		return B_OK;
344	}
345
346	if (cancelSemaphore >= 0)
347		lock.Unlock();
348
349	bool copyAttributesToTarget = copyAttributes;
350		// attributes of the current source to the destination will be copied
351		// when copyAttributes is set to true, but there may be exceptions, so
352		// allow the recursively used copyAttribute parameter to be overridden
353		// for the current target.
354	if (S_ISDIR(sourceInfo.st_mode)) {
355		BDirectory sourceDirectory(&source);
356		ret = sourceDirectory.InitCheck();
357		if (ret != B_OK)
358			return ret;
359
360		if (destination.Exists()) {
361			if (destination.IsDirectory()) {
362				// Do not overwrite attributes on folders that exist.
363				// This should work better when the install target
364				// already contains a Haiku installation.
365				copyAttributesToTarget = false;
366			} else {
367				ret = destination.Remove();
368			}
369
370			if (ret != B_OK) {
371				fprintf(stderr, "Failed to make room for folder '%s': "
372					"%s\n", sourcePath.Path(), strerror(ret));
373				return ret;
374			}
375		}
376
377		ret = create_directory(destPath.Path(), 0777);
378			// Make sure the target path exists, it may have been deleted if
379			// the existing destination was a file instead of a directory.
380		if (ret != B_OK && ret != B_FILE_EXISTS) {
381			fprintf(stderr, "Could not create '%s': %s\n", destPath.Path(),
382				strerror(ret));
383			return ret;
384		}
385
386		BDirectory destDirectory(&destination);
387		ret = destDirectory.InitCheck();
388		if (ret != B_OK)
389			return ret;
390
391		BEntry entry;
392		while (sourceDirectory.GetNextEntry(&entry) == B_OK) {
393			BEntry dest(&destDirectory, entry.Name());
394			ret = dest.InitCheck();
395			if (ret != B_OK)
396				return ret;
397			ret = _Copy(entry, dest, cancelSemaphore, copyAttributes);
398			if (ret != B_OK)
399				return ret;
400		}
401	} else {
402		if (destination.Exists()) {
403			if (destination.IsDirectory())
404				ret = CopyEngine::RemoveFolder(destination);
405			else
406				ret = destination.Remove();
407			if (ret != B_OK) {
408				fprintf(stderr, "Failed to make room for entry '%s': "
409					"%s\n", sourcePath.Path(), strerror(ret));
410				return ret;
411			}
412		}
413
414		fItemsCopied++;
415		BPath destDirectory;
416		ret = destPath.GetParent(&destDirectory);
417		if (ret != B_OK)
418			return ret;
419		fCurrentTargetFolder = destDirectory.Path();
420		fCurrentItem = sourcePath.Leaf();
421		_UpdateProgress();
422
423		if (S_ISLNK(sourceInfo.st_mode)) {
424			// copy symbolic links
425			BSymLink srcLink(&source);
426			ret = srcLink.InitCheck();
427			if (ret != B_OK)
428				return ret;
429
430			char linkPath[B_PATH_NAME_LENGTH];
431			ssize_t read = srcLink.ReadLink(linkPath, B_PATH_NAME_LENGTH - 1);
432			if (read < 0)
433				return (status_t)read;
434
435			BDirectory dstFolder;
436			ret = destination.GetParent(&dstFolder);
437			if (ret != B_OK)
438				return ret;
439			ret = dstFolder.CreateSymLink(sourcePath.Leaf(), linkPath, NULL);
440			if (ret != B_OK)
441				return ret;
442		} else {
443			// copy file data
444			// NOTE: Do not pass the locker, we simply keep holding the lock!
445			ret = _CopyData(source, destination);
446			if (ret != B_OK)
447				return ret;
448		}
449	}
450
451	if (copyAttributesToTarget) {
452		// copy attributes to the current target
453		BNode sourceNode(&source);
454		BNode targetNode(&destination);
455		char attrName[B_ATTR_NAME_LENGTH];
456		while (sourceNode.GetNextAttrName(attrName) == B_OK) {
457			attr_info info;
458			if (sourceNode.GetAttrInfo(attrName, &info) != B_OK)
459				continue;
460			size_t size = 4096;
461			uint8 buffer[size];
462			off_t offset = 0;
463			ssize_t read = sourceNode.ReadAttr(attrName, info.type,
464				offset, buffer, std::min((off_t)size, info.size));
465			// NOTE: It's important to still write the attribute even if
466			// we have read 0 bytes!
467			while (read >= 0) {
468				targetNode.WriteAttr(attrName, info.type, offset, buffer, read);
469				offset += read;
470				read = sourceNode.ReadAttr(attrName, info.type,
471					offset, buffer, std::min((off_t)size, info.size - offset));
472				if (read == 0)
473					break;
474			}
475		}
476
477		// copy basic attributes
478		destination.SetPermissions(sourceInfo.st_mode);
479		destination.SetOwner(sourceInfo.st_uid);
480		destination.SetGroup(sourceInfo.st_gid);
481		destination.SetModificationTime(sourceInfo.st_mtime);
482		destination.SetCreationTime(sourceInfo.st_crtime);
483	}
484
485	return B_OK;
486}
487
488
489const char*
490CopyEngine::_RelativeEntryPath(const char* absoluteSourcePath) const
491{
492	if (strncmp(absoluteSourcePath, fAbsoluteSourcePath,
493			fAbsoluteSourcePath.Length()) != 0) {
494		return absoluteSourcePath;
495	}
496
497	const char* relativePath
498		= absoluteSourcePath + fAbsoluteSourcePath.Length();
499	return relativePath[0] == '/' ? relativePath + 1 : relativePath;
500}
501
502
503void
504CopyEngine::_UpdateProgress()
505{
506	if (fProgressReporter == NULL)
507		return;
508
509	uint64 items = 0;
510	if (fLastItemsCopied < fItemsCopied) {
511		items = fItemsCopied - fLastItemsCopied;
512		fLastItemsCopied = fItemsCopied;
513	}
514
515	off_t bytes = 0;
516	if (fLastBytesRead < fBytesRead) {
517		bytes = fBytesRead - fLastBytesRead;
518		fLastBytesRead = fBytesRead;
519	}
520
521	fProgressReporter->ItemsWritten(items, bytes, fCurrentItem,
522		fCurrentTargetFolder);
523}
524
525
526int32
527CopyEngine::_WriteThreadEntry(void* cookie)
528{
529	CopyEngine* engine = (CopyEngine*)cookie;
530	engine->_WriteThread();
531	return B_OK;
532}
533
534
535void
536CopyEngine::_WriteThread()
537{
538	bigtime_t bufferWaitTimeout = 100000;
539
540	while (!fQuitting) {
541
542		bigtime_t now = system_time();
543
544		Buffer* buffer = NULL;
545		status_t ret = fBufferQueue.Pop(&buffer, bufferWaitTimeout);
546		if (ret == B_TIMED_OUT) {
547			// no buffer, timeout
548			continue;
549		} else if (ret == B_NO_INIT) {
550			// real error
551			return;
552		} else if (ret != B_OK) {
553			// no buffer, queue error
554			snooze(10000);
555			continue;
556		}
557
558		if (!buffer->deleteFile) {
559			ssize_t written = buffer->file->Write(buffer->buffer,
560				buffer->validBytes);
561			if (written != (ssize_t)buffer->validBytes) {
562				// TODO: this should somehow be propagated back
563				// to the main thread!
564				fprintf(stderr, "Failed to write data: %s\n",
565					strerror((status_t)written));
566			}
567			fBytesWritten += written;
568		}
569
570		delete buffer;
571
572		// measure performance
573		fTimeWritten += system_time() - now;
574	}
575
576	double megaBytes = (double)fBytesWritten / (1024 * 1024);
577	double seconds = (double)fTimeWritten / 1000000;
578	if (seconds > 0) {
579		printf("%.2f MB written (%.2f MB/s)\n", megaBytes,
580			megaBytes / seconds);
581	}
582}
583