1/*
2 * Routines that are exclusive to the generator process.
3 *
4 * Copyright (C) 1996-2000 Andrew Tridgell
5 * Copyright (C) 1996 Paul Mackerras
6 * Copyright (C) 2002 Martin Pool <mbp@samba.org>
7 * Copyright (C) 2003, 2004, 2005, 2006 Wayne Davison
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License along
20 * with this program; if not, write to the Free Software Foundation, Inc.,
21 * 51 Franklin Street - Fifth Floor, Boston, MA 02110-1301, USA.
22 */
23
24#include "rsync.h"
25
26extern int verbose;
27extern int dry_run;
28extern int do_xfers;
29extern int stdout_format_has_i;
30extern int logfile_format_has_i;
31extern int am_root;
32extern int am_server;
33extern int am_daemon;
34extern int do_progress;
35extern int relative_paths;
36extern int implied_dirs;
37extern int keep_dirlinks;
38extern int preserve_links;
39extern int preserve_devices;
40extern int preserve_specials;
41extern int preserve_hard_links;
42extern int preserve_perms;
43extern int preserve_uid;
44extern int preserve_gid;
45extern int preserve_times;
46extern int omit_dir_times;
47extern int delete_mode;
48extern int delete_before;
49extern int delete_during;
50extern int delete_after;
51extern int module_id;
52extern int ignore_errors;
53extern int remove_source_files;
54extern int delay_updates;
55extern int update_only;
56extern int ignore_existing;
57extern int ignore_non_existing;
58extern int inplace;
59extern int append_mode;
60extern int make_backups;
61extern int csum_length;
62extern int ignore_times;
63extern int size_only;
64extern OFF_T max_size;
65extern OFF_T min_size;
66extern int io_error;
67extern int allowed_lull;
68extern int sock_f_out;
69extern int ignore_timeout;
70extern int protocol_version;
71extern int fuzzy_basis;
72extern int always_checksum;
73extern int checksum_len;
74extern char *partial_dir;
75extern char *basis_dir[];
76extern int compare_dest;
77extern int copy_dest;
78extern int link_dest;
79extern int whole_file;
80extern int list_only;
81extern int new_root_dir;
82extern int read_batch;
83extern int safe_symlinks;
84extern long block_size; /* "long" because popt can't set an int32. */
85extern int max_delete;
86extern int force_delete;
87extern int one_file_system;
88extern struct stats stats;
89extern dev_t filesystem_dev;
90extern char *backup_dir;
91extern char *backup_suffix;
92extern int backup_suffix_len;
93extern struct file_list *the_file_list;
94extern struct filter_list_struct server_filter_list;
95#ifdef EA_SUPPORT
96extern int extended_attributes;
97#endif
98
99static int deletion_count = 0; /* used to implement --max-delete */
100
101/* For calling delete_file() */
102#define DEL_FORCE_RECURSE	(1<<1) /* recurse even w/o --force */
103#define DEL_TERSE		(1<<3)
104
105
106static int is_backup_file(char *fn)
107{
108	int k = strlen(fn) - backup_suffix_len;
109	return k > 0 && strcmp(fn+k, backup_suffix) == 0;
110}
111
112
113/* Delete a file or directory.  If DEL_FORCE_RECURSE is set in the flags, or if
114 * force_delete is set, this will delete recursively.
115 *
116 * Note that fname must point to a MAXPATHLEN buffer if the mode indicates it's
117 * a directory! (The buffer is used for recursion, but returned unchanged.)
118 */
119static int delete_item(char *fname, int mode, int flags)
120{
121	struct file_list *dirlist;
122	int j, dlen, zap_dir, ok;
123	unsigned remainder;
124	void *save_filters;
125	char *p;
126
127	if (!S_ISDIR(mode)) {
128		if (max_delete && ++deletion_count > max_delete)
129			return 0;
130		if (make_backups && (backup_dir || !is_backup_file(fname)))
131			ok = make_backup(fname);
132		else
133			ok = robust_unlink(fname) == 0;
134		if (ok) {
135			if (!(flags & DEL_TERSE))
136				log_delete(fname, mode);
137			return 0;
138		}
139		if (errno == ENOENT) {
140			deletion_count--;
141			return 0;
142		}
143		rsyserr(FERROR, errno, "delete_file: unlink %s failed",
144			full_fname(fname));
145		return -1;
146	}
147
148	zap_dir = flags & DEL_FORCE_RECURSE || force_delete;
149	if ((max_delete && ++deletion_count > max_delete)
150	    || (dry_run && zap_dir)) {
151		ok = 0;
152		errno = ENOTEMPTY;
153	} else if (make_backups && !backup_dir && !is_backup_file(fname)
154	    && !(flags & DEL_FORCE_RECURSE))
155		ok = make_backup(fname);
156	else
157		ok = do_rmdir(fname) == 0;
158	if (ok) {
159		if (!(flags & DEL_TERSE))
160			log_delete(fname, mode);
161		return 0;
162	}
163	if (errno == ENOENT) {
164		deletion_count--;
165		return 0;
166	}
167	if (!zap_dir) {
168		rsyserr(FERROR, errno, "delete_file: rmdir %s failed",
169			full_fname(fname));
170		return -1;
171	}
172	flags |= DEL_FORCE_RECURSE; /* mark subdir dels as not "in the way" */
173	deletion_count--;
174
175	dlen = strlen(fname);
176	save_filters = push_local_filters(fname, dlen);
177
178	dirlist = get_dirlist(fname, dlen, 0);
179
180	p = fname + dlen;
181	if (dlen != 1 || *fname != '/')
182		*p++ = '/';
183	remainder = MAXPATHLEN - (p - fname);
184
185	for (j = dirlist->count; j--; ) {
186		struct file_struct *fp = dirlist->files[j];
187
188		if (fp->flags & FLAG_MOUNT_POINT)
189			continue;
190
191		strlcpy(p, fp->basename, remainder);
192		delete_item(fname, fp->mode, flags & ~DEL_TERSE);
193	}
194	flist_free(dirlist);
195
196	fname[dlen] = '\0';
197
198	pop_local_filters(save_filters);
199
200	if (max_delete && ++deletion_count > max_delete)
201		return 0;
202
203	if (do_rmdir(fname) == 0) {
204		if (!(flags & DEL_TERSE))
205			log_delete(fname, mode);
206	} else if (errno != ENOTEMPTY && errno != EEXIST && errno != ENOENT) {
207		rsyserr(FERROR, errno, "delete_file: rmdir %s failed",
208			full_fname(fname));
209		return -1;
210	}
211
212	return 0;
213}
214
215
216/* This function is used to implement per-directory deletion, and is used by
217 * all the --delete-WHEN options.  Note that the fbuf pointer must point to a
218 * MAXPATHLEN buffer with the name of the directory in it (the functions we
219 * call will append names onto the end, but the old dir value will be restored
220 * on exit). */
221static void delete_in_dir(struct file_list *flist, char *fbuf,
222			  struct file_struct *file, STRUCT_STAT *stp)
223{
224	static int min_depth = MAXPATHLEN, cur_depth = -1;
225	static void *filt_array[MAXPATHLEN/2+1];
226	static int already_warned = 0;
227	struct file_list *dirlist;
228	char delbuf[MAXPATHLEN];
229	int dlen, i;
230
231	if (!flist) {
232		while (cur_depth >= min_depth)
233			pop_local_filters(filt_array[cur_depth--]);
234		min_depth = MAXPATHLEN;
235		cur_depth = -1;
236		return;
237	}
238
239	if (verbose > 2)
240		rprintf(FINFO, "delete_in_dir(%s)\n", fbuf);
241
242	if (allowed_lull)
243		maybe_send_keepalive();
244
245	if (file->dir.depth >= MAXPATHLEN/2+1)
246		return; /* Impossible... */
247
248	if (io_error && !(lp_ignore_errors(module_id) || ignore_errors)) {
249		if (already_warned)
250			return;
251		rprintf(FINFO,
252			"IO error encountered -- skipping file deletion\n");
253		already_warned = 1;
254		return;
255	}
256
257	while (cur_depth >= file->dir.depth && cur_depth >= min_depth)
258		pop_local_filters(filt_array[cur_depth--]);
259	cur_depth = file->dir.depth;
260	if (min_depth > cur_depth)
261		min_depth = cur_depth;
262	dlen = strlen(fbuf);
263	filt_array[cur_depth] = push_local_filters(fbuf, dlen);
264
265	if (one_file_system) {
266		if (file->flags & FLAG_TOP_DIR)
267			filesystem_dev = stp->st_dev;
268		else if (filesystem_dev != stp->st_dev)
269			return;
270	}
271
272	dirlist = get_dirlist(fbuf, dlen, 0);
273
274	/* If an item in dirlist is not found in flist, delete it
275	 * from the filesystem. */
276	for (i = dirlist->count; i--; ) {
277		struct file_struct *fp = dirlist->files[i];
278		if (!fp->basename || fp->flags & FLAG_MOUNT_POINT)
279			continue;
280		if (flist_find(flist, fp) < 0) {
281			f_name(fp, delbuf);
282			delete_item(delbuf, fp->mode, DEL_FORCE_RECURSE);
283		}
284	}
285
286	flist_free(dirlist);
287}
288
289/* This deletes any files on the receiving side that are not present on the
290 * sending side.  This is used by --delete-before and --delete-after. */
291static void do_delete_pass(struct file_list *flist)
292{
293	char fbuf[MAXPATHLEN];
294	STRUCT_STAT st;
295	int j;
296
297	/* dry_run is incremented when the destination doesn't exist yet. */
298	if (dry_run > 1 || list_only)
299		return;
300
301	for (j = 0; j < flist->count; j++) {
302		struct file_struct *file = flist->files[j];
303
304		if (!(file->flags & FLAG_DEL_HERE))
305			continue;
306
307		f_name(file, fbuf);
308		if (verbose > 1 && file->flags & FLAG_TOP_DIR)
309			rprintf(FINFO, "deleting in %s\n", fbuf);
310
311		if (link_stat(fbuf, &st, keep_dirlinks) < 0
312		 || !S_ISDIR(st.st_mode))
313			continue;
314
315		delete_in_dir(flist, fbuf, file, &st);
316	}
317	delete_in_dir(NULL, NULL, NULL, NULL);
318
319	if (do_progress && !am_server)
320		rprintf(FINFO, "                    \r");
321}
322
323int unchanged_attrs(struct file_struct *file, STRUCT_STAT *st)
324{
325	if (preserve_perms
326	 && (st->st_mode & CHMOD_BITS) != (file->mode & CHMOD_BITS))
327		return 0;
328
329	if (am_root && preserve_uid && st->st_uid != file->uid)
330		return 0;
331
332	if (preserve_gid && file->gid != GID_NONE && st->st_gid != file->gid)
333		return 0;
334
335	return 1;
336}
337
338void itemize(struct file_struct *file, int ndx, int statret, STRUCT_STAT *st,
339	     int32 iflags, uchar fnamecmp_type, char *xname)
340{
341	if (statret >= 0) { /* A from-dest-dir statret can == 1! */
342		int keep_time = !preserve_times ? 0
343		    : S_ISDIR(file->mode) ? !omit_dir_times
344		    : !S_ISLNK(file->mode);
345
346		if (S_ISREG(file->mode) && file->length != st->st_size)
347			iflags |= ITEM_REPORT_SIZE;
348		if ((iflags & (ITEM_TRANSFER|ITEM_LOCAL_CHANGE) && !keep_time
349		     && (!(iflags & ITEM_XNAME_FOLLOWS) || *xname))
350		    || (keep_time && cmp_time(file->modtime, st->st_mtime) != 0))
351			iflags |= ITEM_REPORT_TIME;
352		if ((file->mode & CHMOD_BITS) != (st->st_mode & CHMOD_BITS))
353			iflags |= ITEM_REPORT_PERMS;
354		if (preserve_uid && am_root && file->uid != st->st_uid)
355			iflags |= ITEM_REPORT_OWNER;
356		if (preserve_gid && file->gid != GID_NONE
357		    && st->st_gid != file->gid)
358			iflags |= ITEM_REPORT_GROUP;
359		if (file->flags & FLAG_CLEAR_METADATA)
360			iflags |= ITEM_REPORT_ACL | ITEM_REPORT_XATTR;
361	} else
362		iflags |= ITEM_IS_NEW;
363
364	iflags &= 0xffff;
365	if ((iflags & SIGNIFICANT_ITEM_FLAGS || verbose > 1
366	  || stdout_format_has_i > 1 || (xname && *xname)) && !read_batch) {
367		if (protocol_version >= 29) {
368			if (ndx >= 0)
369				write_int(sock_f_out, ndx);
370			write_shortint(sock_f_out, iflags);
371			if (iflags & ITEM_BASIS_TYPE_FOLLOWS)
372				write_byte(sock_f_out, fnamecmp_type);
373			if (iflags & ITEM_XNAME_FOLLOWS)
374				write_vstring(sock_f_out, xname, strlen(xname));
375		} else if (ndx >= 0) {
376			enum logcode code = logfile_format_has_i ? FINFO : FCLIENT;
377			log_item(code, file, &stats, iflags, xname);
378		}
379	}
380}
381
382
383/* Perform our quick-check heuristic for determining if a file is unchanged. */
384int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
385{
386	if (st->st_size != file->length)
387		return 0;
388
389	/* if always checksum is set then we use the checksum instead
390	   of the file time to determine whether to sync */
391	if (always_checksum && S_ISREG(st->st_mode)) {
392		char sum[MD4_SUM_LENGTH];
393		file_checksum(fn, sum, st->st_size);
394		return memcmp(sum, file->u.sum, checksum_len) == 0;
395	}
396
397	if (size_only)
398		return 1;
399
400	if (ignore_times)
401		return 0;
402
403	return cmp_time(st->st_mtime, file->modtime) == 0;
404}
405
406
407/*
408 * set (initialize) the size entries in the per-file sum_struct
409 * calculating dynamic block and checksum sizes.
410 *
411 * This is only called from generate_and_send_sums() but is a separate
412 * function to encapsulate the logic.
413 *
414 * The block size is a rounded square root of file length.
415 *
416 * The checksum size is determined according to:
417 *     blocksum_bits = BLOCKSUM_BIAS + 2*log2(file_len) - log2(block_len)
418 * provided by Donovan Baarda which gives a probability of rsync
419 * algorithm corrupting data and falling back using the whole md4
420 * checksums.
421 *
422 * This might be made one of several selectable heuristics.
423 */
424static void sum_sizes_sqroot(struct sum_struct *sum, int64 len)
425{
426	int32 blength;
427	int s2length;
428
429	if (block_size)
430		blength = block_size;
431	else if (len <= BLOCK_SIZE * BLOCK_SIZE)
432		blength = BLOCK_SIZE;
433	else {
434		int32 c;
435		int64 l;
436		int cnt;
437		for (c = 1, l = len, cnt = 0; l >>= 2; c <<= 1, cnt++) {}
438		if (cnt >= 31 || c >= MAX_BLOCK_SIZE)
439			blength = MAX_BLOCK_SIZE;
440		else {
441		    blength = 0;
442		    do {
443			    blength |= c;
444			    if (len < (int64)blength * blength)
445				    blength &= ~c;
446			    c >>= 1;
447		    } while (c >= 8);	/* round to multiple of 8 */
448		    blength = MAX(blength, BLOCK_SIZE);
449		}
450	}
451
452	if (protocol_version < 27) {
453		s2length = csum_length;
454	} else if (csum_length == SUM_LENGTH) {
455		s2length = SUM_LENGTH;
456	} else {
457		int32 c;
458		int64 l;
459		int b = BLOCKSUM_BIAS;
460		for (l = len; l >>= 1; b += 2) {}
461		for (c = blength; (c >>= 1) && b; b--) {}
462		/* add a bit, subtract rollsum, round up. */
463		s2length = (b + 1 - 32 + 7) / 8; /* --optimize in compiler-- */
464		s2length = MAX(s2length, csum_length);
465		s2length = MIN(s2length, SUM_LENGTH);
466	}
467
468	sum->flength	= len;
469	sum->blength	= blength;
470	sum->s2length	= s2length;
471	sum->remainder	= len % blength;
472	sum->count	= len / blength + (sum->remainder != 0);
473
474	if (sum->count && verbose > 2) {
475		rprintf(FINFO,
476			"count=%.0f rem=%ld blength=%ld s2length=%d flength=%.0f\n",
477			(double)sum->count, (long)sum->remainder, (long)sum->blength,
478			sum->s2length, (double)sum->flength);
479	}
480}
481
482
483/*
484 * Generate and send a stream of signatures/checksums that describe a buffer
485 *
486 * Generate approximately one checksum every block_len bytes.
487 */
488static void generate_and_send_sums(int fd, OFF_T len, int f_out, int f_copy)
489{
490	int32 i;
491	struct map_struct *mapbuf;
492	struct sum_struct sum;
493	OFF_T offset = 0;
494
495	sum_sizes_sqroot(&sum, len);
496	write_sum_head(f_out, &sum);
497
498	if (append_mode > 0 && f_copy < 0)
499		return;
500
501	if (len > 0)
502		mapbuf = map_file(fd, len, MAX_MAP_SIZE, sum.blength);
503	else
504		mapbuf = NULL;
505
506	for (i = 0; i < sum.count; i++) {
507		int32 n1 = (int32)MIN(len, (OFF_T)sum.blength);
508		char *map = map_ptr(mapbuf, offset, n1);
509		char sum2[SUM_LENGTH];
510		uint32 sum1;
511
512		len -= n1;
513		offset += n1;
514
515		if (f_copy >= 0) {
516			full_write(f_copy, map, n1);
517			if (append_mode > 0)
518				continue;
519		}
520
521		sum1 = get_checksum1(map, n1);
522		get_checksum2(map, n1, sum2);
523
524		if (verbose > 3) {
525			rprintf(FINFO,
526				"chunk[%.0f] offset=%.0f len=%ld sum1=%08lx\n",
527				(double)i, (double)offset - n1, (long)n1,
528				(unsigned long)sum1);
529		}
530		write_int(f_out, sum1);
531		write_buf(f_out, sum2, sum.s2length);
532	}
533
534	if (mapbuf)
535		unmap_file(mapbuf);
536}
537
538
539/* Try to find a filename in the same dir as "fname" with a similar name. */
540static int find_fuzzy(struct file_struct *file, struct file_list *dirlist)
541{
542	int fname_len, fname_suf_len;
543	const char *fname_suf, *fname = file->basename;
544	uint32 lowest_dist = 25 << 16; /* ignore a distance greater than 25 */
545	int j, lowest_j = -1;
546
547	fname_len = strlen(fname);
548	fname_suf = find_filename_suffix(fname, fname_len, &fname_suf_len);
549
550	for (j = 0; j < dirlist->count; j++) {
551		struct file_struct *fp = dirlist->files[j];
552		const char *suf, *name;
553		int len, suf_len;
554		uint32 dist;
555
556		if (!S_ISREG(fp->mode) || !fp->length
557		    || fp->flags & FLAG_NO_FUZZY)
558			continue;
559
560		name = fp->basename;
561
562		if (fp->length == file->length
563		    && cmp_time(fp->modtime, file->modtime) == 0) {
564			if (verbose > 4) {
565				rprintf(FINFO,
566					"fuzzy size/modtime match for %s\n",
567					name);
568			}
569			return j;
570		}
571
572		len = strlen(name);
573		suf = find_filename_suffix(name, len, &suf_len);
574
575		dist = fuzzy_distance(name, len, fname, fname_len);
576		/* Add some extra weight to how well the suffixes match. */
577		dist += fuzzy_distance(suf, suf_len, fname_suf, fname_suf_len)
578		      * 10;
579		if (verbose > 4) {
580			rprintf(FINFO, "fuzzy distance for %s = %d.%05d\n",
581				name, (int)(dist>>16), (int)(dist&0xFFFF));
582		}
583		if (dist <= lowest_dist) {
584			lowest_dist = dist;
585			lowest_j = j;
586		}
587	}
588
589	return lowest_j;
590}
591
592void check_for_finished_hlinks(int itemizing, enum logcode code)
593{
594	struct file_struct *file;
595	int ndx;
596
597	while ((ndx = get_hlink_num()) != -1) {
598		if (ndx < 0 || ndx >= the_file_list->count)
599			continue;
600
601		file = the_file_list->files[ndx];
602		if (!file->link_u.links)
603			continue;
604
605		hard_link_cluster(file, ndx, itemizing, code);
606	}
607}
608
609/* This is only called for regular files.  We return -2 if we've finished
610 * handling the file, -1 if no dest-linking occurred, or a non-negative
611 * value if we found an alternate basis file. */
612static int try_dests_reg(struct file_struct *file, char *fname, int ndx,
613			 char *cmpbuf, STRUCT_STAT *stp, int itemizing,
614			 int maybe_ATTRS_REPORT, enum logcode code)
615{
616	int best_match = -1;
617	int match_level = 0;
618	int j = 0;
619
620	do {
621		pathjoin(cmpbuf, MAXPATHLEN, basis_dir[j], fname);
622		if (link_stat(cmpbuf, stp, 0) < 0 || !S_ISREG(stp->st_mode))
623			continue;
624		switch (match_level) {
625		case 0:
626			best_match = j;
627			match_level = 1;
628			/* FALL THROUGH */
629		case 1:
630			if (!unchanged_file(cmpbuf, file, stp))
631				continue;
632			best_match = j;
633			match_level = 2;
634			/* FALL THROUGH */
635		case 2:
636			if (!unchanged_attrs(file, stp))
637				continue;
638			if (always_checksum && preserve_times
639			 && cmp_time(stp->st_mtime, file->modtime))
640				continue;
641			best_match = j;
642			match_level = 3;
643			break;
644		}
645		break;
646	} while (basis_dir[++j] != NULL);
647
648	if (!match_level)
649		return -1;
650
651	if (j != best_match) {
652		j = best_match;
653		pathjoin(cmpbuf, MAXPATHLEN, basis_dir[j], fname);
654		if (link_stat(cmpbuf, stp, 0) < 0)
655			return -1;
656	}
657
658	if (match_level == 3 && !copy_dest) {
659#ifdef SUPPORT_HARD_LINKS
660		if (link_dest) {
661			if (hard_link_one(file, ndx, fname, 0, stp,
662					  cmpbuf, 1,
663					  itemizing && verbose > 1,
664					  code) < 0)
665				goto try_a_copy;
666			if (preserve_hard_links && file->link_u.links) {
667				if (dry_run)
668					file->link_u.links->link_dest_used = j + 1;
669				hard_link_cluster(file, ndx, itemizing, code);
670			}
671		} else
672#endif
673		if (itemizing)
674			itemize(file, ndx, 0, stp, 0, 0, NULL);
675		if (verbose > 1 && maybe_ATTRS_REPORT) {
676			rprintf(FCLIENT, "%s is uptodate\n", fname);
677		}
678		return -2;
679	}
680
681	if (match_level >= 2) {
682	  try_a_copy: /* Copy the file locally. */
683		if (copy_file(cmpbuf, fname, file->mode) < 0) {
684			if (verbose) {
685				rsyserr(FINFO, errno, "copy_file %s => %s",
686					full_fname(cmpbuf), fname);
687			}
688			return -1;
689		}
690		if (itemizing)
691			itemize(file, ndx, 0, stp, ITEM_LOCAL_CHANGE, 0, NULL);
692		set_file_attrs(fname, file, NULL, 0);
693		if (maybe_ATTRS_REPORT
694		 && ((!itemizing && verbose && match_level == 2)
695		  || (verbose > 1 && match_level == 3))) {
696			code = match_level == 3 ? FCLIENT : FINFO;
697			rprintf(code, "%s%s\n", fname,
698				match_level == 3 ? " is uptodate" : "");
699		}
700		if (preserve_hard_links && file->link_u.links)
701			hard_link_cluster(file, ndx, itemizing, code);
702		return -2;
703	}
704
705	return FNAMECMP_BASIS_DIR_LOW + j;
706}
707
708/* This is only called for non-regular files.  We return -2 if we've finished
709 * handling the file, or -1 if no dest-linking occurred. */
710static int try_dests_non(struct file_struct *file, char *fname, int ndx,
711			 int itemizing, int maybe_ATTRS_REPORT,
712			 enum logcode code)
713{
714	char fnamebuf[MAXPATHLEN];
715	STRUCT_STAT st;
716	int i = 0;
717
718	do {
719		pathjoin(fnamebuf, MAXPATHLEN, basis_dir[i], fname);
720		if (link_stat(fnamebuf, &st, 0) < 0 || S_ISDIR(st.st_mode)
721		 || !unchanged_attrs(file, &st))
722			continue;
723		if (S_ISLNK(file->mode)) {
724#ifdef SUPPORT_LINKS
725			char lnk[MAXPATHLEN];
726			int len;
727			if ((len = readlink(fnamebuf, lnk, MAXPATHLEN-1)) <= 0)
728				continue;
729			lnk[len] = '\0';
730			if (strcmp(lnk, file->u.link) != 0)
731#endif
732				continue;
733		} else if (IS_SPECIAL(file->mode)) {
734			if (!IS_SPECIAL(st.st_mode) || st.st_rdev != file->u.rdev)
735				continue;
736		} else if (IS_DEVICE(file->mode)) {
737			if (!IS_DEVICE(st.st_mode) || st.st_rdev != file->u.rdev)
738				continue;
739		} else {
740			rprintf(FERROR,
741				"internal: try_dests_non() called with invalid mode (%o)\n",
742				(int)file->mode);
743			exit_cleanup(RERR_UNSUPPORTED);
744		}
745#ifdef SUPPORT_HARD_LINKS
746		if (link_dest
747#ifndef CAN_HARDLINK_SYMLINK
748		 && !S_ISLNK(file->mode)
749#endif
750#ifndef CAN_HARDLINK_SPECIAL
751		 && !IS_SPECIAL(file->mode) && !IS_DEVICE(file->mode)
752#endif
753		) {
754			if (do_link(fnamebuf, fname) < 0) {
755				rsyserr(FERROR, errno,
756					"failed to hard-link %s with %s",
757					fnamebuf, fname);
758				break;
759			}
760			if (preserve_hard_links && file->link_u.links)
761				hard_link_cluster(file, ndx, itemizing, code);
762		}
763#endif
764		if (itemizing && stdout_format_has_i && verbose > 1) {
765			int changes = compare_dest ? 0 : ITEM_LOCAL_CHANGE
766				    + (link_dest ? ITEM_XNAME_FOLLOWS : 0);
767			char *lp = link_dest ? "" : NULL;
768			itemize(file, ndx, 0, &st, changes, 0, lp);
769		}
770		if (verbose > 1 && maybe_ATTRS_REPORT) {
771			rprintf(FCLIENT, "%s is uptodate\n", fname);
772		}
773		return -2;
774	} while (basis_dir[++i] != NULL);
775
776	return -1;
777}
778
779static int phase = 0;
780
781/* Acts on the_file_list->file's ndx'th item, whose name is fname.  If a dir,
782 * make sure it exists, and has the right permissions/timestamp info.  For
783 * all other non-regular files (symlinks, etc.) we create them here.  For
784 * regular files that have changed, we try to find a basis file and then
785 * start sending checksums.
786 *
787 * When fname is non-null, it must point to a MAXPATHLEN buffer!
788 *
789 * Note that f_out is set to -1 when doing final directory-permission and
790 * modification-time repair. */
791static void recv_generator(char *fname, struct file_struct *file, int ndx,
792			   int itemizing, int maybe_ATTRS_REPORT,
793			   enum logcode code, int f_out)
794{
795	static int missing_below = -1, excluded_below = -1;
796	static char *parent_dirname = "";
797	static struct file_list *fuzzy_dirlist = NULL;
798	static int need_fuzzy_dirlist = 0;
799	struct file_struct *fuzzy_file = NULL;
800	int fd = -1, f_copy = -1;
801	STRUCT_STAT st, real_st, partial_st;
802	struct file_struct *back_file = NULL;
803	int statret, real_ret, stat_errno;
804	char *fnamecmp, *partialptr, *backupptr = NULL;
805	char fnamecmpbuf[MAXPATHLEN];
806	uchar fnamecmp_type;
807	int del_opts = DEL_TERSE | (delete_mode ? DEL_FORCE_RECURSE : 0);
808
809	if (list_only)
810		return;
811
812	if (!fname) {
813		if (fuzzy_dirlist) {
814			flist_free(fuzzy_dirlist);
815			fuzzy_dirlist = NULL;
816		}
817		if (missing_below >= 0) {
818			if (dry_run)
819				dry_run--;
820			missing_below = -1;
821		}
822		parent_dirname = "";
823		return;
824	}
825
826	if (verbose > 2)
827		rprintf(FINFO, "recv_generator(%s,%d)\n", fname, ndx);
828
829	if (server_filter_list.head) {
830		if (excluded_below >= 0) {
831			if (file->dir.depth > excluded_below)
832				goto skipping;
833			excluded_below = -1;
834		}
835		if (check_filter(&server_filter_list, fname,
836				 S_ISDIR(file->mode)) < 0) {
837			if (S_ISDIR(file->mode))
838				excluded_below = file->dir.depth;
839		  skipping:
840			if (verbose) {
841				rprintf(FINFO,
842					"skipping server-excluded file \"%s\"\n",
843					fname);
844			}
845			return;
846		}
847	}
848
849	if (missing_below >= 0) {
850		if (file->dir.depth <= missing_below) {
851			if (dry_run)
852				dry_run--;
853			missing_below = -1;
854		} else if (!dry_run)
855			return;
856	}
857	if (dry_run > 1) {
858		statret = -1;
859		stat_errno = ENOENT;
860	} else {
861		char *dn = file->dirname ? file->dirname : ".";
862		if (parent_dirname != dn && strcmp(parent_dirname, dn) != 0) {
863			if (relative_paths && !implied_dirs
864			 && do_stat(dn, &st) < 0
865			 && create_directory_path(fname) < 0) {
866				rsyserr(FERROR, errno,
867					"recv_generator: mkdir %s failed",
868					full_fname(dn));
869			}
870			if (fuzzy_dirlist) {
871				flist_free(fuzzy_dirlist);
872				fuzzy_dirlist = NULL;
873			}
874			if (fuzzy_basis)
875				need_fuzzy_dirlist = 1;
876		}
877		parent_dirname = dn;
878
879		if (need_fuzzy_dirlist && S_ISREG(file->mode)) {
880			fuzzy_dirlist = get_dirlist(dn, -1, 1);
881			need_fuzzy_dirlist = 0;
882		}
883
884		statret = link_stat(fname, &st,
885				    keep_dirlinks && S_ISDIR(file->mode));
886		stat_errno = errno;
887	}
888
889	if (ignore_non_existing && statret == -1 && stat_errno == ENOENT) {
890		if (verbose > 1) {
891			rprintf(FINFO, "not creating new %s \"%s\"\n",
892				S_ISDIR(file->mode) ? "directory" : "file",
893				fname);
894		}
895		return;
896	}
897
898	/* If we're not preserving permissions, change the file-list's
899	 * mode based on the local permissions and some heuristics. */
900	if (!preserve_perms) {
901		int exists = statret == 0
902			  && S_ISDIR(st.st_mode) == S_ISDIR(file->mode);
903		file->mode = dest_mode(file->mode, st.st_mode, exists);
904	}
905
906	if (S_ISDIR(file->mode)) {
907		/* The file to be received is a directory, so we need
908		 * to prepare appropriately.  If there is already a
909		 * file of that name and it is *not* a directory, then
910		 * we need to delete it.  If it doesn't exist, then
911		 * (perhaps recursively) create it. */
912		if (statret == 0 && !S_ISDIR(st.st_mode)) {
913			if (delete_item(fname, st.st_mode, del_opts) < 0)
914				return;
915			statret = -1;
916		}
917		if (dry_run && statret != 0 && missing_below < 0) {
918			missing_below = file->dir.depth;
919			dry_run++;
920		}
921		if (itemizing && f_out != -1) {
922			int sr = statret;
923			if (new_root_dir) {
924				if (*fname == '.' && fname[1] == '\0')
925					sr = -1;
926				new_root_dir = 0;
927			}
928			itemize(file, ndx, sr, &st,
929				sr ? ITEM_LOCAL_CHANGE : 0, 0, NULL);
930		}
931		if (statret != 0 && do_mkdir(fname,file->mode) < 0 && errno != EEXIST) {
932			if (!relative_paths || errno != ENOENT
933			    || create_directory_path(fname) < 0
934			    || (do_mkdir(fname, file->mode) < 0 && errno != EEXIST)) {
935				rsyserr(FERROR, errno,
936					"recv_generator: mkdir %s failed",
937					full_fname(fname));
938				file->flags |= FLAG_MISSING;
939				if (ndx+1 < the_file_list->count
940				 && the_file_list->files[ndx+1]->dir.depth > file->dir.depth) {
941					rprintf(FERROR,
942					    "*** Skipping everything below this failed directory ***\n");
943					missing_below = file->dir.depth;
944				}
945				return;
946			}
947		}
948		if (set_file_attrs(fname, file, statret ? NULL : &st, 0)
949		    && verbose && code != FNONE && f_out != -1)
950			rprintf(code, "%s/\n", fname);
951		if (delete_during && f_out != -1 && !phase && dry_run < 2
952		    && (file->flags & FLAG_DEL_HERE))
953			delete_in_dir(the_file_list, fname, file, &st);
954		return;
955	}
956
957	if (preserve_hard_links && file->link_u.links
958	    && hard_link_check(file, ndx, fname, statret, &st,
959			       itemizing, code, HL_CHECK_MASTER))
960		return;
961
962	if (preserve_links && S_ISLNK(file->mode)) {
963#ifdef SUPPORT_LINKS
964		if (safe_symlinks && unsafe_symlink(file->u.link, fname)) {
965			if (verbose) {
966				if (the_file_list->count == 1)
967					fname = f_name(file, NULL);
968				rprintf(FINFO,
969					"ignoring unsafe symlink %s -> \"%s\"\n",
970					full_fname(fname), file->u.link);
971			}
972			return;
973		}
974		if (statret == 0) {
975			char lnk[MAXPATHLEN];
976			int len;
977
978			if (!S_ISDIR(st.st_mode)
979			    && (len = readlink(fname, lnk, MAXPATHLEN-1)) > 0) {
980				lnk[len] = 0;
981				/* A link already pointing to the
982				 * right place -- no further action
983				 * required. */
984				if (strcmp(lnk, file->u.link) == 0) {
985					if (itemizing) {
986						itemize(file, ndx, 0, &st, 0,
987							0, NULL);
988					}
989					set_file_attrs(fname, file, &st,
990						       maybe_ATTRS_REPORT);
991					if (preserve_hard_links
992					    && file->link_u.links) {
993						hard_link_cluster(file, ndx,
994								  itemizing,
995								  code);
996					}
997					if (remove_source_files == 1)
998						goto return_with_success;
999					return;
1000				}
1001			}
1002			/* Not the right symlink (or not a symlink), so
1003			 * delete it. */
1004			if (delete_item(fname, st.st_mode, del_opts) < 0)
1005				return;
1006			if (!S_ISLNK(st.st_mode))
1007				statret = -1;
1008		} else if (basis_dir[0] != NULL) {
1009			if (try_dests_non(file, fname, ndx, itemizing,
1010					  maybe_ATTRS_REPORT, code) == -2) {
1011#ifndef CAN_HARDLINK_SYMLINK
1012				if (link_dest) {
1013					/* Resort to --copy-dest behavior. */
1014				} else
1015#endif
1016				if (!copy_dest)
1017					return;
1018				itemizing = 0;
1019				code = FNONE;
1020			}
1021		}
1022		if (preserve_hard_links && file->link_u.links
1023		    && hard_link_check(file, ndx, fname, -1, &st,
1024				       itemizing, code, HL_SKIP))
1025			return;
1026		if (do_symlink(file->u.link,fname) != 0) {
1027			rsyserr(FERROR, errno, "symlink %s -> \"%s\" failed",
1028				full_fname(fname), file->u.link);
1029		} else {
1030			set_file_attrs(fname, file, NULL, 0);
1031			if (itemizing) {
1032				itemize(file, ndx, statret, &st,
1033					ITEM_LOCAL_CHANGE, 0, NULL);
1034			}
1035			if (code != FNONE && verbose) {
1036				rprintf(code, "%s -> %s\n", fname,
1037					file->u.link);
1038			}
1039			if (preserve_hard_links && file->link_u.links)
1040				hard_link_cluster(file, ndx, itemizing, code);
1041			/* This does not check remove_source_files == 1
1042			 * because this is one of the items that the old
1043			 * --remove-sent-files option would remove. */
1044			if (remove_source_files)
1045				goto return_with_success;
1046		}
1047#endif
1048		return;
1049	}
1050
1051	if ((am_root && preserve_devices && IS_DEVICE(file->mode))
1052	 || (preserve_specials && IS_SPECIAL(file->mode))) {
1053		if (statret != 0 && basis_dir[0] != NULL) {
1054			if (try_dests_non(file, fname, ndx, itemizing,
1055					  maybe_ATTRS_REPORT, code) == -2) {
1056#ifndef CAN_HARDLINK_SPECIAL
1057				if (link_dest) {
1058					/* Resort to --copy-dest behavior. */
1059				} else
1060#endif
1061				if (!copy_dest)
1062					return;
1063				itemizing = 0;
1064				code = FNONE;
1065			}
1066		}
1067		if (statret != 0
1068		 || (st.st_mode & ~CHMOD_BITS) != (file->mode & ~CHMOD_BITS)
1069		 || st.st_rdev != file->u.rdev) {
1070			if (statret == 0
1071			 && delete_item(fname, st.st_mode, del_opts) < 0)
1072				return;
1073			if (preserve_hard_links && file->link_u.links
1074			    && hard_link_check(file, ndx, fname, -1, &st,
1075					       itemizing, code, HL_SKIP))
1076				return;
1077			if ((IS_DEVICE(file->mode) && !IS_DEVICE(st.st_mode))
1078			 || (IS_SPECIAL(file->mode) && !IS_SPECIAL(st.st_mode)))
1079				statret = -1;
1080			if (verbose > 2) {
1081				rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
1082					fname,
1083					(int)file->mode, (int)file->u.rdev);
1084			}
1085			if (do_mknod(fname,file->mode,file->u.rdev) < 0) {
1086				rsyserr(FERROR, errno, "mknod %s failed",
1087					full_fname(fname));
1088			} else {
1089				set_file_attrs(fname, file, NULL, 0);
1090				if (itemizing) {
1091					itemize(file, ndx, statret, &st,
1092						ITEM_LOCAL_CHANGE, 0, NULL);
1093				}
1094				if (code != FNONE && verbose)
1095					rprintf(code, "%s\n", fname);
1096				if (preserve_hard_links && file->link_u.links) {
1097					hard_link_cluster(file, ndx,
1098							  itemizing, code);
1099				}
1100				if (remove_source_files == 1)
1101					goto return_with_success;
1102			}
1103		} else {
1104			if (itemizing)
1105				itemize(file, ndx, statret, &st, 0, 0, NULL);
1106			set_file_attrs(fname, file, &st, maybe_ATTRS_REPORT);
1107			if (preserve_hard_links && file->link_u.links)
1108				hard_link_cluster(file, ndx, itemizing, code);
1109			if (remove_source_files == 1)
1110				goto return_with_success;
1111		}
1112		return;
1113	}
1114
1115	if (!S_ISREG(file->mode)) {
1116		if (the_file_list->count == 1)
1117			fname = f_name(file, NULL);
1118		rprintf(FINFO, "skipping non-regular file \"%s\"\n", fname);
1119		return;
1120	}
1121
1122	if (max_size && file->length > max_size) {
1123		if (verbose > 1) {
1124			if (the_file_list->count == 1)
1125				fname = f_name(file, NULL);
1126			rprintf(FINFO, "%s is over max-size\n", fname);
1127		}
1128		return;
1129	}
1130	if (min_size && file->length < min_size) {
1131		if (verbose > 1) {
1132			if (the_file_list->count == 1)
1133				fname = f_name(file, NULL);
1134			rprintf(FINFO, "%s is under min-size\n", fname);
1135		}
1136		return;
1137	}
1138
1139	if (ignore_existing && statret == 0) {
1140		if (verbose > 1)
1141			rprintf(FINFO, "%s exists\n", fname);
1142		return;
1143	}
1144
1145	if (update_only && statret == 0
1146	    && cmp_time(st.st_mtime, file->modtime) > 0) {
1147		if (verbose > 1)
1148			rprintf(FINFO, "%s is newer\n", fname);
1149		return;
1150	}
1151
1152	fnamecmp = fname;
1153	fnamecmp_type = FNAMECMP_FNAME;
1154#ifdef EA_SUPPORT
1155	if (extended_attributes) {
1156		if (!strncmp("._", file->basename, 2)) {
1157			if (protocol_version >= 29)
1158				itemize(file, ndx, -1, NULL, ITEM_TRANSFER|ITEM_REPORT_ACL|ITEM_REPORT_XATTR, 0, file->basename);
1159			else
1160				write_int(f_out,ndx);
1161			if (!dry_run) {
1162				write_sum_head(f_out, NULL);
1163			}
1164			return;
1165		}
1166	}
1167#endif	/* EA_SUPPORT */
1168	if (statret == 0 && !S_ISREG(st.st_mode)) {
1169		if (delete_item(fname, st.st_mode, del_opts) != 0)
1170			return;
1171		statret = -1;
1172		stat_errno = ENOENT;
1173	}
1174
1175	if (statret != 0 && basis_dir[0] != NULL) {
1176		int j = try_dests_reg(file, fname, ndx, fnamecmpbuf, &st,
1177				      itemizing, maybe_ATTRS_REPORT, code);
1178		if (j == -2) {
1179			if (remove_source_files == 1)
1180				goto return_with_success;
1181			return;
1182		}
1183		if (j >= 0) {
1184			fnamecmp = fnamecmpbuf;
1185			fnamecmp_type = j;
1186			statret = 0;
1187		}
1188	}
1189
1190	real_ret = statret;
1191	real_st = st;
1192
1193	if (partial_dir && (partialptr = partial_dir_fname(fname)) != NULL
1194	    && link_stat(partialptr, &partial_st, 0) == 0
1195	    && S_ISREG(partial_st.st_mode)) {
1196		if (statret != 0)
1197			goto prepare_to_open;
1198	} else
1199		partialptr = NULL;
1200
1201	if (statret != 0 && fuzzy_dirlist && dry_run <= 1) {
1202		int j = find_fuzzy(file, fuzzy_dirlist);
1203		if (j >= 0) {
1204			fuzzy_file = fuzzy_dirlist->files[j];
1205			f_name(fuzzy_file, fnamecmpbuf);
1206			if (verbose > 2) {
1207				rprintf(FINFO, "fuzzy basis selected for %s: %s\n",
1208					fname, fnamecmpbuf);
1209			}
1210			st.st_size = fuzzy_file->length;
1211			statret = 0;
1212			fnamecmp = fnamecmpbuf;
1213			fnamecmp_type = FNAMECMP_FUZZY;
1214		}
1215	}
1216
1217	if (statret != 0) {
1218		if (preserve_hard_links && file->link_u.links
1219		    && hard_link_check(file, ndx, fname, statret, &st,
1220				       itemizing, code, HL_SKIP))
1221			return;
1222		if (stat_errno == ENOENT)
1223			goto notify_others;
1224		rsyserr(FERROR, stat_errno, "recv_generator: failed to stat %s",
1225			full_fname(fname));
1226		return;
1227	}
1228
1229	if (append_mode && st.st_size > file->length)
1230		return;
1231
1232	if (fnamecmp_type <= FNAMECMP_BASIS_DIR_HIGH)
1233		;
1234	else if (fnamecmp_type == FNAMECMP_FUZZY)
1235		;
1236	else if (unchanged_file(fnamecmp, file, &st)) {
1237		if (partialptr) {
1238			do_unlink(partialptr);
1239			handle_partial_dir(partialptr, PDIR_DELETE);
1240		}
1241		if (itemizing) {
1242			itemize(file, ndx, real_ret, &real_st,
1243				0, 0, NULL);
1244		}
1245		set_file_attrs(fname, file, &st, maybe_ATTRS_REPORT);
1246		if (preserve_hard_links && file->link_u.links)
1247			hard_link_cluster(file, ndx, itemizing, code);
1248		if (remove_source_files != 1)
1249			return;
1250	  return_with_success:
1251		if (!dry_run) {
1252			char numbuf[4];
1253			SIVAL(numbuf, 0, ndx);
1254			send_msg(MSG_SUCCESS, numbuf, 4);
1255		}
1256		return;
1257	}
1258
1259  prepare_to_open:
1260	if (partialptr) {
1261		st = partial_st;
1262		fnamecmp = partialptr;
1263		fnamecmp_type = FNAMECMP_PARTIAL_DIR;
1264		statret = 0;
1265	}
1266
1267	if (!do_xfers || read_batch || whole_file)
1268		goto notify_others;
1269
1270	if (fuzzy_dirlist) {
1271		int j = flist_find(fuzzy_dirlist, file);
1272		if (j >= 0) /* don't use changing file as future fuzzy basis */
1273			fuzzy_dirlist->files[j]->flags |= FLAG_NO_FUZZY;
1274	}
1275
1276	/* open the file */
1277	fd = do_open(fnamecmp, O_RDONLY, 0);
1278
1279	if (fd == -1) {
1280		rsyserr(FERROR, errno, "failed to open %s, continuing",
1281			full_fname(fnamecmp));
1282	  pretend_missing:
1283		/* pretend the file didn't exist */
1284		if (preserve_hard_links && file->link_u.links
1285		    && hard_link_check(file, ndx, fname, statret, &st,
1286				       itemizing, code, HL_SKIP))
1287			return;
1288		statret = real_ret = -1;
1289		goto notify_others;
1290	}
1291
1292	if (inplace && make_backups && fnamecmp_type == FNAMECMP_FNAME) {
1293		if (!(backupptr = get_backup_name(fname))) {
1294			close(fd);
1295			return;
1296		}
1297		if (!(back_file = make_file(fname, NULL, NULL, 0, NO_FILTERS))) {
1298			close(fd);
1299			goto pretend_missing;
1300		}
1301		if (robust_unlink(backupptr) && errno != ENOENT) {
1302			rsyserr(FERROR, errno, "unlink %s",
1303				full_fname(backupptr));
1304			free(back_file);
1305			close(fd);
1306			return;
1307		}
1308		if ((f_copy = do_open(backupptr,
1309		    O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600)) < 0) {
1310			rsyserr(FERROR, errno, "open %s",
1311				full_fname(backupptr));
1312			free(back_file);
1313			close(fd);
1314			return;
1315		}
1316		fnamecmp_type = FNAMECMP_BACKUP;
1317	}
1318
1319	if (verbose > 3) {
1320		rprintf(FINFO, "gen mapped %s of size %.0f\n",
1321			fnamecmp, (double)st.st_size);
1322	}
1323
1324	if (verbose > 2)
1325		rprintf(FINFO, "generating and sending sums for %d\n", ndx);
1326
1327  notify_others:
1328	if (remove_source_files && !delay_updates && !phase)
1329		increment_active_files(ndx, itemizing, code);
1330	write_int(f_out, ndx);
1331	if (itemizing) {
1332		int iflags = ITEM_TRANSFER;
1333		if (always_checksum)
1334			iflags |= ITEM_REPORT_CHECKSUM;
1335		if (fnamecmp_type != FNAMECMP_FNAME)
1336			iflags |= ITEM_BASIS_TYPE_FOLLOWS;
1337		if (fnamecmp_type == FNAMECMP_FUZZY)
1338			iflags |= ITEM_XNAME_FOLLOWS;
1339		itemize(file, -1, real_ret, &real_st, iflags, fnamecmp_type,
1340			fuzzy_file ? fuzzy_file->basename : NULL);
1341	}
1342
1343	if (!do_xfers) {
1344		if (preserve_hard_links && file->link_u.links)
1345			hard_link_cluster(file, ndx, itemizing, code);
1346		return;
1347	}
1348	if (read_batch)
1349		return;
1350
1351	if (statret != 0 || whole_file) {
1352		write_sum_head(f_out, NULL);
1353		return;
1354	}
1355
1356	generate_and_send_sums(fd, st.st_size, f_out, f_copy);
1357
1358	if (f_copy >= 0) {
1359		close(f_copy);
1360		set_file_attrs(backupptr, back_file, NULL, 0);
1361		if (verbose > 1) {
1362			rprintf(FINFO, "backed up %s to %s\n",
1363				fname, backupptr);
1364		}
1365		free(back_file);
1366	}
1367
1368	close(fd);
1369}
1370
1371void generate_files(int f_out, struct file_list *flist, char *local_name)
1372{
1373	int i;
1374	char fbuf[MAXPATHLEN];
1375	int itemizing, maybe_ATTRS_REPORT;
1376	enum logcode code;
1377	int lull_mod = allowed_lull * 5;
1378	int need_retouch_dir_times = preserve_times && !omit_dir_times;
1379	int need_retouch_dir_perms = 0;
1380	int save_ignore_existing = ignore_existing;
1381	int save_ignore_non_existing = ignore_non_existing;
1382	int save_do_progress = do_progress;
1383	int save_make_backups = make_backups;
1384	int dir_tweaking = !(list_only || local_name || dry_run);
1385#ifdef HAVE_COPYFILE
1386	int *ea_map = NULL;
1387	int ea_saved = -1;
1388#endif
1389
1390	if (protocol_version >= 29) {
1391		itemizing = 1;
1392		maybe_ATTRS_REPORT = stdout_format_has_i ? 0 : ATTRS_REPORT;
1393		code = logfile_format_has_i ? FNONE : FLOG;
1394	} else if (am_daemon) {
1395		itemizing = logfile_format_has_i && do_xfers;
1396		maybe_ATTRS_REPORT = ATTRS_REPORT;
1397		code = itemizing || !do_xfers ? FCLIENT : FINFO;
1398	} else if (!am_server) {
1399		itemizing = stdout_format_has_i;
1400		maybe_ATTRS_REPORT = stdout_format_has_i ? 0 : ATTRS_REPORT;
1401		code = itemizing ? FNONE : FINFO;
1402	} else {
1403		itemizing = 0;
1404		maybe_ATTRS_REPORT = ATTRS_REPORT;
1405		code = FINFO;
1406	}
1407
1408	if (verbose > 2) {
1409		rprintf(FINFO, "generator starting pid=%ld count=%d\n",
1410			(long)getpid(), flist->count);
1411	}
1412
1413	if (delete_before && !local_name && flist->count > 0)
1414		do_delete_pass(flist);
1415	do_progress = 0;
1416
1417	if (append_mode || whole_file < 0)
1418		whole_file = 0;
1419	if (verbose >= 2) {
1420		rprintf(FINFO, "delta-transmission %s\n",
1421			whole_file
1422			? "disabled for local transfer or --whole-file"
1423			: "enabled");
1424	}
1425
1426	/* Since we often fill up the outgoing socket and then just sit around
1427	 * waiting for the other 2 processes to do their thing, we don't want
1428	 * to exit on a timeout.  If the data stops flowing, the receiver will
1429	 * notice that and let us know via the redo pipe (or its closing). */
1430	ignore_timeout = 1;
1431
1432#ifdef HAVE_COPYFILE
1433	/* APPLE: extended attribute files (._foo) need to be transferred
1434	 * after the corresponding file (foo).  This creates a map the size
1435	 * of flist with the number of the file that preempts the current
1436	 * file's delivery.  Set to -1 if there's nothing to do.
1437	 */
1438	if (extended_attributes) {
1439	    int j;
1440	    struct file_struct *file2;
1441	    struct file_struct *file3;
1442
1443	    if (verbose > 3)
1444		rprintf(FINFO,"initializing extended attribute map\n");
1445	    ea_map = malloc(sizeof(int)*flist->count);
1446	    if (!ea_map)
1447 		    out_of_memory("extended attribute map");
1448	    for (i = 0; i < flist->count; ++i) {
1449		ea_map[i] = -1;
1450		file2 = flist->files[i];
1451
1452		if (!file2->basename || strncmp(file2->basename, "._", 2))
1453		    continue;
1454
1455		for (j = i; j < flist->count; ++j) {
1456 		    file3 = flist->files[j];
1457
1458		    if (!file3->basename)
1459 			continue;
1460
1461		    if(!(file2->dirname || file3->dirname)
1462			|| (file2->dirname && file3->dirname &&
1463			!strcmp(file3->dirname, file2->dirname))) {
1464
1465			if(!strcmp(file3->basename, file2->basename + 2)) {
1466			    ea_map[i] = j;
1467
1468			    if (verbose > 4)
1469				rprintf(FINFO,"mapped %s/%s (%d) -> %s/%s (%d)\n",
1470 					(file2->dirname) ? file2->dirname : ".",
1471					file2->basename, i,
1472					(file2->dirname) ? file2->dirname : ".",
1473					file3->basename, j);
1474			    break;
1475			}
1476		    }
1477		}
1478 	    }
1479	}
1480#endif
1481
1482	for (i = 0; i < flist->count; i++) {
1483		struct file_struct *file = flist->files[i];
1484
1485#ifdef HAVE_COPYFILE
1486		if (extended_attributes) {
1487		    if(ea_map[i] < -1)
1488			continue;
1489
1490		    if(ea_map[i] > 0) {
1491			/* save the current index and set it to the
1492			 * file to skip to
1493			 */
1494			if (verbose > 4)
1495			    rprintf(FINFO,"skipping from %d to %d\n", i, ea_map[i]);
1496
1497			ea_saved = i;
1498			i = ea_map[i];
1499			ea_map[i] = -1;
1500		    }
1501next:		    file = flist->files[i];
1502		}
1503#endif
1504
1505		if (!file->basename)
1506			continue;
1507
1508		if (local_name)
1509			strlcpy(fbuf, local_name, sizeof fbuf);
1510		else
1511			f_name(file, fbuf);
1512		recv_generator(fbuf, file, i, itemizing, maybe_ATTRS_REPORT,
1513			       code, f_out);
1514
1515#ifdef HAVE_COPYFILE
1516		if (extended_attributes) {
1517		    if(ea_saved > -1) {
1518			ea_map[i] = -2;
1519			i = ea_saved;
1520			ea_saved = -1;
1521			goto next;
1522		    }
1523		}
1524#endif
1525		/* We need to ensure that any dirs we create have writeable
1526		 * permissions during the time we are putting files within
1527		 * them.  This is then fixed after the transfer is done. */
1528#ifdef HAVE_CHMOD
1529		if (!am_root && S_ISDIR(file->mode) && !(file->mode & S_IWUSR)
1530		 && dir_tweaking) {
1531			mode_t mode = file->mode | S_IWUSR; /* user write */
1532			char *fname = local_name ? local_name : fbuf;
1533			if (do_chmod(fname, mode) < 0) {
1534				rsyserr(FERROR, errno,
1535					"failed to modify permissions on %s",
1536					full_fname(fname));
1537			}
1538			need_retouch_dir_perms = 1;
1539		}
1540#endif
1541
1542		if (preserve_hard_links)
1543			check_for_finished_hlinks(itemizing, code);
1544
1545		if (allowed_lull && !(i % lull_mod))
1546			maybe_send_keepalive();
1547		else if (!(i % 200))
1548			maybe_flush_socket();
1549	}
1550	recv_generator(NULL, NULL, 0, 0, 0, code, -1);
1551	if (delete_during)
1552		delete_in_dir(NULL, NULL, NULL, NULL);
1553
1554	phase++;
1555	csum_length = SUM_LENGTH;
1556	max_size = min_size = ignore_existing = ignore_non_existing = 0;
1557	update_only = always_checksum = size_only = 0;
1558	ignore_times = 1;
1559	if (append_mode)  /* resend w/o append mode */
1560		append_mode = -1; /* ... but only longer files */
1561	make_backups = 0; /* avoid a duplicate backup for inplace processing */
1562
1563	if (verbose > 2)
1564		rprintf(FINFO,"generate_files phase=%d\n",phase);
1565
1566	write_int(f_out, -1);
1567
1568	/* files can cycle through the system more than once
1569	 * to catch initial checksum errors */
1570	while ((i = get_redo_num(itemizing, code)) != -1) {
1571		struct file_struct *file = flist->files[i];
1572		if (local_name)
1573			strlcpy(fbuf, local_name, sizeof fbuf);
1574		else
1575			f_name(file, fbuf);
1576		recv_generator(fbuf, file, i, itemizing, maybe_ATTRS_REPORT,
1577			       code, f_out);
1578	}
1579
1580	phase++;
1581	ignore_non_existing = save_ignore_non_existing;
1582	ignore_existing = save_ignore_existing;
1583	make_backups = save_make_backups;
1584
1585	if (verbose > 2)
1586		rprintf(FINFO,"generate_files phase=%d\n",phase);
1587
1588	write_int(f_out, -1);
1589	/* Reduce round-trip lag-time for a useless delay-updates phase. */
1590	if (protocol_version >= 29 && !delay_updates)
1591		write_int(f_out, -1);
1592
1593	/* Read MSG_DONE for the redo phase (and any prior messages). */
1594	get_redo_num(itemizing, code);
1595
1596	if (protocol_version >= 29) {
1597		phase++;
1598		if (verbose > 2)
1599			rprintf(FINFO, "generate_files phase=%d\n", phase);
1600		if (delay_updates)
1601			write_int(f_out, -1);
1602		/* Read MSG_DONE for delay-updates phase & prior messages. */
1603		get_redo_num(itemizing, code);
1604	}
1605
1606	do_progress = save_do_progress;
1607	if (delete_after && !local_name && flist->count > 0)
1608		do_delete_pass(flist);
1609
1610	if ((need_retouch_dir_perms || need_retouch_dir_times) && dir_tweaking) {
1611		int j = 0;
1612		/* Now we need to fix any directory permissions that were
1613		 * modified during the transfer and/or re-set any tweaked
1614		 * modified-time values. */
1615		for (i = 0; i < flist->count; i++) {
1616			struct file_struct *file = flist->files[i];
1617
1618			if (!file->basename || !S_ISDIR(file->mode))
1619				continue;
1620			if (!need_retouch_dir_times && file->mode & S_IWUSR)
1621				continue;
1622			if (file->flags & FLAG_MISSING) {
1623				int missing = file->dir.depth;
1624				while (++i < flist->count) {
1625					file = flist->files[i];
1626					if (file->dir.depth <= missing)
1627						break;
1628				}
1629				i--;
1630				continue;
1631			}
1632			recv_generator(f_name(file, NULL), file, i, itemizing,
1633				       maybe_ATTRS_REPORT, code, -1);
1634			if (allowed_lull && !(++j % lull_mod))
1635				maybe_send_keepalive();
1636			else if (!(j % 200))
1637				maybe_flush_socket();
1638		}
1639	}
1640	recv_generator(NULL, NULL, 0, 0, 0, code, -1);
1641
1642	if (max_delete > 0 && deletion_count > max_delete) {
1643		rprintf(FINFO,
1644			"Deletions stopped due to --max-delete limit (%d skipped)\n",
1645			deletion_count - max_delete);
1646		io_error |= IOERR_DEL_LIMIT;
1647	}
1648
1649	if (verbose > 2)
1650		rprintf(FINFO,"generate_files finished\n");
1651#ifdef HAVE_COPYFILE
1652	if (ea_map)
1653		free(ea_map);
1654#endif
1655}
1656