flist.c revision 1.23
1/* $Id: flist.c,v 1.23 2019/03/31 09:26:05 deraadt Exp $ */ 2/* 3 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv> 4 * Copyright (c) 2019 Florian Obser <florian@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18#include <sys/param.h> 19#include <sys/stat.h> 20 21#include <assert.h> 22#include <errno.h> 23#include <fcntl.h> 24#include <fts.h> 25#include <inttypes.h> 26#include <search.h> 27#include <stdio.h> 28#include <stdlib.h> 29#include <string.h> 30#include <unistd.h> 31 32#include "extern.h" 33 34/* 35 * We allocate our file list in chunk sizes so as not to do it one by 36 * one. 37 * Preferrably we get one or two allocation. 38 */ 39#define FLIST_CHUNK_SIZE (1024) 40 41/* 42 * These flags are part of the rsync protocol. 43 * They are sent as the first byte for a file transmission and encode 44 * information that affects subsequent transmissions. 45 */ 46#define FLIST_TOP_LEVEL 0x0001 /* needed for remote --delete */ 47#define FLIST_MODE_SAME 0x0002 /* mode is repeat */ 48#define FLIST_RDEV_SAME 0x0004 /* rdev is repeat */ 49#define FLIST_UID_SAME 0x0008 /* uid is repeat */ 50#define FLIST_GID_SAME 0x0010 /* gid is repeat */ 51#define FLIST_NAME_SAME 0x0020 /* name is repeat */ 52#define FLIST_NAME_LONG 0x0040 /* name >255 bytes */ 53#define FLIST_TIME_SAME 0x0080 /* time is repeat */ 54 55/* 56 * Requied way to sort a filename list. 57 */ 58static int 59flist_cmp(const void *p1, const void *p2) 60{ 61 const struct flist *f1 = p1, *f2 = p2; 62 63 return strcmp(f1->wpath, f2->wpath); 64} 65 66/* 67 * Deduplicate our file list (which may be zero-length). 68 * Returns zero on failure, non-zero on success. 69 */ 70static int 71flist_dedupe(struct sess *sess, struct flist **fl, size_t *sz) 72{ 73 size_t i, j; 74 struct flist *new; 75 struct flist *f, *fnext; 76 77 if (*sz == 0) 78 return 1; 79 80 /* Create a new buffer, "new", and copy. */ 81 82 new = calloc(*sz, sizeof(struct flist)); 83 if (new == NULL) { 84 ERR(sess, "calloc"); 85 return 0; 86 } 87 88 for (i = j = 0; i < *sz - 1; i++) { 89 f = &(*fl)[i]; 90 fnext = &(*fl)[i + 1]; 91 92 if (strcmp(f->wpath, fnext->wpath)) { 93 new[j++] = *f; 94 continue; 95 } 96 97 /* 98 * Our working (destination) paths are the same. 99 * If the actual file is the same (as given on the 100 * command-line), then we can just discard the first. 101 * Otherwise, we need to bail out: it means we have two 102 * different files with the relative path on the 103 * destination side. 104 */ 105 106 if (strcmp(f->path, fnext->path) == 0) { 107 new[j++] = *f; 108 i++; 109 WARNX(sess, "%s: duplicate path: %s", 110 f->wpath, f->path); 111 free(fnext->path); 112 free(fnext->link); 113 fnext->path = fnext->link = NULL; 114 continue; 115 } 116 117 ERRX(sess, "%s: duplicate working path for " 118 "possibly different file: %s, %s", 119 f->wpath, f->path, fnext->path); 120 free(new); 121 return 0; 122 } 123 124 /* Don't forget the last entry. */ 125 126 if (i == *sz - 1) 127 new[j++] = (*fl)[i]; 128 129 /* 130 * Reassign to the deduplicated array. 131 * If we started out with *sz > 0, which we check for at the 132 * beginning, then we'll always continue having *sz > 0. 133 */ 134 135 free(*fl); 136 *fl = new; 137 *sz = j; 138 assert(*sz); 139 return 1; 140} 141 142/* 143 * We're now going to find our top-level directories. 144 * This only applies to recursive mode. 145 * If we have the first element as the ".", then that's the "top 146 * directory" of our transfer. 147 * Otherwise, mark up all top-level directories in the set. 148 * XXX: the FLIST_TOP_LEVEL flag should indicate what is and what isn't 149 * a top-level directory, but I'm not sure if GPL rsync(1) respects it 150 * the same way. 151 */ 152static void 153flist_topdirs(struct sess *sess, struct flist *fl, size_t flsz) 154{ 155 size_t i; 156 const char *cp; 157 158 if (!sess->opts->recursive) 159 return; 160 161 if (flsz && strcmp(fl[0].wpath, ".")) { 162 for (i = 0; i < flsz; i++) { 163 if (!S_ISDIR(fl[i].st.mode)) 164 continue; 165 cp = strchr(fl[i].wpath, '/'); 166 if (cp != NULL && cp[1] != '\0') 167 continue; 168 fl[i].st.flags |= FLSTAT_TOP_DIR; 169 LOG4(sess, "%s: top-level", fl[i].wpath); 170 } 171 } else if (flsz) { 172 fl[0].st.flags |= FLSTAT_TOP_DIR; 173 LOG4(sess, "%s: top-level", fl[0].wpath); 174 } 175} 176 177/* 178 * Filter through the fts() file information. 179 * We want directories (pre-order), regular files, and symlinks. 180 * Everything else is skipped and possibly warned about. 181 * Return zero to skip, non-zero to examine. 182 */ 183static int 184flist_fts_check(struct sess *sess, FTSENT *ent) 185{ 186 187 if (ent->fts_info == FTS_F || 188 ent->fts_info == FTS_D || 189 ent->fts_info == FTS_SL || 190 ent->fts_info == FTS_SLNONE) 191 return 1; 192 193 if (ent->fts_info == FTS_DC) { 194 WARNX(sess, "%s: directory cycle", ent->fts_path); 195 } else if (ent->fts_info == FTS_DNR) { 196 errno = ent->fts_errno; 197 WARN(sess, "%s: unreadable directory", ent->fts_path); 198 } else if (ent->fts_info == FTS_DOT) { 199 WARNX(sess, "%s: skipping dot-file", ent->fts_path); 200 } else if (ent->fts_info == FTS_ERR) { 201 errno = ent->fts_errno; 202 WARN(sess, "%s", ent->fts_path); 203 } else if (ent->fts_info == FTS_DEFAULT) { 204 if ((sess->opts->devices && (S_ISBLK(ent->fts_statp->st_mode) || 205 S_ISCHR(ent->fts_statp->st_mode))) || 206 (sess->opts->specials && 207 (S_ISFIFO(ent->fts_statp->st_mode) || 208 S_ISSOCK(ent->fts_statp->st_mode)))) { 209 return 1; 210 } 211 WARNX(sess, "%s: skipping special", ent->fts_path); 212 } else if (ent->fts_info == FTS_NS) { 213 errno = ent->fts_errno; 214 WARN(sess, "%s: could not stat", ent->fts_path); 215 } 216 217 return 0; 218} 219 220/* 221 * Copy necessary elements in "st" into the fields of "f". 222 */ 223static void 224flist_copy_stat(struct flist *f, const struct stat *st) 225{ 226 f->st.mode = st->st_mode; 227 f->st.uid = st->st_uid; 228 f->st.gid = st->st_gid; 229 f->st.size = st->st_size; 230 f->st.mtime = st->st_mtime; 231 f->st.rdev = st->st_rdev; 232} 233 234void 235flist_free(struct flist *f, size_t sz) 236{ 237 size_t i; 238 239 if (f == NULL) 240 return; 241 242 for (i = 0; i < sz; i++) { 243 free(f[i].path); 244 free(f[i].link); 245 } 246 free(f); 247} 248 249/* 250 * Serialise our file list (which may be zero-length) to the wire. 251 * Makes sure that the receiver isn't going to block on sending us 252 * return messages on the log channel. 253 * Return zero on failure, non-zero on success. 254 */ 255int 256flist_send(struct sess *sess, int fdin, int fdout, const struct flist *fl, 257 size_t flsz) 258{ 259 size_t i, sz, gidsz = 0, uidsz = 0; 260 uint8_t flag; 261 const struct flist *f; 262 const char *fn; 263 struct ident *gids = NULL, *uids = NULL; 264 int rc = 0; 265 266 /* Double-check that we've no pending multiplexed data. */ 267 268 LOG2(sess, "sending file metadata list: %zu", flsz); 269 270 for (i = 0; i < flsz; i++) { 271 f = &fl[i]; 272 fn = f->wpath; 273 sz = strlen(f->wpath); 274 assert(sz > 0); 275 assert(sz < INT32_MAX); 276 277 /* 278 * If applicable, unclog the read buffer. 279 * This happens when the receiver has a lot of log 280 * messages and all we're doing is sending our file list 281 * without checking for messages. 282 */ 283 284 if (sess->mplex_reads && 285 io_read_check(sess, fdin) && 286 !io_read_flush(sess, fdin)) { 287 ERRX1(sess, "io_read_flush"); 288 goto out; 289 } 290 291 /* 292 * For ease, make all of our filenames be "long" 293 * regardless their actual length. 294 * This also makes sure that we don't transmit a zero 295 * byte unintentionally. 296 */ 297 298 flag = FLIST_NAME_LONG; 299 if ((FLSTAT_TOP_DIR & f->st.flags)) 300 flag |= FLIST_TOP_LEVEL; 301 302 LOG3(sess, "%s: sending file metadata: " 303 "size %jd, mtime %jd, mode %o", 304 fn, (intmax_t)f->st.size, 305 (intmax_t)f->st.mtime, f->st.mode); 306 307 /* Now write to the wire. */ 308 /* FIXME: buffer this. */ 309 310 if (!io_write_byte(sess, fdout, flag)) { 311 ERRX1(sess, "io_write_byte"); 312 goto out; 313 } else if (!io_write_int(sess, fdout, sz)) { 314 ERRX1(sess, "io_write_int"); 315 goto out; 316 } else if (!io_write_buf(sess, fdout, fn, sz)) { 317 ERRX1(sess, "io_write_buf"); 318 goto out; 319 } else if (!io_write_long(sess, fdout, f->st.size)) { 320 ERRX1(sess, "io_write_long"); 321 goto out; 322 } else if (!io_write_uint(sess, fdout, (uint32_t)f->st.mtime)) { 323 ERRX1(sess, "io_write_uint"); 324 goto out; 325 } else if (!io_write_uint(sess, fdout, f->st.mode)) { 326 ERRX1(sess, "io_write_uint"); 327 goto out; 328 } 329 330 /* Conditional part: uid. */ 331 332 if (sess->opts->preserve_uids) { 333 if (!io_write_uint(sess, fdout, f->st.uid)) { 334 ERRX1(sess, "io_write_uint"); 335 goto out; 336 } 337 if (!idents_add(sess, 0, &uids, &uidsz, f->st.uid)) { 338 ERRX1(sess, "idents_add"); 339 goto out; 340 } 341 } 342 343 /* Conditional part: gid. */ 344 345 if (sess->opts->preserve_gids) { 346 if (!io_write_uint(sess, fdout, f->st.gid)) { 347 ERRX1(sess, "io_write_uint"); 348 goto out; 349 } 350 if (!idents_add(sess, 1, &gids, &gidsz, f->st.gid)) { 351 ERRX1(sess, "idents_add"); 352 goto out; 353 } 354 } 355 356 /* Conditional part: devices & special files. */ 357 358 if ((sess->opts->devices && (S_ISBLK(f->st.mode) || 359 S_ISCHR(f->st.mode))) || 360 (sess->opts->specials && (S_ISFIFO(f->st.mode) || 361 S_ISSOCK(f->st.mode)))) { 362 if (!io_write_int(sess, fdout, f->st.rdev)) { 363 ERRX1(sess, "io_write_int"); 364 goto out; 365 } 366 } 367 368 /* Conditional part: link. */ 369 370 if (S_ISLNK(f->st.mode) && 371 sess->opts->preserve_links) { 372 fn = f->link; 373 sz = strlen(f->link); 374 assert(sz < INT32_MAX); 375 if (!io_write_int(sess, fdout, sz)) { 376 ERRX1(sess, "io_write_int"); 377 goto out; 378 } 379 if (!io_write_buf(sess, fdout, fn, sz)) { 380 ERRX1(sess, "io_write_buf"); 381 goto out; 382 } 383 } 384 385 if (S_ISREG(f->st.mode)) 386 sess->total_size += f->st.size; 387 } 388 389 /* Signal end of file list. */ 390 391 if (!io_write_byte(sess, fdout, 0)) { 392 ERRX1(sess, "io_write_byte"); 393 goto out; 394 } 395 396 /* Conditionally write identifier lists. */ 397 398 if (sess->opts->preserve_uids && !sess->opts->numeric_ids) { 399 LOG2(sess, "sending uid list: %zu", uidsz); 400 if (!idents_send(sess, fdout, uids, uidsz)) { 401 ERRX1(sess, "idents_send"); 402 goto out; 403 } 404 } 405 406 if (sess->opts->preserve_gids && !sess->opts->numeric_ids) { 407 LOG2(sess, "sending gid list: %zu", gidsz); 408 if (!idents_send(sess, fdout, gids, gidsz)) { 409 ERRX1(sess, "idents_send"); 410 goto out; 411 } 412 } 413 414 rc = 1; 415out: 416 idents_free(gids, gidsz); 417 idents_free(uids, uidsz); 418 return rc; 419} 420 421/* 422 * Read the filename of a file list. 423 * This is the most expensive part of the file list transfer, so a lot 424 * of attention has gone into transmitting as little as possible. 425 * Micro-optimisation, but whatever. 426 * Fills in "f" with the full path on success. 427 * Returns zero on failure, non-zero on success. 428 */ 429static int 430flist_recv_name(struct sess *sess, int fd, struct flist *f, uint8_t flags, 431 char last[MAXPATHLEN]) 432{ 433 uint8_t bval; 434 size_t partial = 0; 435 size_t pathlen = 0, len; 436 437 /* 438 * Read our filename. 439 * If we have FLIST_NAME_SAME, we inherit some of the last 440 * transmitted name. 441 * If we have FLIST_NAME_LONG, then the string length is greater 442 * than byte-size. 443 */ 444 445 if (FLIST_NAME_SAME & flags) { 446 if (!io_read_byte(sess, fd, &bval)) { 447 ERRX1(sess, "io_read_byte"); 448 return 0; 449 } 450 partial = bval; 451 } 452 453 /* Get the (possibly-remaining) filename length. */ 454 455 if (FLIST_NAME_LONG & flags) { 456 if (!io_read_size(sess, fd, &pathlen)) { 457 ERRX1(sess, "io_read_size"); 458 return 0; 459 } 460 } else { 461 if (!io_read_byte(sess, fd, &bval)) { 462 ERRX1(sess, "io_read_byte"); 463 return 0; 464 } 465 pathlen = bval; 466 } 467 468 /* Allocate our full filename length. */ 469 /* FIXME: maximum pathname length. */ 470 471 if ((len = pathlen + partial) == 0) { 472 ERRX(sess, "security violation: zero-length pathname"); 473 return 0; 474 } 475 476 if ((f->path = malloc(len + 1)) == NULL) { 477 ERR(sess, "malloc"); 478 return 0; 479 } 480 f->path[len] = '\0'; 481 482 if (FLIST_NAME_SAME & flags) 483 memcpy(f->path, last, partial); 484 485 if (!io_read_buf(sess, fd, f->path + partial, pathlen)) { 486 ERRX1(sess, "io_read_buf"); 487 return 0; 488 } 489 490 if (f->path[0] == '/') { 491 ERRX(sess, "security violation: absolute pathname: %s", 492 f->path); 493 return 0; 494 } 495 496 if (strstr(f->path, "/../") != NULL || 497 (len > 2 && strcmp(f->path + len - 3, "/..") == 0) || 498 (len > 2 && strncmp(f->path, "../", 3) == 0) || 499 strcmp(f->path, "..") == 0) { 500 ERRX(sess, "%s: security violation: backtracking pathname", 501 f->path); 502 return 0; 503 } 504 505 /* Record our last path and construct our filename. */ 506 507 strlcpy(last, f->path, MAXPATHLEN); 508 f->wpath = f->path; 509 return 1; 510} 511 512/* 513 * Reallocate a file list in chunks of FLIST_CHUNK_SIZE; 514 * Returns zero on failure, non-zero on success. 515 */ 516static int 517flist_realloc(struct sess *sess, struct flist **fl, size_t *sz, size_t *max) 518{ 519 void *pp; 520 521 if (*sz + 1 <= *max) { 522 (*sz)++; 523 return 1; 524 } 525 526 pp = recallocarray(*fl, *max, 527 *max + FLIST_CHUNK_SIZE, sizeof(struct flist)); 528 if (pp == NULL) { 529 ERR(sess, "recallocarray"); 530 return 0; 531 } 532 *fl = pp; 533 *max += FLIST_CHUNK_SIZE; 534 (*sz)++; 535 return 1; 536} 537 538/* 539 * Copy a regular or symbolic link file "path" into "f". 540 * This handles the correct path creation and symbolic linking. 541 * Returns zero on failure, non-zero on success. 542 */ 543static int 544flist_append(struct sess *sess, struct flist *f, struct stat *st, 545 const char *path) 546{ 547 548 /* 549 * Copy the full path for local addressing and transmit 550 * only the filename part for the receiver. 551 */ 552 553 if ((f->path = strdup(path)) == NULL) { 554 ERR(sess, "strdup"); 555 return 0; 556 } 557 558 if ((f->wpath = strrchr(f->path, '/')) == NULL) 559 f->wpath = f->path; 560 else 561 f->wpath++; 562 563 /* 564 * On the receiving end, we'll strip out all bits on the 565 * mode except for the file permissions. 566 * No need to warn about it here. 567 */ 568 569 flist_copy_stat(f, st); 570 571 /* Optionally copy link information. */ 572 573 if (S_ISLNK(st->st_mode)) { 574 f->link = symlink_read(sess, f->path); 575 if (f->link == NULL) { 576 ERRX1(sess, "symlink_read"); 577 return 0; 578 } 579 } 580 581 return 1; 582} 583 584/* 585 * Receive a file list from the wire, filling in length "sz" (which may 586 * possibly be zero) and list "flp" on success. 587 * Return zero on failure, non-zero on success. 588 */ 589int 590flist_recv(struct sess *sess, int fd, struct flist **flp, size_t *sz) 591{ 592 struct flist *fl = NULL; 593 struct flist *ff; 594 const struct flist *fflast = NULL; 595 size_t flsz = 0, flmax = 0, lsz, gidsz = 0, uidsz = 0; 596 uint8_t flag; 597 char last[MAXPATHLEN]; 598 int64_t lval; /* temporary values... */ 599 int32_t ival; 600 uint32_t uival; 601 struct ident *gids = NULL, *uids = NULL; 602 603 last[0] = '\0'; 604 605 for (;;) { 606 if (!io_read_byte(sess, fd, &flag)) { 607 ERRX1(sess, "io_read_byte"); 608 goto out; 609 } else if (flag == 0) 610 break; 611 612 if (!flist_realloc(sess, &fl, &flsz, &flmax)) { 613 ERRX1(sess, "flist_realloc"); 614 goto out; 615 } 616 617 ff = &fl[flsz - 1]; 618 fflast = flsz > 1 ? &fl[flsz - 2] : NULL; 619 620 /* Filename first. */ 621 622 if (!flist_recv_name(sess, fd, ff, flag, last)) { 623 ERRX1(sess, "flist_recv_name"); 624 goto out; 625 } 626 627 /* Read the file size. */ 628 629 if (!io_read_long(sess, fd, &lval)) { 630 ERRX1(sess, "io_read_long"); 631 goto out; 632 } 633 ff->st.size = lval; 634 635 /* Read the modification time. */ 636 637 if (!(FLIST_TIME_SAME & flag)) { 638 if (!io_read_uint(sess, fd, &uival)) { 639 ERRX1(sess, "io_read_int"); 640 goto out; 641 } 642 ff->st.mtime = uival; /* beyond 2038 */ 643 } else if (fflast == NULL) { 644 ERRX(sess, "same time without last entry"); 645 goto out; 646 } else 647 ff->st.mtime = fflast->st.mtime; 648 649 /* Read the file mode. */ 650 651 if (!(FLIST_MODE_SAME & flag)) { 652 if (!io_read_uint(sess, fd, &uival)) { 653 ERRX1(sess, "io_read_int"); 654 goto out; 655 } 656 ff->st.mode = uival; 657 } else if (fflast == NULL) { 658 ERRX(sess, "same mode without last entry"); 659 goto out; 660 } else 661 ff->st.mode = fflast->st.mode; 662 663 /* Conditional part: uid. */ 664 665 if (sess->opts->preserve_uids) { 666 if (!(FLIST_UID_SAME & flag)) { 667 if (!io_read_uint(sess, fd, &uival)) { 668 ERRX1(sess, "io_read_int"); 669 goto out; 670 } 671 ff->st.uid = uival; 672 } else if (fflast == NULL) { 673 ERRX(sess, "same uid without last entry"); 674 goto out; 675 } else 676 ff->st.uid = fflast->st.uid; 677 } 678 679 /* Conditional part: gid. */ 680 681 if (sess->opts->preserve_gids) { 682 if (!(FLIST_GID_SAME & flag)) { 683 if (!io_read_uint(sess, fd, &uival)) { 684 ERRX1(sess, "io_read_int"); 685 goto out; 686 } 687 ff->st.gid = uival; 688 } else if (fflast == NULL) { 689 ERRX(sess, "same gid without last entry"); 690 goto out; 691 } else 692 ff->st.gid = fflast->st.gid; 693 } 694 695 /* Conditional part: devices & special files. */ 696 697 if ((sess->opts->devices && (S_ISBLK(ff->st.mode) || 698 S_ISCHR(ff->st.mode))) || 699 (sess->opts->specials && (S_ISFIFO(ff->st.mode) || 700 S_ISSOCK(ff->st.mode)))) { 701 if (!(FLIST_RDEV_SAME & flag)) { 702 if (!io_read_int(sess, fd, &ival)) { 703 ERRX1(sess, "io_read_int"); 704 goto out; 705 } 706 ff->st.rdev = ival; 707 } else if (fflast == NULL) { 708 ERRX(sess, "same device without last entry"); 709 goto out; 710 } else 711 ff->st.rdev = fflast->st.rdev; 712 } 713 714 /* Conditional part: link. */ 715 716 if (S_ISLNK(ff->st.mode) && 717 sess->opts->preserve_links) { 718 if (!io_read_size(sess, fd, &lsz)) { 719 ERRX1(sess, "io_read_size"); 720 goto out; 721 } else if (lsz == 0) { 722 ERRX(sess, "empty link name"); 723 goto out; 724 } 725 ff->link = calloc(lsz + 1, 1); 726 if (ff->link == NULL) { 727 ERR(sess, "calloc"); 728 goto out; 729 } 730 if (!io_read_buf(sess, fd, ff->link, lsz)) { 731 ERRX1(sess, "io_read_buf"); 732 goto out; 733 } 734 } 735 736 LOG3(sess, "%s: received file metadata: " 737 "size %jd, mtime %jd, mode %o, rdev (%d, %d)", 738 ff->path, (intmax_t)ff->st.size, 739 (intmax_t)ff->st.mtime, ff->st.mode, 740 major(ff->st.rdev), minor(ff->st.rdev)); 741 742 if (S_ISREG(ff->st.mode)) 743 sess->total_size += ff->st.size; 744 } 745 746 /* Conditionally read the user/group list. */ 747 748 if (sess->opts->preserve_uids && !sess->opts->numeric_ids) { 749 if (!idents_recv(sess, fd, &uids, &uidsz)) { 750 ERRX1(sess, "idents_recv"); 751 goto out; 752 } 753 LOG2(sess, "received uid list: %zu", uidsz); 754 } 755 756 if (sess->opts->preserve_gids && !sess->opts->numeric_ids) { 757 if (!idents_recv(sess, fd, &gids, &gidsz)) { 758 ERRX1(sess, "idents_recv"); 759 goto out; 760 } 761 LOG2(sess, "received gid list: %zu", gidsz); 762 } 763 764 /* Remember to order the received list. */ 765 766 LOG2(sess, "received file metadata list: %zu", flsz); 767 qsort(fl, flsz, sizeof(struct flist), flist_cmp); 768 flist_topdirs(sess, fl, flsz); 769 *sz = flsz; 770 *flp = fl; 771 772 /* Conditionally remap and reassign identifiers. */ 773 774 if (sess->opts->preserve_uids && !sess->opts->numeric_ids) { 775 idents_remap(sess, 0, uids, uidsz); 776 idents_assign_uid(sess, fl, flsz, uids, uidsz); 777 } 778 779 if (sess->opts->preserve_gids && !sess->opts->numeric_ids) { 780 idents_remap(sess, 1, gids, gidsz); 781 idents_assign_gid(sess, fl, flsz, gids, gidsz); 782 } 783 784 idents_free(gids, gidsz); 785 idents_free(uids, uidsz); 786 return 1; 787out: 788 flist_free(fl, flsz); 789 idents_free(gids, gidsz); 790 idents_free(uids, uidsz); 791 *sz = 0; 792 *flp = NULL; 793 return 0; 794} 795 796/* 797 * Generate a flist possibly-recursively given a file root, which may 798 * also be a regular file or symlink. 799 * On success, augments the generated list in "flp" of length "sz". 800 * Returns zero on failure, non-zero on success. 801 */ 802static int 803flist_gen_dirent(struct sess *sess, char *root, struct flist **fl, size_t *sz, 804 size_t *max) 805{ 806 char *cargv[2], *cp; 807 int rc = 0; 808 FTS *fts; 809 FTSENT *ent; 810 struct flist *f; 811 size_t flsz = 0, stripdir; 812 struct stat st; 813 814 cargv[0] = root; 815 cargv[1] = NULL; 816 817 /* 818 * If we're a file, then revert to the same actions we use for 819 * the non-recursive scan. 820 */ 821 822 if (lstat(root, &st) == -1) { 823 ERR(sess, "%s: lstat", root); 824 return 0; 825 } else if (S_ISREG(st.st_mode)) { 826 if (!flist_realloc(sess, fl, sz, max)) { 827 ERRX1(sess, "flist_realloc"); 828 return 0; 829 } 830 f = &(*fl)[(*sz) - 1]; 831 assert(f != NULL); 832 833 if (!flist_append(sess, f, &st, root)) { 834 ERRX1(sess, "flist_append"); 835 return 0; 836 } 837 if (unveil(root, "r") == -1) { 838 ERR(sess, "%s: unveil", root); 839 return 0; 840 } 841 return 1; 842 } else if (S_ISLNK(st.st_mode)) { 843 if (!sess->opts->preserve_links) { 844 WARNX(sess, "%s: skipping symlink", root); 845 return 1; 846 } else if (!flist_realloc(sess, fl, sz, max)) { 847 ERRX1(sess, "flist_realloc"); 848 return 0; 849 } 850 f = &(*fl)[(*sz) - 1]; 851 assert(f != NULL); 852 853 if (!flist_append(sess, f, &st, root)) { 854 ERRX1(sess, "flist_append"); 855 return 0; 856 } 857 if (unveil(root, "r") == -1) { 858 ERR(sess, "%s: unveil", root); 859 return 0; 860 } 861 return 1; 862 } else if (!S_ISDIR(st.st_mode)) { 863 WARNX(sess, "%s: skipping special", root); 864 return 1; 865 } 866 867 /* 868 * If we end with a slash, it means that we're not supposed to 869 * copy the directory part itself---only the contents. 870 * So set "stripdir" to be what we take out. 871 */ 872 873 stripdir = strlen(root); 874 assert(stripdir > 0); 875 if (root[stripdir - 1] != '/') 876 stripdir = 0; 877 878 /* 879 * If we're not stripping anything, then see if we need to strip 880 * out the leading material in the path up to and including the 881 * last directory component. 882 */ 883 884 if (stripdir == 0) 885 if ((cp = strrchr(root, '/')) != NULL) 886 stripdir = cp - root + 1; 887 888 /* 889 * If we're recursive, then we need to take down all of the 890 * files and directory components, so use fts(3). 891 * Copying the information file-by-file into the flstat. 892 * We'll make sense of it in flist_send. 893 */ 894 895 if ((fts = fts_open(cargv, FTS_PHYSICAL, NULL)) == NULL) { 896 ERR(sess, "fts_open"); 897 return 0; 898 } 899 900 errno = 0; 901 while ((ent = fts_read(fts)) != NULL) { 902 if (!flist_fts_check(sess, ent)) { 903 errno = 0; 904 continue; 905 } 906 907 /* We don't allow symlinks without -l. */ 908 909 assert(ent->fts_statp != NULL); 910 if (S_ISLNK(ent->fts_statp->st_mode) && 911 !sess->opts->preserve_links) { 912 WARNX(sess, "%s: skipping symlink", ent->fts_path); 913 continue; 914 } 915 916 /* Allocate a new file entry. */ 917 918 if (!flist_realloc(sess, fl, sz, max)) { 919 ERRX1(sess, "flist_realloc"); 920 goto out; 921 } 922 flsz++; 923 f = &(*fl)[*sz - 1]; 924 925 /* Our path defaults to "." for the root. */ 926 927 if (ent->fts_path[stripdir] == '\0') { 928 if (asprintf(&f->path, "%s.", ent->fts_path) < 0) { 929 ERR(sess, "asprintf"); 930 f->path = NULL; 931 goto out; 932 } 933 } else { 934 if ((f->path = strdup(ent->fts_path)) == NULL) { 935 ERR(sess, "strdup"); 936 goto out; 937 } 938 } 939 940 f->wpath = f->path + stripdir; 941 flist_copy_stat(f, ent->fts_statp); 942 943 /* Optionally copy link information. */ 944 945 if (S_ISLNK(ent->fts_statp->st_mode)) { 946 f->link = symlink_read(sess, f->path); 947 if (f->link == NULL) { 948 ERRX1(sess, "symlink_read"); 949 goto out; 950 } 951 } 952 953 /* Reset errno for next fts_read() call. */ 954 errno = 0; 955 } 956 if (errno) { 957 ERR(sess, "fts_read"); 958 goto out; 959 } 960 if (unveil(root, "r") == -1) { 961 ERR(sess, "%s: unveil", root); 962 goto out; 963 } 964 965 LOG3(sess, "generated %zu filenames: %s", flsz, root); 966 rc = 1; 967out: 968 fts_close(fts); 969 return rc; 970} 971 972/* 973 * Generate a flist recursively given the array of directories (or 974 * files, symlinks, doesn't matter) specified in argv (argc >0). 975 * On success, stores the generated list in "flp" with length "sz", 976 * which may be zero. 977 * Returns zero on failure, non-zero on success. 978 */ 979static int 980flist_gen_dirs(struct sess *sess, size_t argc, char **argv, struct flist **flp, 981 size_t *sz) 982{ 983 size_t i, max = 0; 984 985 for (i = 0; i < argc; i++) 986 if (!flist_gen_dirent(sess, argv[i], flp, sz, &max)) 987 break; 988 989 if (i == argc) { 990 LOG2(sess, "recursively generated %zu filenames", *sz); 991 return 1; 992 } 993 994 ERRX1(sess, "flist_gen_dirent"); 995 flist_free(*flp, max); 996 *flp = NULL; 997 *sz = 0; 998 return 0; 999} 1000 1001/* 1002 * Generate list of files from the command-line argc (>0) and argv. 1003 * On success, stores the generated list in "flp" with length "sz", 1004 * which may be zero. 1005 * Returns zero on failure, non-zero on success. 1006 */ 1007static int 1008flist_gen_files(struct sess *sess, size_t argc, char **argv, 1009 struct flist **flp, size_t *sz) 1010{ 1011 struct flist *fl = NULL, *f; 1012 size_t i, flsz = 0; 1013 struct stat st; 1014 1015 assert(argc); 1016 1017 if ((fl = calloc(argc, sizeof(struct flist))) == NULL) { 1018 ERR(sess, "calloc"); 1019 return 0; 1020 } 1021 1022 for (i = 0; i < argc; i++) { 1023 if (argv[i][0] == '\0') 1024 continue; 1025 if (lstat(argv[i], &st) == -1) { 1026 ERR(sess, "%s: lstat", argv[i]); 1027 goto out; 1028 } 1029 1030 /* 1031 * File type checks. 1032 * In non-recursive mode, we don't accept directories. 1033 * We also skip symbolic links without -l. 1034 * Beyond that, we only accept regular files. 1035 */ 1036 1037 if (S_ISDIR(st.st_mode)) { 1038 WARNX(sess, "%s: skipping directory", argv[i]); 1039 continue; 1040 } else if (S_ISLNK(st.st_mode)) { 1041 if (!sess->opts->preserve_links) { 1042 WARNX(sess, "%s: skipping symlink", argv[i]); 1043 continue; 1044 } 1045 } else if (!S_ISREG(st.st_mode)) { 1046 WARNX(sess, "%s: skipping special", argv[i]); 1047 continue; 1048 } 1049 1050 1051 f = &fl[flsz++]; 1052 assert(f != NULL); 1053 1054 /* Add this file to our file-system worldview. */ 1055 1056 if (unveil(argv[i], "r") == -1) { 1057 ERR(sess, "%s: unveil", argv[i]); 1058 goto out; 1059 } 1060 if (!flist_append(sess, f, &st, argv[i])) { 1061 ERRX1(sess, "flist_append"); 1062 goto out; 1063 } 1064 } 1065 1066 LOG2(sess, "non-recursively generated %zu filenames", flsz); 1067 *sz = flsz; 1068 *flp = fl; 1069 return 1; 1070out: 1071 flist_free(fl, argc); 1072 *sz = 0; 1073 *flp = NULL; 1074 return 0; 1075} 1076 1077/* 1078 * Generate a sorted, de-duplicated list of file metadata. 1079 * In non-recursive mode (the default), we use only the files we're 1080 * given. 1081 * Otherwise, directories are recursively examined. 1082 * Returns zero on failure, non-zero on success. 1083 * On success, "fl" will need to be freed with flist_free(). 1084 */ 1085int 1086flist_gen(struct sess *sess, size_t argc, char **argv, struct flist **flp, 1087 size_t *sz) 1088{ 1089 int rc; 1090 1091 assert(argc > 0); 1092 rc = sess->opts->recursive ? 1093 flist_gen_dirs(sess, argc, argv, flp, sz) : 1094 flist_gen_files(sess, argc, argv, flp, sz); 1095 1096 /* After scanning, lock our file-system view. */ 1097 1098 if (unveil(NULL, NULL) == -1) { 1099 ERR(sess, "unveil"); 1100 return 0; 1101 } 1102 if (!rc) 1103 return 0; 1104 1105 qsort(*flp, *sz, sizeof(struct flist), flist_cmp); 1106 1107 if (flist_dedupe(sess, flp, sz)) { 1108 flist_topdirs(sess, *flp, *sz); 1109 return 1; 1110 } 1111 1112 ERRX1(sess, "flist_dedupe"); 1113 flist_free(*flp, *sz); 1114 *flp = NULL; 1115 *sz = 0; 1116 return 0; 1117} 1118 1119/* 1120 * Generate a list of files in root to delete that are within the 1121 * top-level directories stipulated by "wfl". 1122 * Only handles symbolic links, directories, and regular files. 1123 * Returns zero on failure (fl and flsz will be NULL and zero), non-zero 1124 * on success. 1125 * On success, "fl" will need to be freed with flist_free(). 1126 */ 1127int 1128flist_gen_dels(struct sess *sess, const char *root, struct flist **fl, 1129 size_t *sz, const struct flist *wfl, size_t wflsz) 1130{ 1131 char **cargv = NULL; 1132 int rc = 0, c; 1133 FTS *fts = NULL; 1134 FTSENT *ent; 1135 struct flist *f; 1136 size_t cargvs = 0, i, j, max = 0, stripdir; 1137 ENTRY hent; 1138 ENTRY *hentp; 1139 1140 *fl = NULL; 1141 *sz = 0; 1142 1143 /* Only run this code when we're recursive. */ 1144 1145 if (!sess->opts->recursive) 1146 return 1; 1147 1148 /* 1149 * Gather up all top-level directories for scanning. 1150 * This is stipulated by rsync's --delete behaviour, where we 1151 * only delete things in the top-level directories given on the 1152 * command line. 1153 */ 1154 1155 assert(wflsz > 0); 1156 for (i = 0; i < wflsz; i++) 1157 if (FLSTAT_TOP_DIR & wfl[i].st.flags) 1158 cargvs++; 1159 if (cargvs == 0) 1160 return 1; 1161 1162 if ((cargv = calloc(cargvs + 1, sizeof(char *))) == NULL) { 1163 ERR(sess, "calloc"); 1164 return 0; 1165 } 1166 1167 /* 1168 * If we're given just a "." as the first entry, that means 1169 * we're doing a relative copy with a trailing slash. 1170 * Special-case this just for the sake of simplicity. 1171 * Otherwise, look through all top-levels. 1172 */ 1173 1174 if (wflsz && strcmp(wfl[0].wpath, ".") == 0) { 1175 assert(cargvs == 1); 1176 assert(S_ISDIR(wfl[0].st.mode)); 1177 if (asprintf(&cargv[0], "%s/", root) < 0) { 1178 ERR(sess, "asprintf"); 1179 cargv[0] = NULL; 1180 goto out; 1181 } 1182 cargv[1] = NULL; 1183 } else { 1184 for (i = j = 0; i < wflsz; i++) { 1185 if (!(FLSTAT_TOP_DIR & wfl[i].st.flags)) 1186 continue; 1187 assert(S_ISDIR(wfl[i].st.mode)); 1188 assert(strcmp(wfl[i].wpath, ".")); 1189 c = asprintf(&cargv[j], "%s/%s", root, wfl[i].wpath); 1190 if (c < 0) { 1191 ERR(sess, "asprintf"); 1192 cargv[j] = NULL; 1193 goto out; 1194 } 1195 LOG4(sess, "%s: will scan for deletions", cargv[j]); 1196 j++; 1197 } 1198 assert(j == cargvs); 1199 cargv[j] = NULL; 1200 } 1201 1202 LOG2(sess, "delete from %zu directories", cargvs); 1203 1204 /* 1205 * Next, use the standard hcreate(3) hashtable interface to hash 1206 * all of the files that we want to synchronise. 1207 * This way, we'll be able to determine which files we want to 1208 * delete in O(n) time instead of O(n * search) time. 1209 * Plus, we can do the scan in-band and only allocate the files 1210 * we want to delete. 1211 */ 1212 1213 if (!hcreate(wflsz)) { 1214 ERR(sess, "hcreate"); 1215 goto out; 1216 } 1217 1218 for (i = 0; i < wflsz; i++) { 1219 memset(&hent, 0, sizeof(ENTRY)); 1220 if ((hent.key = strdup(wfl[i].wpath)) == NULL) { 1221 ERR(sess, "strdup"); 1222 goto out; 1223 } 1224 if ((hentp = hsearch(hent, ENTER)) == NULL) { 1225 ERR(sess, "hsearch"); 1226 goto out; 1227 } else if (hentp->key != hent.key) { 1228 ERRX(sess, "%s: duplicate", wfl[i].wpath); 1229 free(hent.key); 1230 goto out; 1231 } 1232 } 1233 1234 /* 1235 * Now we're going to try to descend into all of the top-level 1236 * directories stipulated by the file list. 1237 * If the directories don't exist, it's ok. 1238 */ 1239 1240 if ((fts = fts_open(cargv, FTS_PHYSICAL, NULL)) == NULL) { 1241 ERR(sess, "fts_open"); 1242 goto out; 1243 } 1244 1245 stripdir = strlen(root) + 1; 1246 errno = 0; 1247 while ((ent = fts_read(fts)) != NULL) { 1248 if (ent->fts_info == FTS_NS) 1249 continue; 1250 if (!flist_fts_check(sess, ent)) { 1251 errno = 0; 1252 continue; 1253 } else if (stripdir >= ent->fts_pathlen) 1254 continue; 1255 1256 /* Look up in hashtable. */ 1257 1258 memset(&hent, 0, sizeof(ENTRY)); 1259 hent.key = ent->fts_path + stripdir; 1260 if (hsearch(hent, FIND) != NULL) 1261 continue; 1262 1263 /* Not found: we'll delete it. */ 1264 1265 if (!flist_realloc(sess, fl, sz, &max)) { 1266 ERRX1(sess, "flist_realloc"); 1267 goto out; 1268 } 1269 f = &(*fl)[*sz - 1]; 1270 1271 if ((f->path = strdup(ent->fts_path)) == NULL) { 1272 ERR(sess, "strdup"); 1273 goto out; 1274 } 1275 f->wpath = f->path + stripdir; 1276 assert(ent->fts_statp != NULL); 1277 flist_copy_stat(f, ent->fts_statp); 1278 errno = 0; 1279 } 1280 1281 if (errno) { 1282 ERR(sess, "fts_read"); 1283 goto out; 1284 } 1285 1286 qsort(*fl, *sz, sizeof(struct flist), flist_cmp); 1287 rc = 1; 1288out: 1289 if (fts != NULL) 1290 fts_close(fts); 1291 for (i = 0; i < cargvs; i++) 1292 free(cargv[i]); 1293 free(cargv); 1294 hdestroy(); 1295 return rc; 1296} 1297 1298/* 1299 * Delete all files and directories in "fl". 1300 * If called with a zero-length "fl", does nothing. 1301 * If dry_run is specified, simply write what would be done. 1302 * Return zero on failure, non-zero on success. 1303 */ 1304int 1305flist_del(struct sess *sess, int root, const struct flist *fl, size_t flsz) 1306{ 1307 ssize_t i; 1308 int flag; 1309 1310 if (flsz == 0) 1311 return 1; 1312 1313 assert(sess->opts->del); 1314 assert(sess->opts->recursive); 1315 1316 for (i = flsz - 1; i >= 0; i--) { 1317 LOG1(sess, "%s: deleting", fl[i].wpath); 1318 if (sess->opts->dry_run) 1319 continue; 1320 assert(root != -1); 1321 flag = S_ISDIR(fl[i].st.mode) ? AT_REMOVEDIR : 0; 1322 if (unlinkat(root, fl[i].wpath, flag) == -1 && 1323 errno != ENOENT) { 1324 ERR(sess, "%s: unlinkat", fl[i].wpath); 1325 return 0; 1326 } 1327 } 1328 1329 return 1; 1330} 1331