1/* $OpenBSD: repo.c,v 1.60 2024/06/07 08:22:53 claudio Exp $ */ 2/* 3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org> 4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19#include <sys/queue.h> 20#include <sys/tree.h> 21#include <sys/types.h> 22#include <sys/stat.h> 23 24#include <assert.h> 25#include <err.h> 26#include <errno.h> 27#include <fcntl.h> 28#include <fts.h> 29#include <limits.h> 30#include <poll.h> 31#include <stdio.h> 32#include <stdlib.h> 33#include <string.h> 34#include <unistd.h> 35 36#include <imsg.h> 37 38#include "extern.h" 39 40extern struct stats stats; 41extern int rrdpon; 42extern int repo_timeout; 43extern time_t deadline; 44int nofetch; 45 46enum repo_state { 47 REPO_LOADING = 0, 48 REPO_DONE = 1, 49 REPO_FAILED = -1, 50}; 51 52/* 53 * A ta, rsync or rrdp repository. 54 * Depending on what is needed the generic repository is backed by 55 * a ta, rsync or rrdp repository. Multiple repositories can use the 56 * same backend. 57 */ 58struct rrdprepo { 59 SLIST_ENTRY(rrdprepo) entry; 60 char *notifyuri; 61 char *basedir; 62 struct filepath_tree deleted; 63 unsigned int id; 64 enum repo_state state; 65}; 66static SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos); 67 68struct rsyncrepo { 69 SLIST_ENTRY(rsyncrepo) entry; 70 char *repouri; 71 char *basedir; 72 unsigned int id; 73 enum repo_state state; 74}; 75static SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos); 76 77struct tarepo { 78 SLIST_ENTRY(tarepo) entry; 79 char *descr; 80 char *basedir; 81 char **uri; 82 size_t urisz; 83 size_t uriidx; 84 unsigned int id; 85 enum repo_state state; 86}; 87static SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos); 88 89struct repo { 90 SLIST_ENTRY(repo) entry; 91 char *repouri; 92 char *notifyuri; 93 char *basedir; 94 const struct rrdprepo *rrdp; 95 const struct rsyncrepo *rsync; 96 const struct tarepo *ta; 97 struct entityq queue; /* files waiting for repo */ 98 struct repotalstats stats[TALSZ_MAX]; 99 struct repostats repostats; 100 struct timespec start_time; 101 time_t alarm; /* sync timeout */ 102 int talid; 103 int stats_used[TALSZ_MAX]; 104 unsigned int id; /* identifier */ 105}; 106static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); 107 108/* counter for unique repo id */ 109unsigned int repoid; 110 111static struct rsyncrepo *rsync_get(const char *, const char *); 112static void remove_contents(char *); 113 114/* 115 * Database of all file path accessed during a run. 116 */ 117struct filepath { 118 RB_ENTRY(filepath) entry; 119 char *file; 120 time_t mtime; 121 unsigned int talmask; 122}; 123 124static inline int 125filepathcmp(const struct filepath *a, const struct filepath *b) 126{ 127 return strcmp(a->file, b->file); 128} 129 130RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); 131 132/* 133 * Functions to lookup which files have been accessed during computation. 134 */ 135int 136filepath_add(struct filepath_tree *tree, char *file, int id, time_t mtime) 137{ 138 struct filepath *fp, *rfp; 139 140 CTASSERT(TALSZ_MAX < 8 * sizeof(fp->talmask)); 141 assert(id >= 0 && id < 8 * (int)sizeof(fp->talmask)); 142 143 if ((fp = calloc(1, sizeof(*fp))) == NULL) 144 err(1, NULL); 145 if ((fp->file = strdup(file)) == NULL) 146 err(1, NULL); 147 fp->mtime = mtime; 148 149 if ((rfp = RB_INSERT(filepath_tree, tree, fp)) != NULL) { 150 /* already in the tree */ 151 free(fp->file); 152 free(fp); 153 if (rfp->talmask & (1 << id)) 154 return 0; 155 fp = rfp; 156 } 157 fp->talmask |= (1 << id); 158 159 return 1; 160} 161 162/* 163 * Lookup a file path in the tree and return the object if found or NULL. 164 */ 165static struct filepath * 166filepath_find(struct filepath_tree *tree, char *file) 167{ 168 struct filepath needle = { .file = file }; 169 170 return RB_FIND(filepath_tree, tree, &needle); 171} 172 173/* 174 * Returns true if file exists in the tree. 175 */ 176static int 177filepath_exists(struct filepath_tree *tree, char *file) 178{ 179 return filepath_find(tree, file) != NULL; 180} 181 182/* 183 * Remove entry from tree and free it. 184 */ 185static void 186filepath_put(struct filepath_tree *tree, struct filepath *fp) 187{ 188 RB_REMOVE(filepath_tree, tree, fp); 189 free((void *)fp->file); 190 free(fp); 191} 192 193/* 194 * Free all elements of a filepath tree. 195 */ 196static void 197filepath_free(struct filepath_tree *tree) 198{ 199 struct filepath *fp, *nfp; 200 201 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) 202 filepath_put(tree, fp); 203} 204 205RB_GENERATE(filepath_tree, filepath, entry, filepathcmp); 206 207/* 208 * Function to hash a string into a unique directory name. 209 * Returned hash needs to be freed. 210 */ 211static char * 212hash_dir(const char *uri) 213{ 214 unsigned char m[SHA256_DIGEST_LENGTH]; 215 216 SHA256(uri, strlen(uri), m); 217 return hex_encode(m, sizeof(m)); 218} 219 220/* 221 * Function to build the directory name based on URI and a directory 222 * as prefix. Skip the proto:// in URI but keep everything else. 223 */ 224static char * 225repo_dir(const char *uri, const char *dir, int hash) 226{ 227 const char *local; 228 char *out, *hdir = NULL; 229 230 if (hash) { 231 local = hdir = hash_dir(uri); 232 } else { 233 local = strchr(uri, ':'); 234 if (local != NULL) 235 local += strlen("://"); 236 else 237 local = uri; 238 } 239 240 if (dir == NULL) { 241 if ((out = strdup(local)) == NULL) 242 err(1, NULL); 243 } else { 244 if (asprintf(&out, "%s/%s", dir, local) == -1) 245 err(1, NULL); 246 } 247 248 free(hdir); 249 return out; 250} 251 252/* 253 * Function to create all missing directories to a path. 254 * This functions alters the path temporarily. 255 */ 256static int 257repo_mkpath(int fd, char *file) 258{ 259 char *slash; 260 261 /* build directory hierarchy */ 262 slash = strrchr(file, '/'); 263 assert(slash != NULL); 264 *slash = '\0'; 265 if (mkpathat(fd, file) == -1) { 266 warn("mkpath %s", file); 267 return -1; 268 } 269 *slash = '/'; 270 return 0; 271} 272 273/* 274 * Return the state of a repository. 275 */ 276static enum repo_state 277repo_state(const struct repo *rp) 278{ 279 if (rp->ta) 280 return rp->ta->state; 281 if (rp->rsync) 282 return rp->rsync->state; 283 if (rp->rrdp) 284 return rp->rrdp->state; 285 /* No backend so sync is by definition done. */ 286 return REPO_DONE; 287} 288 289/* 290 * Function called once a repository is done with the sync. Either 291 * successfully or after failure. 292 */ 293static void 294repo_done(const void *vp, int ok) 295{ 296 struct repo *rp; 297 struct timespec flush_time; 298 299 SLIST_FOREACH(rp, &repos, entry) { 300 if (vp != rp->ta && vp != rp->rsync && vp != rp->rrdp) 301 continue; 302 303 /* for rrdp try to fall back to rsync */ 304 if (vp == rp->rrdp && !ok && !nofetch) { 305 rp->rrdp = NULL; 306 rp->rsync = rsync_get(rp->repouri, rp->basedir); 307 /* need to check if it was already loaded */ 308 if (repo_state(rp) == REPO_LOADING) 309 continue; 310 } 311 312 entityq_flush(&rp->queue, rp); 313 clock_gettime(CLOCK_MONOTONIC, &flush_time); 314 timespecsub(&flush_time, &rp->start_time, 315 &rp->repostats.sync_time); 316 } 317} 318 319/* 320 * Build TA file name based on the repo info. 321 * If temp is set add Xs for mkostemp. 322 */ 323static char * 324ta_filename(const struct tarepo *tr) 325{ 326 const char *file; 327 char *nfile; 328 329 /* does not matter which URI, all end with same filename */ 330 file = strrchr(tr->uri[0], '/'); 331 assert(file); 332 333 if (asprintf(&nfile, "%s%s", tr->basedir, file) == -1) 334 err(1, NULL); 335 336 return nfile; 337} 338 339static void 340ta_fetch(struct tarepo *tr) 341{ 342 if (!rrdpon) { 343 for (; tr->uriidx < tr->urisz; tr->uriidx++) { 344 if (strncasecmp(tr->uri[tr->uriidx], 345 RSYNC_PROTO, RSYNC_PROTO_LEN) == 0) 346 break; 347 } 348 } 349 350 if (tr->uriidx >= tr->urisz) { 351 tr->state = REPO_FAILED; 352 logx("ta/%s: fallback to cache", tr->descr); 353 354 repo_done(tr, 0); 355 return; 356 } 357 358 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]); 359 360 if (strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO, 361 RSYNC_PROTO_LEN) == 0) { 362 /* 363 * Create destination location. 364 * Build up the tree to this point. 365 */ 366 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL); 367 } else { 368 char *temp; 369 int fd; 370 371 temp = ta_filename(tr); 372 fd = open(temp, 373 O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC, 374 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); 375 if (fd == -1) { 376 warn("open: %s", temp); 377 free(temp); 378 http_finish(tr->id, HTTP_FAILED, NULL); 379 return; 380 } 381 382 free(temp); 383 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd); 384 } 385} 386 387static struct tarepo * 388ta_get(struct tal *tal) 389{ 390 struct tarepo *tr; 391 392 /* no need to look for possible other repo */ 393 394 if ((tr = calloc(1, sizeof(*tr))) == NULL) 395 err(1, NULL); 396 397 tr->id = ++repoid; 398 SLIST_INSERT_HEAD(&tarepos, tr, entry); 399 400 if ((tr->descr = strdup(tal->descr)) == NULL) 401 err(1, NULL); 402 tr->basedir = repo_dir(tal->descr, ".ta", 0); 403 404 /* create base directory */ 405 if (mkpath(tr->basedir) == -1) { 406 warn("mkpath %s", tr->basedir); 407 tr->state = REPO_FAILED; 408 repo_done(tr, 0); 409 return tr; 410 } 411 412 /* steal URI information from TAL */ 413 tr->urisz = tal->urisz; 414 tr->uri = tal->uri; 415 tal->urisz = 0; 416 tal->uri = NULL; 417 418 ta_fetch(tr); 419 420 return tr; 421} 422 423static struct tarepo * 424ta_find(unsigned int id) 425{ 426 struct tarepo *tr; 427 428 SLIST_FOREACH(tr, &tarepos, entry) 429 if (id == tr->id) 430 break; 431 return tr; 432} 433 434static void 435ta_free(void) 436{ 437 struct tarepo *tr; 438 439 while ((tr = SLIST_FIRST(&tarepos)) != NULL) { 440 SLIST_REMOVE_HEAD(&tarepos, entry); 441 free(tr->descr); 442 free(tr->basedir); 443 free(tr->uri); 444 free(tr); 445 } 446} 447 448static struct rsyncrepo * 449rsync_get(const char *uri, const char *validdir) 450{ 451 struct rsyncrepo *rr; 452 char *repo; 453 454 if ((repo = rsync_base_uri(uri)) == NULL) 455 errx(1, "bad caRepository URI: %s", uri); 456 457 SLIST_FOREACH(rr, &rsyncrepos, entry) 458 if (strcmp(rr->repouri, repo) == 0) { 459 free(repo); 460 return rr; 461 } 462 463 if ((rr = calloc(1, sizeof(*rr))) == NULL) 464 err(1, NULL); 465 466 rr->id = ++repoid; 467 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry); 468 469 rr->repouri = repo; 470 rr->basedir = repo_dir(repo, ".rsync", 0); 471 472 /* create base directory */ 473 if (mkpath(rr->basedir) == -1) { 474 warn("mkpath %s", rr->basedir); 475 rsync_finish(rr->id, 0); 476 return rr; 477 } 478 479 logx("%s: pulling from %s", rr->basedir, rr->repouri); 480 rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir); 481 482 return rr; 483} 484 485static struct rsyncrepo * 486rsync_find(unsigned int id) 487{ 488 struct rsyncrepo *rr; 489 490 SLIST_FOREACH(rr, &rsyncrepos, entry) 491 if (id == rr->id) 492 break; 493 return rr; 494} 495 496static void 497rsync_free(void) 498{ 499 struct rsyncrepo *rr; 500 501 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) { 502 SLIST_REMOVE_HEAD(&rsyncrepos, entry); 503 free(rr->repouri); 504 free(rr->basedir); 505 free(rr); 506 } 507} 508 509/* 510 * Build local file name base on the URI and the rrdprepo info. 511 */ 512static char * 513rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid) 514{ 515 char *nfile; 516 const char *dir = rr->basedir; 517 518 if (!valid_uri(uri, strlen(uri), RSYNC_PROTO)) 519 errx(1, "%s: bad URI %s", rr->basedir, uri); 520 uri += RSYNC_PROTO_LEN; /* skip proto */ 521 if (valid) { 522 if ((nfile = strdup(uri)) == NULL) 523 err(1, NULL); 524 } else { 525 if (asprintf(&nfile, "%s/%s", dir, uri) == -1) 526 err(1, NULL); 527 } 528 return nfile; 529} 530 531/* 532 * Build RRDP state file name based on the repo info. 533 * If temp is set add Xs for mkostemp. 534 */ 535static char * 536rrdp_state_filename(const struct rrdprepo *rr, int temp) 537{ 538 char *nfile; 539 540 if (asprintf(&nfile, "%s/.state%s", rr->basedir, 541 temp ? ".XXXXXXXX" : "") == -1) 542 err(1, NULL); 543 544 return nfile; 545} 546 547static struct rrdprepo * 548rrdp_find(unsigned int id) 549{ 550 struct rrdprepo *rr; 551 552 SLIST_FOREACH(rr, &rrdprepos, entry) 553 if (id == rr->id) 554 break; 555 return rr; 556} 557 558static void 559rrdp_free(void) 560{ 561 struct rrdprepo *rr; 562 563 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) { 564 SLIST_REMOVE_HEAD(&rrdprepos, entry); 565 566 free(rr->notifyuri); 567 free(rr->basedir); 568 569 filepath_free(&rr->deleted); 570 571 free(rr); 572 } 573} 574 575/* 576 * Check if a directory is an active rrdp repository. 577 * Returns 1 if found else 0. 578 */ 579static struct repo * 580repo_rrdp_bypath(const char *dir) 581{ 582 struct repo *rp; 583 584 SLIST_FOREACH(rp, &repos, entry) { 585 if (rp->rrdp == NULL) 586 continue; 587 if (strcmp(dir, rp->rrdp->basedir) == 0) 588 return rp; 589 } 590 return NULL; 591} 592 593/* 594 * Check if the URI is actually covered by one of the repositories 595 * that depend on this RRDP repository. 596 * Returns 1 if the URI is valid, 0 if no repouri matches the URI. 597 */ 598static int 599rrdp_uri_valid(struct rrdprepo *rr, const char *uri) 600{ 601 struct repo *rp; 602 603 SLIST_FOREACH(rp, &repos, entry) { 604 if (rp->rrdp != rr) 605 continue; 606 if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0) 607 return 1; 608 } 609 return 0; 610} 611 612/* 613 * Allocate and insert a new repository. 614 */ 615static struct repo * 616repo_alloc(int talid) 617{ 618 struct repo *rp; 619 620 if ((rp = calloc(1, sizeof(*rp))) == NULL) 621 err(1, NULL); 622 623 rp->id = ++repoid; 624 rp->talid = talid; 625 rp->alarm = getmonotime() + repo_timeout; 626 TAILQ_INIT(&rp->queue); 627 SLIST_INSERT_HEAD(&repos, rp, entry); 628 clock_gettime(CLOCK_MONOTONIC, &rp->start_time); 629 630 stats.repos++; 631 return rp; 632} 633 634/* 635 * Parse the RRDP state file if it exists and set the session struct 636 * based on that information. 637 */ 638static struct rrdp_session * 639rrdp_session_parse(const struct rrdprepo *rr) 640{ 641 FILE *f; 642 struct rrdp_session *state; 643 int fd, ln = 0, deltacnt = 0; 644 const char *errstr; 645 char *line = NULL, *file; 646 size_t len = 0; 647 ssize_t n; 648 649 if ((state = calloc(1, sizeof(*state))) == NULL) 650 err(1, NULL); 651 652 file = rrdp_state_filename(rr, 0); 653 if ((fd = open(file, O_RDONLY)) == -1) { 654 if (errno != ENOENT) 655 warn("%s: open state file", rr->basedir); 656 free(file); 657 return state; 658 } 659 free(file); 660 f = fdopen(fd, "r"); 661 if (f == NULL) 662 err(1, "fdopen"); 663 664 while ((n = getline(&line, &len, f)) != -1) { 665 if (line[n - 1] == '\n') 666 line[n - 1] = '\0'; 667 switch (ln) { 668 case 0: 669 if ((state->session_id = strdup(line)) == NULL) 670 err(1, NULL); 671 break; 672 case 1: 673 state->serial = strtonum(line, 1, LLONG_MAX, &errstr); 674 if (errstr) 675 goto fail; 676 break; 677 case 2: 678 if (strcmp(line, "-") == 0) 679 break; 680 if ((state->last_mod = strdup(line)) == NULL) 681 err(1, NULL); 682 break; 683 default: 684 if (deltacnt >= MAX_RRDP_DELTAS) 685 goto fail; 686 if ((state->deltas[deltacnt++] = strdup(line)) == NULL) 687 err(1, NULL); 688 break; 689 } 690 ln++; 691 } 692 693 if (ferror(f)) 694 goto fail; 695 fclose(f); 696 free(line); 697 return state; 698 699 fail: 700 warnx("%s: troubles reading state file", rr->basedir); 701 fclose(f); 702 free(line); 703 free(state->session_id); 704 free(state->last_mod); 705 memset(state, 0, sizeof(*state)); 706 return state; 707} 708 709/* 710 * Carefully write the RRDP session state file back. 711 */ 712void 713rrdp_session_save(unsigned int id, struct rrdp_session *state) 714{ 715 struct rrdprepo *rr; 716 char *temp, *file; 717 FILE *f = NULL; 718 int fd, i; 719 720 rr = rrdp_find(id); 721 if (rr == NULL) 722 errx(1, "non-existent rrdp repo %u", id); 723 724 file = rrdp_state_filename(rr, 0); 725 temp = rrdp_state_filename(rr, 1); 726 727 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) 728 goto fail; 729 (void)fchmod(fd, 0644); 730 f = fdopen(fd, "w"); 731 if (f == NULL) 732 err(1, "fdopen"); 733 734 /* write session state file out */ 735 if (fprintf(f, "%s\n%lld\n", state->session_id, 736 state->serial) < 0) 737 goto fail; 738 739 if (state->last_mod != NULL) { 740 if (fprintf(f, "%s\n", state->last_mod) < 0) 741 goto fail; 742 } else { 743 if (fprintf(f, "-\n") < 0) 744 goto fail; 745 } 746 for (i = 0; i < MAX_RRDP_DELTAS && state->deltas[i] != NULL; i++) { 747 if (fprintf(f, "%s\n", state->deltas[i]) < 0) 748 goto fail; 749 } 750 if (fclose(f) != 0) { 751 f = NULL; 752 goto fail; 753 } 754 755 if (rename(temp, file) == -1) { 756 warn("%s: rename %s to %s", rr->basedir, temp, file); 757 unlink(temp); 758 } 759 760 free(temp); 761 free(file); 762 return; 763 764 fail: 765 warn("%s: save state to %s", rr->basedir, temp); 766 if (f != NULL) 767 fclose(f); 768 unlink(temp); 769 free(temp); 770 free(file); 771} 772 773/* 774 * Free an rrdp_session pointer. Safe to call with NULL. 775 */ 776void 777rrdp_session_free(struct rrdp_session *s) 778{ 779 size_t i; 780 781 if (s == NULL) 782 return; 783 free(s->session_id); 784 free(s->last_mod); 785 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) 786 free(s->deltas[i]); 787 free(s); 788} 789 790void 791rrdp_session_buffer(struct ibuf *b, const struct rrdp_session *s) 792{ 793 size_t i; 794 795 io_str_buffer(b, s->session_id); 796 io_simple_buffer(b, &s->serial, sizeof(s->serial)); 797 io_str_buffer(b, s->last_mod); 798 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) 799 io_str_buffer(b, s->deltas[i]); 800} 801 802struct rrdp_session * 803rrdp_session_read(struct ibuf *b) 804{ 805 struct rrdp_session *s; 806 size_t i; 807 808 if ((s = calloc(1, sizeof(*s))) == NULL) 809 err(1, NULL); 810 811 io_read_str(b, &s->session_id); 812 io_read_buf(b, &s->serial, sizeof(s->serial)); 813 io_read_str(b, &s->last_mod); 814 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) 815 io_read_str(b, &s->deltas[i]); 816 817 return s; 818} 819 820static struct rrdprepo * 821rrdp_get(const char *uri) 822{ 823 struct rrdp_session *state; 824 struct rrdprepo *rr; 825 826 SLIST_FOREACH(rr, &rrdprepos, entry) 827 if (strcmp(rr->notifyuri, uri) == 0) { 828 if (rr->state == REPO_FAILED) 829 return NULL; 830 return rr; 831 } 832 833 if ((rr = calloc(1, sizeof(*rr))) == NULL) 834 err(1, NULL); 835 836 rr->id = ++repoid; 837 SLIST_INSERT_HEAD(&rrdprepos, rr, entry); 838 839 if ((rr->notifyuri = strdup(uri)) == NULL) 840 err(1, NULL); 841 rr->basedir = repo_dir(uri, ".rrdp", 1); 842 843 RB_INIT(&rr->deleted); 844 845 /* create base directory */ 846 if (mkpath(rr->basedir) == -1) { 847 warn("mkpath %s", rr->basedir); 848 rrdp_finish(rr->id, 0); 849 return rr; 850 } 851 852 /* parse state and start the sync */ 853 state = rrdp_session_parse(rr); 854 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, state); 855 rrdp_session_free(state); 856 857 logx("%s: pulling from %s", rr->notifyuri, "network"); 858 859 return rr; 860} 861 862/* 863 * Remove RRDP repo and start over. 864 */ 865void 866rrdp_clear(unsigned int id) 867{ 868 struct rrdprepo *rr; 869 870 rr = rrdp_find(id); 871 if (rr == NULL) 872 errx(1, "non-existent rrdp repo %u", id); 873 874 /* remove rrdp repository contents */ 875 remove_contents(rr->basedir); 876} 877 878/* 879 * Write a file into the temporary RRDP dir but only after checking 880 * its hash (if required). The function also makes sure that the file 881 * tracking is properly adjusted. 882 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error 883 */ 884int 885rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri, 886 char *hash, size_t hlen, char *data, size_t dlen) 887{ 888 struct rrdprepo *rr; 889 struct filepath *fp; 890 ssize_t s; 891 char *fn = NULL; 892 int fd = -1, try = 0, deleted = 0; 893 int flags; 894 895 rr = rrdp_find(id); 896 if (rr == NULL) 897 errx(1, "non-existent rrdp repo %u", id); 898 if (rr->state == REPO_FAILED) 899 return -1; 900 901 /* check hash of original file for updates and deletes */ 902 if (pt == PUB_UPD || pt == PUB_DEL) { 903 if (filepath_exists(&rr->deleted, uri)) { 904 warnx("%s: already deleted", uri); 905 return 0; 906 } 907 /* try to open file first in rrdp then in valid repo */ 908 do { 909 free(fn); 910 if ((fn = rrdp_filename(rr, uri, try++)) == NULL) 911 return 0; 912 fd = open(fn, O_RDONLY); 913 } while (fd == -1 && try < 2); 914 915 if (!valid_filehash(fd, hash, hlen)) { 916 warnx("%s: bad file digest for %s", rr->notifyuri, fn); 917 free(fn); 918 return 0; 919 } 920 free(fn); 921 } 922 923 /* write new content or mark uri as deleted. */ 924 if (pt == PUB_DEL) { 925 filepath_add(&rr->deleted, uri, 0, 0); 926 } else { 927 fp = filepath_find(&rr->deleted, uri); 928 if (fp != NULL) { 929 filepath_put(&rr->deleted, fp); 930 deleted = 1; 931 } 932 933 /* add new file to rrdp dir */ 934 if ((fn = rrdp_filename(rr, uri, 0)) == NULL) 935 return 0; 936 937 if (repo_mkpath(AT_FDCWD, fn) == -1) 938 goto fail; 939 940 flags = O_WRONLY|O_CREAT|O_TRUNC; 941 if (pt == PUB_ADD && !deleted) 942 flags |= O_EXCL; 943 fd = open(fn, flags, 0644); 944 if (fd == -1) { 945 if (errno == EEXIST) { 946 warnx("%s: duplicate publish element for %s", 947 rr->notifyuri, fn); 948 free(fn); 949 return 0; 950 } 951 warn("open %s", fn); 952 goto fail; 953 } 954 955 if ((s = write(fd, data, dlen)) == -1) { 956 warn("write %s", fn); 957 goto fail; 958 } 959 close(fd); 960 if ((size_t)s != dlen) /* impossible */ 961 errx(1, "short write %s", fn); 962 free(fn); 963 } 964 965 return 1; 966 967fail: 968 rr->state = REPO_FAILED; 969 if (fd != -1) 970 close(fd); 971 free(fn); 972 return -1; 973} 974 975/* 976 * RSYNC sync finished, either with or without success. 977 */ 978void 979rsync_finish(unsigned int id, int ok) 980{ 981 struct rsyncrepo *rr; 982 struct tarepo *tr; 983 984 tr = ta_find(id); 985 if (tr != NULL) { 986 /* repository changed state already, ignore request */ 987 if (tr->state != REPO_LOADING) 988 return; 989 if (ok) { 990 logx("ta/%s: loaded from network", tr->descr); 991 stats.rsync_repos++; 992 tr->state = REPO_DONE; 993 repo_done(tr, 1); 994 } else { 995 warnx("ta/%s: load from network failed", tr->descr); 996 stats.rsync_fails++; 997 tr->uriidx++; 998 ta_fetch(tr); 999 } 1000 return; 1001 } 1002 1003 rr = rsync_find(id); 1004 if (rr == NULL) 1005 errx(1, "unknown rsync repo %u", id); 1006 /* repository changed state already, ignore request */ 1007 if (rr->state != REPO_LOADING) 1008 return; 1009 1010 if (ok) { 1011 logx("%s: loaded from network", rr->basedir); 1012 stats.rsync_repos++; 1013 rr->state = REPO_DONE; 1014 } else { 1015 warnx("%s: load from network failed, fallback to cache", 1016 rr->basedir); 1017 stats.rsync_fails++; 1018 rr->state = REPO_FAILED; 1019 /* clear rsync repo since it failed */ 1020 remove_contents(rr->basedir); 1021 } 1022 1023 repo_done(rr, ok); 1024} 1025 1026/* 1027 * RRDP sync finished, either with or without success. 1028 */ 1029void 1030rrdp_finish(unsigned int id, int ok) 1031{ 1032 struct rrdprepo *rr; 1033 1034 rr = rrdp_find(id); 1035 if (rr == NULL) 1036 errx(1, "unknown RRDP repo %u", id); 1037 /* repository changed state already, ignore request */ 1038 if (rr->state != REPO_LOADING) 1039 return; 1040 1041 if (ok) { 1042 logx("%s: loaded from network", rr->notifyuri); 1043 stats.rrdp_repos++; 1044 rr->state = REPO_DONE; 1045 } else { 1046 warnx("%s: load from network failed, fallback to %s", 1047 rr->notifyuri, nofetch ? "cache" : "rsync"); 1048 stats.rrdp_fails++; 1049 rr->state = REPO_FAILED; 1050 /* clear the RRDP repo since it failed */ 1051 remove_contents(rr->basedir); 1052 /* also clear the list of deleted files */ 1053 filepath_free(&rr->deleted); 1054 } 1055 1056 repo_done(rr, ok); 1057} 1058 1059/* 1060 * Handle responses from the http process. For TA file, either rename 1061 * or delete the temporary file. For RRDP requests relay the request 1062 * over to the rrdp process. 1063 */ 1064void 1065http_finish(unsigned int id, enum http_result res, const char *last_mod) 1066{ 1067 struct tarepo *tr; 1068 1069 tr = ta_find(id); 1070 if (tr == NULL) { 1071 /* not a TA fetch therefore RRDP */ 1072 rrdp_http_done(id, res, last_mod); 1073 return; 1074 } 1075 1076 /* repository changed state already, ignore request */ 1077 if (tr->state != REPO_LOADING) 1078 return; 1079 1080 /* Move downloaded TA file into place, or unlink on failure. */ 1081 if (res == HTTP_OK) { 1082 logx("ta/%s: loaded from network", tr->descr); 1083 tr->state = REPO_DONE; 1084 stats.http_repos++; 1085 repo_done(tr, 1); 1086 } else { 1087 remove_contents(tr->basedir); 1088 1089 tr->uriidx++; 1090 warnx("ta/%s: load from network failed", tr->descr); 1091 ta_fetch(tr); 1092 } 1093} 1094 1095/* 1096 * Look up a trust anchor, queueing it for download if not found. 1097 */ 1098struct repo * 1099ta_lookup(int id, struct tal *tal) 1100{ 1101 struct repo *rp; 1102 1103 if (tal->urisz == 0) 1104 errx(1, "TAL %s has no URI", tal->descr); 1105 1106 /* Look up in repository table. (Lookup should actually fail here) */ 1107 SLIST_FOREACH(rp, &repos, entry) { 1108 if (strcmp(rp->repouri, tal->uri[0]) == 0) 1109 return rp; 1110 } 1111 1112 rp = repo_alloc(id); 1113 rp->basedir = repo_dir(tal->descr, "ta", 0); 1114 if ((rp->repouri = strdup(tal->uri[0])) == NULL) 1115 err(1, NULL); 1116 1117 /* check if sync disabled ... */ 1118 if (noop) { 1119 logx("%s: using cache", rp->basedir); 1120 entityq_flush(&rp->queue, rp); 1121 return rp; 1122 } 1123 1124 /* try to create base directory */ 1125 if (mkpath(rp->basedir) == -1) 1126 warn("mkpath %s", rp->basedir); 1127 1128 rp->ta = ta_get(tal); 1129 1130 /* need to check if it was already loaded */ 1131 if (repo_state(rp) != REPO_LOADING) 1132 entityq_flush(&rp->queue, rp); 1133 1134 return rp; 1135} 1136 1137/* 1138 * Look up a repository, queueing it for discovery if not found. 1139 */ 1140struct repo * 1141repo_lookup(int talid, const char *uri, const char *notify) 1142{ 1143 struct repo *rp; 1144 char *repouri; 1145 1146 if ((repouri = rsync_base_uri(uri)) == NULL) 1147 errx(1, "bad caRepository URI: %s", uri); 1148 1149 /* Look up in repository table. */ 1150 SLIST_FOREACH(rp, &repos, entry) { 1151 if (strcmp(rp->repouri, repouri) != 0) 1152 continue; 1153 if (rp->notifyuri != NULL) { 1154 if (notify == NULL) 1155 continue; 1156 if (strcmp(rp->notifyuri, notify) != 0) 1157 continue; 1158 } else if (notify != NULL) 1159 continue; 1160 /* found matching repo */ 1161 free(repouri); 1162 return rp; 1163 } 1164 1165 rp = repo_alloc(talid); 1166 rp->basedir = repo_dir(repouri, NULL, 0); 1167 rp->repouri = repouri; 1168 if (notify != NULL) 1169 if ((rp->notifyuri = strdup(notify)) == NULL) 1170 err(1, NULL); 1171 1172 if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) { 1173 if (talrepocnt[talid] == MAX_REPO_PER_TAL) 1174 warnx("too many repositories under %s", tals[talid]); 1175 nofetch = 1; 1176 } 1177 1178 /* check if sync disabled ... */ 1179 if (noop || nofetch) { 1180 logx("%s: using cache", rp->basedir); 1181 entityq_flush(&rp->queue, rp); 1182 return rp; 1183 } 1184 1185 /* try to create base directory */ 1186 if (mkpath(rp->basedir) == -1) 1187 warn("mkpath %s", rp->basedir); 1188 1189 /* ... else try RRDP first if available then rsync */ 1190 if (notify != NULL) 1191 rp->rrdp = rrdp_get(notify); 1192 if (rp->rrdp == NULL) 1193 rp->rsync = rsync_get(uri, rp->basedir); 1194 1195 /* need to check if it was already loaded */ 1196 if (repo_state(rp) != REPO_LOADING) 1197 entityq_flush(&rp->queue, rp); 1198 1199 return rp; 1200} 1201 1202/* 1203 * Find repository by identifier. 1204 */ 1205struct repo * 1206repo_byid(unsigned int id) 1207{ 1208 struct repo *rp; 1209 1210 SLIST_FOREACH(rp, &repos, entry) { 1211 if (rp->id == id) 1212 return rp; 1213 } 1214 return NULL; 1215} 1216 1217/* 1218 * Find repository by base path. 1219 */ 1220static struct repo * 1221repo_bypath(const char *path) 1222{ 1223 struct repo *rp; 1224 1225 SLIST_FOREACH(rp, &repos, entry) { 1226 if (strcmp(rp->basedir, path) == 0) 1227 return rp; 1228 } 1229 return NULL; 1230} 1231 1232/* 1233 * Return the repository base or alternate directory. 1234 * Returned string must be freed by caller. 1235 */ 1236char * 1237repo_basedir(const struct repo *rp, int wantvalid) 1238{ 1239 char *path = NULL; 1240 1241 if (!wantvalid) { 1242 if (rp->ta) { 1243 if ((path = strdup(rp->ta->basedir)) == NULL) 1244 err(1, NULL); 1245 } else if (rp->rsync) { 1246 if ((path = strdup(rp->rsync->basedir)) == NULL) 1247 err(1, NULL); 1248 } else if (rp->rrdp) { 1249 path = rrdp_filename(rp->rrdp, rp->repouri, 0); 1250 } else 1251 path = NULL; /* only valid repo available */ 1252 } else if (rp->basedir != NULL) { 1253 if ((path = strdup(rp->basedir)) == NULL) 1254 err(1, NULL); 1255 } 1256 1257 return path; 1258} 1259 1260/* 1261 * Return the repository identifier. 1262 */ 1263unsigned int 1264repo_id(const struct repo *rp) 1265{ 1266 return rp->id; 1267} 1268 1269/* 1270 * Return the repository URI. 1271 */ 1272const char * 1273repo_uri(const struct repo *rp) 1274{ 1275 return rp->repouri; 1276} 1277 1278/* 1279 * Return the repository URI. 1280 */ 1281void 1282repo_fetch_uris(const struct repo *rp, const char **carepo, 1283 const char **notifyuri) 1284{ 1285 *carepo = rp->repouri; 1286 *notifyuri = rp->notifyuri; 1287} 1288 1289/* 1290 * Return 1 if repository is synced else 0. 1291 */ 1292int 1293repo_synced(const struct repo *rp) 1294{ 1295 if (repo_state(rp) == REPO_DONE && 1296 !(rp->rrdp == NULL && rp->rsync == NULL && rp->ta == NULL)) 1297 return 1; 1298 return 0; 1299} 1300 1301/* 1302 * Return the protocol string "rrdp", "rsync", "https" which was used to sync. 1303 * Result is only correct if repository was properly synced. 1304 */ 1305const char * 1306repo_proto(const struct repo *rp) 1307{ 1308 1309 if (rp->ta != NULL) { 1310 const struct tarepo *tr = rp->ta; 1311 if (tr->uriidx < tr->urisz && 1312 strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO, 1313 RSYNC_PROTO_LEN) == 0) 1314 return "rsync"; 1315 else 1316 return "https"; 1317 } 1318 if (rp->rrdp != NULL) 1319 return "rrdp"; 1320 return "rsync"; 1321} 1322 1323/* 1324 * Return the repository tal ID. 1325 */ 1326int 1327repo_talid(const struct repo *rp) 1328{ 1329 return rp->talid; 1330} 1331 1332int 1333repo_queued(struct repo *rp, struct entity *p) 1334{ 1335 if (repo_state(rp) == REPO_LOADING) { 1336 TAILQ_INSERT_TAIL(&rp->queue, p, entries); 1337 return 1; 1338 } 1339 return 0; 1340} 1341 1342static void 1343repo_fail(struct repo *rp) 1344{ 1345 /* reset the alarm since code may fallback to rsync */ 1346 rp->alarm = getmonotime() + repo_timeout; 1347 1348 if (rp->ta) 1349 http_finish(rp->ta->id, HTTP_FAILED, NULL); 1350 else if (rp->rsync) 1351 rsync_finish(rp->rsync->id, 0); 1352 else if (rp->rrdp) 1353 rrdp_finish(rp->rrdp->id, 0); 1354 else 1355 errx(1, "%s: bad repo", rp->repouri); 1356} 1357 1358static void 1359repo_abort(struct repo *rp) 1360{ 1361 /* reset the alarm */ 1362 rp->alarm = getmonotime() + repo_timeout; 1363 1364 if (rp->rsync) 1365 rsync_abort(rp->rsync->id); 1366 else if (rp->rrdp) 1367 rrdp_abort(rp->rrdp->id); 1368 else 1369 repo_fail(rp); 1370} 1371 1372int 1373repo_check_timeout(int timeout) 1374{ 1375 struct repo *rp; 1376 time_t now; 1377 int diff; 1378 1379 now = getmonotime(); 1380 1381 /* check against our runtime deadline first */ 1382 if (deadline != 0) { 1383 if (deadline <= now) { 1384 warnx("deadline reached, giving up on repository sync"); 1385 nofetch = 1; 1386 /* clear deadline since nofetch is set */ 1387 deadline = 0; 1388 /* increase now enough so that all pending repos fail */ 1389 now += repo_timeout; 1390 } else { 1391 diff = deadline - now; 1392 diff *= 1000; 1393 if (timeout == INFTIM || diff < timeout) 1394 timeout = diff; 1395 } 1396 } 1397 /* Look up in repository table. (Lookup should actually fail here) */ 1398 SLIST_FOREACH(rp, &repos, entry) { 1399 if (repo_state(rp) == REPO_LOADING) { 1400 if (rp->alarm <= now) { 1401 warnx("%s: synchronisation timeout", 1402 rp->repouri); 1403 repo_abort(rp); 1404 } else { 1405 diff = rp->alarm - now; 1406 diff *= 1000; 1407 if (timeout == INFTIM || diff < timeout) 1408 timeout = diff; 1409 } 1410 } 1411 } 1412 return timeout; 1413} 1414 1415/* 1416 * Update repo-specific stats when files are going to be moved 1417 * from DIR_TEMP to DIR_VALID. 1418 */ 1419void 1420repostats_new_files_inc(struct repo *rp, const char *file) 1421{ 1422 if (strncmp(file, ".rsync/", strlen(".rsync/")) == 0 || 1423 strncmp(file, ".rrdp/", strlen(".rrdp/")) == 0) 1424 rp->repostats.new_files++; 1425} 1426 1427/* 1428 * Update stats object of repository depending on rtype and subtype. 1429 */ 1430void 1431repo_stat_inc(struct repo *rp, int talid, enum rtype type, enum stype subtype) 1432{ 1433 if (rp == NULL) 1434 return; 1435 rp->stats_used[talid] = 1; 1436 switch (type) { 1437 case RTYPE_CER: 1438 if (subtype == STYPE_OK) 1439 rp->stats[talid].certs++; 1440 if (subtype == STYPE_FAIL) 1441 rp->stats[talid].certs_fail++; 1442 if (subtype == STYPE_BGPSEC) { 1443 rp->stats[talid].certs--; 1444 rp->stats[talid].brks++; 1445 } 1446 break; 1447 case RTYPE_MFT: 1448 if (subtype == STYPE_OK) 1449 rp->stats[talid].mfts++; 1450 if (subtype == STYPE_FAIL) 1451 rp->stats[talid].mfts_fail++; 1452 break; 1453 case RTYPE_ROA: 1454 switch (subtype) { 1455 case STYPE_OK: 1456 rp->stats[talid].roas++; 1457 break; 1458 case STYPE_FAIL: 1459 rp->stats[talid].roas_fail++; 1460 break; 1461 case STYPE_INVALID: 1462 rp->stats[talid].roas_invalid++; 1463 break; 1464 case STYPE_TOTAL: 1465 rp->stats[talid].vrps++; 1466 break; 1467 case STYPE_UNIQUE: 1468 rp->stats[talid].vrps_uniqs++; 1469 break; 1470 case STYPE_DEC_UNIQUE: 1471 rp->stats[talid].vrps_uniqs--; 1472 break; 1473 default: 1474 break; 1475 } 1476 break; 1477 case RTYPE_ASPA: 1478 switch (subtype) { 1479 case STYPE_OK: 1480 rp->stats[talid].aspas++; 1481 break; 1482 case STYPE_FAIL: 1483 rp->stats[talid].aspas_fail++; 1484 break; 1485 case STYPE_INVALID: 1486 rp->stats[talid].aspas_invalid++; 1487 break; 1488 case STYPE_TOTAL: 1489 rp->stats[talid].vaps++; 1490 break; 1491 case STYPE_UNIQUE: 1492 rp->stats[talid].vaps_uniqs++; 1493 break; 1494 case STYPE_DEC_UNIQUE: 1495 rp->stats[talid].vaps_uniqs--; 1496 break; 1497 case STYPE_PROVIDERS: 1498 rp->stats[talid].vaps_pas++; 1499 break; 1500 case STYPE_OVERFLOW: 1501 rp->stats[talid].vaps_overflowed++; 1502 break; 1503 default: 1504 break; 1505 } 1506 break; 1507 case RTYPE_SPL: 1508 switch (subtype) { 1509 case STYPE_OK: 1510 rp->stats[talid].spls++; 1511 break; 1512 case STYPE_FAIL: 1513 rp->stats[talid].spls_fail++; 1514 break; 1515 case STYPE_INVALID: 1516 rp->stats[talid].spls_invalid++; 1517 break; 1518 case STYPE_TOTAL: 1519 rp->stats[talid].vsps++; 1520 break; 1521 case STYPE_UNIQUE: 1522 rp->stats[talid].vsps_uniqs++; 1523 break; 1524 case STYPE_DEC_UNIQUE: 1525 rp->stats[talid].vsps_uniqs--; 1526 break; 1527 default: 1528 break; 1529 } 1530 break; 1531 case RTYPE_CRL: 1532 rp->stats[talid].crls++; 1533 break; 1534 case RTYPE_GBR: 1535 rp->stats[talid].gbrs++; 1536 break; 1537 case RTYPE_TAK: 1538 rp->stats[talid].taks++; 1539 break; 1540 default: 1541 break; 1542 } 1543} 1544 1545void 1546repo_tal_stats_collect(void (*cb)(const struct repo *, 1547 const struct repotalstats *, void *), int talid, void *arg) 1548{ 1549 struct repo *rp; 1550 1551 SLIST_FOREACH(rp, &repos, entry) { 1552 if (rp->stats_used[talid]) 1553 cb(rp, &rp->stats[talid], arg); 1554 } 1555} 1556 1557void 1558repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, 1559 void *), void *arg) 1560{ 1561 struct repo *rp; 1562 1563 SLIST_FOREACH(rp, &repos, entry) 1564 cb(rp, &rp->repostats, arg); 1565} 1566 1567/* 1568 * Delayed delete of files from RRDP. Since RRDP has no security built-in 1569 * this code needs to check if this RRDP repository is actually allowed to 1570 * remove the file referenced by the URI. 1571 */ 1572static void 1573repo_cleanup_rrdp(struct filepath_tree *tree) 1574{ 1575 struct repo *rp; 1576 struct rrdprepo *rr; 1577 struct filepath *fp, *nfp; 1578 char *fn; 1579 1580 SLIST_FOREACH(rp, &repos, entry) { 1581 if (rp->rrdp == NULL) 1582 continue; 1583 rr = (struct rrdprepo *)rp->rrdp; 1584 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { 1585 if (!rrdp_uri_valid(rr, fp->file)) { 1586 warnx("%s: external URI %s", rr->notifyuri, 1587 fp->file); 1588 filepath_put(&rr->deleted, fp); 1589 continue; 1590 } 1591 /* try to remove file from rrdp repo ... */ 1592 fn = rrdp_filename(rr, fp->file, 0); 1593 1594 if (unlink(fn) == -1) { 1595 if (errno != ENOENT) 1596 warn("unlink %s", fn); 1597 } else { 1598 if (verbose > 1) 1599 logx("deleted %s", fn); 1600 rp->repostats.del_files++; 1601 } 1602 free(fn); 1603 1604 /* ... and from the valid repository if unused. */ 1605 fn = rrdp_filename(rr, fp->file, 1); 1606 if (!filepath_exists(tree, fn)) { 1607 if (unlink(fn) == -1) { 1608 if (errno != ENOENT) 1609 warn("unlink %s", fn); 1610 } else { 1611 if (verbose > 1) 1612 logx("deleted %s", fn); 1613 rp->repostats.del_files++; 1614 } 1615 } else 1616 warnx("%s: referenced file supposed to be " 1617 "deleted", fn); 1618 1619 free(fn); 1620 filepath_put(&rr->deleted, fp); 1621 } 1622 } 1623} 1624 1625/* 1626 * All files in tree are valid and should be moved to the valid repository 1627 * if not already there. Rename the files to the new path and readd the 1628 * filepath entry with the new path if successful. 1629 */ 1630static void 1631repo_move_valid(struct filepath_tree *tree) 1632{ 1633 struct filepath *fp, *nfp; 1634 size_t rsyncsz = strlen(".rsync/"); 1635 size_t rrdpsz = strlen(".rrdp/"); 1636 size_t tasz = strlen(".ta/"); 1637 char *fn, *base; 1638 1639 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) { 1640 if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 && 1641 strncmp(fp->file, ".rrdp/", rrdpsz) != 0 && 1642 strncmp(fp->file, ".ta/", tasz) != 0) 1643 continue; /* not a temporary file path */ 1644 1645 if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) { 1646 fn = fp->file + rsyncsz; 1647 } else if (strncmp(fp->file, ".ta/", tasz) == 0) { 1648 fn = fp->file + 1; /* just skip the '.' */ 1649 } else { 1650 base = strchr(fp->file + rrdpsz, '/'); 1651 assert(base != NULL); 1652 fn = base + 1; 1653 1654 /* 1655 * Adjust file last modification time in order to 1656 * minimize RSYNC synchronization load after transport 1657 * failover. 1658 * While serializing RRDP datastructures to disk, set 1659 * the last modified timestamp to the CMS signing-time, 1660 * the X.509 notBefore, or CRL lastUpdate timestamp. 1661 */ 1662 if (fp->mtime != 0) { 1663 int ret; 1664 struct timespec ts[2]; 1665 1666 ts[0].tv_nsec = UTIME_OMIT; 1667 ts[1].tv_sec = fp->mtime; 1668 ts[1].tv_nsec = 0; 1669 ret = utimensat(AT_FDCWD, fp->file, ts, 0); 1670 if (ret == -1) { 1671 warn("utimensat %s", fp->file); 1672 continue; 1673 } 1674 } 1675 } 1676 1677 if (repo_mkpath(AT_FDCWD, fn) == -1) 1678 continue; 1679 1680 if (rename(fp->file, fn) == -1) { 1681 warn("rename %s", fp->file); 1682 continue; 1683 } 1684 1685 /* switch filepath node to new path */ 1686 RB_REMOVE(filepath_tree, tree, fp); 1687 base = fp->file; 1688 if ((fp->file = strdup(fn)) == NULL) 1689 err(1, NULL); 1690 free(base); 1691 if (RB_INSERT(filepath_tree, tree, fp) != NULL) 1692 errx(1, "%s: both possibilities of file present", 1693 fp->file); 1694 } 1695} 1696 1697struct fts_state { 1698 enum { BASE_DIR, RSYNC_DIR, TA_DIR, RRDP_DIR } type; 1699 struct repo *rp; 1700} fts_state; 1701 1702static const struct rrdprepo * 1703repo_is_rrdp(struct repo *rp) 1704{ 1705 /* check for special pointers first these are not a repository */ 1706 if (rp != NULL && rp->rrdp != NULL) 1707 return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL; 1708 return NULL; 1709} 1710 1711static inline char * 1712skip_dotslash(char *in) 1713{ 1714 if (memcmp(in, "./", 2) == 0) 1715 return in + 2; 1716 return in; 1717} 1718 1719static void 1720repo_cleanup_entry(FTSENT *e, struct filepath_tree *tree, int cachefd) 1721{ 1722 const struct rrdprepo *rr; 1723 char *path; 1724 1725 path = skip_dotslash(e->fts_path); 1726 switch (e->fts_info) { 1727 case FTS_NSOK: 1728 if (filepath_exists(tree, path)) { 1729 e->fts_parent->fts_number++; 1730 break; 1731 } 1732 if (fts_state.type == RRDP_DIR && fts_state.rp != NULL) { 1733 e->fts_parent->fts_number++; 1734 /* handle rrdp .state files explicitly */ 1735 if (e->fts_level == 3 && 1736 strcmp(e->fts_name, ".state") == 0) 1737 break; 1738 /* can't delete these extra files */ 1739 fts_state.rp->repostats.extra_files++; 1740 if (verbose > 1) 1741 logx("superfluous %s", path); 1742 break; 1743 } 1744 rr = repo_is_rrdp(fts_state.rp); 1745 if (rr != NULL) { 1746 struct stat st; 1747 char *fn; 1748 1749 if (asprintf(&fn, "%s/%s", rr->basedir, path) == -1) 1750 err(1, NULL); 1751 1752 /* 1753 * If the file exists in the rrdp dir 1754 * that file is newer and needs to be kept 1755 * so unlink this file instead of moving 1756 * it over the file in the rrdp dir. 1757 */ 1758 if (fstatat(cachefd, fn, &st, 0) == 0 && 1759 S_ISREG(st.st_mode)) { 1760 free(fn); 1761 goto unlink; 1762 } 1763 if (repo_mkpath(cachefd, fn) == 0) { 1764 if (renameat(AT_FDCWD, e->fts_accpath, 1765 cachefd, fn) == -1) 1766 warn("rename %s to %s", path, fn); 1767 else if (verbose > 1) 1768 logx("moved %s", path); 1769 fts_state.rp->repostats.extra_files++; 1770 } 1771 free(fn); 1772 } else { 1773 unlink: 1774 if (unlink(e->fts_accpath) == -1) { 1775 warn("unlink %s", path); 1776 } else if (fts_state.type == RSYNC_DIR || 1777 fts_state.type == TA_DIR) { 1778 /* no need to keep rsync or ta files */ 1779 if (verbose > 1) 1780 logx("deleted superfluous %s", path); 1781 if (fts_state.rp != NULL) 1782 fts_state.rp->repostats.del_extra_files++; 1783 else 1784 stats.repo_stats.del_extra_files++; 1785 } else { 1786 if (verbose > 1) 1787 logx("deleted %s", path); 1788 if (fts_state.rp != NULL) 1789 fts_state.rp->repostats.del_files++; 1790 else 1791 stats.repo_stats.del_files++; 1792 } 1793 } 1794 break; 1795 case FTS_D: 1796 if (e->fts_level == FTS_ROOTLEVEL) { 1797 fts_state.type = BASE_DIR; 1798 fts_state.rp = NULL; 1799 } 1800 if (e->fts_level == 1) { 1801 /* rpki.example.org or .rrdp / .rsync / .ta */ 1802 if (strcmp(".rsync", e->fts_name) == 0) 1803 fts_state.type = RSYNC_DIR; 1804 else if (strcmp(".ta", e->fts_name) == 0) 1805 fts_state.type = TA_DIR; 1806 else if (strcmp(".rrdp", e->fts_name) == 0) 1807 fts_state.type = RRDP_DIR; 1808 else 1809 fts_state.type = BASE_DIR; 1810 fts_state.rp = NULL; 1811 } 1812 if (e->fts_level == 2) { 1813 /* rpki.example.org/repository or .rrdp/hashdir */ 1814 if (fts_state.type == BASE_DIR) 1815 fts_state.rp = repo_bypath(path); 1816 if (fts_state.type == TA_DIR) 1817 fts_state.rp = repo_bypath(path + 1); 1818 /* 1819 * special handling for rrdp directories, 1820 * clear them if they are not used anymore but 1821 * only if rrdp is active. 1822 * Look them up just using the hash. 1823 */ 1824 if (fts_state.type == RRDP_DIR) 1825 fts_state.rp = repo_rrdp_bypath(path); 1826 } 1827 if (e->fts_level == 3 && fts_state.type == RSYNC_DIR) { 1828 /* .rsync/rpki.example.org/repository */ 1829 fts_state.rp = repo_bypath(path + strlen(".rsync/")); 1830 } 1831 break; 1832 case FTS_DP: 1833 if (e->fts_level == FTS_ROOTLEVEL) 1834 break; 1835 if (e->fts_level == 1) { 1836 /* do not remove .rsync and .rrdp */ 1837 fts_state.rp = NULL; 1838 if (fts_state.type == RRDP_DIR || 1839 fts_state.type == RSYNC_DIR || 1840 fts_state.type == TA_DIR) 1841 break; 1842 } 1843 1844 e->fts_parent->fts_number += e->fts_number; 1845 1846 if (e->fts_number == 0) { 1847 if (rmdir(e->fts_accpath) == -1) 1848 warn("rmdir %s", path); 1849 if (fts_state.rp != NULL) 1850 fts_state.rp->repostats.del_dirs++; 1851 else 1852 stats.repo_stats.del_dirs++; 1853 } 1854 break; 1855 case FTS_SL: 1856 case FTS_SLNONE: 1857 warnx("symlink %s", path); 1858 if (unlink(e->fts_accpath) == -1) 1859 warn("unlink %s", path); 1860 stats.repo_stats.del_extra_files++; 1861 break; 1862 case FTS_NS: 1863 case FTS_ERR: 1864 if (e->fts_errno == ENOENT && e->fts_level == FTS_ROOTLEVEL) 1865 break; 1866 warnx("fts_read %s: %s", path, strerror(e->fts_errno)); 1867 break; 1868 default: 1869 warnx("fts_read %s: unhandled[%x]", path, e->fts_info); 1870 break; 1871 } 1872} 1873 1874void 1875repo_cleanup(struct filepath_tree *tree, int cachefd) 1876{ 1877 char *argv[2] = { ".", NULL }; 1878 FTS *fts; 1879 FTSENT *e; 1880 1881 /* first move temp files which have been used to valid dir */ 1882 repo_move_valid(tree); 1883 /* then delete files requested by rrdp */ 1884 repo_cleanup_rrdp(tree); 1885 1886 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1887 err(1, "fts_open"); 1888 errno = 0; 1889 while ((e = fts_read(fts)) != NULL) { 1890 repo_cleanup_entry(e, tree, cachefd); 1891 errno = 0; 1892 } 1893 if (errno) 1894 err(1, "fts_read"); 1895 if (fts_close(fts) == -1) 1896 err(1, "fts_close"); 1897} 1898 1899void 1900repo_free(void) 1901{ 1902 struct repo *rp; 1903 1904 while ((rp = SLIST_FIRST(&repos)) != NULL) { 1905 SLIST_REMOVE_HEAD(&repos, entry); 1906 free(rp->repouri); 1907 free(rp->notifyuri); 1908 free(rp->basedir); 1909 free(rp); 1910 } 1911 1912 ta_free(); 1913 rrdp_free(); 1914 rsync_free(); 1915} 1916 1917/* 1918 * Remove all files and directories under base. 1919 * Do not remove base directory itself and the .state file. 1920 */ 1921static void 1922remove_contents(char *base) 1923{ 1924 char *argv[2] = { base, NULL }; 1925 FTS *fts; 1926 FTSENT *e; 1927 1928 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1929 err(1, "fts_open"); 1930 errno = 0; 1931 while ((e = fts_read(fts)) != NULL) { 1932 switch (e->fts_info) { 1933 case FTS_NSOK: 1934 case FTS_SL: 1935 case FTS_SLNONE: 1936 if (e->fts_level == 1 && 1937 strcmp(e->fts_name, ".state") == 0) 1938 break; 1939 if (unlink(e->fts_accpath) == -1) 1940 warn("unlink %s", e->fts_path); 1941 break; 1942 case FTS_D: 1943 break; 1944 case FTS_DP: 1945 /* keep root directory */ 1946 if (e->fts_level == FTS_ROOTLEVEL) 1947 break; 1948 if (rmdir(e->fts_accpath) == -1) 1949 warn("rmdir %s", e->fts_path); 1950 break; 1951 case FTS_NS: 1952 case FTS_ERR: 1953 warnx("fts_read %s: %s", e->fts_path, 1954 strerror(e->fts_errno)); 1955 break; 1956 default: 1957 warnx("unhandled[%x] %s", e->fts_info, 1958 e->fts_path); 1959 break; 1960 } 1961 errno = 0; 1962 } 1963 if (errno) 1964 err(1, "fts_read"); 1965 if (fts_close(fts) == -1) 1966 err(1, "fts_close"); 1967} 1968