1/*	$OpenBSD: repo.c,v 1.60 2024/06/07 08:22:53 claudio Exp $ */
2/*
3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org>
4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/queue.h>
20#include <sys/tree.h>
21#include <sys/types.h>
22#include <sys/stat.h>
23
24#include <assert.h>
25#include <err.h>
26#include <errno.h>
27#include <fcntl.h>
28#include <fts.h>
29#include <limits.h>
30#include <poll.h>
31#include <stdio.h>
32#include <stdlib.h>
33#include <string.h>
34#include <unistd.h>
35
36#include <imsg.h>
37
38#include "extern.h"
39
40extern struct stats	stats;
41extern int		rrdpon;
42extern int		repo_timeout;
43extern time_t		deadline;
44int			nofetch;
45
46enum repo_state {
47	REPO_LOADING = 0,
48	REPO_DONE = 1,
49	REPO_FAILED = -1,
50};
51
52/*
53 * A ta, rsync or rrdp repository.
54 * Depending on what is needed the generic repository is backed by
55 * a ta, rsync or rrdp repository. Multiple repositories can use the
56 * same backend.
57 */
58struct rrdprepo {
59	SLIST_ENTRY(rrdprepo)	 entry;
60	char			*notifyuri;
61	char			*basedir;
62	struct filepath_tree	 deleted;
63	unsigned int		 id;
64	enum repo_state		 state;
65};
66static SLIST_HEAD(, rrdprepo)	rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos);
67
68struct rsyncrepo {
69	SLIST_ENTRY(rsyncrepo)	 entry;
70	char			*repouri;
71	char			*basedir;
72	unsigned int		 id;
73	enum repo_state		 state;
74};
75static SLIST_HEAD(, rsyncrepo)	rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos);
76
77struct tarepo {
78	SLIST_ENTRY(tarepo)	 entry;
79	char			*descr;
80	char			*basedir;
81	char			**uri;
82	size_t			 urisz;
83	size_t			 uriidx;
84	unsigned int		 id;
85	enum repo_state		 state;
86};
87static SLIST_HEAD(, tarepo)	tarepos = SLIST_HEAD_INITIALIZER(tarepos);
88
89struct repo {
90	SLIST_ENTRY(repo)	 entry;
91	char			*repouri;
92	char			*notifyuri;
93	char			*basedir;
94	const struct rrdprepo	*rrdp;
95	const struct rsyncrepo	*rsync;
96	const struct tarepo	*ta;
97	struct entityq		 queue;		/* files waiting for repo */
98	struct repotalstats	 stats[TALSZ_MAX];
99	struct repostats	 repostats;
100	struct timespec		 start_time;
101	time_t			 alarm;		/* sync timeout */
102	int			 talid;
103	int			 stats_used[TALSZ_MAX];
104	unsigned int		 id;		/* identifier */
105};
106static SLIST_HEAD(, repo)	repos = SLIST_HEAD_INITIALIZER(repos);
107
108/* counter for unique repo id */
109unsigned int		repoid;
110
111static struct rsyncrepo	*rsync_get(const char *, const char *);
112static void		 remove_contents(char *);
113
114/*
115 * Database of all file path accessed during a run.
116 */
117struct filepath {
118	RB_ENTRY(filepath)	 entry;
119	char			*file;
120	time_t			 mtime;
121	unsigned int		 talmask;
122};
123
124static inline int
125filepathcmp(const struct filepath *a, const struct filepath *b)
126{
127	return strcmp(a->file, b->file);
128}
129
130RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp);
131
132/*
133 * Functions to lookup which files have been accessed during computation.
134 */
135int
136filepath_add(struct filepath_tree *tree, char *file, int id, time_t mtime)
137{
138	struct filepath *fp, *rfp;
139
140	CTASSERT(TALSZ_MAX < 8 * sizeof(fp->talmask));
141	assert(id >= 0 && id < 8 * (int)sizeof(fp->talmask));
142
143	if ((fp = calloc(1, sizeof(*fp))) == NULL)
144		err(1, NULL);
145	if ((fp->file = strdup(file)) == NULL)
146		err(1, NULL);
147	fp->mtime = mtime;
148
149	if ((rfp = RB_INSERT(filepath_tree, tree, fp)) != NULL) {
150		/* already in the tree */
151		free(fp->file);
152		free(fp);
153		if (rfp->talmask & (1 << id))
154			return 0;
155		fp = rfp;
156	}
157	fp->talmask |= (1 << id);
158
159	return 1;
160}
161
162/*
163 * Lookup a file path in the tree and return the object if found or NULL.
164 */
165static struct filepath *
166filepath_find(struct filepath_tree *tree, char *file)
167{
168	struct filepath needle = { .file = file };
169
170	return RB_FIND(filepath_tree, tree, &needle);
171}
172
173/*
174 * Returns true if file exists in the tree.
175 */
176static int
177filepath_exists(struct filepath_tree *tree, char *file)
178{
179	return filepath_find(tree, file) != NULL;
180}
181
182/*
183 * Remove entry from tree and free it.
184 */
185static void
186filepath_put(struct filepath_tree *tree, struct filepath *fp)
187{
188	RB_REMOVE(filepath_tree, tree, fp);
189	free((void *)fp->file);
190	free(fp);
191}
192
193/*
194 * Free all elements of a filepath tree.
195 */
196static void
197filepath_free(struct filepath_tree *tree)
198{
199	struct filepath *fp, *nfp;
200
201	RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp)
202		filepath_put(tree, fp);
203}
204
205RB_GENERATE(filepath_tree, filepath, entry, filepathcmp);
206
207/*
208 * Function to hash a string into a unique directory name.
209 * Returned hash needs to be freed.
210 */
211static char *
212hash_dir(const char *uri)
213{
214	unsigned char m[SHA256_DIGEST_LENGTH];
215
216	SHA256(uri, strlen(uri), m);
217	return hex_encode(m, sizeof(m));
218}
219
220/*
221 * Function to build the directory name based on URI and a directory
222 * as prefix. Skip the proto:// in URI but keep everything else.
223 */
224static char *
225repo_dir(const char *uri, const char *dir, int hash)
226{
227	const char *local;
228	char *out, *hdir = NULL;
229
230	if (hash) {
231		local = hdir = hash_dir(uri);
232	} else {
233		local = strchr(uri, ':');
234		if (local != NULL)
235			local += strlen("://");
236		else
237			local = uri;
238	}
239
240	if (dir == NULL) {
241		if ((out = strdup(local)) == NULL)
242			err(1, NULL);
243	} else {
244		if (asprintf(&out, "%s/%s", dir, local) == -1)
245			err(1, NULL);
246	}
247
248	free(hdir);
249	return out;
250}
251
252/*
253 * Function to create all missing directories to a path.
254 * This functions alters the path temporarily.
255 */
256static int
257repo_mkpath(int fd, char *file)
258{
259	char *slash;
260
261	/* build directory hierarchy */
262	slash = strrchr(file, '/');
263	assert(slash != NULL);
264	*slash = '\0';
265	if (mkpathat(fd, file) == -1) {
266		warn("mkpath %s", file);
267		return -1;
268	}
269	*slash = '/';
270	return 0;
271}
272
273/*
274 * Return the state of a repository.
275 */
276static enum repo_state
277repo_state(const struct repo *rp)
278{
279	if (rp->ta)
280		return rp->ta->state;
281	if (rp->rsync)
282		return rp->rsync->state;
283	if (rp->rrdp)
284		return rp->rrdp->state;
285	/* No backend so sync is by definition done. */
286	return REPO_DONE;
287}
288
289/*
290 * Function called once a repository is done with the sync. Either
291 * successfully or after failure.
292 */
293static void
294repo_done(const void *vp, int ok)
295{
296	struct repo *rp;
297	struct timespec flush_time;
298
299	SLIST_FOREACH(rp, &repos, entry) {
300		if (vp != rp->ta && vp != rp->rsync && vp != rp->rrdp)
301			continue;
302
303		/* for rrdp try to fall back to rsync */
304		if (vp == rp->rrdp && !ok && !nofetch) {
305			rp->rrdp = NULL;
306			rp->rsync = rsync_get(rp->repouri, rp->basedir);
307			/* need to check if it was already loaded */
308			if (repo_state(rp) == REPO_LOADING)
309				continue;
310		}
311
312		entityq_flush(&rp->queue, rp);
313		clock_gettime(CLOCK_MONOTONIC, &flush_time);
314		timespecsub(&flush_time, &rp->start_time,
315		    &rp->repostats.sync_time);
316	}
317}
318
319/*
320 * Build TA file name based on the repo info.
321 * If temp is set add Xs for mkostemp.
322 */
323static char *
324ta_filename(const struct tarepo *tr)
325{
326	const char *file;
327	char *nfile;
328
329	/* does not matter which URI, all end with same filename */
330	file = strrchr(tr->uri[0], '/');
331	assert(file);
332
333	if (asprintf(&nfile, "%s%s", tr->basedir, file) == -1)
334		err(1, NULL);
335
336	return nfile;
337}
338
339static void
340ta_fetch(struct tarepo *tr)
341{
342	if (!rrdpon) {
343		for (; tr->uriidx < tr->urisz; tr->uriidx++) {
344			if (strncasecmp(tr->uri[tr->uriidx],
345			    RSYNC_PROTO, RSYNC_PROTO_LEN) == 0)
346				break;
347		}
348	}
349
350	if (tr->uriidx >= tr->urisz) {
351		tr->state = REPO_FAILED;
352		logx("ta/%s: fallback to cache", tr->descr);
353
354		repo_done(tr, 0);
355		return;
356	}
357
358	logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]);
359
360	if (strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO,
361	    RSYNC_PROTO_LEN) == 0) {
362		/*
363		 * Create destination location.
364		 * Build up the tree to this point.
365		 */
366		rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL);
367	} else {
368		char *temp;
369		int fd;
370
371		temp = ta_filename(tr);
372		fd = open(temp,
373		    O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC,
374		    S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
375		if (fd == -1) {
376			warn("open: %s", temp);
377			free(temp);
378			http_finish(tr->id, HTTP_FAILED, NULL);
379			return;
380		}
381
382		free(temp);
383		http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd);
384	}
385}
386
387static struct tarepo *
388ta_get(struct tal *tal)
389{
390	struct tarepo *tr;
391
392	/* no need to look for possible other repo */
393
394	if ((tr = calloc(1, sizeof(*tr))) == NULL)
395		err(1, NULL);
396
397	tr->id = ++repoid;
398	SLIST_INSERT_HEAD(&tarepos, tr, entry);
399
400	if ((tr->descr = strdup(tal->descr)) == NULL)
401		err(1, NULL);
402	tr->basedir = repo_dir(tal->descr, ".ta", 0);
403
404	/* create base directory */
405	if (mkpath(tr->basedir) == -1) {
406		warn("mkpath %s", tr->basedir);
407		tr->state = REPO_FAILED;
408		repo_done(tr, 0);
409		return tr;
410	}
411
412	/* steal URI information from TAL */
413	tr->urisz = tal->urisz;
414	tr->uri = tal->uri;
415	tal->urisz = 0;
416	tal->uri = NULL;
417
418	ta_fetch(tr);
419
420	return tr;
421}
422
423static struct tarepo *
424ta_find(unsigned int id)
425{
426	struct tarepo *tr;
427
428	SLIST_FOREACH(tr, &tarepos, entry)
429		if (id == tr->id)
430			break;
431	return tr;
432}
433
434static void
435ta_free(void)
436{
437	struct tarepo *tr;
438
439	while ((tr = SLIST_FIRST(&tarepos)) != NULL) {
440		SLIST_REMOVE_HEAD(&tarepos, entry);
441		free(tr->descr);
442		free(tr->basedir);
443		free(tr->uri);
444		free(tr);
445	}
446}
447
448static struct rsyncrepo *
449rsync_get(const char *uri, const char *validdir)
450{
451	struct rsyncrepo *rr;
452	char *repo;
453
454	if ((repo = rsync_base_uri(uri)) == NULL)
455		errx(1, "bad caRepository URI: %s", uri);
456
457	SLIST_FOREACH(rr, &rsyncrepos, entry)
458		if (strcmp(rr->repouri, repo) == 0) {
459			free(repo);
460			return rr;
461		}
462
463	if ((rr = calloc(1, sizeof(*rr))) == NULL)
464		err(1, NULL);
465
466	rr->id = ++repoid;
467	SLIST_INSERT_HEAD(&rsyncrepos, rr, entry);
468
469	rr->repouri = repo;
470	rr->basedir = repo_dir(repo, ".rsync", 0);
471
472	/* create base directory */
473	if (mkpath(rr->basedir) == -1) {
474		warn("mkpath %s", rr->basedir);
475		rsync_finish(rr->id, 0);
476		return rr;
477	}
478
479	logx("%s: pulling from %s", rr->basedir, rr->repouri);
480	rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir);
481
482	return rr;
483}
484
485static struct rsyncrepo *
486rsync_find(unsigned int id)
487{
488	struct rsyncrepo *rr;
489
490	SLIST_FOREACH(rr, &rsyncrepos, entry)
491		if (id == rr->id)
492			break;
493	return rr;
494}
495
496static void
497rsync_free(void)
498{
499	struct rsyncrepo *rr;
500
501	while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) {
502		SLIST_REMOVE_HEAD(&rsyncrepos, entry);
503		free(rr->repouri);
504		free(rr->basedir);
505		free(rr);
506	}
507}
508
509/*
510 * Build local file name base on the URI and the rrdprepo info.
511 */
512static char *
513rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid)
514{
515	char *nfile;
516	const char *dir = rr->basedir;
517
518	if (!valid_uri(uri, strlen(uri), RSYNC_PROTO))
519		errx(1, "%s: bad URI %s", rr->basedir, uri);
520	uri += RSYNC_PROTO_LEN;	/* skip proto */
521	if (valid) {
522		if ((nfile = strdup(uri)) == NULL)
523			err(1, NULL);
524	} else {
525		if (asprintf(&nfile, "%s/%s", dir, uri) == -1)
526			err(1, NULL);
527	}
528	return nfile;
529}
530
531/*
532 * Build RRDP state file name based on the repo info.
533 * If temp is set add Xs for mkostemp.
534 */
535static char *
536rrdp_state_filename(const struct rrdprepo *rr, int temp)
537{
538	char *nfile;
539
540	if (asprintf(&nfile, "%s/.state%s", rr->basedir,
541	    temp ? ".XXXXXXXX" : "") == -1)
542		err(1, NULL);
543
544	return nfile;
545}
546
547static struct rrdprepo *
548rrdp_find(unsigned int id)
549{
550	struct rrdprepo *rr;
551
552	SLIST_FOREACH(rr, &rrdprepos, entry)
553		if (id == rr->id)
554			break;
555	return rr;
556}
557
558static void
559rrdp_free(void)
560{
561	struct rrdprepo *rr;
562
563	while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) {
564		SLIST_REMOVE_HEAD(&rrdprepos, entry);
565
566		free(rr->notifyuri);
567		free(rr->basedir);
568
569		filepath_free(&rr->deleted);
570
571		free(rr);
572	}
573}
574
575/*
576 * Check if a directory is an active rrdp repository.
577 * Returns 1 if found else 0.
578 */
579static struct repo *
580repo_rrdp_bypath(const char *dir)
581{
582	struct repo *rp;
583
584	SLIST_FOREACH(rp, &repos, entry) {
585		if (rp->rrdp == NULL)
586			continue;
587		if (strcmp(dir, rp->rrdp->basedir) == 0)
588			return rp;
589	}
590	return NULL;
591}
592
593/*
594 * Check if the URI is actually covered by one of the repositories
595 * that depend on this RRDP repository.
596 * Returns 1 if the URI is valid, 0 if no repouri matches the URI.
597 */
598static int
599rrdp_uri_valid(struct rrdprepo *rr, const char *uri)
600{
601	struct repo *rp;
602
603	SLIST_FOREACH(rp, &repos, entry) {
604		if (rp->rrdp != rr)
605			continue;
606		if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0)
607			return 1;
608	}
609	return 0;
610}
611
612/*
613 * Allocate and insert a new repository.
614 */
615static struct repo *
616repo_alloc(int talid)
617{
618	struct repo *rp;
619
620	if ((rp = calloc(1, sizeof(*rp))) == NULL)
621		err(1, NULL);
622
623	rp->id = ++repoid;
624	rp->talid = talid;
625	rp->alarm = getmonotime() + repo_timeout;
626	TAILQ_INIT(&rp->queue);
627	SLIST_INSERT_HEAD(&repos, rp, entry);
628	clock_gettime(CLOCK_MONOTONIC, &rp->start_time);
629
630	stats.repos++;
631	return rp;
632}
633
634/*
635 * Parse the RRDP state file if it exists and set the session struct
636 * based on that information.
637 */
638static struct rrdp_session *
639rrdp_session_parse(const struct rrdprepo *rr)
640{
641	FILE *f;
642	struct rrdp_session *state;
643	int fd, ln = 0, deltacnt = 0;
644	const char *errstr;
645	char *line = NULL, *file;
646	size_t len = 0;
647	ssize_t n;
648
649	if ((state = calloc(1, sizeof(*state))) == NULL)
650		err(1, NULL);
651
652	file = rrdp_state_filename(rr, 0);
653	if ((fd = open(file, O_RDONLY)) == -1) {
654		if (errno != ENOENT)
655			warn("%s: open state file", rr->basedir);
656		free(file);
657		return state;
658	}
659	free(file);
660	f = fdopen(fd, "r");
661	if (f == NULL)
662		err(1, "fdopen");
663
664	while ((n = getline(&line, &len, f)) != -1) {
665		if (line[n - 1] == '\n')
666			line[n - 1] = '\0';
667		switch (ln) {
668		case 0:
669			if ((state->session_id = strdup(line)) == NULL)
670				err(1, NULL);
671			break;
672		case 1:
673			state->serial = strtonum(line, 1, LLONG_MAX, &errstr);
674			if (errstr)
675				goto fail;
676			break;
677		case 2:
678			if (strcmp(line, "-") == 0)
679				break;
680			if ((state->last_mod = strdup(line)) == NULL)
681				err(1, NULL);
682			break;
683		default:
684			if (deltacnt >= MAX_RRDP_DELTAS)
685				goto fail;
686			if ((state->deltas[deltacnt++] = strdup(line)) == NULL)
687				err(1, NULL);
688			break;
689		}
690		ln++;
691	}
692
693	if (ferror(f))
694		goto fail;
695	fclose(f);
696	free(line);
697	return state;
698
699 fail:
700	warnx("%s: troubles reading state file", rr->basedir);
701	fclose(f);
702	free(line);
703	free(state->session_id);
704	free(state->last_mod);
705	memset(state, 0, sizeof(*state));
706	return state;
707}
708
709/*
710 * Carefully write the RRDP session state file back.
711 */
712void
713rrdp_session_save(unsigned int id, struct rrdp_session *state)
714{
715	struct rrdprepo *rr;
716	char *temp, *file;
717	FILE *f = NULL;
718	int fd, i;
719
720	rr = rrdp_find(id);
721	if (rr == NULL)
722		errx(1, "non-existent rrdp repo %u", id);
723
724	file = rrdp_state_filename(rr, 0);
725	temp = rrdp_state_filename(rr, 1);
726
727	if ((fd = mkostemp(temp, O_CLOEXEC)) == -1)
728		goto fail;
729	(void)fchmod(fd, 0644);
730	f = fdopen(fd, "w");
731	if (f == NULL)
732		err(1, "fdopen");
733
734	/* write session state file out */
735	if (fprintf(f, "%s\n%lld\n", state->session_id,
736	    state->serial) < 0)
737		goto fail;
738
739	if (state->last_mod != NULL) {
740		if (fprintf(f, "%s\n", state->last_mod) < 0)
741			goto fail;
742	} else {
743		if (fprintf(f, "-\n") < 0)
744			goto fail;
745	}
746	for (i = 0; i < MAX_RRDP_DELTAS && state->deltas[i] != NULL; i++) {
747		if (fprintf(f, "%s\n", state->deltas[i]) < 0)
748			goto fail;
749	}
750	if (fclose(f) != 0) {
751		f = NULL;
752		goto fail;
753	}
754
755	if (rename(temp, file) == -1) {
756		warn("%s: rename %s to %s", rr->basedir, temp, file);
757		unlink(temp);
758	}
759
760	free(temp);
761	free(file);
762	return;
763
764 fail:
765	warn("%s: save state to %s", rr->basedir, temp);
766	if (f != NULL)
767		fclose(f);
768	unlink(temp);
769	free(temp);
770	free(file);
771}
772
773/*
774 * Free an rrdp_session pointer. Safe to call with NULL.
775 */
776void
777rrdp_session_free(struct rrdp_session *s)
778{
779	size_t i;
780
781	if (s == NULL)
782		return;
783	free(s->session_id);
784	free(s->last_mod);
785	for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++)
786		free(s->deltas[i]);
787	free(s);
788}
789
790void
791rrdp_session_buffer(struct ibuf *b, const struct rrdp_session *s)
792{
793	size_t i;
794
795	io_str_buffer(b, s->session_id);
796	io_simple_buffer(b, &s->serial, sizeof(s->serial));
797	io_str_buffer(b, s->last_mod);
798	for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++)
799		io_str_buffer(b, s->deltas[i]);
800}
801
802struct rrdp_session *
803rrdp_session_read(struct ibuf *b)
804{
805	struct rrdp_session *s;
806	size_t i;
807
808	if ((s = calloc(1, sizeof(*s))) == NULL)
809		err(1, NULL);
810
811	io_read_str(b, &s->session_id);
812	io_read_buf(b, &s->serial, sizeof(s->serial));
813	io_read_str(b, &s->last_mod);
814	for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++)
815		io_read_str(b, &s->deltas[i]);
816
817	return s;
818}
819
820static struct rrdprepo *
821rrdp_get(const char *uri)
822{
823	struct rrdp_session *state;
824	struct rrdprepo *rr;
825
826	SLIST_FOREACH(rr, &rrdprepos, entry)
827		if (strcmp(rr->notifyuri, uri) == 0) {
828			if (rr->state == REPO_FAILED)
829				return NULL;
830			return rr;
831		}
832
833	if ((rr = calloc(1, sizeof(*rr))) == NULL)
834		err(1, NULL);
835
836	rr->id = ++repoid;
837	SLIST_INSERT_HEAD(&rrdprepos, rr, entry);
838
839	if ((rr->notifyuri = strdup(uri)) == NULL)
840		err(1, NULL);
841	rr->basedir = repo_dir(uri, ".rrdp", 1);
842
843	RB_INIT(&rr->deleted);
844
845	/* create base directory */
846	if (mkpath(rr->basedir) == -1) {
847		warn("mkpath %s", rr->basedir);
848		rrdp_finish(rr->id, 0);
849		return rr;
850	}
851
852	/* parse state and start the sync */
853	state = rrdp_session_parse(rr);
854	rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, state);
855	rrdp_session_free(state);
856
857	logx("%s: pulling from %s", rr->notifyuri, "network");
858
859	return rr;
860}
861
862/*
863 * Remove RRDP repo and start over.
864 */
865void
866rrdp_clear(unsigned int id)
867{
868	struct rrdprepo *rr;
869
870	rr = rrdp_find(id);
871	if (rr == NULL)
872		errx(1, "non-existent rrdp repo %u", id);
873
874	/* remove rrdp repository contents */
875	remove_contents(rr->basedir);
876}
877
878/*
879 * Write a file into the temporary RRDP dir but only after checking
880 * its hash (if required). The function also makes sure that the file
881 * tracking is properly adjusted.
882 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error
883 */
884int
885rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri,
886    char *hash, size_t hlen, char *data, size_t dlen)
887{
888	struct rrdprepo *rr;
889	struct filepath *fp;
890	ssize_t s;
891	char *fn = NULL;
892	int fd = -1, try = 0, deleted = 0;
893	int flags;
894
895	rr = rrdp_find(id);
896	if (rr == NULL)
897		errx(1, "non-existent rrdp repo %u", id);
898	if (rr->state == REPO_FAILED)
899		return -1;
900
901	/* check hash of original file for updates and deletes */
902	if (pt == PUB_UPD || pt == PUB_DEL) {
903		if (filepath_exists(&rr->deleted, uri)) {
904			warnx("%s: already deleted", uri);
905			return 0;
906		}
907		/* try to open file first in rrdp then in valid repo */
908		do {
909			free(fn);
910			if ((fn = rrdp_filename(rr, uri, try++)) == NULL)
911				return 0;
912			fd = open(fn, O_RDONLY);
913		} while (fd == -1 && try < 2);
914
915		if (!valid_filehash(fd, hash, hlen)) {
916			warnx("%s: bad file digest for %s", rr->notifyuri, fn);
917			free(fn);
918			return 0;
919		}
920		free(fn);
921	}
922
923	/* write new content or mark uri as deleted. */
924	if (pt == PUB_DEL) {
925		filepath_add(&rr->deleted, uri, 0, 0);
926	} else {
927		fp = filepath_find(&rr->deleted, uri);
928		if (fp != NULL) {
929			filepath_put(&rr->deleted, fp);
930			deleted = 1;
931		}
932
933		/* add new file to rrdp dir */
934		if ((fn = rrdp_filename(rr, uri, 0)) == NULL)
935			return 0;
936
937		if (repo_mkpath(AT_FDCWD, fn) == -1)
938			goto fail;
939
940		flags = O_WRONLY|O_CREAT|O_TRUNC;
941		if (pt == PUB_ADD && !deleted)
942			flags |= O_EXCL;
943		fd = open(fn, flags, 0644);
944		if (fd == -1) {
945			if (errno == EEXIST) {
946				warnx("%s: duplicate publish element for %s",
947				    rr->notifyuri, fn);
948				free(fn);
949				return 0;
950			}
951			warn("open %s", fn);
952			goto fail;
953		}
954
955		if ((s = write(fd, data, dlen)) == -1) {
956			warn("write %s", fn);
957			goto fail;
958		}
959		close(fd);
960		if ((size_t)s != dlen)	/* impossible */
961			errx(1, "short write %s", fn);
962		free(fn);
963	}
964
965	return 1;
966
967fail:
968	rr->state = REPO_FAILED;
969	if (fd != -1)
970		close(fd);
971	free(fn);
972	return -1;
973}
974
975/*
976 * RSYNC sync finished, either with or without success.
977 */
978void
979rsync_finish(unsigned int id, int ok)
980{
981	struct rsyncrepo *rr;
982	struct tarepo *tr;
983
984	tr = ta_find(id);
985	if (tr != NULL) {
986		/* repository changed state already, ignore request */
987		if (tr->state != REPO_LOADING)
988			return;
989		if (ok) {
990			logx("ta/%s: loaded from network", tr->descr);
991			stats.rsync_repos++;
992			tr->state = REPO_DONE;
993			repo_done(tr, 1);
994		} else {
995			warnx("ta/%s: load from network failed", tr->descr);
996			stats.rsync_fails++;
997			tr->uriidx++;
998			ta_fetch(tr);
999		}
1000		return;
1001	}
1002
1003	rr = rsync_find(id);
1004	if (rr == NULL)
1005		errx(1, "unknown rsync repo %u", id);
1006	/* repository changed state already, ignore request */
1007	if (rr->state != REPO_LOADING)
1008		return;
1009
1010	if (ok) {
1011		logx("%s: loaded from network", rr->basedir);
1012		stats.rsync_repos++;
1013		rr->state = REPO_DONE;
1014	} else {
1015		warnx("%s: load from network failed, fallback to cache",
1016		    rr->basedir);
1017		stats.rsync_fails++;
1018		rr->state = REPO_FAILED;
1019		/* clear rsync repo since it failed */
1020		remove_contents(rr->basedir);
1021	}
1022
1023	repo_done(rr, ok);
1024}
1025
1026/*
1027 * RRDP sync finished, either with or without success.
1028 */
1029void
1030rrdp_finish(unsigned int id, int ok)
1031{
1032	struct rrdprepo *rr;
1033
1034	rr = rrdp_find(id);
1035	if (rr == NULL)
1036		errx(1, "unknown RRDP repo %u", id);
1037	/* repository changed state already, ignore request */
1038	if (rr->state != REPO_LOADING)
1039		return;
1040
1041	if (ok) {
1042		logx("%s: loaded from network", rr->notifyuri);
1043		stats.rrdp_repos++;
1044		rr->state = REPO_DONE;
1045	} else {
1046		warnx("%s: load from network failed, fallback to %s",
1047		    rr->notifyuri, nofetch ? "cache" : "rsync");
1048		stats.rrdp_fails++;
1049		rr->state = REPO_FAILED;
1050		/* clear the RRDP repo since it failed */
1051		remove_contents(rr->basedir);
1052		/* also clear the list of deleted files */
1053		filepath_free(&rr->deleted);
1054	}
1055
1056	repo_done(rr, ok);
1057}
1058
1059/*
1060 * Handle responses from the http process. For TA file, either rename
1061 * or delete the temporary file. For RRDP requests relay the request
1062 * over to the rrdp process.
1063 */
1064void
1065http_finish(unsigned int id, enum http_result res, const char *last_mod)
1066{
1067	struct tarepo *tr;
1068
1069	tr = ta_find(id);
1070	if (tr == NULL) {
1071		/* not a TA fetch therefore RRDP */
1072		rrdp_http_done(id, res, last_mod);
1073		return;
1074	}
1075
1076	/* repository changed state already, ignore request */
1077	if (tr->state != REPO_LOADING)
1078		return;
1079
1080	/* Move downloaded TA file into place, or unlink on failure. */
1081	if (res == HTTP_OK) {
1082		logx("ta/%s: loaded from network", tr->descr);
1083		tr->state = REPO_DONE;
1084		stats.http_repos++;
1085		repo_done(tr, 1);
1086	} else {
1087		remove_contents(tr->basedir);
1088
1089		tr->uriidx++;
1090		warnx("ta/%s: load from network failed", tr->descr);
1091		ta_fetch(tr);
1092	}
1093}
1094
1095/*
1096 * Look up a trust anchor, queueing it for download if not found.
1097 */
1098struct repo *
1099ta_lookup(int id, struct tal *tal)
1100{
1101	struct repo	*rp;
1102
1103	if (tal->urisz == 0)
1104		errx(1, "TAL %s has no URI", tal->descr);
1105
1106	/* Look up in repository table. (Lookup should actually fail here) */
1107	SLIST_FOREACH(rp, &repos, entry) {
1108		if (strcmp(rp->repouri, tal->uri[0]) == 0)
1109			return rp;
1110	}
1111
1112	rp = repo_alloc(id);
1113	rp->basedir = repo_dir(tal->descr, "ta", 0);
1114	if ((rp->repouri = strdup(tal->uri[0])) == NULL)
1115		err(1, NULL);
1116
1117	/* check if sync disabled ... */
1118	if (noop) {
1119		logx("%s: using cache", rp->basedir);
1120		entityq_flush(&rp->queue, rp);
1121		return rp;
1122	}
1123
1124	/* try to create base directory */
1125	if (mkpath(rp->basedir) == -1)
1126		warn("mkpath %s", rp->basedir);
1127
1128	rp->ta = ta_get(tal);
1129
1130	/* need to check if it was already loaded */
1131	if (repo_state(rp) != REPO_LOADING)
1132		entityq_flush(&rp->queue, rp);
1133
1134	return rp;
1135}
1136
1137/*
1138 * Look up a repository, queueing it for discovery if not found.
1139 */
1140struct repo *
1141repo_lookup(int talid, const char *uri, const char *notify)
1142{
1143	struct repo	*rp;
1144	char		*repouri;
1145
1146	if ((repouri = rsync_base_uri(uri)) == NULL)
1147		errx(1, "bad caRepository URI: %s", uri);
1148
1149	/* Look up in repository table. */
1150	SLIST_FOREACH(rp, &repos, entry) {
1151		if (strcmp(rp->repouri, repouri) != 0)
1152			continue;
1153		if (rp->notifyuri != NULL) {
1154			if (notify == NULL)
1155				continue;
1156			if (strcmp(rp->notifyuri, notify) != 0)
1157				continue;
1158		} else if (notify != NULL)
1159			continue;
1160		/* found matching repo */
1161		free(repouri);
1162		return rp;
1163	}
1164
1165	rp = repo_alloc(talid);
1166	rp->basedir = repo_dir(repouri, NULL, 0);
1167	rp->repouri = repouri;
1168	if (notify != NULL)
1169		if ((rp->notifyuri = strdup(notify)) == NULL)
1170			err(1, NULL);
1171
1172	if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) {
1173		if (talrepocnt[talid] == MAX_REPO_PER_TAL)
1174			warnx("too many repositories under %s", tals[talid]);
1175		nofetch = 1;
1176	}
1177
1178	/* check if sync disabled ... */
1179	if (noop || nofetch) {
1180		logx("%s: using cache", rp->basedir);
1181		entityq_flush(&rp->queue, rp);
1182		return rp;
1183	}
1184
1185	/* try to create base directory */
1186	if (mkpath(rp->basedir) == -1)
1187		warn("mkpath %s", rp->basedir);
1188
1189	/* ... else try RRDP first if available then rsync */
1190	if (notify != NULL)
1191		rp->rrdp = rrdp_get(notify);
1192	if (rp->rrdp == NULL)
1193		rp->rsync = rsync_get(uri, rp->basedir);
1194
1195	/* need to check if it was already loaded */
1196	if (repo_state(rp) != REPO_LOADING)
1197		entityq_flush(&rp->queue, rp);
1198
1199	return rp;
1200}
1201
1202/*
1203 * Find repository by identifier.
1204 */
1205struct repo *
1206repo_byid(unsigned int id)
1207{
1208	struct repo	*rp;
1209
1210	SLIST_FOREACH(rp, &repos, entry) {
1211		if (rp->id == id)
1212			return rp;
1213	}
1214	return NULL;
1215}
1216
1217/*
1218 * Find repository by base path.
1219 */
1220static struct repo *
1221repo_bypath(const char *path)
1222{
1223	struct repo	*rp;
1224
1225	SLIST_FOREACH(rp, &repos, entry) {
1226		if (strcmp(rp->basedir, path) == 0)
1227			return rp;
1228	}
1229	return NULL;
1230}
1231
1232/*
1233 * Return the repository base or alternate directory.
1234 * Returned string must be freed by caller.
1235 */
1236char *
1237repo_basedir(const struct repo *rp, int wantvalid)
1238{
1239	char *path = NULL;
1240
1241	if (!wantvalid) {
1242		if (rp->ta) {
1243			if ((path = strdup(rp->ta->basedir)) == NULL)
1244				err(1, NULL);
1245		} else if (rp->rsync) {
1246			if ((path = strdup(rp->rsync->basedir)) == NULL)
1247				err(1, NULL);
1248		} else if (rp->rrdp) {
1249			path = rrdp_filename(rp->rrdp, rp->repouri, 0);
1250		} else
1251			path = NULL;	/* only valid repo available */
1252	} else if (rp->basedir != NULL) {
1253		if ((path = strdup(rp->basedir)) == NULL)
1254			err(1, NULL);
1255	}
1256
1257	return path;
1258}
1259
1260/*
1261 * Return the repository identifier.
1262 */
1263unsigned int
1264repo_id(const struct repo *rp)
1265{
1266	return rp->id;
1267}
1268
1269/*
1270 * Return the repository URI.
1271 */
1272const char *
1273repo_uri(const struct repo *rp)
1274{
1275	return rp->repouri;
1276}
1277
1278/*
1279 * Return the repository URI.
1280 */
1281void
1282repo_fetch_uris(const struct repo *rp, const char **carepo,
1283    const char **notifyuri)
1284{
1285	*carepo = rp->repouri;
1286	*notifyuri = rp->notifyuri;
1287}
1288
1289/*
1290 * Return 1 if repository is synced else 0.
1291 */
1292int
1293repo_synced(const struct repo *rp)
1294{
1295	if (repo_state(rp) == REPO_DONE &&
1296	    !(rp->rrdp == NULL && rp->rsync == NULL && rp->ta == NULL))
1297		return 1;
1298	return 0;
1299}
1300
1301/*
1302 * Return the protocol string "rrdp", "rsync", "https" which was used to sync.
1303 * Result is only correct if repository was properly synced.
1304 */
1305const char *
1306repo_proto(const struct repo *rp)
1307{
1308
1309	if (rp->ta != NULL) {
1310		const struct tarepo *tr = rp->ta;
1311		if (tr->uriidx < tr->urisz &&
1312		    strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO,
1313		    RSYNC_PROTO_LEN) == 0)
1314			return "rsync";
1315		else
1316			return "https";
1317	}
1318	if (rp->rrdp != NULL)
1319		return "rrdp";
1320	return "rsync";
1321}
1322
1323/*
1324 * Return the repository tal ID.
1325 */
1326int
1327repo_talid(const struct repo *rp)
1328{
1329	return rp->talid;
1330}
1331
1332int
1333repo_queued(struct repo *rp, struct entity *p)
1334{
1335	if (repo_state(rp) == REPO_LOADING) {
1336		TAILQ_INSERT_TAIL(&rp->queue, p, entries);
1337		return 1;
1338	}
1339	return 0;
1340}
1341
1342static void
1343repo_fail(struct repo *rp)
1344{
1345	/* reset the alarm since code may fallback to rsync */
1346	rp->alarm = getmonotime() + repo_timeout;
1347
1348	if (rp->ta)
1349		http_finish(rp->ta->id, HTTP_FAILED, NULL);
1350	else if (rp->rsync)
1351		rsync_finish(rp->rsync->id, 0);
1352	else if (rp->rrdp)
1353		rrdp_finish(rp->rrdp->id, 0);
1354	else
1355		errx(1, "%s: bad repo", rp->repouri);
1356}
1357
1358static void
1359repo_abort(struct repo *rp)
1360{
1361	/* reset the alarm */
1362	rp->alarm = getmonotime() + repo_timeout;
1363
1364	if (rp->rsync)
1365		rsync_abort(rp->rsync->id);
1366	else if (rp->rrdp)
1367		rrdp_abort(rp->rrdp->id);
1368	else
1369		repo_fail(rp);
1370}
1371
1372int
1373repo_check_timeout(int timeout)
1374{
1375	struct repo	*rp;
1376	time_t		 now;
1377	int		 diff;
1378
1379	now = getmonotime();
1380
1381	/* check against our runtime deadline first */
1382	if (deadline != 0) {
1383		if (deadline <= now) {
1384			warnx("deadline reached, giving up on repository sync");
1385			nofetch = 1;
1386			/* clear deadline since nofetch is set */
1387			deadline = 0;
1388			/* increase now enough so that all pending repos fail */
1389			now += repo_timeout;
1390		} else {
1391			diff = deadline - now;
1392			diff *= 1000;
1393			if (timeout == INFTIM || diff < timeout)
1394				timeout = diff;
1395		}
1396	}
1397	/* Look up in repository table. (Lookup should actually fail here) */
1398	SLIST_FOREACH(rp, &repos, entry) {
1399		if (repo_state(rp) == REPO_LOADING) {
1400			if (rp->alarm <= now) {
1401				warnx("%s: synchronisation timeout",
1402				    rp->repouri);
1403				repo_abort(rp);
1404			} else {
1405				diff = rp->alarm - now;
1406				diff *= 1000;
1407				if (timeout == INFTIM || diff < timeout)
1408					timeout = diff;
1409			}
1410		}
1411	}
1412	return timeout;
1413}
1414
1415/*
1416 * Update repo-specific stats when files are going to be moved
1417 * from DIR_TEMP to DIR_VALID.
1418 */
1419void
1420repostats_new_files_inc(struct repo *rp, const char *file)
1421{
1422	if (strncmp(file, ".rsync/", strlen(".rsync/")) == 0 ||
1423	    strncmp(file, ".rrdp/", strlen(".rrdp/")) == 0)
1424		rp->repostats.new_files++;
1425}
1426
1427/*
1428 * Update stats object of repository depending on rtype and subtype.
1429 */
1430void
1431repo_stat_inc(struct repo *rp, int talid, enum rtype type, enum stype subtype)
1432{
1433	if (rp == NULL)
1434		return;
1435	rp->stats_used[talid] = 1;
1436	switch (type) {
1437	case RTYPE_CER:
1438		if (subtype == STYPE_OK)
1439			rp->stats[talid].certs++;
1440		if (subtype == STYPE_FAIL)
1441			rp->stats[talid].certs_fail++;
1442		if (subtype == STYPE_BGPSEC) {
1443			rp->stats[talid].certs--;
1444			rp->stats[talid].brks++;
1445		}
1446		break;
1447	case RTYPE_MFT:
1448		if (subtype == STYPE_OK)
1449			rp->stats[talid].mfts++;
1450		if (subtype == STYPE_FAIL)
1451			rp->stats[talid].mfts_fail++;
1452		break;
1453	case RTYPE_ROA:
1454		switch (subtype) {
1455		case STYPE_OK:
1456			rp->stats[talid].roas++;
1457			break;
1458		case STYPE_FAIL:
1459			rp->stats[talid].roas_fail++;
1460			break;
1461		case STYPE_INVALID:
1462			rp->stats[talid].roas_invalid++;
1463			break;
1464		case STYPE_TOTAL:
1465			rp->stats[talid].vrps++;
1466			break;
1467		case STYPE_UNIQUE:
1468			rp->stats[talid].vrps_uniqs++;
1469			break;
1470		case STYPE_DEC_UNIQUE:
1471			rp->stats[talid].vrps_uniqs--;
1472			break;
1473		default:
1474			break;
1475		}
1476		break;
1477	case RTYPE_ASPA:
1478		switch (subtype) {
1479		case STYPE_OK:
1480			rp->stats[talid].aspas++;
1481			break;
1482		case STYPE_FAIL:
1483			rp->stats[talid].aspas_fail++;
1484			break;
1485		case STYPE_INVALID:
1486			rp->stats[talid].aspas_invalid++;
1487			break;
1488		case STYPE_TOTAL:
1489			rp->stats[talid].vaps++;
1490			break;
1491		case STYPE_UNIQUE:
1492			rp->stats[talid].vaps_uniqs++;
1493			break;
1494		case STYPE_DEC_UNIQUE:
1495			rp->stats[talid].vaps_uniqs--;
1496			break;
1497		case STYPE_PROVIDERS:
1498			rp->stats[talid].vaps_pas++;
1499			break;
1500		case STYPE_OVERFLOW:
1501			rp->stats[talid].vaps_overflowed++;
1502			break;
1503		default:
1504			break;
1505		}
1506		break;
1507	case RTYPE_SPL:
1508		switch (subtype) {
1509		case STYPE_OK:
1510			rp->stats[talid].spls++;
1511			break;
1512		case STYPE_FAIL:
1513			rp->stats[talid].spls_fail++;
1514			break;
1515		case STYPE_INVALID:
1516			rp->stats[talid].spls_invalid++;
1517			break;
1518		case STYPE_TOTAL:
1519			rp->stats[talid].vsps++;
1520			break;
1521		case STYPE_UNIQUE:
1522			rp->stats[talid].vsps_uniqs++;
1523			break;
1524		case STYPE_DEC_UNIQUE:
1525			rp->stats[talid].vsps_uniqs--;
1526			break;
1527		default:
1528			break;
1529		}
1530		break;
1531	case RTYPE_CRL:
1532		rp->stats[talid].crls++;
1533		break;
1534	case RTYPE_GBR:
1535		rp->stats[talid].gbrs++;
1536		break;
1537	case RTYPE_TAK:
1538		rp->stats[talid].taks++;
1539		break;
1540	default:
1541		break;
1542	}
1543}
1544
1545void
1546repo_tal_stats_collect(void (*cb)(const struct repo *,
1547    const struct repotalstats *, void *), int talid, void *arg)
1548{
1549	struct repo	*rp;
1550
1551	SLIST_FOREACH(rp, &repos, entry) {
1552		if (rp->stats_used[talid])
1553			cb(rp, &rp->stats[talid], arg);
1554	}
1555}
1556
1557void
1558repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *,
1559    void *), void *arg)
1560{
1561	struct repo	*rp;
1562
1563	SLIST_FOREACH(rp, &repos, entry)
1564		cb(rp, &rp->repostats, arg);
1565}
1566
1567/*
1568 * Delayed delete of files from RRDP. Since RRDP has no security built-in
1569 * this code needs to check if this RRDP repository is actually allowed to
1570 * remove the file referenced by the URI.
1571 */
1572static void
1573repo_cleanup_rrdp(struct filepath_tree *tree)
1574{
1575	struct repo *rp;
1576	struct rrdprepo *rr;
1577	struct filepath *fp, *nfp;
1578	char *fn;
1579
1580	SLIST_FOREACH(rp, &repos, entry) {
1581		if (rp->rrdp == NULL)
1582			continue;
1583		rr = (struct rrdprepo *)rp->rrdp;
1584		RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) {
1585			if (!rrdp_uri_valid(rr, fp->file)) {
1586				warnx("%s: external URI %s", rr->notifyuri,
1587				    fp->file);
1588				filepath_put(&rr->deleted, fp);
1589				continue;
1590			}
1591			/* try to remove file from rrdp repo ... */
1592			fn = rrdp_filename(rr, fp->file, 0);
1593
1594			if (unlink(fn) == -1) {
1595				if (errno != ENOENT)
1596					warn("unlink %s", fn);
1597			} else {
1598				if (verbose > 1)
1599					logx("deleted %s", fn);
1600				rp->repostats.del_files++;
1601			}
1602			free(fn);
1603
1604			/* ... and from the valid repository if unused. */
1605			fn = rrdp_filename(rr, fp->file, 1);
1606			if (!filepath_exists(tree, fn)) {
1607				if (unlink(fn) == -1) {
1608					if (errno != ENOENT)
1609						warn("unlink %s", fn);
1610				} else {
1611					if (verbose > 1)
1612						logx("deleted %s", fn);
1613					rp->repostats.del_files++;
1614				}
1615			} else
1616				warnx("%s: referenced file supposed to be "
1617				    "deleted", fn);
1618
1619			free(fn);
1620			filepath_put(&rr->deleted, fp);
1621		}
1622	}
1623}
1624
1625/*
1626 * All files in tree are valid and should be moved to the valid repository
1627 * if not already there. Rename the files to the new path and readd the
1628 * filepath entry with the new path if successful.
1629 */
1630static void
1631repo_move_valid(struct filepath_tree *tree)
1632{
1633	struct filepath *fp, *nfp;
1634	size_t rsyncsz = strlen(".rsync/");
1635	size_t rrdpsz = strlen(".rrdp/");
1636	size_t tasz = strlen(".ta/");
1637	char *fn, *base;
1638
1639	RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) {
1640		if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 &&
1641		    strncmp(fp->file, ".rrdp/", rrdpsz) != 0 &&
1642		    strncmp(fp->file, ".ta/", tasz) != 0)
1643			continue; /* not a temporary file path */
1644
1645		if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) {
1646			fn = fp->file + rsyncsz;
1647		} else if (strncmp(fp->file, ".ta/", tasz) == 0) {
1648			fn = fp->file + 1; /* just skip the '.' */
1649		} else {
1650			base = strchr(fp->file + rrdpsz, '/');
1651			assert(base != NULL);
1652			fn = base + 1;
1653
1654			/*
1655			 * Adjust file last modification time in order to
1656			 * minimize RSYNC synchronization load after transport
1657			 * failover.
1658			 * While serializing RRDP datastructures to disk, set
1659			 * the last modified timestamp to the CMS signing-time,
1660			 * the X.509 notBefore, or CRL lastUpdate timestamp.
1661			 */
1662			if (fp->mtime != 0) {
1663				int ret;
1664				struct timespec ts[2];
1665
1666				ts[0].tv_nsec = UTIME_OMIT;
1667				ts[1].tv_sec = fp->mtime;
1668				ts[1].tv_nsec = 0;
1669				ret = utimensat(AT_FDCWD, fp->file, ts, 0);
1670				if (ret == -1) {
1671					warn("utimensat %s", fp->file);
1672					continue;
1673				}
1674			}
1675		}
1676
1677		if (repo_mkpath(AT_FDCWD, fn) == -1)
1678			continue;
1679
1680		if (rename(fp->file, fn) == -1) {
1681			warn("rename %s", fp->file);
1682			continue;
1683		}
1684
1685		/* switch filepath node to new path */
1686		RB_REMOVE(filepath_tree, tree, fp);
1687		base = fp->file;
1688		if ((fp->file = strdup(fn)) == NULL)
1689			err(1, NULL);
1690		free(base);
1691		if (RB_INSERT(filepath_tree, tree, fp) != NULL)
1692			errx(1, "%s: both possibilities of file present",
1693			    fp->file);
1694	}
1695}
1696
1697struct fts_state {
1698	enum { BASE_DIR, RSYNC_DIR, TA_DIR, RRDP_DIR }	type;
1699	struct repo					*rp;
1700} fts_state;
1701
1702static const struct rrdprepo *
1703repo_is_rrdp(struct repo *rp)
1704{
1705	/* check for special pointers first these are not a repository */
1706	if (rp != NULL && rp->rrdp != NULL)
1707		return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL;
1708	return NULL;
1709}
1710
1711static inline char *
1712skip_dotslash(char *in)
1713{
1714	if (memcmp(in, "./", 2) == 0)
1715		return in + 2;
1716	return in;
1717}
1718
1719static void
1720repo_cleanup_entry(FTSENT *e, struct filepath_tree *tree, int cachefd)
1721{
1722	const struct rrdprepo *rr;
1723	char *path;
1724
1725	path = skip_dotslash(e->fts_path);
1726	switch (e->fts_info) {
1727	case FTS_NSOK:
1728		if (filepath_exists(tree, path)) {
1729			e->fts_parent->fts_number++;
1730			break;
1731		}
1732		if (fts_state.type == RRDP_DIR && fts_state.rp != NULL) {
1733			e->fts_parent->fts_number++;
1734			/* handle rrdp .state files explicitly */
1735			if (e->fts_level == 3 &&
1736			    strcmp(e->fts_name, ".state") == 0)
1737				break;
1738			/* can't delete these extra files */
1739			fts_state.rp->repostats.extra_files++;
1740			if (verbose > 1)
1741				logx("superfluous %s", path);
1742			break;
1743		}
1744		rr = repo_is_rrdp(fts_state.rp);
1745		if (rr != NULL) {
1746			struct stat st;
1747			char *fn;
1748
1749			if (asprintf(&fn, "%s/%s", rr->basedir, path) == -1)
1750				err(1, NULL);
1751
1752			/*
1753			 * If the file exists in the rrdp dir
1754			 * that file is newer and needs to be kept
1755			 * so unlink this file instead of moving
1756			 * it over the file in the rrdp dir.
1757			 */
1758			if (fstatat(cachefd, fn, &st, 0) == 0 &&
1759			    S_ISREG(st.st_mode)) {
1760				free(fn);
1761				goto unlink;
1762			}
1763			if (repo_mkpath(cachefd, fn) == 0) {
1764				if (renameat(AT_FDCWD, e->fts_accpath,
1765				    cachefd, fn) == -1)
1766					warn("rename %s to %s", path, fn);
1767				else if (verbose > 1)
1768					logx("moved %s", path);
1769				fts_state.rp->repostats.extra_files++;
1770			}
1771			free(fn);
1772		} else {
1773 unlink:
1774			if (unlink(e->fts_accpath) == -1) {
1775				warn("unlink %s", path);
1776			} else if (fts_state.type == RSYNC_DIR ||
1777			     fts_state.type == TA_DIR) {
1778				/* no need to keep rsync or ta files */
1779				if (verbose > 1)
1780					logx("deleted superfluous %s", path);
1781				if (fts_state.rp != NULL)
1782					fts_state.rp->repostats.del_extra_files++;
1783				else
1784					stats.repo_stats.del_extra_files++;
1785			} else {
1786				if (verbose > 1)
1787					logx("deleted %s", path);
1788				if (fts_state.rp != NULL)
1789					fts_state.rp->repostats.del_files++;
1790				else
1791					stats.repo_stats.del_files++;
1792			}
1793		}
1794		break;
1795	case FTS_D:
1796		if (e->fts_level == FTS_ROOTLEVEL) {
1797			fts_state.type = BASE_DIR;
1798			fts_state.rp = NULL;
1799		}
1800		if (e->fts_level == 1) {
1801			/* rpki.example.org or .rrdp / .rsync / .ta */
1802			if (strcmp(".rsync", e->fts_name) == 0)
1803				fts_state.type = RSYNC_DIR;
1804			else if (strcmp(".ta", e->fts_name) == 0)
1805				fts_state.type = TA_DIR;
1806			else if (strcmp(".rrdp", e->fts_name) == 0)
1807				fts_state.type = RRDP_DIR;
1808			else
1809				fts_state.type = BASE_DIR;
1810			fts_state.rp = NULL;
1811		}
1812		if (e->fts_level == 2) {
1813			/* rpki.example.org/repository or .rrdp/hashdir */
1814			if (fts_state.type == BASE_DIR)
1815				fts_state.rp = repo_bypath(path);
1816			if (fts_state.type == TA_DIR)
1817				fts_state.rp = repo_bypath(path + 1);
1818			/*
1819			 * special handling for rrdp directories,
1820			 * clear them if they are not used anymore but
1821			 * only if rrdp is active.
1822			 * Look them up just using the hash.
1823			 */
1824			if (fts_state.type == RRDP_DIR)
1825				fts_state.rp = repo_rrdp_bypath(path);
1826		}
1827		if (e->fts_level == 3 && fts_state.type == RSYNC_DIR) {
1828			/* .rsync/rpki.example.org/repository */
1829			fts_state.rp = repo_bypath(path + strlen(".rsync/"));
1830		}
1831		break;
1832	case FTS_DP:
1833		if (e->fts_level == FTS_ROOTLEVEL)
1834			break;
1835		if (e->fts_level == 1) {
1836			/* do not remove .rsync and .rrdp */
1837			fts_state.rp = NULL;
1838			if (fts_state.type == RRDP_DIR ||
1839			    fts_state.type == RSYNC_DIR ||
1840			    fts_state.type == TA_DIR)
1841				break;
1842		}
1843
1844		e->fts_parent->fts_number += e->fts_number;
1845
1846		if (e->fts_number == 0) {
1847			if (rmdir(e->fts_accpath) == -1)
1848				warn("rmdir %s", path);
1849			if (fts_state.rp != NULL)
1850				fts_state.rp->repostats.del_dirs++;
1851			else
1852				stats.repo_stats.del_dirs++;
1853		}
1854		break;
1855	case FTS_SL:
1856	case FTS_SLNONE:
1857		warnx("symlink %s", path);
1858		if (unlink(e->fts_accpath) == -1)
1859			warn("unlink %s", path);
1860		stats.repo_stats.del_extra_files++;
1861		break;
1862	case FTS_NS:
1863	case FTS_ERR:
1864		if (e->fts_errno == ENOENT && e->fts_level == FTS_ROOTLEVEL)
1865			break;
1866		warnx("fts_read %s: %s", path, strerror(e->fts_errno));
1867		break;
1868	default:
1869		warnx("fts_read %s: unhandled[%x]", path, e->fts_info);
1870		break;
1871	}
1872}
1873
1874void
1875repo_cleanup(struct filepath_tree *tree, int cachefd)
1876{
1877	char *argv[2] = { ".", NULL };
1878	FTS *fts;
1879	FTSENT *e;
1880
1881	/* first move temp files which have been used to valid dir */
1882	repo_move_valid(tree);
1883	/* then delete files requested by rrdp */
1884	repo_cleanup_rrdp(tree);
1885
1886	if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL)
1887		err(1, "fts_open");
1888	errno = 0;
1889	while ((e = fts_read(fts)) != NULL) {
1890		repo_cleanup_entry(e, tree, cachefd);
1891		errno = 0;
1892	}
1893	if (errno)
1894		err(1, "fts_read");
1895	if (fts_close(fts) == -1)
1896		err(1, "fts_close");
1897}
1898
1899void
1900repo_free(void)
1901{
1902	struct repo *rp;
1903
1904	while ((rp = SLIST_FIRST(&repos)) != NULL) {
1905		SLIST_REMOVE_HEAD(&repos, entry);
1906		free(rp->repouri);
1907		free(rp->notifyuri);
1908		free(rp->basedir);
1909		free(rp);
1910	}
1911
1912	ta_free();
1913	rrdp_free();
1914	rsync_free();
1915}
1916
1917/*
1918 * Remove all files and directories under base.
1919 * Do not remove base directory itself and the .state file.
1920 */
1921static void
1922remove_contents(char *base)
1923{
1924	char *argv[2] = { base, NULL };
1925	FTS *fts;
1926	FTSENT *e;
1927
1928	if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL)
1929		err(1, "fts_open");
1930	errno = 0;
1931	while ((e = fts_read(fts)) != NULL) {
1932		switch (e->fts_info) {
1933		case FTS_NSOK:
1934		case FTS_SL:
1935		case FTS_SLNONE:
1936			if (e->fts_level == 1 &&
1937			    strcmp(e->fts_name, ".state") == 0)
1938				break;
1939			if (unlink(e->fts_accpath) == -1)
1940				warn("unlink %s", e->fts_path);
1941			break;
1942		case FTS_D:
1943			break;
1944		case FTS_DP:
1945			/* keep root directory */
1946			if (e->fts_level == FTS_ROOTLEVEL)
1947				break;
1948			if (rmdir(e->fts_accpath) == -1)
1949				warn("rmdir %s", e->fts_path);
1950			break;
1951		case FTS_NS:
1952		case FTS_ERR:
1953			warnx("fts_read %s: %s", e->fts_path,
1954			    strerror(e->fts_errno));
1955			break;
1956		default:
1957			warnx("unhandled[%x] %s", e->fts_info,
1958			    e->fts_path);
1959			break;
1960		}
1961		errno = 0;
1962	}
1963	if (errno)
1964		err(1, "fts_read");
1965	if (fts_close(fts) == -1)
1966		err(1, "fts_close");
1967}
1968