1/*-
2 * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer
10 * in this position and unchanged.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 134350 2004-08-26 15:51:10Z des $");
31
32#include <sys/param.h>
33#include <sys/socket.h>
34#include <sys/stat.h>
35#include <sys/time.h>
36
37#include <ctype.h>
38#include <err.h>
39#include <errno.h>
40#include <signal.h>
41#include <stdint.h>
42#include <stdio.h>
43#include <stdlib.h>
44#include <string.h>
45#include <sysexits.h>
46#include <termios.h>
47#include <unistd.h>
48
49#include <fetch.h>
50
51#define MINBUFSIZE 4096
52
53/* Option flags */
54int A_flag; /* -A: do not follow 302 redirects */
55int a_flag; /* -a: auto retry */
56off_t B_size; /* -B: buffer size */
57int b_flag; /*! -b: workaround TCP bug */
58char *c_dirname; /* -c: remote directory */
59int d_flag; /* -d: direct connection */
60int F_flag; /* -F: restart without checking mtime */
61char *f_filename; /* -f: file to fetch */
62char *h_hostname; /* -h: host to fetch from */
63int l_flag; /* -l: link rather than copy file: URLs */
64int m_flag; /* -[Mm]: mirror mode */
65char *N_filename; /* -N: netrc file name */
66int n_flag; /* -n: do not preserve modification time */
67int o_flag; /* -o: specify output file */
68int o_directory; /* output file is a directory */
69char *o_filename; /* name of output file */
70int o_stdout; /* output file is stdout */
71int once_flag; /* -1: stop at first successful file */
72int p_flag; /* -[Pp]: use passive FTP */
73int R_flag; /* -R: don't delete partially transferred files */
74int r_flag; /* -r: restart previously interrupted transfer */
75off_t S_size; /* -S: require size to match */
76int s_flag; /* -s: show size, don't fetch */
77long T_secs = 120; /* -T: transfer timeout in seconds */
78int t_flag; /*! -t: workaround TCP bug */
79int U_flag; /* -U: do not use high ports */
80int v_level = 1; /* -v: verbosity level */
81int v_tty; /* stdout is a tty */
82pid_t pgrp; /* our process group */
83long w_secs; /* -w: retry delay */
84int family = PF_UNSPEC; /* -[46]: address family to use */
85
86int sigalrm; /* SIGALRM received */
87int siginfo; /* SIGINFO received */
88int sigint; /* SIGINT received */
89
90long ftp_timeout; /* default timeout for FTP transfers */
91long http_timeout; /* default timeout for HTTP transfers */
92char *buf; /* transfer buffer */
93
94
95/*
96 * Signal handler
97 */
98static void
99sig_handler(int sig)
100{
101 switch (sig) {
102 case SIGALRM:
103 sigalrm = 1;
104 break;
105 case SIGINFO:
106 siginfo = 1;
107 break;
108 case SIGINT:
109 sigint = 1;
110 break;
111 }
112}
113
114struct xferstat {
115 char name[64];
116 struct timeval start;
117 struct timeval last;
118 off_t size;
119 off_t offset;
120 off_t rcvd;
121};
122
123/*
124 * Compute and display ETA
125 */
126static const char *
127stat_eta(struct xferstat *xs)
128{
129 static char str[16];
130 long elapsed, eta;
131 off_t received, expected;
132
133 elapsed = xs->last.tv_sec - xs->start.tv_sec;
134 received = xs->rcvd - xs->offset;
135 expected = xs->size - xs->rcvd;
136 eta = (long)((double)elapsed * expected / received);
137 if (eta > 3600)
138 snprintf(str, sizeof str, "%02ldh%02ldm",
139 eta / 3600, (eta % 3600) / 60);
140 else
141 snprintf(str, sizeof str, "%02ldm%02lds",
142 eta / 60, eta % 60);
143 return (str);
144}
145
146/*
147 * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'...
148 */
149static const char *prefixes = " kMGTP";
150static const char *
151stat_bytes(off_t bytes)
152{
153 static char str[16];
154 const char *prefix = prefixes;
155
156 while (bytes > 9999 && prefix[1] != '\0') {
157 bytes /= 1024;
158 prefix++;
159 }
160 snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix);
161 return (str);
162}
163
164/*
165 * Compute and display transfer rate
166 */
167static const char *
168stat_bps(struct xferstat *xs)
169{
170 static char str[16];
171 double delta, bps;
172
173 delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6))
174 - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6));
175 if (delta == 0.0) {
176 snprintf(str, sizeof str, "?? Bps");
177 } else {
178 bps = (xs->rcvd - xs->offset) / delta;
179 snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps));
180 }
181 return (str);
182}
183
184/*
185 * Update the stats display
186 */
187static void
188stat_display(struct xferstat *xs, int force)
189{
190 struct timeval now;
191 int ctty_pgrp;
192
193 /* check if we're the foreground process */
194 if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 ||
195 (pid_t)ctty_pgrp != pgrp)
196 return;
197
198 gettimeofday(&now, NULL);
199 if (!force && now.tv_sec <= xs->last.tv_sec)
200 return;
201 xs->last = now;
202
203 fprintf(stderr, "\r%-46.46s", xs->name);
204 if (xs->size <= 0) {
205 fprintf(stderr, " %s", stat_bytes(xs->rcvd));
206 } else {
207 fprintf(stderr, "%3d%% of %s",
208 (int)((100.0 * xs->rcvd) / xs->size),
209 stat_bytes(xs->size));
210 }
211 fprintf(stderr, " %s", stat_bps(xs));
212 if (xs->size > 0 && xs->rcvd > 0 &&
213 xs->last.tv_sec >= xs->start.tv_sec + 10)
214 fprintf(stderr, " %s", stat_eta(xs));
215}
216
217/*
218 * Initialize the transfer statistics
219 */
220static void
221stat_start(struct xferstat *xs, const char *name, off_t size, off_t offset)
222{
223 snprintf(xs->name, sizeof xs->name, "%s", name);
224 gettimeofday(&xs->start, NULL);
225 xs->last.tv_sec = xs->last.tv_usec = 0;
226 xs->size = size;
227 xs->offset = offset;
228 xs->rcvd = offset;
229 if (v_tty && v_level > 0)
230 stat_display(xs, 1);
231 else if (v_level > 0)
232 fprintf(stderr, "%-46s", xs->name);
233}
234
235/*
236 * Update the transfer statistics
237 */
238static void
239stat_update(struct xferstat *xs, off_t rcvd)
240{
241 xs->rcvd = rcvd;
242 if (v_tty && v_level > 0)
243 stat_display(xs, 0);
244}
245
246/*
247 * Finalize the transfer statistics
248 */
249static void
250stat_end(struct xferstat *xs)
251{
252 gettimeofday(&xs->last, NULL);
253 if (v_tty && v_level > 0) {
254 stat_display(xs, 1);
255 putc('\n', stderr);
256 } else if (v_level > 0) {
257 fprintf(stderr, " %s %s\n",
258 stat_bytes(xs->size), stat_bps(xs));
259 }
260}
261
262/*
263 * Ask the user for authentication details
264 */
265static int
266query_auth(struct url *URL)
267{
268 struct termios tios;
269 tcflag_t saved_flags;
270 int i, nopwd;
271
272 fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n",
273 URL->scheme, URL->host, URL->port);
274
275 fprintf(stderr, "Login: ");
276 if (fgets(URL->user, sizeof URL->user, stdin) == NULL)
277 return (-1);
278 for (i = strlen(URL->user); i >= 0; --i)
279 if (URL->user[i] == '\r' || URL->user[i] == '\n')
280 URL->user[i] = '\0';
281
282 fprintf(stderr, "Password: ");
283 if (tcgetattr(STDIN_FILENO, &tios) == 0) {
284 saved_flags = tios.c_lflag;
285 tios.c_lflag &= ~ECHO;
286 tios.c_lflag |= ECHONL|ICANON;
287 tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios);
288 nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL);
289 tios.c_lflag = saved_flags;
290 tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios);
291 } else {
292 nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL);
293 }
294 if (nopwd)
295 return (-1);
296 for (i = strlen(URL->pwd); i >= 0; --i)
297 if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n')
298 URL->pwd[i] = '\0';
299
300 return (0);
301}
302
303/*
304 * Fetch a file
305 */
306static int
307fetch(char *URL, const char *path)
308{
309 struct url *url;
310 struct url_stat us;
311 struct stat sb, nsb;
312 struct xferstat xs;
313 FILE *f, *of;
314 size_t size, wr;
315 off_t count;
316 char flags[8];
317 const char *slash;
318 char *tmppath;
319 int r;
320 unsigned timeout;
321 char *ptr;
322
323 f = of = NULL;
324 tmppath = NULL;
325
326 timeout = 0;
327 *flags = 0;
328 count = 0;
329
330 /* set verbosity level */
331 if (v_level > 1)
332 strcat(flags, "v");
333 if (v_level > 2)
334 fetchDebug = 1;
335
336 /* parse URL */
337 if ((url = fetchParseURL(URL)) == NULL) {
338 warnx("%s: parse error", URL);
339 goto failure;
340 }
341
342 /* if no scheme was specified, take a guess */
343 if (!*url->scheme) {
344 if (!*url->host)
345 strcpy(url->scheme, SCHEME_FILE);
346 else if (strncasecmp(url->host, "ftp.", 4) == 0)
347 strcpy(url->scheme, SCHEME_FTP);
348 else if (strncasecmp(url->host, "www.", 4) == 0)
349 strcpy(url->scheme, SCHEME_HTTP);
350 }
351
352 /* common flags */
353 switch (family) {
354 case PF_INET:
355 strcat(flags, "4");
356 break;
357 case PF_INET6:
358 strcat(flags, "6");
359 break;
360 }
361
362 /* FTP specific flags */
363 if (strcmp(url->scheme, "ftp") == 0) {
364 if (p_flag)
365 strcat(flags, "p");
366 if (d_flag)
367 strcat(flags, "d");
368 if (U_flag)
369 strcat(flags, "l");
370 timeout = T_secs ? T_secs : ftp_timeout;
371 }
372
373 /* HTTP specific flags */
374 if (strcmp(url->scheme, "http") == 0) {
375 if (d_flag)
376 strcat(flags, "d");
377 if (A_flag)
378 strcat(flags, "A");
379 timeout = T_secs ? T_secs : http_timeout;
380 }
381
382 /* set the protocol timeout. */
383 fetchTimeout = timeout;
384
385 /* just print size */
386 if (s_flag) {
387 if (timeout)
388 alarm(timeout);
389 r = fetchStat(url, &us, flags);
390 if (timeout)
391 alarm(0);
392 if (sigalrm || sigint)
393 goto signal;
394 if (r == -1) {
395 warnx("%s", fetchLastErrString);
396 goto failure;
397 }
398 if (us.size == -1)
399 printf("Unknown\n");
400 else
401 printf("%jd\n", (intmax_t)us.size);
402 goto success;
403 }
404
405 /*
406 * If the -r flag was specified, we have to compare the local
407 * and remote files, so we should really do a fetchStat()
408 * first, but I know of at least one HTTP server that only
409 * sends the content size in response to GET requests, and
410 * leaves it out of replies to HEAD requests. Also, in the
411 * (frequent) case that the local and remote files match but
412 * the local file is truncated, we have sufficient information
413 * before the compare to issue a correct request. Therefore,
414 * we always issue a GET request as if we were sure the local
415 * file was a truncated copy of the remote file; we can drop
416 * the connection later if we change our minds.
417 */
418 sb.st_size = -1;
419 if (!o_stdout) {
420 r = stat(path, &sb);
421 if (r == 0 && r_flag && S_ISREG(sb.st_mode)) {
422 url->offset = sb.st_size;
423 } else {
424 /*
425 * Whatever value sb.st_size has now is either
426 * wrong (if stat(2) failed) or irrelevant (if the
427 * path does not refer to a regular file)
428 */
429 sb.st_size = -1;
430 if (r == -1 && errno != ENOENT) {
431 warnx("%s: stat()", path);
432 goto failure;
433 }
434 }
435 }
436
437 /* start the transfer */
438 if (timeout)
439 alarm(timeout);
440 f = fetchXGet(url, &us, flags);
441 if (timeout)
442 alarm(0);
443 if (sigalrm || sigint)
444 goto signal;
445 if (f == NULL) {
446 warnx("%s: %s", URL, fetchLastErrString);
447 goto failure;
448 }
449 if (sigint)
450 goto signal;
451
452 /* check that size is as expected */
453 if (S_size) {
454 if (us.size == -1) {
455 warnx("%s: size unknown", URL);
456 } else if (us.size != S_size) {
457 warnx("%s: size mismatch: expected %jd, actual %jd",
458 URL, (intmax_t)S_size, (intmax_t)us.size);
459 goto failure;
460 }
461 }
462
463 /* symlink instead of copy */
464 if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) {
465 if (symlink(url->doc, path) == -1) {
466 warn("%s: symlink()", path);
467 goto failure;
468 }
469 goto success;
470 }
471
472 if (us.size == -1 && !o_stdout && v_level > 0)
473 warnx("%s: size of remote file is not known", URL);
474 if (v_level > 1) {
475 if (sb.st_size != -1)
476 fprintf(stderr, "local size / mtime: %jd / %ld\n",
477 (intmax_t)sb.st_size, (long)sb.st_mtime);
478 if (us.size != -1)
479 fprintf(stderr, "remote size / mtime: %jd / %ld\n",
480 (intmax_t)us.size, (long)us.mtime);
481 }
482
483 /* open output file */
484 if (o_stdout) {
485 /* output to stdout */
486 of = stdout;
487 } else if (r_flag && sb.st_size != -1) {
488 /* resume mode, local file exists */
489 if (!F_flag && us.mtime && sb.st_mtime != us.mtime) {
490 /* no match! have to refetch */
491 fclose(f);
492 /* if precious, warn the user and give up */
493 if (R_flag) {
494 warnx("%s: local modification time "
495 "does not match remote", path);
496 goto failure_keep;
497 }
498 } else if (us.size != -1) {
499 if (us.size == sb.st_size)
500 /* nothing to do */
501 goto success;
502 if (sb.st_size > us.size) {
503 /* local file too long! */
504 warnx("%s: local file (%jd bytes) is longer "
505 "than remote file (%jd bytes)", path,
506 (intmax_t)sb.st_size, (intmax_t)us.size);
507 goto failure;
508 }
509 /* we got it, open local file */
510 if ((of = fopen(path, "a")) == NULL) {
511 warn("%s: fopen()", path);
512 goto failure;
513 }
514 /* check that it didn't move under our feet */
515 if (fstat(fileno(of), &nsb) == -1) {
516 /* can't happen! */
517 warn("%s: fstat()", path);
518 goto failure;
519 }
520 if (nsb.st_dev != sb.st_dev ||
521 nsb.st_ino != nsb.st_ino ||
522 nsb.st_size != sb.st_size) {
523 warnx("%s: file has changed", URL);
524 fclose(of);
525 of = NULL;
526 sb = nsb;
527 }
528 }
529 } else if (m_flag && sb.st_size != -1) {
530 /* mirror mode, local file exists */
531 if (sb.st_size == us.size && sb.st_mtime == us.mtime)
532 goto success;
533 }
534
535 if (of == NULL) {
536 /*
537 * We don't yet have an output file; either this is a
538 * vanilla run with no special flags, or the local and
539 * remote files didn't match.
540 */
541
542 if (url->offset > 0) {
543 /*
544 * We tried to restart a transfer, but for
545 * some reason gave up - so we have to restart
546 * from scratch if we want the whole file
547 */
548 url->offset = 0;
549 if ((f = fetchXGet(url, &us, flags)) == NULL) {
550 warnx("%s: %s", URL, fetchLastErrString);
551 goto failure;
552 }
553 if (sigint)
554 goto signal;
555 }
556
557 /* construct a temp file name */
558 if (sb.st_size != -1 && S_ISREG(sb.st_mode)) {
559 if ((slash = strrchr(path, '/')) == NULL)
560 slash = path;
561 else
562 ++slash;
563 asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s",
564 (int)(slash - path), path, slash);
565 if (tmppath != NULL) {
566 mkstemps(tmppath, strlen(slash) + 1);
567 of = fopen(tmppath, "w");
568 }
569 }
570 if (of == NULL)
571 of = fopen(path, "w");
572 if (of == NULL) {
573 warn("%s: open()", path);
574 goto failure;
575 }
576 }
577 count = url->offset;
578
579 /* start the counter */
580 stat_start(&xs, path, us.size, count);
581
582 sigalrm = siginfo = sigint = 0;
583
584 /* suck in the data */
585 signal(SIGINFO, sig_handler);
586 while (!sigint) {
587 if (us.size != -1 && us.size - count < B_size)
588 size = us.size - count;
589 else
590 size = B_size;
591 if (siginfo) {
592 stat_end(&xs);
593 siginfo = 0;
594 }
595 if ((size = fread(buf, 1, size, f)) == 0) {
596 if (ferror(f) && errno == EINTR && !sigint)
597 clearerr(f);
598 else
599 break;
600 }
601 stat_update(&xs, count += size);
602 for (ptr = buf; size > 0; ptr += wr, size -= wr)
603 if ((wr = fwrite(ptr, 1, size, of)) < size) {
604 if (ferror(of) && errno == EINTR && !sigint)
605 clearerr(of);
606 else
607 break;
608 }
609 if (size != 0)
610 break;
611 }
612 if (!sigalrm)
613 sigalrm = ferror(f) && errno == ETIMEDOUT;
614 signal(SIGINFO, SIG_DFL);
615
616 stat_end(&xs);
617
618 /*
619 * If the transfer timed out or was interrupted, we still want to
620 * set the mtime in case the file is not removed (-r or -R) and
621 * the user later restarts the transfer.
622 */
623 signal:
624 /* set mtime of local file */
625 if (!n_flag && us.mtime && !o_stdout && of != NULL &&
626 (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) {
627 struct timeval tv[2];
628
629 fflush(of);
630 tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime);
631 tv[1].tv_sec = (long)us.mtime;
632 tv[0].tv_usec = tv[1].tv_usec = 0;
633 if (utimes(tmppath ? tmppath : path, tv))
634 warn("%s: utimes()", tmppath ? tmppath : path);
635 }
636
637 /* timed out or interrupted? */
638 if (sigalrm)
639 warnx("transfer timed out");
640 if (sigint) {
641 warnx("transfer interrupted");
642 goto failure;
643 }
644
645 /* timeout / interrupt before connection completley established? */
646 if (f == NULL)
647 goto failure;
648
649 if (!sigalrm) {
650 /* check the status of our files */
651 if (ferror(f))
652 warn("%s", URL);
653 if (ferror(of))
654 warn("%s", path);
655 if (ferror(f) || ferror(of))
656 goto failure;
657 }
658
659 /* did the transfer complete normally? */
660 if (us.size != -1 && count < us.size) {
661 warnx("%s appears to be truncated: %jd/%jd bytes",
662 path, (intmax_t)count, (intmax_t)us.size);
663 goto failure_keep;
664 }
665
666 /*
667 * If the transfer timed out and we didn't know how much to
668 * expect, assume the worst (i.e. we didn't get all of it)
669 */
670 if (sigalrm && us.size == -1) {
671 warnx("%s may be truncated", path);
672 goto failure_keep;
673 }
674
675 success:
676 r = 0;
677 if (tmppath != NULL && rename(tmppath, path) == -1) {
678 warn("%s: rename()", path);
679 goto failure_keep;
680 }
681 goto done;
682 failure:
683 if (of && of != stdout && !R_flag && !r_flag)
684 if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG))
685 unlink(tmppath ? tmppath : path);
686 if (R_flag && tmppath != NULL && sb.st_size == -1)
687 rename(tmppath, path); /* ignore errors here */
688 failure_keep:
689 r = -1;
690 goto done;
691 done:
692 if (f)
693 fclose(f);
694 if (of && of != stdout)
695 fclose(of);
696 if (url)
697 fetchFreeURL(url);
698 if (tmppath != NULL)
699 free(tmppath);
700 return (r);
701}
702
703static void
704usage(void)
705{
706 fprintf(stderr, "%s\n%s\n%s\n",
707 "usage: fetch [-146AFMPRUadlmnpqrsv] [-N netrc] [-o outputfile]",
708 " [-S bytes] [-B bytes] [-T seconds] [-w seconds]",
709 " [-h host -f file [-c dir] | URL ...]");
710}
711
712
713/*
714 * Entry point
715 */
716int
717main(int argc, char *argv[])
718{
719 struct stat sb;
720 struct sigaction sa;
721 const char *p, *s;
722 char *end, *q;
723 int c, e, r;
724
725 while ((c = getopt(argc, argv,
726 "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
727 switch (c) {
728 case '1':
729 once_flag = 1;
730 break;
731 case '4':
732 family = PF_INET;
733 break;
734 case '6':
735 family = PF_INET6;
736 break;
737 case 'A':
738 A_flag = 1;
739 break;
740 case 'a':
741 a_flag = 1;
742 break;
743 case 'B':
744 B_size = (off_t)strtol(optarg, &end, 10);
745 if (*optarg == '\0' || *end != '\0')
746 errx(1, "invalid buffer size (%s)", optarg);
747 break;
748 case 'b':
749 warnx("warning: the -b option is deprecated");
750 b_flag = 1;
751 break;
752 case 'c':
753 c_dirname = optarg;
754 break;
755 case 'd':
756 d_flag = 1;
757 break;
758 case 'F':
759 F_flag = 1;
760 break;
761 case 'f':
762 f_filename = optarg;
763 break;
764 case 'H':
765 warnx("the -H option is now implicit, "
766 "use -U to disable");
767 break;
768 case 'h':
769 h_hostname = optarg;
770 break;
771 case 'l':
772 l_flag = 1;
773 break;
774 case 'o':
775 o_flag = 1;
776 o_filename = optarg;
777 break;
778 case 'M':
779 case 'm':
780 if (r_flag)
781 errx(1, "the -m and -r flags "
782 "are mutually exclusive");
783 m_flag = 1;
784 break;
785 case 'N':
786 N_filename = optarg;
787 break;
788 case 'n':
789 n_flag = 1;
790 break;
791 case 'P':
792 case 'p':
793 p_flag = 1;
794 break;
795 case 'q':
796 v_level = 0;
797 break;
798 case 'R':
799 R_flag = 1;
800 break;
801 case 'r':
802 if (m_flag)
803 errx(1, "the -m and -r flags "
804 "are mutually exclusive");
805 r_flag = 1;
806 break;
807 case 'S':
808 S_size = (off_t)strtol(optarg, &end, 10);
809 if (*optarg == '\0' || *end != '\0')
810 errx(1, "invalid size (%s)", optarg);
811 break;
812 case 's':
813 s_flag = 1;
814 break;
815 case 'T':
816 T_secs = strtol(optarg, &end, 10);
817 if (*optarg == '\0' || *end != '\0')
818 errx(1, "invalid timeout (%s)", optarg);
819 break;
820 case 't':
821 t_flag = 1;
822 warnx("warning: the -t option is deprecated");
823 break;
824 case 'U':
825 U_flag = 1;
826 break;
827 case 'v':
828 v_level++;
829 break;
830 case 'w':
831 a_flag = 1;
832 w_secs = strtol(optarg, &end, 10);
833 if (*optarg == '\0' || *end != '\0')
834 errx(1, "invalid delay (%s)", optarg);
835 break;
836 default:
837 usage();
838 exit(EX_USAGE);
839 }
840
841 argc -= optind;
842 argv += optind;
843
844 if (h_hostname || f_filename || c_dirname) {
845 if (!h_hostname || !f_filename || argc) {
846 usage();
847 exit(EX_USAGE);
848 }
849 /* XXX this is a hack. */
850 if (strcspn(h_hostname, "@:/") != strlen(h_hostname))
851 errx(1, "invalid hostname");
852 if (asprintf(argv, "ftp://%s/%s/%s", h_hostname,
853 c_dirname ? c_dirname : "", f_filename) == -1)
854 errx(1, "%s", strerror(ENOMEM));
855 argc++;
856 }
857
858 if (!argc) {
859 usage();
860 exit(EX_USAGE);
861 }
862
863 /* allocate buffer */
864 if (B_size < MINBUFSIZE)
865 B_size = MINBUFSIZE;
866 if ((buf = malloc(B_size)) == NULL)
867 errx(1, "%s", strerror(ENOMEM));
868
869 /* timeouts */
870 if ((s = getenv("FTP_TIMEOUT")) != NULL) {
871 ftp_timeout = strtol(s, &end, 10);
872 if (*s == '\0' || *end != '\0' || ftp_timeout < 0) {
873 warnx("FTP_TIMEOUT (%s) is not a positive integer", s);
874 ftp_timeout = 0;
875 }
876 }
877 if ((s = getenv("HTTP_TIMEOUT")) != NULL) {
878 http_timeout = strtol(s, &end, 10);
879 if (*s == '\0' || *end != '\0' || http_timeout < 0) {
880 warnx("HTTP_TIMEOUT (%s) is not a positive integer", s);
881 http_timeout = 0;
882 }
883 }
884
885 /* signal handling */
886 sa.sa_flags = 0;
887 sa.sa_handler = sig_handler;
888 sigemptyset(&sa.sa_mask);
889 sigaction(SIGALRM, &sa, NULL);
890 sa.sa_flags = SA_RESETHAND;
891 sigaction(SIGINT, &sa, NULL);
892 fetchRestartCalls = 0;
893
894 /* output file */
895 if (o_flag) {
896 if (strcmp(o_filename, "-") == 0) {
897 o_stdout = 1;
898 } else if (stat(o_filename, &sb) == -1) {
899 if (errno == ENOENT) {
900 if (argc > 1)
901 errx(EX_USAGE, "%s is not a directory",
902 o_filename);
903 } else {
904 err(EX_IOERR, "%s", o_filename);
905 }
906 } else {
907 if (sb.st_mode & S_IFDIR)
908 o_directory = 1;
909 }
910 }
911
912 /* check if output is to a tty (for progress report) */
913 v_tty = isatty(STDERR_FILENO);
914 if (v_tty)
915 pgrp = getpgrp();
916
917 r = 0;
918
919 /* authentication */
920 if (v_tty)
921 fetchAuthMethod = query_auth;
922 if (N_filename != NULL)
923 setenv("NETRC", N_filename, 1);
924
925 while (argc) {
926 if ((p = strrchr(*argv, '/')) == NULL)
927 p = *argv;
928 else
929 p++;
930
931 if (!*p)
932 p = "fetch.out";
933
934 fetchLastErrCode = 0;
935
936 if (o_flag) {
937 if (o_stdout) {
938 e = fetch(*argv, "-");
939 } else if (o_directory) {
940 asprintf(&q, "%s/%s", o_filename, p);
941 e = fetch(*argv, q);
942 free(q);
943 } else {
944 e = fetch(*argv, o_filename);
945 }
946 } else {
947 e = fetch(*argv, p);
948 }
949
950 if (sigint)
951 kill(getpid(), SIGINT);
952
953 if (e == 0 && once_flag)
954 exit(0);
955
956 if (e) {
957 r = 1;
958 if ((fetchLastErrCode
959 && fetchLastErrCode != FETCH_UNAVAIL
960 && fetchLastErrCode != FETCH_MOVED
961 && fetchLastErrCode != FETCH_URL
962 && fetchLastErrCode != FETCH_RESOLV
963 && fetchLastErrCode != FETCH_UNKNOWN)) {
964 if (w_secs && v_level)
965 fprintf(stderr, "Waiting %ld seconds "
966 "before retrying\n", w_secs);
967 if (w_secs)
968 sleep(w_secs);
969 if (a_flag)
970 continue;
971 }
972 }
973
974 argc--, argv++;
975 }
976
977 exit(r);
978}
2 * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer
10 * in this position and unchanged.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 134350 2004-08-26 15:51:10Z des $");
31
32#include <sys/param.h>
33#include <sys/socket.h>
34#include <sys/stat.h>
35#include <sys/time.h>
36
37#include <ctype.h>
38#include <err.h>
39#include <errno.h>
40#include <signal.h>
41#include <stdint.h>
42#include <stdio.h>
43#include <stdlib.h>
44#include <string.h>
45#include <sysexits.h>
46#include <termios.h>
47#include <unistd.h>
48
49#include <fetch.h>
50
51#define MINBUFSIZE 4096
52
53/* Option flags */
54int A_flag; /* -A: do not follow 302 redirects */
55int a_flag; /* -a: auto retry */
56off_t B_size; /* -B: buffer size */
57int b_flag; /*! -b: workaround TCP bug */
58char *c_dirname; /* -c: remote directory */
59int d_flag; /* -d: direct connection */
60int F_flag; /* -F: restart without checking mtime */
61char *f_filename; /* -f: file to fetch */
62char *h_hostname; /* -h: host to fetch from */
63int l_flag; /* -l: link rather than copy file: URLs */
64int m_flag; /* -[Mm]: mirror mode */
65char *N_filename; /* -N: netrc file name */
66int n_flag; /* -n: do not preserve modification time */
67int o_flag; /* -o: specify output file */
68int o_directory; /* output file is a directory */
69char *o_filename; /* name of output file */
70int o_stdout; /* output file is stdout */
71int once_flag; /* -1: stop at first successful file */
72int p_flag; /* -[Pp]: use passive FTP */
73int R_flag; /* -R: don't delete partially transferred files */
74int r_flag; /* -r: restart previously interrupted transfer */
75off_t S_size; /* -S: require size to match */
76int s_flag; /* -s: show size, don't fetch */
77long T_secs = 120; /* -T: transfer timeout in seconds */
78int t_flag; /*! -t: workaround TCP bug */
79int U_flag; /* -U: do not use high ports */
80int v_level = 1; /* -v: verbosity level */
81int v_tty; /* stdout is a tty */
82pid_t pgrp; /* our process group */
83long w_secs; /* -w: retry delay */
84int family = PF_UNSPEC; /* -[46]: address family to use */
85
86int sigalrm; /* SIGALRM received */
87int siginfo; /* SIGINFO received */
88int sigint; /* SIGINT received */
89
90long ftp_timeout; /* default timeout for FTP transfers */
91long http_timeout; /* default timeout for HTTP transfers */
92char *buf; /* transfer buffer */
93
94
95/*
96 * Signal handler
97 */
98static void
99sig_handler(int sig)
100{
101 switch (sig) {
102 case SIGALRM:
103 sigalrm = 1;
104 break;
105 case SIGINFO:
106 siginfo = 1;
107 break;
108 case SIGINT:
109 sigint = 1;
110 break;
111 }
112}
113
114struct xferstat {
115 char name[64];
116 struct timeval start;
117 struct timeval last;
118 off_t size;
119 off_t offset;
120 off_t rcvd;
121};
122
123/*
124 * Compute and display ETA
125 */
126static const char *
127stat_eta(struct xferstat *xs)
128{
129 static char str[16];
130 long elapsed, eta;
131 off_t received, expected;
132
133 elapsed = xs->last.tv_sec - xs->start.tv_sec;
134 received = xs->rcvd - xs->offset;
135 expected = xs->size - xs->rcvd;
136 eta = (long)((double)elapsed * expected / received);
137 if (eta > 3600)
138 snprintf(str, sizeof str, "%02ldh%02ldm",
139 eta / 3600, (eta % 3600) / 60);
140 else
141 snprintf(str, sizeof str, "%02ldm%02lds",
142 eta / 60, eta % 60);
143 return (str);
144}
145
146/*
147 * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'...
148 */
149static const char *prefixes = " kMGTP";
150static const char *
151stat_bytes(off_t bytes)
152{
153 static char str[16];
154 const char *prefix = prefixes;
155
156 while (bytes > 9999 && prefix[1] != '\0') {
157 bytes /= 1024;
158 prefix++;
159 }
160 snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix);
161 return (str);
162}
163
164/*
165 * Compute and display transfer rate
166 */
167static const char *
168stat_bps(struct xferstat *xs)
169{
170 static char str[16];
171 double delta, bps;
172
173 delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6))
174 - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6));
175 if (delta == 0.0) {
176 snprintf(str, sizeof str, "?? Bps");
177 } else {
178 bps = (xs->rcvd - xs->offset) / delta;
179 snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps));
180 }
181 return (str);
182}
183
184/*
185 * Update the stats display
186 */
187static void
188stat_display(struct xferstat *xs, int force)
189{
190 struct timeval now;
191 int ctty_pgrp;
192
193 /* check if we're the foreground process */
194 if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 ||
195 (pid_t)ctty_pgrp != pgrp)
196 return;
197
198 gettimeofday(&now, NULL);
199 if (!force && now.tv_sec <= xs->last.tv_sec)
200 return;
201 xs->last = now;
202
203 fprintf(stderr, "\r%-46.46s", xs->name);
204 if (xs->size <= 0) {
205 fprintf(stderr, " %s", stat_bytes(xs->rcvd));
206 } else {
207 fprintf(stderr, "%3d%% of %s",
208 (int)((100.0 * xs->rcvd) / xs->size),
209 stat_bytes(xs->size));
210 }
211 fprintf(stderr, " %s", stat_bps(xs));
212 if (xs->size > 0 && xs->rcvd > 0 &&
213 xs->last.tv_sec >= xs->start.tv_sec + 10)
214 fprintf(stderr, " %s", stat_eta(xs));
215}
216
217/*
218 * Initialize the transfer statistics
219 */
220static void
221stat_start(struct xferstat *xs, const char *name, off_t size, off_t offset)
222{
223 snprintf(xs->name, sizeof xs->name, "%s", name);
224 gettimeofday(&xs->start, NULL);
225 xs->last.tv_sec = xs->last.tv_usec = 0;
226 xs->size = size;
227 xs->offset = offset;
228 xs->rcvd = offset;
229 if (v_tty && v_level > 0)
230 stat_display(xs, 1);
231 else if (v_level > 0)
232 fprintf(stderr, "%-46s", xs->name);
233}
234
235/*
236 * Update the transfer statistics
237 */
238static void
239stat_update(struct xferstat *xs, off_t rcvd)
240{
241 xs->rcvd = rcvd;
242 if (v_tty && v_level > 0)
243 stat_display(xs, 0);
244}
245
246/*
247 * Finalize the transfer statistics
248 */
249static void
250stat_end(struct xferstat *xs)
251{
252 gettimeofday(&xs->last, NULL);
253 if (v_tty && v_level > 0) {
254 stat_display(xs, 1);
255 putc('\n', stderr);
256 } else if (v_level > 0) {
257 fprintf(stderr, " %s %s\n",
258 stat_bytes(xs->size), stat_bps(xs));
259 }
260}
261
262/*
263 * Ask the user for authentication details
264 */
265static int
266query_auth(struct url *URL)
267{
268 struct termios tios;
269 tcflag_t saved_flags;
270 int i, nopwd;
271
272 fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n",
273 URL->scheme, URL->host, URL->port);
274
275 fprintf(stderr, "Login: ");
276 if (fgets(URL->user, sizeof URL->user, stdin) == NULL)
277 return (-1);
278 for (i = strlen(URL->user); i >= 0; --i)
279 if (URL->user[i] == '\r' || URL->user[i] == '\n')
280 URL->user[i] = '\0';
281
282 fprintf(stderr, "Password: ");
283 if (tcgetattr(STDIN_FILENO, &tios) == 0) {
284 saved_flags = tios.c_lflag;
285 tios.c_lflag &= ~ECHO;
286 tios.c_lflag |= ECHONL|ICANON;
287 tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios);
288 nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL);
289 tios.c_lflag = saved_flags;
290 tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios);
291 } else {
292 nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL);
293 }
294 if (nopwd)
295 return (-1);
296 for (i = strlen(URL->pwd); i >= 0; --i)
297 if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n')
298 URL->pwd[i] = '\0';
299
300 return (0);
301}
302
303/*
304 * Fetch a file
305 */
306static int
307fetch(char *URL, const char *path)
308{
309 struct url *url;
310 struct url_stat us;
311 struct stat sb, nsb;
312 struct xferstat xs;
313 FILE *f, *of;
314 size_t size, wr;
315 off_t count;
316 char flags[8];
317 const char *slash;
318 char *tmppath;
319 int r;
320 unsigned timeout;
321 char *ptr;
322
323 f = of = NULL;
324 tmppath = NULL;
325
326 timeout = 0;
327 *flags = 0;
328 count = 0;
329
330 /* set verbosity level */
331 if (v_level > 1)
332 strcat(flags, "v");
333 if (v_level > 2)
334 fetchDebug = 1;
335
336 /* parse URL */
337 if ((url = fetchParseURL(URL)) == NULL) {
338 warnx("%s: parse error", URL);
339 goto failure;
340 }
341
342 /* if no scheme was specified, take a guess */
343 if (!*url->scheme) {
344 if (!*url->host)
345 strcpy(url->scheme, SCHEME_FILE);
346 else if (strncasecmp(url->host, "ftp.", 4) == 0)
347 strcpy(url->scheme, SCHEME_FTP);
348 else if (strncasecmp(url->host, "www.", 4) == 0)
349 strcpy(url->scheme, SCHEME_HTTP);
350 }
351
352 /* common flags */
353 switch (family) {
354 case PF_INET:
355 strcat(flags, "4");
356 break;
357 case PF_INET6:
358 strcat(flags, "6");
359 break;
360 }
361
362 /* FTP specific flags */
363 if (strcmp(url->scheme, "ftp") == 0) {
364 if (p_flag)
365 strcat(flags, "p");
366 if (d_flag)
367 strcat(flags, "d");
368 if (U_flag)
369 strcat(flags, "l");
370 timeout = T_secs ? T_secs : ftp_timeout;
371 }
372
373 /* HTTP specific flags */
374 if (strcmp(url->scheme, "http") == 0) {
375 if (d_flag)
376 strcat(flags, "d");
377 if (A_flag)
378 strcat(flags, "A");
379 timeout = T_secs ? T_secs : http_timeout;
380 }
381
382 /* set the protocol timeout. */
383 fetchTimeout = timeout;
384
385 /* just print size */
386 if (s_flag) {
387 if (timeout)
388 alarm(timeout);
389 r = fetchStat(url, &us, flags);
390 if (timeout)
391 alarm(0);
392 if (sigalrm || sigint)
393 goto signal;
394 if (r == -1) {
395 warnx("%s", fetchLastErrString);
396 goto failure;
397 }
398 if (us.size == -1)
399 printf("Unknown\n");
400 else
401 printf("%jd\n", (intmax_t)us.size);
402 goto success;
403 }
404
405 /*
406 * If the -r flag was specified, we have to compare the local
407 * and remote files, so we should really do a fetchStat()
408 * first, but I know of at least one HTTP server that only
409 * sends the content size in response to GET requests, and
410 * leaves it out of replies to HEAD requests. Also, in the
411 * (frequent) case that the local and remote files match but
412 * the local file is truncated, we have sufficient information
413 * before the compare to issue a correct request. Therefore,
414 * we always issue a GET request as if we were sure the local
415 * file was a truncated copy of the remote file; we can drop
416 * the connection later if we change our minds.
417 */
418 sb.st_size = -1;
419 if (!o_stdout) {
420 r = stat(path, &sb);
421 if (r == 0 && r_flag && S_ISREG(sb.st_mode)) {
422 url->offset = sb.st_size;
423 } else {
424 /*
425 * Whatever value sb.st_size has now is either
426 * wrong (if stat(2) failed) or irrelevant (if the
427 * path does not refer to a regular file)
428 */
429 sb.st_size = -1;
430 if (r == -1 && errno != ENOENT) {
431 warnx("%s: stat()", path);
432 goto failure;
433 }
434 }
435 }
436
437 /* start the transfer */
438 if (timeout)
439 alarm(timeout);
440 f = fetchXGet(url, &us, flags);
441 if (timeout)
442 alarm(0);
443 if (sigalrm || sigint)
444 goto signal;
445 if (f == NULL) {
446 warnx("%s: %s", URL, fetchLastErrString);
447 goto failure;
448 }
449 if (sigint)
450 goto signal;
451
452 /* check that size is as expected */
453 if (S_size) {
454 if (us.size == -1) {
455 warnx("%s: size unknown", URL);
456 } else if (us.size != S_size) {
457 warnx("%s: size mismatch: expected %jd, actual %jd",
458 URL, (intmax_t)S_size, (intmax_t)us.size);
459 goto failure;
460 }
461 }
462
463 /* symlink instead of copy */
464 if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) {
465 if (symlink(url->doc, path) == -1) {
466 warn("%s: symlink()", path);
467 goto failure;
468 }
469 goto success;
470 }
471
472 if (us.size == -1 && !o_stdout && v_level > 0)
473 warnx("%s: size of remote file is not known", URL);
474 if (v_level > 1) {
475 if (sb.st_size != -1)
476 fprintf(stderr, "local size / mtime: %jd / %ld\n",
477 (intmax_t)sb.st_size, (long)sb.st_mtime);
478 if (us.size != -1)
479 fprintf(stderr, "remote size / mtime: %jd / %ld\n",
480 (intmax_t)us.size, (long)us.mtime);
481 }
482
483 /* open output file */
484 if (o_stdout) {
485 /* output to stdout */
486 of = stdout;
487 } else if (r_flag && sb.st_size != -1) {
488 /* resume mode, local file exists */
489 if (!F_flag && us.mtime && sb.st_mtime != us.mtime) {
490 /* no match! have to refetch */
491 fclose(f);
492 /* if precious, warn the user and give up */
493 if (R_flag) {
494 warnx("%s: local modification time "
495 "does not match remote", path);
496 goto failure_keep;
497 }
498 } else if (us.size != -1) {
499 if (us.size == sb.st_size)
500 /* nothing to do */
501 goto success;
502 if (sb.st_size > us.size) {
503 /* local file too long! */
504 warnx("%s: local file (%jd bytes) is longer "
505 "than remote file (%jd bytes)", path,
506 (intmax_t)sb.st_size, (intmax_t)us.size);
507 goto failure;
508 }
509 /* we got it, open local file */
510 if ((of = fopen(path, "a")) == NULL) {
511 warn("%s: fopen()", path);
512 goto failure;
513 }
514 /* check that it didn't move under our feet */
515 if (fstat(fileno(of), &nsb) == -1) {
516 /* can't happen! */
517 warn("%s: fstat()", path);
518 goto failure;
519 }
520 if (nsb.st_dev != sb.st_dev ||
521 nsb.st_ino != nsb.st_ino ||
522 nsb.st_size != sb.st_size) {
523 warnx("%s: file has changed", URL);
524 fclose(of);
525 of = NULL;
526 sb = nsb;
527 }
528 }
529 } else if (m_flag && sb.st_size != -1) {
530 /* mirror mode, local file exists */
531 if (sb.st_size == us.size && sb.st_mtime == us.mtime)
532 goto success;
533 }
534
535 if (of == NULL) {
536 /*
537 * We don't yet have an output file; either this is a
538 * vanilla run with no special flags, or the local and
539 * remote files didn't match.
540 */
541
542 if (url->offset > 0) {
543 /*
544 * We tried to restart a transfer, but for
545 * some reason gave up - so we have to restart
546 * from scratch if we want the whole file
547 */
548 url->offset = 0;
549 if ((f = fetchXGet(url, &us, flags)) == NULL) {
550 warnx("%s: %s", URL, fetchLastErrString);
551 goto failure;
552 }
553 if (sigint)
554 goto signal;
555 }
556
557 /* construct a temp file name */
558 if (sb.st_size != -1 && S_ISREG(sb.st_mode)) {
559 if ((slash = strrchr(path, '/')) == NULL)
560 slash = path;
561 else
562 ++slash;
563 asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s",
564 (int)(slash - path), path, slash);
565 if (tmppath != NULL) {
566 mkstemps(tmppath, strlen(slash) + 1);
567 of = fopen(tmppath, "w");
568 }
569 }
570 if (of == NULL)
571 of = fopen(path, "w");
572 if (of == NULL) {
573 warn("%s: open()", path);
574 goto failure;
575 }
576 }
577 count = url->offset;
578
579 /* start the counter */
580 stat_start(&xs, path, us.size, count);
581
582 sigalrm = siginfo = sigint = 0;
583
584 /* suck in the data */
585 signal(SIGINFO, sig_handler);
586 while (!sigint) {
587 if (us.size != -1 && us.size - count < B_size)
588 size = us.size - count;
589 else
590 size = B_size;
591 if (siginfo) {
592 stat_end(&xs);
593 siginfo = 0;
594 }
595 if ((size = fread(buf, 1, size, f)) == 0) {
596 if (ferror(f) && errno == EINTR && !sigint)
597 clearerr(f);
598 else
599 break;
600 }
601 stat_update(&xs, count += size);
602 for (ptr = buf; size > 0; ptr += wr, size -= wr)
603 if ((wr = fwrite(ptr, 1, size, of)) < size) {
604 if (ferror(of) && errno == EINTR && !sigint)
605 clearerr(of);
606 else
607 break;
608 }
609 if (size != 0)
610 break;
611 }
612 if (!sigalrm)
613 sigalrm = ferror(f) && errno == ETIMEDOUT;
614 signal(SIGINFO, SIG_DFL);
615
616 stat_end(&xs);
617
618 /*
619 * If the transfer timed out or was interrupted, we still want to
620 * set the mtime in case the file is not removed (-r or -R) and
621 * the user later restarts the transfer.
622 */
623 signal:
624 /* set mtime of local file */
625 if (!n_flag && us.mtime && !o_stdout && of != NULL &&
626 (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) {
627 struct timeval tv[2];
628
629 fflush(of);
630 tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime);
631 tv[1].tv_sec = (long)us.mtime;
632 tv[0].tv_usec = tv[1].tv_usec = 0;
633 if (utimes(tmppath ? tmppath : path, tv))
634 warn("%s: utimes()", tmppath ? tmppath : path);
635 }
636
637 /* timed out or interrupted? */
638 if (sigalrm)
639 warnx("transfer timed out");
640 if (sigint) {
641 warnx("transfer interrupted");
642 goto failure;
643 }
644
645 /* timeout / interrupt before connection completley established? */
646 if (f == NULL)
647 goto failure;
648
649 if (!sigalrm) {
650 /* check the status of our files */
651 if (ferror(f))
652 warn("%s", URL);
653 if (ferror(of))
654 warn("%s", path);
655 if (ferror(f) || ferror(of))
656 goto failure;
657 }
658
659 /* did the transfer complete normally? */
660 if (us.size != -1 && count < us.size) {
661 warnx("%s appears to be truncated: %jd/%jd bytes",
662 path, (intmax_t)count, (intmax_t)us.size);
663 goto failure_keep;
664 }
665
666 /*
667 * If the transfer timed out and we didn't know how much to
668 * expect, assume the worst (i.e. we didn't get all of it)
669 */
670 if (sigalrm && us.size == -1) {
671 warnx("%s may be truncated", path);
672 goto failure_keep;
673 }
674
675 success:
676 r = 0;
677 if (tmppath != NULL && rename(tmppath, path) == -1) {
678 warn("%s: rename()", path);
679 goto failure_keep;
680 }
681 goto done;
682 failure:
683 if (of && of != stdout && !R_flag && !r_flag)
684 if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG))
685 unlink(tmppath ? tmppath : path);
686 if (R_flag && tmppath != NULL && sb.st_size == -1)
687 rename(tmppath, path); /* ignore errors here */
688 failure_keep:
689 r = -1;
690 goto done;
691 done:
692 if (f)
693 fclose(f);
694 if (of && of != stdout)
695 fclose(of);
696 if (url)
697 fetchFreeURL(url);
698 if (tmppath != NULL)
699 free(tmppath);
700 return (r);
701}
702
703static void
704usage(void)
705{
706 fprintf(stderr, "%s\n%s\n%s\n",
707 "usage: fetch [-146AFMPRUadlmnpqrsv] [-N netrc] [-o outputfile]",
708 " [-S bytes] [-B bytes] [-T seconds] [-w seconds]",
709 " [-h host -f file [-c dir] | URL ...]");
710}
711
712
713/*
714 * Entry point
715 */
716int
717main(int argc, char *argv[])
718{
719 struct stat sb;
720 struct sigaction sa;
721 const char *p, *s;
722 char *end, *q;
723 int c, e, r;
724
725 while ((c = getopt(argc, argv,
726 "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
727 switch (c) {
728 case '1':
729 once_flag = 1;
730 break;
731 case '4':
732 family = PF_INET;
733 break;
734 case '6':
735 family = PF_INET6;
736 break;
737 case 'A':
738 A_flag = 1;
739 break;
740 case 'a':
741 a_flag = 1;
742 break;
743 case 'B':
744 B_size = (off_t)strtol(optarg, &end, 10);
745 if (*optarg == '\0' || *end != '\0')
746 errx(1, "invalid buffer size (%s)", optarg);
747 break;
748 case 'b':
749 warnx("warning: the -b option is deprecated");
750 b_flag = 1;
751 break;
752 case 'c':
753 c_dirname = optarg;
754 break;
755 case 'd':
756 d_flag = 1;
757 break;
758 case 'F':
759 F_flag = 1;
760 break;
761 case 'f':
762 f_filename = optarg;
763 break;
764 case 'H':
765 warnx("the -H option is now implicit, "
766 "use -U to disable");
767 break;
768 case 'h':
769 h_hostname = optarg;
770 break;
771 case 'l':
772 l_flag = 1;
773 break;
774 case 'o':
775 o_flag = 1;
776 o_filename = optarg;
777 break;
778 case 'M':
779 case 'm':
780 if (r_flag)
781 errx(1, "the -m and -r flags "
782 "are mutually exclusive");
783 m_flag = 1;
784 break;
785 case 'N':
786 N_filename = optarg;
787 break;
788 case 'n':
789 n_flag = 1;
790 break;
791 case 'P':
792 case 'p':
793 p_flag = 1;
794 break;
795 case 'q':
796 v_level = 0;
797 break;
798 case 'R':
799 R_flag = 1;
800 break;
801 case 'r':
802 if (m_flag)
803 errx(1, "the -m and -r flags "
804 "are mutually exclusive");
805 r_flag = 1;
806 break;
807 case 'S':
808 S_size = (off_t)strtol(optarg, &end, 10);
809 if (*optarg == '\0' || *end != '\0')
810 errx(1, "invalid size (%s)", optarg);
811 break;
812 case 's':
813 s_flag = 1;
814 break;
815 case 'T':
816 T_secs = strtol(optarg, &end, 10);
817 if (*optarg == '\0' || *end != '\0')
818 errx(1, "invalid timeout (%s)", optarg);
819 break;
820 case 't':
821 t_flag = 1;
822 warnx("warning: the -t option is deprecated");
823 break;
824 case 'U':
825 U_flag = 1;
826 break;
827 case 'v':
828 v_level++;
829 break;
830 case 'w':
831 a_flag = 1;
832 w_secs = strtol(optarg, &end, 10);
833 if (*optarg == '\0' || *end != '\0')
834 errx(1, "invalid delay (%s)", optarg);
835 break;
836 default:
837 usage();
838 exit(EX_USAGE);
839 }
840
841 argc -= optind;
842 argv += optind;
843
844 if (h_hostname || f_filename || c_dirname) {
845 if (!h_hostname || !f_filename || argc) {
846 usage();
847 exit(EX_USAGE);
848 }
849 /* XXX this is a hack. */
850 if (strcspn(h_hostname, "@:/") != strlen(h_hostname))
851 errx(1, "invalid hostname");
852 if (asprintf(argv, "ftp://%s/%s/%s", h_hostname,
853 c_dirname ? c_dirname : "", f_filename) == -1)
854 errx(1, "%s", strerror(ENOMEM));
855 argc++;
856 }
857
858 if (!argc) {
859 usage();
860 exit(EX_USAGE);
861 }
862
863 /* allocate buffer */
864 if (B_size < MINBUFSIZE)
865 B_size = MINBUFSIZE;
866 if ((buf = malloc(B_size)) == NULL)
867 errx(1, "%s", strerror(ENOMEM));
868
869 /* timeouts */
870 if ((s = getenv("FTP_TIMEOUT")) != NULL) {
871 ftp_timeout = strtol(s, &end, 10);
872 if (*s == '\0' || *end != '\0' || ftp_timeout < 0) {
873 warnx("FTP_TIMEOUT (%s) is not a positive integer", s);
874 ftp_timeout = 0;
875 }
876 }
877 if ((s = getenv("HTTP_TIMEOUT")) != NULL) {
878 http_timeout = strtol(s, &end, 10);
879 if (*s == '\0' || *end != '\0' || http_timeout < 0) {
880 warnx("HTTP_TIMEOUT (%s) is not a positive integer", s);
881 http_timeout = 0;
882 }
883 }
884
885 /* signal handling */
886 sa.sa_flags = 0;
887 sa.sa_handler = sig_handler;
888 sigemptyset(&sa.sa_mask);
889 sigaction(SIGALRM, &sa, NULL);
890 sa.sa_flags = SA_RESETHAND;
891 sigaction(SIGINT, &sa, NULL);
892 fetchRestartCalls = 0;
893
894 /* output file */
895 if (o_flag) {
896 if (strcmp(o_filename, "-") == 0) {
897 o_stdout = 1;
898 } else if (stat(o_filename, &sb) == -1) {
899 if (errno == ENOENT) {
900 if (argc > 1)
901 errx(EX_USAGE, "%s is not a directory",
902 o_filename);
903 } else {
904 err(EX_IOERR, "%s", o_filename);
905 }
906 } else {
907 if (sb.st_mode & S_IFDIR)
908 o_directory = 1;
909 }
910 }
911
912 /* check if output is to a tty (for progress report) */
913 v_tty = isatty(STDERR_FILENO);
914 if (v_tty)
915 pgrp = getpgrp();
916
917 r = 0;
918
919 /* authentication */
920 if (v_tty)
921 fetchAuthMethod = query_auth;
922 if (N_filename != NULL)
923 setenv("NETRC", N_filename, 1);
924
925 while (argc) {
926 if ((p = strrchr(*argv, '/')) == NULL)
927 p = *argv;
928 else
929 p++;
930
931 if (!*p)
932 p = "fetch.out";
933
934 fetchLastErrCode = 0;
935
936 if (o_flag) {
937 if (o_stdout) {
938 e = fetch(*argv, "-");
939 } else if (o_directory) {
940 asprintf(&q, "%s/%s", o_filename, p);
941 e = fetch(*argv, q);
942 free(q);
943 } else {
944 e = fetch(*argv, o_filename);
945 }
946 } else {
947 e = fetch(*argv, p);
948 }
949
950 if (sigint)
951 kill(getpid(), SIGINT);
952
953 if (e == 0 && once_flag)
954 exit(0);
955
956 if (e) {
957 r = 1;
958 if ((fetchLastErrCode
959 && fetchLastErrCode != FETCH_UNAVAIL
960 && fetchLastErrCode != FETCH_MOVED
961 && fetchLastErrCode != FETCH_URL
962 && fetchLastErrCode != FETCH_RESOLV
963 && fetchLastErrCode != FETCH_UNKNOWN)) {
964 if (w_secs && v_level)
965 fprintf(stderr, "Waiting %ld seconds "
966 "before retrying\n", w_secs);
967 if (w_secs)
968 sleep(w_secs);
969 if (a_flag)
970 continue;
971 }
972 }
973
974 argc--, argv++;
975 }
976
977 exit(r);
978}