fetch.c revision 135546
162216Sdes/*- 2135546Sdes * Copyright (c) 2000-2004 Dag-Erling Co�dan Sm�rgrav 362216Sdes * All rights reserved. 462216Sdes * 562216Sdes * Redistribution and use in source and binary forms, with or without 662216Sdes * modification, are permitted provided that the following conditions 762216Sdes * are met: 862216Sdes * 1. Redistributions of source code must retain the above copyright 962216Sdes * notice, this list of conditions and the following disclaimer 1062216Sdes * in this position and unchanged. 1162216Sdes * 2. Redistributions in binary form must reproduce the above copyright 1262216Sdes * notice, this list of conditions and the following disclaimer in the 1362216Sdes * documentation and/or other materials provided with the distribution. 1462216Sdes * 3. The name of the author may not be used to endorse or promote products 1562216Sdes * derived from this software without specific prior written permission 1662216Sdes * 1762216Sdes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1862216Sdes * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1962216Sdes * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2062216Sdes * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2162216Sdes * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2262216Sdes * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2362216Sdes * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2462216Sdes * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2562216Sdes * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2662216Sdes * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2762216Sdes */ 2862216Sdes 2993213Scharnier#include <sys/cdefs.h> 3093213Scharnier__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 135546 2004-09-21 18:35:21Z des $"); 3193213Scharnier 3262216Sdes#include <sys/param.h> 3391225Sbde#include <sys/socket.h> 3462216Sdes#include <sys/stat.h> 3593257Sbde#include <sys/time.h> 3662216Sdes 3762216Sdes#include <ctype.h> 3862216Sdes#include <err.h> 3962216Sdes#include <errno.h> 4063235Sdes#include <signal.h> 41125976Sdes#include <stdint.h> 4262216Sdes#include <stdio.h> 4362216Sdes#include <stdlib.h> 4462216Sdes#include <string.h> 4562216Sdes#include <sysexits.h> 4677241Sdes#include <termios.h> 4762216Sdes#include <unistd.h> 4862216Sdes 4962216Sdes#include <fetch.h> 5062216Sdes 5162216Sdes#define MINBUFSIZE 4096 5262216Sdes 5362216Sdes/* Option flags */ 5462216Sdesint A_flag; /* -A: do not follow 302 redirects */ 5562216Sdesint a_flag; /* -a: auto retry */ 5679837Sdesoff_t B_size; /* -B: buffer size */ 5762216Sdesint b_flag; /*! -b: workaround TCP bug */ 5862254Sdeschar *c_dirname; /* -c: remote directory */ 5962216Sdesint d_flag; /* -d: direct connection */ 6062216Sdesint F_flag; /* -F: restart without checking mtime */ 6162216Sdeschar *f_filename; /* -f: file to fetch */ 6262216Sdeschar *h_hostname; /* -h: host to fetch from */ 6362216Sdesint l_flag; /* -l: link rather than copy file: URLs */ 6462815Sdesint m_flag; /* -[Mm]: mirror mode */ 65109702Sdeschar *N_filename; /* -N: netrc file name */ 6662815Sdesint n_flag; /* -n: do not preserve modification time */ 6762216Sdesint o_flag; /* -o: specify output file */ 6862216Sdesint o_directory; /* output file is a directory */ 6962216Sdeschar *o_filename; /* name of output file */ 7062216Sdesint o_stdout; /* output file is stdout */ 7162216Sdesint once_flag; /* -1: stop at first successful file */ 7263501Sdesint p_flag; /* -[Pp]: use passive FTP */ 7362216Sdesint R_flag; /* -R: don't delete partially transferred files */ 7462216Sdesint r_flag; /* -r: restart previously interrupted transfer */ 7574717Sdesoff_t S_size; /* -S: require size to match */ 7674717Sdesint s_flag; /* -s: show size, don't fetch */ 77100834Sdeslong T_secs = 120; /* -T: transfer timeout in seconds */ 7862216Sdesint t_flag; /*! -t: workaround TCP bug */ 7974717Sdesint U_flag; /* -U: do not use high ports */ 8062216Sdesint v_level = 1; /* -v: verbosity level */ 8162216Sdesint v_tty; /* stdout is a tty */ 8283863Sdespid_t pgrp; /* our process group */ 83100834Sdeslong w_secs; /* -w: retry delay */ 8462216Sdesint family = PF_UNSPEC; /* -[46]: address family to use */ 8562216Sdes 8663015Sdesint sigalrm; /* SIGALRM received */ 8773937Sdesint siginfo; /* SIGINFO received */ 8863015Sdesint sigint; /* SIGINT received */ 8962216Sdes 90100834Sdeslong ftp_timeout; /* default timeout for FTP transfers */ 91100834Sdeslong http_timeout; /* default timeout for HTTP transfers */ 92125976Sdeschar *buf; /* transfer buffer */ 9362216Sdes 9462216Sdes 9581863Sdes/* 9681863Sdes * Signal handler 9781863Sdes */ 9879837Sdesstatic void 9962216Sdessig_handler(int sig) 10062216Sdes{ 10179837Sdes switch (sig) { 10279837Sdes case SIGALRM: 10379837Sdes sigalrm = 1; 10479837Sdes break; 10579837Sdes case SIGINFO: 10679837Sdes siginfo = 1; 10779837Sdes break; 10879837Sdes case SIGINT: 10979837Sdes sigint = 1; 11079837Sdes break; 11179837Sdes } 11262216Sdes} 11362216Sdes 11462216Sdesstruct xferstat { 115125965Sdes char name[64]; 11679837Sdes struct timeval start; 11779837Sdes struct timeval last; 11879837Sdes off_t size; 11979837Sdes off_t offset; 12079837Sdes off_t rcvd; 12162216Sdes}; 12262216Sdes 12381863Sdes/* 124109702Sdes * Compute and display ETA 125109702Sdes */ 126125965Sdesstatic const char * 127109702Sdesstat_eta(struct xferstat *xs) 128109702Sdes{ 129125965Sdes static char str[16]; 130125976Sdes long elapsed, eta; 131125976Sdes off_t received, expected; 132109702Sdes 133109702Sdes elapsed = xs->last.tv_sec - xs->start.tv_sec; 134112083Sdes received = xs->rcvd - xs->offset; 135112083Sdes expected = xs->size - xs->rcvd; 136112114Sdes eta = (long)((double)elapsed * expected / received); 137125965Sdes if (eta > 3600) 138125965Sdes snprintf(str, sizeof str, "%02ldh%02ldm", 139125965Sdes eta / 3600, (eta % 3600) / 60); 140125965Sdes else 141125965Sdes snprintf(str, sizeof str, "%02ldm%02lds", 142125965Sdes eta / 60, eta % 60); 143125965Sdes return (str); 144125965Sdes} 145125965Sdes 146125965Sdes/* 147125965Sdes * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'... 148125965Sdes */ 149125965Sdesstatic const char *prefixes = " kMGTP"; 150125965Sdesstatic const char * 151129440Slestat_bytes(off_t bytes) 152125965Sdes{ 153125965Sdes static char str[16]; 154125965Sdes const char *prefix = prefixes; 155125965Sdes 156125965Sdes while (bytes > 9999 && prefix[1] != '\0') { 157125965Sdes bytes /= 1024; 158125965Sdes prefix++; 159109702Sdes } 160129440Sle snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix); 161125965Sdes return (str); 162109702Sdes} 163109702Sdes 164109702Sdes/* 165109702Sdes * Compute and display transfer rate 166109702Sdes */ 167125965Sdesstatic const char * 168109702Sdesstat_bps(struct xferstat *xs) 169109702Sdes{ 170125965Sdes static char str[16]; 171109735Sdes double delta, bps; 172109702Sdes 173109735Sdes delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6)) 174109735Sdes - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 175109735Sdes if (delta == 0.0) { 176125965Sdes snprintf(str, sizeof str, "?? Bps"); 177125965Sdes } else { 178125965Sdes bps = (xs->rcvd - xs->offset) / delta; 179129440Sle snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps)); 180109735Sdes } 181125965Sdes return (str); 182109702Sdes} 183109702Sdes 184109702Sdes/* 18581863Sdes * Update the stats display 18681863Sdes */ 18779837Sdesstatic void 18863046Sdesstat_display(struct xferstat *xs, int force) 18962216Sdes{ 19079837Sdes struct timeval now; 19183863Sdes int ctty_pgrp; 19279837Sdes 19383863Sdes /* check if we're the foreground process */ 19483863Sdes if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || 19583863Sdes (pid_t)ctty_pgrp != pgrp) 19683863Sdes return; 197106043Sdes 19879837Sdes gettimeofday(&now, NULL); 19979837Sdes if (!force && now.tv_sec <= xs->last.tv_sec) 20079837Sdes return; 20179837Sdes xs->last = now; 20279837Sdes 203131615Sdes fprintf(stderr, "\r%-46.46s", xs->name); 204106041Sdes if (xs->size <= 0) { 205125965Sdes fprintf(stderr, " %s", stat_bytes(xs->rcvd)); 206106041Sdes } else { 207125965Sdes fprintf(stderr, "%3d%% of %s", 208125965Sdes (int)((100.0 * xs->rcvd) / xs->size), 209125965Sdes stat_bytes(xs->size)); 210106041Sdes } 211125965Sdes fprintf(stderr, " %s", stat_bps(xs)); 212125965Sdes if (xs->size > 0 && xs->rcvd > 0 && 213125965Sdes xs->last.tv_sec >= xs->start.tv_sec + 10) 214125965Sdes fprintf(stderr, " %s", stat_eta(xs)); 21562216Sdes} 21662216Sdes 21781863Sdes/* 21881863Sdes * Initialize the transfer statistics 21981863Sdes */ 22079837Sdesstatic void 22179837Sdesstat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) 22263046Sdes{ 22379837Sdes snprintf(xs->name, sizeof xs->name, "%s", name); 22479837Sdes gettimeofday(&xs->start, NULL); 22579837Sdes xs->last.tv_sec = xs->last.tv_usec = 0; 22679837Sdes xs->size = size; 22779837Sdes xs->offset = offset; 22879837Sdes xs->rcvd = offset; 229125965Sdes if (v_tty && v_level > 0) 230125965Sdes stat_display(xs, 1); 231125965Sdes else if (v_level > 0) 232125965Sdes fprintf(stderr, "%-46s", xs->name); 23363046Sdes} 23463046Sdes 23581863Sdes/* 23681863Sdes * Update the transfer statistics 23781863Sdes */ 23879837Sdesstatic void 23979837Sdesstat_update(struct xferstat *xs, off_t rcvd) 24063046Sdes{ 24179837Sdes xs->rcvd = rcvd; 242125965Sdes if (v_tty && v_level > 0) 243125965Sdes stat_display(xs, 0); 24463046Sdes} 24563046Sdes 24681863Sdes/* 24781863Sdes * Finalize the transfer statistics 24881863Sdes */ 24979837Sdesstatic void 25062216Sdesstat_end(struct xferstat *xs) 25162216Sdes{ 252109735Sdes gettimeofday(&xs->last, NULL); 253125965Sdes if (v_tty && v_level > 0) { 254125965Sdes stat_display(xs, 1); 255125965Sdes putc('\n', stderr); 256125965Sdes } else if (v_level > 0) { 257125965Sdes fprintf(stderr, " %s %s\n", 258125965Sdes stat_bytes(xs->size), stat_bps(xs)); 259125965Sdes } 26062216Sdes} 26162216Sdes 26281863Sdes/* 26381863Sdes * Ask the user for authentication details 26481863Sdes */ 26579837Sdesstatic int 26677241Sdesquery_auth(struct url *URL) 26777241Sdes{ 26879837Sdes struct termios tios; 26979837Sdes tcflag_t saved_flags; 27079837Sdes int i, nopwd; 27177241Sdes 27279837Sdes fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", 27386242Siedowse URL->scheme, URL->host, URL->port); 27479837Sdes 27579837Sdes fprintf(stderr, "Login: "); 27679837Sdes if (fgets(URL->user, sizeof URL->user, stdin) == NULL) 277132695Sdes return (-1); 278132696Sdes for (i = strlen(URL->user); i >= 0; --i) 279132696Sdes if (URL->user[i] == '\r' || URL->user[i] == '\n') 28079837Sdes URL->user[i] = '\0'; 28179837Sdes 28279837Sdes fprintf(stderr, "Password: "); 28379837Sdes if (tcgetattr(STDIN_FILENO, &tios) == 0) { 28479837Sdes saved_flags = tios.c_lflag; 28579837Sdes tios.c_lflag &= ~ECHO; 28679837Sdes tios.c_lflag |= ECHONL|ICANON; 28779837Sdes tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); 28879837Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 28979837Sdes tios.c_lflag = saved_flags; 29079837Sdes tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); 29179837Sdes } else { 29279837Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 29379837Sdes } 29479837Sdes if (nopwd) 295132695Sdes return (-1); 296132696Sdes for (i = strlen(URL->pwd); i >= 0; --i) 297132696Sdes if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n') 298132696Sdes URL->pwd[i] = '\0'; 29979837Sdes 300132695Sdes return (0); 30177241Sdes} 30277241Sdes 30381863Sdes/* 30481863Sdes * Fetch a file 30581863Sdes */ 30679837Sdesstatic int 30779837Sdesfetch(char *URL, const char *path) 30862216Sdes{ 30979837Sdes struct url *url; 31079837Sdes struct url_stat us; 31183217Sdes struct stat sb, nsb; 31279837Sdes struct xferstat xs; 31379837Sdes FILE *f, *of; 31479837Sdes size_t size, wr; 31579837Sdes off_t count; 31679837Sdes char flags[8]; 31783217Sdes const char *slash; 31883217Sdes char *tmppath; 31979837Sdes int r; 320125976Sdes unsigned timeout; 321125976Sdes char *ptr; 32262216Sdes 32379837Sdes f = of = NULL; 32483217Sdes tmppath = NULL; 32562216Sdes 326109702Sdes timeout = 0; 327109702Sdes *flags = 0; 328109702Sdes count = 0; 329109702Sdes 330109702Sdes /* set verbosity level */ 331109702Sdes if (v_level > 1) 332109702Sdes strcat(flags, "v"); 333109702Sdes if (v_level > 2) 334109702Sdes fetchDebug = 1; 335109702Sdes 33679837Sdes /* parse URL */ 33779837Sdes if ((url = fetchParseURL(URL)) == NULL) { 33879837Sdes warnx("%s: parse error", URL); 33979837Sdes goto failure; 34079837Sdes } 34162216Sdes 34279837Sdes /* if no scheme was specified, take a guess */ 34379837Sdes if (!*url->scheme) { 34479837Sdes if (!*url->host) 34579837Sdes strcpy(url->scheme, SCHEME_FILE); 34679837Sdes else if (strncasecmp(url->host, "ftp.", 4) == 0) 34779837Sdes strcpy(url->scheme, SCHEME_FTP); 34879837Sdes else if (strncasecmp(url->host, "www.", 4) == 0) 34979837Sdes strcpy(url->scheme, SCHEME_HTTP); 35079837Sdes } 35169976Sdes 35279837Sdes /* common flags */ 35379837Sdes switch (family) { 35479837Sdes case PF_INET: 35579837Sdes strcat(flags, "4"); 35679837Sdes break; 35779837Sdes case PF_INET6: 35879837Sdes strcat(flags, "6"); 35979837Sdes break; 36079837Sdes } 36162216Sdes 36279837Sdes /* FTP specific flags */ 36379837Sdes if (strcmp(url->scheme, "ftp") == 0) { 36479837Sdes if (p_flag) 36579837Sdes strcat(flags, "p"); 36679837Sdes if (d_flag) 36779837Sdes strcat(flags, "d"); 36879837Sdes if (U_flag) 36979837Sdes strcat(flags, "l"); 37079837Sdes timeout = T_secs ? T_secs : ftp_timeout; 37179837Sdes } 37262216Sdes 37379837Sdes /* HTTP specific flags */ 37479837Sdes if (strcmp(url->scheme, "http") == 0) { 37579837Sdes if (d_flag) 37679837Sdes strcat(flags, "d"); 37779837Sdes if (A_flag) 37879837Sdes strcat(flags, "A"); 37979837Sdes timeout = T_secs ? T_secs : http_timeout; 38079837Sdes } 38162216Sdes 38279837Sdes /* set the protocol timeout. */ 38379837Sdes fetchTimeout = timeout; 38462216Sdes 38579837Sdes /* just print size */ 38679837Sdes if (s_flag) { 387106041Sdes if (timeout) 388106041Sdes alarm(timeout); 389106041Sdes r = fetchStat(url, &us, flags); 390106041Sdes if (timeout) 391106043Sdes alarm(0); 392106041Sdes if (sigalrm || sigint) 393106041Sdes goto signal; 394106041Sdes if (r == -1) { 395106041Sdes warnx("%s", fetchLastErrString); 39679837Sdes goto failure; 397106041Sdes } 39879837Sdes if (us.size == -1) 39979837Sdes printf("Unknown\n"); 40079837Sdes else 401125976Sdes printf("%jd\n", (intmax_t)us.size); 40279837Sdes goto success; 40363345Sdes } 40479837Sdes 40579837Sdes /* 40679837Sdes * If the -r flag was specified, we have to compare the local 40779837Sdes * and remote files, so we should really do a fetchStat() 40879837Sdes * first, but I know of at least one HTTP server that only 40979837Sdes * sends the content size in response to GET requests, and 41079837Sdes * leaves it out of replies to HEAD requests. Also, in the 41179837Sdes * (frequent) case that the local and remote files match but 41279837Sdes * the local file is truncated, we have sufficient information 41379837Sdes * before the compare to issue a correct request. Therefore, 41479837Sdes * we always issue a GET request as if we were sure the local 41579837Sdes * file was a truncated copy of the remote file; we can drop 41679837Sdes * the connection later if we change our minds. 41779837Sdes */ 41883217Sdes sb.st_size = -1; 419133779Sdes if (!o_stdout) { 420133779Sdes r = stat(path, &sb); 421134350Sdes if (r == 0 && r_flag && S_ISREG(sb.st_mode)) { 422133779Sdes url->offset = sb.st_size; 423133779Sdes } else { 424133779Sdes /* 425133779Sdes * Whatever value sb.st_size has now is either 426133779Sdes * wrong (if stat(2) failed) or irrelevant (if the 427133779Sdes * path does not refer to a regular file) 428133779Sdes */ 429133779Sdes sb.st_size = -1; 430133779Sdes if (r == -1 && errno != ENOENT) { 431133779Sdes warnx("%s: stat()", path); 432133779Sdes goto failure; 433133779Sdes } 434133779Sdes } 43562216Sdes } 43662216Sdes 43779837Sdes /* start the transfer */ 438106041Sdes if (timeout) 439106041Sdes alarm(timeout); 440106041Sdes f = fetchXGet(url, &us, flags); 441106042Sdes if (timeout) 442106042Sdes alarm(0); 443106041Sdes if (sigalrm || sigint) 444106041Sdes goto signal; 445106041Sdes if (f == NULL) { 446107353Sdes warnx("%s: %s", URL, fetchLastErrString); 44763345Sdes goto failure; 44879837Sdes } 44979837Sdes if (sigint) 45063345Sdes goto signal; 45179837Sdes 45279837Sdes /* check that size is as expected */ 45379837Sdes if (S_size) { 45479837Sdes if (us.size == -1) { 455107353Sdes warnx("%s: size unknown", URL); 45679837Sdes } else if (us.size != S_size) { 457125976Sdes warnx("%s: size mismatch: expected %jd, actual %jd", 458125976Sdes URL, (intmax_t)S_size, (intmax_t)us.size); 45979837Sdes goto failure; 46079837Sdes } 46179837Sdes } 46279837Sdes 46379837Sdes /* symlink instead of copy */ 46479837Sdes if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 46579837Sdes if (symlink(url->doc, path) == -1) { 46679837Sdes warn("%s: symlink()", path); 46779837Sdes goto failure; 46879837Sdes } 46963568Sdes goto success; 47063345Sdes } 47179837Sdes 472106051Sdes if (us.size == -1 && !o_stdout && v_level > 0) 473107353Sdes warnx("%s: size of remote file is not known", URL); 47479837Sdes if (v_level > 1) { 47579837Sdes if (sb.st_size != -1) 476125976Sdes fprintf(stderr, "local size / mtime: %jd / %ld\n", 477125976Sdes (intmax_t)sb.st_size, (long)sb.st_mtime); 47879837Sdes if (us.size != -1) 479125976Sdes fprintf(stderr, "remote size / mtime: %jd / %ld\n", 480125976Sdes (intmax_t)us.size, (long)us.mtime); 48162216Sdes } 48262216Sdes 48379837Sdes /* open output file */ 48479837Sdes if (o_stdout) { 48579837Sdes /* output to stdout */ 48679837Sdes of = stdout; 48783217Sdes } else if (r_flag && sb.st_size != -1) { 48879837Sdes /* resume mode, local file exists */ 48979837Sdes if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { 49079837Sdes /* no match! have to refetch */ 49179837Sdes fclose(f); 49279837Sdes /* if precious, warn the user and give up */ 49379837Sdes if (R_flag) { 49479837Sdes warnx("%s: local modification time " 49579837Sdes "does not match remote", path); 49679837Sdes goto failure_keep; 49779837Sdes } 498127941Sdes } else if (us.size != -1) { 49979837Sdes if (us.size == sb.st_size) 50079837Sdes /* nothing to do */ 50179837Sdes goto success; 50279837Sdes if (sb.st_size > us.size) { 50379837Sdes /* local file too long! */ 504125976Sdes warnx("%s: local file (%jd bytes) is longer " 505125976Sdes "than remote file (%jd bytes)", path, 506125976Sdes (intmax_t)sb.st_size, (intmax_t)us.size); 50779837Sdes goto failure; 50879837Sdes } 50983217Sdes /* we got it, open local file */ 51079837Sdes if ((of = fopen(path, "a")) == NULL) { 51179837Sdes warn("%s: fopen()", path); 51279837Sdes goto failure; 51379837Sdes } 51483217Sdes /* check that it didn't move under our feet */ 51583217Sdes if (fstat(fileno(of), &nsb) == -1) { 51683217Sdes /* can't happen! */ 51783217Sdes warn("%s: fstat()", path); 51879837Sdes goto failure; 51979837Sdes } 52083217Sdes if (nsb.st_dev != sb.st_dev || 52183217Sdes nsb.st_ino != nsb.st_ino || 52283217Sdes nsb.st_size != sb.st_size) { 523107353Sdes warnx("%s: file has changed", URL); 52483217Sdes fclose(of); 52583217Sdes of = NULL; 52683217Sdes sb = nsb; 52783217Sdes } 52879837Sdes } 52983217Sdes } else if (m_flag && sb.st_size != -1) { 53079837Sdes /* mirror mode, local file exists */ 53179837Sdes if (sb.st_size == us.size && sb.st_mtime == us.mtime) 53279837Sdes goto success; 53379837Sdes } 534106043Sdes 53583217Sdes if (of == NULL) { 53679837Sdes /* 53779837Sdes * We don't yet have an output file; either this is a 53879837Sdes * vanilla run with no special flags, or the local and 53979837Sdes * remote files didn't match. 54079837Sdes */ 541106043Sdes 542109702Sdes if (url->offset > 0) { 54383217Sdes /* 54483217Sdes * We tried to restart a transfer, but for 54583217Sdes * some reason gave up - so we have to restart 54683217Sdes * from scratch if we want the whole file 54783217Sdes */ 54883217Sdes url->offset = 0; 54983217Sdes if ((f = fetchXGet(url, &us, flags)) == NULL) { 550107353Sdes warnx("%s: %s", URL, fetchLastErrString); 55183217Sdes goto failure; 55283217Sdes } 55383217Sdes if (sigint) 55483217Sdes goto signal; 55583217Sdes } 55683217Sdes 55783217Sdes /* construct a temp file name */ 55883217Sdes if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { 55983217Sdes if ((slash = strrchr(path, '/')) == NULL) 56083217Sdes slash = path; 56183217Sdes else 56283217Sdes ++slash; 56383217Sdes asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", 56483307Smike (int)(slash - path), path, slash); 565100834Sdes if (tmppath != NULL) { 566100834Sdes mkstemps(tmppath, strlen(slash) + 1); 567100834Sdes of = fopen(tmppath, "w"); 568100834Sdes } 56983217Sdes } 570100834Sdes if (of == NULL) 57183217Sdes of = fopen(path, "w"); 57283217Sdes if (of == NULL) { 57379837Sdes warn("%s: open()", path); 57479837Sdes goto failure; 57579837Sdes } 57679837Sdes } 57779837Sdes count = url->offset; 57862216Sdes 57979837Sdes /* start the counter */ 58079837Sdes stat_start(&xs, path, us.size, count); 58163046Sdes 58279837Sdes sigalrm = siginfo = sigint = 0; 58379837Sdes 58479837Sdes /* suck in the data */ 58579837Sdes signal(SIGINFO, sig_handler); 586106041Sdes while (!sigint) { 58779837Sdes if (us.size != -1 && us.size - count < B_size) 58879837Sdes size = us.size - count; 58979837Sdes else 59079837Sdes size = B_size; 59179837Sdes if (siginfo) { 59279837Sdes stat_end(&xs); 59379837Sdes siginfo = 0; 59479837Sdes } 59579837Sdes if ((size = fread(buf, 1, size, f)) == 0) { 596106041Sdes if (ferror(f) && errno == EINTR && !sigint) 59779837Sdes clearerr(f); 59879837Sdes else 59979837Sdes break; 60079837Sdes } 60179837Sdes stat_update(&xs, count += size); 60279837Sdes for (ptr = buf; size > 0; ptr += wr, size -= wr) 60379837Sdes if ((wr = fwrite(ptr, 1, size, of)) < size) { 604106041Sdes if (ferror(of) && errno == EINTR && !sigint) 60579837Sdes clearerr(of); 60679837Sdes else 60779837Sdes break; 60879837Sdes } 60979837Sdes if (size != 0) 61079837Sdes break; 61177241Sdes } 612106041Sdes if (!sigalrm) 613106041Sdes sigalrm = ferror(f) && errno == ETIMEDOUT; 61479837Sdes signal(SIGINFO, SIG_DFL); 61579837Sdes 61679837Sdes stat_end(&xs); 61762216Sdes 618106041Sdes /* 619106041Sdes * If the transfer timed out or was interrupted, we still want to 620106041Sdes * set the mtime in case the file is not removed (-r or -R) and 621106041Sdes * the user later restarts the transfer. 622106041Sdes */ 623106041Sdes signal: 62479837Sdes /* set mtime of local file */ 625106857Sdes if (!n_flag && us.mtime && !o_stdout && of != NULL && 626106857Sdes (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { 62779837Sdes struct timeval tv[2]; 628106043Sdes 62979837Sdes fflush(of); 63079837Sdes tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 63179837Sdes tv[1].tv_sec = (long)us.mtime; 63279837Sdes tv[0].tv_usec = tv[1].tv_usec = 0; 63390729Sdes if (utimes(tmppath ? tmppath : path, tv)) 63490729Sdes warn("%s: utimes()", tmppath ? tmppath : path); 63579837Sdes } 63663015Sdes 63779837Sdes /* timed out or interrupted? */ 63879837Sdes if (sigalrm) 63979837Sdes warnx("transfer timed out"); 64079837Sdes if (sigint) { 64179837Sdes warnx("transfer interrupted"); 64279837Sdes goto failure; 64379837Sdes } 64463046Sdes 645106586Sfenner /* timeout / interrupt before connection completley established? */ 646106586Sfenner if (f == NULL) 647106586Sfenner goto failure; 648106586Sfenner 64979837Sdes if (!sigalrm) { 65079837Sdes /* check the status of our files */ 65179837Sdes if (ferror(f)) 65279837Sdes warn("%s", URL); 65379837Sdes if (ferror(of)) 65479837Sdes warn("%s", path); 65579837Sdes if (ferror(f) || ferror(of)) 65679837Sdes goto failure; 65779837Sdes } 65879837Sdes 65979837Sdes /* did the transfer complete normally? */ 66079837Sdes if (us.size != -1 && count < us.size) { 661125976Sdes warnx("%s appears to be truncated: %jd/%jd bytes", 662125976Sdes path, (intmax_t)count, (intmax_t)us.size); 66379837Sdes goto failure_keep; 66479837Sdes } 66579837Sdes 66679837Sdes /* 66779837Sdes * If the transfer timed out and we didn't know how much to 66879837Sdes * expect, assume the worst (i.e. we didn't get all of it) 66979837Sdes */ 67079837Sdes if (sigalrm && us.size == -1) { 67179837Sdes warnx("%s may be truncated", path); 67279837Sdes goto failure_keep; 67379837Sdes } 67479837Sdes 67562216Sdes success: 67679837Sdes r = 0; 67783217Sdes if (tmppath != NULL && rename(tmppath, path) == -1) { 67883217Sdes warn("%s: rename()", path); 67983217Sdes goto failure_keep; 68083217Sdes } 68179837Sdes goto done; 68262216Sdes failure: 68379837Sdes if (of && of != stdout && !R_flag && !r_flag) 68479837Sdes if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) 68583217Sdes unlink(tmppath ? tmppath : path); 68683217Sdes if (R_flag && tmppath != NULL && sb.st_size == -1) 68783217Sdes rename(tmppath, path); /* ignore errors here */ 68863046Sdes failure_keep: 68979837Sdes r = -1; 69079837Sdes goto done; 69162216Sdes done: 69279837Sdes if (f) 69379837Sdes fclose(f); 69479837Sdes if (of && of != stdout) 69579837Sdes fclose(of); 69679837Sdes if (url) 69779837Sdes fetchFreeURL(url); 69883217Sdes if (tmppath != NULL) 69983217Sdes free(tmppath); 700132695Sdes return (r); 70162216Sdes} 70262216Sdes 70379837Sdesstatic void 70462216Sdesusage(void) 70562216Sdes{ 70679837Sdes fprintf(stderr, "%s\n%s\n%s\n", 707109702Sdes "usage: fetch [-146AFMPRUadlmnpqrsv] [-N netrc] [-o outputfile]", 708109702Sdes " [-S bytes] [-B bytes] [-T seconds] [-w seconds]", 70979837Sdes " [-h host -f file [-c dir] | URL ...]"); 71062216Sdes} 71162216Sdes 71262216Sdes 71381863Sdes/* 71481863Sdes * Entry point 71581863Sdes */ 71662216Sdesint 71762216Sdesmain(int argc, char *argv[]) 71862216Sdes{ 71979837Sdes struct stat sb; 72079837Sdes struct sigaction sa; 72179837Sdes const char *p, *s; 722100834Sdes char *end, *q; 72379837Sdes int c, e, r; 72462216Sdes 72579837Sdes while ((c = getopt(argc, argv, 726109702Sdes "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) 72779837Sdes switch (c) { 72879837Sdes case '1': 72979837Sdes once_flag = 1; 73079837Sdes break; 73179837Sdes case '4': 73279837Sdes family = PF_INET; 73379837Sdes break; 73479837Sdes case '6': 73579837Sdes family = PF_INET6; 73679837Sdes break; 73779837Sdes case 'A': 73879837Sdes A_flag = 1; 73979837Sdes break; 74079837Sdes case 'a': 74179837Sdes a_flag = 1; 74279837Sdes break; 74379837Sdes case 'B': 744100834Sdes B_size = (off_t)strtol(optarg, &end, 10); 745100834Sdes if (*optarg == '\0' || *end != '\0') 74680521Sse errx(1, "invalid buffer size (%s)", optarg); 74779837Sdes break; 74879837Sdes case 'b': 74979837Sdes warnx("warning: the -b option is deprecated"); 75079837Sdes b_flag = 1; 75179837Sdes break; 75279837Sdes case 'c': 75379837Sdes c_dirname = optarg; 75479837Sdes break; 75579837Sdes case 'd': 75679837Sdes d_flag = 1; 75779837Sdes break; 75879837Sdes case 'F': 75979837Sdes F_flag = 1; 76079837Sdes break; 76179837Sdes case 'f': 76279837Sdes f_filename = optarg; 76379837Sdes break; 76479837Sdes case 'H': 76593213Scharnier warnx("the -H option is now implicit, " 76679837Sdes "use -U to disable"); 76779837Sdes break; 76879837Sdes case 'h': 76979837Sdes h_hostname = optarg; 77079837Sdes break; 77179837Sdes case 'l': 77279837Sdes l_flag = 1; 77379837Sdes break; 77479837Sdes case 'o': 77579837Sdes o_flag = 1; 77679837Sdes o_filename = optarg; 77779837Sdes break; 77879837Sdes case 'M': 77979837Sdes case 'm': 78079837Sdes if (r_flag) 78179837Sdes errx(1, "the -m and -r flags " 78279837Sdes "are mutually exclusive"); 78379837Sdes m_flag = 1; 78479837Sdes break; 785109702Sdes case 'N': 786109702Sdes N_filename = optarg; 787109702Sdes break; 78879837Sdes case 'n': 78979837Sdes n_flag = 1; 79079837Sdes break; 79179837Sdes case 'P': 79279837Sdes case 'p': 79379837Sdes p_flag = 1; 79479837Sdes break; 79579837Sdes case 'q': 79679837Sdes v_level = 0; 79779837Sdes break; 79879837Sdes case 'R': 79979837Sdes R_flag = 1; 80079837Sdes break; 80179837Sdes case 'r': 80279837Sdes if (m_flag) 80379837Sdes errx(1, "the -m and -r flags " 80479837Sdes "are mutually exclusive"); 80579837Sdes r_flag = 1; 80679837Sdes break; 80779837Sdes case 'S': 808100834Sdes S_size = (off_t)strtol(optarg, &end, 10); 809100834Sdes if (*optarg == '\0' || *end != '\0') 81080521Sse errx(1, "invalid size (%s)", optarg); 81179837Sdes break; 81279837Sdes case 's': 81379837Sdes s_flag = 1; 81479837Sdes break; 815106043Sdes case 'T': 816100834Sdes T_secs = strtol(optarg, &end, 10); 817100834Sdes if (*optarg == '\0' || *end != '\0') 81880521Sse errx(1, "invalid timeout (%s)", optarg); 81979837Sdes break; 82079837Sdes case 't': 82179837Sdes t_flag = 1; 82279837Sdes warnx("warning: the -t option is deprecated"); 82379837Sdes break; 82479837Sdes case 'U': 82579837Sdes U_flag = 1; 82679837Sdes break; 82779837Sdes case 'v': 82879837Sdes v_level++; 82979837Sdes break; 83079837Sdes case 'w': 83179837Sdes a_flag = 1; 832100834Sdes w_secs = strtol(optarg, &end, 10); 833100834Sdes if (*optarg == '\0' || *end != '\0') 83480521Sse errx(1, "invalid delay (%s)", optarg); 83579837Sdes break; 83679837Sdes default: 83779837Sdes usage(); 83879837Sdes exit(EX_USAGE); 83979837Sdes } 84062216Sdes 84179837Sdes argc -= optind; 84279837Sdes argv += optind; 84362216Sdes 84479837Sdes if (h_hostname || f_filename || c_dirname) { 84579837Sdes if (!h_hostname || !f_filename || argc) { 84679837Sdes usage(); 84779837Sdes exit(EX_USAGE); 84879837Sdes } 84979837Sdes /* XXX this is a hack. */ 85079837Sdes if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 85179837Sdes errx(1, "invalid hostname"); 85279837Sdes if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 85379837Sdes c_dirname ? c_dirname : "", f_filename) == -1) 85479837Sdes errx(1, "%s", strerror(ENOMEM)); 85579837Sdes argc++; 85662216Sdes } 85763345Sdes 85879837Sdes if (!argc) { 85979837Sdes usage(); 86079837Sdes exit(EX_USAGE); 86179837Sdes } 86262216Sdes 86379837Sdes /* allocate buffer */ 86479837Sdes if (B_size < MINBUFSIZE) 86579837Sdes B_size = MINBUFSIZE; 86679837Sdes if ((buf = malloc(B_size)) == NULL) 86779837Sdes errx(1, "%s", strerror(ENOMEM)); 86862216Sdes 86979837Sdes /* timeouts */ 87079837Sdes if ((s = getenv("FTP_TIMEOUT")) != NULL) { 871100834Sdes ftp_timeout = strtol(s, &end, 10); 872102478Sdes if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { 873102478Sdes warnx("FTP_TIMEOUT (%s) is not a positive integer", s); 87479837Sdes ftp_timeout = 0; 87579837Sdes } 87662216Sdes } 87779837Sdes if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 878100834Sdes http_timeout = strtol(s, &end, 10); 879102478Sdes if (*s == '\0' || *end != '\0' || http_timeout < 0) { 880102478Sdes warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); 88179837Sdes http_timeout = 0; 88279837Sdes } 88362216Sdes } 88462216Sdes 88579837Sdes /* signal handling */ 88679837Sdes sa.sa_flags = 0; 88779837Sdes sa.sa_handler = sig_handler; 88879837Sdes sigemptyset(&sa.sa_mask); 88979837Sdes sigaction(SIGALRM, &sa, NULL); 89079837Sdes sa.sa_flags = SA_RESETHAND; 89179837Sdes sigaction(SIGINT, &sa, NULL); 89279837Sdes fetchRestartCalls = 0; 89379837Sdes 89479837Sdes /* output file */ 89579837Sdes if (o_flag) { 89679837Sdes if (strcmp(o_filename, "-") == 0) { 89779837Sdes o_stdout = 1; 89879837Sdes } else if (stat(o_filename, &sb) == -1) { 89979837Sdes if (errno == ENOENT) { 90079837Sdes if (argc > 1) 90179837Sdes errx(EX_USAGE, "%s is not a directory", 90279837Sdes o_filename); 90379837Sdes } else { 90479837Sdes err(EX_IOERR, "%s", o_filename); 90579837Sdes } 90679837Sdes } else { 90779837Sdes if (sb.st_mode & S_IFDIR) 90879837Sdes o_directory = 1; 90979837Sdes } 91062216Sdes } 91162216Sdes 91279837Sdes /* check if output is to a tty (for progress report) */ 91379837Sdes v_tty = isatty(STDERR_FILENO); 91483863Sdes if (v_tty) 91583863Sdes pgrp = getpgrp(); 916106043Sdes 91779837Sdes r = 0; 918106043Sdes 91979837Sdes /* authentication */ 92079837Sdes if (v_tty) 92179837Sdes fetchAuthMethod = query_auth; 922109702Sdes if (N_filename != NULL) 923109702Sdes setenv("NETRC", N_filename, 1); 92462216Sdes 92579837Sdes while (argc) { 92679837Sdes if ((p = strrchr(*argv, '/')) == NULL) 92779837Sdes p = *argv; 92879837Sdes else 92979837Sdes p++; 93062216Sdes 93179837Sdes if (!*p) 93279837Sdes p = "fetch.out"; 933106043Sdes 93479837Sdes fetchLastErrCode = 0; 935106043Sdes 93679837Sdes if (o_flag) { 93779837Sdes if (o_stdout) { 93879837Sdes e = fetch(*argv, "-"); 93979837Sdes } else if (o_directory) { 94079837Sdes asprintf(&q, "%s/%s", o_filename, p); 94179837Sdes e = fetch(*argv, q); 94279837Sdes free(q); 94379837Sdes } else { 94479837Sdes e = fetch(*argv, o_filename); 94579837Sdes } 94679837Sdes } else { 94779837Sdes e = fetch(*argv, p); 94879837Sdes } 94962216Sdes 95079837Sdes if (sigint) 95179837Sdes kill(getpid(), SIGINT); 952106043Sdes 95379837Sdes if (e == 0 && once_flag) 95479837Sdes exit(0); 955106043Sdes 95679837Sdes if (e) { 95779837Sdes r = 1; 95879837Sdes if ((fetchLastErrCode 95979837Sdes && fetchLastErrCode != FETCH_UNAVAIL 96079837Sdes && fetchLastErrCode != FETCH_MOVED 96179837Sdes && fetchLastErrCode != FETCH_URL 96279837Sdes && fetchLastErrCode != FETCH_RESOLV 96379837Sdes && fetchLastErrCode != FETCH_UNKNOWN)) { 96479837Sdes if (w_secs && v_level) 965100834Sdes fprintf(stderr, "Waiting %ld seconds " 96679837Sdes "before retrying\n", w_secs); 96779837Sdes if (w_secs) 96879837Sdes sleep(w_secs); 96979837Sdes if (a_flag) 97079837Sdes continue; 97179837Sdes } 97262216Sdes } 97379837Sdes 97479837Sdes argc--, argv++; 97562216Sdes } 97662216Sdes 97779837Sdes exit(r); 97862216Sdes} 979