fetch.c revision 164152
162216Sdes/*- 2135546Sdes * Copyright (c) 2000-2004 Dag-Erling Co�dan Sm�rgrav 362216Sdes * All rights reserved. 462216Sdes * 562216Sdes * Redistribution and use in source and binary forms, with or without 662216Sdes * modification, are permitted provided that the following conditions 762216Sdes * are met: 862216Sdes * 1. Redistributions of source code must retain the above copyright 962216Sdes * notice, this list of conditions and the following disclaimer 1062216Sdes * in this position and unchanged. 1162216Sdes * 2. Redistributions in binary form must reproduce the above copyright 1262216Sdes * notice, this list of conditions and the following disclaimer in the 1362216Sdes * documentation and/or other materials provided with the distribution. 1462216Sdes * 3. The name of the author may not be used to endorse or promote products 1562216Sdes * derived from this software without specific prior written permission 1662216Sdes * 1762216Sdes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1862216Sdes * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1962216Sdes * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2062216Sdes * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2162216Sdes * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2262216Sdes * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2362216Sdes * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2462216Sdes * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2562216Sdes * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2662216Sdes * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2762216Sdes */ 2862216Sdes 2993213Scharnier#include <sys/cdefs.h> 3093213Scharnier__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 164152 2006-11-10 22:05:41Z des $"); 3193213Scharnier 3262216Sdes#include <sys/param.h> 3391225Sbde#include <sys/socket.h> 3462216Sdes#include <sys/stat.h> 3593257Sbde#include <sys/time.h> 3662216Sdes 3762216Sdes#include <ctype.h> 3862216Sdes#include <err.h> 3962216Sdes#include <errno.h> 4063235Sdes#include <signal.h> 41125976Sdes#include <stdint.h> 4262216Sdes#include <stdio.h> 4362216Sdes#include <stdlib.h> 4462216Sdes#include <string.h> 4562216Sdes#include <sysexits.h> 4677241Sdes#include <termios.h> 4762216Sdes#include <unistd.h> 4862216Sdes 4962216Sdes#include <fetch.h> 5062216Sdes 5162216Sdes#define MINBUFSIZE 4096 5262216Sdes 5362216Sdes/* Option flags */ 5462216Sdesint A_flag; /* -A: do not follow 302 redirects */ 5562216Sdesint a_flag; /* -a: auto retry */ 5679837Sdesoff_t B_size; /* -B: buffer size */ 5762216Sdesint b_flag; /*! -b: workaround TCP bug */ 5862254Sdeschar *c_dirname; /* -c: remote directory */ 5962216Sdesint d_flag; /* -d: direct connection */ 6062216Sdesint F_flag; /* -F: restart without checking mtime */ 6162216Sdeschar *f_filename; /* -f: file to fetch */ 6262216Sdeschar *h_hostname; /* -h: host to fetch from */ 6362216Sdesint l_flag; /* -l: link rather than copy file: URLs */ 6462815Sdesint m_flag; /* -[Mm]: mirror mode */ 65109702Sdeschar *N_filename; /* -N: netrc file name */ 6662815Sdesint n_flag; /* -n: do not preserve modification time */ 6762216Sdesint o_flag; /* -o: specify output file */ 6862216Sdesint o_directory; /* output file is a directory */ 6962216Sdeschar *o_filename; /* name of output file */ 7062216Sdesint o_stdout; /* output file is stdout */ 7162216Sdesint once_flag; /* -1: stop at first successful file */ 7263501Sdesint p_flag; /* -[Pp]: use passive FTP */ 7362216Sdesint R_flag; /* -R: don't delete partially transferred files */ 7462216Sdesint r_flag; /* -r: restart previously interrupted transfer */ 7574717Sdesoff_t S_size; /* -S: require size to match */ 7674717Sdesint s_flag; /* -s: show size, don't fetch */ 77100834Sdeslong T_secs = 120; /* -T: transfer timeout in seconds */ 7862216Sdesint t_flag; /*! -t: workaround TCP bug */ 7974717Sdesint U_flag; /* -U: do not use high ports */ 8062216Sdesint v_level = 1; /* -v: verbosity level */ 8162216Sdesint v_tty; /* stdout is a tty */ 8283863Sdespid_t pgrp; /* our process group */ 83100834Sdeslong w_secs; /* -w: retry delay */ 8462216Sdesint family = PF_UNSPEC; /* -[46]: address family to use */ 8562216Sdes 8663015Sdesint sigalrm; /* SIGALRM received */ 8773937Sdesint siginfo; /* SIGINFO received */ 8863015Sdesint sigint; /* SIGINT received */ 8962216Sdes 90100834Sdeslong ftp_timeout; /* default timeout for FTP transfers */ 91100834Sdeslong http_timeout; /* default timeout for HTTP transfers */ 92125976Sdeschar *buf; /* transfer buffer */ 9362216Sdes 9462216Sdes 9581863Sdes/* 9681863Sdes * Signal handler 9781863Sdes */ 9879837Sdesstatic void 9962216Sdessig_handler(int sig) 10062216Sdes{ 10179837Sdes switch (sig) { 10279837Sdes case SIGALRM: 10379837Sdes sigalrm = 1; 10479837Sdes break; 10579837Sdes case SIGINFO: 10679837Sdes siginfo = 1; 10779837Sdes break; 10879837Sdes case SIGINT: 10979837Sdes sigint = 1; 11079837Sdes break; 11179837Sdes } 11262216Sdes} 11362216Sdes 11462216Sdesstruct xferstat { 115125965Sdes char name[64]; 11679837Sdes struct timeval start; 11779837Sdes struct timeval last; 11879837Sdes off_t size; 11979837Sdes off_t offset; 12079837Sdes off_t rcvd; 12162216Sdes}; 12262216Sdes 12381863Sdes/* 124109702Sdes * Compute and display ETA 125109702Sdes */ 126125965Sdesstatic const char * 127109702Sdesstat_eta(struct xferstat *xs) 128109702Sdes{ 129125965Sdes static char str[16]; 130125976Sdes long elapsed, eta; 131125976Sdes off_t received, expected; 132109702Sdes 133109702Sdes elapsed = xs->last.tv_sec - xs->start.tv_sec; 134112083Sdes received = xs->rcvd - xs->offset; 135112083Sdes expected = xs->size - xs->rcvd; 136112114Sdes eta = (long)((double)elapsed * expected / received); 137125965Sdes if (eta > 3600) 138125965Sdes snprintf(str, sizeof str, "%02ldh%02ldm", 139125965Sdes eta / 3600, (eta % 3600) / 60); 140125965Sdes else 141125965Sdes snprintf(str, sizeof str, "%02ldm%02lds", 142125965Sdes eta / 60, eta % 60); 143125965Sdes return (str); 144125965Sdes} 145125965Sdes 146125965Sdes/* 147125965Sdes * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'... 148125965Sdes */ 149125965Sdesstatic const char *prefixes = " kMGTP"; 150125965Sdesstatic const char * 151129440Slestat_bytes(off_t bytes) 152125965Sdes{ 153125965Sdes static char str[16]; 154125965Sdes const char *prefix = prefixes; 155125965Sdes 156125965Sdes while (bytes > 9999 && prefix[1] != '\0') { 157125965Sdes bytes /= 1024; 158125965Sdes prefix++; 159109702Sdes } 160129440Sle snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix); 161125965Sdes return (str); 162109702Sdes} 163109702Sdes 164109702Sdes/* 165109702Sdes * Compute and display transfer rate 166109702Sdes */ 167125965Sdesstatic const char * 168109702Sdesstat_bps(struct xferstat *xs) 169109702Sdes{ 170125965Sdes static char str[16]; 171109735Sdes double delta, bps; 172109702Sdes 173109735Sdes delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6)) 174109735Sdes - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 175109735Sdes if (delta == 0.0) { 176125965Sdes snprintf(str, sizeof str, "?? Bps"); 177125965Sdes } else { 178125965Sdes bps = (xs->rcvd - xs->offset) / delta; 179129440Sle snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps)); 180109735Sdes } 181125965Sdes return (str); 182109702Sdes} 183109702Sdes 184109702Sdes/* 18581863Sdes * Update the stats display 18681863Sdes */ 18779837Sdesstatic void 18863046Sdesstat_display(struct xferstat *xs, int force) 18962216Sdes{ 19079837Sdes struct timeval now; 19183863Sdes int ctty_pgrp; 19279837Sdes 19383863Sdes /* check if we're the foreground process */ 19483863Sdes if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || 19583863Sdes (pid_t)ctty_pgrp != pgrp) 19683863Sdes return; 197106043Sdes 19879837Sdes gettimeofday(&now, NULL); 19979837Sdes if (!force && now.tv_sec <= xs->last.tv_sec) 20079837Sdes return; 20179837Sdes xs->last = now; 20279837Sdes 203131615Sdes fprintf(stderr, "\r%-46.46s", xs->name); 204106041Sdes if (xs->size <= 0) { 205153894Sdes setproctitle("%s [%s]", xs->name, stat_bytes(xs->rcvd)); 206125965Sdes fprintf(stderr, " %s", stat_bytes(xs->rcvd)); 207106041Sdes } else { 208153894Sdes setproctitle("%s [%d%% of %s]", xs->name, 209153894Sdes (int)((100.0 * xs->rcvd) / xs->size), 210153894Sdes stat_bytes(xs->size)); 211125965Sdes fprintf(stderr, "%3d%% of %s", 212125965Sdes (int)((100.0 * xs->rcvd) / xs->size), 213125965Sdes stat_bytes(xs->size)); 214106041Sdes } 215125965Sdes fprintf(stderr, " %s", stat_bps(xs)); 216125965Sdes if (xs->size > 0 && xs->rcvd > 0 && 217125965Sdes xs->last.tv_sec >= xs->start.tv_sec + 10) 218125965Sdes fprintf(stderr, " %s", stat_eta(xs)); 21962216Sdes} 22062216Sdes 22181863Sdes/* 22281863Sdes * Initialize the transfer statistics 22381863Sdes */ 22479837Sdesstatic void 22579837Sdesstat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) 22663046Sdes{ 22779837Sdes snprintf(xs->name, sizeof xs->name, "%s", name); 22879837Sdes gettimeofday(&xs->start, NULL); 22979837Sdes xs->last.tv_sec = xs->last.tv_usec = 0; 23079837Sdes xs->size = size; 23179837Sdes xs->offset = offset; 23279837Sdes xs->rcvd = offset; 233125965Sdes if (v_tty && v_level > 0) 234125965Sdes stat_display(xs, 1); 235125965Sdes else if (v_level > 0) 236125965Sdes fprintf(stderr, "%-46s", xs->name); 23763046Sdes} 23863046Sdes 23981863Sdes/* 24081863Sdes * Update the transfer statistics 24181863Sdes */ 24279837Sdesstatic void 24379837Sdesstat_update(struct xferstat *xs, off_t rcvd) 24463046Sdes{ 24579837Sdes xs->rcvd = rcvd; 246125965Sdes if (v_tty && v_level > 0) 247125965Sdes stat_display(xs, 0); 24863046Sdes} 24963046Sdes 25081863Sdes/* 25181863Sdes * Finalize the transfer statistics 25281863Sdes */ 25379837Sdesstatic void 25462216Sdesstat_end(struct xferstat *xs) 25562216Sdes{ 256109735Sdes gettimeofday(&xs->last, NULL); 257125965Sdes if (v_tty && v_level > 0) { 258125965Sdes stat_display(xs, 1); 259125965Sdes putc('\n', stderr); 260125965Sdes } else if (v_level > 0) { 261125965Sdes fprintf(stderr, " %s %s\n", 262125965Sdes stat_bytes(xs->size), stat_bps(xs)); 263125965Sdes } 26462216Sdes} 26562216Sdes 26681863Sdes/* 26781863Sdes * Ask the user for authentication details 26881863Sdes */ 26979837Sdesstatic int 27077241Sdesquery_auth(struct url *URL) 27177241Sdes{ 27279837Sdes struct termios tios; 27379837Sdes tcflag_t saved_flags; 27479837Sdes int i, nopwd; 27577241Sdes 27679837Sdes fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", 27786242Siedowse URL->scheme, URL->host, URL->port); 27879837Sdes 27979837Sdes fprintf(stderr, "Login: "); 28079837Sdes if (fgets(URL->user, sizeof URL->user, stdin) == NULL) 281132695Sdes return (-1); 282132696Sdes for (i = strlen(URL->user); i >= 0; --i) 283132696Sdes if (URL->user[i] == '\r' || URL->user[i] == '\n') 28479837Sdes URL->user[i] = '\0'; 28579837Sdes 28679837Sdes fprintf(stderr, "Password: "); 28779837Sdes if (tcgetattr(STDIN_FILENO, &tios) == 0) { 28879837Sdes saved_flags = tios.c_lflag; 28979837Sdes tios.c_lflag &= ~ECHO; 29079837Sdes tios.c_lflag |= ECHONL|ICANON; 29179837Sdes tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); 29279837Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 29379837Sdes tios.c_lflag = saved_flags; 29479837Sdes tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); 29579837Sdes } else { 29679837Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 29779837Sdes } 29879837Sdes if (nopwd) 299132695Sdes return (-1); 300132696Sdes for (i = strlen(URL->pwd); i >= 0; --i) 301132696Sdes if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n') 302132696Sdes URL->pwd[i] = '\0'; 30379837Sdes 304132695Sdes return (0); 30577241Sdes} 30677241Sdes 30781863Sdes/* 30881863Sdes * Fetch a file 30981863Sdes */ 31079837Sdesstatic int 31179837Sdesfetch(char *URL, const char *path) 31262216Sdes{ 31379837Sdes struct url *url; 31479837Sdes struct url_stat us; 31583217Sdes struct stat sb, nsb; 31679837Sdes struct xferstat xs; 31779837Sdes FILE *f, *of; 31879837Sdes size_t size, wr; 31979837Sdes off_t count; 32079837Sdes char flags[8]; 32183217Sdes const char *slash; 32283217Sdes char *tmppath; 32379837Sdes int r; 324125976Sdes unsigned timeout; 325125976Sdes char *ptr; 32662216Sdes 32779837Sdes f = of = NULL; 32883217Sdes tmppath = NULL; 32962216Sdes 330109702Sdes timeout = 0; 331109702Sdes *flags = 0; 332109702Sdes count = 0; 333109702Sdes 334109702Sdes /* set verbosity level */ 335109702Sdes if (v_level > 1) 336109702Sdes strcat(flags, "v"); 337109702Sdes if (v_level > 2) 338109702Sdes fetchDebug = 1; 339109702Sdes 34079837Sdes /* parse URL */ 34179837Sdes if ((url = fetchParseURL(URL)) == NULL) { 34279837Sdes warnx("%s: parse error", URL); 34379837Sdes goto failure; 34479837Sdes } 34562216Sdes 34679837Sdes /* if no scheme was specified, take a guess */ 34779837Sdes if (!*url->scheme) { 34879837Sdes if (!*url->host) 34979837Sdes strcpy(url->scheme, SCHEME_FILE); 35079837Sdes else if (strncasecmp(url->host, "ftp.", 4) == 0) 35179837Sdes strcpy(url->scheme, SCHEME_FTP); 35279837Sdes else if (strncasecmp(url->host, "www.", 4) == 0) 35379837Sdes strcpy(url->scheme, SCHEME_HTTP); 35479837Sdes } 35569976Sdes 35679837Sdes /* common flags */ 35779837Sdes switch (family) { 35879837Sdes case PF_INET: 35979837Sdes strcat(flags, "4"); 36079837Sdes break; 36179837Sdes case PF_INET6: 36279837Sdes strcat(flags, "6"); 36379837Sdes break; 36479837Sdes } 36562216Sdes 36679837Sdes /* FTP specific flags */ 36779837Sdes if (strcmp(url->scheme, "ftp") == 0) { 36879837Sdes if (p_flag) 36979837Sdes strcat(flags, "p"); 37079837Sdes if (d_flag) 37179837Sdes strcat(flags, "d"); 37279837Sdes if (U_flag) 37379837Sdes strcat(flags, "l"); 37479837Sdes timeout = T_secs ? T_secs : ftp_timeout; 37579837Sdes } 37662216Sdes 37779837Sdes /* HTTP specific flags */ 37879837Sdes if (strcmp(url->scheme, "http") == 0) { 37979837Sdes if (d_flag) 38079837Sdes strcat(flags, "d"); 38179837Sdes if (A_flag) 38279837Sdes strcat(flags, "A"); 38379837Sdes timeout = T_secs ? T_secs : http_timeout; 38479837Sdes } 38562216Sdes 38679837Sdes /* set the protocol timeout. */ 38779837Sdes fetchTimeout = timeout; 38862216Sdes 38979837Sdes /* just print size */ 39079837Sdes if (s_flag) { 391106041Sdes if (timeout) 392106041Sdes alarm(timeout); 393106041Sdes r = fetchStat(url, &us, flags); 394106041Sdes if (timeout) 395106043Sdes alarm(0); 396106041Sdes if (sigalrm || sigint) 397106041Sdes goto signal; 398106041Sdes if (r == -1) { 399106041Sdes warnx("%s", fetchLastErrString); 40079837Sdes goto failure; 401106041Sdes } 40279837Sdes if (us.size == -1) 40379837Sdes printf("Unknown\n"); 40479837Sdes else 405125976Sdes printf("%jd\n", (intmax_t)us.size); 40679837Sdes goto success; 40763345Sdes } 40879837Sdes 40979837Sdes /* 41079837Sdes * If the -r flag was specified, we have to compare the local 41179837Sdes * and remote files, so we should really do a fetchStat() 41279837Sdes * first, but I know of at least one HTTP server that only 41379837Sdes * sends the content size in response to GET requests, and 41479837Sdes * leaves it out of replies to HEAD requests. Also, in the 41579837Sdes * (frequent) case that the local and remote files match but 41679837Sdes * the local file is truncated, we have sufficient information 41779837Sdes * before the compare to issue a correct request. Therefore, 41879837Sdes * we always issue a GET request as if we were sure the local 41979837Sdes * file was a truncated copy of the remote file; we can drop 42079837Sdes * the connection later if we change our minds. 42179837Sdes */ 42283217Sdes sb.st_size = -1; 423133779Sdes if (!o_stdout) { 424133779Sdes r = stat(path, &sb); 425134350Sdes if (r == 0 && r_flag && S_ISREG(sb.st_mode)) { 426133779Sdes url->offset = sb.st_size; 427153919Sdes } else if (r == -1 || !S_ISREG(sb.st_mode)) { 428133779Sdes /* 429133779Sdes * Whatever value sb.st_size has now is either 430133779Sdes * wrong (if stat(2) failed) or irrelevant (if the 431133779Sdes * path does not refer to a regular file) 432133779Sdes */ 433133779Sdes sb.st_size = -1; 434133779Sdes } 435153919Sdes if (r == -1 && errno != ENOENT) { 436153919Sdes warnx("%s: stat()", path); 437153919Sdes goto failure; 438153919Sdes } 43962216Sdes } 44062216Sdes 44179837Sdes /* start the transfer */ 442106041Sdes if (timeout) 443106041Sdes alarm(timeout); 444106041Sdes f = fetchXGet(url, &us, flags); 445106042Sdes if (timeout) 446106042Sdes alarm(0); 447106041Sdes if (sigalrm || sigint) 448106041Sdes goto signal; 449106041Sdes if (f == NULL) { 450107353Sdes warnx("%s: %s", URL, fetchLastErrString); 45163345Sdes goto failure; 45279837Sdes } 45379837Sdes if (sigint) 45463345Sdes goto signal; 45579837Sdes 45679837Sdes /* check that size is as expected */ 45779837Sdes if (S_size) { 45879837Sdes if (us.size == -1) { 459107353Sdes warnx("%s: size unknown", URL); 46079837Sdes } else if (us.size != S_size) { 461125976Sdes warnx("%s: size mismatch: expected %jd, actual %jd", 462125976Sdes URL, (intmax_t)S_size, (intmax_t)us.size); 46379837Sdes goto failure; 46479837Sdes } 46579837Sdes } 46679837Sdes 46779837Sdes /* symlink instead of copy */ 46879837Sdes if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 46979837Sdes if (symlink(url->doc, path) == -1) { 47079837Sdes warn("%s: symlink()", path); 47179837Sdes goto failure; 47279837Sdes } 47363568Sdes goto success; 47463345Sdes } 47579837Sdes 476106051Sdes if (us.size == -1 && !o_stdout && v_level > 0) 477107353Sdes warnx("%s: size of remote file is not known", URL); 47879837Sdes if (v_level > 1) { 47979837Sdes if (sb.st_size != -1) 480125976Sdes fprintf(stderr, "local size / mtime: %jd / %ld\n", 481125976Sdes (intmax_t)sb.st_size, (long)sb.st_mtime); 48279837Sdes if (us.size != -1) 483125976Sdes fprintf(stderr, "remote size / mtime: %jd / %ld\n", 484125976Sdes (intmax_t)us.size, (long)us.mtime); 48562216Sdes } 48662216Sdes 48779837Sdes /* open output file */ 48879837Sdes if (o_stdout) { 48979837Sdes /* output to stdout */ 49079837Sdes of = stdout; 49183217Sdes } else if (r_flag && sb.st_size != -1) { 49279837Sdes /* resume mode, local file exists */ 49379837Sdes if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { 49479837Sdes /* no match! have to refetch */ 49579837Sdes fclose(f); 49679837Sdes /* if precious, warn the user and give up */ 49779837Sdes if (R_flag) { 49879837Sdes warnx("%s: local modification time " 49979837Sdes "does not match remote", path); 50079837Sdes goto failure_keep; 50179837Sdes } 502127941Sdes } else if (us.size != -1) { 50379837Sdes if (us.size == sb.st_size) 50479837Sdes /* nothing to do */ 50579837Sdes goto success; 50679837Sdes if (sb.st_size > us.size) { 50779837Sdes /* local file too long! */ 508125976Sdes warnx("%s: local file (%jd bytes) is longer " 509125976Sdes "than remote file (%jd bytes)", path, 510125976Sdes (intmax_t)sb.st_size, (intmax_t)us.size); 51179837Sdes goto failure; 51279837Sdes } 51383217Sdes /* we got it, open local file */ 51479837Sdes if ((of = fopen(path, "a")) == NULL) { 51579837Sdes warn("%s: fopen()", path); 51679837Sdes goto failure; 51779837Sdes } 51883217Sdes /* check that it didn't move under our feet */ 51983217Sdes if (fstat(fileno(of), &nsb) == -1) { 52083217Sdes /* can't happen! */ 52183217Sdes warn("%s: fstat()", path); 52279837Sdes goto failure; 52379837Sdes } 52483217Sdes if (nsb.st_dev != sb.st_dev || 52583217Sdes nsb.st_ino != nsb.st_ino || 52683217Sdes nsb.st_size != sb.st_size) { 527107353Sdes warnx("%s: file has changed", URL); 52883217Sdes fclose(of); 52983217Sdes of = NULL; 53083217Sdes sb = nsb; 53183217Sdes } 53279837Sdes } 53383217Sdes } else if (m_flag && sb.st_size != -1) { 53479837Sdes /* mirror mode, local file exists */ 53579837Sdes if (sb.st_size == us.size && sb.st_mtime == us.mtime) 53679837Sdes goto success; 53779837Sdes } 538106043Sdes 53983217Sdes if (of == NULL) { 54079837Sdes /* 54179837Sdes * We don't yet have an output file; either this is a 54279837Sdes * vanilla run with no special flags, or the local and 54379837Sdes * remote files didn't match. 54479837Sdes */ 545106043Sdes 546109702Sdes if (url->offset > 0) { 54783217Sdes /* 54883217Sdes * We tried to restart a transfer, but for 54983217Sdes * some reason gave up - so we have to restart 55083217Sdes * from scratch if we want the whole file 55183217Sdes */ 55283217Sdes url->offset = 0; 55383217Sdes if ((f = fetchXGet(url, &us, flags)) == NULL) { 554107353Sdes warnx("%s: %s", URL, fetchLastErrString); 55583217Sdes goto failure; 55683217Sdes } 55783217Sdes if (sigint) 55883217Sdes goto signal; 55983217Sdes } 56083217Sdes 56183217Sdes /* construct a temp file name */ 56283217Sdes if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { 56383217Sdes if ((slash = strrchr(path, '/')) == NULL) 56483217Sdes slash = path; 56583217Sdes else 56683217Sdes ++slash; 56783217Sdes asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", 56883307Smike (int)(slash - path), path, slash); 569100834Sdes if (tmppath != NULL) { 570100834Sdes mkstemps(tmppath, strlen(slash) + 1); 571100834Sdes of = fopen(tmppath, "w"); 572164152Sdes chown(tmppath, sb.st_uid, sb.st_gid); 573164152Sdes chmod(tmppath, sb.st_mode & ALLPERMS); 574100834Sdes } 57583217Sdes } 576100834Sdes if (of == NULL) 57783217Sdes of = fopen(path, "w"); 57883217Sdes if (of == NULL) { 57979837Sdes warn("%s: open()", path); 58079837Sdes goto failure; 58179837Sdes } 58279837Sdes } 58379837Sdes count = url->offset; 58462216Sdes 58579837Sdes /* start the counter */ 58679837Sdes stat_start(&xs, path, us.size, count); 58763046Sdes 58879837Sdes sigalrm = siginfo = sigint = 0; 58979837Sdes 59079837Sdes /* suck in the data */ 59179837Sdes signal(SIGINFO, sig_handler); 592106041Sdes while (!sigint) { 593137854Scperciva if (us.size != -1 && us.size - count < B_size && 594137854Scperciva us.size - count >= 0) 59579837Sdes size = us.size - count; 59679837Sdes else 59779837Sdes size = B_size; 59879837Sdes if (siginfo) { 59979837Sdes stat_end(&xs); 60079837Sdes siginfo = 0; 60179837Sdes } 60279837Sdes if ((size = fread(buf, 1, size, f)) == 0) { 603106041Sdes if (ferror(f) && errno == EINTR && !sigint) 60479837Sdes clearerr(f); 60579837Sdes else 60679837Sdes break; 60779837Sdes } 60879837Sdes stat_update(&xs, count += size); 60979837Sdes for (ptr = buf; size > 0; ptr += wr, size -= wr) 61079837Sdes if ((wr = fwrite(ptr, 1, size, of)) < size) { 611106041Sdes if (ferror(of) && errno == EINTR && !sigint) 61279837Sdes clearerr(of); 61379837Sdes else 61479837Sdes break; 61579837Sdes } 61679837Sdes if (size != 0) 61779837Sdes break; 61877241Sdes } 619106041Sdes if (!sigalrm) 620106041Sdes sigalrm = ferror(f) && errno == ETIMEDOUT; 62179837Sdes signal(SIGINFO, SIG_DFL); 62279837Sdes 62379837Sdes stat_end(&xs); 62462216Sdes 625106041Sdes /* 626106041Sdes * If the transfer timed out or was interrupted, we still want to 627106041Sdes * set the mtime in case the file is not removed (-r or -R) and 628106041Sdes * the user later restarts the transfer. 629106041Sdes */ 630106041Sdes signal: 63179837Sdes /* set mtime of local file */ 632106857Sdes if (!n_flag && us.mtime && !o_stdout && of != NULL && 633106857Sdes (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { 63479837Sdes struct timeval tv[2]; 635106043Sdes 63679837Sdes fflush(of); 63779837Sdes tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 63879837Sdes tv[1].tv_sec = (long)us.mtime; 63979837Sdes tv[0].tv_usec = tv[1].tv_usec = 0; 64090729Sdes if (utimes(tmppath ? tmppath : path, tv)) 64190729Sdes warn("%s: utimes()", tmppath ? tmppath : path); 64279837Sdes } 64363015Sdes 64479837Sdes /* timed out or interrupted? */ 64579837Sdes if (sigalrm) 64679837Sdes warnx("transfer timed out"); 64779837Sdes if (sigint) { 64879837Sdes warnx("transfer interrupted"); 64979837Sdes goto failure; 65079837Sdes } 65163046Sdes 652106586Sfenner /* timeout / interrupt before connection completley established? */ 653106586Sfenner if (f == NULL) 654106586Sfenner goto failure; 655106586Sfenner 65679837Sdes if (!sigalrm) { 65779837Sdes /* check the status of our files */ 65879837Sdes if (ferror(f)) 65979837Sdes warn("%s", URL); 66079837Sdes if (ferror(of)) 66179837Sdes warn("%s", path); 66279837Sdes if (ferror(f) || ferror(of)) 66379837Sdes goto failure; 66479837Sdes } 66579837Sdes 66679837Sdes /* did the transfer complete normally? */ 66779837Sdes if (us.size != -1 && count < us.size) { 668125976Sdes warnx("%s appears to be truncated: %jd/%jd bytes", 669125976Sdes path, (intmax_t)count, (intmax_t)us.size); 67079837Sdes goto failure_keep; 67179837Sdes } 67279837Sdes 67379837Sdes /* 67479837Sdes * If the transfer timed out and we didn't know how much to 67579837Sdes * expect, assume the worst (i.e. we didn't get all of it) 67679837Sdes */ 67779837Sdes if (sigalrm && us.size == -1) { 67879837Sdes warnx("%s may be truncated", path); 67979837Sdes goto failure_keep; 68079837Sdes } 68179837Sdes 68262216Sdes success: 68379837Sdes r = 0; 68483217Sdes if (tmppath != NULL && rename(tmppath, path) == -1) { 68583217Sdes warn("%s: rename()", path); 68683217Sdes goto failure_keep; 68783217Sdes } 68879837Sdes goto done; 68962216Sdes failure: 69079837Sdes if (of && of != stdout && !R_flag && !r_flag) 69179837Sdes if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) 69283217Sdes unlink(tmppath ? tmppath : path); 69383217Sdes if (R_flag && tmppath != NULL && sb.st_size == -1) 69483217Sdes rename(tmppath, path); /* ignore errors here */ 69563046Sdes failure_keep: 69679837Sdes r = -1; 69779837Sdes goto done; 69862216Sdes done: 69979837Sdes if (f) 70079837Sdes fclose(f); 70179837Sdes if (of && of != stdout) 70279837Sdes fclose(of); 70379837Sdes if (url) 70479837Sdes fetchFreeURL(url); 70583217Sdes if (tmppath != NULL) 70683217Sdes free(tmppath); 707132695Sdes return (r); 70862216Sdes} 70962216Sdes 71079837Sdesstatic void 71162216Sdesusage(void) 71262216Sdes{ 71379837Sdes fprintf(stderr, "%s\n%s\n%s\n", 714109702Sdes "usage: fetch [-146AFMPRUadlmnpqrsv] [-N netrc] [-o outputfile]", 715109702Sdes " [-S bytes] [-B bytes] [-T seconds] [-w seconds]", 71679837Sdes " [-h host -f file [-c dir] | URL ...]"); 71762216Sdes} 71862216Sdes 71962216Sdes 72081863Sdes/* 72181863Sdes * Entry point 72281863Sdes */ 72362216Sdesint 72462216Sdesmain(int argc, char *argv[]) 72562216Sdes{ 72679837Sdes struct stat sb; 72779837Sdes struct sigaction sa; 72879837Sdes const char *p, *s; 729100834Sdes char *end, *q; 73079837Sdes int c, e, r; 73162216Sdes 73279837Sdes while ((c = getopt(argc, argv, 733109702Sdes "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) 73479837Sdes switch (c) { 73579837Sdes case '1': 73679837Sdes once_flag = 1; 73779837Sdes break; 73879837Sdes case '4': 73979837Sdes family = PF_INET; 74079837Sdes break; 74179837Sdes case '6': 74279837Sdes family = PF_INET6; 74379837Sdes break; 74479837Sdes case 'A': 74579837Sdes A_flag = 1; 74679837Sdes break; 74779837Sdes case 'a': 74879837Sdes a_flag = 1; 74979837Sdes break; 75079837Sdes case 'B': 751100834Sdes B_size = (off_t)strtol(optarg, &end, 10); 752100834Sdes if (*optarg == '\0' || *end != '\0') 75380521Sse errx(1, "invalid buffer size (%s)", optarg); 75479837Sdes break; 75579837Sdes case 'b': 75679837Sdes warnx("warning: the -b option is deprecated"); 75779837Sdes b_flag = 1; 75879837Sdes break; 75979837Sdes case 'c': 76079837Sdes c_dirname = optarg; 76179837Sdes break; 76279837Sdes case 'd': 76379837Sdes d_flag = 1; 76479837Sdes break; 76579837Sdes case 'F': 76679837Sdes F_flag = 1; 76779837Sdes break; 76879837Sdes case 'f': 76979837Sdes f_filename = optarg; 77079837Sdes break; 77179837Sdes case 'H': 77293213Scharnier warnx("the -H option is now implicit, " 77379837Sdes "use -U to disable"); 77479837Sdes break; 77579837Sdes case 'h': 77679837Sdes h_hostname = optarg; 77779837Sdes break; 77879837Sdes case 'l': 77979837Sdes l_flag = 1; 78079837Sdes break; 78179837Sdes case 'o': 78279837Sdes o_flag = 1; 78379837Sdes o_filename = optarg; 78479837Sdes break; 78579837Sdes case 'M': 78679837Sdes case 'm': 78779837Sdes if (r_flag) 78879837Sdes errx(1, "the -m and -r flags " 78979837Sdes "are mutually exclusive"); 79079837Sdes m_flag = 1; 79179837Sdes break; 792109702Sdes case 'N': 793109702Sdes N_filename = optarg; 794109702Sdes break; 79579837Sdes case 'n': 79679837Sdes n_flag = 1; 79779837Sdes break; 79879837Sdes case 'P': 79979837Sdes case 'p': 80079837Sdes p_flag = 1; 80179837Sdes break; 80279837Sdes case 'q': 80379837Sdes v_level = 0; 80479837Sdes break; 80579837Sdes case 'R': 80679837Sdes R_flag = 1; 80779837Sdes break; 80879837Sdes case 'r': 80979837Sdes if (m_flag) 81079837Sdes errx(1, "the -m and -r flags " 81179837Sdes "are mutually exclusive"); 81279837Sdes r_flag = 1; 81379837Sdes break; 81479837Sdes case 'S': 815100834Sdes S_size = (off_t)strtol(optarg, &end, 10); 816100834Sdes if (*optarg == '\0' || *end != '\0') 81780521Sse errx(1, "invalid size (%s)", optarg); 81879837Sdes break; 81979837Sdes case 's': 82079837Sdes s_flag = 1; 82179837Sdes break; 822106043Sdes case 'T': 823100834Sdes T_secs = strtol(optarg, &end, 10); 824100834Sdes if (*optarg == '\0' || *end != '\0') 82580521Sse errx(1, "invalid timeout (%s)", optarg); 82679837Sdes break; 82779837Sdes case 't': 82879837Sdes t_flag = 1; 82979837Sdes warnx("warning: the -t option is deprecated"); 83079837Sdes break; 83179837Sdes case 'U': 83279837Sdes U_flag = 1; 83379837Sdes break; 83479837Sdes case 'v': 83579837Sdes v_level++; 83679837Sdes break; 83779837Sdes case 'w': 83879837Sdes a_flag = 1; 839100834Sdes w_secs = strtol(optarg, &end, 10); 840100834Sdes if (*optarg == '\0' || *end != '\0') 84180521Sse errx(1, "invalid delay (%s)", optarg); 84279837Sdes break; 84379837Sdes default: 84479837Sdes usage(); 84579837Sdes exit(EX_USAGE); 84679837Sdes } 84762216Sdes 84879837Sdes argc -= optind; 84979837Sdes argv += optind; 85062216Sdes 85179837Sdes if (h_hostname || f_filename || c_dirname) { 85279837Sdes if (!h_hostname || !f_filename || argc) { 85379837Sdes usage(); 85479837Sdes exit(EX_USAGE); 85579837Sdes } 85679837Sdes /* XXX this is a hack. */ 85779837Sdes if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 85879837Sdes errx(1, "invalid hostname"); 85979837Sdes if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 86079837Sdes c_dirname ? c_dirname : "", f_filename) == -1) 86179837Sdes errx(1, "%s", strerror(ENOMEM)); 86279837Sdes argc++; 86362216Sdes } 86463345Sdes 86579837Sdes if (!argc) { 86679837Sdes usage(); 86779837Sdes exit(EX_USAGE); 86879837Sdes } 86962216Sdes 87079837Sdes /* allocate buffer */ 87179837Sdes if (B_size < MINBUFSIZE) 87279837Sdes B_size = MINBUFSIZE; 87379837Sdes if ((buf = malloc(B_size)) == NULL) 87479837Sdes errx(1, "%s", strerror(ENOMEM)); 87562216Sdes 87679837Sdes /* timeouts */ 87779837Sdes if ((s = getenv("FTP_TIMEOUT")) != NULL) { 878100834Sdes ftp_timeout = strtol(s, &end, 10); 879102478Sdes if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { 880102478Sdes warnx("FTP_TIMEOUT (%s) is not a positive integer", s); 88179837Sdes ftp_timeout = 0; 88279837Sdes } 88362216Sdes } 88479837Sdes if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 885100834Sdes http_timeout = strtol(s, &end, 10); 886102478Sdes if (*s == '\0' || *end != '\0' || http_timeout < 0) { 887102478Sdes warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); 88879837Sdes http_timeout = 0; 88979837Sdes } 89062216Sdes } 89162216Sdes 89279837Sdes /* signal handling */ 89379837Sdes sa.sa_flags = 0; 89479837Sdes sa.sa_handler = sig_handler; 89579837Sdes sigemptyset(&sa.sa_mask); 89679837Sdes sigaction(SIGALRM, &sa, NULL); 89779837Sdes sa.sa_flags = SA_RESETHAND; 89879837Sdes sigaction(SIGINT, &sa, NULL); 89979837Sdes fetchRestartCalls = 0; 90079837Sdes 90179837Sdes /* output file */ 90279837Sdes if (o_flag) { 90379837Sdes if (strcmp(o_filename, "-") == 0) { 90479837Sdes o_stdout = 1; 90579837Sdes } else if (stat(o_filename, &sb) == -1) { 90679837Sdes if (errno == ENOENT) { 90779837Sdes if (argc > 1) 90879837Sdes errx(EX_USAGE, "%s is not a directory", 90979837Sdes o_filename); 91079837Sdes } else { 91179837Sdes err(EX_IOERR, "%s", o_filename); 91279837Sdes } 91379837Sdes } else { 91479837Sdes if (sb.st_mode & S_IFDIR) 91579837Sdes o_directory = 1; 91679837Sdes } 91762216Sdes } 91862216Sdes 91979837Sdes /* check if output is to a tty (for progress report) */ 92079837Sdes v_tty = isatty(STDERR_FILENO); 92183863Sdes if (v_tty) 92283863Sdes pgrp = getpgrp(); 923106043Sdes 92479837Sdes r = 0; 925106043Sdes 92679837Sdes /* authentication */ 92779837Sdes if (v_tty) 92879837Sdes fetchAuthMethod = query_auth; 929109702Sdes if (N_filename != NULL) 930109702Sdes setenv("NETRC", N_filename, 1); 93162216Sdes 93279837Sdes while (argc) { 93379837Sdes if ((p = strrchr(*argv, '/')) == NULL) 93479837Sdes p = *argv; 93579837Sdes else 93679837Sdes p++; 93762216Sdes 93879837Sdes if (!*p) 93979837Sdes p = "fetch.out"; 940106043Sdes 94179837Sdes fetchLastErrCode = 0; 942106043Sdes 94379837Sdes if (o_flag) { 94479837Sdes if (o_stdout) { 94579837Sdes e = fetch(*argv, "-"); 94679837Sdes } else if (o_directory) { 94779837Sdes asprintf(&q, "%s/%s", o_filename, p); 94879837Sdes e = fetch(*argv, q); 94979837Sdes free(q); 95079837Sdes } else { 95179837Sdes e = fetch(*argv, o_filename); 95279837Sdes } 95379837Sdes } else { 95479837Sdes e = fetch(*argv, p); 95579837Sdes } 95662216Sdes 95779837Sdes if (sigint) 95879837Sdes kill(getpid(), SIGINT); 959106043Sdes 96079837Sdes if (e == 0 && once_flag) 96179837Sdes exit(0); 962106043Sdes 96379837Sdes if (e) { 96479837Sdes r = 1; 96579837Sdes if ((fetchLastErrCode 96679837Sdes && fetchLastErrCode != FETCH_UNAVAIL 96779837Sdes && fetchLastErrCode != FETCH_MOVED 96879837Sdes && fetchLastErrCode != FETCH_URL 96979837Sdes && fetchLastErrCode != FETCH_RESOLV 97079837Sdes && fetchLastErrCode != FETCH_UNKNOWN)) { 97179837Sdes if (w_secs && v_level) 972100834Sdes fprintf(stderr, "Waiting %ld seconds " 97379837Sdes "before retrying\n", w_secs); 97479837Sdes if (w_secs) 97579837Sdes sleep(w_secs); 97679837Sdes if (a_flag) 97779837Sdes continue; 97879837Sdes } 97962216Sdes } 98079837Sdes 98179837Sdes argc--, argv++; 98262216Sdes } 98362216Sdes 98479837Sdes exit(r); 98562216Sdes} 986