fetch.c revision 186043
162216Sdes/*- 2185912Sdes * Copyright (c) 2000-2004 Dag-Erling Co��dan Sm��rgrav 362216Sdes * All rights reserved. 462216Sdes * 562216Sdes * Redistribution and use in source and binary forms, with or without 662216Sdes * modification, are permitted provided that the following conditions 762216Sdes * are met: 862216Sdes * 1. Redistributions of source code must retain the above copyright 962216Sdes * notice, this list of conditions and the following disclaimer 1062216Sdes * in this position and unchanged. 1162216Sdes * 2. Redistributions in binary form must reproduce the above copyright 1262216Sdes * notice, this list of conditions and the following disclaimer in the 1362216Sdes * documentation and/or other materials provided with the distribution. 1462216Sdes * 3. The name of the author may not be used to endorse or promote products 1562216Sdes * derived from this software without specific prior written permission 1662216Sdes * 1762216Sdes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1862216Sdes * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1962216Sdes * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2062216Sdes * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2162216Sdes * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2262216Sdes * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2362216Sdes * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2462216Sdes * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2562216Sdes * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2662216Sdes * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2762216Sdes */ 2862216Sdes 2993213Scharnier#include <sys/cdefs.h> 3093213Scharnier__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 186043 2008-12-13 17:48:06Z ru $"); 3193213Scharnier 3262216Sdes#include <sys/param.h> 3391225Sbde#include <sys/socket.h> 3462216Sdes#include <sys/stat.h> 3593257Sbde#include <sys/time.h> 3662216Sdes 3762216Sdes#include <ctype.h> 3862216Sdes#include <err.h> 3962216Sdes#include <errno.h> 4063235Sdes#include <signal.h> 41125976Sdes#include <stdint.h> 4262216Sdes#include <stdio.h> 4362216Sdes#include <stdlib.h> 4462216Sdes#include <string.h> 4562216Sdes#include <sysexits.h> 4677241Sdes#include <termios.h> 4762216Sdes#include <unistd.h> 4862216Sdes 4962216Sdes#include <fetch.h> 5062216Sdes 5162216Sdes#define MINBUFSIZE 4096 5262216Sdes 5362216Sdes/* Option flags */ 5462216Sdesint A_flag; /* -A: do not follow 302 redirects */ 5562216Sdesint a_flag; /* -a: auto retry */ 5679837Sdesoff_t B_size; /* -B: buffer size */ 5762216Sdesint b_flag; /*! -b: workaround TCP bug */ 5862254Sdeschar *c_dirname; /* -c: remote directory */ 5962216Sdesint d_flag; /* -d: direct connection */ 6062216Sdesint F_flag; /* -F: restart without checking mtime */ 6162216Sdeschar *f_filename; /* -f: file to fetch */ 6262216Sdeschar *h_hostname; /* -h: host to fetch from */ 6362216Sdesint l_flag; /* -l: link rather than copy file: URLs */ 6462815Sdesint m_flag; /* -[Mm]: mirror mode */ 65109702Sdeschar *N_filename; /* -N: netrc file name */ 6662815Sdesint n_flag; /* -n: do not preserve modification time */ 6762216Sdesint o_flag; /* -o: specify output file */ 6862216Sdesint o_directory; /* output file is a directory */ 6962216Sdeschar *o_filename; /* name of output file */ 7062216Sdesint o_stdout; /* output file is stdout */ 7162216Sdesint once_flag; /* -1: stop at first successful file */ 7263501Sdesint p_flag; /* -[Pp]: use passive FTP */ 7362216Sdesint R_flag; /* -R: don't delete partially transferred files */ 7462216Sdesint r_flag; /* -r: restart previously interrupted transfer */ 7574717Sdesoff_t S_size; /* -S: require size to match */ 7674717Sdesint s_flag; /* -s: show size, don't fetch */ 77100834Sdeslong T_secs = 120; /* -T: transfer timeout in seconds */ 7862216Sdesint t_flag; /*! -t: workaround TCP bug */ 7974717Sdesint U_flag; /* -U: do not use high ports */ 8062216Sdesint v_level = 1; /* -v: verbosity level */ 8162216Sdesint v_tty; /* stdout is a tty */ 8283863Sdespid_t pgrp; /* our process group */ 83100834Sdeslong w_secs; /* -w: retry delay */ 8462216Sdesint family = PF_UNSPEC; /* -[46]: address family to use */ 8562216Sdes 8663015Sdesint sigalrm; /* SIGALRM received */ 8773937Sdesint siginfo; /* SIGINFO received */ 8863015Sdesint sigint; /* SIGINT received */ 8962216Sdes 90100834Sdeslong ftp_timeout; /* default timeout for FTP transfers */ 91100834Sdeslong http_timeout; /* default timeout for HTTP transfers */ 92125976Sdeschar *buf; /* transfer buffer */ 9362216Sdes 9462216Sdes 9581863Sdes/* 9681863Sdes * Signal handler 9781863Sdes */ 9879837Sdesstatic void 9962216Sdessig_handler(int sig) 10062216Sdes{ 10179837Sdes switch (sig) { 10279837Sdes case SIGALRM: 10379837Sdes sigalrm = 1; 10479837Sdes break; 10579837Sdes case SIGINFO: 10679837Sdes siginfo = 1; 10779837Sdes break; 10879837Sdes case SIGINT: 10979837Sdes sigint = 1; 11079837Sdes break; 11179837Sdes } 11262216Sdes} 11362216Sdes 11462216Sdesstruct xferstat { 115125965Sdes char name[64]; 11679837Sdes struct timeval start; 11779837Sdes struct timeval last; 11879837Sdes off_t size; 11979837Sdes off_t offset; 12079837Sdes off_t rcvd; 12162216Sdes}; 12262216Sdes 12381863Sdes/* 124109702Sdes * Compute and display ETA 125109702Sdes */ 126125965Sdesstatic const char * 127109702Sdesstat_eta(struct xferstat *xs) 128109702Sdes{ 129125965Sdes static char str[16]; 130125976Sdes long elapsed, eta; 131125976Sdes off_t received, expected; 132109702Sdes 133109702Sdes elapsed = xs->last.tv_sec - xs->start.tv_sec; 134112083Sdes received = xs->rcvd - xs->offset; 135112083Sdes expected = xs->size - xs->rcvd; 136112114Sdes eta = (long)((double)elapsed * expected / received); 137125965Sdes if (eta > 3600) 138125965Sdes snprintf(str, sizeof str, "%02ldh%02ldm", 139125965Sdes eta / 3600, (eta % 3600) / 60); 140125965Sdes else 141125965Sdes snprintf(str, sizeof str, "%02ldm%02lds", 142125965Sdes eta / 60, eta % 60); 143125965Sdes return (str); 144125965Sdes} 145125965Sdes 146125965Sdes/* 147125965Sdes * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'... 148125965Sdes */ 149125965Sdesstatic const char *prefixes = " kMGTP"; 150125965Sdesstatic const char * 151129440Slestat_bytes(off_t bytes) 152125965Sdes{ 153125965Sdes static char str[16]; 154125965Sdes const char *prefix = prefixes; 155125965Sdes 156125965Sdes while (bytes > 9999 && prefix[1] != '\0') { 157125965Sdes bytes /= 1024; 158125965Sdes prefix++; 159109702Sdes } 160129440Sle snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix); 161125965Sdes return (str); 162109702Sdes} 163109702Sdes 164109702Sdes/* 165109702Sdes * Compute and display transfer rate 166109702Sdes */ 167125965Sdesstatic const char * 168109702Sdesstat_bps(struct xferstat *xs) 169109702Sdes{ 170125965Sdes static char str[16]; 171109735Sdes double delta, bps; 172109702Sdes 173109735Sdes delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6)) 174109735Sdes - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 175109735Sdes if (delta == 0.0) { 176125965Sdes snprintf(str, sizeof str, "?? Bps"); 177125965Sdes } else { 178125965Sdes bps = (xs->rcvd - xs->offset) / delta; 179129440Sle snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps)); 180109735Sdes } 181125965Sdes return (str); 182109702Sdes} 183109702Sdes 184109702Sdes/* 18581863Sdes * Update the stats display 18681863Sdes */ 18779837Sdesstatic void 18863046Sdesstat_display(struct xferstat *xs, int force) 18962216Sdes{ 19079837Sdes struct timeval now; 19183863Sdes int ctty_pgrp; 19279837Sdes 19383863Sdes /* check if we're the foreground process */ 19483863Sdes if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || 19583863Sdes (pid_t)ctty_pgrp != pgrp) 19683863Sdes return; 197106043Sdes 19879837Sdes gettimeofday(&now, NULL); 19979837Sdes if (!force && now.tv_sec <= xs->last.tv_sec) 20079837Sdes return; 20179837Sdes xs->last = now; 20279837Sdes 203131615Sdes fprintf(stderr, "\r%-46.46s", xs->name); 204106041Sdes if (xs->size <= 0) { 205153894Sdes setproctitle("%s [%s]", xs->name, stat_bytes(xs->rcvd)); 206125965Sdes fprintf(stderr, " %s", stat_bytes(xs->rcvd)); 207106041Sdes } else { 208153894Sdes setproctitle("%s [%d%% of %s]", xs->name, 209153894Sdes (int)((100.0 * xs->rcvd) / xs->size), 210153894Sdes stat_bytes(xs->size)); 211125965Sdes fprintf(stderr, "%3d%% of %s", 212125965Sdes (int)((100.0 * xs->rcvd) / xs->size), 213125965Sdes stat_bytes(xs->size)); 214106041Sdes } 215125965Sdes fprintf(stderr, " %s", stat_bps(xs)); 216125965Sdes if (xs->size > 0 && xs->rcvd > 0 && 217125965Sdes xs->last.tv_sec >= xs->start.tv_sec + 10) 218125965Sdes fprintf(stderr, " %s", stat_eta(xs)); 21962216Sdes} 22062216Sdes 22181863Sdes/* 22281863Sdes * Initialize the transfer statistics 22381863Sdes */ 22479837Sdesstatic void 22579837Sdesstat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) 22663046Sdes{ 22779837Sdes snprintf(xs->name, sizeof xs->name, "%s", name); 22879837Sdes gettimeofday(&xs->start, NULL); 22979837Sdes xs->last.tv_sec = xs->last.tv_usec = 0; 23079837Sdes xs->size = size; 23179837Sdes xs->offset = offset; 23279837Sdes xs->rcvd = offset; 233125965Sdes if (v_tty && v_level > 0) 234125965Sdes stat_display(xs, 1); 235125965Sdes else if (v_level > 0) 236125965Sdes fprintf(stderr, "%-46s", xs->name); 23763046Sdes} 23863046Sdes 23981863Sdes/* 24081863Sdes * Update the transfer statistics 24181863Sdes */ 24279837Sdesstatic void 24379837Sdesstat_update(struct xferstat *xs, off_t rcvd) 24463046Sdes{ 24579837Sdes xs->rcvd = rcvd; 246125965Sdes if (v_tty && v_level > 0) 247125965Sdes stat_display(xs, 0); 24863046Sdes} 24963046Sdes 25081863Sdes/* 25181863Sdes * Finalize the transfer statistics 25281863Sdes */ 25379837Sdesstatic void 25462216Sdesstat_end(struct xferstat *xs) 25562216Sdes{ 256109735Sdes gettimeofday(&xs->last, NULL); 257125965Sdes if (v_tty && v_level > 0) { 258125965Sdes stat_display(xs, 1); 259125965Sdes putc('\n', stderr); 260125965Sdes } else if (v_level > 0) { 261125965Sdes fprintf(stderr, " %s %s\n", 262125965Sdes stat_bytes(xs->size), stat_bps(xs)); 263125965Sdes } 26462216Sdes} 26562216Sdes 26681863Sdes/* 26781863Sdes * Ask the user for authentication details 26881863Sdes */ 26979837Sdesstatic int 27077241Sdesquery_auth(struct url *URL) 27177241Sdes{ 27279837Sdes struct termios tios; 27379837Sdes tcflag_t saved_flags; 27479837Sdes int i, nopwd; 27577241Sdes 27679837Sdes fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", 27786242Siedowse URL->scheme, URL->host, URL->port); 27879837Sdes 27979837Sdes fprintf(stderr, "Login: "); 28079837Sdes if (fgets(URL->user, sizeof URL->user, stdin) == NULL) 281132695Sdes return (-1); 282132696Sdes for (i = strlen(URL->user); i >= 0; --i) 283132696Sdes if (URL->user[i] == '\r' || URL->user[i] == '\n') 28479837Sdes URL->user[i] = '\0'; 28579837Sdes 28679837Sdes fprintf(stderr, "Password: "); 28779837Sdes if (tcgetattr(STDIN_FILENO, &tios) == 0) { 28879837Sdes saved_flags = tios.c_lflag; 28979837Sdes tios.c_lflag &= ~ECHO; 29079837Sdes tios.c_lflag |= ECHONL|ICANON; 29179837Sdes tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); 29279837Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 29379837Sdes tios.c_lflag = saved_flags; 29479837Sdes tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); 29579837Sdes } else { 29679837Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 29779837Sdes } 29879837Sdes if (nopwd) 299132695Sdes return (-1); 300132696Sdes for (i = strlen(URL->pwd); i >= 0; --i) 301132696Sdes if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n') 302132696Sdes URL->pwd[i] = '\0'; 30379837Sdes 304132695Sdes return (0); 30577241Sdes} 30677241Sdes 30781863Sdes/* 30881863Sdes * Fetch a file 30981863Sdes */ 31079837Sdesstatic int 31179837Sdesfetch(char *URL, const char *path) 31262216Sdes{ 31379837Sdes struct url *url; 31479837Sdes struct url_stat us; 31583217Sdes struct stat sb, nsb; 31679837Sdes struct xferstat xs; 31779837Sdes FILE *f, *of; 31879837Sdes size_t size, wr; 31979837Sdes off_t count; 32079837Sdes char flags[8]; 32183217Sdes const char *slash; 32283217Sdes char *tmppath; 32379837Sdes int r; 324125976Sdes unsigned timeout; 325125976Sdes char *ptr; 32662216Sdes 32779837Sdes f = of = NULL; 32883217Sdes tmppath = NULL; 32962216Sdes 330109702Sdes timeout = 0; 331109702Sdes *flags = 0; 332109702Sdes count = 0; 333109702Sdes 334109702Sdes /* set verbosity level */ 335109702Sdes if (v_level > 1) 336109702Sdes strcat(flags, "v"); 337109702Sdes if (v_level > 2) 338109702Sdes fetchDebug = 1; 339109702Sdes 34079837Sdes /* parse URL */ 34179837Sdes if ((url = fetchParseURL(URL)) == NULL) { 34279837Sdes warnx("%s: parse error", URL); 34379837Sdes goto failure; 34479837Sdes } 34562216Sdes 34679837Sdes /* if no scheme was specified, take a guess */ 34779837Sdes if (!*url->scheme) { 34879837Sdes if (!*url->host) 34979837Sdes strcpy(url->scheme, SCHEME_FILE); 35079837Sdes else if (strncasecmp(url->host, "ftp.", 4) == 0) 35179837Sdes strcpy(url->scheme, SCHEME_FTP); 35279837Sdes else if (strncasecmp(url->host, "www.", 4) == 0) 35379837Sdes strcpy(url->scheme, SCHEME_HTTP); 35479837Sdes } 35569976Sdes 35679837Sdes /* common flags */ 35779837Sdes switch (family) { 35879837Sdes case PF_INET: 35979837Sdes strcat(flags, "4"); 36079837Sdes break; 36179837Sdes case PF_INET6: 36279837Sdes strcat(flags, "6"); 36379837Sdes break; 36479837Sdes } 36562216Sdes 36679837Sdes /* FTP specific flags */ 367181962Sobrien if (strcmp(url->scheme, SCHEME_FTP) == 0) { 36879837Sdes if (p_flag) 36979837Sdes strcat(flags, "p"); 37079837Sdes if (d_flag) 37179837Sdes strcat(flags, "d"); 37279837Sdes if (U_flag) 37379837Sdes strcat(flags, "l"); 37479837Sdes timeout = T_secs ? T_secs : ftp_timeout; 37579837Sdes } 37662216Sdes 37779837Sdes /* HTTP specific flags */ 378185912Sdes if (strcmp(url->scheme, SCHEME_HTTP) == 0 || 379185912Sdes strcmp(url->scheme, SCHEME_HTTPS) == 0) { 38079837Sdes if (d_flag) 38179837Sdes strcat(flags, "d"); 38279837Sdes if (A_flag) 38379837Sdes strcat(flags, "A"); 38479837Sdes timeout = T_secs ? T_secs : http_timeout; 38579837Sdes } 38662216Sdes 38779837Sdes /* set the protocol timeout. */ 38879837Sdes fetchTimeout = timeout; 38962216Sdes 39079837Sdes /* just print size */ 39179837Sdes if (s_flag) { 392106041Sdes if (timeout) 393106041Sdes alarm(timeout); 394106041Sdes r = fetchStat(url, &us, flags); 395106041Sdes if (timeout) 396106043Sdes alarm(0); 397106041Sdes if (sigalrm || sigint) 398106041Sdes goto signal; 399106041Sdes if (r == -1) { 400106041Sdes warnx("%s", fetchLastErrString); 40179837Sdes goto failure; 402106041Sdes } 40379837Sdes if (us.size == -1) 40479837Sdes printf("Unknown\n"); 40579837Sdes else 406125976Sdes printf("%jd\n", (intmax_t)us.size); 40779837Sdes goto success; 40863345Sdes } 40979837Sdes 41079837Sdes /* 41179837Sdes * If the -r flag was specified, we have to compare the local 41279837Sdes * and remote files, so we should really do a fetchStat() 41379837Sdes * first, but I know of at least one HTTP server that only 41479837Sdes * sends the content size in response to GET requests, and 41579837Sdes * leaves it out of replies to HEAD requests. Also, in the 41679837Sdes * (frequent) case that the local and remote files match but 41779837Sdes * the local file is truncated, we have sufficient information 41879837Sdes * before the compare to issue a correct request. Therefore, 41979837Sdes * we always issue a GET request as if we were sure the local 42079837Sdes * file was a truncated copy of the remote file; we can drop 42179837Sdes * the connection later if we change our minds. 42279837Sdes */ 42383217Sdes sb.st_size = -1; 424133779Sdes if (!o_stdout) { 425133779Sdes r = stat(path, &sb); 426134350Sdes if (r == 0 && r_flag && S_ISREG(sb.st_mode)) { 427133779Sdes url->offset = sb.st_size; 428153919Sdes } else if (r == -1 || !S_ISREG(sb.st_mode)) { 429133779Sdes /* 430133779Sdes * Whatever value sb.st_size has now is either 431133779Sdes * wrong (if stat(2) failed) or irrelevant (if the 432133779Sdes * path does not refer to a regular file) 433133779Sdes */ 434133779Sdes sb.st_size = -1; 435133779Sdes } 436153919Sdes if (r == -1 && errno != ENOENT) { 437153919Sdes warnx("%s: stat()", path); 438153919Sdes goto failure; 439153919Sdes } 44062216Sdes } 44162216Sdes 44279837Sdes /* start the transfer */ 443106041Sdes if (timeout) 444106041Sdes alarm(timeout); 445106041Sdes f = fetchXGet(url, &us, flags); 446106042Sdes if (timeout) 447106042Sdes alarm(0); 448106041Sdes if (sigalrm || sigint) 449106041Sdes goto signal; 450106041Sdes if (f == NULL) { 451107353Sdes warnx("%s: %s", URL, fetchLastErrString); 45263345Sdes goto failure; 45379837Sdes } 45479837Sdes if (sigint) 45563345Sdes goto signal; 45679837Sdes 45779837Sdes /* check that size is as expected */ 45879837Sdes if (S_size) { 45979837Sdes if (us.size == -1) { 460107353Sdes warnx("%s: size unknown", URL); 46179837Sdes } else if (us.size != S_size) { 462125976Sdes warnx("%s: size mismatch: expected %jd, actual %jd", 463125976Sdes URL, (intmax_t)S_size, (intmax_t)us.size); 46479837Sdes goto failure; 46579837Sdes } 46679837Sdes } 46779837Sdes 46879837Sdes /* symlink instead of copy */ 46979837Sdes if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 47079837Sdes if (symlink(url->doc, path) == -1) { 47179837Sdes warn("%s: symlink()", path); 47279837Sdes goto failure; 47379837Sdes } 47463568Sdes goto success; 47563345Sdes } 47679837Sdes 477106051Sdes if (us.size == -1 && !o_stdout && v_level > 0) 478107353Sdes warnx("%s: size of remote file is not known", URL); 47979837Sdes if (v_level > 1) { 48079837Sdes if (sb.st_size != -1) 481125976Sdes fprintf(stderr, "local size / mtime: %jd / %ld\n", 482125976Sdes (intmax_t)sb.st_size, (long)sb.st_mtime); 48379837Sdes if (us.size != -1) 484125976Sdes fprintf(stderr, "remote size / mtime: %jd / %ld\n", 485125976Sdes (intmax_t)us.size, (long)us.mtime); 48662216Sdes } 48762216Sdes 48879837Sdes /* open output file */ 48979837Sdes if (o_stdout) { 49079837Sdes /* output to stdout */ 49179837Sdes of = stdout; 49283217Sdes } else if (r_flag && sb.st_size != -1) { 49379837Sdes /* resume mode, local file exists */ 49479837Sdes if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { 49579837Sdes /* no match! have to refetch */ 49679837Sdes fclose(f); 49779837Sdes /* if precious, warn the user and give up */ 49879837Sdes if (R_flag) { 49979837Sdes warnx("%s: local modification time " 50079837Sdes "does not match remote", path); 50179837Sdes goto failure_keep; 50279837Sdes } 503127941Sdes } else if (us.size != -1) { 50479837Sdes if (us.size == sb.st_size) 50579837Sdes /* nothing to do */ 50679837Sdes goto success; 50779837Sdes if (sb.st_size > us.size) { 50879837Sdes /* local file too long! */ 509125976Sdes warnx("%s: local file (%jd bytes) is longer " 510125976Sdes "than remote file (%jd bytes)", path, 511125976Sdes (intmax_t)sb.st_size, (intmax_t)us.size); 51279837Sdes goto failure; 51379837Sdes } 51483217Sdes /* we got it, open local file */ 51579837Sdes if ((of = fopen(path, "a")) == NULL) { 51679837Sdes warn("%s: fopen()", path); 51779837Sdes goto failure; 51879837Sdes } 51983217Sdes /* check that it didn't move under our feet */ 52083217Sdes if (fstat(fileno(of), &nsb) == -1) { 52183217Sdes /* can't happen! */ 52283217Sdes warn("%s: fstat()", path); 52379837Sdes goto failure; 52479837Sdes } 52583217Sdes if (nsb.st_dev != sb.st_dev || 52683217Sdes nsb.st_ino != nsb.st_ino || 52783217Sdes nsb.st_size != sb.st_size) { 528107353Sdes warnx("%s: file has changed", URL); 52983217Sdes fclose(of); 53083217Sdes of = NULL; 53183217Sdes sb = nsb; 53283217Sdes } 53379837Sdes } 53483217Sdes } else if (m_flag && sb.st_size != -1) { 53579837Sdes /* mirror mode, local file exists */ 53679837Sdes if (sb.st_size == us.size && sb.st_mtime == us.mtime) 53779837Sdes goto success; 53879837Sdes } 539106043Sdes 54083217Sdes if (of == NULL) { 54179837Sdes /* 54279837Sdes * We don't yet have an output file; either this is a 54379837Sdes * vanilla run with no special flags, or the local and 54479837Sdes * remote files didn't match. 54579837Sdes */ 546106043Sdes 547109702Sdes if (url->offset > 0) { 54883217Sdes /* 54983217Sdes * We tried to restart a transfer, but for 55083217Sdes * some reason gave up - so we have to restart 55183217Sdes * from scratch if we want the whole file 55283217Sdes */ 55383217Sdes url->offset = 0; 55483217Sdes if ((f = fetchXGet(url, &us, flags)) == NULL) { 555107353Sdes warnx("%s: %s", URL, fetchLastErrString); 55683217Sdes goto failure; 55783217Sdes } 55883217Sdes if (sigint) 55983217Sdes goto signal; 56083217Sdes } 56183217Sdes 56283217Sdes /* construct a temp file name */ 56383217Sdes if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { 56483217Sdes if ((slash = strrchr(path, '/')) == NULL) 56583217Sdes slash = path; 56683217Sdes else 56783217Sdes ++slash; 56883217Sdes asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", 56983307Smike (int)(slash - path), path, slash); 570100834Sdes if (tmppath != NULL) { 571100834Sdes mkstemps(tmppath, strlen(slash) + 1); 572100834Sdes of = fopen(tmppath, "w"); 573164152Sdes chown(tmppath, sb.st_uid, sb.st_gid); 574164152Sdes chmod(tmppath, sb.st_mode & ALLPERMS); 575100834Sdes } 57683217Sdes } 577100834Sdes if (of == NULL) 57883217Sdes of = fopen(path, "w"); 57983217Sdes if (of == NULL) { 58079837Sdes warn("%s: open()", path); 58179837Sdes goto failure; 58279837Sdes } 58379837Sdes } 58479837Sdes count = url->offset; 58562216Sdes 58679837Sdes /* start the counter */ 58779837Sdes stat_start(&xs, path, us.size, count); 58863046Sdes 58979837Sdes sigalrm = siginfo = sigint = 0; 59079837Sdes 59179837Sdes /* suck in the data */ 59279837Sdes signal(SIGINFO, sig_handler); 593106041Sdes while (!sigint) { 594137854Scperciva if (us.size != -1 && us.size - count < B_size && 595137854Scperciva us.size - count >= 0) 59679837Sdes size = us.size - count; 59779837Sdes else 59879837Sdes size = B_size; 59979837Sdes if (siginfo) { 60079837Sdes stat_end(&xs); 60179837Sdes siginfo = 0; 60279837Sdes } 60379837Sdes if ((size = fread(buf, 1, size, f)) == 0) { 604106041Sdes if (ferror(f) && errno == EINTR && !sigint) 60579837Sdes clearerr(f); 60679837Sdes else 60779837Sdes break; 60879837Sdes } 60979837Sdes stat_update(&xs, count += size); 61079837Sdes for (ptr = buf; size > 0; ptr += wr, size -= wr) 61179837Sdes if ((wr = fwrite(ptr, 1, size, of)) < size) { 612106041Sdes if (ferror(of) && errno == EINTR && !sigint) 61379837Sdes clearerr(of); 61479837Sdes else 61579837Sdes break; 61679837Sdes } 61779837Sdes if (size != 0) 61879837Sdes break; 61977241Sdes } 620106041Sdes if (!sigalrm) 621106041Sdes sigalrm = ferror(f) && errno == ETIMEDOUT; 62279837Sdes signal(SIGINFO, SIG_DFL); 62379837Sdes 62479837Sdes stat_end(&xs); 62562216Sdes 626106041Sdes /* 627106041Sdes * If the transfer timed out or was interrupted, we still want to 628106041Sdes * set the mtime in case the file is not removed (-r or -R) and 629106041Sdes * the user later restarts the transfer. 630106041Sdes */ 631106041Sdes signal: 63279837Sdes /* set mtime of local file */ 633106857Sdes if (!n_flag && us.mtime && !o_stdout && of != NULL && 634106857Sdes (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { 63579837Sdes struct timeval tv[2]; 636106043Sdes 63779837Sdes fflush(of); 63879837Sdes tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 63979837Sdes tv[1].tv_sec = (long)us.mtime; 64079837Sdes tv[0].tv_usec = tv[1].tv_usec = 0; 64190729Sdes if (utimes(tmppath ? tmppath : path, tv)) 64290729Sdes warn("%s: utimes()", tmppath ? tmppath : path); 64379837Sdes } 64463015Sdes 64579837Sdes /* timed out or interrupted? */ 64679837Sdes if (sigalrm) 64779837Sdes warnx("transfer timed out"); 64879837Sdes if (sigint) { 64979837Sdes warnx("transfer interrupted"); 65079837Sdes goto failure; 65179837Sdes } 65263046Sdes 653106586Sfenner /* timeout / interrupt before connection completley established? */ 654106586Sfenner if (f == NULL) 655106586Sfenner goto failure; 656106586Sfenner 65779837Sdes if (!sigalrm) { 65879837Sdes /* check the status of our files */ 65979837Sdes if (ferror(f)) 66079837Sdes warn("%s", URL); 66179837Sdes if (ferror(of)) 66279837Sdes warn("%s", path); 66379837Sdes if (ferror(f) || ferror(of)) 66479837Sdes goto failure; 66579837Sdes } 66679837Sdes 66779837Sdes /* did the transfer complete normally? */ 66879837Sdes if (us.size != -1 && count < us.size) { 669125976Sdes warnx("%s appears to be truncated: %jd/%jd bytes", 670125976Sdes path, (intmax_t)count, (intmax_t)us.size); 67179837Sdes goto failure_keep; 67279837Sdes } 67379837Sdes 67479837Sdes /* 67579837Sdes * If the transfer timed out and we didn't know how much to 67679837Sdes * expect, assume the worst (i.e. we didn't get all of it) 67779837Sdes */ 67879837Sdes if (sigalrm && us.size == -1) { 67979837Sdes warnx("%s may be truncated", path); 68079837Sdes goto failure_keep; 68179837Sdes } 68279837Sdes 68362216Sdes success: 68479837Sdes r = 0; 68583217Sdes if (tmppath != NULL && rename(tmppath, path) == -1) { 68683217Sdes warn("%s: rename()", path); 68783217Sdes goto failure_keep; 68883217Sdes } 68979837Sdes goto done; 69062216Sdes failure: 69179837Sdes if (of && of != stdout && !R_flag && !r_flag) 69279837Sdes if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) 69383217Sdes unlink(tmppath ? tmppath : path); 69483217Sdes if (R_flag && tmppath != NULL && sb.st_size == -1) 69583217Sdes rename(tmppath, path); /* ignore errors here */ 69663046Sdes failure_keep: 69779837Sdes r = -1; 69879837Sdes goto done; 69962216Sdes done: 70079837Sdes if (f) 70179837Sdes fclose(f); 70279837Sdes if (of && of != stdout) 70379837Sdes fclose(of); 70479837Sdes if (url) 70579837Sdes fetchFreeURL(url); 70683217Sdes if (tmppath != NULL) 70783217Sdes free(tmppath); 708132695Sdes return (r); 70962216Sdes} 71062216Sdes 71179837Sdesstatic void 71262216Sdesusage(void) 71362216Sdes{ 714186043Sru fprintf(stderr, "%s\n%s\n%s\n%s\n", 715186043Sru"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", 716186043Sru" [-T seconds] [-w seconds] URL ...", 717186043Sru" fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", 718186043Sru" [-T seconds] [-w seconds] -h host -f file [-c dir]"); 71962216Sdes} 72062216Sdes 72162216Sdes 72281863Sdes/* 72381863Sdes * Entry point 72481863Sdes */ 72562216Sdesint 72662216Sdesmain(int argc, char *argv[]) 72762216Sdes{ 72879837Sdes struct stat sb; 72979837Sdes struct sigaction sa; 73079837Sdes const char *p, *s; 731100834Sdes char *end, *q; 73279837Sdes int c, e, r; 73362216Sdes 73479837Sdes while ((c = getopt(argc, argv, 735109702Sdes "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) 73679837Sdes switch (c) { 73779837Sdes case '1': 73879837Sdes once_flag = 1; 73979837Sdes break; 74079837Sdes case '4': 74179837Sdes family = PF_INET; 74279837Sdes break; 74379837Sdes case '6': 74479837Sdes family = PF_INET6; 74579837Sdes break; 74679837Sdes case 'A': 74779837Sdes A_flag = 1; 74879837Sdes break; 74979837Sdes case 'a': 75079837Sdes a_flag = 1; 75179837Sdes break; 75279837Sdes case 'B': 753100834Sdes B_size = (off_t)strtol(optarg, &end, 10); 754100834Sdes if (*optarg == '\0' || *end != '\0') 75580521Sse errx(1, "invalid buffer size (%s)", optarg); 75679837Sdes break; 75779837Sdes case 'b': 75879837Sdes warnx("warning: the -b option is deprecated"); 75979837Sdes b_flag = 1; 76079837Sdes break; 76179837Sdes case 'c': 76279837Sdes c_dirname = optarg; 76379837Sdes break; 76479837Sdes case 'd': 76579837Sdes d_flag = 1; 76679837Sdes break; 76779837Sdes case 'F': 76879837Sdes F_flag = 1; 76979837Sdes break; 77079837Sdes case 'f': 77179837Sdes f_filename = optarg; 77279837Sdes break; 77379837Sdes case 'H': 77493213Scharnier warnx("the -H option is now implicit, " 77579837Sdes "use -U to disable"); 77679837Sdes break; 77779837Sdes case 'h': 77879837Sdes h_hostname = optarg; 77979837Sdes break; 78079837Sdes case 'l': 78179837Sdes l_flag = 1; 78279837Sdes break; 78379837Sdes case 'o': 78479837Sdes o_flag = 1; 78579837Sdes o_filename = optarg; 78679837Sdes break; 78779837Sdes case 'M': 78879837Sdes case 'm': 78979837Sdes if (r_flag) 79079837Sdes errx(1, "the -m and -r flags " 79179837Sdes "are mutually exclusive"); 79279837Sdes m_flag = 1; 79379837Sdes break; 794109702Sdes case 'N': 795109702Sdes N_filename = optarg; 796109702Sdes break; 79779837Sdes case 'n': 79879837Sdes n_flag = 1; 79979837Sdes break; 80079837Sdes case 'P': 80179837Sdes case 'p': 80279837Sdes p_flag = 1; 80379837Sdes break; 80479837Sdes case 'q': 80579837Sdes v_level = 0; 80679837Sdes break; 80779837Sdes case 'R': 80879837Sdes R_flag = 1; 80979837Sdes break; 81079837Sdes case 'r': 81179837Sdes if (m_flag) 81279837Sdes errx(1, "the -m and -r flags " 81379837Sdes "are mutually exclusive"); 81479837Sdes r_flag = 1; 81579837Sdes break; 81679837Sdes case 'S': 817100834Sdes S_size = (off_t)strtol(optarg, &end, 10); 818100834Sdes if (*optarg == '\0' || *end != '\0') 81980521Sse errx(1, "invalid size (%s)", optarg); 82079837Sdes break; 82179837Sdes case 's': 82279837Sdes s_flag = 1; 82379837Sdes break; 824106043Sdes case 'T': 825100834Sdes T_secs = strtol(optarg, &end, 10); 826100834Sdes if (*optarg == '\0' || *end != '\0') 82780521Sse errx(1, "invalid timeout (%s)", optarg); 82879837Sdes break; 82979837Sdes case 't': 83079837Sdes t_flag = 1; 83179837Sdes warnx("warning: the -t option is deprecated"); 83279837Sdes break; 83379837Sdes case 'U': 83479837Sdes U_flag = 1; 83579837Sdes break; 83679837Sdes case 'v': 83779837Sdes v_level++; 83879837Sdes break; 83979837Sdes case 'w': 84079837Sdes a_flag = 1; 841100834Sdes w_secs = strtol(optarg, &end, 10); 842100834Sdes if (*optarg == '\0' || *end != '\0') 84380521Sse errx(1, "invalid delay (%s)", optarg); 84479837Sdes break; 84579837Sdes default: 84679837Sdes usage(); 84779837Sdes exit(EX_USAGE); 84879837Sdes } 84962216Sdes 85079837Sdes argc -= optind; 85179837Sdes argv += optind; 85262216Sdes 85379837Sdes if (h_hostname || f_filename || c_dirname) { 85479837Sdes if (!h_hostname || !f_filename || argc) { 85579837Sdes usage(); 85679837Sdes exit(EX_USAGE); 85779837Sdes } 85879837Sdes /* XXX this is a hack. */ 85979837Sdes if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 86079837Sdes errx(1, "invalid hostname"); 86179837Sdes if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 86279837Sdes c_dirname ? c_dirname : "", f_filename) == -1) 86379837Sdes errx(1, "%s", strerror(ENOMEM)); 86479837Sdes argc++; 86562216Sdes } 86663345Sdes 86779837Sdes if (!argc) { 86879837Sdes usage(); 86979837Sdes exit(EX_USAGE); 87079837Sdes } 87162216Sdes 87279837Sdes /* allocate buffer */ 87379837Sdes if (B_size < MINBUFSIZE) 87479837Sdes B_size = MINBUFSIZE; 87579837Sdes if ((buf = malloc(B_size)) == NULL) 87679837Sdes errx(1, "%s", strerror(ENOMEM)); 87762216Sdes 87879837Sdes /* timeouts */ 87979837Sdes if ((s = getenv("FTP_TIMEOUT")) != NULL) { 880100834Sdes ftp_timeout = strtol(s, &end, 10); 881102478Sdes if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { 882102478Sdes warnx("FTP_TIMEOUT (%s) is not a positive integer", s); 88379837Sdes ftp_timeout = 0; 88479837Sdes } 88562216Sdes } 88679837Sdes if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 887100834Sdes http_timeout = strtol(s, &end, 10); 888102478Sdes if (*s == '\0' || *end != '\0' || http_timeout < 0) { 889102478Sdes warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); 89079837Sdes http_timeout = 0; 89179837Sdes } 89262216Sdes } 89362216Sdes 89479837Sdes /* signal handling */ 89579837Sdes sa.sa_flags = 0; 89679837Sdes sa.sa_handler = sig_handler; 89779837Sdes sigemptyset(&sa.sa_mask); 89879837Sdes sigaction(SIGALRM, &sa, NULL); 89979837Sdes sa.sa_flags = SA_RESETHAND; 90079837Sdes sigaction(SIGINT, &sa, NULL); 90179837Sdes fetchRestartCalls = 0; 90279837Sdes 90379837Sdes /* output file */ 90479837Sdes if (o_flag) { 90579837Sdes if (strcmp(o_filename, "-") == 0) { 90679837Sdes o_stdout = 1; 90779837Sdes } else if (stat(o_filename, &sb) == -1) { 90879837Sdes if (errno == ENOENT) { 90979837Sdes if (argc > 1) 91079837Sdes errx(EX_USAGE, "%s is not a directory", 91179837Sdes o_filename); 91279837Sdes } else { 91379837Sdes err(EX_IOERR, "%s", o_filename); 91479837Sdes } 91579837Sdes } else { 91679837Sdes if (sb.st_mode & S_IFDIR) 91779837Sdes o_directory = 1; 91879837Sdes } 91962216Sdes } 92062216Sdes 92179837Sdes /* check if output is to a tty (for progress report) */ 92279837Sdes v_tty = isatty(STDERR_FILENO); 92383863Sdes if (v_tty) 92483863Sdes pgrp = getpgrp(); 925106043Sdes 92679837Sdes r = 0; 927106043Sdes 92879837Sdes /* authentication */ 92979837Sdes if (v_tty) 93079837Sdes fetchAuthMethod = query_auth; 931109702Sdes if (N_filename != NULL) 932109702Sdes setenv("NETRC", N_filename, 1); 93362216Sdes 93479837Sdes while (argc) { 93579837Sdes if ((p = strrchr(*argv, '/')) == NULL) 93679837Sdes p = *argv; 93779837Sdes else 93879837Sdes p++; 93962216Sdes 94079837Sdes if (!*p) 94179837Sdes p = "fetch.out"; 942106043Sdes 94379837Sdes fetchLastErrCode = 0; 944106043Sdes 94579837Sdes if (o_flag) { 94679837Sdes if (o_stdout) { 94779837Sdes e = fetch(*argv, "-"); 94879837Sdes } else if (o_directory) { 94979837Sdes asprintf(&q, "%s/%s", o_filename, p); 95079837Sdes e = fetch(*argv, q); 95179837Sdes free(q); 95279837Sdes } else { 95379837Sdes e = fetch(*argv, o_filename); 95479837Sdes } 95579837Sdes } else { 95679837Sdes e = fetch(*argv, p); 95779837Sdes } 95862216Sdes 95979837Sdes if (sigint) 96079837Sdes kill(getpid(), SIGINT); 961106043Sdes 96279837Sdes if (e == 0 && once_flag) 96379837Sdes exit(0); 964106043Sdes 96579837Sdes if (e) { 96679837Sdes r = 1; 96779837Sdes if ((fetchLastErrCode 96879837Sdes && fetchLastErrCode != FETCH_UNAVAIL 96979837Sdes && fetchLastErrCode != FETCH_MOVED 97079837Sdes && fetchLastErrCode != FETCH_URL 97179837Sdes && fetchLastErrCode != FETCH_RESOLV 97279837Sdes && fetchLastErrCode != FETCH_UNKNOWN)) { 97379837Sdes if (w_secs && v_level) 974100834Sdes fprintf(stderr, "Waiting %ld seconds " 97579837Sdes "before retrying\n", w_secs); 97679837Sdes if (w_secs) 97779837Sdes sleep(w_secs); 97879837Sdes if (a_flag) 97979837Sdes continue; 98079837Sdes } 98162216Sdes } 98279837Sdes 98379837Sdes argc--, argv++; 98462216Sdes } 98562216Sdes 98679837Sdes exit(r); 98762216Sdes} 988