fetch.c revision 225599
162216Sdes/*- 2185912Sdes * Copyright (c) 2000-2004 Dag-Erling Co��dan Sm��rgrav 362216Sdes * All rights reserved. 462216Sdes * 562216Sdes * Redistribution and use in source and binary forms, with or without 662216Sdes * modification, are permitted provided that the following conditions 762216Sdes * are met: 862216Sdes * 1. Redistributions of source code must retain the above copyright 962216Sdes * notice, this list of conditions and the following disclaimer 1062216Sdes * in this position and unchanged. 1162216Sdes * 2. Redistributions in binary form must reproduce the above copyright 1262216Sdes * notice, this list of conditions and the following disclaimer in the 1362216Sdes * documentation and/or other materials provided with the distribution. 1462216Sdes * 3. The name of the author may not be used to endorse or promote products 1562216Sdes * derived from this software without specific prior written permission 1662216Sdes * 1762216Sdes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1862216Sdes * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1962216Sdes * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2062216Sdes * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2162216Sdes * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2262216Sdes * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2362216Sdes * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2462216Sdes * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2562216Sdes * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2662216Sdes * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2762216Sdes */ 2862216Sdes 2993213Scharnier#include <sys/cdefs.h> 3093213Scharnier__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 225599 2011-09-15 22:50:31Z des $"); 3193213Scharnier 3262216Sdes#include <sys/param.h> 3391225Sbde#include <sys/socket.h> 3462216Sdes#include <sys/stat.h> 3593257Sbde#include <sys/time.h> 3662216Sdes 37200462Sdelphij#include <ctype.h> 3862216Sdes#include <err.h> 3962216Sdes#include <errno.h> 4063235Sdes#include <signal.h> 41125976Sdes#include <stdint.h> 4262216Sdes#include <stdio.h> 4362216Sdes#include <stdlib.h> 4462216Sdes#include <string.h> 4577241Sdes#include <termios.h> 4662216Sdes#include <unistd.h> 4762216Sdes 4862216Sdes#include <fetch.h> 4962216Sdes 5062216Sdes#define MINBUFSIZE 4096 51187361Sdes#define TIMEOUT 120 5262216Sdes 5362216Sdes/* Option flags */ 5462216Sdesint A_flag; /* -A: do not follow 302 redirects */ 5562216Sdesint a_flag; /* -a: auto retry */ 5679837Sdesoff_t B_size; /* -B: buffer size */ 5762216Sdesint b_flag; /*! -b: workaround TCP bug */ 5862254Sdeschar *c_dirname; /* -c: remote directory */ 5962216Sdesint d_flag; /* -d: direct connection */ 6062216Sdesint F_flag; /* -F: restart without checking mtime */ 6162216Sdeschar *f_filename; /* -f: file to fetch */ 6262216Sdeschar *h_hostname; /* -h: host to fetch from */ 63186124Smurrayint i_flag; /* -i: specify input file for mtime comparison */ 64186124Smurraychar *i_filename; /* name of input file */ 6562216Sdesint l_flag; /* -l: link rather than copy file: URLs */ 6662815Sdesint m_flag; /* -[Mm]: mirror mode */ 67109702Sdeschar *N_filename; /* -N: netrc file name */ 6862815Sdesint n_flag; /* -n: do not preserve modification time */ 6962216Sdesint o_flag; /* -o: specify output file */ 7062216Sdesint o_directory; /* output file is a directory */ 7162216Sdeschar *o_filename; /* name of output file */ 7262216Sdesint o_stdout; /* output file is stdout */ 7362216Sdesint once_flag; /* -1: stop at first successful file */ 7463501Sdesint p_flag; /* -[Pp]: use passive FTP */ 7562216Sdesint R_flag; /* -R: don't delete partially transferred files */ 7662216Sdesint r_flag; /* -r: restart previously interrupted transfer */ 7774717Sdesoff_t S_size; /* -S: require size to match */ 7874717Sdesint s_flag; /* -s: show size, don't fetch */ 79187361Sdeslong T_secs; /* -T: transfer timeout in seconds */ 8062216Sdesint t_flag; /*! -t: workaround TCP bug */ 8174717Sdesint U_flag; /* -U: do not use high ports */ 8262216Sdesint v_level = 1; /* -v: verbosity level */ 8362216Sdesint v_tty; /* stdout is a tty */ 8483863Sdespid_t pgrp; /* our process group */ 85100834Sdeslong w_secs; /* -w: retry delay */ 8662216Sdesint family = PF_UNSPEC; /* -[46]: address family to use */ 8762216Sdes 8863015Sdesint sigalrm; /* SIGALRM received */ 8973937Sdesint siginfo; /* SIGINFO received */ 9063015Sdesint sigint; /* SIGINT received */ 9162216Sdes 92187361Sdeslong ftp_timeout = TIMEOUT; /* default timeout for FTP transfers */ 93187361Sdeslong http_timeout = TIMEOUT; /* default timeout for HTTP transfers */ 94125976Sdeschar *buf; /* transfer buffer */ 9562216Sdes 9662216Sdes 9781863Sdes/* 9881863Sdes * Signal handler 9981863Sdes */ 10079837Sdesstatic void 10162216Sdessig_handler(int sig) 10262216Sdes{ 10379837Sdes switch (sig) { 10479837Sdes case SIGALRM: 10579837Sdes sigalrm = 1; 10679837Sdes break; 10779837Sdes case SIGINFO: 10879837Sdes siginfo = 1; 10979837Sdes break; 11079837Sdes case SIGINT: 11179837Sdes sigint = 1; 11279837Sdes break; 11379837Sdes } 11462216Sdes} 11562216Sdes 11662216Sdesstruct xferstat { 117125965Sdes char name[64]; 11879837Sdes struct timeval start; 11979837Sdes struct timeval last; 12079837Sdes off_t size; 12179837Sdes off_t offset; 12279837Sdes off_t rcvd; 12362216Sdes}; 12462216Sdes 12581863Sdes/* 126109702Sdes * Compute and display ETA 127109702Sdes */ 128125965Sdesstatic const char * 129109702Sdesstat_eta(struct xferstat *xs) 130109702Sdes{ 131125965Sdes static char str[16]; 132125976Sdes long elapsed, eta; 133125976Sdes off_t received, expected; 134109702Sdes 135109702Sdes elapsed = xs->last.tv_sec - xs->start.tv_sec; 136112083Sdes received = xs->rcvd - xs->offset; 137112083Sdes expected = xs->size - xs->rcvd; 138112114Sdes eta = (long)((double)elapsed * expected / received); 139125965Sdes if (eta > 3600) 140125965Sdes snprintf(str, sizeof str, "%02ldh%02ldm", 141125965Sdes eta / 3600, (eta % 3600) / 60); 142125965Sdes else 143125965Sdes snprintf(str, sizeof str, "%02ldm%02lds", 144125965Sdes eta / 60, eta % 60); 145125965Sdes return (str); 146125965Sdes} 147125965Sdes 148125965Sdes/* 149125965Sdes * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'... 150125965Sdes */ 151125965Sdesstatic const char *prefixes = " kMGTP"; 152125965Sdesstatic const char * 153129440Slestat_bytes(off_t bytes) 154125965Sdes{ 155125965Sdes static char str[16]; 156125965Sdes const char *prefix = prefixes; 157125965Sdes 158125965Sdes while (bytes > 9999 && prefix[1] != '\0') { 159125965Sdes bytes /= 1024; 160125965Sdes prefix++; 161109702Sdes } 162129440Sle snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix); 163125965Sdes return (str); 164109702Sdes} 165109702Sdes 166109702Sdes/* 167109702Sdes * Compute and display transfer rate 168109702Sdes */ 169125965Sdesstatic const char * 170109702Sdesstat_bps(struct xferstat *xs) 171109702Sdes{ 172125965Sdes static char str[16]; 173109735Sdes double delta, bps; 174109702Sdes 175109735Sdes delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6)) 176109735Sdes - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 177109735Sdes if (delta == 0.0) { 178125965Sdes snprintf(str, sizeof str, "?? Bps"); 179125965Sdes } else { 180125965Sdes bps = (xs->rcvd - xs->offset) / delta; 181129440Sle snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps)); 182109735Sdes } 183125965Sdes return (str); 184109702Sdes} 185109702Sdes 186109702Sdes/* 18781863Sdes * Update the stats display 18881863Sdes */ 18979837Sdesstatic void 19063046Sdesstat_display(struct xferstat *xs, int force) 19162216Sdes{ 19279837Sdes struct timeval now; 19383863Sdes int ctty_pgrp; 19479837Sdes 19583863Sdes /* check if we're the foreground process */ 19683863Sdes if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || 19783863Sdes (pid_t)ctty_pgrp != pgrp) 19883863Sdes return; 199106043Sdes 20079837Sdes gettimeofday(&now, NULL); 20179837Sdes if (!force && now.tv_sec <= xs->last.tv_sec) 20279837Sdes return; 20379837Sdes xs->last = now; 20479837Sdes 205131615Sdes fprintf(stderr, "\r%-46.46s", xs->name); 206106041Sdes if (xs->size <= 0) { 207153894Sdes setproctitle("%s [%s]", xs->name, stat_bytes(xs->rcvd)); 208125965Sdes fprintf(stderr, " %s", stat_bytes(xs->rcvd)); 209106041Sdes } else { 210153894Sdes setproctitle("%s [%d%% of %s]", xs->name, 211153894Sdes (int)((100.0 * xs->rcvd) / xs->size), 212153894Sdes stat_bytes(xs->size)); 213125965Sdes fprintf(stderr, "%3d%% of %s", 214125965Sdes (int)((100.0 * xs->rcvd) / xs->size), 215125965Sdes stat_bytes(xs->size)); 216106041Sdes } 217125965Sdes fprintf(stderr, " %s", stat_bps(xs)); 218125965Sdes if (xs->size > 0 && xs->rcvd > 0 && 219125965Sdes xs->last.tv_sec >= xs->start.tv_sec + 10) 220125965Sdes fprintf(stderr, " %s", stat_eta(xs)); 22162216Sdes} 22262216Sdes 22381863Sdes/* 22481863Sdes * Initialize the transfer statistics 22581863Sdes */ 22679837Sdesstatic void 22779837Sdesstat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) 22863046Sdes{ 22979837Sdes snprintf(xs->name, sizeof xs->name, "%s", name); 23079837Sdes gettimeofday(&xs->start, NULL); 23179837Sdes xs->last.tv_sec = xs->last.tv_usec = 0; 23279837Sdes xs->size = size; 23379837Sdes xs->offset = offset; 23479837Sdes xs->rcvd = offset; 235125965Sdes if (v_tty && v_level > 0) 236125965Sdes stat_display(xs, 1); 237125965Sdes else if (v_level > 0) 238125965Sdes fprintf(stderr, "%-46s", xs->name); 23963046Sdes} 24063046Sdes 24181863Sdes/* 24281863Sdes * Update the transfer statistics 24381863Sdes */ 24479837Sdesstatic void 24579837Sdesstat_update(struct xferstat *xs, off_t rcvd) 24663046Sdes{ 24779837Sdes xs->rcvd = rcvd; 248125965Sdes if (v_tty && v_level > 0) 249125965Sdes stat_display(xs, 0); 25063046Sdes} 25163046Sdes 25281863Sdes/* 25381863Sdes * Finalize the transfer statistics 25481863Sdes */ 25579837Sdesstatic void 25662216Sdesstat_end(struct xferstat *xs) 25762216Sdes{ 258109735Sdes gettimeofday(&xs->last, NULL); 259125965Sdes if (v_tty && v_level > 0) { 260125965Sdes stat_display(xs, 1); 261125965Sdes putc('\n', stderr); 262125965Sdes } else if (v_level > 0) { 263125965Sdes fprintf(stderr, " %s %s\n", 264125965Sdes stat_bytes(xs->size), stat_bps(xs)); 265125965Sdes } 26662216Sdes} 26762216Sdes 26881863Sdes/* 26981863Sdes * Ask the user for authentication details 27081863Sdes */ 27179837Sdesstatic int 27277241Sdesquery_auth(struct url *URL) 27377241Sdes{ 27479837Sdes struct termios tios; 27579837Sdes tcflag_t saved_flags; 27679837Sdes int i, nopwd; 27777241Sdes 27879837Sdes fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", 27986242Siedowse URL->scheme, URL->host, URL->port); 28079837Sdes 28179837Sdes fprintf(stderr, "Login: "); 28279837Sdes if (fgets(URL->user, sizeof URL->user, stdin) == NULL) 283132695Sdes return (-1); 284132696Sdes for (i = strlen(URL->user); i >= 0; --i) 285132696Sdes if (URL->user[i] == '\r' || URL->user[i] == '\n') 28679837Sdes URL->user[i] = '\0'; 28779837Sdes 28879837Sdes fprintf(stderr, "Password: "); 28979837Sdes if (tcgetattr(STDIN_FILENO, &tios) == 0) { 29079837Sdes saved_flags = tios.c_lflag; 29179837Sdes tios.c_lflag &= ~ECHO; 29279837Sdes tios.c_lflag |= ECHONL|ICANON; 29379837Sdes tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); 29479837Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 29579837Sdes tios.c_lflag = saved_flags; 29679837Sdes tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); 29779837Sdes } else { 29879837Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 29979837Sdes } 30079837Sdes if (nopwd) 301132695Sdes return (-1); 302132696Sdes for (i = strlen(URL->pwd); i >= 0; --i) 303132696Sdes if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n') 304132696Sdes URL->pwd[i] = '\0'; 30579837Sdes 306132695Sdes return (0); 30777241Sdes} 30877241Sdes 30981863Sdes/* 31081863Sdes * Fetch a file 31181863Sdes */ 31279837Sdesstatic int 31379837Sdesfetch(char *URL, const char *path) 31462216Sdes{ 31579837Sdes struct url *url; 31679837Sdes struct url_stat us; 31783217Sdes struct stat sb, nsb; 31879837Sdes struct xferstat xs; 31979837Sdes FILE *f, *of; 32079837Sdes size_t size, wr; 32179837Sdes off_t count; 32279837Sdes char flags[8]; 32383217Sdes const char *slash; 32483217Sdes char *tmppath; 32579837Sdes int r; 326125976Sdes unsigned timeout; 327125976Sdes char *ptr; 32862216Sdes 32979837Sdes f = of = NULL; 33083217Sdes tmppath = NULL; 33162216Sdes 332109702Sdes timeout = 0; 333109702Sdes *flags = 0; 334109702Sdes count = 0; 335109702Sdes 336109702Sdes /* set verbosity level */ 337109702Sdes if (v_level > 1) 338109702Sdes strcat(flags, "v"); 339109702Sdes if (v_level > 2) 340109702Sdes fetchDebug = 1; 341109702Sdes 34279837Sdes /* parse URL */ 343201290Sru url = NULL; 344201290Sru if (*URL == '\0') { 345201290Sru warnx("empty URL"); 346201290Sru goto failure; 347201290Sru } 34879837Sdes if ((url = fetchParseURL(URL)) == NULL) { 34979837Sdes warnx("%s: parse error", URL); 35079837Sdes goto failure; 35179837Sdes } 35262216Sdes 35379837Sdes /* if no scheme was specified, take a guess */ 35479837Sdes if (!*url->scheme) { 35579837Sdes if (!*url->host) 35679837Sdes strcpy(url->scheme, SCHEME_FILE); 35779837Sdes else if (strncasecmp(url->host, "ftp.", 4) == 0) 35879837Sdes strcpy(url->scheme, SCHEME_FTP); 35979837Sdes else if (strncasecmp(url->host, "www.", 4) == 0) 36079837Sdes strcpy(url->scheme, SCHEME_HTTP); 36179837Sdes } 36269976Sdes 36379837Sdes /* common flags */ 36479837Sdes switch (family) { 36579837Sdes case PF_INET: 36679837Sdes strcat(flags, "4"); 36779837Sdes break; 36879837Sdes case PF_INET6: 36979837Sdes strcat(flags, "6"); 37079837Sdes break; 37179837Sdes } 37262216Sdes 37379837Sdes /* FTP specific flags */ 374181962Sobrien if (strcmp(url->scheme, SCHEME_FTP) == 0) { 37579837Sdes if (p_flag) 37679837Sdes strcat(flags, "p"); 37779837Sdes if (d_flag) 37879837Sdes strcat(flags, "d"); 37979837Sdes if (U_flag) 38079837Sdes strcat(flags, "l"); 38179837Sdes timeout = T_secs ? T_secs : ftp_timeout; 38279837Sdes } 38362216Sdes 38479837Sdes /* HTTP specific flags */ 385185912Sdes if (strcmp(url->scheme, SCHEME_HTTP) == 0 || 386185912Sdes strcmp(url->scheme, SCHEME_HTTPS) == 0) { 38779837Sdes if (d_flag) 38879837Sdes strcat(flags, "d"); 38979837Sdes if (A_flag) 39079837Sdes strcat(flags, "A"); 39179837Sdes timeout = T_secs ? T_secs : http_timeout; 392186124Smurray if (i_flag) { 393186124Smurray if (stat(i_filename, &sb)) { 394186124Smurray warn("%s: stat()", i_filename); 395186124Smurray goto failure; 396186124Smurray } 397186124Smurray url->ims_time = sb.st_mtime; 398186124Smurray strcat(flags, "i"); 399186124Smurray } 40079837Sdes } 40162216Sdes 40279837Sdes /* set the protocol timeout. */ 40379837Sdes fetchTimeout = timeout; 40462216Sdes 40579837Sdes /* just print size */ 40679837Sdes if (s_flag) { 407106041Sdes if (timeout) 408106041Sdes alarm(timeout); 409106041Sdes r = fetchStat(url, &us, flags); 410106041Sdes if (timeout) 411106043Sdes alarm(0); 412106041Sdes if (sigalrm || sigint) 413106041Sdes goto signal; 414106041Sdes if (r == -1) { 415106041Sdes warnx("%s", fetchLastErrString); 41679837Sdes goto failure; 417106041Sdes } 41879837Sdes if (us.size == -1) 41979837Sdes printf("Unknown\n"); 42079837Sdes else 421125976Sdes printf("%jd\n", (intmax_t)us.size); 42279837Sdes goto success; 42363345Sdes } 42479837Sdes 42579837Sdes /* 42679837Sdes * If the -r flag was specified, we have to compare the local 42779837Sdes * and remote files, so we should really do a fetchStat() 42879837Sdes * first, but I know of at least one HTTP server that only 42979837Sdes * sends the content size in response to GET requests, and 43079837Sdes * leaves it out of replies to HEAD requests. Also, in the 43179837Sdes * (frequent) case that the local and remote files match but 43279837Sdes * the local file is truncated, we have sufficient information 43379837Sdes * before the compare to issue a correct request. Therefore, 43479837Sdes * we always issue a GET request as if we were sure the local 43579837Sdes * file was a truncated copy of the remote file; we can drop 43679837Sdes * the connection later if we change our minds. 43779837Sdes */ 43883217Sdes sb.st_size = -1; 439133779Sdes if (!o_stdout) { 440133779Sdes r = stat(path, &sb); 441134350Sdes if (r == 0 && r_flag && S_ISREG(sb.st_mode)) { 442133779Sdes url->offset = sb.st_size; 443153919Sdes } else if (r == -1 || !S_ISREG(sb.st_mode)) { 444133779Sdes /* 445133779Sdes * Whatever value sb.st_size has now is either 446133779Sdes * wrong (if stat(2) failed) or irrelevant (if the 447133779Sdes * path does not refer to a regular file) 448133779Sdes */ 449133779Sdes sb.st_size = -1; 450133779Sdes } 451153919Sdes if (r == -1 && errno != ENOENT) { 452153919Sdes warnx("%s: stat()", path); 453153919Sdes goto failure; 454153919Sdes } 45562216Sdes } 45662216Sdes 45779837Sdes /* start the transfer */ 458106041Sdes if (timeout) 459106041Sdes alarm(timeout); 460106041Sdes f = fetchXGet(url, &us, flags); 461106042Sdes if (timeout) 462106042Sdes alarm(0); 463106041Sdes if (sigalrm || sigint) 464106041Sdes goto signal; 465106041Sdes if (f == NULL) { 466107353Sdes warnx("%s: %s", URL, fetchLastErrString); 467186124Smurray if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0 468186124Smurray && fetchLastErrCode == FETCH_OK 469186124Smurray && strcmp(fetchLastErrString, "Not Modified") == 0) { 470186124Smurray /* HTTP Not Modified Response, return OK. */ 471186124Smurray r = 0; 472186124Smurray goto done; 473186124Smurray } else 474186124Smurray goto failure; 47579837Sdes } 47679837Sdes if (sigint) 47763345Sdes goto signal; 47879837Sdes 47979837Sdes /* check that size is as expected */ 48079837Sdes if (S_size) { 48179837Sdes if (us.size == -1) { 482107353Sdes warnx("%s: size unknown", URL); 48379837Sdes } else if (us.size != S_size) { 484125976Sdes warnx("%s: size mismatch: expected %jd, actual %jd", 485125976Sdes URL, (intmax_t)S_size, (intmax_t)us.size); 48679837Sdes goto failure; 48779837Sdes } 48879837Sdes } 48979837Sdes 49079837Sdes /* symlink instead of copy */ 49179837Sdes if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 49279837Sdes if (symlink(url->doc, path) == -1) { 49379837Sdes warn("%s: symlink()", path); 49479837Sdes goto failure; 49579837Sdes } 49663568Sdes goto success; 49763345Sdes } 49879837Sdes 499106051Sdes if (us.size == -1 && !o_stdout && v_level > 0) 500107353Sdes warnx("%s: size of remote file is not known", URL); 50179837Sdes if (v_level > 1) { 50279837Sdes if (sb.st_size != -1) 503125976Sdes fprintf(stderr, "local size / mtime: %jd / %ld\n", 504125976Sdes (intmax_t)sb.st_size, (long)sb.st_mtime); 50579837Sdes if (us.size != -1) 506125976Sdes fprintf(stderr, "remote size / mtime: %jd / %ld\n", 507125976Sdes (intmax_t)us.size, (long)us.mtime); 50862216Sdes } 50962216Sdes 51079837Sdes /* open output file */ 51179837Sdes if (o_stdout) { 51279837Sdes /* output to stdout */ 51379837Sdes of = stdout; 51483217Sdes } else if (r_flag && sb.st_size != -1) { 51579837Sdes /* resume mode, local file exists */ 51679837Sdes if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { 51779837Sdes /* no match! have to refetch */ 51879837Sdes fclose(f); 51979837Sdes /* if precious, warn the user and give up */ 52079837Sdes if (R_flag) { 52179837Sdes warnx("%s: local modification time " 52279837Sdes "does not match remote", path); 52379837Sdes goto failure_keep; 52479837Sdes } 525225599Sdes } else if (url->offset > sb.st_size) { 526225599Sdes /* gap between what we asked for and what we got */ 527225599Sdes warnx("%s: gap in resume mode", URL); 528225599Sdes fclose(of); 529225599Sdes of = NULL; 530225599Sdes /* picked up again later */ 531127941Sdes } else if (us.size != -1) { 53279837Sdes if (us.size == sb.st_size) 53379837Sdes /* nothing to do */ 53479837Sdes goto success; 53579837Sdes if (sb.st_size > us.size) { 53679837Sdes /* local file too long! */ 537125976Sdes warnx("%s: local file (%jd bytes) is longer " 538125976Sdes "than remote file (%jd bytes)", path, 539125976Sdes (intmax_t)sb.st_size, (intmax_t)us.size); 54079837Sdes goto failure; 54179837Sdes } 54283217Sdes /* we got it, open local file */ 54379837Sdes if ((of = fopen(path, "a")) == NULL) { 54479837Sdes warn("%s: fopen()", path); 54579837Sdes goto failure; 54679837Sdes } 54783217Sdes /* check that it didn't move under our feet */ 54883217Sdes if (fstat(fileno(of), &nsb) == -1) { 54983217Sdes /* can't happen! */ 55083217Sdes warn("%s: fstat()", path); 55179837Sdes goto failure; 55279837Sdes } 55383217Sdes if (nsb.st_dev != sb.st_dev || 55483217Sdes nsb.st_ino != nsb.st_ino || 55583217Sdes nsb.st_size != sb.st_size) { 556107353Sdes warnx("%s: file has changed", URL); 55783217Sdes fclose(of); 55883217Sdes of = NULL; 55983217Sdes sb = nsb; 560225599Sdes /* picked up again later */ 56183217Sdes } 562225599Sdes /* seek to where we left off */ 563225599Sdes if (of != NULL && fseek(of, url->offset, SEEK_SET) != 0) { 564225599Sdes warn("%s: fseek()", path); 565225599Sdes fclose(of); 566225599Sdes of = NULL; 567225599Sdes /* picked up again later */ 568225599Sdes } 56979837Sdes } 57083217Sdes } else if (m_flag && sb.st_size != -1) { 57179837Sdes /* mirror mode, local file exists */ 57279837Sdes if (sb.st_size == us.size && sb.st_mtime == us.mtime) 57379837Sdes goto success; 57479837Sdes } 575106043Sdes 57683217Sdes if (of == NULL) { 57779837Sdes /* 57879837Sdes * We don't yet have an output file; either this is a 57979837Sdes * vanilla run with no special flags, or the local and 58079837Sdes * remote files didn't match. 58179837Sdes */ 582106043Sdes 583109702Sdes if (url->offset > 0) { 58483217Sdes /* 58583217Sdes * We tried to restart a transfer, but for 58683217Sdes * some reason gave up - so we have to restart 58783217Sdes * from scratch if we want the whole file 58883217Sdes */ 58983217Sdes url->offset = 0; 59083217Sdes if ((f = fetchXGet(url, &us, flags)) == NULL) { 591107353Sdes warnx("%s: %s", URL, fetchLastErrString); 59283217Sdes goto failure; 59383217Sdes } 59483217Sdes if (sigint) 59583217Sdes goto signal; 59683217Sdes } 59783217Sdes 59883217Sdes /* construct a temp file name */ 59983217Sdes if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { 60083217Sdes if ((slash = strrchr(path, '/')) == NULL) 60183217Sdes slash = path; 60283217Sdes else 60383217Sdes ++slash; 60483217Sdes asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", 60583307Smike (int)(slash - path), path, slash); 606100834Sdes if (tmppath != NULL) { 607100834Sdes mkstemps(tmppath, strlen(slash) + 1); 608100834Sdes of = fopen(tmppath, "w"); 609164152Sdes chown(tmppath, sb.st_uid, sb.st_gid); 610164152Sdes chmod(tmppath, sb.st_mode & ALLPERMS); 611100834Sdes } 61283217Sdes } 613100834Sdes if (of == NULL) 61483217Sdes of = fopen(path, "w"); 61583217Sdes if (of == NULL) { 61679837Sdes warn("%s: open()", path); 61779837Sdes goto failure; 61879837Sdes } 61979837Sdes } 62079837Sdes count = url->offset; 62162216Sdes 62279837Sdes /* start the counter */ 62379837Sdes stat_start(&xs, path, us.size, count); 62463046Sdes 62579837Sdes sigalrm = siginfo = sigint = 0; 62679837Sdes 62779837Sdes /* suck in the data */ 62879837Sdes signal(SIGINFO, sig_handler); 629106041Sdes while (!sigint) { 630137854Scperciva if (us.size != -1 && us.size - count < B_size && 631137854Scperciva us.size - count >= 0) 63279837Sdes size = us.size - count; 63379837Sdes else 63479837Sdes size = B_size; 63579837Sdes if (siginfo) { 63679837Sdes stat_end(&xs); 63779837Sdes siginfo = 0; 63879837Sdes } 63979837Sdes if ((size = fread(buf, 1, size, f)) == 0) { 640106041Sdes if (ferror(f) && errno == EINTR && !sigint) 64179837Sdes clearerr(f); 64279837Sdes else 64379837Sdes break; 64479837Sdes } 64579837Sdes stat_update(&xs, count += size); 64679837Sdes for (ptr = buf; size > 0; ptr += wr, size -= wr) 64779837Sdes if ((wr = fwrite(ptr, 1, size, of)) < size) { 648106041Sdes if (ferror(of) && errno == EINTR && !sigint) 64979837Sdes clearerr(of); 65079837Sdes else 65179837Sdes break; 65279837Sdes } 65379837Sdes if (size != 0) 65479837Sdes break; 65577241Sdes } 656106041Sdes if (!sigalrm) 657106041Sdes sigalrm = ferror(f) && errno == ETIMEDOUT; 65879837Sdes signal(SIGINFO, SIG_DFL); 65979837Sdes 66079837Sdes stat_end(&xs); 66162216Sdes 662106041Sdes /* 663106041Sdes * If the transfer timed out or was interrupted, we still want to 664106041Sdes * set the mtime in case the file is not removed (-r or -R) and 665106041Sdes * the user later restarts the transfer. 666106041Sdes */ 667106041Sdes signal: 66879837Sdes /* set mtime of local file */ 669106857Sdes if (!n_flag && us.mtime && !o_stdout && of != NULL && 670106857Sdes (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { 67179837Sdes struct timeval tv[2]; 672106043Sdes 67379837Sdes fflush(of); 67479837Sdes tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 67579837Sdes tv[1].tv_sec = (long)us.mtime; 67679837Sdes tv[0].tv_usec = tv[1].tv_usec = 0; 67790729Sdes if (utimes(tmppath ? tmppath : path, tv)) 67890729Sdes warn("%s: utimes()", tmppath ? tmppath : path); 67979837Sdes } 68063015Sdes 68179837Sdes /* timed out or interrupted? */ 68279837Sdes if (sigalrm) 68379837Sdes warnx("transfer timed out"); 68479837Sdes if (sigint) { 68579837Sdes warnx("transfer interrupted"); 68679837Sdes goto failure; 68779837Sdes } 68863046Sdes 689106586Sfenner /* timeout / interrupt before connection completley established? */ 690106586Sfenner if (f == NULL) 691106586Sfenner goto failure; 692106586Sfenner 69379837Sdes if (!sigalrm) { 69479837Sdes /* check the status of our files */ 69579837Sdes if (ferror(f)) 69679837Sdes warn("%s", URL); 69779837Sdes if (ferror(of)) 69879837Sdes warn("%s", path); 69979837Sdes if (ferror(f) || ferror(of)) 70079837Sdes goto failure; 70179837Sdes } 70279837Sdes 70379837Sdes /* did the transfer complete normally? */ 70479837Sdes if (us.size != -1 && count < us.size) { 705125976Sdes warnx("%s appears to be truncated: %jd/%jd bytes", 706125976Sdes path, (intmax_t)count, (intmax_t)us.size); 70779837Sdes goto failure_keep; 70879837Sdes } 70979837Sdes 71079837Sdes /* 71179837Sdes * If the transfer timed out and we didn't know how much to 71279837Sdes * expect, assume the worst (i.e. we didn't get all of it) 71379837Sdes */ 71479837Sdes if (sigalrm && us.size == -1) { 71579837Sdes warnx("%s may be truncated", path); 71679837Sdes goto failure_keep; 71779837Sdes } 71879837Sdes 71962216Sdes success: 72079837Sdes r = 0; 72183217Sdes if (tmppath != NULL && rename(tmppath, path) == -1) { 72283217Sdes warn("%s: rename()", path); 72383217Sdes goto failure_keep; 72483217Sdes } 72579837Sdes goto done; 72662216Sdes failure: 72779837Sdes if (of && of != stdout && !R_flag && !r_flag) 72879837Sdes if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) 72983217Sdes unlink(tmppath ? tmppath : path); 73083217Sdes if (R_flag && tmppath != NULL && sb.st_size == -1) 73183217Sdes rename(tmppath, path); /* ignore errors here */ 73263046Sdes failure_keep: 73379837Sdes r = -1; 73479837Sdes goto done; 73562216Sdes done: 73679837Sdes if (f) 73779837Sdes fclose(f); 73879837Sdes if (of && of != stdout) 73979837Sdes fclose(of); 74079837Sdes if (url) 74179837Sdes fetchFreeURL(url); 74283217Sdes if (tmppath != NULL) 74383217Sdes free(tmppath); 744132695Sdes return (r); 74562216Sdes} 74662216Sdes 74779837Sdesstatic void 74862216Sdesusage(void) 74962216Sdes{ 750186043Sru fprintf(stderr, "%s\n%s\n%s\n%s\n", 751186043Sru"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", 752186124Smurray" [-T seconds] [-w seconds] [-i file] URL ...", 753186043Sru" fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", 754186124Smurray" [-T seconds] [-w seconds] [-i file] -h host -f file [-c dir]"); 75562216Sdes} 75662216Sdes 75762216Sdes 75881863Sdes/* 75981863Sdes * Entry point 76081863Sdes */ 76162216Sdesint 76262216Sdesmain(int argc, char *argv[]) 76362216Sdes{ 76479837Sdes struct stat sb; 76579837Sdes struct sigaction sa; 76679837Sdes const char *p, *s; 767100834Sdes char *end, *q; 76879837Sdes int c, e, r; 76962216Sdes 77079837Sdes while ((c = getopt(argc, argv, 771186124Smurray "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) 77279837Sdes switch (c) { 77379837Sdes case '1': 77479837Sdes once_flag = 1; 77579837Sdes break; 77679837Sdes case '4': 77779837Sdes family = PF_INET; 77879837Sdes break; 77979837Sdes case '6': 78079837Sdes family = PF_INET6; 78179837Sdes break; 78279837Sdes case 'A': 78379837Sdes A_flag = 1; 78479837Sdes break; 78579837Sdes case 'a': 78679837Sdes a_flag = 1; 78779837Sdes break; 78879837Sdes case 'B': 789100834Sdes B_size = (off_t)strtol(optarg, &end, 10); 790100834Sdes if (*optarg == '\0' || *end != '\0') 79180521Sse errx(1, "invalid buffer size (%s)", optarg); 79279837Sdes break; 79379837Sdes case 'b': 79479837Sdes warnx("warning: the -b option is deprecated"); 79579837Sdes b_flag = 1; 79679837Sdes break; 79779837Sdes case 'c': 79879837Sdes c_dirname = optarg; 79979837Sdes break; 80079837Sdes case 'd': 80179837Sdes d_flag = 1; 80279837Sdes break; 80379837Sdes case 'F': 80479837Sdes F_flag = 1; 80579837Sdes break; 80679837Sdes case 'f': 80779837Sdes f_filename = optarg; 80879837Sdes break; 80979837Sdes case 'H': 81093213Scharnier warnx("the -H option is now implicit, " 81179837Sdes "use -U to disable"); 81279837Sdes break; 81379837Sdes case 'h': 81479837Sdes h_hostname = optarg; 81579837Sdes break; 816186124Smurray case 'i': 817186124Smurray i_flag = 1; 818186124Smurray i_filename = optarg; 819186124Smurray break; 82079837Sdes case 'l': 82179837Sdes l_flag = 1; 82279837Sdes break; 82379837Sdes case 'o': 82479837Sdes o_flag = 1; 82579837Sdes o_filename = optarg; 82679837Sdes break; 82779837Sdes case 'M': 82879837Sdes case 'm': 82979837Sdes if (r_flag) 83079837Sdes errx(1, "the -m and -r flags " 83179837Sdes "are mutually exclusive"); 83279837Sdes m_flag = 1; 83379837Sdes break; 834109702Sdes case 'N': 835109702Sdes N_filename = optarg; 836109702Sdes break; 83779837Sdes case 'n': 83879837Sdes n_flag = 1; 83979837Sdes break; 84079837Sdes case 'P': 84179837Sdes case 'p': 84279837Sdes p_flag = 1; 84379837Sdes break; 84479837Sdes case 'q': 84579837Sdes v_level = 0; 84679837Sdes break; 84779837Sdes case 'R': 84879837Sdes R_flag = 1; 84979837Sdes break; 85079837Sdes case 'r': 85179837Sdes if (m_flag) 85279837Sdes errx(1, "the -m and -r flags " 85379837Sdes "are mutually exclusive"); 85479837Sdes r_flag = 1; 85579837Sdes break; 85679837Sdes case 'S': 857100834Sdes S_size = (off_t)strtol(optarg, &end, 10); 858100834Sdes if (*optarg == '\0' || *end != '\0') 85980521Sse errx(1, "invalid size (%s)", optarg); 86079837Sdes break; 86179837Sdes case 's': 86279837Sdes s_flag = 1; 86379837Sdes break; 864106043Sdes case 'T': 865100834Sdes T_secs = strtol(optarg, &end, 10); 866100834Sdes if (*optarg == '\0' || *end != '\0') 86780521Sse errx(1, "invalid timeout (%s)", optarg); 86879837Sdes break; 86979837Sdes case 't': 87079837Sdes t_flag = 1; 87179837Sdes warnx("warning: the -t option is deprecated"); 87279837Sdes break; 87379837Sdes case 'U': 87479837Sdes U_flag = 1; 87579837Sdes break; 87679837Sdes case 'v': 87779837Sdes v_level++; 87879837Sdes break; 87979837Sdes case 'w': 88079837Sdes a_flag = 1; 881100834Sdes w_secs = strtol(optarg, &end, 10); 882100834Sdes if (*optarg == '\0' || *end != '\0') 88380521Sse errx(1, "invalid delay (%s)", optarg); 88479837Sdes break; 88579837Sdes default: 88679837Sdes usage(); 887186241Smurray exit(1); 88879837Sdes } 88962216Sdes 89079837Sdes argc -= optind; 89179837Sdes argv += optind; 89262216Sdes 89379837Sdes if (h_hostname || f_filename || c_dirname) { 89479837Sdes if (!h_hostname || !f_filename || argc) { 89579837Sdes usage(); 896186241Smurray exit(1); 89779837Sdes } 89879837Sdes /* XXX this is a hack. */ 89979837Sdes if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 90079837Sdes errx(1, "invalid hostname"); 90179837Sdes if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 90279837Sdes c_dirname ? c_dirname : "", f_filename) == -1) 90379837Sdes errx(1, "%s", strerror(ENOMEM)); 90479837Sdes argc++; 90562216Sdes } 90663345Sdes 90779837Sdes if (!argc) { 90879837Sdes usage(); 909186241Smurray exit(1); 91079837Sdes } 91162216Sdes 91279837Sdes /* allocate buffer */ 91379837Sdes if (B_size < MINBUFSIZE) 91479837Sdes B_size = MINBUFSIZE; 91579837Sdes if ((buf = malloc(B_size)) == NULL) 91679837Sdes errx(1, "%s", strerror(ENOMEM)); 91762216Sdes 91879837Sdes /* timeouts */ 91979837Sdes if ((s = getenv("FTP_TIMEOUT")) != NULL) { 920100834Sdes ftp_timeout = strtol(s, &end, 10); 921102478Sdes if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { 922102478Sdes warnx("FTP_TIMEOUT (%s) is not a positive integer", s); 92379837Sdes ftp_timeout = 0; 92479837Sdes } 92562216Sdes } 92679837Sdes if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 927100834Sdes http_timeout = strtol(s, &end, 10); 928102478Sdes if (*s == '\0' || *end != '\0' || http_timeout < 0) { 929102478Sdes warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); 93079837Sdes http_timeout = 0; 93179837Sdes } 93262216Sdes } 93362216Sdes 93479837Sdes /* signal handling */ 93579837Sdes sa.sa_flags = 0; 93679837Sdes sa.sa_handler = sig_handler; 93779837Sdes sigemptyset(&sa.sa_mask); 93879837Sdes sigaction(SIGALRM, &sa, NULL); 93979837Sdes sa.sa_flags = SA_RESETHAND; 94079837Sdes sigaction(SIGINT, &sa, NULL); 94179837Sdes fetchRestartCalls = 0; 94279837Sdes 94379837Sdes /* output file */ 94479837Sdes if (o_flag) { 94579837Sdes if (strcmp(o_filename, "-") == 0) { 94679837Sdes o_stdout = 1; 94779837Sdes } else if (stat(o_filename, &sb) == -1) { 94879837Sdes if (errno == ENOENT) { 94979837Sdes if (argc > 1) 950186241Smurray errx(1, "%s is not a directory", 95179837Sdes o_filename); 95279837Sdes } else { 953186241Smurray err(1, "%s", o_filename); 95479837Sdes } 95579837Sdes } else { 95679837Sdes if (sb.st_mode & S_IFDIR) 95779837Sdes o_directory = 1; 95879837Sdes } 95962216Sdes } 96062216Sdes 96179837Sdes /* check if output is to a tty (for progress report) */ 96279837Sdes v_tty = isatty(STDERR_FILENO); 96383863Sdes if (v_tty) 96483863Sdes pgrp = getpgrp(); 965106043Sdes 96679837Sdes r = 0; 967106043Sdes 96879837Sdes /* authentication */ 96979837Sdes if (v_tty) 97079837Sdes fetchAuthMethod = query_auth; 971109702Sdes if (N_filename != NULL) 972109702Sdes setenv("NETRC", N_filename, 1); 97362216Sdes 97479837Sdes while (argc) { 97579837Sdes if ((p = strrchr(*argv, '/')) == NULL) 97679837Sdes p = *argv; 97779837Sdes else 97879837Sdes p++; 97962216Sdes 98079837Sdes if (!*p) 98179837Sdes p = "fetch.out"; 982106043Sdes 98379837Sdes fetchLastErrCode = 0; 984106043Sdes 98579837Sdes if (o_flag) { 98679837Sdes if (o_stdout) { 98779837Sdes e = fetch(*argv, "-"); 98879837Sdes } else if (o_directory) { 98979837Sdes asprintf(&q, "%s/%s", o_filename, p); 99079837Sdes e = fetch(*argv, q); 99179837Sdes free(q); 99279837Sdes } else { 99379837Sdes e = fetch(*argv, o_filename); 99479837Sdes } 99579837Sdes } else { 99679837Sdes e = fetch(*argv, p); 99779837Sdes } 99862216Sdes 99979837Sdes if (sigint) 100079837Sdes kill(getpid(), SIGINT); 1001106043Sdes 100279837Sdes if (e == 0 && once_flag) 100379837Sdes exit(0); 1004106043Sdes 100579837Sdes if (e) { 100679837Sdes r = 1; 100779837Sdes if ((fetchLastErrCode 100879837Sdes && fetchLastErrCode != FETCH_UNAVAIL 100979837Sdes && fetchLastErrCode != FETCH_MOVED 101079837Sdes && fetchLastErrCode != FETCH_URL 101179837Sdes && fetchLastErrCode != FETCH_RESOLV 101279837Sdes && fetchLastErrCode != FETCH_UNKNOWN)) { 101379837Sdes if (w_secs && v_level) 1014100834Sdes fprintf(stderr, "Waiting %ld seconds " 101579837Sdes "before retrying\n", w_secs); 101679837Sdes if (w_secs) 101779837Sdes sleep(w_secs); 101879837Sdes if (a_flag) 101979837Sdes continue; 102079837Sdes } 102162216Sdes } 102279837Sdes 102379837Sdes argc--, argv++; 102462216Sdes } 102562216Sdes 102679837Sdes exit(r); 102762216Sdes} 1028