fetch.c revision 107353
162216Sdes/*- 262216Sdes * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav 362216Sdes * All rights reserved. 462216Sdes * 562216Sdes * Redistribution and use in source and binary forms, with or without 662216Sdes * modification, are permitted provided that the following conditions 762216Sdes * are met: 862216Sdes * 1. Redistributions of source code must retain the above copyright 962216Sdes * notice, this list of conditions and the following disclaimer 1062216Sdes * in this position and unchanged. 1162216Sdes * 2. Redistributions in binary form must reproduce the above copyright 1262216Sdes * notice, this list of conditions and the following disclaimer in the 1362216Sdes * documentation and/or other materials provided with the distribution. 1462216Sdes * 3. The name of the author may not be used to endorse or promote products 1562216Sdes * derived from this software without specific prior written permission 1662216Sdes * 1762216Sdes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1862216Sdes * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1962216Sdes * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2062216Sdes * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2162216Sdes * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2262216Sdes * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2362216Sdes * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2462216Sdes * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2562216Sdes * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2662216Sdes * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2762216Sdes */ 2862216Sdes 2993213Scharnier#include <sys/cdefs.h> 3093213Scharnier__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 107353 2002-11-27 20:52:07Z des $"); 3193213Scharnier 3262216Sdes#include <sys/param.h> 3391225Sbde#include <sys/socket.h> 3462216Sdes#include <sys/stat.h> 3593257Sbde#include <sys/time.h> 3662216Sdes 3762216Sdes#include <ctype.h> 3862216Sdes#include <err.h> 3962216Sdes#include <errno.h> 4063235Sdes#include <signal.h> 4162216Sdes#include <stdio.h> 4262216Sdes#include <stdlib.h> 4362216Sdes#include <string.h> 4462216Sdes#include <sysexits.h> 4577241Sdes#include <termios.h> 4662216Sdes#include <unistd.h> 4762216Sdes 4862216Sdes#include <fetch.h> 4962216Sdes 5062216Sdes#define MINBUFSIZE 4096 5162216Sdes 5262216Sdes/* Option flags */ 5362216Sdesint A_flag; /* -A: do not follow 302 redirects */ 5462216Sdesint a_flag; /* -a: auto retry */ 5579837Sdesoff_t B_size; /* -B: buffer size */ 5662216Sdesint b_flag; /*! -b: workaround TCP bug */ 5762254Sdeschar *c_dirname; /* -c: remote directory */ 5862216Sdesint d_flag; /* -d: direct connection */ 5962216Sdesint F_flag; /* -F: restart without checking mtime */ 6062216Sdeschar *f_filename; /* -f: file to fetch */ 6162216Sdeschar *h_hostname; /* -h: host to fetch from */ 6262216Sdesint l_flag; /* -l: link rather than copy file: URLs */ 6362815Sdesint m_flag; /* -[Mm]: mirror mode */ 6462815Sdesint n_flag; /* -n: do not preserve modification time */ 6562216Sdesint o_flag; /* -o: specify output file */ 6662216Sdesint o_directory; /* output file is a directory */ 6762216Sdeschar *o_filename; /* name of output file */ 6862216Sdesint o_stdout; /* output file is stdout */ 6962216Sdesint once_flag; /* -1: stop at first successful file */ 7063501Sdesint p_flag; /* -[Pp]: use passive FTP */ 7162216Sdesint R_flag; /* -R: don't delete partially transferred files */ 7262216Sdesint r_flag; /* -r: restart previously interrupted transfer */ 7374717Sdesoff_t S_size; /* -S: require size to match */ 7474717Sdesint s_flag; /* -s: show size, don't fetch */ 75100834Sdeslong T_secs = 120; /* -T: transfer timeout in seconds */ 7662216Sdesint t_flag; /*! -t: workaround TCP bug */ 7774717Sdesint U_flag; /* -U: do not use high ports */ 7862216Sdesint v_level = 1; /* -v: verbosity level */ 7962216Sdesint v_tty; /* stdout is a tty */ 8083863Sdespid_t pgrp; /* our process group */ 81100834Sdeslong w_secs; /* -w: retry delay */ 8262216Sdesint family = PF_UNSPEC; /* -[46]: address family to use */ 8362216Sdes 8463015Sdesint sigalrm; /* SIGALRM received */ 8573937Sdesint siginfo; /* SIGINFO received */ 8663015Sdesint sigint; /* SIGINT received */ 8762216Sdes 88100834Sdeslong ftp_timeout; /* default timeout for FTP transfers */ 89100834Sdeslong http_timeout; /* default timeout for HTTP transfers */ 9062216Sdesu_char *buf; /* transfer buffer */ 9162216Sdes 9262216Sdes 9381863Sdes/* 9481863Sdes * Signal handler 9581863Sdes */ 9679837Sdesstatic void 9762216Sdessig_handler(int sig) 9862216Sdes{ 9979837Sdes switch (sig) { 10079837Sdes case SIGALRM: 10179837Sdes sigalrm = 1; 10279837Sdes break; 10379837Sdes case SIGINFO: 10479837Sdes siginfo = 1; 10579837Sdes break; 10679837Sdes case SIGINT: 10779837Sdes sigint = 1; 10879837Sdes break; 10979837Sdes } 11062216Sdes} 11162216Sdes 11262216Sdesstruct xferstat { 11379837Sdes char name[40]; 11479837Sdes struct timeval start; 11579837Sdes struct timeval end; 11679837Sdes struct timeval last; 11779837Sdes off_t size; 11879837Sdes off_t offset; 11979837Sdes off_t rcvd; 12062216Sdes}; 12162216Sdes 12281863Sdes/* 12381863Sdes * Update the stats display 12481863Sdes */ 12579837Sdesstatic void 12663046Sdesstat_display(struct xferstat *xs, int force) 12762216Sdes{ 12879837Sdes struct timeval now; 12983863Sdes int ctty_pgrp; 13079837Sdes 13179837Sdes if (!v_tty || !v_level) 13279837Sdes return; 13379837Sdes 13483863Sdes /* check if we're the foreground process */ 13583863Sdes if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || 13683863Sdes (pid_t)ctty_pgrp != pgrp) 13783863Sdes return; 138106043Sdes 13979837Sdes gettimeofday(&now, NULL); 14079837Sdes if (!force && now.tv_sec <= xs->last.tv_sec) 14179837Sdes return; 14279837Sdes xs->last = now; 14379837Sdes 14479837Sdes fprintf(stderr, "\rReceiving %s", xs->name); 145106041Sdes if (xs->size <= 0) { 14679837Sdes fprintf(stderr, ": %lld bytes", (long long)xs->rcvd); 147106041Sdes } else { 148106041Sdes long elapsed; 149106041Sdes 15079837Sdes fprintf(stderr, " (%lld bytes): %d%%", (long long)xs->size, 15179837Sdes (int)((100.0 * xs->rcvd) / xs->size)); 152106041Sdes elapsed = xs->last.tv_sec - xs->start.tv_sec; 153106796Sfenner if (elapsed > 30 && xs->rcvd > 0) { 154106041Sdes long remaining; 155106041Sdes 156106041Sdes remaining = ((xs->size * elapsed) / xs->rcvd) - elapsed; 157106041Sdes fprintf(stderr, " (ETA "); 158106041Sdes if (remaining > 3600) { 159106041Sdes fprintf(stderr, "%02ld:", remaining / 3600); 160106041Sdes remaining %= 3600; 161106041Sdes } 162106041Sdes fprintf(stderr, "%02ld:%02ld) ", 163106041Sdes remaining / 60, remaining % 60); 164106041Sdes } 165106041Sdes } 16662216Sdes} 16762216Sdes 16881863Sdes/* 16981863Sdes * Initialize the transfer statistics 17081863Sdes */ 17179837Sdesstatic void 17279837Sdesstat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) 17363046Sdes{ 17479837Sdes snprintf(xs->name, sizeof xs->name, "%s", name); 17579837Sdes gettimeofday(&xs->start, NULL); 17679837Sdes xs->last.tv_sec = xs->last.tv_usec = 0; 17779837Sdes xs->end = xs->last; 17879837Sdes xs->size = size; 17979837Sdes xs->offset = offset; 18079837Sdes xs->rcvd = offset; 18179837Sdes stat_display(xs, 1); 18263046Sdes} 18363046Sdes 18481863Sdes/* 18581863Sdes * Update the transfer statistics 18681863Sdes */ 18779837Sdesstatic void 18879837Sdesstat_update(struct xferstat *xs, off_t rcvd) 18963046Sdes{ 19079837Sdes xs->rcvd = rcvd; 19179837Sdes stat_display(xs, 0); 19263046Sdes} 19363046Sdes 19481863Sdes/* 19581863Sdes * Finalize the transfer statistics 19681863Sdes */ 19779837Sdesstatic void 19862216Sdesstat_end(struct xferstat *xs) 19962216Sdes{ 20079837Sdes double delta; 20179837Sdes double bps; 20263720Sdes 20379837Sdes if (!v_level) 20479837Sdes return; 20579837Sdes 20679837Sdes gettimeofday(&xs->end, NULL); 20779837Sdes 20879837Sdes stat_display(xs, 1); 20979837Sdes fputc('\n', stderr); 21079837Sdes delta = (xs->end.tv_sec + (xs->end.tv_usec / 1.e6)) 21179837Sdes - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 21279837Sdes fprintf(stderr, "%lld bytes transferred in %.1f seconds ", 21379837Sdes (long long)(xs->rcvd - xs->offset), delta); 21479837Sdes bps = (xs->rcvd - xs->offset) / delta; 21579837Sdes if (bps > 1024*1024) 21679837Sdes fprintf(stderr, "(%.2f MBps)\n", bps / (1024*1024)); 21779837Sdes else if (bps > 1024) 21879837Sdes fprintf(stderr, "(%.2f kBps)\n", bps / 1024); 21979837Sdes else 22079837Sdes fprintf(stderr, "(%.2f Bps)\n", bps); 22162216Sdes} 22262216Sdes 22381863Sdes/* 22481863Sdes * Ask the user for authentication details 22581863Sdes */ 22679837Sdesstatic int 22777241Sdesquery_auth(struct url *URL) 22877241Sdes{ 22979837Sdes struct termios tios; 23079837Sdes tcflag_t saved_flags; 23179837Sdes int i, nopwd; 23277241Sdes 23379837Sdes 23479837Sdes fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", 23586242Siedowse URL->scheme, URL->host, URL->port); 23679837Sdes 23779837Sdes fprintf(stderr, "Login: "); 23879837Sdes if (fgets(URL->user, sizeof URL->user, stdin) == NULL) 23979837Sdes return -1; 24079837Sdes for (i = 0; URL->user[i]; ++i) 24179837Sdes if (isspace(URL->user[i])) 24279837Sdes URL->user[i] = '\0'; 24379837Sdes 24479837Sdes fprintf(stderr, "Password: "); 24579837Sdes if (tcgetattr(STDIN_FILENO, &tios) == 0) { 24679837Sdes saved_flags = tios.c_lflag; 24779837Sdes tios.c_lflag &= ~ECHO; 24879837Sdes tios.c_lflag |= ECHONL|ICANON; 24979837Sdes tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); 25079837Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 25179837Sdes tios.c_lflag = saved_flags; 25279837Sdes tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); 25379837Sdes } else { 25479837Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 25579837Sdes } 25679837Sdes if (nopwd) 25779837Sdes return -1; 25879837Sdes 25979837Sdes for (i = 0; URL->pwd[i]; ++i) 26079837Sdes if (isspace(URL->pwd[i])) 26179837Sdes URL->pwd[i] = '\0'; 26279837Sdes return 0; 26377241Sdes} 26477241Sdes 26581863Sdes/* 26681863Sdes * Fetch a file 26781863Sdes */ 26879837Sdesstatic int 26979837Sdesfetch(char *URL, const char *path) 27062216Sdes{ 27179837Sdes struct url *url; 27279837Sdes struct url_stat us; 27383217Sdes struct stat sb, nsb; 27479837Sdes struct xferstat xs; 27579837Sdes FILE *f, *of; 27679837Sdes size_t size, wr; 27779837Sdes off_t count; 27879837Sdes char flags[8]; 27983217Sdes const char *slash; 28083217Sdes char *tmppath; 28179837Sdes int r; 28279837Sdes u_int timeout; 28379837Sdes u_char *ptr; 28462216Sdes 28579837Sdes f = of = NULL; 28683217Sdes tmppath = NULL; 28762216Sdes 28879837Sdes /* parse URL */ 28979837Sdes if ((url = fetchParseURL(URL)) == NULL) { 29079837Sdes warnx("%s: parse error", URL); 29179837Sdes goto failure; 29279837Sdes } 29362216Sdes 29479837Sdes /* if no scheme was specified, take a guess */ 29579837Sdes if (!*url->scheme) { 29679837Sdes if (!*url->host) 29779837Sdes strcpy(url->scheme, SCHEME_FILE); 29879837Sdes else if (strncasecmp(url->host, "ftp.", 4) == 0) 29979837Sdes strcpy(url->scheme, SCHEME_FTP); 30079837Sdes else if (strncasecmp(url->host, "www.", 4) == 0) 30179837Sdes strcpy(url->scheme, SCHEME_HTTP); 30279837Sdes } 30369976Sdes 30479837Sdes timeout = 0; 30579837Sdes *flags = 0; 30679837Sdes count = 0; 30762216Sdes 30879837Sdes /* common flags */ 30979837Sdes if (v_level > 1) 31079837Sdes strcat(flags, "v"); 31187563Sdes if (v_level > 2) 31287563Sdes fetchDebug = 1; 31379837Sdes switch (family) { 31479837Sdes case PF_INET: 31579837Sdes strcat(flags, "4"); 31679837Sdes break; 31779837Sdes case PF_INET6: 31879837Sdes strcat(flags, "6"); 31979837Sdes break; 32079837Sdes } 32162216Sdes 32279837Sdes /* FTP specific flags */ 32379837Sdes if (strcmp(url->scheme, "ftp") == 0) { 32479837Sdes if (p_flag) 32579837Sdes strcat(flags, "p"); 32679837Sdes if (d_flag) 32779837Sdes strcat(flags, "d"); 32879837Sdes if (U_flag) 32979837Sdes strcat(flags, "l"); 33079837Sdes timeout = T_secs ? T_secs : ftp_timeout; 33179837Sdes } 33262216Sdes 33379837Sdes /* HTTP specific flags */ 33479837Sdes if (strcmp(url->scheme, "http") == 0) { 33579837Sdes if (d_flag) 33679837Sdes strcat(flags, "d"); 33779837Sdes if (A_flag) 33879837Sdes strcat(flags, "A"); 33979837Sdes timeout = T_secs ? T_secs : http_timeout; 34079837Sdes } 34162216Sdes 34279837Sdes /* set the protocol timeout. */ 34379837Sdes fetchTimeout = timeout; 34462216Sdes 34579837Sdes /* just print size */ 34679837Sdes if (s_flag) { 347106041Sdes if (timeout) 348106041Sdes alarm(timeout); 349106041Sdes r = fetchStat(url, &us, flags); 350106041Sdes if (timeout) 351106043Sdes alarm(0); 352106041Sdes if (sigalrm || sigint) 353106041Sdes goto signal; 354106041Sdes if (r == -1) { 355106041Sdes warnx("%s", fetchLastErrString); 35679837Sdes goto failure; 357106041Sdes } 35879837Sdes if (us.size == -1) 35979837Sdes printf("Unknown\n"); 36079837Sdes else 36179837Sdes printf("%lld\n", (long long)us.size); 36279837Sdes goto success; 36363345Sdes } 36479837Sdes 36579837Sdes /* 36679837Sdes * If the -r flag was specified, we have to compare the local 36779837Sdes * and remote files, so we should really do a fetchStat() 36879837Sdes * first, but I know of at least one HTTP server that only 36979837Sdes * sends the content size in response to GET requests, and 37079837Sdes * leaves it out of replies to HEAD requests. Also, in the 37179837Sdes * (frequent) case that the local and remote files match but 37279837Sdes * the local file is truncated, we have sufficient information 37379837Sdes * before the compare to issue a correct request. Therefore, 37479837Sdes * we always issue a GET request as if we were sure the local 37579837Sdes * file was a truncated copy of the remote file; we can drop 37679837Sdes * the connection later if we change our minds. 37779837Sdes */ 37883217Sdes sb.st_size = -1; 37983217Sdes if (!o_stdout && stat(path, &sb) == -1 && errno != ENOENT) { 38083217Sdes warnx("%s: stat()", path); 38183217Sdes goto failure; 38262216Sdes } 38383217Sdes if (!o_stdout && r_flag && S_ISREG(sb.st_mode)) 38483217Sdes url->offset = sb.st_size; 38562216Sdes 38679837Sdes /* start the transfer */ 387106041Sdes if (timeout) 388106041Sdes alarm(timeout); 389106041Sdes f = fetchXGet(url, &us, flags); 390106042Sdes if (timeout) 391106042Sdes alarm(0); 392106041Sdes if (sigalrm || sigint) 393106041Sdes goto signal; 394106041Sdes if (f == NULL) { 395107353Sdes warnx("%s: %s", URL, fetchLastErrString); 39663345Sdes goto failure; 39779837Sdes } 39879837Sdes if (sigint) 39963345Sdes goto signal; 40079837Sdes 40179837Sdes /* check that size is as expected */ 40279837Sdes if (S_size) { 40379837Sdes if (us.size == -1) { 404107353Sdes warnx("%s: size unknown", URL); 40579837Sdes goto failure; 40679837Sdes } else if (us.size != S_size) { 40779837Sdes warnx("%s: size mismatch: expected %lld, actual %lld", 408107353Sdes URL, (long long)S_size, (long long)us.size); 40979837Sdes goto failure; 41079837Sdes } 41179837Sdes } 41279837Sdes 41379837Sdes /* symlink instead of copy */ 41479837Sdes if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 41579837Sdes if (symlink(url->doc, path) == -1) { 41679837Sdes warn("%s: symlink()", path); 41779837Sdes goto failure; 41879837Sdes } 41963568Sdes goto success; 42063345Sdes } 42179837Sdes 422106051Sdes if (us.size == -1 && !o_stdout && v_level > 0) 423107353Sdes warnx("%s: size of remote file is not known", URL); 42479837Sdes if (v_level > 1) { 42579837Sdes if (sb.st_size != -1) 42679837Sdes fprintf(stderr, "local size / mtime: %lld / %ld\n", 42779837Sdes (long long)sb.st_size, (long)sb.st_mtime); 42879837Sdes if (us.size != -1) 42979837Sdes fprintf(stderr, "remote size / mtime: %lld / %ld\n", 43079837Sdes (long long)us.size, (long)us.mtime); 43162216Sdes } 43262216Sdes 43379837Sdes /* open output file */ 43479837Sdes if (o_stdout) { 43579837Sdes /* output to stdout */ 43679837Sdes of = stdout; 43783217Sdes } else if (r_flag && sb.st_size != -1) { 43879837Sdes /* resume mode, local file exists */ 43979837Sdes if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { 44079837Sdes /* no match! have to refetch */ 44179837Sdes fclose(f); 44279837Sdes /* if precious, warn the user and give up */ 44379837Sdes if (R_flag) { 44479837Sdes warnx("%s: local modification time " 44579837Sdes "does not match remote", path); 44679837Sdes goto failure_keep; 44779837Sdes } 44879837Sdes } else { 44979837Sdes if (us.size == sb.st_size) 45079837Sdes /* nothing to do */ 45179837Sdes goto success; 45279837Sdes if (sb.st_size > us.size) { 45379837Sdes /* local file too long! */ 45479837Sdes warnx("%s: local file (%lld bytes) is longer " 45579837Sdes "than remote file (%lld bytes)", path, 45679837Sdes (long long)sb.st_size, (long long)us.size); 45779837Sdes goto failure; 45879837Sdes } 45983217Sdes /* we got it, open local file */ 46079837Sdes if ((of = fopen(path, "a")) == NULL) { 46179837Sdes warn("%s: fopen()", path); 46279837Sdes goto failure; 46379837Sdes } 46483217Sdes /* check that it didn't move under our feet */ 46583217Sdes if (fstat(fileno(of), &nsb) == -1) { 46683217Sdes /* can't happen! */ 46783217Sdes warn("%s: fstat()", path); 46879837Sdes goto failure; 46979837Sdes } 47083217Sdes if (nsb.st_dev != sb.st_dev || 47183217Sdes nsb.st_ino != nsb.st_ino || 47283217Sdes nsb.st_size != sb.st_size) { 473107353Sdes warnx("%s: file has changed", URL); 47483217Sdes fclose(of); 47583217Sdes of = NULL; 47683217Sdes sb = nsb; 47783217Sdes } 47879837Sdes } 47983217Sdes } else if (m_flag && sb.st_size != -1) { 48079837Sdes /* mirror mode, local file exists */ 48179837Sdes if (sb.st_size == us.size && sb.st_mtime == us.mtime) 48279837Sdes goto success; 48379837Sdes } 484106043Sdes 48583217Sdes if (of == NULL) { 48679837Sdes /* 48779837Sdes * We don't yet have an output file; either this is a 48879837Sdes * vanilla run with no special flags, or the local and 48979837Sdes * remote files didn't match. 49079837Sdes */ 491106043Sdes 49283217Sdes if (url->offset != 0) { 49383217Sdes /* 49483217Sdes * We tried to restart a transfer, but for 49583217Sdes * some reason gave up - so we have to restart 49683217Sdes * from scratch if we want the whole file 49783217Sdes */ 49883217Sdes url->offset = 0; 49983217Sdes if ((f = fetchXGet(url, &us, flags)) == NULL) { 500107353Sdes warnx("%s: %s", URL, fetchLastErrString); 50183217Sdes goto failure; 50283217Sdes } 50383217Sdes if (sigint) 50483217Sdes goto signal; 50583217Sdes } 50683217Sdes 50783217Sdes /* construct a temp file name */ 50883217Sdes if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { 50983217Sdes if ((slash = strrchr(path, '/')) == NULL) 51083217Sdes slash = path; 51183217Sdes else 51283217Sdes ++slash; 51383217Sdes asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", 51483307Smike (int)(slash - path), path, slash); 515100834Sdes if (tmppath != NULL) { 516100834Sdes mkstemps(tmppath, strlen(slash) + 1); 517100834Sdes of = fopen(tmppath, "w"); 518100834Sdes } 51983217Sdes } 520100834Sdes if (of == NULL) 52183217Sdes of = fopen(path, "w"); 52283217Sdes if (of == NULL) { 52379837Sdes warn("%s: open()", path); 52479837Sdes goto failure; 52579837Sdes } 52679837Sdes } 52779837Sdes count = url->offset; 52862216Sdes 52979837Sdes /* start the counter */ 53079837Sdes stat_start(&xs, path, us.size, count); 53163046Sdes 53279837Sdes sigalrm = siginfo = sigint = 0; 53379837Sdes 53479837Sdes /* suck in the data */ 53579837Sdes signal(SIGINFO, sig_handler); 536106041Sdes while (!sigint) { 53779837Sdes if (us.size != -1 && us.size - count < B_size) 53879837Sdes size = us.size - count; 53979837Sdes else 54079837Sdes size = B_size; 54179837Sdes if (siginfo) { 54279837Sdes stat_end(&xs); 54379837Sdes siginfo = 0; 54479837Sdes } 54579837Sdes if ((size = fread(buf, 1, size, f)) == 0) { 546106041Sdes if (ferror(f) && errno == EINTR && !sigint) 54779837Sdes clearerr(f); 54879837Sdes else 54979837Sdes break; 55079837Sdes } 55179837Sdes stat_update(&xs, count += size); 55279837Sdes for (ptr = buf; size > 0; ptr += wr, size -= wr) 55379837Sdes if ((wr = fwrite(ptr, 1, size, of)) < size) { 554106041Sdes if (ferror(of) && errno == EINTR && !sigint) 55579837Sdes clearerr(of); 55679837Sdes else 55779837Sdes break; 55879837Sdes } 55979837Sdes if (size != 0) 56079837Sdes break; 56177241Sdes } 562106041Sdes if (!sigalrm) 563106041Sdes sigalrm = ferror(f) && errno == ETIMEDOUT; 56479837Sdes signal(SIGINFO, SIG_DFL); 56579837Sdes 56679837Sdes stat_end(&xs); 56762216Sdes 568106041Sdes /* 569106041Sdes * If the transfer timed out or was interrupted, we still want to 570106041Sdes * set the mtime in case the file is not removed (-r or -R) and 571106041Sdes * the user later restarts the transfer. 572106041Sdes */ 573106041Sdes signal: 57479837Sdes /* set mtime of local file */ 575106857Sdes if (!n_flag && us.mtime && !o_stdout && of != NULL && 576106857Sdes (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { 57779837Sdes struct timeval tv[2]; 578106043Sdes 57979837Sdes fflush(of); 58079837Sdes tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 58179837Sdes tv[1].tv_sec = (long)us.mtime; 58279837Sdes tv[0].tv_usec = tv[1].tv_usec = 0; 58390729Sdes if (utimes(tmppath ? tmppath : path, tv)) 58490729Sdes warn("%s: utimes()", tmppath ? tmppath : path); 58579837Sdes } 58663015Sdes 58779837Sdes /* timed out or interrupted? */ 58879837Sdes if (sigalrm) 58979837Sdes warnx("transfer timed out"); 59079837Sdes if (sigint) { 59179837Sdes warnx("transfer interrupted"); 59279837Sdes goto failure; 59379837Sdes } 59463046Sdes 595106586Sfenner /* timeout / interrupt before connection completley established? */ 596106586Sfenner if (f == NULL) 597106586Sfenner goto failure; 598106586Sfenner 59979837Sdes if (!sigalrm) { 60079837Sdes /* check the status of our files */ 60179837Sdes if (ferror(f)) 60279837Sdes warn("%s", URL); 60379837Sdes if (ferror(of)) 60479837Sdes warn("%s", path); 60579837Sdes if (ferror(f) || ferror(of)) 60679837Sdes goto failure; 60779837Sdes } 60879837Sdes 60979837Sdes /* did the transfer complete normally? */ 61079837Sdes if (us.size != -1 && count < us.size) { 61179837Sdes warnx("%s appears to be truncated: %lld/%lld bytes", 61279837Sdes path, (long long)count, (long long)us.size); 61379837Sdes goto failure_keep; 61479837Sdes } 61579837Sdes 61679837Sdes /* 61779837Sdes * If the transfer timed out and we didn't know how much to 61879837Sdes * expect, assume the worst (i.e. we didn't get all of it) 61979837Sdes */ 62079837Sdes if (sigalrm && us.size == -1) { 62179837Sdes warnx("%s may be truncated", path); 62279837Sdes goto failure_keep; 62379837Sdes } 62479837Sdes 62562216Sdes success: 62679837Sdes r = 0; 62783217Sdes if (tmppath != NULL && rename(tmppath, path) == -1) { 62883217Sdes warn("%s: rename()", path); 62983217Sdes goto failure_keep; 63083217Sdes } 63179837Sdes goto done; 63262216Sdes failure: 63379837Sdes if (of && of != stdout && !R_flag && !r_flag) 63479837Sdes if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) 63583217Sdes unlink(tmppath ? tmppath : path); 63683217Sdes if (R_flag && tmppath != NULL && sb.st_size == -1) 63783217Sdes rename(tmppath, path); /* ignore errors here */ 63863046Sdes failure_keep: 63979837Sdes r = -1; 64079837Sdes goto done; 64162216Sdes done: 64279837Sdes if (f) 64379837Sdes fclose(f); 64479837Sdes if (of && of != stdout) 64579837Sdes fclose(of); 64679837Sdes if (url) 64779837Sdes fetchFreeURL(url); 64883217Sdes if (tmppath != NULL) 64983217Sdes free(tmppath); 65079837Sdes return r; 65162216Sdes} 65262216Sdes 65379837Sdesstatic void 65462216Sdesusage(void) 65562216Sdes{ 65679837Sdes fprintf(stderr, "%s\n%s\n%s\n", 65795258Sdes "usage: fetch [-146AFMPRUadlmnpqrsv] [-o outputfile] [-S bytes]", 65879837Sdes " [-B bytes] [-T seconds] [-w seconds]", 65979837Sdes " [-h host -f file [-c dir] | URL ...]"); 66062216Sdes} 66162216Sdes 66262216Sdes 66381863Sdes/* 66481863Sdes * Entry point 66581863Sdes */ 66662216Sdesint 66762216Sdesmain(int argc, char *argv[]) 66862216Sdes{ 66979837Sdes struct stat sb; 67079837Sdes struct sigaction sa; 67179837Sdes const char *p, *s; 672100834Sdes char *end, *q; 67379837Sdes int c, e, r; 67462216Sdes 67579837Sdes while ((c = getopt(argc, argv, 67693213Scharnier "146AaB:bc:dFf:Hh:lMmnPpo:qRrS:sT:tUvw:")) != -1) 67779837Sdes switch (c) { 67879837Sdes case '1': 67979837Sdes once_flag = 1; 68079837Sdes break; 68179837Sdes case '4': 68279837Sdes family = PF_INET; 68379837Sdes break; 68479837Sdes case '6': 68579837Sdes family = PF_INET6; 68679837Sdes break; 68779837Sdes case 'A': 68879837Sdes A_flag = 1; 68979837Sdes break; 69079837Sdes case 'a': 69179837Sdes a_flag = 1; 69279837Sdes break; 69379837Sdes case 'B': 694100834Sdes B_size = (off_t)strtol(optarg, &end, 10); 695100834Sdes if (*optarg == '\0' || *end != '\0') 69680521Sse errx(1, "invalid buffer size (%s)", optarg); 69779837Sdes break; 69879837Sdes case 'b': 69979837Sdes warnx("warning: the -b option is deprecated"); 70079837Sdes b_flag = 1; 70179837Sdes break; 70279837Sdes case 'c': 70379837Sdes c_dirname = optarg; 70479837Sdes break; 70579837Sdes case 'd': 70679837Sdes d_flag = 1; 70779837Sdes break; 70879837Sdes case 'F': 70979837Sdes F_flag = 1; 71079837Sdes break; 71179837Sdes case 'f': 71279837Sdes f_filename = optarg; 71379837Sdes break; 71479837Sdes case 'H': 71593213Scharnier warnx("the -H option is now implicit, " 71679837Sdes "use -U to disable"); 71779837Sdes break; 71879837Sdes case 'h': 71979837Sdes h_hostname = optarg; 72079837Sdes break; 72179837Sdes case 'l': 72279837Sdes l_flag = 1; 72379837Sdes break; 72479837Sdes case 'o': 72579837Sdes o_flag = 1; 72679837Sdes o_filename = optarg; 72779837Sdes break; 72879837Sdes case 'M': 72979837Sdes case 'm': 73079837Sdes if (r_flag) 73179837Sdes errx(1, "the -m and -r flags " 73279837Sdes "are mutually exclusive"); 73379837Sdes m_flag = 1; 73479837Sdes break; 73579837Sdes case 'n': 73679837Sdes n_flag = 1; 73779837Sdes break; 73879837Sdes case 'P': 73979837Sdes case 'p': 74079837Sdes p_flag = 1; 74179837Sdes break; 74279837Sdes case 'q': 74379837Sdes v_level = 0; 74479837Sdes break; 74579837Sdes case 'R': 74679837Sdes R_flag = 1; 74779837Sdes break; 74879837Sdes case 'r': 74979837Sdes if (m_flag) 75079837Sdes errx(1, "the -m and -r flags " 75179837Sdes "are mutually exclusive"); 75279837Sdes r_flag = 1; 75379837Sdes break; 75479837Sdes case 'S': 755100834Sdes S_size = (off_t)strtol(optarg, &end, 10); 756100834Sdes if (*optarg == '\0' || *end != '\0') 75780521Sse errx(1, "invalid size (%s)", optarg); 75879837Sdes break; 75979837Sdes case 's': 76079837Sdes s_flag = 1; 76179837Sdes break; 762106043Sdes case 'T': 763100834Sdes T_secs = strtol(optarg, &end, 10); 764100834Sdes if (*optarg == '\0' || *end != '\0') 76580521Sse errx(1, "invalid timeout (%s)", optarg); 76679837Sdes break; 76779837Sdes case 't': 76879837Sdes t_flag = 1; 76979837Sdes warnx("warning: the -t option is deprecated"); 77079837Sdes break; 77179837Sdes case 'U': 77279837Sdes U_flag = 1; 77379837Sdes break; 77479837Sdes case 'v': 77579837Sdes v_level++; 77679837Sdes break; 77779837Sdes case 'w': 77879837Sdes a_flag = 1; 779100834Sdes w_secs = strtol(optarg, &end, 10); 780100834Sdes if (*optarg == '\0' || *end != '\0') 78180521Sse errx(1, "invalid delay (%s)", optarg); 78279837Sdes break; 78379837Sdes default: 78479837Sdes usage(); 78579837Sdes exit(EX_USAGE); 78679837Sdes } 78762216Sdes 78879837Sdes argc -= optind; 78979837Sdes argv += optind; 79062216Sdes 79179837Sdes if (h_hostname || f_filename || c_dirname) { 79279837Sdes if (!h_hostname || !f_filename || argc) { 79379837Sdes usage(); 79479837Sdes exit(EX_USAGE); 79579837Sdes } 79679837Sdes /* XXX this is a hack. */ 79779837Sdes if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 79879837Sdes errx(1, "invalid hostname"); 79979837Sdes if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 80079837Sdes c_dirname ? c_dirname : "", f_filename) == -1) 80179837Sdes errx(1, "%s", strerror(ENOMEM)); 80279837Sdes argc++; 80362216Sdes } 80463345Sdes 80579837Sdes if (!argc) { 80679837Sdes usage(); 80779837Sdes exit(EX_USAGE); 80879837Sdes } 80962216Sdes 81079837Sdes /* allocate buffer */ 81179837Sdes if (B_size < MINBUFSIZE) 81279837Sdes B_size = MINBUFSIZE; 81379837Sdes if ((buf = malloc(B_size)) == NULL) 81479837Sdes errx(1, "%s", strerror(ENOMEM)); 81562216Sdes 81679837Sdes /* timeouts */ 81779837Sdes if ((s = getenv("FTP_TIMEOUT")) != NULL) { 818100834Sdes ftp_timeout = strtol(s, &end, 10); 819102478Sdes if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { 820102478Sdes warnx("FTP_TIMEOUT (%s) is not a positive integer", s); 82179837Sdes ftp_timeout = 0; 82279837Sdes } 82362216Sdes } 82479837Sdes if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 825100834Sdes http_timeout = strtol(s, &end, 10); 826102478Sdes if (*s == '\0' || *end != '\0' || http_timeout < 0) { 827102478Sdes warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); 82879837Sdes http_timeout = 0; 82979837Sdes } 83062216Sdes } 83162216Sdes 83279837Sdes /* signal handling */ 83379837Sdes sa.sa_flags = 0; 83479837Sdes sa.sa_handler = sig_handler; 83579837Sdes sigemptyset(&sa.sa_mask); 83679837Sdes sigaction(SIGALRM, &sa, NULL); 83779837Sdes sa.sa_flags = SA_RESETHAND; 83879837Sdes sigaction(SIGINT, &sa, NULL); 83979837Sdes fetchRestartCalls = 0; 84079837Sdes 84179837Sdes /* output file */ 84279837Sdes if (o_flag) { 84379837Sdes if (strcmp(o_filename, "-") == 0) { 84479837Sdes o_stdout = 1; 84579837Sdes } else if (stat(o_filename, &sb) == -1) { 84679837Sdes if (errno == ENOENT) { 84779837Sdes if (argc > 1) 84879837Sdes errx(EX_USAGE, "%s is not a directory", 84979837Sdes o_filename); 85079837Sdes } else { 85179837Sdes err(EX_IOERR, "%s", o_filename); 85279837Sdes } 85379837Sdes } else { 85479837Sdes if (sb.st_mode & S_IFDIR) 85579837Sdes o_directory = 1; 85679837Sdes } 85762216Sdes } 85862216Sdes 85979837Sdes /* check if output is to a tty (for progress report) */ 86079837Sdes v_tty = isatty(STDERR_FILENO); 86183863Sdes if (v_tty) 86283863Sdes pgrp = getpgrp(); 863106043Sdes 86479837Sdes r = 0; 865106043Sdes 86679837Sdes /* authentication */ 86779837Sdes if (v_tty) 86879837Sdes fetchAuthMethod = query_auth; 86962216Sdes 87079837Sdes while (argc) { 87179837Sdes if ((p = strrchr(*argv, '/')) == NULL) 87279837Sdes p = *argv; 87379837Sdes else 87479837Sdes p++; 87562216Sdes 87679837Sdes if (!*p) 87779837Sdes p = "fetch.out"; 878106043Sdes 87979837Sdes fetchLastErrCode = 0; 880106043Sdes 88179837Sdes if (o_flag) { 88279837Sdes if (o_stdout) { 88379837Sdes e = fetch(*argv, "-"); 88479837Sdes } else if (o_directory) { 88579837Sdes asprintf(&q, "%s/%s", o_filename, p); 88679837Sdes e = fetch(*argv, q); 88779837Sdes free(q); 88879837Sdes } else { 88979837Sdes e = fetch(*argv, o_filename); 89079837Sdes } 89179837Sdes } else { 89279837Sdes e = fetch(*argv, p); 89379837Sdes } 89462216Sdes 89579837Sdes if (sigint) 89679837Sdes kill(getpid(), SIGINT); 897106043Sdes 89879837Sdes if (e == 0 && once_flag) 89979837Sdes exit(0); 900106043Sdes 90179837Sdes if (e) { 90279837Sdes r = 1; 90379837Sdes if ((fetchLastErrCode 90479837Sdes && fetchLastErrCode != FETCH_UNAVAIL 90579837Sdes && fetchLastErrCode != FETCH_MOVED 90679837Sdes && fetchLastErrCode != FETCH_URL 90779837Sdes && fetchLastErrCode != FETCH_RESOLV 90879837Sdes && fetchLastErrCode != FETCH_UNKNOWN)) { 90979837Sdes if (w_secs && v_level) 910100834Sdes fprintf(stderr, "Waiting %ld seconds " 91179837Sdes "before retrying\n", w_secs); 91279837Sdes if (w_secs) 91379837Sdes sleep(w_secs); 91479837Sdes if (a_flag) 91579837Sdes continue; 91679837Sdes } 91762216Sdes } 91879837Sdes 91979837Sdes argc--, argv++; 92062216Sdes } 92162216Sdes 92279837Sdes exit(r); 92362216Sdes} 924