fetch.c revision 106857
133965Sjdp/*- 289857Sobrien * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav 389857Sobrien * All rights reserved. 433965Sjdp * 533965Sjdp * Redistribution and use in source and binary forms, with or without 633965Sjdp * modification, are permitted provided that the following conditions 733965Sjdp * are met: 833965Sjdp * 1. Redistributions of source code must retain the above copyright 933965Sjdp * notice, this list of conditions and the following disclaimer 1033965Sjdp * in this position and unchanged. 1133965Sjdp * 2. Redistributions in binary form must reproduce the above copyright 1233965Sjdp * notice, this list of conditions and the following disclaimer in the 1333965Sjdp * documentation and/or other materials provided with the distribution. 1433965Sjdp * 3. The name of the author may not be used to endorse or promote products 1533965Sjdp * derived from this software without specific prior written permission 1633965Sjdp * 1733965Sjdp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1833965Sjdp * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1933965Sjdp * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2033965Sjdp * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2133965Sjdp * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2233965Sjdp * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2333965Sjdp * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2433965Sjdp * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2533965Sjdp * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2633965Sjdp * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2733965Sjdp */ 2833965Sjdp 2933965Sjdp#include <sys/cdefs.h> 3078828Sobrien__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 106857 2002-11-13 16:04:20Z des $"); 3189857Sobrien 3278828Sobrien#include <sys/param.h> 3378828Sobrien#include <sys/socket.h> 3478828Sobrien#include <sys/stat.h> 3533965Sjdp#include <sys/time.h> 3689857Sobrien 3789857Sobrien#include <ctype.h> 3889857Sobrien#include <err.h> 3933965Sjdp#include <errno.h> 4033965Sjdp#include <signal.h> 4189857Sobrien#include <stdio.h> 4233965Sjdp#include <stdlib.h> 4333965Sjdp#include <string.h> 4489857Sobrien#include <sysexits.h> 4533965Sjdp#include <termios.h> 4633965Sjdp#include <unistd.h> 4733965Sjdp 4833965Sjdp#include <fetch.h> 4989857Sobrien 5033965Sjdp#define MINBUFSIZE 4096 5133965Sjdp 5233965Sjdp/* Option flags */ 5333965Sjdpint A_flag; /* -A: do not follow 302 redirects */ 5433965Sjdpint a_flag; /* -a: auto retry */ 5533965Sjdpoff_t B_size; /* -B: buffer size */ 5633965Sjdpint b_flag; /*! -b: workaround TCP bug */ 5733965Sjdpchar *c_dirname; /* -c: remote directory */ 5833965Sjdpint d_flag; /* -d: direct connection */ 5933965Sjdpint F_flag; /* -F: restart without checking mtime */ 6033965Sjdpchar *f_filename; /* -f: file to fetch */ 6133965Sjdpchar *h_hostname; /* -h: host to fetch from */ 6233965Sjdpint l_flag; /* -l: link rather than copy file: URLs */ 6333965Sjdpint m_flag; /* -[Mm]: mirror mode */ 6433965Sjdpint n_flag; /* -n: do not preserve modification time */ 6533965Sjdpint o_flag; /* -o: specify output file */ 6633965Sjdpint o_directory; /* output file is a directory */ 6733965Sjdpchar *o_filename; /* name of output file */ 6833965Sjdpint o_stdout; /* output file is stdout */ 6933965Sjdpint once_flag; /* -1: stop at first successful file */ 7033965Sjdpint p_flag; /* -[Pp]: use passive FTP */ 7133965Sjdpint R_flag; /* -R: don't delete partially transferred files */ 7233965Sjdpint r_flag; /* -r: restart previously interrupted transfer */ 7333965Sjdpoff_t S_size; /* -S: require size to match */ 7433965Sjdpint s_flag; /* -s: show size, don't fetch */ 7533965Sjdplong T_secs = 120; /* -T: transfer timeout in seconds */ 7633965Sjdpint t_flag; /*! -t: workaround TCP bug */ 7733965Sjdpint U_flag; /* -U: do not use high ports */ 7833965Sjdpint v_level = 1; /* -v: verbosity level */ 7933965Sjdpint v_tty; /* stdout is a tty */ 8033965Sjdppid_t pgrp; /* our process group */ 8133965Sjdplong w_secs; /* -w: retry delay */ 8289857Sobrienint family = PF_UNSPEC; /* -[46]: address family to use */ 8389857Sobrien 8433965Sjdpint sigalrm; /* SIGALRM received */ 8533965Sjdpint siginfo; /* SIGINFO received */ 8633965Sjdpint sigint; /* SIGINT received */ 8733965Sjdp 8833965Sjdplong ftp_timeout; /* default timeout for FTP transfers */ 8933965Sjdplong http_timeout; /* default timeout for HTTP transfers */ 9033965Sjdpu_char *buf; /* transfer buffer */ 9133965Sjdp 9233965Sjdp 9333965Sjdp/* 9433965Sjdp * Signal handler 9533965Sjdp */ 9660484Sobrienstatic void 9760484Sobriensig_handler(int sig) 9860484Sobrien{ 9960484Sobrien switch (sig) { 10060484Sobrien case SIGALRM: 10160484Sobrien sigalrm = 1; 10260484Sobrien break; 10360484Sobrien case SIGINFO: 10460484Sobrien siginfo = 1; 10560484Sobrien break; 10633965Sjdp case SIGINT: 10733965Sjdp sigint = 1; 10833965Sjdp break; 10933965Sjdp } 11033965Sjdp} 11189857Sobrien 11289857Sobrienstruct xferstat { 11389857Sobrien char name[40]; 11433965Sjdp struct timeval start; 11533965Sjdp struct timeval end; 11633965Sjdp struct timeval last; 11733965Sjdp off_t size; 11833965Sjdp off_t offset; 11933965Sjdp off_t rcvd; 12033965Sjdp}; 12133965Sjdp 12233965Sjdp/* 12333965Sjdp * Update the stats display 12433965Sjdp */ 12533965Sjdpstatic void 12633965Sjdpstat_display(struct xferstat *xs, int force) 12733965Sjdp{ 12833965Sjdp struct timeval now; 12933965Sjdp int ctty_pgrp; 13033965Sjdp 13133965Sjdp if (!v_tty || !v_level) 13233965Sjdp return; 13333965Sjdp 13433965Sjdp /* check if we're the foreground process */ 13533965Sjdp if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || 13633965Sjdp (pid_t)ctty_pgrp != pgrp) 13760484Sobrien return; 13860484Sobrien 13960484Sobrien gettimeofday(&now, NULL); 14060484Sobrien if (!force && now.tv_sec <= xs->last.tv_sec) 14160484Sobrien return; 14260484Sobrien xs->last = now; 14360484Sobrien 14460484Sobrien fprintf(stderr, "\rReceiving %s", xs->name); 14560484Sobrien if (xs->size <= 0) { 14660484Sobrien fprintf(stderr, ": %lld bytes", (long long)xs->rcvd); 14789857Sobrien } else { 14889857Sobrien long elapsed; 14989857Sobrien 15089857Sobrien fprintf(stderr, " (%lld bytes): %d%%", (long long)xs->size, 15189857Sobrien (int)((100.0 * xs->rcvd) / xs->size)); 15289857Sobrien elapsed = xs->last.tv_sec - xs->start.tv_sec; 15389857Sobrien if (elapsed > 30 && xs->rcvd > 0) { 15489857Sobrien long remaining; 15589857Sobrien 15689857Sobrien remaining = ((xs->size * elapsed) / xs->rcvd) - elapsed; 15789857Sobrien fprintf(stderr, " (ETA "); 15889857Sobrien if (remaining > 3600) { 15989857Sobrien fprintf(stderr, "%02ld:", remaining / 3600); 16089857Sobrien remaining %= 3600; 16189857Sobrien } 16289857Sobrien fprintf(stderr, "%02ld:%02ld) ", 16389857Sobrien remaining / 60, remaining % 60); 16489857Sobrien } 16589857Sobrien } 16689857Sobrien} 16789857Sobrien 16889857Sobrien/* 16989857Sobrien * Initialize the transfer statistics 17033965Sjdp */ 17133965Sjdpstatic void 17233965Sjdpstat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) 17360484Sobrien{ 17433965Sjdp snprintf(xs->name, sizeof xs->name, "%s", name); 17533965Sjdp gettimeofday(&xs->start, NULL); 17633965Sjdp xs->last.tv_sec = xs->last.tv_usec = 0; 17733965Sjdp xs->end = xs->last; 17833965Sjdp xs->size = size; 17933965Sjdp xs->offset = offset; 18033965Sjdp xs->rcvd = offset; 18133965Sjdp stat_display(xs, 1); 18233965Sjdp} 18333965Sjdp 18433965Sjdp/* 18533965Sjdp * Update the transfer statistics 18660484Sobrien */ 18760484Sobrienstatic void 18833965Sjdpstat_update(struct xferstat *xs, off_t rcvd) 18960484Sobrien{ 19033965Sjdp xs->rcvd = rcvd; 19160484Sobrien stat_display(xs, 0); 19233965Sjdp} 19360484Sobrien 19433965Sjdp/* 19560484Sobrien * Finalize the transfer statistics 19633965Sjdp */ 19760484Sobrienstatic void 19833965Sjdpstat_end(struct xferstat *xs) 19960484Sobrien{ 20060484Sobrien double delta; 20160484Sobrien double bps; 20260484Sobrien 20360484Sobrien if (!v_level) 20460484Sobrien return; 20533965Sjdp 20633965Sjdp gettimeofday(&xs->end, NULL); 20760484Sobrien 20860484Sobrien stat_display(xs, 1); 20960484Sobrien fputc('\n', stderr); 21060484Sobrien delta = (xs->end.tv_sec + (xs->end.tv_usec / 1.e6)) 21160484Sobrien - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 21260484Sobrien fprintf(stderr, "%lld bytes transferred in %.1f seconds ", 21360484Sobrien (long long)(xs->rcvd - xs->offset), delta); 21460484Sobrien bps = (xs->rcvd - xs->offset) / delta; 21560484Sobrien if (bps > 1024*1024) 21660484Sobrien fprintf(stderr, "(%.2f MBps)\n", bps / (1024*1024)); 21760484Sobrien else if (bps > 1024) 21860484Sobrien fprintf(stderr, "(%.2f kBps)\n", bps / 1024); 21960484Sobrien else 22060484Sobrien fprintf(stderr, "(%.2f Bps)\n", bps); 22177298Sobrien} 22260484Sobrien 22360484Sobrien/* 22460484Sobrien * Ask the user for authentication details 22560484Sobrien */ 22689857Sobrienstatic int 22789857Sobrienquery_auth(struct url *URL) 22889857Sobrien{ 22989857Sobrien struct termios tios; 23089857Sobrien tcflag_t saved_flags; 23189857Sobrien int i, nopwd; 23289857Sobrien 23389857Sobrien 23489857Sobrien fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", 23589857Sobrien URL->scheme, URL->host, URL->port); 23633965Sjdp 23733965Sjdp fprintf(stderr, "Login: "); 23833965Sjdp if (fgets(URL->user, sizeof URL->user, stdin) == NULL) 23933965Sjdp return -1; 24033965Sjdp for (i = 0; URL->user[i]; ++i) 24189857Sobrien if (isspace(URL->user[i])) 24233965Sjdp URL->user[i] = '\0'; 24333965Sjdp 24433965Sjdp fprintf(stderr, "Password: "); 24589857Sobrien if (tcgetattr(STDIN_FILENO, &tios) == 0) { 24633965Sjdp saved_flags = tios.c_lflag; 24733965Sjdp tios.c_lflag &= ~ECHO; 24833965Sjdp tios.c_lflag |= ECHONL|ICANON; 24933965Sjdp tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); 25089857Sobrien nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 25189857Sobrien tios.c_lflag = saved_flags; 25289857Sobrien tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); 25389857Sobrien } else { 25489857Sobrien nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 25589857Sobrien } 25689857Sobrien if (nopwd) 25733965Sjdp return -1; 25833965Sjdp 25933965Sjdp for (i = 0; URL->pwd[i]; ++i) 26089857Sobrien if (isspace(URL->pwd[i])) 26133965Sjdp URL->pwd[i] = '\0'; 26233965Sjdp return 0; 26389857Sobrien} 26489857Sobrien 26533965Sjdp/* 26633965Sjdp * Fetch a file 26733965Sjdp */ 26889857Sobrienstatic int 26933965Sjdpfetch(char *URL, const char *path) 27033965Sjdp{ 27133965Sjdp struct url *url; 27289857Sobrien struct url_stat us; 27333965Sjdp struct stat sb, nsb; 27433965Sjdp struct xferstat xs; 27589857Sobrien FILE *f, *of; 27633965Sjdp size_t size, wr; 27733965Sjdp off_t count; 27889857Sobrien char flags[8]; 27989857Sobrien const char *slash; 28089857Sobrien char *tmppath; 28189857Sobrien int r; 28260484Sobrien u_int timeout; 28360484Sobrien u_char *ptr; 28489857Sobrien 28577298Sobrien f = of = NULL; 28677298Sobrien tmppath = NULL; 28777298Sobrien 28833965Sjdp /* parse URL */ 28933965Sjdp if ((url = fetchParseURL(URL)) == NULL) { 29033965Sjdp warnx("%s: parse error", URL); 29133965Sjdp goto failure; 29233965Sjdp } 29333965Sjdp 29433965Sjdp /* if no scheme was specified, take a guess */ 29533965Sjdp if (!*url->scheme) { 29633965Sjdp if (!*url->host) 29733965Sjdp strcpy(url->scheme, SCHEME_FILE); 29833965Sjdp else if (strncasecmp(url->host, "ftp.", 4) == 0) 29933965Sjdp strcpy(url->scheme, SCHEME_FTP); 30033965Sjdp else if (strncasecmp(url->host, "www.", 4) == 0) 30133965Sjdp strcpy(url->scheme, SCHEME_HTTP); 30233965Sjdp } 30333965Sjdp 30433965Sjdp timeout = 0; 30533965Sjdp *flags = 0; 30633965Sjdp count = 0; 30733965Sjdp 30889857Sobrien /* common flags */ 30989857Sobrien if (v_level > 1) 31089857Sobrien strcat(flags, "v"); 31189857Sobrien if (v_level > 2) 31289857Sobrien fetchDebug = 1; 31389857Sobrien switch (family) { 31489857Sobrien case PF_INET: 31589857Sobrien strcat(flags, "4"); 31689857Sobrien break; 31789857Sobrien case PF_INET6: 31889857Sobrien strcat(flags, "6"); 31989857Sobrien break; 32089857Sobrien } 32189857Sobrien 32233965Sjdp /* FTP specific flags */ 32333965Sjdp if (strcmp(url->scheme, "ftp") == 0) { 32433965Sjdp if (p_flag) 32533965Sjdp strcat(flags, "p"); 32633965Sjdp if (d_flag) 32733965Sjdp strcat(flags, "d"); 32833965Sjdp if (U_flag) 32960484Sobrien strcat(flags, "l"); 33060484Sobrien timeout = T_secs ? T_secs : ftp_timeout; 33160484Sobrien } 33260484Sobrien 33360484Sobrien /* HTTP specific flags */ 33460484Sobrien if (strcmp(url->scheme, "http") == 0) { 33560484Sobrien if (d_flag) 33633965Sjdp strcat(flags, "d"); 33733965Sjdp if (A_flag) 33889857Sobrien strcat(flags, "A"); 33989857Sobrien timeout = T_secs ? T_secs : http_timeout; 34089857Sobrien } 34189857Sobrien 34289857Sobrien /* set the protocol timeout. */ 34389857Sobrien fetchTimeout = timeout; 34489857Sobrien 34589857Sobrien /* just print size */ 34633965Sjdp if (s_flag) { 34733965Sjdp if (timeout) 34889857Sobrien alarm(timeout); 34989857Sobrien r = fetchStat(url, &us, flags); 35089857Sobrien if (timeout) 35189857Sobrien alarm(0); 35289857Sobrien if (sigalrm || sigint) 35389857Sobrien goto signal; 35460484Sobrien if (r == -1) { 35560484Sobrien warnx("%s", fetchLastErrString); 35660484Sobrien goto failure; 35760484Sobrien } 35860484Sobrien if (us.size == -1) 35960484Sobrien printf("Unknown\n"); 36060484Sobrien else 36160484Sobrien printf("%lld\n", (long long)us.size); 36260484Sobrien goto success; 36360484Sobrien } 36460484Sobrien 36560484Sobrien /* 36660484Sobrien * If the -r flag was specified, we have to compare the local 36760484Sobrien * and remote files, so we should really do a fetchStat() 36860484Sobrien * first, but I know of at least one HTTP server that only 36960484Sobrien * sends the content size in response to GET requests, and 37060484Sobrien * leaves it out of replies to HEAD requests. Also, in the 37160484Sobrien * (frequent) case that the local and remote files match but 37260484Sobrien * the local file is truncated, we have sufficient information 37360484Sobrien * before the compare to issue a correct request. Therefore, 37460484Sobrien * we always issue a GET request as if we were sure the local 37560484Sobrien * file was a truncated copy of the remote file; we can drop 37660484Sobrien * the connection later if we change our minds. 37760484Sobrien */ 37860484Sobrien sb.st_size = -1; 37933965Sjdp if (!o_stdout && stat(path, &sb) == -1 && errno != ENOENT) { 38033965Sjdp warnx("%s: stat()", path); 38133965Sjdp goto failure; 38233965Sjdp } 38333965Sjdp if (!o_stdout && r_flag && S_ISREG(sb.st_mode)) 38433965Sjdp url->offset = sb.st_size; 38533965Sjdp 38633965Sjdp /* start the transfer */ 38789857Sobrien if (timeout) 38889857Sobrien alarm(timeout); 38989857Sobrien f = fetchXGet(url, &us, flags); 39089857Sobrien if (timeout) 39189857Sobrien alarm(0); 39289857Sobrien if (sigalrm || sigint) 39389857Sobrien goto signal; 39489857Sobrien if (f == NULL) { 39589857Sobrien warnx("%s: %s", path, fetchLastErrString); 39689857Sobrien goto failure; 39789857Sobrien } 39889857Sobrien if (sigint) 39989857Sobrien goto signal; 40089857Sobrien 40189857Sobrien /* check that size is as expected */ 40289857Sobrien if (S_size) { 40389857Sobrien if (us.size == -1) { 40489857Sobrien warnx("%s: size unknown", path); 40589857Sobrien goto failure; 40633965Sjdp } else if (us.size != S_size) { 40733965Sjdp warnx("%s: size mismatch: expected %lld, actual %lld", 40833965Sjdp path, (long long)S_size, (long long)us.size); 40933965Sjdp goto failure; 41033965Sjdp } 41133965Sjdp } 41233965Sjdp 41333965Sjdp /* symlink instead of copy */ 41433965Sjdp if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 41533965Sjdp if (symlink(url->doc, path) == -1) { 41633965Sjdp warn("%s: symlink()", path); 41733965Sjdp goto failure; 41833965Sjdp } 41989857Sobrien goto success; 42089857Sobrien } 42133965Sjdp 42233965Sjdp if (us.size == -1 && !o_stdout && v_level > 0) 42333965Sjdp warnx("%s: size of remote file is not known", path); 42489857Sobrien if (v_level > 1) { 42589857Sobrien if (sb.st_size != -1) 42633965Sjdp fprintf(stderr, "local size / mtime: %lld / %ld\n", 42733965Sjdp (long long)sb.st_size, (long)sb.st_mtime); 42833965Sjdp if (us.size != -1) 42933965Sjdp fprintf(stderr, "remote size / mtime: %lld / %ld\n", 43033965Sjdp (long long)us.size, (long)us.mtime); 43133965Sjdp } 43289857Sobrien 43389857Sobrien /* open output file */ 43433965Sjdp if (o_stdout) { 43533965Sjdp /* output to stdout */ 43633965Sjdp of = stdout; 43733965Sjdp } else if (r_flag && sb.st_size != -1) { 43833965Sjdp /* resume mode, local file exists */ 43933965Sjdp if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { 44033965Sjdp /* no match! have to refetch */ 44133965Sjdp fclose(f); 44233965Sjdp /* if precious, warn the user and give up */ 44333965Sjdp if (R_flag) { 44489857Sobrien warnx("%s: local modification time " 44589857Sobrien "does not match remote", path); 44633965Sjdp goto failure_keep; 44733965Sjdp } 44833965Sjdp } else { 44933965Sjdp if (us.size == sb.st_size) 45089857Sobrien /* nothing to do */ 45189857Sobrien goto success; 45233965Sjdp if (sb.st_size > us.size) { 45333965Sjdp /* local file too long! */ 45433965Sjdp warnx("%s: local file (%lld bytes) is longer " 45589857Sobrien "than remote file (%lld bytes)", path, 45689857Sobrien (long long)sb.st_size, (long long)us.size); 45733965Sjdp goto failure; 45860484Sobrien } 45977298Sobrien /* we got it, open local file */ 46089857Sobrien if ((of = fopen(path, "a")) == NULL) { 46189857Sobrien warn("%s: fopen()", path); 46277298Sobrien goto failure; 46333965Sjdp } 46433965Sjdp /* check that it didn't move under our feet */ 46533965Sjdp if (fstat(fileno(of), &nsb) == -1) { 46633965Sjdp /* can't happen! */ 46789857Sobrien warn("%s: fstat()", path); 46889857Sobrien goto failure; 46933965Sjdp } 47033965Sjdp if (nsb.st_dev != sb.st_dev || 47133965Sjdp nsb.st_ino != nsb.st_ino || 47289857Sobrien nsb.st_size != sb.st_size) { 47389857Sobrien warnx("%s: file has changed", path); 47433965Sjdp fclose(of); 47577298Sobrien of = NULL; 47677298Sobrien sb = nsb; 47789857Sobrien } 47889857Sobrien } 47977298Sobrien } else if (m_flag && sb.st_size != -1) { 48060484Sobrien /* mirror mode, local file exists */ 48177298Sobrien if (sb.st_size == us.size && sb.st_mtime == us.mtime) 48289857Sobrien goto success; 48389857Sobrien } 48460484Sobrien 48533965Sjdp if (of == NULL) { 48633965Sjdp /* 48733965Sjdp * We don't yet have an output file; either this is a 48889857Sobrien * vanilla run with no special flags, or the local and 48989857Sobrien * remote files didn't match. 49033965Sjdp */ 49133965Sjdp 49233965Sjdp if (url->offset != 0) { 49333965Sjdp /* 49433965Sjdp * We tried to restart a transfer, but for 49533965Sjdp * some reason gave up - so we have to restart 49689857Sobrien * from scratch if we want the whole file 49733965Sjdp */ 49833965Sjdp url->offset = 0; 49933965Sjdp if ((f = fetchXGet(url, &us, flags)) == NULL) { 50033965Sjdp warnx("%s: %s", path, fetchLastErrString); 50133965Sjdp goto failure; 50233965Sjdp } 50333965Sjdp if (sigint) 50433965Sjdp goto signal; 50533965Sjdp } 50633965Sjdp 50733965Sjdp /* construct a temp file name */ 50833965Sjdp if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { 50933965Sjdp if ((slash = strrchr(path, '/')) == NULL) 51033965Sjdp slash = path; 51133965Sjdp else 51233965Sjdp ++slash; 51333965Sjdp asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", 51433965Sjdp (int)(slash - path), path, slash); 51533965Sjdp if (tmppath != NULL) { 51633965Sjdp mkstemps(tmppath, strlen(slash) + 1); 51733965Sjdp of = fopen(tmppath, "w"); 51833965Sjdp } 51933965Sjdp } 52033965Sjdp if (of == NULL) 52133965Sjdp of = fopen(path, "w"); 52233965Sjdp if (of == NULL) { 52333965Sjdp warn("%s: open()", path); 52433965Sjdp goto failure; 52533965Sjdp } 52633965Sjdp } 52733965Sjdp count = url->offset; 52833965Sjdp 52933965Sjdp /* start the counter */ 53033965Sjdp stat_start(&xs, path, us.size, count); 53133965Sjdp 53233965Sjdp sigalrm = siginfo = sigint = 0; 53333965Sjdp 53433965Sjdp /* suck in the data */ 53533965Sjdp signal(SIGINFO, sig_handler); 53633965Sjdp while (!sigint) { 53733965Sjdp if (us.size != -1 && us.size - count < B_size) 53833965Sjdp size = us.size - count; 53933965Sjdp else 54033965Sjdp size = B_size; 54133965Sjdp if (siginfo) { 54233965Sjdp stat_end(&xs); 54333965Sjdp siginfo = 0; 54433965Sjdp } 54533965Sjdp if ((size = fread(buf, 1, size, f)) == 0) { 54633965Sjdp if (ferror(f) && errno == EINTR && !sigint) 54733965Sjdp clearerr(f); 54833965Sjdp else 54933965Sjdp break; 55033965Sjdp } 55133965Sjdp stat_update(&xs, count += size); 55233965Sjdp for (ptr = buf; size > 0; ptr += wr, size -= wr) 55333965Sjdp if ((wr = fwrite(ptr, 1, size, of)) < size) { 55433965Sjdp if (ferror(of) && errno == EINTR && !sigint) 55533965Sjdp clearerr(of); 55633965Sjdp else 55733965Sjdp break; 55833965Sjdp } 55933965Sjdp if (size != 0) 56033965Sjdp break; 56133965Sjdp } 56233965Sjdp if (!sigalrm) 56333965Sjdp sigalrm = ferror(f) && errno == ETIMEDOUT; 56433965Sjdp signal(SIGINFO, SIG_DFL); 56533965Sjdp 56633965Sjdp stat_end(&xs); 56733965Sjdp 56833965Sjdp /* 56933965Sjdp * If the transfer timed out or was interrupted, we still want to 57033965Sjdp * set the mtime in case the file is not removed (-r or -R) and 57133965Sjdp * the user later restarts the transfer. 57233965Sjdp */ 57333965Sjdp signal: 57433965Sjdp /* set mtime of local file */ 57533965Sjdp if (!n_flag && us.mtime && !o_stdout && of != NULL && 57633965Sjdp (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { 57733965Sjdp struct timeval tv[2]; 57833965Sjdp 57933965Sjdp fflush(of); 58033965Sjdp tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 58133965Sjdp tv[1].tv_sec = (long)us.mtime; 58233965Sjdp tv[0].tv_usec = tv[1].tv_usec = 0; 58333965Sjdp if (utimes(tmppath ? tmppath : path, tv)) 58433965Sjdp warn("%s: utimes()", tmppath ? tmppath : path); 58533965Sjdp } 58633965Sjdp 58733965Sjdp /* timed out or interrupted? */ 58833965Sjdp if (sigalrm) 58933965Sjdp warnx("transfer timed out"); 59033965Sjdp if (sigint) { 59133965Sjdp warnx("transfer interrupted"); 59233965Sjdp goto failure; 59333965Sjdp } 59433965Sjdp 59533965Sjdp /* timeout / interrupt before connection completley established? */ 59633965Sjdp if (f == NULL) 59733965Sjdp goto failure; 59833965Sjdp 59933965Sjdp if (!sigalrm) { 60033965Sjdp /* check the status of our files */ 60133965Sjdp if (ferror(f)) 60233965Sjdp warn("%s", URL); 60333965Sjdp if (ferror(of)) 60433965Sjdp warn("%s", path); 60533965Sjdp if (ferror(f) || ferror(of)) 60633965Sjdp goto failure; 60733965Sjdp } 60833965Sjdp 60933965Sjdp /* did the transfer complete normally? */ 61033965Sjdp if (us.size != -1 && count < us.size) { 61133965Sjdp warnx("%s appears to be truncated: %lld/%lld bytes", 61233965Sjdp path, (long long)count, (long long)us.size); 61333965Sjdp goto failure_keep; 61433965Sjdp } 61533965Sjdp 61633965Sjdp /* 61733965Sjdp * If the transfer timed out and we didn't know how much to 61833965Sjdp * expect, assume the worst (i.e. we didn't get all of it) 61933965Sjdp */ 62033965Sjdp if (sigalrm && us.size == -1) { 62133965Sjdp warnx("%s may be truncated", path); 62233965Sjdp goto failure_keep; 62333965Sjdp } 62433965Sjdp 62533965Sjdp success: 62633965Sjdp r = 0; 62733965Sjdp if (tmppath != NULL && rename(tmppath, path) == -1) { 62833965Sjdp warn("%s: rename()", path); 62933965Sjdp goto failure_keep; 63033965Sjdp } 63133965Sjdp goto done; 63233965Sjdp failure: 63333965Sjdp if (of && of != stdout && !R_flag && !r_flag) 63433965Sjdp if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) 63533965Sjdp unlink(tmppath ? tmppath : path); 63633965Sjdp if (R_flag && tmppath != NULL && sb.st_size == -1) 63733965Sjdp rename(tmppath, path); /* ignore errors here */ 63833965Sjdp failure_keep: 63933965Sjdp r = -1; 64033965Sjdp goto done; 64133965Sjdp done: 64233965Sjdp if (f) 64389857Sobrien fclose(f); 64489857Sobrien if (of && of != stdout) 64533965Sjdp fclose(of); 64633965Sjdp if (url) 64733965Sjdp fetchFreeURL(url); 64889857Sobrien if (tmppath != NULL) 64989857Sobrien free(tmppath); 65033965Sjdp return r; 65160484Sobrien} 65260484Sobrien 65360484Sobrienstatic void 65460484Sobrienusage(void) 65560484Sobrien{ 65660484Sobrien fprintf(stderr, "%s\n%s\n%s\n", 65760484Sobrien "usage: fetch [-146AFMPRUadlmnpqrsv] [-o outputfile] [-S bytes]", 65860484Sobrien " [-B bytes] [-T seconds] [-w seconds]", 65960484Sobrien " [-h host -f file [-c dir] | URL ...]"); 66060484Sobrien} 66160484Sobrien 66260484Sobrien 66360484Sobrien/* 66460484Sobrien * Entry point 66560484Sobrien */ 66660484Sobrienint 66760484Sobrienmain(int argc, char *argv[]) 66860484Sobrien{ 66960484Sobrien struct stat sb; 67060484Sobrien struct sigaction sa; 67160484Sobrien const char *p, *s; 67260484Sobrien char *end, *q; 67360484Sobrien int c, e, r; 67460484Sobrien 67560484Sobrien while ((c = getopt(argc, argv, 67660484Sobrien "146AaB:bc:dFf:Hh:lMmnPpo:qRrS:sT:tUvw:")) != -1) 67760484Sobrien switch (c) { 67860484Sobrien case '1': 67960484Sobrien once_flag = 1; 68060484Sobrien break; 68160484Sobrien case '4': 68260484Sobrien family = PF_INET; 68389857Sobrien break; 68460484Sobrien case '6': 68589857Sobrien family = PF_INET6; 68689857Sobrien break; 68789857Sobrien case 'A': 68889857Sobrien A_flag = 1; 68989857Sobrien break; 69060484Sobrien case 'a': 69160484Sobrien a_flag = 1; 69260484Sobrien break; 69360484Sobrien case 'B': 69460484Sobrien B_size = (off_t)strtol(optarg, &end, 10); 69560484Sobrien if (*optarg == '\0' || *end != '\0') 69689857Sobrien errx(1, "invalid buffer size (%s)", optarg); 69760484Sobrien break; 69889857Sobrien case 'b': 69989857Sobrien warnx("warning: the -b option is deprecated"); 70089857Sobrien b_flag = 1; 70189857Sobrien break; 70289857Sobrien case 'c': 70389857Sobrien c_dirname = optarg; 70489857Sobrien break; 70589857Sobrien case 'd': 70689857Sobrien d_flag = 1; 70789857Sobrien break; 70889857Sobrien case 'F': 70989857Sobrien F_flag = 1; 71089857Sobrien break; 71189857Sobrien case 'f': 71289857Sobrien f_filename = optarg; 71389857Sobrien break; 71489857Sobrien case 'H': 71589857Sobrien warnx("the -H option is now implicit, " 71689857Sobrien "use -U to disable"); 71789857Sobrien break; 71889857Sobrien case 'h': 71989857Sobrien h_hostname = optarg; 72089857Sobrien break; 72189857Sobrien case 'l': 72289857Sobrien l_flag = 1; 72389857Sobrien break; 72489857Sobrien case 'o': 72589857Sobrien o_flag = 1; 72689857Sobrien o_filename = optarg; 72789857Sobrien break; 72889857Sobrien case 'M': 72989857Sobrien case 'm': 73089857Sobrien if (r_flag) 73189857Sobrien errx(1, "the -m and -r flags " 73289857Sobrien "are mutually exclusive"); 73389857Sobrien m_flag = 1; 73489857Sobrien break; 73589857Sobrien case 'n': 73689857Sobrien n_flag = 1; 73789857Sobrien break; 73889857Sobrien case 'P': 73989857Sobrien case 'p': 74089857Sobrien p_flag = 1; 74189857Sobrien break; 74289857Sobrien case 'q': 74333965Sjdp v_level = 0; 74433965Sjdp break; 74533965Sjdp case 'R': 74633965Sjdp R_flag = 1; 74733965Sjdp break; 74833965Sjdp case 'r': 74933965Sjdp if (m_flag) 75033965Sjdp errx(1, "the -m and -r flags " 75133965Sjdp "are mutually exclusive"); 75233965Sjdp r_flag = 1; 75338889Sjdp break; 75438889Sjdp case 'S': 75538889Sjdp S_size = (off_t)strtol(optarg, &end, 10); 75638889Sjdp if (*optarg == '\0' || *end != '\0') 75760484Sobrien errx(1, "invalid size (%s)", optarg); 75860484Sobrien break; 75960484Sobrien case 's': 76060484Sobrien s_flag = 1; 76160484Sobrien break; 76260484Sobrien case 'T': 76360484Sobrien T_secs = strtol(optarg, &end, 10); 76460484Sobrien if (*optarg == '\0' || *end != '\0') 76560484Sobrien errx(1, "invalid timeout (%s)", optarg); 76660484Sobrien break; 76760484Sobrien case 't': 76860484Sobrien t_flag = 1; 76960484Sobrien warnx("warning: the -t option is deprecated"); 77060484Sobrien break; 77160484Sobrien case 'U': 77260484Sobrien U_flag = 1; 77360484Sobrien break; 77460484Sobrien case 'v': 77560484Sobrien v_level++; 77660484Sobrien break; 77760484Sobrien case 'w': 77877298Sobrien a_flag = 1; 77960484Sobrien w_secs = strtol(optarg, &end, 10); 78060484Sobrien if (*optarg == '\0' || *end != '\0') 78160484Sobrien errx(1, "invalid delay (%s)", optarg); 78260484Sobrien break; 78360484Sobrien default: 78460484Sobrien usage(); 78560484Sobrien exit(EX_USAGE); 78660484Sobrien } 78777298Sobrien 78877298Sobrien argc -= optind; 78960484Sobrien argv += optind; 79060484Sobrien 79199461Sobrien if (h_hostname || f_filename || c_dirname) { 79299461Sobrien if (!h_hostname || !f_filename || argc) { 79399461Sobrien usage(); 79499461Sobrien exit(EX_USAGE); 79599461Sobrien } 79660484Sobrien /* XXX this is a hack. */ 79760484Sobrien if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 79860484Sobrien errx(1, "invalid hostname"); 79960484Sobrien if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 80033965Sjdp c_dirname ? c_dirname : "", f_filename) == -1) 80133965Sjdp errx(1, "%s", strerror(ENOMEM)); 80233965Sjdp argc++; 80338889Sjdp } 80438889Sjdp 80560484Sobrien if (!argc) { 80689857Sobrien usage(); 80789857Sobrien exit(EX_USAGE); 80860484Sobrien } 80933965Sjdp 81033965Sjdp /* allocate buffer */ 81133965Sjdp if (B_size < MINBUFSIZE) 81233965Sjdp B_size = MINBUFSIZE; 81333965Sjdp if ((buf = malloc(B_size)) == NULL) 81433965Sjdp errx(1, "%s", strerror(ENOMEM)); 81533965Sjdp 81633965Sjdp /* timeouts */ 81733965Sjdp if ((s = getenv("FTP_TIMEOUT")) != NULL) { 81889857Sobrien ftp_timeout = strtol(s, &end, 10); 81933965Sjdp if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { 82033965Sjdp warnx("FTP_TIMEOUT (%s) is not a positive integer", s); 82133965Sjdp ftp_timeout = 0; 82289857Sobrien } 82333965Sjdp } 82433965Sjdp if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 82533965Sjdp http_timeout = strtol(s, &end, 10); 82689857Sobrien if (*s == '\0' || *end != '\0' || http_timeout < 0) { 82760484Sobrien warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); 82860484Sobrien http_timeout = 0; 82989857Sobrien } 83060484Sobrien } 83160484Sobrien 83289857Sobrien /* signal handling */ 83389857Sobrien sa.sa_flags = 0; 83489857Sobrien sa.sa_handler = sig_handler; 83589857Sobrien sigemptyset(&sa.sa_mask); 83689857Sobrien sigaction(SIGALRM, &sa, NULL); 83789857Sobrien sa.sa_flags = SA_RESETHAND; 83889857Sobrien sigaction(SIGINT, &sa, NULL); 83989857Sobrien fetchRestartCalls = 0; 84033965Sjdp 84133965Sjdp /* output file */ 84233965Sjdp if (o_flag) { 84389857Sobrien if (strcmp(o_filename, "-") == 0) { 84460484Sobrien o_stdout = 1; 84560484Sobrien } else if (stat(o_filename, &sb) == -1) { 84633965Sjdp if (errno == ENOENT) { 84789857Sobrien if (argc > 1) 84860484Sobrien errx(EX_USAGE, "%s is not a directory", 84960484Sobrien o_filename); 85060484Sobrien } else { 85189857Sobrien err(EX_IOERR, "%s", o_filename); 85233965Sjdp } 85333965Sjdp } else { 85433965Sjdp if (sb.st_mode & S_IFDIR) 85589857Sobrien o_directory = 1; 85633965Sjdp } 85733965Sjdp } 85833965Sjdp 85933965Sjdp /* check if output is to a tty (for progress report) */ 86089857Sobrien v_tty = isatty(STDERR_FILENO); 86133965Sjdp if (v_tty) 86233965Sjdp pgrp = getpgrp(); 86360484Sobrien 86460484Sobrien r = 0; 86533965Sjdp 86689857Sobrien /* authentication */ 86789857Sobrien if (v_tty) 86889857Sobrien fetchAuthMethod = query_auth; 86989857Sobrien 87089857Sobrien while (argc) { 87189857Sobrien if ((p = strrchr(*argv, '/')) == NULL) 87289857Sobrien p = *argv; 87389857Sobrien else 87489857Sobrien p++; 87589857Sobrien 87689857Sobrien if (!*p) 87789857Sobrien p = "fetch.out"; 87889857Sobrien 87989857Sobrien fetchLastErrCode = 0; 88033965Sjdp 88133965Sjdp if (o_flag) { 88289857Sobrien if (o_stdout) { 88360484Sobrien e = fetch(*argv, "-"); 88460484Sobrien } else if (o_directory) { 88533965Sjdp asprintf(&q, "%s/%s", o_filename, p); 88633965Sjdp e = fetch(*argv, q); 88733965Sjdp free(q); 88889857Sobrien } else { 88989857Sobrien e = fetch(*argv, o_filename); 89033965Sjdp } 89189857Sobrien } else { 89289857Sobrien e = fetch(*argv, p); 89389857Sobrien } 89489857Sobrien 89589857Sobrien if (sigint) 89689857Sobrien kill(getpid(), SIGINT); 89733965Sjdp 89833965Sjdp if (e == 0 && once_flag) 89933965Sjdp exit(0); 90033965Sjdp 90133965Sjdp if (e) { 90233965Sjdp r = 1; 90333965Sjdp if ((fetchLastErrCode 90433965Sjdp && fetchLastErrCode != FETCH_UNAVAIL 90533965Sjdp && fetchLastErrCode != FETCH_MOVED 90633965Sjdp && fetchLastErrCode != FETCH_URL 90733965Sjdp && fetchLastErrCode != FETCH_RESOLV 90833965Sjdp && fetchLastErrCode != FETCH_UNKNOWN)) { 90933965Sjdp if (w_secs && v_level) 91033965Sjdp fprintf(stderr, "Waiting %ld seconds " 91133965Sjdp "before retrying\n", w_secs); 91233965Sjdp if (w_secs) 91333965Sjdp sleep(w_secs); 91433965Sjdp if (a_flag) 91533965Sjdp continue; 91633965Sjdp } 91733965Sjdp } 91833965Sjdp 91933965Sjdp argc--, argv++; 92033965Sjdp } 92133965Sjdp 92233965Sjdp exit(r); 92333965Sjdp} 92433965Sjdp