fetch.c revision 225599
1212420Sken/*- 2212420Sken * Copyright (c) 2000-2004 Dag-Erling Co��dan Sm��rgrav 3237683Sken * All rights reserved. 4212420Sken * 5212420Sken * Redistribution and use in source and binary forms, with or without 6212420Sken * modification, are permitted provided that the following conditions 7212420Sken * are met: 8212420Sken * 1. Redistributions of source code must retain the above copyright 9212420Sken * notice, this list of conditions and the following disclaimer 10212420Sken * in this position and unchanged. 11212420Sken * 2. Redistributions in binary form must reproduce the above copyright 12212420Sken * notice, this list of conditions and the following disclaimer in the 13212420Sken * documentation and/or other materials provided with the distribution. 14212420Sken * 3. The name of the author may not be used to endorse or promote products 15212420Sken * derived from this software without specific prior written permission 16212420Sken * 17212420Sken * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18212420Sken * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19212420Sken * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20212420Sken * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21212420Sken * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22212420Sken * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23212420Sken * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24212420Sken * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25212420Sken * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26230592Sken * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27230592Sken */ 28230592Sken 29230592Sken#include <sys/cdefs.h> 30230592Sken__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 225599 2011-09-15 22:50:31Z des $"); 31212420Sken 32212420Sken#include <sys/param.h> 33212420Sken#include <sys/socket.h> 34212420Sken#include <sys/stat.h> 35212420Sken#include <sys/time.h> 36212420Sken 37230592Sken#include <ctype.h> 38212420Sken#include <err.h> 39212420Sken#include <errno.h> 40212420Sken#include <signal.h> 41212420Sken#include <stdint.h> 42212420Sken#include <stdio.h> 43212420Sken#include <stdlib.h> 44212420Sken#include <string.h> 45212420Sken#include <termios.h> 46212420Sken#include <unistd.h> 47212420Sken 48212420Sken#include <fetch.h> 49212420Sken 50216088Sken#define MINBUFSIZE 4096 51230592Sken#define TIMEOUT 120 52230592Sken 53230592Sken/* Option flags */ 54230592Skenint A_flag; /* -A: do not follow 302 redirects */ 55212420Skenint a_flag; /* -a: auto retry */ 56212420Skenoff_t B_size; /* -B: buffer size */ 57212420Skenint b_flag; /*! -b: workaround TCP bug */ 58212420Skenchar *c_dirname; /* -c: remote directory */ 59212420Skenint d_flag; /* -d: direct connection */ 60230592Skenint F_flag; /* -F: restart without checking mtime */ 61230592Skenchar *f_filename; /* -f: file to fetch */ 62212420Skenchar *h_hostname; /* -h: host to fetch from */ 63212420Skenint i_flag; /* -i: specify input file for mtime comparison */ 64230592Skenchar *i_filename; /* name of input file */ 65212420Skenint l_flag; /* -l: link rather than copy file: URLs */ 66212420Skenint m_flag; /* -[Mm]: mirror mode */ 67212420Skenchar *N_filename; /* -N: netrc file name */ 68212420Skenint n_flag; /* -n: do not preserve modification time */ 69212420Skenint o_flag; /* -o: specify output file */ 70212420Skenint o_directory; /* output file is a directory */ 71212420Skenchar *o_filename; /* name of output file */ 72216088Skenint o_stdout; /* output file is stdout */ 73216088Skenint once_flag; /* -1: stop at first successful file */ 74216088Skenint p_flag; /* -[Pp]: use passive FTP */ 75212420Skenint R_flag; /* -R: don't delete partially transferred files */ 76212420Skenint r_flag; /* -r: restart previously interrupted transfer */ 77212420Skenoff_t S_size; /* -S: require size to match */ 78212420Skenint s_flag; /* -s: show size, don't fetch */ 79212420Skenlong T_secs; /* -T: transfer timeout in seconds */ 80212420Skenint t_flag; /*! -t: workaround TCP bug */ 81212420Skenint U_flag; /* -U: do not use high ports */ 82230592Skenint v_level = 1; /* -v: verbosity level */ 83230592Skenint v_tty; /* stdout is a tty */ 84212420Skenpid_t pgrp; /* our process group */ 85212420Skenlong w_secs; /* -w: retry delay */ 86230592Skenint family = PF_UNSPEC; /* -[46]: address family to use */ 87212420Sken 88230592Skenint sigalrm; /* SIGALRM received */ 89230592Skenint siginfo; /* SIGINFO received */ 90212420Skenint sigint; /* SIGINT received */ 91230592Sken 92230592Skenlong ftp_timeout = TIMEOUT; /* default timeout for FTP transfers */ 93230592Skenlong http_timeout = TIMEOUT; /* default timeout for HTTP transfers */ 94230592Skenchar *buf; /* transfer buffer */ 95230592Sken 96230592Sken 97230592Sken/* 98230592Sken * Signal handler 99230592Sken */ 100230592Skenstatic void 101230592Skensig_handler(int sig) 102230592Sken{ 103230592Sken switch (sig) { 104230592Sken case SIGALRM: 105230592Sken sigalrm = 1; 106230592Sken break; 107230592Sken case SIGINFO: 108230592Sken siginfo = 1; 109230592Sken break; 110230592Sken case SIGINT: 111230592Sken sigint = 1; 112230592Sken break; 113230592Sken } 114212420Sken} 115212420Sken 116230592Skenstruct xferstat { 117212420Sken char name[64]; 118212420Sken struct timeval start; 119212420Sken struct timeval last; 120212420Sken off_t size; 121212420Sken off_t offset; 122212420Sken off_t rcvd; 123212420Sken}; 124230592Sken 125230592Sken/* 126212420Sken * Compute and display ETA 127212420Sken */ 128230592Skenstatic const char * 129216088Skenstat_eta(struct xferstat *xs) 130216088Sken{ 131216088Sken static char str[16]; 132216088Sken long elapsed, eta; 133216088Sken off_t received, expected; 134230592Sken 135212420Sken elapsed = xs->last.tv_sec - xs->start.tv_sec; 136230592Sken received = xs->rcvd - xs->offset; 137230592Sken expected = xs->size - xs->rcvd; 138230592Sken eta = (long)((double)elapsed * expected / received); 139230592Sken if (eta > 3600) 140253549Sken snprintf(str, sizeof str, "%02ldh%02ldm", 141253549Sken eta / 3600, (eta % 3600) / 60); 142253549Sken else 143253549Sken snprintf(str, sizeof str, "%02ldm%02lds", 144230592Sken eta / 60, eta % 60); 145230592Sken return (str); 146230592Sken} 147230592Sken 148230592Sken/* 149212420Sken * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'... 150231240Sken */ 151230592Skenstatic const char *prefixes = " kMGTP"; 152216368Skenstatic const char * 153230592Skenstat_bytes(off_t bytes) 154230592Sken{ 155216368Sken static char str[16]; 156264492Sscottl const char *prefix = prefixes; 157230592Sken 158230592Sken while (bytes > 9999 && prefix[1] != '\0') { 159230592Sken bytes /= 1024; 160216368Sken prefix++; 161216368Sken } 162230592Sken snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix); 163216368Sken return (str); 164216368Sken} 165230592Sken 166230592Sken/* 167230592Sken * Compute and display transfer rate 168230592Sken */ 169230592Skenstatic const char * 170230592Skenstat_bps(struct xferstat *xs) 171230592Sken{ 172230592Sken static char str[16]; 173230592Sken double delta, bps; 174212420Sken 175253460Sscottl delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6)) 176253460Sscottl - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 177230592Sken if (delta == 0.0) { 178230592Sken snprintf(str, sizeof str, "?? Bps"); 179230592Sken } else { 180253460Sscottl bps = (xs->rcvd - xs->offset) / delta; 181230592Sken snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps)); 182262853Smav } 183262853Smav return (str); 184262853Smav} 185230592Sken 186230592Sken/* 187253460Sscottl * Update the stats display 188230592Sken */ 189230592Skenstatic void 190230592Skenstat_display(struct xferstat *xs, int force) 191212420Sken{ 192230592Sken struct timeval now; 193270250Sslm int ctty_pgrp; 194270250Sslm 195270250Sslm /* check if we're the foreground process */ 196270250Sslm if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || 197270250Sslm (pid_t)ctty_pgrp != pgrp) 198270250Sslm return; 199270250Sslm 200270250Sslm gettimeofday(&now, NULL); 201270250Sslm if (!force && now.tv_sec <= xs->last.tv_sec) 202270250Sslm return; 203230592Sken xs->last = now; 204230592Sken 205253460Sscottl fprintf(stderr, "\r%-46.46s", xs->name); 206253460Sscottl if (xs->size <= 0) { 207230592Sken setproctitle("%s [%s]", xs->name, stat_bytes(xs->rcvd)); 208230592Sken fprintf(stderr, " %s", stat_bytes(xs->rcvd)); 209230592Sken } else { 210230592Sken setproctitle("%s [%d%% of %s]", xs->name, 211230592Sken (int)((100.0 * xs->rcvd) / xs->size), 212253460Sscottl stat_bytes(xs->size)); 213230592Sken fprintf(stderr, "%3d%% of %s", 214230592Sken (int)((100.0 * xs->rcvd) / xs->size), 215262853Smav stat_bytes(xs->size)); 216253549Sken } 217253549Sken fprintf(stderr, " %s", stat_bps(xs)); 218253549Sken if (xs->size > 0 && xs->rcvd > 0 && 219230592Sken xs->last.tv_sec >= xs->start.tv_sec + 10) 220253549Sken fprintf(stderr, " %s", stat_eta(xs)); 221230592Sken} 222253460Sscottl 223230592Sken/* 224212420Sken * Initialize the transfer statistics 225230592Sken */ 226212420Skenstatic void 227230592Skenstat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) 228230592Sken{ 229230592Sken snprintf(xs->name, sizeof xs->name, "%s", name); 230230592Sken gettimeofday(&xs->start, NULL); 231230592Sken xs->last.tv_sec = xs->last.tv_usec = 0; 232230592Sken xs->size = size; 233230592Sken xs->offset = offset; 234230592Sken xs->rcvd = offset; 235212420Sken if (v_tty && v_level > 0) 236253460Sscottl stat_display(xs, 1); 237230592Sken else if (v_level > 0) 238230592Sken fprintf(stderr, "%-46s", xs->name); 239230592Sken} 240253460Sscottl 241253460Sscottl/* 242230592Sken * Update the transfer statistics 243230592Sken */ 244253460Sscottlstatic void 245230592Skenstat_update(struct xferstat *xs, off_t rcvd) 246230592Sken{ 247230592Sken xs->rcvd = rcvd; 248212420Sken if (v_tty && v_level > 0) 249212420Sken stat_display(xs, 0); 250230592Sken} 251230592Sken 252212420Sken/* 253253460Sscottl * Finalize the transfer statistics 254230592Sken */ 255230592Skenstatic void 256212420Skenstat_end(struct xferstat *xs) 257230592Sken{ 258230592Sken gettimeofday(&xs->last, NULL); 259230592Sken if (v_tty && v_level > 0) { 260230592Sken stat_display(xs, 1); 261230592Sken putc('\n', stderr); 262253460Sscottl } else if (v_level > 0) { 263230592Sken fprintf(stderr, " %s %s\n", 264212420Sken stat_bytes(xs->size), stat_bps(xs)); 265253460Sscottl } 266230592Sken} 267212420Sken 268230592Sken/* 269212420Sken * Ask the user for authentication details 270212420Sken */ 271230592Skenstatic int 272230592Skenquery_auth(struct url *URL) 273212420Sken{ 274230592Sken struct termios tios; 275230592Sken tcflag_t saved_flags; 276230592Sken int i, nopwd; 277230592Sken 278212420Sken fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", 279253460Sscottl URL->scheme, URL->host, URL->port); 280230592Sken 281230592Sken fprintf(stderr, "Login: "); 282230592Sken if (fgets(URL->user, sizeof URL->user, stdin) == NULL) 283230592Sken return (-1); 284230592Sken for (i = strlen(URL->user); i >= 0; --i) 285212420Sken if (URL->user[i] == '\r' || URL->user[i] == '\n') 286230592Sken URL->user[i] = '\0'; 287230592Sken 288230592Sken fprintf(stderr, "Password: "); 289230592Sken if (tcgetattr(STDIN_FILENO, &tios) == 0) { 290230592Sken saved_flags = tios.c_lflag; 291253460Sscottl tios.c_lflag &= ~ECHO; 292212420Sken tios.c_lflag |= ECHONL|ICANON; 293212420Sken tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); 294212420Sken nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 295249468Smav tios.c_lflag = saved_flags; 296253550Sken tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); 297253460Sscottl } else { 298230592Sken nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 299212420Sken } 300212420Sken if (nopwd) 301230592Sken return (-1); 302237800Sken for (i = strlen(URL->pwd); i >= 0; --i) 303237800Sken if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n') 304237800Sken URL->pwd[i] = '\0'; 305237800Sken 306237800Sken return (0); 307230592Sken} 308253549Sken 309212420Sken/* 310212420Sken * Fetch a file 311212420Sken */ 312253460Sscottlstatic int 313212420Skenfetch(char *URL, const char *path) 314230592Sken{ 315230592Sken struct url *url; 316230592Sken struct url_stat us; 317230592Sken struct stat sb, nsb; 318212420Sken struct xferstat xs; 319230592Sken FILE *f, *of; 320230592Sken size_t size, wr; 321212420Sken off_t count; 322254253Sscottl char flags[8]; 323254257Smav const char *slash; 324254253Sscottl char *tmppath; 325254253Sscottl int r; 326230592Sken unsigned timeout; 327212420Sken char *ptr; 328230592Sken 329212420Sken f = of = NULL; 330230592Sken tmppath = NULL; 331230592Sken 332230592Sken timeout = 0; 333230592Sken *flags = 0; 334230592Sken count = 0; 335230592Sken 336230592Sken /* set verbosity level */ 337230592Sken if (v_level > 1) 338212420Sken strcat(flags, "v"); 339212420Sken if (v_level > 2) 340230592Sken fetchDebug = 1; 341230592Sken 342230592Sken /* parse URL */ 343230592Sken url = NULL; 344230592Sken if (*URL == '\0') { 345230592Sken warnx("empty URL"); 346230592Sken goto failure; 347230592Sken } 348212420Sken if ((url = fetchParseURL(URL)) == NULL) { 349230592Sken warnx("%s: parse error", URL); 350230592Sken goto failure; 351230592Sken } 352253460Sscottl 353212420Sken /* if no scheme was specified, take a guess */ 354230592Sken if (!*url->scheme) { 355230592Sken if (!*url->host) 356212420Sken strcpy(url->scheme, SCHEME_FILE); 357231240Sken else if (strncasecmp(url->host, "ftp.", 4) == 0) 358230592Sken strcpy(url->scheme, SCHEME_FTP); 359231240Sken else if (strncasecmp(url->host, "www.", 4) == 0) 360230592Sken strcpy(url->scheme, SCHEME_HTTP); 361231240Sken } 362231240Sken 363231240Sken /* common flags */ 364212420Sken switch (family) { 365253460Sscottl case PF_INET: 366231240Sken strcat(flags, "4"); 367231240Sken break; 368231240Sken case PF_INET6: 369231240Sken strcat(flags, "6"); 370231240Sken break; 371231240Sken } 372231240Sken 373253460Sscottl /* FTP specific flags */ 374253460Sscottl if (strcmp(url->scheme, SCHEME_FTP) == 0) { 375231240Sken if (p_flag) 376230592Sken strcat(flags, "p"); 377230592Sken if (d_flag) 378212420Sken strcat(flags, "d"); 379231240Sken if (U_flag) 380253460Sscottl strcat(flags, "l"); 381253460Sscottl timeout = T_secs ? T_secs : ftp_timeout; 382231240Sken } 383231240Sken 384231240Sken /* HTTP specific flags */ 385231240Sken if (strcmp(url->scheme, SCHEME_HTTP) == 0 || 386231240Sken strcmp(url->scheme, SCHEME_HTTPS) == 0) { 387253460Sscottl if (d_flag) 388253460Sscottl strcat(flags, "d"); 389231240Sken if (A_flag) 390240518Seadler strcat(flags, "A"); 391231240Sken timeout = T_secs ? T_secs : http_timeout; 392253460Sscottl if (i_flag) { 393253460Sscottl if (stat(i_filename, &sb)) { 394231240Sken warn("%s: stat()", i_filename); 395231240Sken goto failure; 396231240Sken } 397231240Sken url->ims_time = sb.st_mtime; 398231240Sken strcat(flags, "i"); 399231240Sken } 400231240Sken } 401231240Sken 402231240Sken /* set the protocol timeout. */ 403231240Sken fetchTimeout = timeout; 404231240Sken 405231240Sken /* just print size */ 406231240Sken if (s_flag) { 407231240Sken if (timeout) 408231240Sken alarm(timeout); 409231240Sken r = fetchStat(url, &us, flags); 410231240Sken if (timeout) 411231240Sken alarm(0); 412231240Sken if (sigalrm || sigint) 413231240Sken goto signal; 414212420Sken if (r == -1) { 415212420Sken warnx("%s", fetchLastErrString); 416231240Sken goto failure; 417212420Sken } 418231240Sken if (us.size == -1) 419231240Sken printf("Unknown\n"); 420231240Sken else 421231240Sken printf("%jd\n", (intmax_t)us.size); 422231240Sken goto success; 423231240Sken } 424231240Sken 425231240Sken /* 426231240Sken * If the -r flag was specified, we have to compare the local 427231240Sken * and remote files, so we should really do a fetchStat() 428231240Sken * first, but I know of at least one HTTP server that only 429253460Sscottl * sends the content size in response to GET requests, and 430231240Sken * leaves it out of replies to HEAD requests. Also, in the 431231240Sken * (frequent) case that the local and remote files match but 432231240Sken * the local file is truncated, we have sufficient information 433231240Sken * before the compare to issue a correct request. Therefore, 434231240Sken * we always issue a GET request as if we were sure the local 435231240Sken * file was a truncated copy of the remote file; we can drop 436231240Sken * the connection later if we change our minds. 437231240Sken */ 438231240Sken sb.st_size = -1; 439231240Sken if (!o_stdout) { 440231240Sken r = stat(path, &sb); 441231240Sken if (r == 0 && r_flag && S_ISREG(sb.st_mode)) { 442231240Sken url->offset = sb.st_size; 443231240Sken } else if (r == -1 || !S_ISREG(sb.st_mode)) { 444231240Sken /* 445231240Sken * Whatever value sb.st_size has now is either 446231240Sken * wrong (if stat(2) failed) or irrelevant (if the 447231240Sken * path does not refer to a regular file) 448253460Sscottl */ 449253460Sscottl sb.st_size = -1; 450231240Sken } 451231240Sken if (r == -1 && errno != ENOENT) { 452231240Sken warnx("%s: stat()", path); 453231240Sken goto failure; 454231240Sken } 455231240Sken } 456231240Sken 457253460Sscottl /* start the transfer */ 458253460Sscottl if (timeout) 459231240Sken alarm(timeout); 460231240Sken f = fetchXGet(url, &us, flags); 461231240Sken if (timeout) 462231240Sken alarm(0); 463231240Sken if (sigalrm || sigint) 464231240Sken goto signal; 465231240Sken if (f == NULL) { 466231240Sken warnx("%s: %s", URL, fetchLastErrString); 467231240Sken if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0 468231240Sken && fetchLastErrCode == FETCH_OK 469231240Sken && strcmp(fetchLastErrString, "Not Modified") == 0) { 470231240Sken /* HTTP Not Modified Response, return OK. */ 471231240Sken r = 0; 472231240Sken goto done; 473231240Sken } else 474231240Sken goto failure; 475231240Sken } 476231240Sken if (sigint) 477231240Sken goto signal; 478231240Sken 479231240Sken /* check that size is as expected */ 480231240Sken if (S_size) { 481231240Sken if (us.size == -1) { 482230592Sken warnx("%s: size unknown", URL); 483230592Sken } else if (us.size != S_size) { 484230592Sken warnx("%s: size mismatch: expected %jd, actual %jd", 485230592Sken URL, (intmax_t)S_size, (intmax_t)us.size); 486230592Sken goto failure; 487230592Sken } 488230592Sken } 489212420Sken 490230592Sken /* symlink instead of copy */ 491230592Sken if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 492212420Sken if (symlink(url->doc, path) == -1) { 493212420Sken warn("%s: symlink()", path); 494212420Sken goto failure; 495212420Sken } 496212420Sken goto success; 497212420Sken } 498253460Sscottl 499212420Sken if (us.size == -1 && !o_stdout && v_level > 0) 500230592Sken warnx("%s: size of remote file is not known", URL); 501230592Sken if (v_level > 1) { 502230592Sken if (sb.st_size != -1) 503230592Sken fprintf(stderr, "local size / mtime: %jd / %ld\n", 504230592Sken (intmax_t)sb.st_size, (long)sb.st_mtime); 505212420Sken if (us.size != -1) 506253460Sscottl fprintf(stderr, "remote size / mtime: %jd / %ld\n", 507253460Sscottl (intmax_t)us.size, (long)us.mtime); 508212420Sken } 509230592Sken 510212420Sken /* open output file */ 511230592Sken if (o_stdout) { 512230592Sken /* output to stdout */ 513230592Sken of = stdout; 514212420Sken } else if (r_flag && sb.st_size != -1) { 515253460Sscottl /* resume mode, local file exists */ 516253460Sscottl if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { 517212420Sken /* no match! have to refetch */ 518212420Sken fclose(f); 519212420Sken /* if precious, warn the user and give up */ 520231240Sken if (R_flag) { 521218811Sken warnx("%s: local modification time " 522212420Sken "does not match remote", path); 523218811Sken goto failure_keep; 524237683Sken } 525212420Sken } else if (url->offset > sb.st_size) { 526212420Sken /* gap between what we asked for and what we got */ 527212420Sken warnx("%s: gap in resume mode", URL); 528212420Sken fclose(of); 529212420Sken of = NULL; 530212420Sken /* picked up again later */ 531230592Sken } else if (us.size != -1) { 532212420Sken if (us.size == sb.st_size) 533230592Sken /* nothing to do */ 534212420Sken goto success; 535230592Sken if (sb.st_size > us.size) { 536230592Sken /* local file too long! */ 537212420Sken warnx("%s: local file (%jd bytes) is longer " 538212420Sken "than remote file (%jd bytes)", path, 539212420Sken (intmax_t)sb.st_size, (intmax_t)us.size); 540230592Sken goto failure; 541212420Sken } 542212420Sken /* we got it, open local file */ 543212420Sken if ((of = fopen(path, "a")) == NULL) { 544212420Sken warn("%s: fopen()", path); 545218811Sken goto failure; 546212420Sken } 547212420Sken /* check that it didn't move under our feet */ 548253460Sscottl if (fstat(fileno(of), &nsb) == -1) { 549212420Sken /* can't happen! */ 550230592Sken warn("%s: fstat()", path); 551230592Sken goto failure; 552230592Sken } 553213535Sken if (nsb.st_dev != sb.st_dev || 554218812Sken nsb.st_ino != nsb.st_ino || 555218812Sken nsb.st_size != sb.st_size) { 556218812Sken warnx("%s: file has changed", URL); 557218812Sken fclose(of); 558218812Sken of = NULL; 559230592Sken sb = nsb; 560253460Sscottl /* picked up again later */ 561253460Sscottl } 562253460Sscottl /* seek to where we left off */ 563253460Sscottl if (of != NULL && fseek(of, url->offset, SEEK_SET) != 0) { 564230592Sken warn("%s: fseek()", path); 565218812Sken fclose(of); 566218812Sken of = NULL; 567218812Sken /* picked up again later */ 568230592Sken } 569230592Sken } 570253460Sscottl } else if (m_flag && sb.st_size != -1) { 571253460Sscottl /* mirror mode, local file exists */ 572230592Sken if (sb.st_size == us.size && sb.st_mtime == us.mtime) 573230592Sken goto success; 574230592Sken } 575230592Sken 576237683Sken if (of == NULL) { 577253460Sscottl /* 578253460Sscottl * We don't yet have an output file; either this is a 579237683Sken * vanilla run with no special flags, or the local and 580230592Sken * remote files didn't match. 581212420Sken */ 582212420Sken 583212420Sken if (url->offset > 0) { 584253460Sscottl /* 585237683Sken * We tried to restart a transfer, but for 586230592Sken * some reason gave up - so we have to restart 587240518Seadler * from scratch if we want the whole file 588212420Sken */ 589212420Sken url->offset = 0; 590230592Sken if ((f = fetchXGet(url, &us, flags)) == NULL) { 591218811Sken warnx("%s: %s", URL, fetchLastErrString); 592212420Sken goto failure; 593212420Sken } 594237683Sken if (sigint) 595230592Sken goto signal; 596230592Sken } 597230592Sken 598230592Sken /* construct a temp file name */ 599212420Sken if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { 600230592Sken if ((slash = strrchr(path, '/')) == NULL) 601212420Sken slash = path; 602253460Sscottl else 603230592Sken ++slash; 604230592Sken asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", 605218811Sken (int)(slash - path), path, slash); 606218811Sken if (tmppath != NULL) { 607253460Sscottl mkstemps(tmppath, strlen(slash) + 1); 608230592Sken of = fopen(tmppath, "w"); 609268197Sscottl chown(tmppath, sb.st_uid, sb.st_gid); 610230592Sken chmod(tmppath, sb.st_mode & ALLPERMS); 611218811Sken } 612212420Sken } 613212420Sken if (of == NULL) 614212420Sken of = fopen(path, "w"); 615230592Sken if (of == NULL) { 616212420Sken warn("%s: open()", path); 617212420Sken goto failure; 618230592Sken } 619230592Sken } 620237683Sken count = url->offset; 621212420Sken 622253460Sscottl /* start the counter */ 623212420Sken stat_start(&xs, path, us.size, count); 624230592Sken 625230592Sken sigalrm = siginfo = sigint = 0; 626212420Sken 627230592Sken /* suck in the data */ 628230592Sken signal(SIGINFO, sig_handler); 629230592Sken while (!sigint) { 630230592Sken if (us.size != -1 && us.size - count < B_size && 631230592Sken us.size - count >= 0) 632230592Sken size = us.size - count; 633253460Sscottl else 634253460Sscottl size = B_size; 635230592Sken if (siginfo) { 636230592Sken stat_end(&xs); 637230592Sken siginfo = 0; 638230592Sken } 639230592Sken if ((size = fread(buf, 1, size, f)) == 0) { 640212420Sken if (ferror(f) && errno == EINTR && !sigint) 641230592Sken clearerr(f); 642230592Sken else 643253460Sscottl break; 644253460Sscottl } 645230592Sken stat_update(&xs, count += size); 646230592Sken for (ptr = buf; size > 0; ptr += wr, size -= wr) 647212420Sken if ((wr = fwrite(ptr, 1, size, of)) < size) { 648212420Sken if (ferror(of) && errno == EINTR && !sigint) 649253460Sscottl clearerr(of); 650253460Sscottl else 651237683Sken break; 652212420Sken } 653230592Sken if (size != 0) 654230592Sken break; 655230592Sken } 656230592Sken if (!sigalrm) 657230592Sken sigalrm = ferror(f) && errno == ETIMEDOUT; 658230592Sken signal(SIGINFO, SIG_DFL); 659237683Sken 660230592Sken stat_end(&xs); 661230592Sken 662230592Sken /* 663230592Sken * If the transfer timed out or was interrupted, we still want to 664230592Sken * set the mtime in case the file is not removed (-r or -R) and 665230592Sken * the user later restarts the transfer. 666230592Sken */ 667230592Sken signal: 668231240Sken /* set mtime of local file */ 669237683Sken if (!n_flag && us.mtime && !o_stdout && of != NULL && 670237683Sken (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { 671237683Sken struct timeval tv[2]; 672237683Sken 673237683Sken fflush(of); 674237683Sken tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 675212420Sken tv[1].tv_sec = (long)us.mtime; 676237683Sken tv[0].tv_usec = tv[1].tv_usec = 0; 677212420Sken if (utimes(tmppath ? tmppath : path, tv)) 678230592Sken warn("%s: utimes()", tmppath ? tmppath : path); 679212420Sken } 680212420Sken 681212420Sken /* timed out or interrupted? */ 682212420Sken if (sigalrm) 683212420Sken warnx("transfer timed out"); 684237683Sken if (sigint) { 685212420Sken warnx("transfer interrupted"); 686212420Sken goto failure; 687212420Sken } 688212420Sken 689212420Sken /* timeout / interrupt before connection completley established? */ 690212420Sken if (f == NULL) 691212420Sken goto failure; 692212420Sken 693212420Sken if (!sigalrm) { 694230592Sken /* check the status of our files */ 695230592Sken if (ferror(f)) 696230592Sken warn("%s", URL); 697230592Sken if (ferror(of)) 698230592Sken warn("%s", path); 699212420Sken if (ferror(f) || ferror(of)) 700212420Sken goto failure; 701212420Sken } 702212420Sken 703212420Sken /* did the transfer complete normally? */ 704212420Sken if (us.size != -1 && count < us.size) { 705212420Sken warnx("%s appears to be truncated: %jd/%jd bytes", 706212420Sken path, (intmax_t)count, (intmax_t)us.size); 707212420Sken goto failure_keep; 708212420Sken } 709212420Sken 710230592Sken /* 711230592Sken * If the transfer timed out and we didn't know how much to 712212420Sken * expect, assume the worst (i.e. we didn't get all of it) 713253460Sscottl */ 714212420Sken if (sigalrm && us.size == -1) { 715212420Sken warnx("%s may be truncated", path); 716237683Sken goto failure_keep; 717237683Sken } 718237683Sken 719237683Sken success: 720237683Sken r = 0; 721264492Sscottl if (tmppath != NULL && rename(tmppath, path) == -1) { 722264492Sscottl warn("%s: rename()", path); 723264492Sscottl goto failure_keep; 724264492Sscottl } 725264492Sscottl goto done; 726264492Sscottl failure: 727264492Sscottl if (of && of != stdout && !R_flag && !r_flag) 728264492Sscottl if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) 729212420Sken unlink(tmppath ? tmppath : path); 730264492Sscottl if (R_flag && tmppath != NULL && sb.st_size == -1) 731237683Sken rename(tmppath, path); /* ignore errors here */ 732237683Sken failure_keep: 733237683Sken r = -1; 734237683Sken goto done; 735237683Sken done: 736237683Sken if (f) 737212420Sken fclose(f); 738212420Sken if (of && of != stdout) 739212420Sken fclose(of); 740230592Sken if (url) 741253460Sscottl fetchFreeURL(url); 742212420Sken if (tmppath != NULL) 743212420Sken free(tmppath); 744212420Sken return (r); 745212420Sken} 746230592Sken 747212420Skenstatic void 748230592Skenusage(void) 749212420Sken{ 750253460Sscottl fprintf(stderr, "%s\n%s\n%s\n%s\n", 751212420Sken"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", 752212420Sken" [-T seconds] [-w seconds] [-i file] URL ...", 753212420Sken" fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [-N file] [-o file] [-S bytes]", 754212420Sken" [-T seconds] [-w seconds] [-i file] -h host -f file [-c dir]"); 755230592Sken} 756230592Sken 757230592Sken 758230592Sken/* 759230592Sken * Entry point 760230592Sken */ 761230592Skenint 762230592Skenmain(int argc, char *argv[]) 763230592Sken{ 764230592Sken struct stat sb; 765230592Sken struct sigaction sa; 766230592Sken const char *p, *s; 767230592Sken char *end, *q; 768212420Sken int c, e, r; 769212420Sken 770212420Sken while ((c = getopt(argc, argv, 771212420Sken "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) 772212420Sken switch (c) { 773212420Sken case '1': 774253460Sscottl once_flag = 1; 775212420Sken break; 776212420Sken case '4': 777212420Sken family = PF_INET; 778212420Sken break; 779212420Sken case '6': 780212420Sken family = PF_INET6; 781253549Sken break; 782253549Sken case 'A': 783253549Sken A_flag = 1; 784212420Sken break; 785230592Sken case 'a': 786230592Sken a_flag = 1; 787262853Smav break; 788212420Sken case 'B': 789212420Sken B_size = (off_t)strtol(optarg, &end, 10); 790212420Sken if (*optarg == '\0' || *end != '\0') 791212420Sken errx(1, "invalid buffer size (%s)", optarg); 792230592Sken break; 793230592Sken case 'b': 794253549Sken warnx("warning: the -b option is deprecated"); 795253549Sken b_flag = 1; 796253549Sken break; 797253549Sken case 'c': 798253549Sken c_dirname = optarg; 799253549Sken break; 800253549Sken case 'd': 801230592Sken d_flag = 1; 802253549Sken break; 803253549Sken case 'F': 804253549Sken F_flag = 1; 805253549Sken break; 806253549Sken case 'f': 807253549Sken f_filename = optarg; 808253549Sken break; 809253549Sken case 'H': 810253549Sken warnx("the -H option is now implicit, " 811253549Sken "use -U to disable"); 812253549Sken break; 813253549Sken case 'h': 814253549Sken h_hostname = optarg; 815253549Sken break; 816253549Sken case 'i': 817253549Sken i_flag = 1; 818253549Sken i_filename = optarg; 819253549Sken break; 820253549Sken case 'l': 821253549Sken l_flag = 1; 822230592Sken break; 823253549Sken case 'o': 824253549Sken o_flag = 1; 825253549Sken o_filename = optarg; 826253549Sken break; 827253549Sken case 'M': 828253549Sken case 'm': 829253549Sken if (r_flag) 830230592Sken errx(1, "the -m and -r flags " 831230592Sken "are mutually exclusive"); 832230592Sken m_flag = 1; 833212420Sken break; 834212420Sken case 'N': 835212420Sken N_filename = optarg; 836212420Sken break; 837212420Sken case 'n': 838212420Sken n_flag = 1; 839212420Sken break; 840212420Sken case 'P': 841212420Sken case 'p': 842212420Sken p_flag = 1; 843212420Sken break; 844237683Sken case 'q': 845237683Sken v_level = 0; 846237683Sken break; 847212420Sken case 'R': 848253460Sscottl R_flag = 1; 849212420Sken break; 850212420Sken case 'r': 851212420Sken if (m_flag) 852212420Sken errx(1, "the -m and -r flags " 853212420Sken "are mutually exclusive"); 854230592Sken r_flag = 1; 855212420Sken break; 856230592Sken case 'S': 857230592Sken S_size = (off_t)strtol(optarg, &end, 10); 858230592Sken if (*optarg == '\0' || *end != '\0') 859230592Sken errx(1, "invalid size (%s)", optarg); 860230592Sken break; 861230592Sken case 's': 862230592Sken s_flag = 1; 863230592Sken break; 864212420Sken case 'T': 865212420Sken T_secs = strtol(optarg, &end, 10); 866230592Sken if (*optarg == '\0' || *end != '\0') 867230592Sken errx(1, "invalid timeout (%s)", optarg); 868253549Sken break; 869253549Sken case 't': 870253549Sken t_flag = 1; 871253549Sken warnx("warning: the -t option is deprecated"); 872253549Sken break; 873230592Sken case 'U': 874212420Sken U_flag = 1; 875212420Sken break; 876212420Sken case 'v': 877212420Sken v_level++; 878212420Sken break; 879212420Sken case 'w': 880212420Sken a_flag = 1; 881230592Sken w_secs = strtol(optarg, &end, 10); 882253549Sken if (*optarg == '\0' || *end != '\0') 883212420Sken errx(1, "invalid delay (%s)", optarg); 884212420Sken break; 885212420Sken default: 886212420Sken usage(); 887212420Sken exit(1); 888264492Sscottl } 889237683Sken 890237683Sken argc -= optind; 891237683Sken argv += optind; 892237683Sken 893237683Sken if (h_hostname || f_filename || c_dirname) { 894212420Sken if (!h_hostname || !f_filename || argc) { 895212420Sken usage(); 896212420Sken exit(1); 897212420Sken } 898212420Sken /* XXX this is a hack. */ 899212420Sken if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 900212420Sken errx(1, "invalid hostname"); 901230592Sken if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 902212420Sken c_dirname ? c_dirname : "", f_filename) == -1) 903212420Sken errx(1, "%s", strerror(ENOMEM)); 904212420Sken argc++; 905212420Sken } 906253460Sscottl 907212420Sken if (!argc) { 908212420Sken usage(); 909212420Sken exit(1); 910212420Sken } 911212420Sken 912212420Sken /* allocate buffer */ 913212420Sken if (B_size < MINBUFSIZE) 914212420Sken B_size = MINBUFSIZE; 915212420Sken if ((buf = malloc(B_size)) == NULL) 916212420Sken errx(1, "%s", strerror(ENOMEM)); 917212420Sken 918212420Sken /* timeouts */ 919212420Sken if ((s = getenv("FTP_TIMEOUT")) != NULL) { 920253460Sscottl ftp_timeout = strtol(s, &end, 10); 921253460Sscottl if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { 922212420Sken warnx("FTP_TIMEOUT (%s) is not a positive integer", s); 923230592Sken ftp_timeout = 0; 924212420Sken } 925212420Sken } 926212420Sken if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 927212420Sken http_timeout = strtol(s, &end, 10); 928212420Sken if (*s == '\0' || *end != '\0' || http_timeout < 0) { 929212420Sken warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); 930212420Sken http_timeout = 0; 931212420Sken } 932212420Sken } 933253549Sken 934253549Sken /* signal handling */ 935253549Sken sa.sa_flags = 0; 936248825Smav sa.sa_handler = sig_handler; 937253549Sken sigemptyset(&sa.sa_mask); 938212420Sken sigaction(SIGALRM, &sa, NULL); 939264492Sscottl sa.sa_flags = SA_RESETHAND; 940237683Sken sigaction(SIGINT, &sa, NULL); 941264492Sscottl fetchRestartCalls = 0; 942212420Sken 943212420Sken /* output file */ 944212420Sken if (o_flag) { 945212420Sken if (strcmp(o_filename, "-") == 0) { 946212420Sken o_stdout = 1; 947212420Sken } else if (stat(o_filename, &sb) == -1) { 948212420Sken if (errno == ENOENT) { 949212420Sken if (argc > 1) 950212420Sken errx(1, "%s is not a directory", 951212420Sken o_filename); 952230592Sken } else { 953230592Sken err(1, "%s", o_filename); 954230592Sken } 955230592Sken } else { 956230592Sken if (sb.st_mode & S_IFDIR) 957230592Sken o_directory = 1; 958268197Sscottl } 959212420Sken } 960212420Sken 961212420Sken /* check if output is to a tty (for progress report) */ 962212420Sken v_tty = isatty(STDERR_FILENO); 963212420Sken if (v_tty) 964212420Sken pgrp = getpgrp(); 965212420Sken 966212420Sken r = 0; 967212420Sken 968212420Sken /* authentication */ 969212420Sken if (v_tty) 970212420Sken fetchAuthMethod = query_auth; 971212420Sken if (N_filename != NULL) 972264492Sscottl setenv("NETRC", N_filename, 1); 973264492Sscottl 974264492Sscottl while (argc) { 975212420Sken if ((p = strrchr(*argv, '/')) == NULL) 976212420Sken p = *argv; 977270250Sslm else 978212420Sken p++; 979212420Sken 980212420Sken if (!*p) 981212420Sken p = "fetch.out"; 982212420Sken 983212420Sken fetchLastErrCode = 0; 984212420Sken 985212420Sken if (o_flag) { 986212420Sken if (o_stdout) { 987212420Sken e = fetch(*argv, "-"); 988212420Sken } else if (o_directory) { 989212420Sken asprintf(&q, "%s/%s", o_filename, p); 990212420Sken e = fetch(*argv, q); 991212420Sken free(q); 992212420Sken } else { 993212420Sken e = fetch(*argv, o_filename); 994212420Sken } 995212420Sken } else { 996212420Sken e = fetch(*argv, p); 997212420Sken } 998212420Sken 999212420Sken if (sigint) 1000212420Sken kill(getpid(), SIGINT); 1001212420Sken 1002212420Sken if (e == 0 && once_flag) 1003212420Sken exit(0); 1004268197Sscottl 1005212420Sken if (e) { 1006212420Sken r = 1; 1007212420Sken if ((fetchLastErrCode 1008212420Sken && fetchLastErrCode != FETCH_UNAVAIL 1009268197Sscottl && fetchLastErrCode != FETCH_MOVED 1010212420Sken && fetchLastErrCode != FETCH_URL 1011212420Sken && fetchLastErrCode != FETCH_RESOLV 1012253460Sscottl && fetchLastErrCode != FETCH_UNKNOWN)) { 1013212420Sken if (w_secs && v_level) 1014212420Sken fprintf(stderr, "Waiting %ld seconds " 1015212420Sken "before retrying\n", w_secs); 1016212420Sken if (w_secs) 1017212420Sken sleep(w_secs); 1018253460Sscottl if (a_flag) 1019253460Sscottl continue; 1020268197Sscottl } 1021212420Sken } 1022212420Sken 1023212420Sken argc--, argv++; 1024212420Sken } 1025216088Sken 1026216088Sken exit(r); 1027216088Sken} 1028216088Sken