fetch.c revision 129440
1180740Sdes/*- 2207311Sdes * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav 3180740Sdes * All rights reserved. 4204861Sdes * 5180740Sdes * Redistribution and use in source and binary forms, with or without 6180740Sdes * modification, are permitted provided that the following conditions 7180740Sdes * are met: 8180740Sdes * 1. Redistributions of source code must retain the above copyright 9204861Sdes * notice, this list of conditions and the following disclaimer 10180740Sdes * in this position and unchanged. 11180740Sdes * 2. Redistributions in binary form must reproduce the above copyright 12180740Sdes * notice, this list of conditions and the following disclaimer in the 13180740Sdes * documentation and/or other materials provided with the distribution. 14180740Sdes * 3. The name of the author may not be used to endorse or promote products 15180740Sdes * derived from this software without specific prior written permission 16180740Sdes * 17180740Sdes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18180740Sdes * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19180740Sdes * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20180740Sdes * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21204861Sdes * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22180740Sdes * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23180740Sdes * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24180740Sdes * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25180740Sdes * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26180740Sdes * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27180740Sdes */ 28180740Sdes 29180740Sdes#include <sys/cdefs.h> 30180740Sdes__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 129440 2004-05-19 11:07:30Z le $"); 31180740Sdes 32180740Sdes#include <sys/param.h> 33180740Sdes#include <sys/socket.h> 34180740Sdes#include <sys/stat.h> 35180740Sdes#include <sys/time.h> 36180740Sdes 37180740Sdes#include <ctype.h> 38180740Sdes#include <err.h> 39180740Sdes#include <errno.h> 40180740Sdes#include <signal.h> 41180740Sdes#include <stdint.h> 42180740Sdes#include <stdio.h> 43204861Sdes#include <stdlib.h> 44204861Sdes#include <string.h> 45204861Sdes#include <sysexits.h> 46204861Sdes#include <termios.h> 47204861Sdes#include <unistd.h> 48204861Sdes 49204861Sdes#include <fetch.h> 50180740Sdes 51204861Sdes#define MINBUFSIZE 4096 52180740Sdes 53204861Sdes/* Option flags */ 54180740Sdesint A_flag; /* -A: do not follow 302 redirects */ 55180740Sdesint a_flag; /* -a: auto retry */ 56180740Sdesoff_t B_size; /* -B: buffer size */ 57180740Sdesint b_flag; /*! -b: workaround TCP bug */ 58180740Sdeschar *c_dirname; /* -c: remote directory */ 59180740Sdesint d_flag; /* -d: direct connection */ 60180740Sdesint F_flag; /* -F: restart without checking mtime */ 61180740Sdeschar *f_filename; /* -f: file to fetch */ 62180740Sdeschar *h_hostname; /* -h: host to fetch from */ 63180740Sdesint l_flag; /* -l: link rather than copy file: URLs */ 64180740Sdesint m_flag; /* -[Mm]: mirror mode */ 65180740Sdeschar *N_filename; /* -N: netrc file name */ 66180740Sdesint n_flag; /* -n: do not preserve modification time */ 67180740Sdesint o_flag; /* -o: specify output file */ 68180740Sdesint o_directory; /* output file is a directory */ 69204861Sdeschar *o_filename; /* name of output file */ 70204861Sdesint o_stdout; /* output file is stdout */ 71180740Sdesint once_flag; /* -1: stop at first successful file */ 72180740Sdesint p_flag; /* -[Pp]: use passive FTP */ 73180740Sdesint R_flag; /* -R: don't delete partially transferred files */ 74180740Sdesint r_flag; /* -r: restart previously interrupted transfer */ 75180740Sdesoff_t S_size; /* -S: require size to match */ 76180740Sdesint s_flag; /* -s: show size, don't fetch */ 77180740Sdeslong T_secs = 120; /* -T: transfer timeout in seconds */ 78180740Sdesint t_flag; /*! -t: workaround TCP bug */ 79180740Sdesint U_flag; /* -U: do not use high ports */ 80180740Sdesint v_level = 1; /* -v: verbosity level */ 81180740Sdesint v_tty; /* stdout is a tty */ 82180740Sdespid_t pgrp; /* our process group */ 83180740Sdeslong w_secs; /* -w: retry delay */ 84180740Sdesint family = PF_UNSPEC; /* -[46]: address family to use */ 85180740Sdes 86180740Sdesint sigalrm; /* SIGALRM received */ 87180740Sdesint siginfo; /* SIGINFO received */ 88180740Sdesint sigint; /* SIGINT received */ 89180740Sdes 90180740Sdeslong ftp_timeout; /* default timeout for FTP transfers */ 91180740Sdeslong http_timeout; /* default timeout for HTTP transfers */ 92180740Sdeschar *buf; /* transfer buffer */ 93204861Sdes 94180740Sdes 95180740Sdes/* 96180740Sdes * Signal handler 97180740Sdes */ 98180740Sdesstatic void 99180740Sdessig_handler(int sig) 100180740Sdes{ 101180740Sdes switch (sig) { 102180740Sdes case SIGALRM: 103180740Sdes sigalrm = 1; 104180740Sdes break; 105180740Sdes case SIGINFO: 106204861Sdes siginfo = 1; 107204861Sdes break; 108204861Sdes case SIGINT: 109204861Sdes sigint = 1; 110204861Sdes break; 111204861Sdes } 112204861Sdes} 113204861Sdes 114204861Sdesstruct xferstat { 115204861Sdes char name[64]; 116204861Sdes struct timeval start; 117180740Sdes struct timeval last; 118180740Sdes off_t size; 119180740Sdes off_t offset; 120180740Sdes off_t rcvd; 121180740Sdes}; 122180740Sdes 123180740Sdes/* 124180740Sdes * Compute and display ETA 125180740Sdes */ 126180740Sdesstatic const char * 127180740Sdesstat_eta(struct xferstat *xs) 128180740Sdes{ 129180740Sdes static char str[16]; 130180740Sdes long elapsed, eta; 131180740Sdes off_t received, expected; 132180740Sdes 133180740Sdes elapsed = xs->last.tv_sec - xs->start.tv_sec; 134180740Sdes received = xs->rcvd - xs->offset; 135180740Sdes expected = xs->size - xs->rcvd; 136180740Sdes eta = (long)((double)elapsed * expected / received); 137180740Sdes if (eta > 3600) 138204861Sdes snprintf(str, sizeof str, "%02ldh%02ldm", 139180740Sdes eta / 3600, (eta % 3600) / 60); 140180740Sdes else 141180740Sdes snprintf(str, sizeof str, "%02ldm%02lds", 142180740Sdes eta / 60, eta % 60); 143180740Sdes return (str); 144180740Sdes} 145180740Sdes 146180740Sdes/* 147180740Sdes * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'... 148180740Sdes */ 149180740Sdesstatic const char *prefixes = " kMGTP"; 150180740Sdesstatic const char * 151180740Sdesstat_bytes(off_t bytes) 152180740Sdes{ 153180740Sdes static char str[16]; 154180740Sdes const char *prefix = prefixes; 155180740Sdes 156180740Sdes while (bytes > 9999 && prefix[1] != '\0') { 157180740Sdes bytes /= 1024; 158180740Sdes prefix++; 159180740Sdes } 160180740Sdes snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix); 161180740Sdes return (str); 162180740Sdes} 163180740Sdes 164204861Sdes/* 165180740Sdes * Compute and display transfer rate 166180740Sdes */ 167180740Sdesstatic const char * 168180740Sdesstat_bps(struct xferstat *xs) 169180740Sdes{ 170180740Sdes static char str[16]; 171180740Sdes double delta, bps; 172180740Sdes 173180740Sdes delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6)) 174180740Sdes - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 175180740Sdes if (delta == 0.0) { 176180740Sdes snprintf(str, sizeof str, "?? Bps"); 177180740Sdes } else { 178180740Sdes bps = (xs->rcvd - xs->offset) / delta; 179180740Sdes snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps)); 180180740Sdes } 181180740Sdes return (str); 182180740Sdes} 183180740Sdes 184180740Sdes/* 185180740Sdes * Update the stats display 186180740Sdes */ 187180740Sdesstatic void 188180740Sdesstat_display(struct xferstat *xs, int force) 189180740Sdes{ 190180740Sdes struct timeval now; 191180740Sdes int ctty_pgrp; 192180740Sdes 193180740Sdes /* check if we're the foreground process */ 194180740Sdes if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || 195180740Sdes (pid_t)ctty_pgrp != pgrp) 196180740Sdes return; 197180740Sdes 198180740Sdes gettimeofday(&now, NULL); 199180740Sdes if (!force && now.tv_sec <= xs->last.tv_sec) 200180740Sdes return; 201180740Sdes xs->last = now; 202180740Sdes 203180740Sdes fprintf(stderr, "\r%-46s", xs->name); 204180740Sdes if (xs->size <= 0) { 205180740Sdes fprintf(stderr, " %s", stat_bytes(xs->rcvd)); 206189006Sdes } else { 207180740Sdes fprintf(stderr, "%3d%% of %s", 208180740Sdes (int)((100.0 * xs->rcvd) / xs->size), 209180740Sdes stat_bytes(xs->size)); 210180740Sdes } 211180740Sdes fprintf(stderr, " %s", stat_bps(xs)); 212180740Sdes if (xs->size > 0 && xs->rcvd > 0 && 213180740Sdes xs->last.tv_sec >= xs->start.tv_sec + 10) 214180740Sdes fprintf(stderr, " %s", stat_eta(xs)); 215180740Sdes} 216180740Sdes 217180740Sdes/* 218180740Sdes * Initialize the transfer statistics 219180740Sdes */ 220180740Sdesstatic void 221180740Sdesstat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) 222180740Sdes{ 223180740Sdes snprintf(xs->name, sizeof xs->name, "%s", name); 224180740Sdes gettimeofday(&xs->start, NULL); 225180740Sdes xs->last.tv_sec = xs->last.tv_usec = 0; 226180740Sdes xs->size = size; 227180740Sdes xs->offset = offset; 228180740Sdes xs->rcvd = offset; 229180740Sdes if (v_tty && v_level > 0) 230180740Sdes stat_display(xs, 1); 231180740Sdes else if (v_level > 0) 232180740Sdes fprintf(stderr, "%-46s", xs->name); 233180740Sdes} 234180740Sdes 235180740Sdes/* 236180740Sdes * Update the transfer statistics 237180740Sdes */ 238180740Sdesstatic void 239180740Sdesstat_update(struct xferstat *xs, off_t rcvd) 240180740Sdes{ 241180740Sdes xs->rcvd = rcvd; 242180740Sdes if (v_tty && v_level > 0) 243180740Sdes stat_display(xs, 0); 244180740Sdes} 245180740Sdes 246204861Sdes/* 247180740Sdes * Finalize the transfer statistics 248180740Sdes */ 249180740Sdesstatic void 250180740Sdesstat_end(struct xferstat *xs) 251180740Sdes{ 252180740Sdes gettimeofday(&xs->last, NULL); 253180740Sdes if (v_tty && v_level > 0) { 254180740Sdes stat_display(xs, 1); 255180740Sdes putc('\n', stderr); 256180740Sdes } else if (v_level > 0) { 257180740Sdes fprintf(stderr, " %s %s\n", 258180740Sdes stat_bytes(xs->size), stat_bps(xs)); 259180740Sdes } 260180740Sdes} 261180740Sdes 262180740Sdes/* 263180740Sdes * Ask the user for authentication details 264180740Sdes */ 265180740Sdesstatic int 266180740Sdesquery_auth(struct url *URL) 267204861Sdes{ 268180740Sdes struct termios tios; 269180740Sdes tcflag_t saved_flags; 270180740Sdes int i, nopwd; 271180740Sdes 272180740Sdes 273180740Sdes fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", 274180740Sdes URL->scheme, URL->host, URL->port); 275180740Sdes 276180740Sdes fprintf(stderr, "Login: "); 277180740Sdes if (fgets(URL->user, sizeof URL->user, stdin) == NULL) 278180740Sdes return -1; 279180740Sdes for (i = 0; URL->user[i]; ++i) 280180740Sdes if (isspace(URL->user[i])) 281180740Sdes URL->user[i] = '\0'; 282180740Sdes 283180740Sdes fprintf(stderr, "Password: "); 284180740Sdes if (tcgetattr(STDIN_FILENO, &tios) == 0) { 285180740Sdes saved_flags = tios.c_lflag; 286180740Sdes tios.c_lflag &= ~ECHO; 287180740Sdes tios.c_lflag |= ECHONL|ICANON; 288180740Sdes tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); 289180740Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 290180740Sdes tios.c_lflag = saved_flags; 291180740Sdes tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); 292180740Sdes } else { 293180740Sdes nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 294180740Sdes } 295180740Sdes if (nopwd) 296180740Sdes return -1; 297180740Sdes 298180740Sdes for (i = 0; URL->pwd[i]; ++i) 299180740Sdes if (isspace(URL->pwd[i])) 300180740Sdes URL->pwd[i] = '\0'; 301180740Sdes return 0; 302180740Sdes} 303180740Sdes 304180740Sdes/* 305180740Sdes * Fetch a file 306180740Sdes */ 307180740Sdesstatic int 308180740Sdesfetch(char *URL, const char *path) 309180740Sdes{ 310180740Sdes struct url *url; 311180740Sdes struct url_stat us; 312180740Sdes struct stat sb, nsb; 313180740Sdes struct xferstat xs; 314180740Sdes FILE *f, *of; 315180740Sdes size_t size, wr; 316180740Sdes off_t count; 317180740Sdes char flags[8]; 318180740Sdes const char *slash; 319180740Sdes char *tmppath; 320180740Sdes int r; 321180740Sdes unsigned timeout; 322180740Sdes char *ptr; 323180740Sdes 324180740Sdes f = of = NULL; 325180740Sdes tmppath = NULL; 326180740Sdes 327180740Sdes timeout = 0; 328180740Sdes *flags = 0; 329180740Sdes count = 0; 330180740Sdes 331180740Sdes /* set verbosity level */ 332180740Sdes if (v_level > 1) 333180740Sdes strcat(flags, "v"); 334180740Sdes if (v_level > 2) 335180740Sdes fetchDebug = 1; 336180740Sdes 337180740Sdes /* parse URL */ 338180740Sdes if ((url = fetchParseURL(URL)) == NULL) { 339180740Sdes warnx("%s: parse error", URL); 340180740Sdes goto failure; 341180740Sdes } 342180740Sdes 343180740Sdes /* if no scheme was specified, take a guess */ 344180740Sdes if (!*url->scheme) { 345180740Sdes if (!*url->host) 346180740Sdes strcpy(url->scheme, SCHEME_FILE); 347204861Sdes else if (strncasecmp(url->host, "ftp.", 4) == 0) 348204861Sdes strcpy(url->scheme, SCHEME_FTP); 349204861Sdes else if (strncasecmp(url->host, "www.", 4) == 0) 350204861Sdes strcpy(url->scheme, SCHEME_HTTP); 351180740Sdes } 352180740Sdes 353180740Sdes /* common flags */ 354180740Sdes switch (family) { 355180740Sdes case PF_INET: 356180740Sdes strcat(flags, "4"); 357180740Sdes break; 358180740Sdes case PF_INET6: 359180740Sdes strcat(flags, "6"); 360180740Sdes break; 361180740Sdes } 362180740Sdes 363180740Sdes /* FTP specific flags */ 364180740Sdes if (strcmp(url->scheme, "ftp") == 0) { 365180740Sdes if (p_flag) 366180740Sdes strcat(flags, "p"); 367180740Sdes if (d_flag) 368180740Sdes strcat(flags, "d"); 369180740Sdes if (U_flag) 370180740Sdes strcat(flags, "l"); 371180740Sdes timeout = T_secs ? T_secs : ftp_timeout; 372180740Sdes } 373180740Sdes 374180740Sdes /* HTTP specific flags */ 375180740Sdes if (strcmp(url->scheme, "http") == 0) { 376180740Sdes if (d_flag) 377180740Sdes strcat(flags, "d"); 378180740Sdes if (A_flag) 379180740Sdes strcat(flags, "A"); 380180740Sdes timeout = T_secs ? T_secs : http_timeout; 381180740Sdes } 382180740Sdes 383180740Sdes /* set the protocol timeout. */ 384180740Sdes fetchTimeout = timeout; 385180740Sdes 386180740Sdes /* just print size */ 387180740Sdes if (s_flag) { 388180740Sdes if (timeout) 389180740Sdes alarm(timeout); 390180740Sdes r = fetchStat(url, &us, flags); 391180740Sdes if (timeout) 392180740Sdes alarm(0); 393180740Sdes if (sigalrm || sigint) 394180740Sdes goto signal; 395180740Sdes if (r == -1) { 396180740Sdes warnx("%s", fetchLastErrString); 397180740Sdes goto failure; 398180740Sdes } 399180740Sdes if (us.size == -1) 400180740Sdes printf("Unknown\n"); 401180740Sdes else 402180740Sdes printf("%jd\n", (intmax_t)us.size); 403180740Sdes goto success; 404180740Sdes } 405180740Sdes 406180740Sdes /* 407180740Sdes * If the -r flag was specified, we have to compare the local 408180740Sdes * and remote files, so we should really do a fetchStat() 409180740Sdes * first, but I know of at least one HTTP server that only 410189006Sdes * sends the content size in response to GET requests, and 411180740Sdes * leaves it out of replies to HEAD requests. Also, in the 412180740Sdes * (frequent) case that the local and remote files match but 413180740Sdes * the local file is truncated, we have sufficient information 414180740Sdes * before the compare to issue a correct request. Therefore, 415180740Sdes * we always issue a GET request as if we were sure the local 416180740Sdes * file was a truncated copy of the remote file; we can drop 417180740Sdes * the connection later if we change our minds. 418180740Sdes */ 419204861Sdes sb.st_size = -1; 420204861Sdes if (!o_stdout && stat(path, &sb) == -1 && errno != ENOENT) { 421204861Sdes warnx("%s: stat()", path); 422180740Sdes goto failure; 423180740Sdes } 424180740Sdes if (!o_stdout && r_flag && S_ISREG(sb.st_mode)) 425180740Sdes url->offset = sb.st_size; 426180740Sdes 427180740Sdes /* start the transfer */ 428180740Sdes if (timeout) 429180740Sdes alarm(timeout); 430180740Sdes f = fetchXGet(url, &us, flags); 431180740Sdes if (timeout) 432180740Sdes alarm(0); 433180740Sdes if (sigalrm || sigint) 434180740Sdes goto signal; 435180740Sdes if (f == NULL) { 436180740Sdes warnx("%s: %s", URL, fetchLastErrString); 437180740Sdes goto failure; 438180740Sdes } 439180740Sdes if (sigint) 440180740Sdes goto signal; 441180740Sdes 442180740Sdes /* check that size is as expected */ 443180740Sdes if (S_size) { 444180740Sdes if (us.size == -1) { 445180740Sdes warnx("%s: size unknown", URL); 446180740Sdes goto failure; 447180740Sdes } else if (us.size != S_size) { 448180740Sdes warnx("%s: size mismatch: expected %jd, actual %jd", 449180740Sdes URL, (intmax_t)S_size, (intmax_t)us.size); 450180740Sdes goto failure; 451180740Sdes } 452180740Sdes } 453180740Sdes 454180740Sdes /* symlink instead of copy */ 455180740Sdes if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 456180740Sdes if (symlink(url->doc, path) == -1) { 457204861Sdes warn("%s: symlink()", path); 458180740Sdes goto failure; 459180740Sdes } 460180740Sdes goto success; 461180740Sdes } 462180740Sdes 463180740Sdes if (us.size == -1 && !o_stdout && v_level > 0) 464180740Sdes warnx("%s: size of remote file is not known", URL); 465180740Sdes if (v_level > 1) { 466180740Sdes if (sb.st_size != -1) 467180740Sdes fprintf(stderr, "local size / mtime: %jd / %ld\n", 468180740Sdes (intmax_t)sb.st_size, (long)sb.st_mtime); 469180740Sdes if (us.size != -1) 470180740Sdes fprintf(stderr, "remote size / mtime: %jd / %ld\n", 471180740Sdes (intmax_t)us.size, (long)us.mtime); 472180740Sdes } 473180740Sdes 474180740Sdes /* open output file */ 475180740Sdes if (o_stdout) { 476180740Sdes /* output to stdout */ 477180740Sdes of = stdout; 478180740Sdes } else if (r_flag && sb.st_size != -1) { 479180740Sdes /* resume mode, local file exists */ 480180740Sdes if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { 481180740Sdes /* no match! have to refetch */ 482180740Sdes fclose(f); 483180740Sdes /* if precious, warn the user and give up */ 484180740Sdes if (R_flag) { 485204861Sdes warnx("%s: local modification time " 486180740Sdes "does not match remote", path); 487180740Sdes goto failure_keep; 488180740Sdes } 489180740Sdes } else if (us.size != -1) { 490180740Sdes if (us.size == sb.st_size) 491180740Sdes /* nothing to do */ 492180740Sdes goto success; 493180740Sdes if (sb.st_size > us.size) { 494180740Sdes /* local file too long! */ 495180740Sdes warnx("%s: local file (%jd bytes) is longer " 496180740Sdes "than remote file (%jd bytes)", path, 497180740Sdes (intmax_t)sb.st_size, (intmax_t)us.size); 498204861Sdes goto failure; 499180740Sdes } 500204861Sdes /* we got it, open local file */ 501204861Sdes if ((of = fopen(path, "a")) == NULL) { 502204861Sdes warn("%s: fopen()", path); 503204861Sdes goto failure; 504204861Sdes } 505204861Sdes /* check that it didn't move under our feet */ 506204861Sdes if (fstat(fileno(of), &nsb) == -1) { 507204861Sdes /* can't happen! */ 508180740Sdes warn("%s: fstat()", path); 509204861Sdes goto failure; 510204861Sdes } 511180740Sdes if (nsb.st_dev != sb.st_dev || 512180740Sdes nsb.st_ino != nsb.st_ino || 513180740Sdes nsb.st_size != sb.st_size) { 514180740Sdes warnx("%s: file has changed", URL); 515180740Sdes fclose(of); 516180740Sdes of = NULL; 517180740Sdes sb = nsb; 518180740Sdes } 519180740Sdes } 520180740Sdes } else if (m_flag && sb.st_size != -1) { 521180740Sdes /* mirror mode, local file exists */ 522180740Sdes if (sb.st_size == us.size && sb.st_mtime == us.mtime) 523180740Sdes goto success; 524180740Sdes } 525180740Sdes 526180740Sdes if (of == NULL) { 527180740Sdes /* 528180740Sdes * We don't yet have an output file; either this is a 529180740Sdes * vanilla run with no special flags, or the local and 530180740Sdes * remote files didn't match. 531180740Sdes */ 532180740Sdes 533180740Sdes if (url->offset > 0) { 534180740Sdes /* 535204861Sdes * We tried to restart a transfer, but for 536180740Sdes * some reason gave up - so we have to restart 537180740Sdes * from scratch if we want the whole file 538204861Sdes */ 539180740Sdes url->offset = 0; 540180740Sdes if ((f = fetchXGet(url, &us, flags)) == NULL) { 541180740Sdes warnx("%s: %s", URL, fetchLastErrString); 542180740Sdes goto failure; 543180740Sdes } 544180740Sdes if (sigint) 545180740Sdes goto signal; 546180740Sdes } 547180740Sdes 548180740Sdes /* construct a temp file name */ 549180740Sdes if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { 550180740Sdes if ((slash = strrchr(path, '/')) == NULL) 551180740Sdes slash = path; 552180740Sdes else 553180740Sdes ++slash; 554180740Sdes asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", 555180740Sdes (int)(slash - path), path, slash); 556180740Sdes if (tmppath != NULL) { 557180740Sdes mkstemps(tmppath, strlen(slash) + 1); 558180740Sdes of = fopen(tmppath, "w"); 559180740Sdes } 560180740Sdes } 561180740Sdes if (of == NULL) 562180740Sdes of = fopen(path, "w"); 563180740Sdes if (of == NULL) { 564180740Sdes warn("%s: open()", path); 565180740Sdes goto failure; 566180740Sdes } 567180740Sdes } 568180740Sdes count = url->offset; 569180740Sdes 570180740Sdes /* start the counter */ 571180740Sdes stat_start(&xs, path, us.size, count); 572180740Sdes 573180740Sdes sigalrm = siginfo = sigint = 0; 574180740Sdes 575180740Sdes /* suck in the data */ 576180740Sdes signal(SIGINFO, sig_handler); 577180740Sdes while (!sigint) { 578180740Sdes if (us.size != -1 && us.size - count < B_size) 579180740Sdes size = us.size - count; 580180740Sdes else 581180740Sdes size = B_size; 582180740Sdes if (siginfo) { 583180740Sdes stat_end(&xs); 584180740Sdes siginfo = 0; 585180740Sdes } 586180740Sdes if ((size = fread(buf, 1, size, f)) == 0) { 587180740Sdes if (ferror(f) && errno == EINTR && !sigint) 588180740Sdes clearerr(f); 589180740Sdes else 590180740Sdes break; 591180740Sdes } 592180740Sdes stat_update(&xs, count += size); 593180740Sdes for (ptr = buf; size > 0; ptr += wr, size -= wr) 594180740Sdes if ((wr = fwrite(ptr, 1, size, of)) < size) { 595180740Sdes if (ferror(of) && errno == EINTR && !sigint) 596180740Sdes clearerr(of); 597180740Sdes else 598180740Sdes break; 599180740Sdes } 600180740Sdes if (size != 0) 601180740Sdes break; 602180740Sdes } 603180740Sdes if (!sigalrm) 604180740Sdes sigalrm = ferror(f) && errno == ETIMEDOUT; 605180740Sdes signal(SIGINFO, SIG_DFL); 606180740Sdes 607180740Sdes stat_end(&xs); 608180740Sdes 609180740Sdes /* 610180740Sdes * If the transfer timed out or was interrupted, we still want to 611180740Sdes * set the mtime in case the file is not removed (-r or -R) and 612180740Sdes * the user later restarts the transfer. 613180740Sdes */ 614180740Sdes signal: 615180740Sdes /* set mtime of local file */ 616180740Sdes if (!n_flag && us.mtime && !o_stdout && of != NULL && 617180740Sdes (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { 618180740Sdes struct timeval tv[2]; 619180740Sdes 620180740Sdes fflush(of); 621180740Sdes tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 622180740Sdes tv[1].tv_sec = (long)us.mtime; 623180740Sdes tv[0].tv_usec = tv[1].tv_usec = 0; 624180740Sdes if (utimes(tmppath ? tmppath : path, tv)) 625180740Sdes warn("%s: utimes()", tmppath ? tmppath : path); 626180740Sdes } 627180740Sdes 628180740Sdes /* timed out or interrupted? */ 629180740Sdes if (sigalrm) 630180740Sdes warnx("transfer timed out"); 631180740Sdes if (sigint) { 632180740Sdes warnx("transfer interrupted"); 633180740Sdes goto failure; 634180740Sdes } 635180740Sdes 636180740Sdes /* timeout / interrupt before connection completley established? */ 637180740Sdes if (f == NULL) 638180740Sdes goto failure; 639180740Sdes 640180740Sdes if (!sigalrm) { 641180740Sdes /* check the status of our files */ 642180740Sdes if (ferror(f)) 643180740Sdes warn("%s", URL); 644180740Sdes if (ferror(of)) 645180740Sdes warn("%s", path); 646180740Sdes if (ferror(f) || ferror(of)) 647180740Sdes goto failure; 648180740Sdes } 649180740Sdes 650180740Sdes /* did the transfer complete normally? */ 651180740Sdes if (us.size != -1 && count < us.size) { 652180740Sdes warnx("%s appears to be truncated: %jd/%jd bytes", 653180740Sdes path, (intmax_t)count, (intmax_t)us.size); 654180740Sdes goto failure_keep; 655180740Sdes } 656180740Sdes 657180740Sdes /* 658180740Sdes * If the transfer timed out and we didn't know how much to 659180740Sdes * expect, assume the worst (i.e. we didn't get all of it) 660180740Sdes */ 661180740Sdes if (sigalrm && us.size == -1) { 662180740Sdes warnx("%s may be truncated", path); 663180740Sdes goto failure_keep; 664180740Sdes } 665180740Sdes 666180740Sdes success: 667180740Sdes r = 0; 668180740Sdes if (tmppath != NULL && rename(tmppath, path) == -1) { 669180740Sdes warn("%s: rename()", path); 670180740Sdes goto failure_keep; 671180740Sdes } 672180740Sdes goto done; 673180740Sdes failure: 674180740Sdes if (of && of != stdout && !R_flag && !r_flag) 675180740Sdes if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) 676180740Sdes unlink(tmppath ? tmppath : path); 677180740Sdes if (R_flag && tmppath != NULL && sb.st_size == -1) 678180740Sdes rename(tmppath, path); /* ignore errors here */ 679180740Sdes failure_keep: 680180740Sdes r = -1; 681180740Sdes goto done; 682180740Sdes done: 683180740Sdes if (f) 684180740Sdes fclose(f); 685180740Sdes if (of && of != stdout) 686180740Sdes fclose(of); 687180740Sdes if (url) 688180740Sdes fetchFreeURL(url); 689180740Sdes if (tmppath != NULL) 690180740Sdes free(tmppath); 691180740Sdes return r; 692180740Sdes} 693180740Sdes 694180740Sdesstatic void 695180740Sdesusage(void) 696207311Sdes{ 697180740Sdes fprintf(stderr, "%s\n%s\n%s\n", 698180740Sdes "usage: fetch [-146AFMPRUadlmnpqrsv] [-N netrc] [-o outputfile]", 699180740Sdes " [-S bytes] [-B bytes] [-T seconds] [-w seconds]", 700180740Sdes " [-h host -f file [-c dir] | URL ...]"); 701180740Sdes} 702180740Sdes 703180740Sdes 704180740Sdes/* 705180740Sdes * Entry point 706180740Sdes */ 707180740Sdesint 708180740Sdesmain(int argc, char *argv[]) 709180740Sdes{ 710180740Sdes struct stat sb; 711180740Sdes struct sigaction sa; 712180740Sdes const char *p, *s; 713180740Sdes char *end, *q; 714180740Sdes int c, e, r; 715180740Sdes 716180740Sdes while ((c = getopt(argc, argv, 717197670Sdes "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1) 718180740Sdes switch (c) { 719180740Sdes case '1': 720180740Sdes once_flag = 1; 721180740Sdes break; 722180740Sdes case '4': 723180740Sdes family = PF_INET; 724180740Sdes break; 725180740Sdes case '6': 726180740Sdes family = PF_INET6; 727180750Sdes break; 728180740Sdes case 'A': 729180740Sdes A_flag = 1; 730180740Sdes break; 731180740Sdes case 'a': 732180740Sdes a_flag = 1; 733180740Sdes break; 734180740Sdes case 'B': 735180740Sdes B_size = (off_t)strtol(optarg, &end, 10); 736180740Sdes if (*optarg == '\0' || *end != '\0') 737180740Sdes errx(1, "invalid buffer size (%s)", optarg); 738180740Sdes break; 739180740Sdes case 'b': 740180740Sdes warnx("warning: the -b option is deprecated"); 741180740Sdes b_flag = 1; 742180740Sdes break; 743180740Sdes case 'c': 744180740Sdes c_dirname = optarg; 745180740Sdes break; 746180740Sdes case 'd': 747180740Sdes d_flag = 1; 748180740Sdes break; 749180740Sdes case 'F': 750180740Sdes F_flag = 1; 751180740Sdes break; 752180740Sdes case 'f': 753180740Sdes f_filename = optarg; 754180740Sdes break; 755180740Sdes case 'H': 756180740Sdes warnx("the -H option is now implicit, " 757180740Sdes "use -U to disable"); 758180740Sdes break; 759180740Sdes case 'h': 760180740Sdes h_hostname = optarg; 761180740Sdes break; 762180740Sdes case 'l': 763180740Sdes l_flag = 1; 764180740Sdes break; 765180740Sdes case 'o': 766180740Sdes o_flag = 1; 767180740Sdes o_filename = optarg; 768180740Sdes break; 769180740Sdes case 'M': 770180740Sdes case 'm': 771180740Sdes if (r_flag) 772180740Sdes errx(1, "the -m and -r flags " 773180740Sdes "are mutually exclusive"); 774180740Sdes m_flag = 1; 775180740Sdes break; 776180740Sdes case 'N': 777180740Sdes N_filename = optarg; 778180740Sdes break; 779180740Sdes case 'n': 780180740Sdes n_flag = 1; 781180740Sdes break; 782180740Sdes case 'P': 783180740Sdes case 'p': 784180740Sdes p_flag = 1; 785180740Sdes break; 786180740Sdes case 'q': 787180740Sdes v_level = 0; 788180740Sdes break; 789180740Sdes case 'R': 790180740Sdes R_flag = 1; 791180740Sdes break; 792180740Sdes case 'r': 793180740Sdes if (m_flag) 794180740Sdes errx(1, "the -m and -r flags " 795180740Sdes "are mutually exclusive"); 796180740Sdes r_flag = 1; 797180740Sdes break; 798180740Sdes case 'S': 799180740Sdes S_size = (off_t)strtol(optarg, &end, 10); 800180740Sdes if (*optarg == '\0' || *end != '\0') 801180740Sdes errx(1, "invalid size (%s)", optarg); 802180740Sdes break; 803180740Sdes case 's': 804180740Sdes s_flag = 1; 805180740Sdes break; 806180740Sdes case 'T': 807180740Sdes T_secs = strtol(optarg, &end, 10); 808180740Sdes if (*optarg == '\0' || *end != '\0') 809180740Sdes errx(1, "invalid timeout (%s)", optarg); 810180740Sdes break; 811180740Sdes case 't': 812180740Sdes t_flag = 1; 813180740Sdes warnx("warning: the -t option is deprecated"); 814180740Sdes break; 815180740Sdes case 'U': 816180740Sdes U_flag = 1; 817180740Sdes break; 818180740Sdes case 'v': 819180740Sdes v_level++; 820180740Sdes break; 821180740Sdes case 'w': 822180740Sdes a_flag = 1; 823180740Sdes w_secs = strtol(optarg, &end, 10); 824180740Sdes if (*optarg == '\0' || *end != '\0') 825180740Sdes errx(1, "invalid delay (%s)", optarg); 826180740Sdes break; 827180740Sdes default: 828180740Sdes usage(); 829180740Sdes exit(EX_USAGE); 830180740Sdes } 831180740Sdes 832180740Sdes argc -= optind; 833180740Sdes argv += optind; 834180740Sdes 835180740Sdes if (h_hostname || f_filename || c_dirname) { 836180740Sdes if (!h_hostname || !f_filename || argc) { 837180740Sdes usage(); 838180740Sdes exit(EX_USAGE); 839180740Sdes } 840180740Sdes /* XXX this is a hack. */ 841180740Sdes if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 842180740Sdes errx(1, "invalid hostname"); 843204861Sdes if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 844180740Sdes c_dirname ? c_dirname : "", f_filename) == -1) 845204861Sdes errx(1, "%s", strerror(ENOMEM)); 846204861Sdes argc++; 847180740Sdes } 848204861Sdes 849204861Sdes if (!argc) { 850180740Sdes usage(); 851180740Sdes exit(EX_USAGE); 852180740Sdes } 853180740Sdes 854180740Sdes /* allocate buffer */ 855180740Sdes if (B_size < MINBUFSIZE) 856180740Sdes B_size = MINBUFSIZE; 857180740Sdes if ((buf = malloc(B_size)) == NULL) 858180740Sdes errx(1, "%s", strerror(ENOMEM)); 859180740Sdes 860180740Sdes /* timeouts */ 861180740Sdes if ((s = getenv("FTP_TIMEOUT")) != NULL) { 862204861Sdes ftp_timeout = strtol(s, &end, 10); 863180740Sdes if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { 864204861Sdes warnx("FTP_TIMEOUT (%s) is not a positive integer", s); 865204861Sdes ftp_timeout = 0; 866180740Sdes } 867204861Sdes } 868204861Sdes if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 869180740Sdes http_timeout = strtol(s, &end, 10); 870180740Sdes if (*s == '\0' || *end != '\0' || http_timeout < 0) { 871180740Sdes warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); 872180740Sdes http_timeout = 0; 873180740Sdes } 874180740Sdes } 875180740Sdes 876180740Sdes /* signal handling */ 877180740Sdes sa.sa_flags = 0; 878180740Sdes sa.sa_handler = sig_handler; 879180740Sdes sigemptyset(&sa.sa_mask); 880180740Sdes sigaction(SIGALRM, &sa, NULL); 881180740Sdes sa.sa_flags = SA_RESETHAND; 882180740Sdes sigaction(SIGINT, &sa, NULL); 883180740Sdes fetchRestartCalls = 0; 884180740Sdes 885180740Sdes /* output file */ 886180740Sdes if (o_flag) { 887180740Sdes if (strcmp(o_filename, "-") == 0) { 888180740Sdes o_stdout = 1; 889180740Sdes } else if (stat(o_filename, &sb) == -1) { 890180740Sdes if (errno == ENOENT) { 891180740Sdes if (argc > 1) 892180740Sdes errx(EX_USAGE, "%s is not a directory", 893180740Sdes o_filename); 894180740Sdes } else { 895180740Sdes err(EX_IOERR, "%s", o_filename); 896180740Sdes } 897180740Sdes } else { 898180740Sdes if (sb.st_mode & S_IFDIR) 899180740Sdes o_directory = 1; 900180740Sdes } 901180740Sdes } 902180740Sdes 903180740Sdes /* check if output is to a tty (for progress report) */ 904180740Sdes v_tty = isatty(STDERR_FILENO); 905180740Sdes if (v_tty) 906180740Sdes pgrp = getpgrp(); 907180740Sdes 908180740Sdes r = 0; 909180740Sdes 910180740Sdes /* authentication */ 911180740Sdes if (v_tty) 912180740Sdes fetchAuthMethod = query_auth; 913180740Sdes if (N_filename != NULL) 914180740Sdes setenv("NETRC", N_filename, 1); 915180740Sdes 916180740Sdes while (argc) { 917180740Sdes if ((p = strrchr(*argv, '/')) == NULL) 918180740Sdes p = *argv; 919180740Sdes else 920180740Sdes p++; 921180740Sdes 922180740Sdes if (!*p) 923180740Sdes p = "fetch.out"; 924180740Sdes 925180740Sdes fetchLastErrCode = 0; 926180740Sdes 927180740Sdes if (o_flag) { 928180740Sdes if (o_stdout) { 929180740Sdes e = fetch(*argv, "-"); 930180740Sdes } else if (o_directory) { 931180740Sdes asprintf(&q, "%s/%s", o_filename, p); 932180740Sdes e = fetch(*argv, q); 933180740Sdes free(q); 934180740Sdes } else { 935180740Sdes e = fetch(*argv, o_filename); 936180740Sdes } 937180740Sdes } else { 938180740Sdes e = fetch(*argv, p); 939180740Sdes } 940180740Sdes 941180740Sdes if (sigint) 942180740Sdes kill(getpid(), SIGINT); 943180740Sdes 944180740Sdes if (e == 0 && once_flag) 945180740Sdes exit(0); 946180740Sdes 947180740Sdes if (e) { 948180740Sdes r = 1; 949180740Sdes if ((fetchLastErrCode 950180740Sdes && fetchLastErrCode != FETCH_UNAVAIL 951180740Sdes && fetchLastErrCode != FETCH_MOVED 952180740Sdes && fetchLastErrCode != FETCH_URL 953180740Sdes && fetchLastErrCode != FETCH_RESOLV 954180740Sdes && fetchLastErrCode != FETCH_UNKNOWN)) { 955180740Sdes if (w_secs && v_level) 956180740Sdes fprintf(stderr, "Waiting %ld seconds " 957180740Sdes "before retrying\n", w_secs); 958180740Sdes if (w_secs) 959180740Sdes sleep(w_secs); 960180740Sdes if (a_flag) 961180740Sdes continue; 962180740Sdes } 963180740Sdes } 964180740Sdes 965180740Sdes argc--, argv++; 966180740Sdes } 967180740Sdes 968180740Sdes exit(r); 969180740Sdes} 970180740Sdes