fetch.c revision 106042
1106266Sjulian/*- 2106266Sjulian * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav 3139823Simp * All rights reserved. 4139823Simp * 5139823Simp * Redistribution and use in source and binary forms, with or without 6144674Sglebius * modification, are permitted provided that the following conditions 7106266Sjulian * are met: 8106266Sjulian * 1. Redistributions of source code must retain the above copyright 9106266Sjulian * notice, this list of conditions and the following disclaimer 10106266Sjulian * in this position and unchanged. 11106266Sjulian * 2. Redistributions in binary form must reproduce the above copyright 12106319Sjulian * notice, this list of conditions and the following disclaimer in the 13106266Sjulian * documentation and/or other materials provided with the distribution. 14106319Sjulian * 3. The name of the author may not be used to endorse or promote products 15106319Sjulian * derived from this software without specific prior written permission 16106266Sjulian * 17106319Sjulian * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18106319Sjulian * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19106266Sjulian * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20106266Sjulian * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21106266Sjulian * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22106319Sjulian * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23106319Sjulian * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24106319Sjulian * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25106266Sjulian * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26106266Sjulian * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27106266Sjulian */ 28106266Sjulian 29106266Sjulian#include <sys/cdefs.h> 30106319Sjulian__FBSDID("$FreeBSD: head/usr.bin/fetch/fetch.c 106042 2002-10-27 15:32:06Z des $"); 31106266Sjulian 32106266Sjulian#include <sys/param.h> 33106266Sjulian#include <sys/socket.h> 34106266Sjulian#include <sys/stat.h> 35106266Sjulian#include <sys/time.h> 36106266Sjulian 37106266Sjulian#include <ctype.h> 38106266Sjulian#include <err.h> 39106266Sjulian#include <errno.h> 40106266Sjulian#include <signal.h> 41125077Sharti#include <stdio.h> 42125077Sharti#include <stdlib.h> 43125077Sharti#include <string.h> 44106266Sjulian#include <sysexits.h> 45106266Sjulian#include <termios.h> 46143387Sbmilekic#include <unistd.h> 47143387Sbmilekic 48143387Sbmilekic#include <fetch.h> 49143387Sbmilekic 50106266Sjulian#define MINBUFSIZE 4096 51106266Sjulian 52106266Sjulian/* Option flags */ 53106266Sjulianint A_flag; /* -A: do not follow 302 redirects */ 54106266Sjulianint a_flag; /* -a: auto retry */ 55106266Sjulianoff_t B_size; /* -B: buffer size */ 56143387Sbmilekicint b_flag; /*! -b: workaround TCP bug */ 57106266Sjulianchar *c_dirname; /* -c: remote directory */ 58106266Sjulianint d_flag; /* -d: direct connection */ 59106266Sjulianint F_flag; /* -F: restart without checking mtime */ 60106266Sjulianchar *f_filename; /* -f: file to fetch */ 61106266Sjulianchar *h_hostname; /* -h: host to fetch from */ 62106266Sjulianint l_flag; /* -l: link rather than copy file: URLs */ 63106266Sjulianint m_flag; /* -[Mm]: mirror mode */ 64106266Sjulianint n_flag; /* -n: do not preserve modification time */ 65106266Sjulianint o_flag; /* -o: specify output file */ 66106266Sjulianint o_directory; /* output file is a directory */ 67106266Sjulianchar *o_filename; /* name of output file */ 68144674Sglebiusint o_stdout; /* output file is stdout */ 69106266Sjulianint once_flag; /* -1: stop at first successful file */ 70106266Sjulianint p_flag; /* -[Pp]: use passive FTP */ 71106266Sjulianint R_flag; /* -R: don't delete partially transferred files */ 72106266Sjulianint r_flag; /* -r: restart previously interrupted transfer */ 73106266Sjulianoff_t S_size; /* -S: require size to match */ 74106266Sjulianint s_flag; /* -s: show size, don't fetch */ 75106266Sjulianlong T_secs = 120; /* -T: transfer timeout in seconds */ 76106266Sjulianint t_flag; /*! -t: workaround TCP bug */ 77106266Sjulianint U_flag; /* -U: do not use high ports */ 78106266Sjulianint v_level = 1; /* -v: verbosity level */ 79106266Sjulianint v_tty; /* stdout is a tty */ 80167156Semastepid_t pgrp; /* our process group */ 81167156Semastelong w_secs; /* -w: retry delay */ 82106266Sjulianint family = PF_UNSPEC; /* -[46]: address family to use */ 83106266Sjulian 84106266Sjulianint sigalrm; /* SIGALRM received */ 85144674Sglebiusint siginfo; /* SIGINFO received */ 86144674Sglebiusint sigint; /* SIGINT received */ 87106266Sjulian 88106319Sjulianlong ftp_timeout; /* default timeout for FTP transfers */ 89106266Sjulianlong http_timeout; /* default timeout for HTTP transfers */ 90137138Sglebiusu_char *buf; /* transfer buffer */ 91144674Sglebius 92144674Sglebius 93167156Semaste/* 94106266Sjulian * Signal handler 95106266Sjulian */ 96106266Sjulianstatic void 97106266Sjuliansig_handler(int sig) 98106266Sjulian{ 99106266Sjulian switch (sig) { 100106266Sjulian case SIGALRM: 101106266Sjulian sigalrm = 1; 102106266Sjulian break; 103106266Sjulian case SIGINFO: 104106266Sjulian siginfo = 1; 105144674Sglebius break; 106106266Sjulian case SIGINT: 107106266Sjulian sigint = 1; 108106266Sjulian break; 109106266Sjulian } 110125243Sharti} 111106266Sjulian 112144674Sglebiusstruct xferstat { 113106266Sjulian char name[40]; 114106266Sjulian struct timeval start; 115144674Sglebius struct timeval end; 116167156Semaste struct timeval last; 117167156Semaste off_t size; 118167156Semaste off_t offset; 119167156Semaste off_t rcvd; 120106266Sjulian}; 121106266Sjulian 122125077Sharti/* 123106266Sjulian * Update the stats display 124106266Sjulian */ 125106266Sjulianstatic void 126106266Sjulianstat_display(struct xferstat *xs, int force) 127106266Sjulian{ 128106266Sjulian struct timeval now; 129106266Sjulian int ctty_pgrp; 130106266Sjulian 131106266Sjulian if (!v_tty || !v_level) 132106266Sjulian return; 133106266Sjulian 134106266Sjulian /* check if we're the foreground process */ 135106266Sjulian if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || 136106266Sjulian (pid_t)ctty_pgrp != pgrp) 137106266Sjulian return; 138106266Sjulian 139106266Sjulian gettimeofday(&now, NULL); 140167156Semaste if (!force && now.tv_sec <= xs->last.tv_sec) 141167156Semaste return; 142167156Semaste xs->last = now; 143167156Semaste 144167156Semaste fprintf(stderr, "\rReceiving %s", xs->name); 145167156Semaste if (xs->size <= 0) { 146167156Semaste fprintf(stderr, ": %lld bytes", (long long)xs->rcvd); 147167156Semaste } else { 148106266Sjulian long elapsed; 149106266Sjulian 150106266Sjulian fprintf(stderr, " (%lld bytes): %d%%", (long long)xs->size, 151106266Sjulian (int)((100.0 * xs->rcvd) / xs->size)); 152106266Sjulian elapsed = xs->last.tv_sec - xs->start.tv_sec; 153106266Sjulian if (elapsed > 30) { 154106266Sjulian long remaining; 155106266Sjulian 156106266Sjulian remaining = ((xs->size * elapsed) / xs->rcvd) - elapsed; 157106266Sjulian fprintf(stderr, " (ETA "); 158106266Sjulian if (remaining > 3600) { 159106266Sjulian fprintf(stderr, "%02ld:", remaining / 3600); 160106266Sjulian remaining %= 3600; 161106266Sjulian } 162106266Sjulian fprintf(stderr, "%02ld:%02ld) ", 163106266Sjulian remaining / 60, remaining % 60); 164106266Sjulian } 165106266Sjulian } 166106266Sjulian} 167106266Sjulian 168106266Sjulian/* 169106266Sjulian * Initialize the transfer statistics 170106266Sjulian */ 171106266Sjulianstatic void 172106266Sjulianstat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) 173106266Sjulian{ 174106266Sjulian snprintf(xs->name, sizeof xs->name, "%s", name); 175106266Sjulian gettimeofday(&xs->start, NULL); 176106266Sjulian xs->last.tv_sec = xs->last.tv_usec = 0; 177106266Sjulian xs->end = xs->last; 178106266Sjulian xs->size = size; 179106266Sjulian xs->offset = offset; 180106266Sjulian xs->rcvd = offset; 181106266Sjulian stat_display(xs, 1); 182106266Sjulian} 183106266Sjulian 184106266Sjulian/* 185106266Sjulian * Update the transfer statistics 186106266Sjulian */ 187106266Sjulianstatic void 188106266Sjulianstat_update(struct xferstat *xs, off_t rcvd) 189106266Sjulian{ 190106266Sjulian xs->rcvd = rcvd; 191106266Sjulian stat_display(xs, 0); 192125033Sharti} 193125033Sharti 194144674Sglebius/* 195144674Sglebius * Finalize the transfer statistics 196144674Sglebius */ 197125033Shartistatic void 198125033Shartistat_end(struct xferstat *xs) 199153690Sglebius{ 200153690Sglebius double delta; 201153690Sglebius double bps; 202153690Sglebius 203153690Sglebius if (!v_level) 204153690Sglebius return; 205153690Sglebius 206167156Semaste gettimeofday(&xs->end, NULL); 207167156Semaste 208167156Semaste stat_display(xs, 1); 209167156Semaste fputc('\n', stderr); 210167156Semaste delta = (xs->end.tv_sec + (xs->end.tv_usec / 1.e6)) 211167156Semaste - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 212167156Semaste fprintf(stderr, "%lld bytes transferred in %.1f seconds ", 213167156Semaste (long long)(xs->rcvd - xs->offset), delta); 214167156Semaste bps = (xs->rcvd - xs->offset) / delta; 215167156Semaste if (bps > 1024*1024) 216167156Semaste fprintf(stderr, "(%.2f MBps)\n", bps / (1024*1024)); 217167156Semaste else if (bps > 1024) 218167156Semaste fprintf(stderr, "(%.2f kBps)\n", bps / 1024); 219167156Semaste else 220106266Sjulian fprintf(stderr, "(%.2f Bps)\n", bps); 221106266Sjulian} 222106266Sjulian 223106266Sjulian/* 224106266Sjulian * Ask the user for authentication details 225129823Sjulian */ 226129823Sjulianstatic int 227129823Sjulianquery_auth(struct url *URL) 228129823Sjulian{ 229129823Sjulian struct termios tios; 230129823Sjulian tcflag_t saved_flags; 231144674Sglebius int i, nopwd; 232129823Sjulian 233129823Sjulian 234129823Sjulian fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", 235106266Sjulian URL->scheme, URL->host, URL->port); 236106266Sjulian 237106266Sjulian fprintf(stderr, "Login: "); 238144674Sglebius if (fgets(URL->user, sizeof URL->user, stdin) == NULL) 239125032Sharti return -1; 240106266Sjulian for (i = 0; URL->user[i]; ++i) 241106266Sjulian if (isspace(URL->user[i])) 242106266Sjulian URL->user[i] = '\0'; 243106266Sjulian 244106321Sjulian fprintf(stderr, "Password: "); 245106266Sjulian if (tcgetattr(STDIN_FILENO, &tios) == 0) { 246106266Sjulian saved_flags = tios.c_lflag; 247106266Sjulian tios.c_lflag &= ~ECHO; 248125030Sharti tios.c_lflag |= ECHONL|ICANON; 249106266Sjulian tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); 250106266Sjulian nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 251106266Sjulian tios.c_lflag = saved_flags; 252106321Sjulian tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); 253106321Sjulian } else { 254106266Sjulian nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); 255137138Sglebius } 256137138Sglebius if (nopwd) 257106266Sjulian return -1; 258106266Sjulian 259106266Sjulian for (i = 0; URL->pwd[i]; ++i) 260106266Sjulian if (isspace(URL->pwd[i])) 261106266Sjulian URL->pwd[i] = '\0'; 262106266Sjulian return 0; 263106266Sjulian} 264106266Sjulian 265106266Sjulian/* 266144674Sglebius * Fetch a file 267106266Sjulian */ 268106266Sjulianstatic int 269144674Sglebiusfetch(char *URL, const char *path) 270106266Sjulian{ 271144674Sglebius struct url *url; 272106266Sjulian struct url_stat us; 273106266Sjulian struct stat sb, nsb; 274106266Sjulian struct xferstat xs; 275106266Sjulian FILE *f, *of; 276144674Sglebius size_t size, wr; 277106266Sjulian off_t count; 278106266Sjulian char flags[8]; 279106266Sjulian const char *slash; 280106266Sjulian char *tmppath; 281144674Sglebius int r; 282144674Sglebius u_int timeout; 283144674Sglebius u_char *ptr; 284144674Sglebius 285144674Sglebius f = of = NULL; 286144674Sglebius tmppath = NULL; 287144674Sglebius 288144674Sglebius /* parse URL */ 289144674Sglebius if ((url = fetchParseURL(URL)) == NULL) { 290144674Sglebius warnx("%s: parse error", URL); 291144674Sglebius goto failure; 292144674Sglebius } 293144674Sglebius 294144674Sglebius /* if no scheme was specified, take a guess */ 295144674Sglebius if (!*url->scheme) { 296144674Sglebius if (!*url->host) 297144674Sglebius strcpy(url->scheme, SCHEME_FILE); 298144674Sglebius else if (strncasecmp(url->host, "ftp.", 4) == 0) 299144674Sglebius strcpy(url->scheme, SCHEME_FTP); 300144674Sglebius else if (strncasecmp(url->host, "www.", 4) == 0) 301144674Sglebius strcpy(url->scheme, SCHEME_HTTP); 302144674Sglebius } 303144674Sglebius 304144674Sglebius timeout = 0; 305144674Sglebius *flags = 0; 306144674Sglebius count = 0; 307144674Sglebius 308144674Sglebius /* common flags */ 309144674Sglebius if (v_level > 1) 310144674Sglebius strcat(flags, "v"); 311144674Sglebius if (v_level > 2) 312106266Sjulian fetchDebug = 1; 313106266Sjulian switch (family) { 314106266Sjulian case PF_INET: 315106321Sjulian strcat(flags, "4"); 316106266Sjulian break; 317144674Sglebius case PF_INET6: 318144674Sglebius strcat(flags, "6"); 319106266Sjulian break; 320106266Sjulian } 321106321Sjulian 322144674Sglebius /* FTP specific flags */ 323106266Sjulian if (strcmp(url->scheme, "ftp") == 0) { 324106266Sjulian if (p_flag) 325106435Sjulian strcat(flags, "p"); 326106435Sjulian if (d_flag) 327106435Sjulian strcat(flags, "d"); 328106435Sjulian if (U_flag) 329106266Sjulian strcat(flags, "l"); 330106266Sjulian timeout = T_secs ? T_secs : ftp_timeout; 331106266Sjulian } 332106266Sjulian 333106266Sjulian /* HTTP specific flags */ 334106266Sjulian if (strcmp(url->scheme, "http") == 0) { 335106266Sjulian if (d_flag) 336106266Sjulian strcat(flags, "d"); 337106266Sjulian if (A_flag) 338106266Sjulian strcat(flags, "A"); 339106266Sjulian timeout = T_secs ? T_secs : http_timeout; 340106266Sjulian } 341106266Sjulian 342106266Sjulian /* set the protocol timeout. */ 343106266Sjulian fetchTimeout = timeout; 344106319Sjulian 345106321Sjulian /* just print size */ 346106266Sjulian if (s_flag) { 347106319Sjulian if (timeout) 348106266Sjulian alarm(timeout); 349106319Sjulian r = fetchStat(url, &us, flags); 350106266Sjulian if (timeout) 351106319Sjulian alarm(0); 352106266Sjulian if (sigalrm || sigint) 353106266Sjulian goto signal; 354106266Sjulian if (r == -1) { 355106266Sjulian warnx("%s", fetchLastErrString); 356106266Sjulian goto failure; 357106266Sjulian } 358106266Sjulian if (us.size == -1) 359106266Sjulian printf("Unknown\n"); 360144674Sglebius else 361144674Sglebius printf("%lld\n", (long long)us.size); 362144674Sglebius goto success; 363106266Sjulian } 364106266Sjulian 365106266Sjulian /* 366125033Sharti * If the -r flag was specified, we have to compare the local 367144674Sglebius * and remote files, so we should really do a fetchStat() 368144674Sglebius * first, but I know of at least one HTTP server that only 369144674Sglebius * sends the content size in response to GET requests, and 370144674Sglebius * leaves it out of replies to HEAD requests. Also, in the 371144674Sglebius * (frequent) case that the local and remote files match but 372125033Sharti * the local file is truncated, we have sufficient information 373106266Sjulian * before the compare to issue a correct request. Therefore, 374106266Sjulian * we always issue a GET request as if we were sure the local 375106266Sjulian * file was a truncated copy of the remote file; we can drop 376106266Sjulian * the connection later if we change our minds. 377106266Sjulian */ 378106266Sjulian sb.st_size = -1; 379144674Sglebius if (!o_stdout && stat(path, &sb) == -1 && errno != ENOENT) { 380144674Sglebius warnx("%s: stat()", path); 381144674Sglebius goto failure; 382144674Sglebius } 383144674Sglebius if (!o_stdout && r_flag && S_ISREG(sb.st_mode)) 384144674Sglebius url->offset = sb.st_size; 385144674Sglebius 386144674Sglebius /* start the transfer */ 387144674Sglebius if (timeout) 388144674Sglebius alarm(timeout); 389144674Sglebius f = fetchXGet(url, &us, flags); 390144674Sglebius if (timeout) 391153690Sglebius alarm(0); 392153690Sglebius if (sigalrm || sigint) 393153690Sglebius goto signal; 394153690Sglebius if (f == NULL) { 395153690Sglebius warnx("%s: %s", path, fetchLastErrString); 396153690Sglebius goto failure; 397153690Sglebius } 398153690Sglebius if (sigint) 399153690Sglebius goto signal; 400153690Sglebius 401153690Sglebius /* check that size is as expected */ 402153690Sglebius if (S_size) { 403153690Sglebius if (us.size == -1) { 404153690Sglebius warnx("%s: size unknown", path); 405153690Sglebius goto failure; 406167156Semaste } else if (us.size != S_size) { 407167156Semaste warnx("%s: size mismatch: expected %lld, actual %lld", 408167156Semaste path, (long long)S_size, (long long)us.size); 409167156Semaste goto failure; 410167156Semaste } 411167156Semaste } 412167156Semaste 413167156Semaste /* symlink instead of copy */ 414167156Semaste if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 415167156Semaste if (symlink(url->doc, path) == -1) { 416167156Semaste warn("%s: symlink()", path); 417167156Semaste goto failure; 418167156Semaste } 419167156Semaste goto success; 420167156Semaste } 421167156Semaste 422167156Semaste if (us.size == -1 && !o_stdout) 423167156Semaste warnx("%s: size of remote file is not known", path); 424167156Semaste if (v_level > 1) { 425167156Semaste if (sb.st_size != -1) 426167156Semaste fprintf(stderr, "local size / mtime: %lld / %ld\n", 427167156Semaste (long long)sb.st_size, (long)sb.st_mtime); 428167156Semaste if (us.size != -1) 429106266Sjulian fprintf(stderr, "remote size / mtime: %lld / %ld\n", 430106266Sjulian (long long)us.size, (long)us.mtime); 431106266Sjulian } 432106266Sjulian 433106266Sjulian /* open output file */ 434106435Sjulian if (o_stdout) { 435106435Sjulian /* output to stdout */ 436106435Sjulian of = stdout; 437106435Sjulian } else if (r_flag && sb.st_size != -1) { 438106435Sjulian /* resume mode, local file exists */ 439106435Sjulian if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { 440144674Sglebius /* no match! have to refetch */ 441144674Sglebius fclose(f); 442144674Sglebius /* if precious, warn the user and give up */ 443144674Sglebius if (R_flag) { 444144674Sglebius warnx("%s: local modification time " 445144674Sglebius "does not match remote", path); 446144674Sglebius goto failure_keep; 447144674Sglebius } 448144674Sglebius } else { 449144674Sglebius if (us.size == sb.st_size) 450106435Sjulian /* nothing to do */ 451106435Sjulian goto success; 452144674Sglebius if (sb.st_size > us.size) { 453106435Sjulian /* local file too long! */ 454106435Sjulian warnx("%s: local file (%lld bytes) is longer " 455106435Sjulian "than remote file (%lld bytes)", path, 456106435Sjulian (long long)sb.st_size, (long long)us.size); 457106266Sjulian goto failure; 458106266Sjulian } 459106266Sjulian /* we got it, open local file */ 460106266Sjulian if ((of = fopen(path, "a")) == NULL) { 461106266Sjulian warn("%s: fopen()", path); 462106266Sjulian goto failure; 463144674Sglebius } 464106321Sjulian /* check that it didn't move under our feet */ 465144674Sglebius if (fstat(fileno(of), &nsb) == -1) { 466106321Sjulian /* can't happen! */ 467106266Sjulian warn("%s: fstat()", path); 468106266Sjulian goto failure; 469106266Sjulian } 470106266Sjulian if (nsb.st_dev != sb.st_dev || 471106266Sjulian nsb.st_ino != nsb.st_ino || 472106266Sjulian nsb.st_size != sb.st_size) { 473106266Sjulian warnx("%s: file has changed", path); 474106266Sjulian fclose(of); 475106266Sjulian of = NULL; 476106266Sjulian sb = nsb; 477106321Sjulian } 478106266Sjulian } 479144674Sglebius } else if (m_flag && sb.st_size != -1) { 480144674Sglebius /* mirror mode, local file exists */ 481106266Sjulian if (sb.st_size == us.size && sb.st_mtime == us.mtime) 482106266Sjulian goto success; 483106321Sjulian } 484106321Sjulian 485106266Sjulian if (of == NULL) { 486106266Sjulian /* 487144674Sglebius * We don't yet have an output file; either this is a 488106266Sjulian * vanilla run with no special flags, or the local and 489106321Sjulian * remote files didn't match. 490106266Sjulian */ 491106266Sjulian 492144674Sglebius if (url->offset != 0) { 493106266Sjulian /* 494144674Sglebius * We tried to restart a transfer, but for 495106266Sjulian * some reason gave up - so we have to restart 496125031Sharti * from scratch if we want the whole file 497106266Sjulian */ 498106266Sjulian url->offset = 0; 499106266Sjulian if ((f = fetchXGet(url, &us, flags)) == NULL) { 500106266Sjulian warnx("%s: %s", path, fetchLastErrString); 501106266Sjulian goto failure; 502106266Sjulian } 503106266Sjulian if (sigint) 504106266Sjulian goto signal; 505106266Sjulian } 506106266Sjulian 507106266Sjulian /* construct a temp file name */ 508144674Sglebius if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { 509106266Sjulian if ((slash = strrchr(path, '/')) == NULL) 510106266Sjulian slash = path; 511106266Sjulian else 512106321Sjulian ++slash; 513106321Sjulian asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", 514125077Sharti (int)(slash - path), path, slash); 515144674Sglebius if (tmppath != NULL) { 516106266Sjulian mkstemps(tmppath, strlen(slash) + 1); 517106266Sjulian of = fopen(tmppath, "w"); 518106266Sjulian } 519106266Sjulian } 520106266Sjulian if (of == NULL) 521106266Sjulian of = fopen(path, "w"); 522106266Sjulian if (of == NULL) { 523106266Sjulian warn("%s: open()", path); 524106266Sjulian goto failure; 525106319Sjulian } 526106266Sjulian } 527106321Sjulian count = url->offset; 528125077Sharti 529144674Sglebius /* start the counter */ 530106321Sjulian stat_start(&xs, path, us.size, count); 531106266Sjulian 532106266Sjulian sigalrm = siginfo = sigint = 0; 533106266Sjulian 534106266Sjulian /* suck in the data */ 535106435Sjulian signal(SIGINFO, sig_handler); 536106435Sjulian while (!sigint) { 537106435Sjulian if (us.size != -1 && us.size - count < B_size) 538106435Sjulian size = us.size - count; 539144674Sglebius else 540106435Sjulian size = B_size; 541106435Sjulian if (siginfo) { 542106435Sjulian stat_end(&xs); 543106266Sjulian siginfo = 0; 544144674Sglebius } 545106266Sjulian if ((size = fread(buf, 1, size, f)) == 0) { 546106266Sjulian if (ferror(f) && errno == EINTR && !sigint) 547125077Sharti clearerr(f); 548106266Sjulian else 549106266Sjulian break; 550106266Sjulian } 551106266Sjulian stat_update(&xs, count += size); 552106266Sjulian for (ptr = buf; size > 0; ptr += wr, size -= wr) 553106266Sjulian if ((wr = fwrite(ptr, 1, size, of)) < size) { 554106266Sjulian if (ferror(of) && errno == EINTR && !sigint) 555106266Sjulian clearerr(of); 556106266Sjulian else 557106266Sjulian break; 558106266Sjulian } 559125077Sharti if (size != 0) 560106266Sjulian break; 561106319Sjulian } 562106266Sjulian if (!sigalrm) 563106266Sjulian sigalrm = ferror(f) && errno == ETIMEDOUT; 564106266Sjulian signal(SIGINFO, SIG_DFL); 565106266Sjulian 566106266Sjulian stat_end(&xs); 567106266Sjulian 568106266Sjulian /* 569106266Sjulian * If the transfer timed out or was interrupted, we still want to 570106266Sjulian * set the mtime in case the file is not removed (-r or -R) and 571106266Sjulian * the user later restarts the transfer. 572106266Sjulian */ 573144674Sglebius signal: 574106266Sjulian /* set mtime of local file */ 575106266Sjulian if (!n_flag && us.mtime && !o_stdout 576106266Sjulian && (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { 577106266Sjulian struct timeval tv[2]; 578106266Sjulian 579144674Sglebius fflush(of); 580106266Sjulian tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 581106266Sjulian tv[1].tv_sec = (long)us.mtime; 582106266Sjulian tv[0].tv_usec = tv[1].tv_usec = 0; 583144674Sglebius if (utimes(tmppath ? tmppath : path, tv)) 584144674Sglebius warn("%s: utimes()", tmppath ? tmppath : path); 585106266Sjulian } 586106266Sjulian 587106266Sjulian /* timed out or interrupted? */ 588106266Sjulian if (sigalrm) 589106266Sjulian warnx("transfer timed out"); 590106266Sjulian if (sigint) { 591106266Sjulian warnx("transfer interrupted"); 592106266Sjulian goto failure; 593106266Sjulian } 594106266Sjulian 595106266Sjulian if (!sigalrm) { 596106266Sjulian /* check the status of our files */ 597125031Sharti if (ferror(f)) 598106266Sjulian warn("%s", URL); 599106266Sjulian if (ferror(of)) 600106266Sjulian warn("%s", path); 601106266Sjulian if (ferror(f) || ferror(of)) 602106266Sjulian goto failure; 603106266Sjulian } 604106266Sjulian 605106266Sjulian /* did the transfer complete normally? */ 606106266Sjulian if (us.size != -1 && count < us.size) { 607106266Sjulian warnx("%s appears to be truncated: %lld/%lld bytes", 608144674Sglebius path, (long long)count, (long long)us.size); 609144674Sglebius goto failure_keep; 610106266Sjulian } 611144674Sglebius 612144674Sglebius /* 613144674Sglebius * If the transfer timed out and we didn't know how much to 614144674Sglebius * expect, assume the worst (i.e. we didn't get all of it) 615144674Sglebius */ 616144674Sglebius if (sigalrm && us.size == -1) { 617144674Sglebius warnx("%s may be truncated", path); 618144674Sglebius goto failure_keep; 619144674Sglebius } 620144674Sglebius 621144674Sglebius success: 622144674Sglebius r = 0; 623144674Sglebius if (tmppath != NULL && rename(tmppath, path) == -1) { 624144674Sglebius warn("%s: rename()", path); 625153690Sglebius goto failure_keep; 626144674Sglebius } 627144674Sglebius goto done; 628144674Sglebius failure: 629144674Sglebius if (of && of != stdout && !R_flag && !r_flag) 630106266Sjulian if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) 631106266Sjulian unlink(tmppath ? tmppath : path); 632106266Sjulian if (R_flag && tmppath != NULL && sb.st_size == -1) 633106266Sjulian rename(tmppath, path); /* ignore errors here */ 634106266Sjulian failure_keep: 635106266Sjulian r = -1; 636144674Sglebius goto done; 637106266Sjulian done: 638144674Sglebius if (f) 639144674Sglebius fclose(f); 640144674Sglebius if (of && of != stdout) 641144674Sglebius fclose(of); 642144674Sglebius if (url) 643106266Sjulian fetchFreeURL(url); 644106266Sjulian if (tmppath != NULL) 645106266Sjulian free(tmppath); 646106266Sjulian return r; 647106266Sjulian} 648106266Sjulian 649106266Sjulianstatic void 650106266Sjulianusage(void) 651125243Sharti{ 652106266Sjulian fprintf(stderr, "%s\n%s\n%s\n", 653125243Sharti "usage: fetch [-146AFMPRUadlmnpqrsv] [-o outputfile] [-S bytes]", 654106266Sjulian " [-B bytes] [-T seconds] [-w seconds]", 655106266Sjulian " [-h host -f file [-c dir] | URL ...]"); 656106266Sjulian} 657125077Sharti 658106266Sjulian 659144674Sglebius/* 660106321Sjulian * Entry point 661106266Sjulian */ 662106266Sjulianint 663106266Sjulianmain(int argc, char *argv[]) 664106266Sjulian{ 665125033Sharti struct stat sb; 666141745Sru struct sigaction sa; 667125033Sharti const char *p, *s; 668125033Sharti char *end, *q; 669125033Sharti int c, e, r; 670125033Sharti 671153690Sglebius while ((c = getopt(argc, argv, 672153690Sglebius "146AaB:bc:dFf:Hh:lMmnPpo:qRrS:sT:tUvw:")) != -1) 673153690Sglebius switch (c) { 674153690Sglebius case '1': 675153690Sglebius once_flag = 1; 676153690Sglebius break; 677153690Sglebius case '4': 678153690Sglebius family = PF_INET; 679153690Sglebius break; 680153690Sglebius case '6': 681153690Sglebius family = PF_INET6; 682153690Sglebius break; 683153690Sglebius case 'A': 684153690Sglebius A_flag = 1; 685153690Sglebius break; 686106266Sjulian case 'a': 687125243Sharti a_flag = 1; 688106266Sjulian break; 689125243Sharti case 'B': 690138268Sglebius B_size = (off_t)strtol(optarg, &end, 10); 691137136Sglebius if (*optarg == '\0' || *end != '\0') 692106266Sjulian errx(1, "invalid buffer size (%s)", optarg); 693106266Sjulian break; 694106266Sjulian case 'b': 695154707Sglebius warnx("warning: the -b option is deprecated"); 696106266Sjulian b_flag = 1; 697106266Sjulian break; 698154707Sglebius case 'c': 699106266Sjulian c_dirname = optarg; 700106266Sjulian break; 701154707Sglebius case 'd': 702106266Sjulian d_flag = 1; 703106266Sjulian break; 704125077Sharti case 'F': 705106321Sjulian F_flag = 1; 706125077Sharti break; 707106266Sjulian case 'f': 708144674Sglebius f_filename = optarg; 709106266Sjulian break; 710106266Sjulian case 'H': 711154707Sglebius warnx("the -H option is now implicit, " 712106266Sjulian "use -U to disable"); 713125031Sharti break; 714106266Sjulian case 'h': 715106266Sjulian h_hostname = optarg; 716106266Sjulian break; 717167156Semaste case 'l': 718167156Semaste l_flag = 1; 719167156Semaste break; 720167156Semaste case 'o': 721167156Semaste o_flag = 1; 722167156Semaste o_filename = optarg; 723167156Semaste break; 724106266Sjulian case 'M': 725106266Sjulian case 'm': 726106266Sjulian if (r_flag) 727144674Sglebius errx(1, "the -m and -r flags " 728125031Sharti "are mutually exclusive"); 729106266Sjulian m_flag = 1; 730154707Sglebius break; 731154707Sglebius case 'n': 732154707Sglebius n_flag = 1; 733154707Sglebius break; 734106266Sjulian case 'P': 735106266Sjulian case 'p': 736106266Sjulian p_flag = 1; 737106266Sjulian break; 738106266Sjulian case 'q': 739106266Sjulian v_level = 0; 740106266Sjulian break; 741106266Sjulian case 'R': 742167156Semaste R_flag = 1; 743167156Semaste break; 744167156Semaste case 'r': 745167156Semaste if (m_flag) 746167156Semaste errx(1, "the -m and -r flags " 747167156Semaste "are mutually exclusive"); 748167156Semaste r_flag = 1; 749167156Semaste break; 750167156Semaste case 'S': 751167156Semaste S_size = (off_t)strtol(optarg, &end, 10); 752167156Semaste if (*optarg == '\0' || *end != '\0') 753167156Semaste errx(1, "invalid size (%s)", optarg); 754167156Semaste break; 755167156Semaste case 's': 756167156Semaste s_flag = 1; 757167156Semaste break; 758167156Semaste case 'T': 759167156Semaste T_secs = strtol(optarg, &end, 10); 760167156Semaste if (*optarg == '\0' || *end != '\0') 761167156Semaste errx(1, "invalid timeout (%s)", optarg); 762167156Semaste break; 763167156Semaste case 't': 764167156Semaste t_flag = 1; 765167156Semaste warnx("warning: the -t option is deprecated"); 766167156Semaste break; 767167156Semaste case 'U': 768167156Semaste U_flag = 1; 769167156Semaste break; 770167156Semaste case 'v': 771167156Semaste v_level++; 772167156Semaste break; 773167156Semaste case 'w': 774167156Semaste a_flag = 1; 775167156Semaste w_secs = strtol(optarg, &end, 10); 776167156Semaste if (*optarg == '\0' || *end != '\0') 777167156Semaste errx(1, "invalid delay (%s)", optarg); 778167156Semaste break; 779167156Semaste default: 780167156Semaste usage(); 781167156Semaste exit(EX_USAGE); 782167156Semaste } 783167156Semaste 784167156Semaste argc -= optind; 785167156Semaste argv += optind; 786167156Semaste 787167156Semaste if (h_hostname || f_filename || c_dirname) { 788167156Semaste if (!h_hostname || !f_filename || argc) { 789167156Semaste usage(); 790167156Semaste exit(EX_USAGE); 791167156Semaste } 792167156Semaste /* XXX this is a hack. */ 793167156Semaste if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 794167156Semaste errx(1, "invalid hostname"); 795167156Semaste if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 796167156Semaste c_dirname ? c_dirname : "", f_filename) == -1) 797167156Semaste errx(1, "%s", strerror(ENOMEM)); 798167156Semaste argc++; 799167156Semaste } 800167156Semaste 801167156Semaste if (!argc) { 802167156Semaste usage(); 803167156Semaste exit(EX_USAGE); 804167156Semaste } 805 806 /* allocate buffer */ 807 if (B_size < MINBUFSIZE) 808 B_size = MINBUFSIZE; 809 if ((buf = malloc(B_size)) == NULL) 810 errx(1, "%s", strerror(ENOMEM)); 811 812 /* timeouts */ 813 if ((s = getenv("FTP_TIMEOUT")) != NULL) { 814 ftp_timeout = strtol(s, &end, 10); 815 if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { 816 warnx("FTP_TIMEOUT (%s) is not a positive integer", s); 817 ftp_timeout = 0; 818 } 819 } 820 if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 821 http_timeout = strtol(s, &end, 10); 822 if (*s == '\0' || *end != '\0' || http_timeout < 0) { 823 warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); 824 http_timeout = 0; 825 } 826 } 827 828 /* signal handling */ 829 sa.sa_flags = 0; 830 sa.sa_handler = sig_handler; 831 sigemptyset(&sa.sa_mask); 832 sigaction(SIGALRM, &sa, NULL); 833 sa.sa_flags = SA_RESETHAND; 834 sigaction(SIGINT, &sa, NULL); 835 fetchRestartCalls = 0; 836 837 /* output file */ 838 if (o_flag) { 839 if (strcmp(o_filename, "-") == 0) { 840 o_stdout = 1; 841 } else if (stat(o_filename, &sb) == -1) { 842 if (errno == ENOENT) { 843 if (argc > 1) 844 errx(EX_USAGE, "%s is not a directory", 845 o_filename); 846 } else { 847 err(EX_IOERR, "%s", o_filename); 848 } 849 } else { 850 if (sb.st_mode & S_IFDIR) 851 o_directory = 1; 852 } 853 } 854 855 /* check if output is to a tty (for progress report) */ 856 v_tty = isatty(STDERR_FILENO); 857 if (v_tty) 858 pgrp = getpgrp(); 859 860 r = 0; 861 862 /* authentication */ 863 if (v_tty) 864 fetchAuthMethod = query_auth; 865 866 while (argc) { 867 if ((p = strrchr(*argv, '/')) == NULL) 868 p = *argv; 869 else 870 p++; 871 872 if (!*p) 873 p = "fetch.out"; 874 875 fetchLastErrCode = 0; 876 877 if (o_flag) { 878 if (o_stdout) { 879 e = fetch(*argv, "-"); 880 } else if (o_directory) { 881 asprintf(&q, "%s/%s", o_filename, p); 882 e = fetch(*argv, q); 883 free(q); 884 } else { 885 e = fetch(*argv, o_filename); 886 } 887 } else { 888 e = fetch(*argv, p); 889 } 890 891 if (sigint) 892 kill(getpid(), SIGINT); 893 894 if (e == 0 && once_flag) 895 exit(0); 896 897 if (e) { 898 r = 1; 899 if ((fetchLastErrCode 900 && fetchLastErrCode != FETCH_UNAVAIL 901 && fetchLastErrCode != FETCH_MOVED 902 && fetchLastErrCode != FETCH_URL 903 && fetchLastErrCode != FETCH_RESOLV 904 && fetchLastErrCode != FETCH_UNKNOWN)) { 905 if (w_secs && v_level) 906 fprintf(stderr, "Waiting %ld seconds " 907 "before retrying\n", w_secs); 908 if (w_secs) 909 sleep(w_secs); 910 if (a_flag) 911 continue; 912 } 913 } 914 915 argc--, argv++; 916 } 917 918 exit(r); 919} 920