fetch.c revision 63067
1/*- 2 * Copyright (c) 2000 Dag-Erling Co�dan Sm�rgrav 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer 10 * in this position and unchanged. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * $FreeBSD: head/usr.bin/fetch/fetch.c 63067 2000-07-13 08:37:39Z des $ 29 */ 30 31#include <sys/param.h> 32#include <sys/stat.h> 33#include <sys/socket.h> 34 35#include <ctype.h> 36#include <err.h> 37#include <errno.h> 38#include <stdio.h> 39#include <stdlib.h> 40#include <string.h> 41#include <sysexits.h> 42#include <unistd.h> 43 44#include <fetch.h> 45 46#define MINBUFSIZE 4096 47 48/* Option flags */ 49int A_flag; /* -A: do not follow 302 redirects */ 50int a_flag; /* -a: auto retry */ 51size_t B_size; /* -B: buffer size */ 52int b_flag; /*! -b: workaround TCP bug */ 53char *c_dirname; /* -c: remote directory */ 54int d_flag; /* -d: direct connection */ 55int F_flag; /* -F: restart without checking mtime */ 56char *f_filename; /* -f: file to fetch */ 57int H_flag; /* -H: use high port */ 58char *h_hostname; /* -h: host to fetch from */ 59int l_flag; /* -l: link rather than copy file: URLs */ 60int m_flag; /* -[Mm]: mirror mode */ 61int n_flag; /* -n: do not preserve modification time */ 62int o_flag; /* -o: specify output file */ 63int o_directory; /* output file is a directory */ 64char *o_filename; /* name of output file */ 65int o_stdout; /* output file is stdout */ 66int once_flag; /* -1: stop at first successful file */ 67int p_flag = 1; /* -[Pp]: use passive FTP */ 68int R_flag; /* -R: don't delete partially transferred files */ 69int r_flag; /* -r: restart previously interrupted transfer */ 70u_int T_secs = 0; /* -T: transfer timeout in seconds */ 71int s_flag; /* -s: show size, don't fetch */ 72off_t S_size; /* -S: require size to match */ 73int t_flag; /*! -t: workaround TCP bug */ 74int v_level = 1; /* -v: verbosity level */ 75int v_tty; /* stdout is a tty */ 76u_int w_secs; /* -w: retry delay */ 77int family = PF_UNSPEC; /* -[46]: address family to use */ 78 79int sigalrm; /* SIGALRM received */ 80int sigint; /* SIGINT received */ 81 82u_int ftp_timeout; /* default timeout for FTP transfers */ 83u_int http_timeout; /* default timeout for HTTP transfers */ 84u_char *buf; /* transfer buffer */ 85 86 87void 88sig_handler(int sig) 89{ 90 switch (sig) { 91 case SIGALRM: 92 sigalrm = 1; 93 break; 94 case SIGINT: 95 sigint = 1; 96 break; 97 } 98} 99 100struct xferstat { 101 char name[40]; 102 struct timeval start; 103 struct timeval end; 104 struct timeval last; 105 off_t size; 106 off_t offset; 107 off_t rcvd; 108}; 109 110void 111stat_display(struct xferstat *xs, int force) 112{ 113 struct timeval now; 114 115 if (!v_tty) 116 return; 117 118 gettimeofday(&now, NULL); 119 if (!force && now.tv_sec <= xs->last.tv_sec) 120 return; 121 xs->last = now; 122 123 fprintf(stderr, "\rReceiving %s", xs->name); 124 if (xs->size == -1) 125 fprintf(stderr, ": %lld bytes", xs->rcvd); 126 else 127 fprintf(stderr, " (%lld bytes): %d%%", xs->size, 128 (int)((100.0 * xs->rcvd) / xs->size)); 129} 130 131void 132stat_start(struct xferstat *xs, char *name, off_t size, off_t offset) 133{ 134 snprintf(xs->name, sizeof xs->name, "%s", name); 135 gettimeofday(&xs->start, NULL); 136 xs->last.tv_sec = xs->last.tv_usec = 0; 137 xs->end = xs->last; 138 xs->size = size; 139 xs->offset = offset; 140 xs->rcvd = offset; 141 stat_display(xs, 1); 142} 143 144void 145stat_update(struct xferstat *xs, off_t rcvd, int force) 146{ 147 xs->rcvd = rcvd; 148 stat_display(xs, 0); 149} 150 151void 152stat_end(struct xferstat *xs) 153{ 154 double delta; 155 double bps; 156 157 gettimeofday(&xs->end, NULL); 158 159 stat_display(xs, 1); 160 fputc('\n', stderr); 161 delta = (xs->end.tv_sec + (xs->end.tv_usec / 1.e6)) 162 - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); 163 fprintf(stderr, "%lld bytes transferred in %.1f seconds ", 164 xs->rcvd - xs->offset, delta); 165 bps = (xs->rcvd - xs->offset) / delta; 166 if (bps > 1024*1024) 167 fprintf(stderr, "(%.2f MBps)\n", bps / (1024*1024)); 168 else if (bps > 1024) 169 fprintf(stderr, "(%.2f kBps)\n", bps / 1024); 170 else 171 fprintf(stderr, "(%.2f Bps)\n", bps); 172} 173 174int 175fetch(char *URL, char *path) 176{ 177 struct url *url; 178 struct url_stat us; 179 struct stat sb; 180 struct xferstat xs; 181 FILE *f, *of; 182 size_t size; 183 off_t count; 184 char flags[8]; 185 int n, r; 186 u_int timeout; 187 188 f = of = NULL; 189 190 /* parse URL */ 191 if ((url = fetchParseURL(URL)) == NULL) { 192 warnx("%s: parse error", URL); 193 goto failure; 194 } 195 196 timeout = 0; 197 *flags = 0; 198 199 /* common flags */ 200 if (v_level > 2) 201 strcat(flags, "v"); 202 switch (family) { 203 case PF_INET: 204 strcat(flags, "4"); 205 break; 206 case PF_INET6: 207 strcat(flags, "6"); 208 break; 209 } 210 211 /* FTP specific flags */ 212 if (strcmp(url->scheme, "ftp") == 0) { 213 if (p_flag) 214 strcat(flags, "p"); 215 if (d_flag) 216 strcat(flags, "d"); 217 if (H_flag) 218 strcat(flags, "h"); 219 timeout = T_secs ? T_secs : ftp_timeout; 220 } 221 222 /* HTTP specific flags */ 223 if (strcmp(url->scheme, "http") == 0) { 224 if (d_flag) 225 strcat(flags, "d"); 226 if (A_flag) 227 strcat(flags, "A"); 228 timeout = T_secs ? T_secs : http_timeout; 229 } 230 231 /* set the protocol timeout. */ 232 fetchTimeout = timeout; 233 234 /* stat remote file */ 235 if (fetchStat(url, &us, flags) == -1) 236 goto failure; 237 238 /* just print size */ 239 if (s_flag) { 240 if (us.size == -1) 241 printf("Unknown\n"); 242 else 243 printf("%lld\n", us.size); 244 goto success; 245 } 246 247 /* check that size is as expected */ 248 if (S_size && us.size != -1 && us.size != S_size) { 249 warnx("%s: size mismatch: expected %lld, actual %lld", 250 path, S_size, us.size); 251 goto failure; 252 } 253 254 /* symlink instead of copy */ 255 if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { 256 if (symlink(url->doc, path) == -1) { 257 warn("%s: symlink()", path); 258 goto failure; 259 } 260 goto success; 261 } 262 263 if (o_stdout) { 264 /* output to stdout */ 265 of = stdout; 266 } else if (r_flag && us.size != -1 && stat(path, &sb) != -1 267 && (F_flag || (us.mtime && sb.st_mtime == us.mtime))) { 268 /* output to file, restart aborted transfer */ 269 if (us.size == sb.st_size) 270 goto success; 271 else if (sb.st_size > us.size && truncate(path, us.size) == -1) { 272 warn("%s: truncate()", path); 273 goto failure; 274 } 275 if ((of = fopen(path, "a")) == NULL) { 276 warn("%s: open()", path); 277 goto failure; 278 } 279 url->offset = sb.st_size; 280 } else if (m_flag && us.size != -1 && stat(path, &sb) != -1) { 281 /* output to file, mirror mode */ 282 if (sb.st_size == us.size && sb.st_mtime == us.mtime) 283 return 0; 284 if ((of = fopen(path, "w")) == NULL) { 285 warn("%s: open()", path); 286 goto failure; 287 } 288 } else { 289 /* output to file, all other cases */ 290 if ((of = fopen(path, "w")) == NULL) { 291 warn("%s: open()", path); 292 goto failure; 293 } 294 } 295 count = url->offset; 296 297 /* start the transfer */ 298 if ((f = fetchGet(url, flags)) == NULL) { 299 warnx("%s", fetchLastErrString); 300 if (!R_flag && !r_flag && !o_stdout) 301 unlink(path); 302 goto failure; 303 } 304 305 /* start the counter */ 306 stat_start(&xs, path, us.size, count); 307 308 sigint = sigalrm = 0; 309 310 /* suck in the data */ 311 for (n = 0; !sigint && !sigalrm; ++n) { 312 if (us.size != -1 && us.size - count < B_size) 313 size = us.size - count; 314 else 315 size = B_size; 316 if (timeout) 317 alarm(timeout); 318 if ((size = fread(buf, 1, size, f)) <= 0) 319 break; 320 stat_update(&xs, count += size, 0); 321 if (fwrite(buf, size, 1, of) != 1) 322 break; 323 } 324 325 if (timeout) 326 alarm(0); 327 328 stat_end(&xs); 329 330 /* Set mtime of local file */ 331 if (!n_flag && us.mtime && !o_stdout) { 332 struct timeval tv[2]; 333 334 fflush(of); 335 tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); 336 tv[1].tv_sec = (long)us.mtime; 337 tv[0].tv_usec = tv[1].tv_usec = 0; 338 if (utimes(path, tv)) 339 warn("%s: utimes()", path); 340 } 341 342 /* timed out or interrupted? */ 343 if (sigalrm) 344 warnx("transfer timed out"); 345 if (sigint) 346 warnx("transfer interrupted"); 347 348 /* check the status of our files */ 349 if (ferror(f)) 350 warn("%s", URL); 351 if (ferror(of)) 352 warn("%s", path); 353 if (ferror(f) || ferror(of)) 354 goto failure; 355 356 /* did the transfer complete normally? */ 357 if (us.size != -1 && count < us.size) { 358 warnx("%s appears to be truncated: %lld/%lld bytes", 359 path, count, us.size); 360 goto failure_keep; 361 } 362 363 success: 364 r = 0; 365 goto done; 366 failure: 367 if (of && of != stdout && !R_flag && !r_flag) 368 unlink(path); 369 failure_keep: 370 r = -1; 371 goto done; 372 done: 373 if (f) 374 fclose(f); 375 if (of && of != stdout) 376 fclose(of); 377 if (url) 378 fetchFreeURL(url); 379 return r; 380} 381 382void 383usage(void) 384{ 385 /* XXX badly out of synch */ 386 fprintf(stderr, 387 "Usage: fetch [-1AFHMPRabdlmnpqrstv] [-o outputfile] [-S bytes]\n" 388 " [-B bytes] [-T seconds] [-w seconds]\n" 389 " [-f file -h host [-c dir] | URL ...]\n" 390 ); 391} 392 393 394#define PARSENUM(NAME, TYPE) \ 395int \ 396NAME(char *s, TYPE *v) \ 397{ \ 398 *v = 0; \ 399 for (*v = 0; *s; s++) \ 400 if (isdigit(*s)) \ 401 *v = *v * 10 + *s - '0'; \ 402 else \ 403 return -1; \ 404 return 0; \ 405} 406 407PARSENUM(parseint, u_int) 408PARSENUM(parsesize, size_t) 409PARSENUM(parseoff, off_t) 410 411int 412main(int argc, char *argv[]) 413{ 414 struct stat sb; 415 char *p, *q, *s; 416 int c, e, r; 417 418 while ((c = getopt(argc, argv, 419 "146AaB:bc:dFf:h:lHMmnPpo:qRrS:sT:tvw:")) != EOF) 420 switch (c) { 421 case '1': 422 once_flag = 1; 423 break; 424 case '4': 425 family = PF_INET; 426 break; 427 case '6': 428 family = PF_INET6; 429 break; 430 case 'A': 431 A_flag = 1; 432 break; 433 case 'a': 434 a_flag = 1; 435 break; 436 case 'B': 437 if (parsesize(optarg, &B_size) == -1) 438 errx(1, "invalid buffer size"); 439 break; 440 case 'b': 441 warnx("warning: the -b option is deprecated"); 442 b_flag = 1; 443 break; 444 case 'c': 445 c_dirname = optarg; 446 break; 447 case 'd': 448 d_flag = 1; 449 break; 450 case 'F': 451 F_flag = 1; 452 break; 453 case 'f': 454 f_filename = optarg; 455 break; 456 case 'H': 457 H_flag = 1; 458 break; 459 case 'h': 460 h_hostname = optarg; 461 break; 462 case 'l': 463 l_flag = 1; 464 break; 465 case 'o': 466 o_flag = 1; 467 o_filename = optarg; 468 break; 469 case 'M': 470 case 'm': 471 m_flag = 1; 472 break; 473 case 'n': 474 n_flag = 1; 475 break; 476 case 'P': 477 case 'p': 478 p_flag = 1; 479 break; 480 case 'q': 481 v_level = 0; 482 break; 483 case 'R': 484 R_flag = 1; 485 break; 486 case 'r': 487 r_flag = 1; 488 break; 489 case 'S': 490 if (parseoff(optarg, &S_size) == -1) 491 errx(1, "invalid size"); 492 break; 493 case 's': 494 s_flag = 1; 495 break; 496 case 'T': 497 if (parseint(optarg, &T_secs) == -1) 498 errx(1, "invalid timeout"); 499 break; 500 case 't': 501 t_flag = 1; 502 warnx("warning: the -t option is deprecated"); 503 break; 504 case 'v': 505 v_level++; 506 break; 507 case 'w': 508 a_flag = 1; 509 if (parseint(optarg, &w_secs) == -1) 510 errx(1, "invalid delay"); 511 break; 512 default: 513 usage(); 514 exit(EX_USAGE); 515 } 516 517 argc -= optind; 518 argv += optind; 519 520 if (h_hostname || f_filename || c_dirname) { 521 if (!h_hostname || !f_filename || argc) { 522 usage(); 523 exit(EX_USAGE); 524 } 525 /* XXX this is a hack. */ 526 if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) 527 errx(1, "invalid hostname"); 528 if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, 529 c_dirname ? c_dirname : "", f_filename) == -1) 530 errx(1, strerror(ENOMEM)); 531 argc++; 532 } 533 534 if (!argc) { 535 usage(); 536 exit(EX_USAGE); 537 } 538 539 /* allocate buffer */ 540 if (B_size < MINBUFSIZE) 541 B_size = MINBUFSIZE; 542 if ((buf = malloc(B_size)) == NULL) 543 errx(1, strerror(ENOMEM)); 544 545 /* timeout handling */ 546 signal(SIGALRM, sig_handler); 547 if ((s = getenv("FTP_TIMEOUT")) != NULL) { 548 if (parseint(s, &ftp_timeout) == -1) { 549 warnx("FTP_TIMEOUT is not a positive integer"); 550 ftp_timeout = 0; 551 } 552 } 553 if ((s = getenv("HTTP_TIMEOUT")) != NULL) { 554 if (parseint(s, &http_timeout) == -1) { 555 warnx("HTTP_TIMEOUT is not a positive integer"); 556 http_timeout = 0; 557 } 558 } 559 560 /* interrupt handling */ 561 signal(SIGINT, sig_handler); 562 563 /* output file */ 564 if (o_flag) { 565 if (strcmp(o_filename, "-") == 0) { 566 o_stdout = 1; 567 } else if (stat(o_filename, &sb) == -1) { 568 if (errno == ENOENT) { 569 if (argc > 1) 570 errx(EX_USAGE, "%s is not a directory", o_filename); 571 } else { 572 err(EX_IOERR, "%s", o_filename); 573 } 574 } else { 575 if (sb.st_mode & S_IFDIR) 576 o_directory = 1; 577 } 578 } 579 580 /* check if output is to a tty (for progress report) */ 581 v_tty = isatty(STDERR_FILENO); 582 r = 0; 583 584 while (argc) { 585 if ((p = strrchr(*argv, '/')) == NULL) 586 p = *argv; 587 else 588 p++; 589 590 if (!*p) 591 p = "fetch.out"; 592 593 fetchLastErrCode = 0; 594 595 if (o_flag) { 596 if (o_stdout) { 597 e = fetch(*argv, "-"); 598 } else if (o_directory) { 599 asprintf(&q, "%s/%s", o_filename, p); 600 e = fetch(*argv, q); 601 free(q); 602 } else { 603 e = fetch(*argv, o_filename); 604 } 605 } else { 606 e = fetch(*argv, p); 607 } 608 609 if (sigint) 610 exit(1); 611 612 if (e == 0 && once_flag) 613 exit(0); 614 615 if (e) { 616 r = 1; 617 if ((fetchLastErrCode 618 && fetchLastErrCode != FETCH_UNAVAIL 619 && fetchLastErrCode != FETCH_MOVED 620 && fetchLastErrCode != FETCH_URL 621 && fetchLastErrCode != FETCH_RESOLV 622 && fetchLastErrCode != FETCH_UNKNOWN)) { 623 if (w_secs) { 624 if (v_level) 625 fprintf(stderr, "Waiting %d seconds before retrying\n", w_secs); 626 sleep(w_secs); 627 } 628 if (a_flag) 629 continue; 630 fprintf(stderr, "Skipping %s\n", *argv); 631 } 632 } 633 634 argc--, argv++; 635 } 636 637 exit(r); 638} 639