1/*************************************************************************** 2 * _ _ ____ _ 3 * Project ___| | | | _ \| | 4 * / __| | | | |_) | | 5 * | (__| |_| | _ <| |___ 6 * \___|\___/|_| \_\_____| 7 * 8 * Copyright (C) 1998 - 2013, Daniel Stenberg, <daniel@haxx.se>, et al. 9 * 10 * This software is licensed as described in the file COPYING, which 11 * you should have received as part of this distribution. The terms 12 * are also available at http://curl.haxx.se/docs/copyright.html. 13 * 14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell 15 * copies of the Software, and permit persons to whom the Software is 16 * furnished to do so, under the terms of the COPYING file. 17 * 18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 19 * KIND, either express or implied. 20 * 21 ***************************************************************************/ 22#include "tool_setup.h" 23 24#ifdef HAVE_FCNTL_H 25# include <fcntl.h> 26#endif 27 28#ifdef HAVE_UTIME_H 29# include <utime.h> 30#elif defined(HAVE_SYS_UTIME_H) 31# include <sys/utime.h> 32#endif 33 34#ifdef HAVE_LOCALE_H 35# include <locale.h> 36#endif 37 38#ifdef HAVE_NETINET_TCP_H 39# include <netinet/tcp.h> 40#endif 41 42#include "rawstr.h" 43 44#define ENABLE_CURLX_PRINTF 45/* use our own printf() functions */ 46#include "curlx.h" 47 48#include "tool_binmode.h" 49#include "tool_cfgable.h" 50#include "tool_cb_dbg.h" 51#include "tool_cb_hdr.h" 52#include "tool_cb_prg.h" 53#include "tool_cb_rea.h" 54#include "tool_cb_see.h" 55#include "tool_cb_wrt.h" 56#include "tool_dirhie.h" 57#include "tool_doswin.h" 58#include "tool_easysrc.h" 59#include "tool_getparam.h" 60#include "tool_helpers.h" 61#include "tool_homedir.h" 62#include "tool_libinfo.h" 63#include "tool_main.h" 64#include "tool_metalink.h" 65#include "tool_msgs.h" 66#include "tool_operate.h" 67#include "tool_operhlp.h" 68#include "tool_parsecfg.h" 69#include "tool_setopt.h" 70#include "tool_sleep.h" 71#include "tool_urlglob.h" 72#include "tool_util.h" 73#include "tool_writeenv.h" 74#include "tool_writeout.h" 75#include "tool_xattr.h" 76#include "tool_vms.h" 77 78#include "memdebug.h" /* keep this as LAST include */ 79 80#define CURLseparator "--_curl_--" 81 82#ifndef O_BINARY 83/* since O_BINARY as used in bitmasks, setting it to zero makes it usable in 84 source code but yet it doesn't ruin anything */ 85# define O_BINARY 0 86#endif 87 88#define CURL_CA_CERT_ERRORMSG1 \ 89 "More details here: http://curl.haxx.se/docs/sslcerts.html\n\n" \ 90 "curl performs SSL certificate verification by default, " \ 91 "using a \"bundle\"\n" \ 92 " of Certificate Authority (CA) public keys (CA certs). If the default\n" \ 93 " bundle file isn't adequate, you can specify an alternate file\n" \ 94 " using the --cacert option.\n" 95 96#define CURL_CA_CERT_ERRORMSG2 \ 97 "If this HTTPS server uses a certificate signed by a CA represented in\n" \ 98 " the bundle, the certificate verification probably failed due to a\n" \ 99 " problem with the certificate (it might be expired, or the name might\n" \ 100 " not match the domain name in the URL).\n" \ 101 "If you'd like to turn off curl's verification of the certificate, use\n" \ 102 " the -k (or --insecure) option.\n" 103 104static int is_fatal_error(int code) 105{ 106 switch(code) { 107 /* TODO: Should CURLE_SSL_CACERT be included as critical error ? */ 108 case CURLE_FAILED_INIT: 109 case CURLE_OUT_OF_MEMORY: 110 case CURLE_UNKNOWN_OPTION: 111 case CURLE_FUNCTION_NOT_FOUND: 112 case CURLE_BAD_FUNCTION_ARGUMENT: 113 /* critical error */ 114 return 1; 115 default: 116 break; 117 } 118 /* no error or not critical */ 119 return 0; 120} 121 122int operate(struct Configurable *config, int argc, argv_item_t argv[]) 123{ 124 char errorbuffer[CURL_ERROR_SIZE]; 125 struct ProgressData progressbar; 126 struct getout *urlnode; 127 128 struct HdrCbData hdrcbdata; 129 struct OutStruct heads; 130 131 metalinkfile *mlfile_last = NULL; 132 133 CURL *curl = NULL; 134 char *httpgetfields = NULL; 135 136 bool stillflags; 137 int res = 0; 138 int i; 139 140 bool orig_noprogress; 141 bool orig_isatty; 142 143 errorbuffer[0] = '\0'; 144 /* default headers output stream is stdout */ 145 memset(&hdrcbdata, 0, sizeof(struct HdrCbData)); 146 memset(&heads, 0, sizeof(struct OutStruct)); 147 heads.stream = stdout; 148 heads.config = config; 149 150 memory_tracking_init(); 151 152 /* 153 ** Initialize curl library - do not call any libcurl functions before 154 ** this point. Note that the memory_tracking_init() magic above is an 155 ** exception, but then that's not part of the official public API. 156 */ 157 if(main_init() != CURLE_OK) { 158 helpf(config->errors, "error initializing curl library\n"); 159 return CURLE_FAILED_INIT; 160 } 161 162 /* Get libcurl info right away */ 163 if(get_libcurl_info() != CURLE_OK) { 164 helpf(config->errors, "error retrieving curl library information\n"); 165 main_free(); 166 return CURLE_FAILED_INIT; 167 } 168 169 /* Get a curl handle to use for all forthcoming curl transfers */ 170 curl = curl_easy_init(); 171 if(!curl) { 172 helpf(config->errors, "error initializing curl easy handle\n"); 173 main_free(); 174 return CURLE_FAILED_INIT; 175 } 176 config->easy = curl; 177 178 /* 179 ** Beyond this point no return'ing from this function allowed. 180 ** Jump to label 'quit_curl' in order to abandon this function 181 ** from outside of nested loops further down below. 182 */ 183 184 /* setup proper locale from environment */ 185#ifdef HAVE_SETLOCALE 186 setlocale(LC_ALL, ""); 187#endif 188 189 /* inits */ 190 config->postfieldsize = -1; 191 config->showerror = -1; /* will show errors */ 192 config->use_httpget = FALSE; 193 config->create_dirs = FALSE; 194 config->maxredirs = DEFAULT_MAXREDIRS; 195 config->proto = CURLPROTO_ALL; /* FIXME: better to read from library */ 196 config->proto_present = FALSE; 197 config->proto_redir = 198 CURLPROTO_ALL & ~(CURLPROTO_FILE|CURLPROTO_SCP); /* not FILE or SCP */ 199 config->proto_redir_present = FALSE; 200 201 if((argc > 1) && 202 (!curlx_strnequal("--", argv[1], 2) && (argv[1][0] == '-')) && 203 strchr(argv[1], 'q')) { 204 /* 205 * The first flag, that is not a verbose name, but a shortname 206 * and it includes the 'q' flag! 207 */ 208 ; 209 } 210 else { 211 parseconfig(NULL, config); /* ignore possible failure */ 212 } 213 214 if((argc < 2) && !config->url_list) { 215 helpf(config->errors, NULL); 216 res = CURLE_FAILED_INIT; 217 goto quit_curl; 218 } 219 220 /* Parse options */ 221 for(i = 1, stillflags = TRUE; i < argc; i++) { 222 if(stillflags && 223 ('-' == argv[i][0])) { 224 char *nextarg; 225 bool passarg; 226 char *orig_opt = argv[i]; 227 228 char *flag = argv[i]; 229 230 if(curlx_strequal("--", argv[i])) 231 /* this indicates the end of the flags and thus enables the 232 following (URL) argument to start with -. */ 233 stillflags = FALSE; 234 else { 235 nextarg = (i < (argc-1)) ? argv[i+1] : NULL; 236 237 res = getparameter(flag, nextarg, &passarg, config); 238 if(res) { 239 int retval = CURLE_OK; 240 if(res != PARAM_HELP_REQUESTED) { 241 const char *reason = param2text(res); 242 helpf(config->errors, "option %s: %s\n", orig_opt, reason); 243 retval = CURLE_FAILED_INIT; 244 } 245 res = retval; 246 goto quit_curl; 247 } 248 249 if(passarg) /* we're supposed to skip this */ 250 i++; 251 } 252 } 253 else { 254 bool used; 255 /* just add the URL please */ 256 res = getparameter((char *)"--url", argv[i], &used, config); 257 if(res) 258 goto quit_curl; 259 } 260 } 261 262 if((!config->url_list || !config->url_list->url) && !config->list_engines) { 263 helpf(config->errors, "no URL specified!\n"); 264 res = CURLE_FAILED_INIT; 265 goto quit_curl; 266 } 267 268 if(!config->useragent) 269 config->useragent = my_useragent(); 270 if(!config->useragent) { 271 helpf(config->errors, "out of memory\n"); 272 res = CURLE_OUT_OF_MEMORY; 273 goto quit_curl; 274 } 275 276 /* On WIN32 we can't set the path to curl-ca-bundle.crt 277 * at compile time. So we look here for the file in two ways: 278 * 1: look at the environment variable CURL_CA_BUNDLE for a path 279 * 2: if #1 isn't found, use the windows API function SearchPath() 280 * to find it along the app's path (includes app's dir and CWD) 281 * 282 * We support the environment variable thing for non-Windows platforms 283 * too. Just for the sake of it. 284 */ 285 if(!config->cacert && 286 !config->capath && 287 !config->insecure_ok) { 288 char *env; 289 env = curlx_getenv("CURL_CA_BUNDLE"); 290 if(env) { 291 config->cacert = strdup(env); 292 if(!config->cacert) { 293 curl_free(env); 294 helpf(config->errors, "out of memory\n"); 295 res = CURLE_OUT_OF_MEMORY; 296 goto quit_curl; 297 } 298 } 299 else { 300 env = curlx_getenv("SSL_CERT_DIR"); 301 if(env) { 302 config->capath = strdup(env); 303 if(!config->capath) { 304 curl_free(env); 305 helpf(config->errors, "out of memory\n"); 306 res = CURLE_OUT_OF_MEMORY; 307 goto quit_curl; 308 } 309 } 310 else { 311 env = curlx_getenv("SSL_CERT_FILE"); 312 if(env) { 313 config->cacert = strdup(env); 314 if(!config->cacert) { 315 curl_free(env); 316 helpf(config->errors, "out of memory\n"); 317 res = CURLE_OUT_OF_MEMORY; 318 goto quit_curl; 319 } 320 } 321 } 322 } 323 324 if(env) 325 curl_free(env); 326#ifdef WIN32 327 else { 328 res = FindWin32CACert(config, "curl-ca-bundle.crt"); 329 if(res) 330 goto quit_curl; 331 } 332#endif 333 } 334 335 if(config->postfields) { 336 if(config->use_httpget) { 337 /* Use the postfields data for a http get */ 338 httpgetfields = strdup(config->postfields); 339 Curl_safefree(config->postfields); 340 if(!httpgetfields) { 341 helpf(config->errors, "out of memory\n"); 342 res = CURLE_OUT_OF_MEMORY; 343 goto quit_curl; 344 } 345 if(SetHTTPrequest(config, 346 (config->no_body?HTTPREQ_HEAD:HTTPREQ_GET), 347 &config->httpreq)) { 348 res = PARAM_BAD_USE; 349 goto quit_curl; 350 } 351 } 352 else { 353 if(SetHTTPrequest(config, HTTPREQ_SIMPLEPOST, &config->httpreq)) { 354 res = PARAM_BAD_USE; 355 goto quit_curl; 356 } 357 } 358 } 359 360#ifndef CURL_DISABLE_LIBCURL_OPTION 361 res = easysrc_init(); 362 if(res) { 363 helpf(config->errors, "out of memory\n"); 364 goto quit_curl; 365 } 366#endif 367 368 if(config->list_engines) { 369 struct curl_slist *engines = NULL; 370 curl_easy_getinfo(curl, CURLINFO_SSL_ENGINES, &engines); 371 list_engines(engines); 372 curl_slist_free_all(engines); 373 res = CURLE_OK; 374 goto quit_curl; 375 } 376 377 /* Single header file for all URLs */ 378 if(config->headerfile) { 379 /* open file for output: */ 380 if(!curlx_strequal(config->headerfile, "-")) { 381 FILE *newfile = fopen(config->headerfile, "wb"); 382 if(!newfile) { 383 warnf(config, "Failed to open %s\n", config->headerfile); 384 res = CURLE_WRITE_ERROR; 385 goto quit_curl; 386 } 387 else { 388 heads.filename = config->headerfile; 389 heads.s_isreg = TRUE; 390 heads.fopened = TRUE; 391 heads.stream = newfile; 392 } 393 } 394 } 395 396 /* save the values of noprogress and isatty to restore them later on */ 397 orig_noprogress = config->noprogress; 398 orig_isatty = config->isatty; 399 400 /* 401 ** Nested loops start here. 402 */ 403 404 /* loop through the list of given URLs */ 405 406 for(urlnode = config->url_list; urlnode; urlnode = urlnode->next) { 407 408 int up; /* upload file counter within a single upload glob */ 409 char *infiles; /* might be a glob pattern */ 410 char *outfiles; 411 int infilenum; 412 URLGlob *inglob; 413 414 int metalink = 0; /* nonzero for metalink download. */ 415 metalinkfile *mlfile; 416 metalink_resource *mlres; 417 418 outfiles = NULL; 419 infilenum = 1; 420 inglob = NULL; 421 422 if(urlnode->flags & GETOUT_METALINK) { 423 metalink = 1; 424 if(mlfile_last == NULL) { 425 mlfile_last = config->metalinkfile_list; 426 } 427 mlfile = mlfile_last; 428 mlfile_last = mlfile_last->next; 429 mlres = mlfile->resource; 430 } 431 else { 432 mlfile = NULL; 433 mlres = NULL; 434 } 435 436 /* urlnode->url is the full URL (it might be NULL) */ 437 438 if(!urlnode->url) { 439 /* This node has no URL. Free node data without destroying the 440 node itself nor modifying next pointer and continue to next */ 441 Curl_safefree(urlnode->outfile); 442 Curl_safefree(urlnode->infile); 443 urlnode->flags = 0; 444 continue; /* next URL please */ 445 } 446 447 /* save outfile pattern before expansion */ 448 if(urlnode->outfile) { 449 outfiles = strdup(urlnode->outfile); 450 if(!outfiles) { 451 helpf(config->errors, "out of memory\n"); 452 res = CURLE_OUT_OF_MEMORY; 453 break; 454 } 455 } 456 457 infiles = urlnode->infile; 458 459 if(!config->globoff && infiles) { 460 /* Unless explicitly shut off */ 461 res = glob_url(&inglob, infiles, &infilenum, 462 config->showerror?config->errors:NULL); 463 if(res) { 464 Curl_safefree(outfiles); 465 break; 466 } 467 } 468 469 /* Here's the loop for uploading multiple files within the same 470 single globbed string. If no upload, we enter the loop once anyway. */ 471 for(up = 0 ; up < infilenum; up++) { 472 473 char *uploadfile; /* a single file, never a glob */ 474 int separator; 475 URLGlob *urls; 476 int urlnum; 477 478 uploadfile = NULL; 479 urls = NULL; 480 urlnum = 0; 481 482 if(!up && !infiles) 483 Curl_nop_stmt; 484 else { 485 if(inglob) { 486 res = glob_next_url(&uploadfile, inglob); 487 if(res == CURLE_OUT_OF_MEMORY) 488 helpf(config->errors, "out of memory\n"); 489 } 490 else if(!up) { 491 uploadfile = strdup(infiles); 492 if(!uploadfile) { 493 helpf(config->errors, "out of memory\n"); 494 res = CURLE_OUT_OF_MEMORY; 495 } 496 } 497 else 498 uploadfile = NULL; 499 if(!uploadfile) 500 break; 501 } 502 503 if(metalink) { 504 /* For Metalink download, we don't use glob. Instead we use 505 the number of resources as urlnum. */ 506 urlnum = count_next_metalink_resource(mlfile); 507 } 508 else 509 if(!config->globoff) { 510 /* Unless explicitly shut off, we expand '{...}' and '[...]' 511 expressions and return total number of URLs in pattern set */ 512 res = glob_url(&urls, urlnode->url, &urlnum, 513 config->showerror?config->errors:NULL); 514 if(res) { 515 Curl_safefree(uploadfile); 516 break; 517 } 518 } 519 else 520 urlnum = 1; /* without globbing, this is a single URL */ 521 522 /* if multiple files extracted to stdout, insert separators! */ 523 separator= ((!outfiles || curlx_strequal(outfiles, "-")) && urlnum > 1); 524 525 /* Here's looping around each globbed URL */ 526 for(i = 0 ; i < urlnum; i++) { 527 528 int infd; 529 bool infdopen; 530 char *outfile; 531 struct OutStruct outs; 532 struct InStruct input; 533 struct timeval retrystart; 534 curl_off_t uploadfilesize; 535 long retry_numretries; 536 long retry_sleep_default; 537 long retry_sleep; 538 char *this_url = NULL; 539 int metalink_next_res = 0; 540 541 outfile = NULL; 542 infdopen = FALSE; 543 infd = STDIN_FILENO; 544 uploadfilesize = -1; /* -1 means unknown */ 545 546 /* default output stream is stdout */ 547 memset(&outs, 0, sizeof(struct OutStruct)); 548 outs.stream = stdout; 549 outs.config = config; 550 551 if(metalink) { 552 /* For Metalink download, use name in Metalink file as 553 filename. */ 554 outfile = strdup(mlfile->filename); 555 if(!outfile) { 556 res = CURLE_OUT_OF_MEMORY; 557 goto show_error; 558 } 559 this_url = strdup(mlres->url); 560 if(!this_url) { 561 res = CURLE_OUT_OF_MEMORY; 562 goto show_error; 563 } 564 } 565 else { 566 if(urls) { 567 res = glob_next_url(&this_url, urls); 568 if(res) 569 goto show_error; 570 } 571 else if(!i) { 572 this_url = strdup(urlnode->url); 573 if(!this_url) { 574 res = CURLE_OUT_OF_MEMORY; 575 goto show_error; 576 } 577 } 578 else 579 this_url = NULL; 580 if(!this_url) 581 break; 582 583 if(outfiles) { 584 outfile = strdup(outfiles); 585 if(!outfile) { 586 res = CURLE_OUT_OF_MEMORY; 587 goto show_error; 588 } 589 } 590 } 591 592 if(((urlnode->flags&GETOUT_USEREMOTE) || 593 (outfile && !curlx_strequal("-", outfile))) && 594 (metalink || !config->use_metalink)) { 595 596 /* 597 * We have specified a file name to store the result in, or we have 598 * decided we want to use the remote file name. 599 */ 600 601 if(!outfile) { 602 /* extract the file name from the URL */ 603 res = get_url_file_name(&outfile, this_url); 604 if(res) 605 goto show_error; 606 if((!outfile || !*outfile) && !config->content_disposition) { 607 helpf(config->errors, "Remote file name has no length!\n"); 608 res = CURLE_WRITE_ERROR; 609 goto quit_urls; 610 } 611#if defined(MSDOS) || defined(WIN32) 612 /* For DOS and WIN32, we do some major replacing of 613 bad characters in the file name before using it */ 614 outfile = sanitize_dos_name(outfile); 615 if(!outfile) { 616 res = CURLE_OUT_OF_MEMORY; 617 goto show_error; 618 } 619#endif /* MSDOS || WIN32 */ 620 } 621 else if(urls) { 622 /* fill '#1' ... '#9' terms from URL pattern */ 623 char *storefile = outfile; 624 res = glob_match_url(&outfile, storefile, urls); 625 Curl_safefree(storefile); 626 if(res) { 627 /* bad globbing */ 628 warnf(config, "bad output glob!\n"); 629 goto quit_urls; 630 } 631 } 632 633 /* Create the directory hierarchy, if not pre-existent to a multiple 634 file output call */ 635 636 if(config->create_dirs || metalink) { 637 res = create_dir_hierarchy(outfile, config->errors); 638 /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */ 639 if(res == CURLE_WRITE_ERROR) 640 goto quit_urls; 641 if(res) { 642 goto show_error; 643 } 644 } 645 646 if((urlnode->flags & GETOUT_USEREMOTE) 647 && config->content_disposition) { 648 /* Our header callback MIGHT set the filename */ 649 DEBUGASSERT(!outs.filename); 650 } 651 652 if(config->resume_from_current) { 653 /* We're told to continue from where we are now. Get the size 654 of the file as it is now and open it for append instead */ 655 struct_stat fileinfo; 656 /* VMS -- Danger, the filesize is only valid for stream files */ 657 if(0 == stat(outfile, &fileinfo)) 658 /* set offset to current file size: */ 659 config->resume_from = fileinfo.st_size; 660 else 661 /* let offset be 0 */ 662 config->resume_from = 0; 663 } 664 665 if(config->resume_from) { 666 /* open file for output: */ 667 FILE *file = fopen(outfile, config->resume_from?"ab":"wb"); 668 if(!file) { 669 helpf(config->errors, "Can't open '%s'!\n", outfile); 670 res = CURLE_WRITE_ERROR; 671 goto quit_urls; 672 } 673 outs.fopened = TRUE; 674 outs.stream = file; 675 outs.init = config->resume_from; 676 } 677 else { 678 outs.stream = NULL; /* open when needed */ 679 } 680 outs.filename = outfile; 681 outs.s_isreg = TRUE; 682 } 683 684 if(uploadfile && !stdin_upload(uploadfile)) { 685 /* 686 * We have specified a file to upload and it isn't "-". 687 */ 688 struct_stat fileinfo; 689 690 this_url = add_file_name_to_url(curl, this_url, uploadfile); 691 if(!this_url) { 692 res = CURLE_OUT_OF_MEMORY; 693 goto show_error; 694 } 695 /* VMS Note: 696 * 697 * Reading binary from files can be a problem... Only FIXED, VAR 698 * etc WITHOUT implied CC will work Others need a \n appended to a 699 * line 700 * 701 * - Stat gives a size but this is UNRELIABLE in VMS As a f.e. a 702 * fixed file with implied CC needs to have a byte added for every 703 * record processed, this can by derived from Filesize & recordsize 704 * for VARiable record files the records need to be counted! for 705 * every record add 1 for linefeed and subtract 2 for the record 706 * header for VARIABLE header files only the bare record data needs 707 * to be considered with one appended if implied CC 708 */ 709 710 infd = open(uploadfile, O_RDONLY | O_BINARY); 711 if((infd == -1) || fstat(infd, &fileinfo)) { 712 helpf(config->errors, "Can't open '%s'!\n", uploadfile); 713 if(infd != -1) { 714 close(infd); 715 infd = STDIN_FILENO; 716 } 717 res = CURLE_READ_ERROR; 718 goto quit_urls; 719 } 720 infdopen = TRUE; 721 722 /* we ignore file size for char/block devices, sockets, etc. */ 723 if(S_ISREG(fileinfo.st_mode)) 724 uploadfilesize = fileinfo.st_size; 725 726 } 727 else if(uploadfile && stdin_upload(uploadfile)) { 728 /* count to see if there are more than one auth bit set 729 in the authtype field */ 730 int authbits = 0; 731 int bitcheck = 0; 732 while(bitcheck < 32) { 733 if(config->authtype & (1UL << bitcheck++)) { 734 authbits++; 735 if(authbits > 1) { 736 /* more than one, we're done! */ 737 break; 738 } 739 } 740 } 741 742 /* 743 * If the user has also selected --anyauth or --proxy-anyauth 744 * we should warn him/her. 745 */ 746 if(config->proxyanyauth || (authbits>1)) { 747 warnf(config, 748 "Using --anyauth or --proxy-anyauth with upload from stdin" 749 " involves a big risk of it not working. Use a temporary" 750 " file or a fixed auth type instead!\n"); 751 } 752 753 DEBUGASSERT(infdopen == FALSE); 754 DEBUGASSERT(infd == STDIN_FILENO); 755 756 set_binmode(stdin); 757 if(curlx_strequal(uploadfile, ".")) { 758 if(curlx_nonblock((curl_socket_t)infd, TRUE) < 0) 759 warnf(config, 760 "fcntl failed on fd=%d: %s\n", infd, strerror(errno)); 761 } 762 } 763 764 if(uploadfile && config->resume_from_current) 765 config->resume_from = -1; /* -1 will then force get-it-yourself */ 766 767 if(output_expected(this_url, uploadfile) 768 && outs.stream && isatty(fileno(outs.stream))) 769 /* we send the output to a tty, therefore we switch off the progress 770 meter */ 771 config->noprogress = config->isatty = TRUE; 772 else { 773 /* progress meter is per download, so restore config 774 values */ 775 config->noprogress = orig_noprogress; 776 config->isatty = orig_isatty; 777 } 778 779 if(urlnum > 1 && !(config->mute)) { 780 fprintf(config->errors, "\n[%d/%d]: %s --> %s\n", 781 i+1, urlnum, this_url, outfile ? outfile : "<stdout>"); 782 if(separator) 783 printf("%s%s\n", CURLseparator, this_url); 784 } 785 if(httpgetfields) { 786 char *urlbuffer; 787 /* Find out whether the url contains a file name */ 788 const char *pc = strstr(this_url, "://"); 789 char sep = '?'; 790 if(pc) 791 pc += 3; 792 else 793 pc = this_url; 794 795 pc = strrchr(pc, '/'); /* check for a slash */ 796 797 if(pc) { 798 /* there is a slash present in the URL */ 799 800 if(strchr(pc, '?')) 801 /* Ouch, there's already a question mark in the URL string, we 802 then append the data with an ampersand separator instead! */ 803 sep='&'; 804 } 805 /* 806 * Then append ? followed by the get fields to the url. 807 */ 808 if(pc) 809 urlbuffer = aprintf("%s%c%s", this_url, sep, httpgetfields); 810 else 811 /* Append / before the ? to create a well-formed url 812 if the url contains a hostname only 813 */ 814 urlbuffer = aprintf("%s/?%s", this_url, httpgetfields); 815 816 if(!urlbuffer) { 817 res = CURLE_OUT_OF_MEMORY; 818 goto show_error; 819 } 820 821 Curl_safefree(this_url); /* free previous URL */ 822 this_url = urlbuffer; /* use our new URL instead! */ 823 } 824 825 if(!config->errors) 826 config->errors = stderr; 827 828 if((!outfile || !strcmp(outfile, "-")) && !config->use_ascii) { 829 /* We get the output to stdout and we have not got the ASCII/text 830 flag, then set stdout to be binary */ 831 set_binmode(stdout); 832 } 833 834 if(config->tcp_nodelay) 835 my_setopt(curl, CURLOPT_TCP_NODELAY, 1); 836 837 /* where to store */ 838 my_setopt(curl, CURLOPT_WRITEDATA, &outs); 839 if(metalink || !config->use_metalink) 840 /* what call to write */ 841 my_setopt(curl, CURLOPT_WRITEFUNCTION, tool_write_cb); 842#ifdef USE_METALINK 843 else 844 /* Set Metalink specific write callback function to parse 845 XML data progressively. */ 846 my_setopt(curl, CURLOPT_WRITEFUNCTION, metalink_write_cb); 847#endif /* USE_METALINK */ 848 849 /* for uploads */ 850 input.fd = infd; 851 input.config = config; 852 /* Note that if CURLOPT_READFUNCTION is fread (the default), then 853 * lib/telnet.c will Curl_poll() on the input file descriptor 854 * rather then calling the READFUNCTION at regular intervals. 855 * The circumstances in which it is preferable to enable this 856 * behaviour, by omitting to set the READFUNCTION & READDATA options, 857 * have not been determined. 858 */ 859 my_setopt(curl, CURLOPT_READDATA, &input); 860 /* what call to read */ 861 my_setopt(curl, CURLOPT_READFUNCTION, tool_read_cb); 862 863 /* in 7.18.0, the CURLOPT_SEEKFUNCTION/DATA pair is taking over what 864 CURLOPT_IOCTLFUNCTION/DATA pair previously provided for seeking */ 865 my_setopt(curl, CURLOPT_SEEKDATA, &input); 866 my_setopt(curl, CURLOPT_SEEKFUNCTION, tool_seek_cb); 867 868 if(config->recvpersecond) 869 /* tell libcurl to use a smaller sized buffer as it allows us to 870 make better sleeps! 7.9.9 stuff! */ 871 my_setopt(curl, CURLOPT_BUFFERSIZE, config->recvpersecond); 872 873 /* size of uploaded file: */ 874 if(uploadfilesize != -1) 875 my_setopt(curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize); 876 my_setopt_str(curl, CURLOPT_URL, this_url); /* what to fetch */ 877 my_setopt(curl, CURLOPT_NOPROGRESS, config->noprogress); 878 if(config->no_body) { 879 my_setopt(curl, CURLOPT_NOBODY, 1); 880 my_setopt(curl, CURLOPT_HEADER, 1); 881 } 882 /* If --metalink is used, we ignore --include (headers in 883 output) option because mixing headers to the body will 884 confuse XML parser and/or hash check will fail. */ 885 else if(!config->use_metalink) 886 my_setopt(curl, CURLOPT_HEADER, config->include_headers); 887 888#if !defined(CURL_DISABLE_PROXY) 889 { 890 /* TODO: Make this a run-time check instead of compile-time one. */ 891 892 my_setopt_str(curl, CURLOPT_PROXY, config->proxy); 893 my_setopt_str(curl, CURLOPT_PROXYUSERPWD, config->proxyuserpwd); 894 895 /* new in libcurl 7.3 */ 896 my_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, config->proxytunnel); 897 898 /* new in libcurl 7.5 */ 899 if(config->proxy) 900 my_setopt_enum(curl, CURLOPT_PROXYTYPE, config->proxyver); 901 902 /* new in libcurl 7.10 */ 903 if(config->socksproxy) { 904 my_setopt_str(curl, CURLOPT_PROXY, config->socksproxy); 905 my_setopt_enum(curl, CURLOPT_PROXYTYPE, config->socksver); 906 } 907 908 /* new in libcurl 7.10.6 */ 909 if(config->proxyanyauth) 910 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, 911 (long) CURLAUTH_ANY); 912 else if(config->proxynegotiate) 913 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, 914 (long) CURLAUTH_GSSNEGOTIATE); 915 else if(config->proxyntlm) 916 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, 917 (long) CURLAUTH_NTLM); 918 else if(config->proxydigest) 919 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, 920 (long) CURLAUTH_DIGEST); 921 else if(config->proxybasic) 922 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, 923 (long) CURLAUTH_BASIC); 924 925 /* new in libcurl 7.19.4 */ 926 my_setopt(curl, CURLOPT_NOPROXY, config->noproxy); 927 } 928#endif 929 930 my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror); 931 my_setopt(curl, CURLOPT_UPLOAD, uploadfile?TRUE:FALSE); 932 my_setopt(curl, CURLOPT_DIRLISTONLY, config->dirlistonly); 933 my_setopt(curl, CURLOPT_APPEND, config->ftp_append); 934 935 if(config->netrc_opt) 936 my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); 937 else if(config->netrc || config->netrc_file) 938 my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_REQUIRED); 939 else 940 my_setopt(curl, CURLOPT_NETRC, CURL_NETRC_IGNORED); 941 942 if(config->netrc_file) 943 my_setopt(curl, CURLOPT_NETRC_FILE, config->netrc_file); 944 945 my_setopt(curl, CURLOPT_TRANSFERTEXT, config->use_ascii); 946 my_setopt_str(curl, CURLOPT_USERPWD, config->userpwd); 947 my_setopt_str(curl, CURLOPT_RANGE, config->range); 948 my_setopt(curl, CURLOPT_ERRORBUFFER, errorbuffer); 949 my_setopt(curl, CURLOPT_TIMEOUT, config->timeout); 950 951 if(built_in_protos & CURLPROTO_HTTP) { 952 953 long postRedir = 0; 954 955 my_setopt(curl, CURLOPT_FOLLOWLOCATION, 956 config->followlocation); 957 my_setopt(curl, CURLOPT_UNRESTRICTED_AUTH, 958 config->unrestricted_auth); 959 960 switch(config->httpreq) { 961 case HTTPREQ_SIMPLEPOST: 962 my_setopt_str(curl, CURLOPT_POSTFIELDS, 963 config->postfields); 964 my_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, 965 config->postfieldsize); 966 break; 967 case HTTPREQ_POST: 968 my_setopt_httppost(curl, CURLOPT_HTTPPOST, config->httppost); 969 break; 970 default: 971 break; 972 } 973 974 my_setopt_str(curl, CURLOPT_REFERER, config->referer); 975 my_setopt(curl, CURLOPT_AUTOREFERER, config->autoreferer); 976 my_setopt_str(curl, CURLOPT_USERAGENT, config->useragent); 977 my_setopt_slist(curl, CURLOPT_HTTPHEADER, config->headers); 978 979 /* new in libcurl 7.5 */ 980 my_setopt(curl, CURLOPT_MAXREDIRS, config->maxredirs); 981 982 /* new in libcurl 7.9.1 */ 983 if(config->httpversion) 984 my_setopt_enum(curl, CURLOPT_HTTP_VERSION, config->httpversion); 985 986 /* new in libcurl 7.10.6 (default is Basic) */ 987 if(config->authtype) 988 my_setopt_bitmask(curl, CURLOPT_HTTPAUTH, (long) config->authtype); 989 990 /* curl 7.19.1 (the 301 version existed in 7.18.2), 991 303 was added in 7.26.0 */ 992 if(config->post301) 993 postRedir |= CURL_REDIR_POST_301; 994 if(config->post302) 995 postRedir |= CURL_REDIR_POST_302; 996 if(config->post303) 997 postRedir |= CURL_REDIR_POST_303; 998 my_setopt(curl, CURLOPT_POSTREDIR, postRedir); 999 1000 /* new in libcurl 7.21.6 */ 1001 if(config->encoding) 1002 my_setopt_str(curl, CURLOPT_ACCEPT_ENCODING, ""); 1003 1004 /* new in libcurl 7.21.6 */ 1005 if(config->tr_encoding) 1006 my_setopt(curl, CURLOPT_TRANSFER_ENCODING, 1); 1007 1008 } /* (built_in_protos & CURLPROTO_HTTP) */ 1009 1010 my_setopt_str(curl, CURLOPT_FTPPORT, config->ftpport); 1011 my_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1012 config->low_speed_limit); 1013 my_setopt(curl, CURLOPT_LOW_SPEED_TIME, config->low_speed_time); 1014 my_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE, 1015 config->sendpersecond); 1016 my_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, 1017 config->recvpersecond); 1018 my_setopt(curl, CURLOPT_RESUME_FROM_LARGE, 1019 config->use_resume?config->resume_from:0); 1020 1021 my_setopt(curl, CURLOPT_SSLCERT, config->cert); 1022 my_setopt_str(curl, CURLOPT_SSLCERTTYPE, config->cert_type); 1023 my_setopt(curl, CURLOPT_SSLKEY, config->key); 1024 my_setopt_str(curl, CURLOPT_SSLKEYTYPE, config->key_type); 1025 my_setopt_str(curl, CURLOPT_KEYPASSWD, config->key_passwd); 1026 1027 if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) { 1028 1029 /* SSH and SSL private key uses same command-line option */ 1030 /* new in libcurl 7.16.1 */ 1031 my_setopt_str(curl, CURLOPT_SSH_PRIVATE_KEYFILE, config->key); 1032 /* new in libcurl 7.16.1 */ 1033 my_setopt_str(curl, CURLOPT_SSH_PUBLIC_KEYFILE, config->pubkey); 1034 1035 /* new in libcurl 7.17.1: SSH host key md5 checking allows us 1036 to fail if we are not talking to who we think we should */ 1037 my_setopt_str(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_MD5, 1038 config->hostpubmd5); 1039 } 1040 1041 if(config->cacert) 1042 my_setopt_str(curl, CURLOPT_CAINFO, config->cacert); 1043 if(config->capath) 1044 my_setopt_str(curl, CURLOPT_CAPATH, config->capath); 1045 if(config->crlfile) 1046 my_setopt_str(curl, CURLOPT_CRLFILE, config->crlfile); 1047 1048 if(curlinfo->features & CURL_VERSION_SSL) { 1049 if(config->insecure_ok) { 1050 my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); 1051 my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); 1052 } 1053 else { 1054 my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); 1055 /* libcurl default is strict verifyhost -> 2L */ 1056 /* my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); */ 1057 } 1058 } 1059 1060 if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) { 1061 if(!config->insecure_ok) { 1062 char *home; 1063 char *file; 1064 res = CURLE_OUT_OF_MEMORY; 1065 home = homedir(); 1066 if(home) { 1067 file = aprintf("%s/%sssh/known_hosts", home, DOT_CHAR); 1068 if(file) { 1069 /* new in curl 7.19.6 */ 1070 res = res_setopt_str(curl, CURLOPT_SSH_KNOWNHOSTS, file); 1071 curl_free(file); 1072 if(res == CURLE_UNKNOWN_OPTION) 1073 /* libssh2 version older than 1.1.1 */ 1074 res = CURLE_OK; 1075 } 1076 Curl_safefree(home); 1077 } 1078 if(res) 1079 goto show_error; 1080 } 1081 } 1082 1083 if(config->no_body || config->remote_time) { 1084 /* no body or use remote time */ 1085 my_setopt(curl, CURLOPT_FILETIME, TRUE); 1086 } 1087 1088 my_setopt(curl, CURLOPT_CRLF, config->crlf); 1089 my_setopt_slist(curl, CURLOPT_QUOTE, config->quote); 1090 my_setopt_slist(curl, CURLOPT_POSTQUOTE, config->postquote); 1091 my_setopt_slist(curl, CURLOPT_PREQUOTE, config->prequote); 1092 1093#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES) 1094 { 1095 /* TODO: Make this a run-time check instead of compile-time one. */ 1096 1097 if(config->cookie) 1098 my_setopt_str(curl, CURLOPT_COOKIE, config->cookie); 1099 1100 if(config->cookiefile) 1101 my_setopt_str(curl, CURLOPT_COOKIEFILE, config->cookiefile); 1102 1103 /* new in libcurl 7.9 */ 1104 if(config->cookiejar) 1105 my_setopt_str(curl, CURLOPT_COOKIEJAR, config->cookiejar); 1106 1107 /* new in libcurl 7.9.7 */ 1108 my_setopt(curl, CURLOPT_COOKIESESSION, config->cookiesession); 1109 } 1110#endif 1111 1112 my_setopt_enum(curl, CURLOPT_SSLVERSION, config->ssl_version); 1113 my_setopt_enum(curl, CURLOPT_TIMECONDITION, config->timecond); 1114 my_setopt(curl, CURLOPT_TIMEVALUE, config->condtime); 1115 my_setopt_str(curl, CURLOPT_CUSTOMREQUEST, config->customrequest); 1116 my_setopt(curl, CURLOPT_STDERR, config->errors); 1117 1118 /* three new ones in libcurl 7.3: */ 1119 my_setopt_str(curl, CURLOPT_INTERFACE, config->iface); 1120 my_setopt_str(curl, CURLOPT_KRBLEVEL, config->krblevel); 1121 1122 progressbarinit(&progressbar, config); 1123 if((config->progressmode == CURL_PROGRESS_BAR) && 1124 !config->noprogress && !config->mute) { 1125 /* we want the alternative style, then we have to implement it 1126 ourselves! */ 1127 my_setopt(curl, CURLOPT_PROGRESSFUNCTION, tool_progress_cb); 1128 my_setopt(curl, CURLOPT_PROGRESSDATA, &progressbar); 1129 } 1130 1131 /* new in libcurl 7.6.2: */ 1132 my_setopt_slist(curl, CURLOPT_TELNETOPTIONS, config->telnet_options); 1133 1134 /* new in libcurl 7.7: */ 1135 my_setopt_str(curl, CURLOPT_RANDOM_FILE, config->random_file); 1136 my_setopt(curl, CURLOPT_EGDSOCKET, config->egd_file); 1137 my_setopt(curl, CURLOPT_CONNECTTIMEOUT, config->connecttimeout); 1138 1139 if(config->cipher_list) 1140 my_setopt_str(curl, CURLOPT_SSL_CIPHER_LIST, config->cipher_list); 1141 1142 /* new in libcurl 7.9.2: */ 1143 if(config->disable_epsv) 1144 /* disable it */ 1145 my_setopt(curl, CURLOPT_FTP_USE_EPSV, FALSE); 1146 1147 /* new in libcurl 7.10.5 */ 1148 if(config->disable_eprt) 1149 /* disable it */ 1150 my_setopt(curl, CURLOPT_FTP_USE_EPRT, FALSE); 1151 1152 if(config->tracetype != TRACE_NONE) { 1153 my_setopt(curl, CURLOPT_DEBUGFUNCTION, tool_debug_cb); 1154 my_setopt(curl, CURLOPT_DEBUGDATA, config); 1155 my_setopt(curl, CURLOPT_VERBOSE, TRUE); 1156 } 1157 1158 /* new in curl 7.9.3 */ 1159 if(config->engine) { 1160 res = res_setopt_str(curl, CURLOPT_SSLENGINE, config->engine); 1161 if(res) 1162 goto show_error; 1163 my_setopt(curl, CURLOPT_SSLENGINE_DEFAULT, 1); 1164 } 1165 1166 /* new in curl 7.10.7, extended in 7.19.4 but this only sets 0 or 1 */ 1167 my_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS, 1168 config->ftp_create_dirs); 1169 1170 /* new in curl 7.10.8 */ 1171 if(config->max_filesize) 1172 my_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, 1173 config->max_filesize); 1174 1175 if(4 == config->ip_version) 1176 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); 1177 else if(6 == config->ip_version) 1178 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); 1179 else 1180 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_WHATEVER); 1181 1182 /* new in curl 7.15.5 */ 1183 if(config->ftp_ssl_reqd) 1184 my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL); 1185 1186 /* new in curl 7.11.0 */ 1187 else if(config->ftp_ssl) 1188 my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY); 1189 1190 /* new in curl 7.16.0 */ 1191 else if(config->ftp_ssl_control) 1192 my_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_CONTROL); 1193 1194 /* new in curl 7.16.1 */ 1195 if(config->ftp_ssl_ccc) 1196 my_setopt_enum(curl, CURLOPT_FTP_SSL_CCC, config->ftp_ssl_ccc_mode); 1197 1198#if defined(HAVE_GSSAPI) || defined(USE_WINDOWS_SSPI) 1199 { 1200 /* TODO: Make this a run-time check instead of compile-time one. */ 1201 1202 /* new in curl 7.19.4 */ 1203 if(config->socks5_gssapi_service) 1204 my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_SERVICE, 1205 config->socks5_gssapi_service); 1206 1207 /* new in curl 7.19.4 */ 1208 if(config->socks5_gssapi_nec) 1209 my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_NEC, 1210 config->socks5_gssapi_nec); 1211 } 1212#endif 1213 /* curl 7.13.0 */ 1214 my_setopt_str(curl, CURLOPT_FTP_ACCOUNT, config->ftp_account); 1215 1216 my_setopt(curl, CURLOPT_IGNORE_CONTENT_LENGTH, config->ignorecl); 1217 1218 /* curl 7.14.2 */ 1219 my_setopt(curl, CURLOPT_FTP_SKIP_PASV_IP, config->ftp_skip_ip); 1220 1221 /* curl 7.15.1 */ 1222 my_setopt(curl, CURLOPT_FTP_FILEMETHOD, config->ftp_filemethod); 1223 1224 /* curl 7.15.2 */ 1225 if(config->localport) { 1226 my_setopt(curl, CURLOPT_LOCALPORT, config->localport); 1227 my_setopt_str(curl, CURLOPT_LOCALPORTRANGE, 1228 config->localportrange); 1229 } 1230 1231 /* curl 7.15.5 */ 1232 my_setopt_str(curl, CURLOPT_FTP_ALTERNATIVE_TO_USER, 1233 config->ftp_alternative_to_user); 1234 1235 /* curl 7.16.0 */ 1236 if(config->disable_sessionid) 1237 my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE, 1238 !config->disable_sessionid); 1239 1240 /* curl 7.16.2 */ 1241 if(config->raw) { 1242 my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, FALSE); 1243 my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, FALSE); 1244 } 1245 1246 /* curl 7.17.1 */ 1247 if(!config->nokeepalive) { 1248 my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L); 1249 if(config->alivetime != 0) { 1250#if !defined(TCP_KEEPIDLE) || !defined(TCP_KEEPINTVL) 1251 warnf(config, "Keep-alive functionality somewhat crippled due to " 1252 "missing support in your operating system!\n"); 1253#endif 1254 my_setopt(curl, CURLOPT_TCP_KEEPIDLE, config->alivetime); 1255 my_setopt(curl, CURLOPT_TCP_KEEPINTVL, config->alivetime); 1256 } 1257 } 1258 else 1259 my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 0L); 1260 1261 /* curl 7.20.0 */ 1262 if(config->tftp_blksize) 1263 my_setopt(curl, CURLOPT_TFTP_BLKSIZE, config->tftp_blksize); 1264 1265 if(config->mail_from) 1266 my_setopt_str(curl, CURLOPT_MAIL_FROM, config->mail_from); 1267 1268 if(config->mail_rcpt) 1269 my_setopt_slist(curl, CURLOPT_MAIL_RCPT, config->mail_rcpt); 1270 1271 /* curl 7.20.x */ 1272 if(config->ftp_pret) 1273 my_setopt(curl, CURLOPT_FTP_USE_PRET, TRUE); 1274 1275 if(config->proto_present) 1276 my_setopt_flags(curl, CURLOPT_PROTOCOLS, config->proto); 1277 if(config->proto_redir_present) 1278 my_setopt_flags(curl, CURLOPT_REDIR_PROTOCOLS, config->proto_redir); 1279 1280 if(config->content_disposition 1281 && (urlnode->flags & GETOUT_USEREMOTE) 1282 && (checkprefix("http://", this_url) || 1283 checkprefix("https://", this_url))) 1284 hdrcbdata.honor_cd_filename = TRUE; 1285 else 1286 hdrcbdata.honor_cd_filename = FALSE; 1287 1288 hdrcbdata.outs = &outs; 1289 hdrcbdata.heads = &heads; 1290 1291 my_setopt(curl, CURLOPT_HEADERFUNCTION, tool_header_cb); 1292 my_setopt(curl, CURLOPT_HEADERDATA, &hdrcbdata); 1293 1294 if(config->resolve) 1295 /* new in 7.21.3 */ 1296 my_setopt_slist(curl, CURLOPT_RESOLVE, config->resolve); 1297 1298 /* new in 7.21.4 */ 1299 if(curlinfo->features & CURL_VERSION_TLSAUTH_SRP) { 1300 if(config->tls_username) 1301 my_setopt_str(curl, CURLOPT_TLSAUTH_USERNAME, 1302 config->tls_username); 1303 if(config->tls_password) 1304 my_setopt_str(curl, CURLOPT_TLSAUTH_PASSWORD, 1305 config->tls_password); 1306 if(config->tls_authtype) 1307 my_setopt_str(curl, CURLOPT_TLSAUTH_TYPE, 1308 config->tls_authtype); 1309 } 1310 1311 /* new in 7.22.0 */ 1312 if(config->gssapi_delegation) 1313 my_setopt_str(curl, CURLOPT_GSSAPI_DELEGATION, 1314 config->gssapi_delegation); 1315 1316 /* new in 7.25.0 */ 1317 if(config->ssl_allow_beast) 1318 my_setopt(curl, CURLOPT_SSL_OPTIONS, (long)CURLSSLOPT_ALLOW_BEAST); 1319 1320 if(config->mail_auth) 1321 my_setopt_str(curl, CURLOPT_MAIL_AUTH, config->mail_auth); 1322 1323 /* initialize retry vars for loop below */ 1324 retry_sleep_default = (config->retry_delay) ? 1325 config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */ 1326 1327 retry_numretries = config->req_retry; 1328 retry_sleep = retry_sleep_default; /* ms */ 1329 retrystart = tvnow(); 1330 1331#ifndef CURL_DISABLE_LIBCURL_OPTION 1332 res = easysrc_perform(); 1333 if(res) { 1334 goto show_error; 1335 } 1336#endif 1337 1338 for(;;) { 1339#ifdef USE_METALINK 1340 if(!metalink && config->use_metalink) { 1341 /* If outs.metalink_parser is non-NULL, delete it first. */ 1342 if(outs.metalink_parser) 1343 metalink_parser_context_delete(outs.metalink_parser); 1344 outs.metalink_parser = metalink_parser_context_new(); 1345 if(outs.metalink_parser == NULL) { 1346 res = CURLE_OUT_OF_MEMORY; 1347 goto show_error; 1348 } 1349 fprintf(config->errors, "Metalink: parsing (%s) metalink/XML...\n", 1350 this_url); 1351 } 1352 else if(metalink) 1353 fprintf(config->errors, "Metalink: fetching (%s) from (%s)...\n", 1354 mlfile->filename, this_url); 1355#endif /* USE_METALINK */ 1356 1357 res = curl_easy_perform(curl); 1358 1359 if(outs.is_cd_filename && outs.stream && !config->mute && 1360 outs.filename) 1361 printf("curl: Saved to filename '%s'\n", outs.filename); 1362 1363 /* if retry-max-time is non-zero, make sure we haven't exceeded the 1364 time */ 1365 if(retry_numretries && 1366 (!config->retry_maxtime || 1367 (tvdiff(tvnow(), retrystart) < 1368 config->retry_maxtime*1000L)) ) { 1369 enum { 1370 RETRY_NO, 1371 RETRY_TIMEOUT, 1372 RETRY_HTTP, 1373 RETRY_FTP, 1374 RETRY_LAST /* not used */ 1375 } retry = RETRY_NO; 1376 long response; 1377 if((CURLE_OPERATION_TIMEDOUT == res) || 1378 (CURLE_COULDNT_RESOLVE_HOST == res) || 1379 (CURLE_COULDNT_RESOLVE_PROXY == res) || 1380 (CURLE_FTP_ACCEPT_TIMEOUT == res)) 1381 /* retry timeout always */ 1382 retry = RETRY_TIMEOUT; 1383 else if((CURLE_OK == res) || 1384 (config->failonerror && 1385 (CURLE_HTTP_RETURNED_ERROR == res))) { 1386 /* If it returned OK. _or_ failonerror was enabled and it 1387 returned due to such an error, check for HTTP transient 1388 errors to retry on. */ 1389 char *effective_url = NULL; 1390 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); 1391 if(effective_url && 1392 checkprefix("http", effective_url)) { 1393 /* This was HTTP(S) */ 1394 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); 1395 1396 switch(response) { 1397 case 500: /* Internal Server Error */ 1398 case 502: /* Bad Gateway */ 1399 case 503: /* Service Unavailable */ 1400 case 504: /* Gateway Timeout */ 1401 retry = RETRY_HTTP; 1402 /* 1403 * At this point, we have already written data to the output 1404 * file (or terminal). If we write to a file, we must rewind 1405 * or close/re-open the file so that the next attempt starts 1406 * over from the beginning. 1407 * 1408 * TODO: similar action for the upload case. We might need 1409 * to start over reading from a previous point if we have 1410 * uploaded something when this was returned. 1411 */ 1412 break; 1413 } 1414 } 1415 } /* if CURLE_OK */ 1416 else if(res) { 1417 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); 1418 1419 if(response/100 == 4) 1420 /* 1421 * This is typically when the FTP server only allows a certain 1422 * amount of users and we are not one of them. All 4xx codes 1423 * are transient. 1424 */ 1425 retry = RETRY_FTP; 1426 } 1427 1428 if(retry) { 1429 static const char * const m[]={ 1430 NULL, "timeout", "HTTP error", "FTP error" 1431 }; 1432 warnf(config, "Transient problem: %s " 1433 "Will retry in %ld seconds. " 1434 "%ld retries left.\n", 1435 m[retry], retry_sleep/1000L, retry_numretries); 1436 1437 tool_go_sleep(retry_sleep); 1438 retry_numretries--; 1439 if(!config->retry_delay) { 1440 retry_sleep *= 2; 1441 if(retry_sleep > RETRY_SLEEP_MAX) 1442 retry_sleep = RETRY_SLEEP_MAX; 1443 } 1444 if(outs.bytes && outs.filename) { 1445 /* We have written data to a output file, we truncate file 1446 */ 1447 if(!config->mute) 1448 fprintf(config->errors, "Throwing away %" 1449 CURL_FORMAT_CURL_OFF_T " bytes\n", 1450 outs.bytes); 1451 fflush(outs.stream); 1452 /* truncate file at the position where we started appending */ 1453#ifdef HAVE_FTRUNCATE 1454 if(ftruncate( fileno(outs.stream), outs.init)) { 1455 /* when truncate fails, we can't just append as then we'll 1456 create something strange, bail out */ 1457 if(!config->mute) 1458 fprintf(config->errors, 1459 "failed to truncate, exiting\n"); 1460 res = CURLE_WRITE_ERROR; 1461 goto quit_urls; 1462 } 1463 /* now seek to the end of the file, the position where we 1464 just truncated the file in a large file-safe way */ 1465 fseek(outs.stream, 0, SEEK_END); 1466#else 1467 /* ftruncate is not available, so just reposition the file 1468 to the location we would have truncated it. This won't 1469 work properly with large files on 32-bit systems, but 1470 most of those will have ftruncate. */ 1471 fseek(outs.stream, (long)outs.init, SEEK_SET); 1472#endif 1473 outs.bytes = 0; /* clear for next round */ 1474 } 1475 continue; /* curl_easy_perform loop */ 1476 } 1477 } /* if retry_numretries */ 1478 else if(metalink) { 1479 /* Metalink: Decide to try the next resource or 1480 not. Basically, we want to try the next resource if 1481 download was not successful. */ 1482 long response; 1483 if(CURLE_OK == res) { 1484 /* TODO We want to try next resource when download was 1485 not successful. How to know that? */ 1486 char *effective_url = NULL; 1487 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); 1488 if(effective_url && 1489 curlx_strnequal(effective_url, "http", 4)) { 1490 /* This was HTTP(S) */ 1491 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); 1492 if(response != 200 && response != 206) { 1493 metalink_next_res = 1; 1494 fprintf(config->errors, 1495 "Metalink: fetching (%s) from (%s) FAILED " 1496 "(HTTP status code %d)\n", 1497 mlfile->filename, this_url, response); 1498 } 1499 } 1500 } 1501 else { 1502 metalink_next_res = 1; 1503 fprintf(config->errors, 1504 "Metalink: fetching (%s) from (%s) FAILED (%s)\n", 1505 mlfile->filename, this_url, 1506 (errorbuffer[0]) ? 1507 errorbuffer : curl_easy_strerror((CURLcode)res)); 1508 } 1509 } 1510 if(metalink && !metalink_next_res) 1511 fprintf(config->errors, "Metalink: fetching (%s) from (%s) OK\n", 1512 mlfile->filename, this_url); 1513 1514 /* In all ordinary cases, just break out of loop here */ 1515 break; /* curl_easy_perform loop */ 1516 1517 } 1518 1519 if((config->progressmode == CURL_PROGRESS_BAR) && 1520 progressbar.calls) 1521 /* if the custom progress bar has been displayed, we output a 1522 newline here */ 1523 fputs("\n", progressbar.out); 1524 1525 if(config->writeout) 1526 ourWriteOut(curl, &outs, config->writeout); 1527 1528 if(config->writeenv) 1529 ourWriteEnv(curl); 1530 1531 /* 1532 ** Code within this loop may jump directly here to label 'show_error' 1533 ** in order to display an error message for CURLcode stored in 'res' 1534 ** variable and exit loop once that necessary writing and cleanup 1535 ** in label 'quit_urls' has been done. 1536 */ 1537 1538 show_error: 1539 1540#ifdef __VMS 1541 if(is_vms_shell()) { 1542 /* VMS DCL shell behavior */ 1543 if(!config->showerror) 1544 vms_show = VMSSTS_HIDE; 1545 } 1546 else 1547#endif 1548 if(res && config->showerror) { 1549 fprintf(config->errors, "curl: (%d) %s\n", res, (errorbuffer[0]) ? 1550 errorbuffer : curl_easy_strerror((CURLcode)res)); 1551 if(res == CURLE_SSL_CACERT) 1552 fprintf(config->errors, "%s%s", 1553 CURL_CA_CERT_ERRORMSG1, CURL_CA_CERT_ERRORMSG2); 1554 } 1555 1556 /* Fall through comment to 'quit_urls' label */ 1557 1558 /* 1559 ** Upon error condition and always that a message has already been 1560 ** displayed, code within this loop may jump directly here to label 1561 ** 'quit_urls' otherwise it should jump to 'show_error' label above. 1562 ** 1563 ** When 'res' variable is _not_ CURLE_OK loop will exit once that 1564 ** all code following 'quit_urls' has been executed. Otherwise it 1565 ** will loop to the beginning from where it may exit if there are 1566 ** no more urls left. 1567 */ 1568 1569 quit_urls: 1570 1571 /* Set file extended attributes */ 1572 if(!res && config->xattr && outs.fopened && outs.stream) { 1573 int rc = fwrite_xattr(curl, fileno(outs.stream)); 1574 if(rc) 1575 warnf(config, "Error setting extended attributes: %s\n", 1576 strerror(errno)); 1577 } 1578 1579 /* Close the file */ 1580 if(outs.fopened && outs.stream) { 1581 int rc = fclose(outs.stream); 1582 if(!res && rc) { 1583 /* something went wrong in the writing process */ 1584 res = CURLE_WRITE_ERROR; 1585 fprintf(config->errors, "(%d) Failed writing body\n", res); 1586 } 1587 } 1588 else if(!outs.s_isreg && outs.stream) { 1589 /* Dump standard stream buffered data */ 1590 int rc = fflush(outs.stream); 1591 if(!res && rc) { 1592 /* something went wrong in the writing process */ 1593 res = CURLE_WRITE_ERROR; 1594 fprintf(config->errors, "(%d) Failed writing body\n", res); 1595 } 1596 } 1597 1598#ifdef __AMIGA__ 1599 if(!res && outs.s_isreg && outs.filename) { 1600 /* Set the url (up to 80 chars) as comment for the file */ 1601 if(strlen(url) > 78) 1602 url[79] = '\0'; 1603 SetComment(outs.filename, url); 1604 } 1605#endif 1606 1607#ifdef HAVE_UTIME 1608 /* File time can only be set _after_ the file has been closed */ 1609 if(!res && config->remote_time && outs.s_isreg && outs.filename) { 1610 /* Ask libcurl if we got a remote file time */ 1611 long filetime = -1; 1612 curl_easy_getinfo(curl, CURLINFO_FILETIME, &filetime); 1613 if(filetime >= 0) { 1614 struct utimbuf times; 1615 times.actime = (time_t)filetime; 1616 times.modtime = (time_t)filetime; 1617 utime(outs.filename, ×); /* set the time we got */ 1618 } 1619 } 1620#endif 1621 1622#ifdef USE_METALINK 1623 if(!metalink && config->use_metalink && res == CURLE_OK) { 1624 int rv = parse_metalink(config, &outs, this_url); 1625 if(rv == 0) 1626 fprintf(config->errors, "Metalink: parsing (%s) OK\n", this_url); 1627 else if(rv == -1) 1628 fprintf(config->errors, "Metalink: parsing (%s) FAILED\n", 1629 this_url); 1630 } 1631 else if(metalink && res == CURLE_OK && !metalink_next_res) { 1632 int rv = metalink_check_hash(config, mlfile, outs.filename); 1633 if(rv == 0) { 1634 metalink_next_res = 1; 1635 } 1636 } 1637#endif /* USE_METALINK */ 1638 1639 /* No more business with this output struct */ 1640 if(outs.alloc_filename) 1641 Curl_safefree(outs.filename); 1642#ifdef USE_METALINK 1643 if(outs.metalink_parser) 1644 metalink_parser_context_delete(outs.metalink_parser); 1645#endif /* USE_METALINK */ 1646 memset(&outs, 0, sizeof(struct OutStruct)); 1647 hdrcbdata.outs = NULL; 1648 1649 /* Free loop-local allocated memory and close loop-local opened fd */ 1650 1651 Curl_safefree(outfile); 1652 Curl_safefree(this_url); 1653 1654 if(infdopen) 1655 close(infd); 1656 1657 if(metalink) { 1658 /* Should exit if error is fatal. */ 1659 if(is_fatal_error(res)) { 1660 break; 1661 } 1662 if(!metalink_next_res) 1663 break; 1664 mlres = mlres->next; 1665 if(mlres == NULL) 1666 /* TODO If metalink_next_res is 1 and mlres is NULL, 1667 * set res to error code 1668 */ 1669 break; 1670 } 1671 else 1672 if(urlnum > 1) { 1673 /* when url globbing, exit loop upon critical error */ 1674 if(is_fatal_error(res)) 1675 break; 1676 } 1677 else if(res) 1678 /* when not url globbing, exit loop upon any error */ 1679 break; 1680 1681 } /* loop to the next URL */ 1682 1683 /* Free loop-local allocated memory */ 1684 1685 Curl_safefree(uploadfile); 1686 1687 if(urls) { 1688 /* Free list of remaining URLs */ 1689 glob_cleanup(urls); 1690 urls = NULL; 1691 } 1692 1693 if(infilenum > 1) { 1694 /* when file globbing, exit loop upon critical error */ 1695 if(is_fatal_error(res)) 1696 break; 1697 } 1698 else if(res) 1699 /* when not file globbing, exit loop upon any error */ 1700 break; 1701 1702 } /* loop to the next globbed upload file */ 1703 1704 /* Free loop-local allocated memory */ 1705 1706 Curl_safefree(outfiles); 1707 1708 if(inglob) { 1709 /* Free list of globbed upload files */ 1710 glob_cleanup(inglob); 1711 inglob = NULL; 1712 } 1713 1714 /* Free this URL node data without destroying the 1715 the node itself nor modifying next pointer. */ 1716 Curl_safefree(urlnode->url); 1717 Curl_safefree(urlnode->outfile); 1718 Curl_safefree(urlnode->infile); 1719 urlnode->flags = 0; 1720 1721 /* 1722 ** Bail out upon critical errors 1723 */ 1724 if(is_fatal_error(res)) 1725 goto quit_curl; 1726 1727 } /* for-loop through all URLs */ 1728 1729 /* 1730 ** Nested loops end here. 1731 */ 1732 1733 quit_curl: 1734 1735 /* Free function-local referenced allocated memory */ 1736 Curl_safefree(httpgetfields); 1737 1738 /* Free list of given URLs */ 1739 clean_getout(config); 1740 1741 /* Cleanup the curl handle now that our 1742 progressbar struct is still in scope */ 1743 if(curl) { 1744 curl_easy_cleanup(curl); 1745 config->easy = curl = NULL; 1746 } 1747#ifndef CURL_DISABLE_LIBCURL_OPTION 1748 easysrc_cleanup(); 1749#endif 1750 1751 hdrcbdata.heads = NULL; 1752 1753 /* Close function-local opened file descriptors */ 1754 1755 if(heads.fopened && heads.stream) 1756 fclose(heads.stream); 1757 if(heads.alloc_filename) 1758 Curl_safefree(heads.filename); 1759 1760 if(config->trace_fopened && config->trace_stream) 1761 fclose(config->trace_stream); 1762 1763#ifndef CURL_DISABLE_LIBCURL_OPTION 1764 /* Dump the libcurl code if previously enabled. 1765 NOTE: that this function relies on config->errors amongst other things 1766 so not everything can be closed and cleaned before this is called */ 1767 dumpeasysrc(config); 1768#endif 1769 1770 if(config->errors_fopened && config->errors) 1771 fclose(config->errors); 1772 1773 /* Release metalink related resources here */ 1774 clean_metalink(config); 1775 1776 main_free(); /* cleanup */ 1777 1778 return res; 1779} 1780 1781