1/*************************************************************************** 2 * _ _ ____ _ 3 * Project ___| | | | _ \| | 4 * / __| | | | |_) | | 5 * | (__| |_| | _ <| |___ 6 * \___|\___/|_| \_\_____| 7 * 8 * Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al. 9 * 10 * This software is licensed as described in the file COPYING, which 11 * you should have received as part of this distribution. The terms 12 * are also available at http://curl.haxx.se/docs/copyright.html. 13 * 14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell 15 * copies of the Software, and permit persons to whom the Software is 16 * furnished to do so, under the terms of the COPYING file. 17 * 18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 19 * KIND, either express or implied. 20 * 21 ***************************************************************************/ 22#include "tool_setup.h" 23 24#ifdef HAVE_FCNTL_H 25# include <fcntl.h> 26#endif 27 28#ifdef HAVE_UTIME_H 29# include <utime.h> 30#elif defined(HAVE_SYS_UTIME_H) 31# include <sys/utime.h> 32#endif 33 34#ifdef HAVE_LOCALE_H 35# include <locale.h> 36#endif 37 38#ifdef HAVE_NETINET_TCP_H 39# include <netinet/tcp.h> 40#endif 41 42#ifdef __VMS 43# include <fabdef.h> 44#endif 45 46#include "rawstr.h" 47 48#define ENABLE_CURLX_PRINTF 49/* use our own printf() functions */ 50#include "curlx.h" 51 52#include "tool_binmode.h" 53#include "tool_cfgable.h" 54#include "tool_cb_dbg.h" 55#include "tool_cb_hdr.h" 56#include "tool_cb_prg.h" 57#include "tool_cb_rea.h" 58#include "tool_cb_see.h" 59#include "tool_cb_wrt.h" 60#include "tool_dirhie.h" 61#include "tool_doswin.h" 62#include "tool_easysrc.h" 63#include "tool_getparam.h" 64#include "tool_helpers.h" 65#include "tool_homedir.h" 66#include "tool_libinfo.h" 67#include "tool_main.h" 68#include "tool_metalink.h" 69#include "tool_msgs.h" 70#include "tool_operate.h" 71#include "tool_operhlp.h" 72#include "tool_paramhlp.h" 73#include "tool_parsecfg.h" 74#include "tool_setopt.h" 75#include "tool_sleep.h" 76#include "tool_urlglob.h" 77#include "tool_util.h" 78#include "tool_writeenv.h" 79#include "tool_writeout.h" 80#include "tool_xattr.h" 81#include "tool_vms.h" 82#include "tool_help.h" 83#include "tool_hugehelp.h" 84 85#include "memdebug.h" /* keep this as LAST include */ 86 87#ifdef CURLDEBUG 88/* libcurl's debug builds provide an extra function */ 89CURLcode curl_easy_perform_ev(CURL *easy); 90#endif 91 92#define CURLseparator "--_curl_--" 93 94#ifndef O_BINARY 95/* since O_BINARY as used in bitmasks, setting it to zero makes it usable in 96 source code but yet it doesn't ruin anything */ 97# define O_BINARY 0 98#endif 99 100#define CURL_CA_CERT_ERRORMSG1 \ 101 "More details here: http://curl.haxx.se/docs/sslcerts.html\n\n" \ 102 "curl performs SSL certificate verification by default, " \ 103 "using a \"bundle\"\n" \ 104 " of Certificate Authority (CA) public keys (CA certs). If the default\n" \ 105 " bundle file isn't adequate, you can specify an alternate file\n" \ 106 " using the --cacert option.\n" 107 108#define CURL_CA_CERT_ERRORMSG2 \ 109 "If this HTTPS server uses a certificate signed by a CA represented in\n" \ 110 " the bundle, the certificate verification probably failed due to a\n" \ 111 " problem with the certificate (it might be expired, or the name might\n" \ 112 " not match the domain name in the URL).\n" \ 113 "If you'd like to turn off curl's verification of the certificate, use\n" \ 114 " the -k (or --insecure) option.\n" 115 116static bool is_fatal_error(CURLcode code) 117{ 118 switch(code) { 119 /* TODO: Should CURLE_SSL_CACERT be included as critical error ? */ 120 case CURLE_FAILED_INIT: 121 case CURLE_OUT_OF_MEMORY: 122 case CURLE_UNKNOWN_OPTION: 123 case CURLE_FUNCTION_NOT_FOUND: 124 case CURLE_BAD_FUNCTION_ARGUMENT: 125 /* critical error */ 126 return TRUE; 127 default: 128 break; 129 } 130 131 /* no error or not critical */ 132 return FALSE; 133} 134 135#ifdef __VMS 136/* 137 * get_vms_file_size does what it takes to get the real size of the file 138 * 139 * For fixed files, find out the size of the EOF block and adjust. 140 * 141 * For all others, have to read the entire file in, discarding the contents. 142 * Most posted text files will be small, and binary files like zlib archives 143 * and CD/DVD images should be either a STREAM_LF format or a fixed format. 144 * 145 */ 146static curl_off_t vms_realfilesize(const char * name, 147 const struct_stat * stat_buf) 148{ 149 char buffer[8192]; 150 curl_off_t count; 151 int ret_stat; 152 FILE * file; 153 154 file = fopen(name, "r"); 155 if(file == NULL) { 156 return 0; 157 } 158 count = 0; 159 ret_stat = 1; 160 while(ret_stat > 0) { 161 ret_stat = fread(buffer, 1, sizeof(buffer), file); 162 if(ret_stat != 0) 163 count += ret_stat; 164 } 165 fclose(file); 166 167 return count; 168} 169 170/* 171 * 172 * VmsSpecialSize checks to see if the stat st_size can be trusted and 173 * if not to call a routine to get the correct size. 174 * 175 */ 176static curl_off_t VmsSpecialSize(const char * name, 177 const struct_stat * stat_buf) 178{ 179 switch(stat_buf->st_fab_rfm) { 180 case FAB$C_VAR: 181 case FAB$C_VFC: 182 return vms_realfilesize(name, stat_buf); 183 break; 184 default: 185 return stat_buf->st_size; 186 } 187} 188#endif /* __VMS */ 189 190static CURLcode operate_do(struct GlobalConfig *global, 191 struct OperationConfig *config) 192{ 193 char errorbuffer[CURL_ERROR_SIZE]; 194 struct ProgressData progressbar; 195 struct getout *urlnode; 196 197 struct HdrCbData hdrcbdata; 198 struct OutStruct heads; 199 200 metalinkfile *mlfile_last = NULL; 201 202 CURL *curl = config->easy; 203 char *httpgetfields = NULL; 204 205 CURLcode res = CURLE_OK; 206 unsigned long li; 207 208 /* Save the values of noprogress and isatty to restore them later on */ 209 bool orig_noprogress = global->noprogress; 210 bool orig_isatty = global->isatty; 211 212 errorbuffer[0] = '\0'; 213 214 /* default headers output stream is stdout */ 215 memset(&hdrcbdata, 0, sizeof(struct HdrCbData)); 216 memset(&heads, 0, sizeof(struct OutStruct)); 217 heads.stream = stdout; 218 heads.config = config; 219 220 /* 221 ** Beyond this point no return'ing from this function allowed. 222 ** Jump to label 'quit_curl' in order to abandon this function 223 ** from outside of nested loops further down below. 224 */ 225 226 /* Check we have a url */ 227 if(!config->url_list || !config->url_list->url) { 228 helpf(global->errors, "no URL specified!\n"); 229 res = CURLE_FAILED_INIT; 230 goto quit_curl; 231 } 232 233 /* On WIN32 we can't set the path to curl-ca-bundle.crt 234 * at compile time. So we look here for the file in two ways: 235 * 1: look at the environment variable CURL_CA_BUNDLE for a path 236 * 2: if #1 isn't found, use the windows API function SearchPath() 237 * to find it along the app's path (includes app's dir and CWD) 238 * 239 * We support the environment variable thing for non-Windows platforms 240 * too. Just for the sake of it. 241 */ 242 if(!config->cacert && 243 !config->capath && 244 !config->insecure_ok) { 245 char *env; 246 env = curlx_getenv("CURL_CA_BUNDLE"); 247 if(env) { 248 config->cacert = strdup(env); 249 if(!config->cacert) { 250 curl_free(env); 251 helpf(global->errors, "out of memory\n"); 252 res = CURLE_OUT_OF_MEMORY; 253 goto quit_curl; 254 } 255 } 256 else { 257 env = curlx_getenv("SSL_CERT_DIR"); 258 if(env) { 259 config->capath = strdup(env); 260 if(!config->capath) { 261 curl_free(env); 262 helpf(global->errors, "out of memory\n"); 263 res = CURLE_OUT_OF_MEMORY; 264 goto quit_curl; 265 } 266 } 267 else { 268 env = curlx_getenv("SSL_CERT_FILE"); 269 if(env) { 270 config->cacert = strdup(env); 271 if(!config->cacert) { 272 curl_free(env); 273 helpf(global->errors, "out of memory\n"); 274 res = CURLE_OUT_OF_MEMORY; 275 goto quit_curl; 276 } 277 } 278 } 279 } 280 281 if(env) 282 curl_free(env); 283#ifdef WIN32 284 else { 285 res = FindWin32CACert(config, "curl-ca-bundle.crt"); 286 if(res) 287 goto quit_curl; 288 } 289#endif 290 } 291 292 if(config->postfields) { 293 if(config->use_httpget) { 294 /* Use the postfields data for a http get */ 295 httpgetfields = strdup(config->postfields); 296 Curl_safefree(config->postfields); 297 if(!httpgetfields) { 298 helpf(global->errors, "out of memory\n"); 299 res = CURLE_OUT_OF_MEMORY; 300 goto quit_curl; 301 } 302 if(SetHTTPrequest(config, 303 (config->no_body?HTTPREQ_HEAD:HTTPREQ_GET), 304 &config->httpreq)) { 305 res = CURLE_FAILED_INIT; 306 goto quit_curl; 307 } 308 } 309 else { 310 if(SetHTTPrequest(config, HTTPREQ_SIMPLEPOST, &config->httpreq)) { 311 res = CURLE_FAILED_INIT; 312 goto quit_curl; 313 } 314 } 315 } 316 317 /* Single header file for all URLs */ 318 if(config->headerfile) { 319 /* open file for output: */ 320 if(!curlx_strequal(config->headerfile, "-")) { 321 FILE *newfile = fopen(config->headerfile, "wb"); 322 if(!newfile) { 323 warnf(config, "Failed to open %s\n", config->headerfile); 324 res = CURLE_WRITE_ERROR; 325 goto quit_curl; 326 } 327 else { 328 heads.filename = config->headerfile; 329 heads.s_isreg = TRUE; 330 heads.fopened = TRUE; 331 heads.stream = newfile; 332 } 333 } 334 else { 335 /* always use binary mode for protocol header output */ 336 set_binmode(heads.stream); 337 } 338 } 339 340 /* 341 ** Nested loops start here. 342 */ 343 344 /* loop through the list of given URLs */ 345 346 for(urlnode = config->url_list; urlnode; urlnode = urlnode->next) { 347 348 unsigned long up; /* upload file counter within a single upload glob */ 349 char *infiles; /* might be a glob pattern */ 350 char *outfiles; 351 unsigned long infilenum; 352 URLGlob *inglob; 353 354 int metalink = 0; /* nonzero for metalink download. */ 355 metalinkfile *mlfile; 356 metalink_resource *mlres; 357 358 outfiles = NULL; 359 infilenum = 1; 360 inglob = NULL; 361 362 if(urlnode->flags & GETOUT_METALINK) { 363 metalink = 1; 364 if(mlfile_last == NULL) { 365 mlfile_last = config->metalinkfile_list; 366 } 367 mlfile = mlfile_last; 368 mlfile_last = mlfile_last->next; 369 mlres = mlfile->resource; 370 } 371 else { 372 mlfile = NULL; 373 mlres = NULL; 374 } 375 376 /* urlnode->url is the full URL (it might be NULL) */ 377 378 if(!urlnode->url) { 379 /* This node has no URL. Free node data without destroying the 380 node itself nor modifying next pointer and continue to next */ 381 Curl_safefree(urlnode->outfile); 382 Curl_safefree(urlnode->infile); 383 urlnode->flags = 0; 384 continue; /* next URL please */ 385 } 386 387 /* save outfile pattern before expansion */ 388 if(urlnode->outfile) { 389 outfiles = strdup(urlnode->outfile); 390 if(!outfiles) { 391 helpf(global->errors, "out of memory\n"); 392 res = CURLE_OUT_OF_MEMORY; 393 break; 394 } 395 } 396 397 infiles = urlnode->infile; 398 399 if(!config->globoff && infiles) { 400 /* Unless explicitly shut off */ 401 res = (CURLcode) glob_url(&inglob, infiles, &infilenum, 402 global->showerror?global->errors:NULL); 403 if(res) { 404 Curl_safefree(outfiles); 405 break; 406 } 407 } 408 409 /* Here's the loop for uploading multiple files within the same 410 single globbed string. If no upload, we enter the loop once anyway. */ 411 for(up = 0 ; up < infilenum; up++) { 412 413 char *uploadfile; /* a single file, never a glob */ 414 int separator; 415 URLGlob *urls; 416 unsigned long urlnum; 417 418 uploadfile = NULL; 419 urls = NULL; 420 urlnum = 0; 421 422 if(!up && !infiles) 423 Curl_nop_stmt; 424 else { 425 if(inglob) { 426 res = (CURLcode) glob_next_url(&uploadfile, inglob); 427 if(res == CURLE_OUT_OF_MEMORY) 428 helpf(global->errors, "out of memory\n"); 429 } 430 else if(!up) { 431 uploadfile = strdup(infiles); 432 if(!uploadfile) { 433 helpf(global->errors, "out of memory\n"); 434 res = CURLE_OUT_OF_MEMORY; 435 } 436 } 437 else 438 uploadfile = NULL; 439 if(!uploadfile) 440 break; 441 } 442 443 if(metalink) { 444 /* For Metalink download, we don't use glob. Instead we use 445 the number of resources as urlnum. */ 446 urlnum = count_next_metalink_resource(mlfile); 447 } 448 else 449 if(!config->globoff) { 450 /* Unless explicitly shut off, we expand '{...}' and '[...]' 451 expressions and return total number of URLs in pattern set */ 452 res = (CURLcode) glob_url(&urls, urlnode->url, &urlnum, 453 global->showerror?global->errors:NULL); 454 if(res) { 455 Curl_safefree(uploadfile); 456 break; 457 } 458 } 459 else 460 urlnum = 1; /* without globbing, this is a single URL */ 461 462 /* if multiple files extracted to stdout, insert separators! */ 463 separator= ((!outfiles || curlx_strequal(outfiles, "-")) && urlnum > 1); 464 465 /* Here's looping around each globbed URL */ 466 for(li = 0 ; li < urlnum; li++) { 467 468 int infd; 469 bool infdopen; 470 char *outfile; 471 struct OutStruct outs; 472 struct InStruct input; 473 struct timeval retrystart; 474 curl_off_t uploadfilesize; 475 long retry_numretries; 476 long retry_sleep_default; 477 long retry_sleep; 478 char *this_url = NULL; 479 int metalink_next_res = 0; 480 481 outfile = NULL; 482 infdopen = FALSE; 483 infd = STDIN_FILENO; 484 uploadfilesize = -1; /* -1 means unknown */ 485 486 /* default output stream is stdout */ 487 memset(&outs, 0, sizeof(struct OutStruct)); 488 outs.stream = stdout; 489 outs.config = config; 490 491 if(metalink) { 492 /* For Metalink download, use name in Metalink file as 493 filename. */ 494 outfile = strdup(mlfile->filename); 495 if(!outfile) { 496 res = CURLE_OUT_OF_MEMORY; 497 goto show_error; 498 } 499 this_url = strdup(mlres->url); 500 if(!this_url) { 501 res = CURLE_OUT_OF_MEMORY; 502 goto show_error; 503 } 504 } 505 else { 506 if(urls) { 507 res = (CURLcode) glob_next_url(&this_url, urls); 508 if(res) 509 goto show_error; 510 } 511 else if(!li) { 512 this_url = strdup(urlnode->url); 513 if(!this_url) { 514 res = CURLE_OUT_OF_MEMORY; 515 goto show_error; 516 } 517 } 518 else 519 this_url = NULL; 520 if(!this_url) 521 break; 522 523 if(outfiles) { 524 outfile = strdup(outfiles); 525 if(!outfile) { 526 res = CURLE_OUT_OF_MEMORY; 527 goto show_error; 528 } 529 } 530 } 531 532 if(((urlnode->flags&GETOUT_USEREMOTE) || 533 (outfile && !curlx_strequal("-", outfile))) && 534 (metalink || !config->use_metalink)) { 535 536 /* 537 * We have specified a file name to store the result in, or we have 538 * decided we want to use the remote file name. 539 */ 540 541 if(!outfile) { 542 /* extract the file name from the URL */ 543 res = get_url_file_name(&outfile, this_url); 544 if(res) 545 goto show_error; 546 if((!outfile || !*outfile) && !config->content_disposition) { 547 helpf(global->errors, "Remote file name has no length!\n"); 548 res = CURLE_WRITE_ERROR; 549 goto quit_urls; 550 } 551#if defined(MSDOS) || defined(WIN32) 552 /* For DOS and WIN32, we do some major replacing of 553 bad characters in the file name before using it */ 554 outfile = sanitize_dos_name(outfile); 555 if(!outfile) { 556 res = CURLE_OUT_OF_MEMORY; 557 goto show_error; 558 } 559#endif /* MSDOS || WIN32 */ 560 } 561 else if(urls) { 562 /* fill '#1' ... '#9' terms from URL pattern */ 563 char *storefile = outfile; 564 res = (CURLcode) glob_match_url(&outfile, storefile, urls); 565 Curl_safefree(storefile); 566 if(res) { 567 /* bad globbing */ 568 warnf(config, "bad output glob!\n"); 569 goto quit_urls; 570 } 571 } 572 573 /* Create the directory hierarchy, if not pre-existent to a multiple 574 file output call */ 575 576 if(config->create_dirs || metalink) { 577 res = create_dir_hierarchy(outfile, global->errors); 578 /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */ 579 if(res == CURLE_WRITE_ERROR) 580 goto quit_urls; 581 if(res) { 582 goto show_error; 583 } 584 } 585 586 if((urlnode->flags & GETOUT_USEREMOTE) 587 && config->content_disposition) { 588 /* Our header callback MIGHT set the filename */ 589 DEBUGASSERT(!outs.filename); 590 } 591 592 if(config->resume_from_current) { 593 /* We're told to continue from where we are now. Get the size 594 of the file as it is now and open it for append instead */ 595 struct_stat fileinfo; 596 /* VMS -- Danger, the filesize is only valid for stream files */ 597 if(0 == stat(outfile, &fileinfo)) 598 /* set offset to current file size: */ 599 config->resume_from = fileinfo.st_size; 600 else 601 /* let offset be 0 */ 602 config->resume_from = 0; 603 } 604 605 if(config->resume_from) { 606#ifdef __VMS 607 /* open file for output, forcing VMS output format into stream 608 mode which is needed for stat() call above to always work. */ 609 FILE *file = fopen(outfile, config->resume_from?"ab":"wb", 610 "ctx=stm", "rfm=stmlf", "rat=cr", "mrs=0"); 611#else 612 /* open file for output: */ 613 FILE *file = fopen(outfile, config->resume_from?"ab":"wb"); 614#endif 615 if(!file) { 616 helpf(global->errors, "Can't open '%s'!\n", outfile); 617 res = CURLE_WRITE_ERROR; 618 goto quit_urls; 619 } 620 outs.fopened = TRUE; 621 outs.stream = file; 622 outs.init = config->resume_from; 623 } 624 else { 625 outs.stream = NULL; /* open when needed */ 626 } 627 outs.filename = outfile; 628 outs.s_isreg = TRUE; 629 } 630 631 if(uploadfile && !stdin_upload(uploadfile)) { 632 /* 633 * We have specified a file to upload and it isn't "-". 634 */ 635 struct_stat fileinfo; 636 637 this_url = add_file_name_to_url(curl, this_url, uploadfile); 638 if(!this_url) { 639 res = CURLE_OUT_OF_MEMORY; 640 goto show_error; 641 } 642 /* VMS Note: 643 * 644 * Reading binary from files can be a problem... Only FIXED, VAR 645 * etc WITHOUT implied CC will work Others need a \n appended to a 646 * line 647 * 648 * - Stat gives a size but this is UNRELIABLE in VMS As a f.e. a 649 * fixed file with implied CC needs to have a byte added for every 650 * record processed, this can by derived from Filesize & recordsize 651 * for VARiable record files the records need to be counted! for 652 * every record add 1 for linefeed and subtract 2 for the record 653 * header for VARIABLE header files only the bare record data needs 654 * to be considered with one appended if implied CC 655 */ 656#ifdef __VMS 657 /* Calculate the real upload site for VMS */ 658 infd = -1; 659 if(stat(uploadfile, &fileinfo) == 0) { 660 fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo); 661 switch (fileinfo.st_fab_rfm) { 662 case FAB$C_VAR: 663 case FAB$C_VFC: 664 case FAB$C_STMCR: 665 infd = open(uploadfile, O_RDONLY | O_BINARY); 666 break; 667 default: 668 infd = open(uploadfile, O_RDONLY | O_BINARY, 669 "rfm=stmlf", "ctx=stm"); 670 } 671 } 672 if(infd == -1) 673#else 674 infd = open(uploadfile, O_RDONLY | O_BINARY); 675 if((infd == -1) || fstat(infd, &fileinfo)) 676#endif 677 { 678 helpf(global->errors, "Can't open '%s'!\n", uploadfile); 679 if(infd != -1) { 680 close(infd); 681 infd = STDIN_FILENO; 682 } 683 res = CURLE_READ_ERROR; 684 goto quit_urls; 685 } 686 infdopen = TRUE; 687 688 /* we ignore file size for char/block devices, sockets, etc. */ 689 if(S_ISREG(fileinfo.st_mode)) 690 uploadfilesize = fileinfo.st_size; 691 692 } 693 else if(uploadfile && stdin_upload(uploadfile)) { 694 /* count to see if there are more than one auth bit set 695 in the authtype field */ 696 int authbits = 0; 697 int bitcheck = 0; 698 while(bitcheck < 32) { 699 if(config->authtype & (1UL << bitcheck++)) { 700 authbits++; 701 if(authbits > 1) { 702 /* more than one, we're done! */ 703 break; 704 } 705 } 706 } 707 708 /* 709 * If the user has also selected --anyauth or --proxy-anyauth 710 * we should warn him/her. 711 */ 712 if(config->proxyanyauth || (authbits>1)) { 713 warnf(config, 714 "Using --anyauth or --proxy-anyauth with upload from stdin" 715 " involves a big risk of it not working. Use a temporary" 716 " file or a fixed auth type instead!\n"); 717 } 718 719 DEBUGASSERT(infdopen == FALSE); 720 DEBUGASSERT(infd == STDIN_FILENO); 721 722 set_binmode(stdin); 723 if(curlx_strequal(uploadfile, ".")) { 724 if(curlx_nonblock((curl_socket_t)infd, TRUE) < 0) 725 warnf(config, 726 "fcntl failed on fd=%d: %s\n", infd, strerror(errno)); 727 } 728 } 729 730 if(uploadfile && config->resume_from_current) 731 config->resume_from = -1; /* -1 will then force get-it-yourself */ 732 733 if(output_expected(this_url, uploadfile) && outs.stream && 734 isatty(fileno(outs.stream))) 735 /* we send the output to a tty, therefore we switch off the progress 736 meter */ 737 global->noprogress = global->isatty = TRUE; 738 else { 739 /* progress meter is per download, so restore config 740 values */ 741 global->noprogress = orig_noprogress; 742 global->isatty = orig_isatty; 743 } 744 745 if(urlnum > 1 && !global->mute) { 746 fprintf(global->errors, "\n[%lu/%lu]: %s --> %s\n", 747 li+1, urlnum, this_url, outfile ? outfile : "<stdout>"); 748 if(separator) 749 printf("%s%s\n", CURLseparator, this_url); 750 } 751 if(httpgetfields) { 752 char *urlbuffer; 753 /* Find out whether the url contains a file name */ 754 const char *pc = strstr(this_url, "://"); 755 char sep = '?'; 756 if(pc) 757 pc += 3; 758 else 759 pc = this_url; 760 761 pc = strrchr(pc, '/'); /* check for a slash */ 762 763 if(pc) { 764 /* there is a slash present in the URL */ 765 766 if(strchr(pc, '?')) 767 /* Ouch, there's already a question mark in the URL string, we 768 then append the data with an ampersand separator instead! */ 769 sep='&'; 770 } 771 /* 772 * Then append ? followed by the get fields to the url. 773 */ 774 if(pc) 775 urlbuffer = aprintf("%s%c%s", this_url, sep, httpgetfields); 776 else 777 /* Append / before the ? to create a well-formed url 778 if the url contains a hostname only 779 */ 780 urlbuffer = aprintf("%s/?%s", this_url, httpgetfields); 781 782 if(!urlbuffer) { 783 res = CURLE_OUT_OF_MEMORY; 784 goto show_error; 785 } 786 787 Curl_safefree(this_url); /* free previous URL */ 788 this_url = urlbuffer; /* use our new URL instead! */ 789 } 790 791 if(!global->errors) 792 global->errors = stderr; 793 794 if((!outfile || !strcmp(outfile, "-")) && !config->use_ascii) { 795 /* We get the output to stdout and we have not got the ASCII/text 796 flag, then set stdout to be binary */ 797 set_binmode(stdout); 798 } 799 800 if(config->tcp_nodelay) 801 my_setopt(curl, CURLOPT_TCP_NODELAY, 1L); 802 803 /* where to store */ 804 my_setopt(curl, CURLOPT_WRITEDATA, &outs); 805 if(metalink || !config->use_metalink) 806 /* what call to write */ 807 my_setopt(curl, CURLOPT_WRITEFUNCTION, tool_write_cb); 808#ifdef USE_METALINK 809 else 810 /* Set Metalink specific write callback function to parse 811 XML data progressively. */ 812 my_setopt(curl, CURLOPT_WRITEFUNCTION, metalink_write_cb); 813#endif /* USE_METALINK */ 814 815 /* for uploads */ 816 input.fd = infd; 817 input.config = config; 818 /* Note that if CURLOPT_READFUNCTION is fread (the default), then 819 * lib/telnet.c will Curl_poll() on the input file descriptor 820 * rather then calling the READFUNCTION at regular intervals. 821 * The circumstances in which it is preferable to enable this 822 * behaviour, by omitting to set the READFUNCTION & READDATA options, 823 * have not been determined. 824 */ 825 my_setopt(curl, CURLOPT_READDATA, &input); 826 /* what call to read */ 827 my_setopt(curl, CURLOPT_READFUNCTION, tool_read_cb); 828 829 /* in 7.18.0, the CURLOPT_SEEKFUNCTION/DATA pair is taking over what 830 CURLOPT_IOCTLFUNCTION/DATA pair previously provided for seeking */ 831 my_setopt(curl, CURLOPT_SEEKDATA, &input); 832 my_setopt(curl, CURLOPT_SEEKFUNCTION, tool_seek_cb); 833 834 if(config->recvpersecond) 835 /* tell libcurl to use a smaller sized buffer as it allows us to 836 make better sleeps! 7.9.9 stuff! */ 837 my_setopt(curl, CURLOPT_BUFFERSIZE, (long)config->recvpersecond); 838 839 /* size of uploaded file: */ 840 if(uploadfilesize != -1) 841 my_setopt(curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize); 842 my_setopt_str(curl, CURLOPT_URL, this_url); /* what to fetch */ 843 my_setopt(curl, CURLOPT_NOPROGRESS, global->noprogress?1L:0L); 844 if(config->no_body) { 845 my_setopt(curl, CURLOPT_NOBODY, 1L); 846 my_setopt(curl, CURLOPT_HEADER, 1L); 847 } 848 /* If --metalink is used, we ignore --include (headers in 849 output) option because mixing headers to the body will 850 confuse XML parser and/or hash check will fail. */ 851 else if(!config->use_metalink) 852 my_setopt(curl, CURLOPT_HEADER, config->include_headers?1L:0L); 853 854 if(config->xoauth2_bearer) 855 my_setopt_str(curl, CURLOPT_XOAUTH2_BEARER, config->xoauth2_bearer); 856 857#if !defined(CURL_DISABLE_PROXY) 858 { 859 /* TODO: Make this a run-time check instead of compile-time one. */ 860 861 my_setopt_str(curl, CURLOPT_PROXY, config->proxy); 862 my_setopt_str(curl, CURLOPT_PROXYUSERPWD, config->proxyuserpwd); 863 864 /* new in libcurl 7.3 */ 865 my_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, config->proxytunnel?1L:0L); 866 867 /* new in libcurl 7.5 */ 868 if(config->proxy) 869 my_setopt_enum(curl, CURLOPT_PROXYTYPE, (long)config->proxyver); 870 871 /* new in libcurl 7.10 */ 872 if(config->socksproxy) { 873 my_setopt_str(curl, CURLOPT_PROXY, config->socksproxy); 874 my_setopt_enum(curl, CURLOPT_PROXYTYPE, (long)config->socksver); 875 } 876 877 /* new in libcurl 7.10.6 */ 878 if(config->proxyanyauth) 879 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, 880 (long)CURLAUTH_ANY); 881 else if(config->proxynegotiate) 882 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, 883 (long)CURLAUTH_GSSNEGOTIATE); 884 else if(config->proxyntlm) 885 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, 886 (long)CURLAUTH_NTLM); 887 else if(config->proxydigest) 888 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, 889 (long)CURLAUTH_DIGEST); 890 else if(config->proxybasic) 891 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH, 892 (long)CURLAUTH_BASIC); 893 894 /* new in libcurl 7.19.4 */ 895 my_setopt(curl, CURLOPT_NOPROXY, config->noproxy); 896 } 897#endif 898 899 my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror?1L:0L); 900 my_setopt(curl, CURLOPT_UPLOAD, uploadfile?1L:0L); 901 my_setopt(curl, CURLOPT_DIRLISTONLY, config->dirlistonly?1L:0L); 902 my_setopt(curl, CURLOPT_APPEND, config->ftp_append?1L:0L); 903 904 if(config->netrc_opt) 905 my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_OPTIONAL); 906 else if(config->netrc || config->netrc_file) 907 my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_REQUIRED); 908 else 909 my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_IGNORED); 910 911 if(config->netrc_file) 912 my_setopt(curl, CURLOPT_NETRC_FILE, config->netrc_file); 913 914 my_setopt(curl, CURLOPT_TRANSFERTEXT, config->use_ascii?1L:0L); 915 if(config->login_options) 916 my_setopt_str(curl, CURLOPT_LOGIN_OPTIONS, config->login_options); 917 my_setopt_str(curl, CURLOPT_USERPWD, config->userpwd); 918 my_setopt_str(curl, CURLOPT_RANGE, config->range); 919 my_setopt(curl, CURLOPT_ERRORBUFFER, errorbuffer); 920 my_setopt(curl, CURLOPT_TIMEOUT_MS, (long)(config->timeout * 1000)); 921 922 if(built_in_protos & CURLPROTO_HTTP) { 923 924 long postRedir = 0; 925 926 my_setopt(curl, CURLOPT_FOLLOWLOCATION, 927 config->followlocation?1L:0L); 928 my_setopt(curl, CURLOPT_UNRESTRICTED_AUTH, 929 config->unrestricted_auth?1L:0L); 930 931 switch(config->httpreq) { 932 case HTTPREQ_SIMPLEPOST: 933 my_setopt_str(curl, CURLOPT_POSTFIELDS, 934 config->postfields); 935 my_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, 936 config->postfieldsize); 937 break; 938 case HTTPREQ_POST: 939 my_setopt_httppost(curl, CURLOPT_HTTPPOST, config->httppost); 940 break; 941 default: 942 break; 943 } 944 945 my_setopt_str(curl, CURLOPT_REFERER, config->referer); 946 my_setopt(curl, CURLOPT_AUTOREFERER, config->autoreferer?1L:0L); 947 my_setopt_str(curl, CURLOPT_USERAGENT, config->useragent); 948 my_setopt_slist(curl, CURLOPT_HTTPHEADER, config->headers); 949 950 /* new in libcurl 7.36.0 */ 951 if(config->proxyheaders) { 952 my_setopt_slist(curl, CURLOPT_PROXYHEADER, config->proxyheaders); 953 my_setopt(curl, CURLOPT_HEADEROPT, CURLHEADER_SEPARATE); 954 } 955 956 /* new in libcurl 7.5 */ 957 my_setopt(curl, CURLOPT_MAXREDIRS, config->maxredirs); 958 959 /* new in libcurl 7.9.1 */ 960 if(config->httpversion) 961 my_setopt_enum(curl, CURLOPT_HTTP_VERSION, config->httpversion); 962 963 /* new in libcurl 7.10.6 (default is Basic) */ 964 if(config->authtype) 965 my_setopt_bitmask(curl, CURLOPT_HTTPAUTH, (long)config->authtype); 966 967 /* curl 7.19.1 (the 301 version existed in 7.18.2), 968 303 was added in 7.26.0 */ 969 if(config->post301) 970 postRedir |= CURL_REDIR_POST_301; 971 if(config->post302) 972 postRedir |= CURL_REDIR_POST_302; 973 if(config->post303) 974 postRedir |= CURL_REDIR_POST_303; 975 my_setopt(curl, CURLOPT_POSTREDIR, postRedir); 976 977 /* new in libcurl 7.21.6 */ 978 if(config->encoding) 979 my_setopt_str(curl, CURLOPT_ACCEPT_ENCODING, ""); 980 981 /* new in libcurl 7.21.6 */ 982 if(config->tr_encoding) 983 my_setopt(curl, CURLOPT_TRANSFER_ENCODING, 1L); 984 985 } /* (built_in_protos & CURLPROTO_HTTP) */ 986 987 my_setopt_str(curl, CURLOPT_FTPPORT, config->ftpport); 988 my_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 989 config->low_speed_limit); 990 my_setopt(curl, CURLOPT_LOW_SPEED_TIME, config->low_speed_time); 991 my_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE, 992 config->sendpersecond); 993 my_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, 994 config->recvpersecond); 995 996 if(config->use_resume) 997 my_setopt(curl, CURLOPT_RESUME_FROM_LARGE, config->resume_from); 998 else 999 my_setopt(curl, CURLOPT_RESUME_FROM_LARGE, CURL_OFF_T_C(0)); 1000 1001 my_setopt_str(curl, CURLOPT_SSLCERT, config->cert); 1002 my_setopt_str(curl, CURLOPT_SSLCERTTYPE, config->cert_type); 1003 my_setopt_str(curl, CURLOPT_SSLKEY, config->key); 1004 my_setopt_str(curl, CURLOPT_SSLKEYTYPE, config->key_type); 1005 my_setopt_str(curl, CURLOPT_KEYPASSWD, config->key_passwd); 1006 1007 if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) { 1008 1009 /* SSH and SSL private key uses same command-line option */ 1010 /* new in libcurl 7.16.1 */ 1011 my_setopt_str(curl, CURLOPT_SSH_PRIVATE_KEYFILE, config->key); 1012 /* new in libcurl 7.16.1 */ 1013 my_setopt_str(curl, CURLOPT_SSH_PUBLIC_KEYFILE, config->pubkey); 1014 1015 /* new in libcurl 7.17.1: SSH host key md5 checking allows us 1016 to fail if we are not talking to who we think we should */ 1017 my_setopt_str(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_MD5, 1018 config->hostpubmd5); 1019 } 1020 1021 if(config->cacert) 1022 my_setopt_str(curl, CURLOPT_CAINFO, config->cacert); 1023 if(config->capath) 1024 my_setopt_str(curl, CURLOPT_CAPATH, config->capath); 1025 if(config->crlfile) 1026 my_setopt_str(curl, CURLOPT_CRLFILE, config->crlfile); 1027 1028 if(curlinfo->features & CURL_VERSION_SSL) { 1029 if(config->insecure_ok) { 1030 my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); 1031 my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); 1032 } 1033 else { 1034 my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); 1035 /* libcurl default is strict verifyhost -> 2L */ 1036 /* my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); */ 1037 } 1038 } 1039 1040 if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) { 1041 if(!config->insecure_ok) { 1042 char *home; 1043 char *file; 1044 res = CURLE_OUT_OF_MEMORY; 1045 home = homedir(); 1046 if(home) { 1047 file = aprintf("%s/%sssh/known_hosts", home, DOT_CHAR); 1048 if(file) { 1049 /* new in curl 7.19.6 */ 1050 res = res_setopt_str(curl, CURLOPT_SSH_KNOWNHOSTS, file); 1051 curl_free(file); 1052 if(res == CURLE_UNKNOWN_OPTION) 1053 /* libssh2 version older than 1.1.1 */ 1054 res = CURLE_OK; 1055 } 1056 Curl_safefree(home); 1057 } 1058 if(res) 1059 goto show_error; 1060 } 1061 } 1062 1063 if(config->no_body || config->remote_time) { 1064 /* no body or use remote time */ 1065 my_setopt(curl, CURLOPT_FILETIME, 1L); 1066 } 1067 1068 my_setopt(curl, CURLOPT_CRLF, config->crlf?1L:0L); 1069 my_setopt_slist(curl, CURLOPT_QUOTE, config->quote); 1070 my_setopt_slist(curl, CURLOPT_POSTQUOTE, config->postquote); 1071 my_setopt_slist(curl, CURLOPT_PREQUOTE, config->prequote); 1072 1073#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES) 1074 if(config->cookie) 1075 my_setopt_str(curl, CURLOPT_COOKIE, config->cookie); 1076 1077 if(config->cookiefile) 1078 my_setopt_str(curl, CURLOPT_COOKIEFILE, config->cookiefile); 1079 1080 /* new in libcurl 7.9 */ 1081 if(config->cookiejar) 1082 my_setopt_str(curl, CURLOPT_COOKIEJAR, config->cookiejar); 1083 1084 /* new in libcurl 7.9.7 */ 1085 my_setopt(curl, CURLOPT_COOKIESESSION, config->cookiesession?1L:0L); 1086#else 1087 if(config->cookie || config->cookiefile || config->cookiejar) { 1088 warnf(config, "cookie option(s) used even though cookie support " 1089 "is disabled!\n"); 1090 return CURLE_NOT_BUILT_IN; 1091 } 1092#endif 1093 1094 my_setopt_enum(curl, CURLOPT_SSLVERSION, config->ssl_version); 1095 my_setopt_enum(curl, CURLOPT_TIMECONDITION, (long)config->timecond); 1096 my_setopt(curl, CURLOPT_TIMEVALUE, (long)config->condtime); 1097 my_setopt_str(curl, CURLOPT_CUSTOMREQUEST, config->customrequest); 1098 my_setopt(curl, CURLOPT_STDERR, global->errors); 1099 1100 /* three new ones in libcurl 7.3: */ 1101 my_setopt_str(curl, CURLOPT_INTERFACE, config->iface); 1102 my_setopt_str(curl, CURLOPT_KRBLEVEL, config->krblevel); 1103 1104 progressbarinit(&progressbar, config); 1105 if((global->progressmode == CURL_PROGRESS_BAR) && 1106 !global->noprogress && !global->mute) { 1107 /* we want the alternative style, then we have to implement it 1108 ourselves! */ 1109 my_setopt(curl, CURLOPT_XFERINFOFUNCTION, tool_progress_cb); 1110 my_setopt(curl, CURLOPT_XFERINFODATA, &progressbar); 1111 } 1112 1113 /* new in libcurl 7.24.0: */ 1114 if(config->dns_servers) 1115 my_setopt_str(curl, CURLOPT_DNS_SERVERS, config->dns_servers); 1116 1117 /* new in libcurl 7.33.0: */ 1118 if(config->dns_interface) 1119 my_setopt_str(curl, CURLOPT_DNS_INTERFACE, config->dns_interface); 1120 if(config->dns_ipv4_addr) 1121 my_setopt_str(curl, CURLOPT_DNS_LOCAL_IP4, config->dns_ipv4_addr); 1122 if(config->dns_ipv6_addr) 1123 my_setopt_str(curl, CURLOPT_DNS_LOCAL_IP6, config->dns_ipv6_addr); 1124 1125 /* new in libcurl 7.6.2: */ 1126 my_setopt_slist(curl, CURLOPT_TELNETOPTIONS, config->telnet_options); 1127 1128 /* new in libcurl 7.7: */ 1129 my_setopt_str(curl, CURLOPT_RANDOM_FILE, config->random_file); 1130 my_setopt_str(curl, CURLOPT_EGDSOCKET, config->egd_file); 1131 my_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 1132 (long)(config->connecttimeout * 1000)); 1133 1134 if(config->cipher_list) 1135 my_setopt_str(curl, CURLOPT_SSL_CIPHER_LIST, config->cipher_list); 1136 1137 /* new in libcurl 7.9.2: */ 1138 if(config->disable_epsv) 1139 /* disable it */ 1140 my_setopt(curl, CURLOPT_FTP_USE_EPSV, 0L); 1141 1142 /* new in libcurl 7.10.5 */ 1143 if(config->disable_eprt) 1144 /* disable it */ 1145 my_setopt(curl, CURLOPT_FTP_USE_EPRT, 0L); 1146 1147 if(global->tracetype != TRACE_NONE) { 1148 my_setopt(curl, CURLOPT_DEBUGFUNCTION, tool_debug_cb); 1149 my_setopt(curl, CURLOPT_DEBUGDATA, config); 1150 my_setopt(curl, CURLOPT_VERBOSE, 1L); 1151 } 1152 1153 /* new in curl 7.9.3 */ 1154 if(config->engine) { 1155 res = res_setopt_str(curl, CURLOPT_SSLENGINE, config->engine); 1156 if(res) 1157 goto show_error; 1158 my_setopt(curl, CURLOPT_SSLENGINE_DEFAULT, 1L); 1159 } 1160 1161 /* new in curl 7.10.7, extended in 7.19.4 but this only sets 0 or 1 */ 1162 my_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS, 1163 config->ftp_create_dirs?1L:0L); 1164 1165 /* new in curl 7.10.8 */ 1166 if(config->max_filesize) 1167 my_setopt(curl, CURLOPT_MAXFILESIZE_LARGE, 1168 config->max_filesize); 1169 1170 if(4 == config->ip_version) 1171 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); 1172 else if(6 == config->ip_version) 1173 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); 1174 else 1175 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_WHATEVER); 1176 1177 /* new in curl 7.15.5 */ 1178 if(config->ftp_ssl_reqd) 1179 my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_ALL); 1180 1181 /* new in curl 7.11.0 */ 1182 else if(config->ftp_ssl) 1183 my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_TRY); 1184 1185 /* new in curl 7.16.0 */ 1186 else if(config->ftp_ssl_control) 1187 my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_CONTROL); 1188 1189 /* new in curl 7.16.1 */ 1190 if(config->ftp_ssl_ccc) 1191 my_setopt_enum(curl, CURLOPT_FTP_SSL_CCC, 1192 (long)config->ftp_ssl_ccc_mode); 1193 1194#if defined(HAVE_GSSAPI) || defined(USE_WINDOWS_SSPI) 1195 { 1196 /* TODO: Make this a run-time check instead of compile-time one. */ 1197 1198 /* new in curl 7.19.4 */ 1199 if(config->socks5_gssapi_service) 1200 my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_SERVICE, 1201 config->socks5_gssapi_service); 1202 1203 /* new in curl 7.19.4 */ 1204 if(config->socks5_gssapi_nec) 1205 my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_NEC, 1206 config->socks5_gssapi_nec); 1207 } 1208#endif 1209 /* curl 7.13.0 */ 1210 my_setopt_str(curl, CURLOPT_FTP_ACCOUNT, config->ftp_account); 1211 1212 my_setopt(curl, CURLOPT_IGNORE_CONTENT_LENGTH, config->ignorecl?1L:0L); 1213 1214 /* curl 7.14.2 */ 1215 my_setopt(curl, CURLOPT_FTP_SKIP_PASV_IP, config->ftp_skip_ip?1L:0L); 1216 1217 /* curl 7.15.1 */ 1218 my_setopt(curl, CURLOPT_FTP_FILEMETHOD, (long)config->ftp_filemethod); 1219 1220 /* curl 7.15.2 */ 1221 if(config->localport) { 1222 my_setopt(curl, CURLOPT_LOCALPORT, (long)config->localport); 1223 my_setopt_str(curl, CURLOPT_LOCALPORTRANGE, 1224 (long)config->localportrange); 1225 } 1226 1227 /* curl 7.15.5 */ 1228 my_setopt_str(curl, CURLOPT_FTP_ALTERNATIVE_TO_USER, 1229 config->ftp_alternative_to_user); 1230 1231 /* curl 7.16.0 */ 1232 if(config->disable_sessionid) 1233 /* disable it */ 1234 my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE, 0L); 1235 1236 /* curl 7.16.2 */ 1237 if(config->raw) { 1238 my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 0L); 1239 my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, 0L); 1240 } 1241 1242 /* curl 7.17.1 */ 1243 if(!config->nokeepalive) { 1244 my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L); 1245 if(config->alivetime != 0) { 1246#if !defined(TCP_KEEPIDLE) || !defined(TCP_KEEPINTVL) 1247 warnf(config, "Keep-alive functionality somewhat crippled due to " 1248 "missing support in your operating system!\n"); 1249#endif 1250 my_setopt(curl, CURLOPT_TCP_KEEPIDLE, config->alivetime); 1251 my_setopt(curl, CURLOPT_TCP_KEEPINTVL, config->alivetime); 1252 } 1253 } 1254 else 1255 my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 0L); 1256 1257 /* curl 7.20.0 */ 1258 if(config->tftp_blksize) 1259 my_setopt(curl, CURLOPT_TFTP_BLKSIZE, config->tftp_blksize); 1260 1261 if(config->mail_from) 1262 my_setopt_str(curl, CURLOPT_MAIL_FROM, config->mail_from); 1263 1264 if(config->mail_rcpt) 1265 my_setopt_slist(curl, CURLOPT_MAIL_RCPT, config->mail_rcpt); 1266 1267 /* curl 7.20.x */ 1268 if(config->ftp_pret) 1269 my_setopt(curl, CURLOPT_FTP_USE_PRET, 1L); 1270 1271 if(config->proto_present) 1272 my_setopt_flags(curl, CURLOPT_PROTOCOLS, config->proto); 1273 if(config->proto_redir_present) 1274 my_setopt_flags(curl, CURLOPT_REDIR_PROTOCOLS, config->proto_redir); 1275 1276 if(config->content_disposition 1277 && (urlnode->flags & GETOUT_USEREMOTE) 1278 && (checkprefix("http://", this_url) || 1279 checkprefix("https://", this_url))) 1280 hdrcbdata.honor_cd_filename = TRUE; 1281 else 1282 hdrcbdata.honor_cd_filename = FALSE; 1283 1284 hdrcbdata.outs = &outs; 1285 hdrcbdata.heads = &heads; 1286 1287 my_setopt(curl, CURLOPT_HEADERFUNCTION, tool_header_cb); 1288 my_setopt(curl, CURLOPT_HEADERDATA, &hdrcbdata); 1289 1290 if(config->resolve) 1291 /* new in 7.21.3 */ 1292 my_setopt_slist(curl, CURLOPT_RESOLVE, config->resolve); 1293 1294 /* new in 7.21.4 */ 1295 if(curlinfo->features & CURL_VERSION_TLSAUTH_SRP) { 1296 if(config->tls_username) 1297 my_setopt_str(curl, CURLOPT_TLSAUTH_USERNAME, 1298 config->tls_username); 1299 if(config->tls_password) 1300 my_setopt_str(curl, CURLOPT_TLSAUTH_PASSWORD, 1301 config->tls_password); 1302 if(config->tls_authtype) 1303 my_setopt_str(curl, CURLOPT_TLSAUTH_TYPE, 1304 config->tls_authtype); 1305 } 1306 1307 /* new in 7.22.0 */ 1308 if(config->gssapi_delegation) 1309 my_setopt_str(curl, CURLOPT_GSSAPI_DELEGATION, 1310 config->gssapi_delegation); 1311 1312 /* new in 7.25.0 */ 1313 if(config->ssl_allow_beast) 1314 my_setopt(curl, CURLOPT_SSL_OPTIONS, (long)CURLSSLOPT_ALLOW_BEAST); 1315 1316 if(config->mail_auth) 1317 my_setopt_str(curl, CURLOPT_MAIL_AUTH, config->mail_auth); 1318 1319 /* new in 7.31.0 */ 1320 if(config->sasl_ir) 1321 my_setopt(curl, CURLOPT_SASL_IR, 1L); 1322 1323 if(config->nonpn) { 1324 my_setopt(curl, CURLOPT_SSL_ENABLE_NPN, 0L); 1325 } 1326 1327 if(config->noalpn) { 1328 my_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, 0L); 1329 } 1330 1331 /* initialize retry vars for loop below */ 1332 retry_sleep_default = (config->retry_delay) ? 1333 config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */ 1334 1335 retry_numretries = config->req_retry; 1336 retry_sleep = retry_sleep_default; /* ms */ 1337 retrystart = tvnow(); 1338 1339#ifndef CURL_DISABLE_LIBCURL_OPTION 1340 res = easysrc_perform(); 1341 if(res) { 1342 goto show_error; 1343 } 1344#endif 1345 1346 for(;;) { 1347#ifdef USE_METALINK 1348 if(!metalink && config->use_metalink) { 1349 /* If outs.metalink_parser is non-NULL, delete it first. */ 1350 if(outs.metalink_parser) 1351 metalink_parser_context_delete(outs.metalink_parser); 1352 outs.metalink_parser = metalink_parser_context_new(); 1353 if(outs.metalink_parser == NULL) { 1354 res = CURLE_OUT_OF_MEMORY; 1355 goto show_error; 1356 } 1357 fprintf(config->global->errors, 1358 "Metalink: parsing (%s) metalink/XML...\n", this_url); 1359 } 1360 else if(metalink) 1361 fprintf(config->global->errors, 1362 "Metalink: fetching (%s) from (%s)...\n", 1363 mlfile->filename, this_url); 1364#endif /* USE_METALINK */ 1365 1366#ifdef CURLDEBUG 1367 if(config->test_event_based) 1368 res = curl_easy_perform_ev(curl); 1369 else 1370#endif 1371 res = curl_easy_perform(curl); 1372 1373 if(outs.is_cd_filename && outs.stream && !global->mute && 1374 outs.filename) 1375 printf("curl: Saved to filename '%s'\n", outs.filename); 1376 1377 /* if retry-max-time is non-zero, make sure we haven't exceeded the 1378 time */ 1379 if(retry_numretries && 1380 (!config->retry_maxtime || 1381 (tvdiff(tvnow(), retrystart) < 1382 config->retry_maxtime*1000L)) ) { 1383 enum { 1384 RETRY_NO, 1385 RETRY_TIMEOUT, 1386 RETRY_HTTP, 1387 RETRY_FTP, 1388 RETRY_LAST /* not used */ 1389 } retry = RETRY_NO; 1390 long response; 1391 if((CURLE_OPERATION_TIMEDOUT == res) || 1392 (CURLE_COULDNT_RESOLVE_HOST == res) || 1393 (CURLE_COULDNT_RESOLVE_PROXY == res) || 1394 (CURLE_FTP_ACCEPT_TIMEOUT == res)) 1395 /* retry timeout always */ 1396 retry = RETRY_TIMEOUT; 1397 else if((CURLE_OK == res) || 1398 (config->failonerror && 1399 (CURLE_HTTP_RETURNED_ERROR == res))) { 1400 /* If it returned OK. _or_ failonerror was enabled and it 1401 returned due to such an error, check for HTTP transient 1402 errors to retry on. */ 1403 char *effective_url = NULL; 1404 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); 1405 if(effective_url && 1406 checkprefix("http", effective_url)) { 1407 /* This was HTTP(S) */ 1408 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); 1409 1410 switch(response) { 1411 case 500: /* Internal Server Error */ 1412 case 502: /* Bad Gateway */ 1413 case 503: /* Service Unavailable */ 1414 case 504: /* Gateway Timeout */ 1415 retry = RETRY_HTTP; 1416 /* 1417 * At this point, we have already written data to the output 1418 * file (or terminal). If we write to a file, we must rewind 1419 * or close/re-open the file so that the next attempt starts 1420 * over from the beginning. 1421 * 1422 * TODO: similar action for the upload case. We might need 1423 * to start over reading from a previous point if we have 1424 * uploaded something when this was returned. 1425 */ 1426 break; 1427 } 1428 } 1429 } /* if CURLE_OK */ 1430 else if(res) { 1431 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); 1432 1433 if(response/100 == 4) 1434 /* 1435 * This is typically when the FTP server only allows a certain 1436 * amount of users and we are not one of them. All 4xx codes 1437 * are transient. 1438 */ 1439 retry = RETRY_FTP; 1440 } 1441 1442 if(retry) { 1443 static const char * const m[]={ 1444 NULL, "timeout", "HTTP error", "FTP error" 1445 }; 1446 warnf(config, "Transient problem: %s " 1447 "Will retry in %ld seconds. " 1448 "%ld retries left.\n", 1449 m[retry], retry_sleep/1000L, retry_numretries); 1450 1451 tool_go_sleep(retry_sleep); 1452 retry_numretries--; 1453 if(!config->retry_delay) { 1454 retry_sleep *= 2; 1455 if(retry_sleep > RETRY_SLEEP_MAX) 1456 retry_sleep = RETRY_SLEEP_MAX; 1457 } 1458 if(outs.bytes && outs.filename) { 1459 /* We have written data to a output file, we truncate file 1460 */ 1461 if(!global->mute) 1462 fprintf(global->errors, "Throwing away %" 1463 CURL_FORMAT_CURL_OFF_T " bytes\n", 1464 outs.bytes); 1465 fflush(outs.stream); 1466 /* truncate file at the position where we started appending */ 1467#ifdef HAVE_FTRUNCATE 1468 if(ftruncate( fileno(outs.stream), outs.init)) { 1469 /* when truncate fails, we can't just append as then we'll 1470 create something strange, bail out */ 1471 if(!global->mute) 1472 fprintf(global->errors, 1473 "failed to truncate, exiting\n"); 1474 res = CURLE_WRITE_ERROR; 1475 goto quit_urls; 1476 } 1477 /* now seek to the end of the file, the position where we 1478 just truncated the file in a large file-safe way */ 1479 fseek(outs.stream, 0, SEEK_END); 1480#else 1481 /* ftruncate is not available, so just reposition the file 1482 to the location we would have truncated it. This won't 1483 work properly with large files on 32-bit systems, but 1484 most of those will have ftruncate. */ 1485 fseek(outs.stream, (long)outs.init, SEEK_SET); 1486#endif 1487 outs.bytes = 0; /* clear for next round */ 1488 } 1489 continue; /* curl_easy_perform loop */ 1490 } 1491 } /* if retry_numretries */ 1492 else if(metalink) { 1493 /* Metalink: Decide to try the next resource or 1494 not. Basically, we want to try the next resource if 1495 download was not successful. */ 1496 long response; 1497 if(CURLE_OK == res) { 1498 /* TODO We want to try next resource when download was 1499 not successful. How to know that? */ 1500 char *effective_url = NULL; 1501 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); 1502 if(effective_url && 1503 curlx_strnequal(effective_url, "http", 4)) { 1504 /* This was HTTP(S) */ 1505 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); 1506 if(response != 200 && response != 206) { 1507 metalink_next_res = 1; 1508 fprintf(global->errors, 1509 "Metalink: fetching (%s) from (%s) FAILED " 1510 "(HTTP status code %d)\n", 1511 mlfile->filename, this_url, response); 1512 } 1513 } 1514 } 1515 else { 1516 metalink_next_res = 1; 1517 fprintf(global->errors, 1518 "Metalink: fetching (%s) from (%s) FAILED (%s)\n", 1519 mlfile->filename, this_url, 1520 (errorbuffer[0]) ? 1521 errorbuffer : curl_easy_strerror((CURLcode)res)); 1522 } 1523 } 1524 if(metalink && !metalink_next_res) 1525 fprintf(global->errors, "Metalink: fetching (%s) from (%s) OK\n", 1526 mlfile->filename, this_url); 1527 1528 /* In all ordinary cases, just break out of loop here */ 1529 break; /* curl_easy_perform loop */ 1530 1531 } 1532 1533 if((global->progressmode == CURL_PROGRESS_BAR) && 1534 progressbar.calls) 1535 /* if the custom progress bar has been displayed, we output a 1536 newline here */ 1537 fputs("\n", progressbar.out); 1538 1539 if(config->writeout) 1540 ourWriteOut(curl, &outs, config->writeout); 1541 1542 if(config->writeenv) 1543 ourWriteEnv(curl); 1544 1545 /* 1546 ** Code within this loop may jump directly here to label 'show_error' 1547 ** in order to display an error message for CURLcode stored in 'res' 1548 ** variable and exit loop once that necessary writing and cleanup 1549 ** in label 'quit_urls' has been done. 1550 */ 1551 1552 show_error: 1553 1554#ifdef __VMS 1555 if(is_vms_shell()) { 1556 /* VMS DCL shell behavior */ 1557 if(!global->showerror) 1558 vms_show = VMSSTS_HIDE; 1559 } 1560 else 1561#endif 1562 if(res && global->showerror) { 1563 fprintf(global->errors, "curl: (%d) %s\n", res, (errorbuffer[0]) ? 1564 errorbuffer : curl_easy_strerror((CURLcode)res)); 1565 if(res == CURLE_SSL_CACERT) 1566 fprintf(global->errors, "%s%s", 1567 CURL_CA_CERT_ERRORMSG1, CURL_CA_CERT_ERRORMSG2); 1568 } 1569 1570 /* Fall through comment to 'quit_urls' label */ 1571 1572 /* 1573 ** Upon error condition and always that a message has already been 1574 ** displayed, code within this loop may jump directly here to label 1575 ** 'quit_urls' otherwise it should jump to 'show_error' label above. 1576 ** 1577 ** When 'res' variable is _not_ CURLE_OK loop will exit once that 1578 ** all code following 'quit_urls' has been executed. Otherwise it 1579 ** will loop to the beginning from where it may exit if there are 1580 ** no more urls left. 1581 */ 1582 1583 quit_urls: 1584 1585 /* Set file extended attributes */ 1586 if(!res && config->xattr && outs.fopened && outs.stream) { 1587 int rc = fwrite_xattr(curl, fileno(outs.stream)); 1588 if(rc) 1589 warnf(config, "Error setting extended attributes: %s\n", 1590 strerror(errno)); 1591 } 1592 1593 /* Close the file */ 1594 if(outs.fopened && outs.stream) { 1595 int rc = fclose(outs.stream); 1596 if(!res && rc) { 1597 /* something went wrong in the writing process */ 1598 res = CURLE_WRITE_ERROR; 1599 fprintf(global->errors, "(%d) Failed writing body\n", res); 1600 } 1601 } 1602 else if(!outs.s_isreg && outs.stream) { 1603 /* Dump standard stream buffered data */ 1604 int rc = fflush(outs.stream); 1605 if(!res && rc) { 1606 /* something went wrong in the writing process */ 1607 res = CURLE_WRITE_ERROR; 1608 fprintf(global->errors, "(%d) Failed writing body\n", res); 1609 } 1610 } 1611 1612#ifdef __AMIGA__ 1613 if(!res && outs.s_isreg && outs.filename) { 1614 /* Set the url (up to 80 chars) as comment for the file */ 1615 if(strlen(url) > 78) 1616 url[79] = '\0'; 1617 SetComment(outs.filename, url); 1618 } 1619#endif 1620 1621#ifdef HAVE_UTIME 1622 /* File time can only be set _after_ the file has been closed */ 1623 if(!res && config->remote_time && outs.s_isreg && outs.filename) { 1624 /* Ask libcurl if we got a remote file time */ 1625 long filetime = -1; 1626 curl_easy_getinfo(curl, CURLINFO_FILETIME, &filetime); 1627 if(filetime >= 0) { 1628 struct utimbuf times; 1629 times.actime = (time_t)filetime; 1630 times.modtime = (time_t)filetime; 1631 utime(outs.filename, ×); /* set the time we got */ 1632 } 1633 } 1634#endif 1635 1636#ifdef USE_METALINK 1637 if(!metalink && config->use_metalink && res == CURLE_OK) { 1638 int rv = parse_metalink(config, &outs, this_url); 1639 if(rv == 0) 1640 fprintf(config->global->errors, "Metalink: parsing (%s) OK\n", 1641 this_url); 1642 else if(rv == -1) 1643 fprintf(config->global->errors, "Metalink: parsing (%s) FAILED\n", 1644 this_url); 1645 } 1646 else if(metalink && res == CURLE_OK && !metalink_next_res) { 1647 int rv = metalink_check_hash(global, mlfile, outs.filename); 1648 if(rv == 0) { 1649 metalink_next_res = 1; 1650 } 1651 } 1652#endif /* USE_METALINK */ 1653 1654 /* No more business with this output struct */ 1655 if(outs.alloc_filename) 1656 Curl_safefree(outs.filename); 1657#ifdef USE_METALINK 1658 if(outs.metalink_parser) 1659 metalink_parser_context_delete(outs.metalink_parser); 1660#endif /* USE_METALINK */ 1661 memset(&outs, 0, sizeof(struct OutStruct)); 1662 hdrcbdata.outs = NULL; 1663 1664 /* Free loop-local allocated memory and close loop-local opened fd */ 1665 1666 Curl_safefree(outfile); 1667 Curl_safefree(this_url); 1668 1669 if(infdopen) 1670 close(infd); 1671 1672 if(metalink) { 1673 /* Should exit if error is fatal. */ 1674 if(is_fatal_error(res)) { 1675 break; 1676 } 1677 if(!metalink_next_res) 1678 break; 1679 mlres = mlres->next; 1680 if(mlres == NULL) 1681 /* TODO If metalink_next_res is 1 and mlres is NULL, 1682 * set res to error code 1683 */ 1684 break; 1685 } 1686 else 1687 if(urlnum > 1) { 1688 /* when url globbing, exit loop upon critical error */ 1689 if(is_fatal_error(res)) 1690 break; 1691 } 1692 else if(res) 1693 /* when not url globbing, exit loop upon any error */ 1694 break; 1695 1696 } /* loop to the next URL */ 1697 1698 /* Free loop-local allocated memory */ 1699 1700 Curl_safefree(uploadfile); 1701 1702 if(urls) { 1703 /* Free list of remaining URLs */ 1704 glob_cleanup(urls); 1705 urls = NULL; 1706 } 1707 1708 if(infilenum > 1) { 1709 /* when file globbing, exit loop upon critical error */ 1710 if(is_fatal_error(res)) 1711 break; 1712 } 1713 else if(res) 1714 /* when not file globbing, exit loop upon any error */ 1715 break; 1716 1717 } /* loop to the next globbed upload file */ 1718 1719 /* Free loop-local allocated memory */ 1720 1721 Curl_safefree(outfiles); 1722 1723 if(inglob) { 1724 /* Free list of globbed upload files */ 1725 glob_cleanup(inglob); 1726 inglob = NULL; 1727 } 1728 1729 /* Free this URL node data without destroying the 1730 the node itself nor modifying next pointer. */ 1731 Curl_safefree(urlnode->url); 1732 Curl_safefree(urlnode->outfile); 1733 Curl_safefree(urlnode->infile); 1734 urlnode->flags = 0; 1735 1736 /* 1737 ** Bail out upon critical errors 1738 */ 1739 if(is_fatal_error(res)) 1740 goto quit_curl; 1741 1742 } /* for-loop through all URLs */ 1743 1744 /* 1745 ** Nested loops end here. 1746 */ 1747 1748 quit_curl: 1749 1750 /* Reset the global config variables */ 1751 global->noprogress = orig_noprogress; 1752 global->isatty = orig_isatty; 1753 1754 /* Free function-local referenced allocated memory */ 1755 Curl_safefree(httpgetfields); 1756 1757 /* Free list of given URLs */ 1758 clean_getout(config); 1759 1760 hdrcbdata.heads = NULL; 1761 1762 /* Close function-local opened file descriptors */ 1763 if(heads.fopened && heads.stream) 1764 fclose(heads.stream); 1765 1766 if(heads.alloc_filename) 1767 Curl_safefree(heads.filename); 1768 1769 /* Release metalink related resources here */ 1770 clean_metalink(config); 1771 1772 return res; 1773} 1774 1775CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[]) 1776{ 1777 CURLcode result = CURLE_OK; 1778 1779 /* Setup proper locale from environment */ 1780#ifdef HAVE_SETLOCALE 1781 setlocale(LC_ALL, ""); 1782#endif 1783 1784 /* Parse .curlrc if necessary */ 1785 if((argc == 1) || (!curlx_strequal(argv[1], "-q"))) { 1786 parseconfig(NULL, config); /* ignore possible failure */ 1787 1788 /* If we had no arguments then make sure a url was specified in .curlrc */ 1789 if((argc < 2) && (!config->first->url_list)) { 1790 helpf(config->errors, NULL); 1791 result = CURLE_FAILED_INIT; 1792 } 1793 } 1794 1795 if(!result) { 1796 /* Parse the command line arguments */ 1797 ParameterError res = parse_args(config, argc, argv); 1798 if(res) { 1799 result = CURLE_OK; 1800 1801 /* Check if we were asked for the help */ 1802 if(res == PARAM_HELP_REQUESTED) 1803 tool_help(); 1804 /* Check if we were asked for the manual */ 1805 else if(res == PARAM_MANUAL_REQUESTED) 1806 hugehelp(); 1807 /* Check if we were asked for the version information */ 1808 else if(res == PARAM_VERSION_INFO_REQUESTED) 1809 tool_version_info(); 1810 /* Check if we were asked to list the SSL engines */ 1811 else if(res == PARAM_ENGINES_REQUESTED) 1812 tool_list_engines(config->easy); 1813 else 1814 result = CURLE_FAILED_INIT; 1815 } 1816 else { 1817#ifndef CURL_DISABLE_LIBCURL_OPTION 1818 /* Initialise the libcurl source output */ 1819 result = easysrc_init(); 1820#endif 1821 1822 /* Perform the main operations */ 1823 if(!result) { 1824 size_t count = 0; 1825 struct OperationConfig *operation = config->first; 1826 1827 /* Get the required aguments for each operation */ 1828 while(!result && operation) { 1829 result = get_args(operation, count++); 1830 1831 operation = operation->next; 1832 } 1833 1834 /* Set the current operation pointer */ 1835 config->current = config->first; 1836 1837 /* Perform each operation */ 1838 while(!result && config->current) { 1839 result = operate_do(config, config->current); 1840 1841 config->current = config->current->next; 1842 } 1843 1844#ifndef CURL_DISABLE_LIBCURL_OPTION 1845 /* Cleanup the libcurl source output */ 1846 easysrc_cleanup(); 1847 1848 /* Dump the libcurl code if previously enabled */ 1849 dumpeasysrc(config); 1850#endif 1851 } 1852 else 1853 helpf(config->errors, "out of memory\n"); 1854 } 1855 } 1856 1857 return result; 1858} 1859