1/*************************************************************************** 2 * _ _ ____ _ 3 * Project ___| | | | _ \| | 4 * / __| | | | |_) | | 5 * | (__| |_| | _ <| |___ 6 * \___|\___/|_| \_\_____| 7 * 8 * Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al. 9 * 10 * This software is licensed as described in the file COPYING, which 11 * you should have received as part of this distribution. The terms 12 * are also available at http://curl.haxx.se/docs/copyright.html. 13 * 14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell 15 * copies of the Software, and permit persons to whom the Software is 16 * furnished to do so, under the terms of the COPYING file. 17 * 18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 19 * KIND, either express or implied. 20 * 21 ***************************************************************************/ 22 23#include "curl_setup.h" 24 25#include "strtoofft.h" 26#include "strequal.h" 27#include "rawstr.h" 28 29#ifdef HAVE_NETINET_IN_H 30#include <netinet/in.h> 31#endif 32#ifdef HAVE_NETDB_H 33#include <netdb.h> 34#endif 35#ifdef HAVE_ARPA_INET_H 36#include <arpa/inet.h> 37#endif 38#ifdef HAVE_NET_IF_H 39#include <net/if.h> 40#endif 41#ifdef HAVE_SYS_IOCTL_H 42#include <sys/ioctl.h> 43#endif 44#ifdef HAVE_SIGNAL_H 45#include <signal.h> 46#endif 47 48#ifdef HAVE_SYS_PARAM_H 49#include <sys/param.h> 50#endif 51 52#ifdef HAVE_SYS_SELECT_H 53#include <sys/select.h> 54#endif 55 56#ifndef HAVE_SOCKET 57#error "We can't compile without socket() support!" 58#endif 59 60#include "urldata.h" 61#include <curl/curl.h> 62#include "netrc.h" 63 64#include "content_encoding.h" 65#include "hostip.h" 66#include "transfer.h" 67#include "sendf.h" 68#include "speedcheck.h" 69#include "progress.h" 70#include "http.h" 71#include "url.h" 72#include "getinfo.h" 73#include "vtls/vtls.h" 74#include "http_digest.h" 75#include "curl_ntlm.h" 76#include "http_negotiate.h" 77#include "share.h" 78#include "curl_memory.h" 79#include "select.h" 80#include "multiif.h" 81#include "connect.h" 82#include "non-ascii.h" 83 84#define _MPRINTF_REPLACE /* use our functions only */ 85#include <curl/mprintf.h> 86 87/* The last #include file should be: */ 88#include "memdebug.h" 89 90/* 91 * This function will call the read callback to fill our buffer with data 92 * to upload. 93 */ 94CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp) 95{ 96 struct SessionHandle *data = conn->data; 97 size_t buffersize = (size_t)bytes; 98 int nread; 99#ifdef CURL_DOES_CONVERSIONS 100 bool sending_http_headers = FALSE; 101 102 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) { 103 const struct HTTP *http = data->req.protop; 104 105 if(http->sending == HTTPSEND_REQUEST) 106 /* We're sending the HTTP request headers, not the data. 107 Remember that so we don't re-translate them into garbage. */ 108 sending_http_headers = TRUE; 109 } 110#endif 111 112 if(data->req.upload_chunky) { 113 /* if chunked Transfer-Encoding */ 114 buffersize -= (8 + 2 + 2); /* 32bit hex + CRLF + CRLF */ 115 data->req.upload_fromhere += (8 + 2); /* 32bit hex + CRLF */ 116 } 117 118 /* this function returns a size_t, so we typecast to int to prevent warnings 119 with picky compilers */ 120 nread = (int)conn->fread_func(data->req.upload_fromhere, 1, 121 buffersize, conn->fread_in); 122 123 if(nread == CURL_READFUNC_ABORT) { 124 failf(data, "operation aborted by callback"); 125 *nreadp = 0; 126 return CURLE_ABORTED_BY_CALLBACK; 127 } 128 else if(nread == CURL_READFUNC_PAUSE) { 129 130 if(conn->handler->flags & PROTOPT_NONETWORK) { 131 /* protocols that work without network cannot be paused. This is 132 actually only FILE:// just now, and it can't pause since the transfer 133 isn't done using the "normal" procedure. */ 134 failf(data, "Read callback asked for PAUSE when not supported!"); 135 return CURLE_READ_ERROR; 136 } 137 else { 138 struct SingleRequest *k = &data->req; 139 /* CURL_READFUNC_PAUSE pauses read callbacks that feed socket writes */ 140 k->keepon |= KEEP_SEND_PAUSE; /* mark socket send as paused */ 141 if(data->req.upload_chunky) { 142 /* Back out the preallocation done above */ 143 data->req.upload_fromhere -= (8 + 2); 144 } 145 *nreadp = 0; 146 } 147 return CURLE_OK; /* nothing was read */ 148 } 149 else if((size_t)nread > buffersize) { 150 /* the read function returned a too large value */ 151 *nreadp = 0; 152 failf(data, "read function returned funny value"); 153 return CURLE_READ_ERROR; 154 } 155 156 if(!data->req.forbidchunk && data->req.upload_chunky) { 157 /* if chunked Transfer-Encoding 158 * build chunk: 159 * 160 * <HEX SIZE> CRLF 161 * <DATA> CRLF 162 */ 163 /* On non-ASCII platforms the <DATA> may or may not be 164 translated based on set.prefer_ascii while the protocol 165 portion must always be translated to the network encoding. 166 To further complicate matters, line end conversion might be 167 done later on, so we need to prevent CRLFs from becoming 168 CRCRLFs if that's the case. To do this we use bare LFs 169 here, knowing they'll become CRLFs later on. 170 */ 171 172 char hexbuffer[11]; 173 const char *endofline_native; 174 const char *endofline_network; 175 int hexlen; 176 177 if( 178#ifdef CURL_DO_LINEEND_CONV 179 (data->set.prefer_ascii) || 180#endif 181 (data->set.crlf)) { 182 /* \n will become \r\n later on */ 183 endofline_native = "\n"; 184 endofline_network = "\x0a"; 185 } 186 else { 187 endofline_native = "\r\n"; 188 endofline_network = "\x0d\x0a"; 189 } 190 hexlen = snprintf(hexbuffer, sizeof(hexbuffer), 191 "%x%s", nread, endofline_native); 192 193 /* move buffer pointer */ 194 data->req.upload_fromhere -= hexlen; 195 nread += hexlen; 196 197 /* copy the prefix to the buffer, leaving out the NUL */ 198 memcpy(data->req.upload_fromhere, hexbuffer, hexlen); 199 200 /* always append ASCII CRLF to the data */ 201 memcpy(data->req.upload_fromhere + nread, 202 endofline_network, 203 strlen(endofline_network)); 204 205#ifdef CURL_DOES_CONVERSIONS 206 CURLcode res; 207 int length; 208 if(data->set.prefer_ascii) { 209 /* translate the protocol and data */ 210 length = nread; 211 } 212 else { 213 /* just translate the protocol portion */ 214 length = strlen(hexbuffer); 215 } 216 res = Curl_convert_to_network(data, data->req.upload_fromhere, length); 217 /* Curl_convert_to_network calls failf if unsuccessful */ 218 if(res) 219 return(res); 220#endif /* CURL_DOES_CONVERSIONS */ 221 222 if((nread - hexlen) == 0) 223 /* mark this as done once this chunk is transferred */ 224 data->req.upload_done = TRUE; 225 226 nread+=(int)strlen(endofline_native); /* for the added end of line */ 227 } 228#ifdef CURL_DOES_CONVERSIONS 229 else if((data->set.prefer_ascii) && (!sending_http_headers)) { 230 CURLcode res; 231 res = Curl_convert_to_network(data, data->req.upload_fromhere, nread); 232 /* Curl_convert_to_network calls failf if unsuccessful */ 233 if(res != CURLE_OK) 234 return(res); 235 } 236#endif /* CURL_DOES_CONVERSIONS */ 237 238 *nreadp = nread; 239 240 return CURLE_OK; 241} 242 243 244/* 245 * Curl_readrewind() rewinds the read stream. This is typically used for HTTP 246 * POST/PUT with multi-pass authentication when a sending was denied and a 247 * resend is necessary. 248 */ 249CURLcode Curl_readrewind(struct connectdata *conn) 250{ 251 struct SessionHandle *data = conn->data; 252 253 conn->bits.rewindaftersend = FALSE; /* we rewind now */ 254 255 /* explicitly switch off sending data on this connection now since we are 256 about to restart a new transfer and thus we want to avoid inadvertently 257 sending more data on the existing connection until the next transfer 258 starts */ 259 data->req.keepon &= ~KEEP_SEND; 260 261 /* We have sent away data. If not using CURLOPT_POSTFIELDS or 262 CURLOPT_HTTPPOST, call app to rewind 263 */ 264 if(data->set.postfields || 265 (data->set.httpreq == HTTPREQ_POST_FORM)) 266 ; /* do nothing */ 267 else { 268 if(data->set.seek_func) { 269 int err; 270 271 err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET); 272 if(err) { 273 failf(data, "seek callback returned error %d", (int)err); 274 return CURLE_SEND_FAIL_REWIND; 275 } 276 } 277 else if(data->set.ioctl_func) { 278 curlioerr err; 279 280 err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD, 281 data->set.ioctl_client); 282 infof(data, "the ioctl callback returned %d\n", (int)err); 283 284 if(err) { 285 /* FIXME: convert to a human readable error message */ 286 failf(data, "ioctl callback returned error %d", (int)err); 287 return CURLE_SEND_FAIL_REWIND; 288 } 289 } 290 else { 291 /* If no CURLOPT_READFUNCTION is used, we know that we operate on a 292 given FILE * stream and we can actually attempt to rewind that 293 ourselves with fseek() */ 294 if(data->set.fread_func == (curl_read_callback)fread) { 295 if(-1 != fseek(data->set.in, 0, SEEK_SET)) 296 /* successful rewind */ 297 return CURLE_OK; 298 } 299 300 /* no callback set or failure above, makes us fail at once */ 301 failf(data, "necessary data rewind wasn't possible"); 302 return CURLE_SEND_FAIL_REWIND; 303 } 304 } 305 return CURLE_OK; 306} 307 308static int data_pending(const struct connectdata *conn) 309{ 310 /* in the case of libssh2, we can never be really sure that we have emptied 311 its internal buffers so we MUST always try until we get EAGAIN back */ 312 return conn->handler->protocol&(CURLPROTO_SCP|CURLPROTO_SFTP) || 313#if defined(USE_NGHTTP2) 314 Curl_ssl_data_pending(conn, FIRSTSOCKET) || 315 /* For HTTP/2, we may read up everything including responde body 316 with header fields in Curl_http_readwrite_headers. If no 317 content-length is provided, curl waits for the connection 318 close, which we emulate it using conn->proto.httpc.closed = 319 TRUE. The thing is if we read everything, then http2_recv won't 320 be called and we cannot signal the HTTP/2 stream has closed. As 321 a workaround, we return nonzero here to call http2_recv. */ 322 ((conn->handler->protocol&PROTO_FAMILY_HTTP) && conn->httpversion == 20 && 323 conn->proto.httpc.closed); 324#else 325 Curl_ssl_data_pending(conn, FIRSTSOCKET); 326#endif 327} 328 329static void read_rewind(struct connectdata *conn, 330 size_t thismuch) 331{ 332 DEBUGASSERT(conn->read_pos >= thismuch); 333 334 conn->read_pos -= thismuch; 335 conn->bits.stream_was_rewound = TRUE; 336 337#ifdef DEBUGBUILD 338 { 339 char buf[512 + 1]; 340 size_t show; 341 342 show = CURLMIN(conn->buf_len - conn->read_pos, sizeof(buf)-1); 343 if(conn->master_buffer) { 344 memcpy(buf, conn->master_buffer + conn->read_pos, show); 345 buf[show] = '\0'; 346 } 347 else { 348 buf[0] = '\0'; 349 } 350 351 DEBUGF(infof(conn->data, 352 "Buffer after stream rewind (read_pos = %zu): [%s]\n", 353 conn->read_pos, buf)); 354 } 355#endif 356} 357 358/* 359 * Check to see if CURLOPT_TIMECONDITION was met by comparing the time of the 360 * remote document with the time provided by CURLOPT_TIMEVAL 361 */ 362bool Curl_meets_timecondition(struct SessionHandle *data, time_t timeofdoc) 363{ 364 if((timeofdoc == 0) || (data->set.timevalue == 0)) 365 return TRUE; 366 367 switch(data->set.timecondition) { 368 case CURL_TIMECOND_IFMODSINCE: 369 default: 370 if(timeofdoc <= data->set.timevalue) { 371 infof(data, 372 "The requested document is not new enough\n"); 373 data->info.timecond = TRUE; 374 return FALSE; 375 } 376 break; 377 case CURL_TIMECOND_IFUNMODSINCE: 378 if(timeofdoc >= data->set.timevalue) { 379 infof(data, 380 "The requested document is not old enough\n"); 381 data->info.timecond = TRUE; 382 return FALSE; 383 } 384 break; 385 } 386 387 return TRUE; 388} 389 390/* 391 * Go ahead and do a read if we have a readable socket or if 392 * the stream was rewound (in which case we have data in a 393 * buffer) 394 */ 395static CURLcode readwrite_data(struct SessionHandle *data, 396 struct connectdata *conn, 397 struct SingleRequest *k, 398 int *didwhat, bool *done) 399{ 400 CURLcode result = CURLE_OK; 401 ssize_t nread; /* number of bytes read */ 402 size_t excess = 0; /* excess bytes read */ 403 bool is_empty_data = FALSE; 404 bool readmore = FALSE; /* used by RTP to signal for more data */ 405 406 *done = FALSE; 407 408 /* This is where we loop until we have read everything there is to 409 read or we get a CURLE_AGAIN */ 410 do { 411 size_t buffersize = data->set.buffer_size? 412 data->set.buffer_size : BUFSIZE; 413 size_t bytestoread = buffersize; 414 415 if(k->size != -1 && !k->header) { 416 /* make sure we don't read "too much" if we can help it since we 417 might be pipelining and then someone else might want to read what 418 follows! */ 419 curl_off_t totalleft = k->size - k->bytecount; 420 if(totalleft < (curl_off_t)bytestoread) 421 bytestoread = (size_t)totalleft; 422 } 423 424 if(bytestoread) { 425 /* receive data from the network! */ 426 result = Curl_read(conn, conn->sockfd, k->buf, bytestoread, &nread); 427 428 /* read would've blocked */ 429 if(CURLE_AGAIN == result) 430 break; /* get out of loop */ 431 432 if(result>0) 433 return result; 434 } 435 else { 436 /* read nothing but since we wanted nothing we consider this an OK 437 situation to proceed from */ 438 nread = 0; 439 } 440 441 if((k->bytecount == 0) && (k->writebytecount == 0)) { 442 Curl_pgrsTime(data, TIMER_STARTTRANSFER); 443 if(k->exp100 > EXP100_SEND_DATA) 444 /* set time stamp to compare with when waiting for the 100 */ 445 k->start100 = Curl_tvnow(); 446 } 447 448 *didwhat |= KEEP_RECV; 449 /* indicates data of zero size, i.e. empty file */ 450 is_empty_data = ((nread == 0) && (k->bodywrites == 0)) ? TRUE : FALSE; 451 452 /* NUL terminate, allowing string ops to be used */ 453 if(0 < nread || is_empty_data) { 454 k->buf[nread] = 0; 455 } 456 else if(0 >= nread) { 457 /* if we receive 0 or less here, the server closed the connection 458 and we bail out from this! */ 459 DEBUGF(infof(data, "nread <= 0, server closed connection, bailing\n")); 460 k->keepon &= ~KEEP_RECV; 461 break; 462 } 463 464 /* Default buffer to use when we write the buffer, it may be changed 465 in the flow below before the actual storing is done. */ 466 k->str = k->buf; 467 468 if(conn->handler->readwrite) { 469 result = conn->handler->readwrite(data, conn, &nread, &readmore); 470 if(result) 471 return result; 472 if(readmore) 473 break; 474 } 475 476#ifndef CURL_DISABLE_HTTP 477 /* Since this is a two-state thing, we check if we are parsing 478 headers at the moment or not. */ 479 if(k->header) { 480 /* we are in parse-the-header-mode */ 481 bool stop_reading = FALSE; 482 result = Curl_http_readwrite_headers(data, conn, &nread, &stop_reading); 483 if(result) 484 return result; 485 486 if(conn->handler->readwrite && 487 (k->maxdownload <= 0 && nread > 0)) { 488 result = conn->handler->readwrite(data, conn, &nread, &readmore); 489 if(result) 490 return result; 491 if(readmore) 492 break; 493 } 494 495 if(stop_reading) { 496 /* We've stopped dealing with input, get out of the do-while loop */ 497 498 if(nread > 0) { 499 if(Curl_multi_pipeline_enabled(conn->data->multi)) { 500 infof(data, 501 "Rewinding stream by : %zd" 502 " bytes on url %s (zero-length body)\n", 503 nread, data->state.path); 504 read_rewind(conn, (size_t)nread); 505 } 506 else { 507 infof(data, 508 "Excess found in a non pipelined read:" 509 " excess = %zd" 510 " url = %s (zero-length body)\n", 511 nread, data->state.path); 512 } 513 } 514 515 break; 516 } 517 } 518#endif /* CURL_DISABLE_HTTP */ 519 520 521 /* This is not an 'else if' since it may be a rest from the header 522 parsing, where the beginning of the buffer is headers and the end 523 is non-headers. */ 524 if(k->str && !k->header && (nread > 0 || is_empty_data)) { 525 526#ifndef CURL_DISABLE_HTTP 527 if(0 == k->bodywrites && !is_empty_data) { 528 /* These checks are only made the first time we are about to 529 write a piece of the body */ 530 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) { 531 /* HTTP-only checks */ 532 533 if(data->req.newurl) { 534 if(conn->bits.close) { 535 /* Abort after the headers if "follow Location" is set 536 and we're set to close anyway. */ 537 k->keepon &= ~KEEP_RECV; 538 *done = TRUE; 539 return CURLE_OK; 540 } 541 /* We have a new url to load, but since we want to be able 542 to re-use this connection properly, we read the full 543 response in "ignore more" */ 544 k->ignorebody = TRUE; 545 infof(data, "Ignoring the response-body\n"); 546 } 547 if(data->state.resume_from && !k->content_range && 548 (data->set.httpreq==HTTPREQ_GET) && 549 !k->ignorebody) { 550 /* we wanted to resume a download, although the server doesn't 551 * seem to support this and we did this with a GET (if it 552 * wasn't a GET we did a POST or PUT resume) */ 553 failf(data, "HTTP server doesn't seem to support " 554 "byte ranges. Cannot resume."); 555 return CURLE_RANGE_ERROR; 556 } 557 558 if(data->set.timecondition && !data->state.range) { 559 /* A time condition has been set AND no ranges have been 560 requested. This seems to be what chapter 13.3.4 of 561 RFC 2616 defines to be the correct action for a 562 HTTP/1.1 client */ 563 564 if(!Curl_meets_timecondition(data, k->timeofdoc)) { 565 *done = TRUE; 566 /* We're simulating a http 304 from server so we return 567 what should have been returned from the server */ 568 data->info.httpcode = 304; 569 infof(data, "Simulate a HTTP 304 response!\n"); 570 /* we abort the transfer before it is completed == we ruin the 571 re-use ability. Close the connection */ 572 connclose(conn, "Simulated 304 handling"); 573 return CURLE_OK; 574 } 575 } /* we have a time condition */ 576 577 } /* this is HTTP or RTSP */ 578 } /* this is the first time we write a body part */ 579#endif /* CURL_DISABLE_HTTP */ 580 581 k->bodywrites++; 582 583 /* pass data to the debug function before it gets "dechunked" */ 584 if(data->set.verbose) { 585 if(k->badheader) { 586 Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff, 587 (size_t)k->hbuflen, conn); 588 if(k->badheader == HEADER_PARTHEADER) 589 Curl_debug(data, CURLINFO_DATA_IN, 590 k->str, (size_t)nread, conn); 591 } 592 else 593 Curl_debug(data, CURLINFO_DATA_IN, 594 k->str, (size_t)nread, conn); 595 } 596 597#ifndef CURL_DISABLE_HTTP 598 if(k->chunk) { 599 /* 600 * Here comes a chunked transfer flying and we need to decode this 601 * properly. While the name says read, this function both reads 602 * and writes away the data. The returned 'nread' holds the number 603 * of actual data it wrote to the client. 604 */ 605 606 CHUNKcode res = 607 Curl_httpchunk_read(conn, k->str, nread, &nread); 608 609 if(CHUNKE_OK < res) { 610 if(CHUNKE_WRITE_ERROR == res) { 611 failf(data, "Failed writing data"); 612 return CURLE_WRITE_ERROR; 613 } 614 failf(data, "%s in chunked-encoding", Curl_chunked_strerror(res)); 615 return CURLE_RECV_ERROR; 616 } 617 else if(CHUNKE_STOP == res) { 618 size_t dataleft; 619 /* we're done reading chunks! */ 620 k->keepon &= ~KEEP_RECV; /* read no more */ 621 622 /* There are now possibly N number of bytes at the end of the 623 str buffer that weren't written to the client. 624 625 We DO care about this data if we are pipelining. 626 Push it back to be read on the next pass. */ 627 628 dataleft = conn->chunk.dataleft; 629 if(dataleft != 0) { 630 infof(conn->data, "Leftovers after chunking: %zu bytes\n", 631 dataleft); 632 if(Curl_multi_pipeline_enabled(conn->data->multi)) { 633 /* only attempt the rewind if we truly are pipelining */ 634 infof(conn->data, "Rewinding %zu bytes\n",dataleft); 635 read_rewind(conn, dataleft); 636 } 637 } 638 } 639 /* If it returned OK, we just keep going */ 640 } 641#endif /* CURL_DISABLE_HTTP */ 642 643 /* Account for body content stored in the header buffer */ 644 if(k->badheader && !k->ignorebody) { 645 DEBUGF(infof(data, "Increasing bytecount by %zu from hbuflen\n", 646 k->hbuflen)); 647 k->bytecount += k->hbuflen; 648 } 649 650 if((-1 != k->maxdownload) && 651 (k->bytecount + nread >= k->maxdownload)) { 652 653 excess = (size_t)(k->bytecount + nread - k->maxdownload); 654 if(excess > 0 && !k->ignorebody) { 655 if(Curl_multi_pipeline_enabled(conn->data->multi)) { 656 /* The 'excess' amount below can't be more than BUFSIZE which 657 always will fit in a size_t */ 658 infof(data, 659 "Rewinding stream by : %zu" 660 " bytes on url %s (size = %" CURL_FORMAT_CURL_OFF_T 661 ", maxdownload = %" CURL_FORMAT_CURL_OFF_T 662 ", bytecount = %" CURL_FORMAT_CURL_OFF_T ", nread = %zd)\n", 663 excess, data->state.path, 664 k->size, k->maxdownload, k->bytecount, nread); 665 read_rewind(conn, excess); 666 } 667 else { 668 infof(data, 669 "Excess found in a non pipelined read:" 670 " excess = %zu" 671 ", size = %" CURL_FORMAT_CURL_OFF_T 672 ", maxdownload = %" CURL_FORMAT_CURL_OFF_T 673 ", bytecount = %" CURL_FORMAT_CURL_OFF_T "\n", 674 excess, k->size, k->maxdownload, k->bytecount); 675 } 676 } 677 678 nread = (ssize_t) (k->maxdownload - k->bytecount); 679 if(nread < 0 ) /* this should be unusual */ 680 nread = 0; 681 682 k->keepon &= ~KEEP_RECV; /* we're done reading */ 683 } 684 685 k->bytecount += nread; 686 687 Curl_pgrsSetDownloadCounter(data, k->bytecount); 688 689 if(!k->chunk && (nread || k->badheader || is_empty_data)) { 690 /* If this is chunky transfer, it was already written */ 691 692 if(k->badheader && !k->ignorebody) { 693 /* we parsed a piece of data wrongly assuming it was a header 694 and now we output it as body instead */ 695 696 /* Don't let excess data pollute body writes */ 697 if(k->maxdownload == -1 || (curl_off_t)k->hbuflen <= k->maxdownload) 698 result = Curl_client_write(conn, CLIENTWRITE_BODY, 699 data->state.headerbuff, 700 k->hbuflen); 701 else 702 result = Curl_client_write(conn, CLIENTWRITE_BODY, 703 data->state.headerbuff, 704 (size_t)k->maxdownload); 705 706 if(result) 707 return result; 708 } 709 if(k->badheader < HEADER_ALLBAD) { 710 /* This switch handles various content encodings. If there's an 711 error here, be sure to check over the almost identical code 712 in http_chunks.c. 713 Make sure that ALL_CONTENT_ENCODINGS contains all the 714 encodings handled here. */ 715#ifdef HAVE_LIBZ 716 switch (conn->data->set.http_ce_skip ? 717 IDENTITY : k->auto_decoding) { 718 case IDENTITY: 719#endif 720 /* This is the default when the server sends no 721 Content-Encoding header. See Curl_readwrite_init; the 722 memset() call initializes k->auto_decoding to zero. */ 723 if(!k->ignorebody) { 724 725#ifndef CURL_DISABLE_POP3 726 if(conn->handler->protocol&PROTO_FAMILY_POP3) 727 result = Curl_pop3_write(conn, k->str, nread); 728 else 729#endif /* CURL_DISABLE_POP3 */ 730 731 result = Curl_client_write(conn, CLIENTWRITE_BODY, k->str, 732 nread); 733 } 734#ifdef HAVE_LIBZ 735 break; 736 737 case DEFLATE: 738 /* Assume CLIENTWRITE_BODY; headers are not encoded. */ 739 if(!k->ignorebody) 740 result = Curl_unencode_deflate_write(conn, k, nread); 741 break; 742 743 case GZIP: 744 /* Assume CLIENTWRITE_BODY; headers are not encoded. */ 745 if(!k->ignorebody) 746 result = Curl_unencode_gzip_write(conn, k, nread); 747 break; 748 749 case COMPRESS: 750 default: 751 failf (data, "Unrecognized content encoding type. " 752 "libcurl understands `identity', `deflate' and `gzip' " 753 "content encodings."); 754 result = CURLE_BAD_CONTENT_ENCODING; 755 break; 756 } 757#endif 758 } 759 k->badheader = HEADER_NORMAL; /* taken care of now */ 760 761 if(result) 762 return result; 763 } 764 765 } /* if(! header and data to read ) */ 766 767 if(conn->handler->readwrite && 768 (excess > 0 && !conn->bits.stream_was_rewound)) { 769 /* Parse the excess data */ 770 k->str += nread; 771 nread = (ssize_t)excess; 772 773 result = conn->handler->readwrite(data, conn, &nread, &readmore); 774 if(result) 775 return result; 776 777 if(readmore) 778 k->keepon |= KEEP_RECV; /* we're not done reading */ 779 break; 780 } 781 782 if(is_empty_data) { 783 /* if we received nothing, the server closed the connection and we 784 are done */ 785 k->keepon &= ~KEEP_RECV; 786 } 787 788 } while(data_pending(conn)); 789 790 if(((k->keepon & (KEEP_RECV|KEEP_SEND)) == KEEP_SEND) && 791 conn->bits.close ) { 792 /* When we've read the entire thing and the close bit is set, the server 793 may now close the connection. If there's now any kind of sending going 794 on from our side, we need to stop that immediately. */ 795 infof(data, "we are done reading and this is set to close, stop send\n"); 796 k->keepon &= ~KEEP_SEND; /* no writing anymore either */ 797 } 798 799 return CURLE_OK; 800} 801 802/* 803 * Send data to upload to the server, when the socket is writable. 804 */ 805static CURLcode readwrite_upload(struct SessionHandle *data, 806 struct connectdata *conn, 807 struct SingleRequest *k, 808 int *didwhat) 809{ 810 ssize_t i, si; 811 ssize_t bytes_written; 812 CURLcode result; 813 ssize_t nread; /* number of bytes read */ 814 bool sending_http_headers = FALSE; 815 816 if((k->bytecount == 0) && (k->writebytecount == 0)) 817 Curl_pgrsTime(data, TIMER_STARTTRANSFER); 818 819 *didwhat |= KEEP_SEND; 820 821 /* 822 * We loop here to do the READ and SEND loop until we run out of 823 * data to send or until we get EWOULDBLOCK back 824 * 825 * FIXME: above comment is misleading. Currently no looping is 826 * actually done in do-while loop below. 827 */ 828 do { 829 830 /* only read more data if there's no upload data already 831 present in the upload buffer */ 832 if(0 == data->req.upload_present) { 833 /* init the "upload from here" pointer */ 834 data->req.upload_fromhere = k->uploadbuf; 835 836 if(!k->upload_done) { 837 /* HTTP pollution, this should be written nicer to become more 838 protocol agnostic. */ 839 int fillcount; 840 struct HTTP *http = data->req.protop; 841 842 if((k->exp100 == EXP100_SENDING_REQUEST) && 843 (http->sending == HTTPSEND_BODY)) { 844 /* If this call is to send body data, we must take some action: 845 We have sent off the full HTTP 1.1 request, and we shall now 846 go into the Expect: 100 state and await such a header */ 847 k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */ 848 k->keepon &= ~KEEP_SEND; /* disable writing */ 849 k->start100 = Curl_tvnow(); /* timeout count starts now */ 850 *didwhat &= ~KEEP_SEND; /* we didn't write anything actually */ 851 852 /* set a timeout for the multi interface */ 853 Curl_expire(data, data->set.expect_100_timeout); 854 break; 855 } 856 857 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) { 858 if(http->sending == HTTPSEND_REQUEST) 859 /* We're sending the HTTP request headers, not the data. 860 Remember that so we don't change the line endings. */ 861 sending_http_headers = TRUE; 862 else 863 sending_http_headers = FALSE; 864 } 865 866 result = Curl_fillreadbuffer(conn, BUFSIZE, &fillcount); 867 if(result) 868 return result; 869 870 nread = (ssize_t)fillcount; 871 } 872 else 873 nread = 0; /* we're done uploading/reading */ 874 875 if(!nread && (k->keepon & KEEP_SEND_PAUSE)) { 876 /* this is a paused transfer */ 877 break; 878 } 879 else if(nread<=0) { 880 /* done */ 881 k->keepon &= ~KEEP_SEND; /* we're done writing */ 882 883 if(conn->bits.rewindaftersend) { 884 result = Curl_readrewind(conn); 885 if(result) 886 return result; 887 } 888 break; 889 } 890 891 /* store number of bytes available for upload */ 892 data->req.upload_present = nread; 893 894#ifndef CURL_DISABLE_SMTP 895 if(conn->handler->protocol & PROTO_FAMILY_SMTP) { 896 result = Curl_smtp_escape_eob(conn, nread); 897 if(result) 898 return result; 899 } 900 else 901#endif /* CURL_DISABLE_SMTP */ 902 903 /* convert LF to CRLF if so asked */ 904 if((!sending_http_headers) && ( 905#ifdef CURL_DO_LINEEND_CONV 906 /* always convert if we're FTPing in ASCII mode */ 907 (data->set.prefer_ascii) || 908#endif 909 (data->set.crlf))) { 910 if(data->state.scratch == NULL) 911 data->state.scratch = malloc(2*BUFSIZE); 912 if(data->state.scratch == NULL) { 913 failf (data, "Failed to alloc scratch buffer!"); 914 return CURLE_OUT_OF_MEMORY; 915 } 916 /* 917 * ASCII/EBCDIC Note: This is presumably a text (not binary) 918 * transfer so the data should already be in ASCII. 919 * That means the hex values for ASCII CR (0x0d) & LF (0x0a) 920 * must be used instead of the escape sequences \r & \n. 921 */ 922 for(i = 0, si = 0; i < nread; i++, si++) { 923 if(data->req.upload_fromhere[i] == 0x0a) { 924 data->state.scratch[si++] = 0x0d; 925 data->state.scratch[si] = 0x0a; 926 if(!data->set.crlf) { 927 /* we're here only because FTP is in ASCII mode... 928 bump infilesize for the LF we just added */ 929 data->state.infilesize++; 930 } 931 } 932 else 933 data->state.scratch[si] = data->req.upload_fromhere[i]; 934 } 935 if(si != nread) { 936 /* only perform the special operation if we really did replace 937 anything */ 938 nread = si; 939 940 /* upload from the new (replaced) buffer instead */ 941 data->req.upload_fromhere = data->state.scratch; 942 943 /* set the new amount too */ 944 data->req.upload_present = nread; 945 } 946 } 947 } /* if 0 == data->req.upload_present */ 948 else { 949 /* We have a partial buffer left from a previous "round". Use 950 that instead of reading more data */ 951 } 952 953 /* write to socket (send away data) */ 954 result = Curl_write(conn, 955 conn->writesockfd, /* socket to send to */ 956 data->req.upload_fromhere, /* buffer pointer */ 957 data->req.upload_present, /* buffer size */ 958 &bytes_written); /* actually sent */ 959 960 if(result) 961 return result; 962 963 if(data->set.verbose) 964 /* show the data before we change the pointer upload_fromhere */ 965 Curl_debug(data, CURLINFO_DATA_OUT, data->req.upload_fromhere, 966 (size_t)bytes_written, conn); 967 968 k->writebytecount += bytes_written; 969 970 if(k->writebytecount == data->state.infilesize) { 971 /* we have sent all data we were supposed to */ 972 k->upload_done = TRUE; 973 infof(data, "We are completely uploaded and fine\n"); 974 } 975 976 if(data->req.upload_present != bytes_written) { 977 /* we only wrote a part of the buffer (if anything), deal with it! */ 978 979 /* store the amount of bytes left in the buffer to write */ 980 data->req.upload_present -= bytes_written; 981 982 /* advance the pointer where to find the buffer when the next send 983 is to happen */ 984 data->req.upload_fromhere += bytes_written; 985 } 986 else { 987 /* we've uploaded that buffer now */ 988 data->req.upload_fromhere = k->uploadbuf; 989 data->req.upload_present = 0; /* no more bytes left */ 990 991 if(k->upload_done) { 992 /* switch off writing, we're done! */ 993 k->keepon &= ~KEEP_SEND; /* we're done writing */ 994 } 995 } 996 997 Curl_pgrsSetUploadCounter(data, k->writebytecount); 998 999 } WHILE_FALSE; /* just to break out from! */ 1000 1001 return CURLE_OK; 1002} 1003 1004/* 1005 * Curl_readwrite() is the low-level function to be called when data is to 1006 * be read and written to/from the connection. 1007 */ 1008CURLcode Curl_readwrite(struct connectdata *conn, 1009 bool *done) 1010{ 1011 struct SessionHandle *data = conn->data; 1012 struct SingleRequest *k = &data->req; 1013 CURLcode result; 1014 int didwhat=0; 1015 1016 curl_socket_t fd_read; 1017 curl_socket_t fd_write; 1018 int select_res = conn->cselect_bits; 1019 1020 conn->cselect_bits = 0; 1021 1022 /* only use the proper socket if the *_HOLD bit is not set simultaneously as 1023 then we are in rate limiting state in that transfer direction */ 1024 1025 if((k->keepon & KEEP_RECVBITS) == KEEP_RECV) 1026 fd_read = conn->sockfd; 1027 else 1028 fd_read = CURL_SOCKET_BAD; 1029 1030 if((k->keepon & KEEP_SENDBITS) == KEEP_SEND) 1031 fd_write = conn->writesockfd; 1032 else 1033 fd_write = CURL_SOCKET_BAD; 1034 1035 if(!select_res) /* Call for select()/poll() only, if read/write/error 1036 status is not known. */ 1037 select_res = Curl_socket_ready(fd_read, fd_write, 0); 1038 1039 if(select_res == CURL_CSELECT_ERR) { 1040 failf(data, "select/poll returned error"); 1041 return CURLE_SEND_ERROR; 1042 } 1043 1044 /* We go ahead and do a read if we have a readable socket or if 1045 the stream was rewound (in which case we have data in a 1046 buffer) */ 1047 if((k->keepon & KEEP_RECV) && 1048 ((select_res & CURL_CSELECT_IN) || conn->bits.stream_was_rewound)) { 1049 1050 result = readwrite_data(data, conn, k, &didwhat, done); 1051 if(result || *done) 1052 return result; 1053 } 1054 1055 /* If we still have writing to do, we check if we have a writable socket. */ 1056 if((k->keepon & KEEP_SEND) && (select_res & CURL_CSELECT_OUT)) { 1057 /* write */ 1058 1059 result = readwrite_upload(data, conn, k, &didwhat); 1060 if(result) 1061 return result; 1062 } 1063 1064 k->now = Curl_tvnow(); 1065 if(didwhat) { 1066 /* Update read/write counters */ 1067 if(k->bytecountp) 1068 *k->bytecountp = k->bytecount; /* read count */ 1069 if(k->writebytecountp) 1070 *k->writebytecountp = k->writebytecount; /* write count */ 1071 } 1072 else { 1073 /* no read no write, this is a timeout? */ 1074 if(k->exp100 == EXP100_AWAITING_CONTINUE) { 1075 /* This should allow some time for the header to arrive, but only a 1076 very short time as otherwise it'll be too much wasted time too 1077 often. */ 1078 1079 /* Quoting RFC2616, section "8.2.3 Use of the 100 (Continue) Status": 1080 1081 Therefore, when a client sends this header field to an origin server 1082 (possibly via a proxy) from which it has never seen a 100 (Continue) 1083 status, the client SHOULD NOT wait for an indefinite period before 1084 sending the request body. 1085 1086 */ 1087 1088 long ms = Curl_tvdiff(k->now, k->start100); 1089 if(ms >= data->set.expect_100_timeout) { 1090 /* we've waited long enough, continue anyway */ 1091 k->exp100 = EXP100_SEND_DATA; 1092 k->keepon |= KEEP_SEND; 1093 infof(data, "Done waiting for 100-continue\n"); 1094 } 1095 } 1096 } 1097 1098 if(Curl_pgrsUpdate(conn)) 1099 result = CURLE_ABORTED_BY_CALLBACK; 1100 else 1101 result = Curl_speedcheck(data, k->now); 1102 if(result) 1103 return result; 1104 1105 if(k->keepon) { 1106 if(0 > Curl_timeleft(data, &k->now, FALSE)) { 1107 if(k->size != -1) { 1108 failf(data, "Operation timed out after %ld milliseconds with %" 1109 CURL_FORMAT_CURL_OFF_T " out of %" 1110 CURL_FORMAT_CURL_OFF_T " bytes received", 1111 Curl_tvdiff(k->now, data->progress.t_startsingle), k->bytecount, 1112 k->size); 1113 } 1114 else { 1115 failf(data, "Operation timed out after %ld milliseconds with %" 1116 CURL_FORMAT_CURL_OFF_T " bytes received", 1117 Curl_tvdiff(k->now, data->progress.t_startsingle), k->bytecount); 1118 } 1119 return CURLE_OPERATION_TIMEDOUT; 1120 } 1121 } 1122 else { 1123 /* 1124 * The transfer has been performed. Just make some general checks before 1125 * returning. 1126 */ 1127 1128 if(!(data->set.opt_no_body) && (k->size != -1) && 1129 (k->bytecount != k->size) && 1130#ifdef CURL_DO_LINEEND_CONV 1131 /* Most FTP servers don't adjust their file SIZE response for CRLFs, 1132 so we'll check to see if the discrepancy can be explained 1133 by the number of CRLFs we've changed to LFs. 1134 */ 1135 (k->bytecount != (k->size + data->state.crlf_conversions)) && 1136#endif /* CURL_DO_LINEEND_CONV */ 1137 !data->req.newurl) { 1138 failf(data, "transfer closed with %" CURL_FORMAT_CURL_OFF_T 1139 " bytes remaining to read", 1140 k->size - k->bytecount); 1141 return CURLE_PARTIAL_FILE; 1142 } 1143 else if(!(data->set.opt_no_body) && 1144 k->chunk && 1145 (conn->chunk.state != CHUNK_STOP)) { 1146 /* 1147 * In chunked mode, return an error if the connection is closed prior to 1148 * the empty (terminating) chunk is read. 1149 * 1150 * The condition above used to check for 1151 * conn->proto.http->chunk.datasize != 0 which is true after reading 1152 * *any* chunk, not just the empty chunk. 1153 * 1154 */ 1155 failf(data, "transfer closed with outstanding read data remaining"); 1156 return CURLE_PARTIAL_FILE; 1157 } 1158 if(Curl_pgrsUpdate(conn)) 1159 return CURLE_ABORTED_BY_CALLBACK; 1160 } 1161 1162 /* Now update the "done" boolean we return */ 1163 *done = (0 == (k->keepon&(KEEP_RECV|KEEP_SEND| 1164 KEEP_RECV_PAUSE|KEEP_SEND_PAUSE))) ? TRUE : FALSE; 1165 1166 return CURLE_OK; 1167} 1168 1169/* 1170 * Curl_single_getsock() gets called by the multi interface code when the app 1171 * has requested to get the sockets for the current connection. This function 1172 * will then be called once for every connection that the multi interface 1173 * keeps track of. This function will only be called for connections that are 1174 * in the proper state to have this information available. 1175 */ 1176int Curl_single_getsock(const struct connectdata *conn, 1177 curl_socket_t *sock, /* points to numsocks number 1178 of sockets */ 1179 int numsocks) 1180{ 1181 const struct SessionHandle *data = conn->data; 1182 int bitmap = GETSOCK_BLANK; 1183 unsigned sockindex = 0; 1184 1185 if(conn->handler->perform_getsock) 1186 return conn->handler->perform_getsock(conn, sock, numsocks); 1187 1188 if(numsocks < 2) 1189 /* simple check but we might need two slots */ 1190 return GETSOCK_BLANK; 1191 1192 /* don't include HOLD and PAUSE connections */ 1193 if((data->req.keepon & KEEP_RECVBITS) == KEEP_RECV) { 1194 1195 DEBUGASSERT(conn->sockfd != CURL_SOCKET_BAD); 1196 1197 bitmap |= GETSOCK_READSOCK(sockindex); 1198 sock[sockindex] = conn->sockfd; 1199 } 1200 1201 /* don't include HOLD and PAUSE connections */ 1202 if((data->req.keepon & KEEP_SENDBITS) == KEEP_SEND) { 1203 1204 if((conn->sockfd != conn->writesockfd) || 1205 !(data->req.keepon & KEEP_RECV)) { 1206 /* only if they are not the same socket or we didn't have a readable 1207 one, we increase index */ 1208 if(data->req.keepon & KEEP_RECV) 1209 sockindex++; /* increase index if we need two entries */ 1210 1211 DEBUGASSERT(conn->writesockfd != CURL_SOCKET_BAD); 1212 1213 sock[sockindex] = conn->writesockfd; 1214 } 1215 1216 bitmap |= GETSOCK_WRITESOCK(sockindex); 1217 } 1218 1219 return bitmap; 1220} 1221 1222/* 1223 * Determine optimum sleep time based on configured rate, current rate, 1224 * and packet size. 1225 * Returns value in milliseconds. 1226 * 1227 * The basic idea is to adjust the desired rate up/down in this method 1228 * based on whether we are running too slow or too fast. Then, calculate 1229 * how many milliseconds to wait for the next packet to achieve this new 1230 * rate. 1231 */ 1232long Curl_sleep_time(curl_off_t rate_bps, curl_off_t cur_rate_bps, 1233 int pkt_size) 1234{ 1235 curl_off_t min_sleep = 0; 1236 curl_off_t rv = 0; 1237 1238 if(rate_bps == 0) 1239 return 0; 1240 1241 /* If running faster than about .1% of the desired speed, slow 1242 * us down a bit. Use shift instead of division as the 0.1% 1243 * cutoff is arbitrary anyway. 1244 */ 1245 if(cur_rate_bps > (rate_bps + (rate_bps >> 10))) { 1246 /* running too fast, decrease target rate by 1/64th of rate */ 1247 rate_bps -= rate_bps >> 6; 1248 min_sleep = 1; 1249 } 1250 else if(cur_rate_bps < (rate_bps - (rate_bps >> 10))) { 1251 /* running too slow, increase target rate by 1/64th of rate */ 1252 rate_bps += rate_bps >> 6; 1253 } 1254 1255 /* Determine number of milliseconds to wait until we do 1256 * the next packet at the adjusted rate. We should wait 1257 * longer when using larger packets, for instance. 1258 */ 1259 rv = ((curl_off_t)((pkt_size * 8) * 1000) / rate_bps); 1260 1261 /* Catch rounding errors and always slow down at least 1ms if 1262 * we are running too fast. 1263 */ 1264 if(rv < min_sleep) 1265 rv = min_sleep; 1266 1267 /* Bound value to fit in 'long' on 32-bit platform. That's 1268 * plenty long enough anyway! 1269 */ 1270 if(rv > 0x7fffffff) 1271 rv = 0x7fffffff; 1272 1273 return (long)rv; 1274} 1275 1276/* 1277 * Curl_pretransfer() is called immediately before a transfer starts. 1278 */ 1279CURLcode Curl_pretransfer(struct SessionHandle *data) 1280{ 1281 CURLcode res; 1282 if(!data->change.url) { 1283 /* we can't do anything without URL */ 1284 failf(data, "No URL set!"); 1285 return CURLE_URL_MALFORMAT; 1286 } 1287 1288 /* Init the SSL session ID cache here. We do it here since we want to do it 1289 after the *_setopt() calls (that could specify the size of the cache) but 1290 before any transfer takes place. */ 1291 res = Curl_ssl_initsessions(data, data->set.ssl.max_ssl_sessions); 1292 if(res) 1293 return res; 1294 1295 data->set.followlocation=0; /* reset the location-follow counter */ 1296 data->state.this_is_a_follow = FALSE; /* reset this */ 1297 data->state.errorbuf = FALSE; /* no error has occurred */ 1298 data->state.httpversion = 0; /* don't assume any particular server version */ 1299 1300 data->state.ssl_connect_retry = FALSE; 1301 1302 data->state.authproblem = FALSE; 1303 data->state.authhost.want = data->set.httpauth; 1304 data->state.authproxy.want = data->set.proxyauth; 1305 Curl_safefree(data->info.wouldredirect); 1306 data->info.wouldredirect = NULL; 1307 1308 /* If there is a list of cookie files to read, do it now! */ 1309 if(data->change.cookielist) 1310 Curl_cookie_loadfiles(data); 1311 1312 /* If there is a list of host pairs to deal with */ 1313 if(data->change.resolve) 1314 res = Curl_loadhostpairs(data); 1315 1316 if(!res) { 1317 /* Allow data->set.use_port to set which port to use. This needs to be 1318 * disabled for example when we follow Location: headers to URLs using 1319 * different ports! */ 1320 data->state.allow_port = TRUE; 1321 1322#if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL) 1323 /************************************************************* 1324 * Tell signal handler to ignore SIGPIPE 1325 *************************************************************/ 1326 if(!data->set.no_signal) 1327 data->state.prev_signal = signal(SIGPIPE, SIG_IGN); 1328#endif 1329 1330 Curl_initinfo(data); /* reset session-specific information "variables" */ 1331 Curl_pgrsStartNow(data); 1332 1333 if(data->set.timeout) 1334 Curl_expire(data, data->set.timeout); 1335 1336 if(data->set.connecttimeout) 1337 Curl_expire(data, data->set.connecttimeout); 1338 1339 /* In case the handle is re-used and an authentication method was picked 1340 in the session we need to make sure we only use the one(s) we now 1341 consider to be fine */ 1342 data->state.authhost.picked &= data->state.authhost.want; 1343 data->state.authproxy.picked &= data->state.authproxy.want; 1344 } 1345 1346 return res; 1347} 1348 1349/* 1350 * Curl_posttransfer() is called immediately after a transfer ends 1351 */ 1352CURLcode Curl_posttransfer(struct SessionHandle *data) 1353{ 1354#if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL) 1355 /* restore the signal handler for SIGPIPE before we get back */ 1356 if(!data->set.no_signal) 1357 signal(SIGPIPE, data->state.prev_signal); 1358#else 1359 (void)data; /* unused parameter */ 1360#endif 1361 1362 return CURLE_OK; 1363} 1364 1365#ifndef CURL_DISABLE_HTTP 1366/* 1367 * strlen_url() returns the length of the given URL if the spaces within the 1368 * URL were properly URL encoded. 1369 */ 1370static size_t strlen_url(const char *url) 1371{ 1372 const char *ptr; 1373 size_t newlen=0; 1374 bool left=TRUE; /* left side of the ? */ 1375 1376 for(ptr=url; *ptr; ptr++) { 1377 switch(*ptr) { 1378 case '?': 1379 left=FALSE; 1380 /* fall through */ 1381 default: 1382 newlen++; 1383 break; 1384 case ' ': 1385 if(left) 1386 newlen+=3; 1387 else 1388 newlen++; 1389 break; 1390 } 1391 } 1392 return newlen; 1393} 1394 1395/* strcpy_url() copies a url to a output buffer and URL-encodes the spaces in 1396 * the source URL accordingly. 1397 */ 1398static void strcpy_url(char *output, const char *url) 1399{ 1400 /* we must add this with whitespace-replacing */ 1401 bool left=TRUE; 1402 const char *iptr; 1403 char *optr = output; 1404 for(iptr = url; /* read from here */ 1405 *iptr; /* until zero byte */ 1406 iptr++) { 1407 switch(*iptr) { 1408 case '?': 1409 left=FALSE; 1410 /* fall through */ 1411 default: 1412 *optr++=*iptr; 1413 break; 1414 case ' ': 1415 if(left) { 1416 *optr++='%'; /* add a '%' */ 1417 *optr++='2'; /* add a '2' */ 1418 *optr++='0'; /* add a '0' */ 1419 } 1420 else 1421 *optr++='+'; /* add a '+' here */ 1422 break; 1423 } 1424 } 1425 *optr=0; /* zero terminate output buffer */ 1426 1427} 1428 1429/* 1430 * Returns true if the given URL is absolute (as opposed to relative) 1431 */ 1432static bool is_absolute_url(const char *url) 1433{ 1434 char prot[16]; /* URL protocol string storage */ 1435 char letter; /* used for a silly sscanf */ 1436 1437 return (2 == sscanf(url, "%15[^?&/:]://%c", prot, &letter)) ? TRUE : FALSE; 1438} 1439 1440/* 1441 * Concatenate a relative URL to a base URL making it absolute. 1442 * URL-encodes any spaces. 1443 * The returned pointer must be freed by the caller unless NULL 1444 * (returns NULL on out of memory). 1445 */ 1446static char *concat_url(const char *base, const char *relurl) 1447{ 1448 /*** 1449 TRY to append this new path to the old URL 1450 to the right of the host part. Oh crap, this is doomed to cause 1451 problems in the future... 1452 */ 1453 char *newest; 1454 char *protsep; 1455 char *pathsep; 1456 size_t newlen; 1457 1458 const char *useurl = relurl; 1459 size_t urllen; 1460 1461 /* we must make our own copy of the URL to play with, as it may 1462 point to read-only data */ 1463 char *url_clone=strdup(base); 1464 1465 if(!url_clone) 1466 return NULL; /* skip out of this NOW */ 1467 1468 /* protsep points to the start of the host name */ 1469 protsep=strstr(url_clone, "//"); 1470 if(!protsep) 1471 protsep=url_clone; 1472 else 1473 protsep+=2; /* pass the slashes */ 1474 1475 if('/' != relurl[0]) { 1476 int level=0; 1477 1478 /* First we need to find out if there's a ?-letter in the URL, 1479 and cut it and the right-side of that off */ 1480 pathsep = strchr(protsep, '?'); 1481 if(pathsep) 1482 *pathsep=0; 1483 1484 /* we have a relative path to append to the last slash if there's one 1485 available, or if the new URL is just a query string (starts with a 1486 '?') we append the new one at the end of the entire currently worked 1487 out URL */ 1488 if(useurl[0] != '?') { 1489 pathsep = strrchr(protsep, '/'); 1490 if(pathsep) 1491 *pathsep=0; 1492 } 1493 1494 /* Check if there's any slash after the host name, and if so, remember 1495 that position instead */ 1496 pathsep = strchr(protsep, '/'); 1497 if(pathsep) 1498 protsep = pathsep+1; 1499 else 1500 protsep = NULL; 1501 1502 /* now deal with one "./" or any amount of "../" in the newurl 1503 and act accordingly */ 1504 1505 if((useurl[0] == '.') && (useurl[1] == '/')) 1506 useurl+=2; /* just skip the "./" */ 1507 1508 while((useurl[0] == '.') && 1509 (useurl[1] == '.') && 1510 (useurl[2] == '/')) { 1511 level++; 1512 useurl+=3; /* pass the "../" */ 1513 } 1514 1515 if(protsep) { 1516 while(level--) { 1517 /* cut off one more level from the right of the original URL */ 1518 pathsep = strrchr(protsep, '/'); 1519 if(pathsep) 1520 *pathsep=0; 1521 else { 1522 *protsep=0; 1523 break; 1524 } 1525 } 1526 } 1527 } 1528 else { 1529 /* We got a new absolute path for this server */ 1530 1531 if((relurl[0] == '/') && (relurl[1] == '/')) { 1532 /* the new URL starts with //, just keep the protocol part from the 1533 original one */ 1534 *protsep=0; 1535 useurl = &relurl[2]; /* we keep the slashes from the original, so we 1536 skip the new ones */ 1537 } 1538 else { 1539 /* cut off the original URL from the first slash, or deal with URLs 1540 without slash */ 1541 pathsep = strchr(protsep, '/'); 1542 if(pathsep) { 1543 /* When people use badly formatted URLs, such as 1544 "http://www.url.com?dir=/home/daniel" we must not use the first 1545 slash, if there's a ?-letter before it! */ 1546 char *sep = strchr(protsep, '?'); 1547 if(sep && (sep < pathsep)) 1548 pathsep = sep; 1549 *pathsep=0; 1550 } 1551 else { 1552 /* There was no slash. Now, since we might be operating on a badly 1553 formatted URL, such as "http://www.url.com?id=2380" which doesn't 1554 use a slash separator as it is supposed to, we need to check for a 1555 ?-letter as well! */ 1556 pathsep = strchr(protsep, '?'); 1557 if(pathsep) 1558 *pathsep=0; 1559 } 1560 } 1561 } 1562 1563 /* If the new part contains a space, this is a mighty stupid redirect 1564 but we still make an effort to do "right". To the left of a '?' 1565 letter we replace each space with %20 while it is replaced with '+' 1566 on the right side of the '?' letter. 1567 */ 1568 newlen = strlen_url(useurl); 1569 1570 urllen = strlen(url_clone); 1571 1572 newest = malloc(urllen + 1 + /* possible slash */ 1573 newlen + 1 /* zero byte */); 1574 1575 if(!newest) { 1576 free(url_clone); /* don't leak this */ 1577 return NULL; 1578 } 1579 1580 /* copy over the root url part */ 1581 memcpy(newest, url_clone, urllen); 1582 1583 /* check if we need to append a slash */ 1584 if(('/' == useurl[0]) || (protsep && !*protsep) || ('?' == useurl[0])) 1585 ; 1586 else 1587 newest[urllen++]='/'; 1588 1589 /* then append the new piece on the right side */ 1590 strcpy_url(&newest[urllen], useurl); 1591 1592 free(url_clone); 1593 1594 return newest; 1595} 1596#endif /* CURL_DISABLE_HTTP */ 1597 1598/* 1599 * Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string 1600 * as given by the remote server and set up the new URL to request. 1601 */ 1602CURLcode Curl_follow(struct SessionHandle *data, 1603 char *newurl, /* this 'newurl' is the Location: string, 1604 and it must be malloc()ed before passed 1605 here */ 1606 followtype type) /* see transfer.h */ 1607{ 1608#ifdef CURL_DISABLE_HTTP 1609 (void)data; 1610 (void)newurl; 1611 (void)type; 1612 /* Location: following will not happen when HTTP is disabled */ 1613 return CURLE_TOO_MANY_REDIRECTS; 1614#else 1615 1616 /* Location: redirect */ 1617 bool disallowport = FALSE; 1618 1619 if(type == FOLLOW_REDIR) { 1620 if((data->set.maxredirs != -1) && 1621 (data->set.followlocation >= data->set.maxredirs)) { 1622 failf(data,"Maximum (%ld) redirects followed", data->set.maxredirs); 1623 return CURLE_TOO_MANY_REDIRECTS; 1624 } 1625 1626 /* mark the next request as a followed location: */ 1627 data->state.this_is_a_follow = TRUE; 1628 1629 data->set.followlocation++; /* count location-followers */ 1630 1631 if(data->set.http_auto_referer) { 1632 /* We are asked to automatically set the previous URL as the referer 1633 when we get the next URL. We pick the ->url field, which may or may 1634 not be 100% correct */ 1635 1636 if(data->change.referer_alloc) { 1637 Curl_safefree(data->change.referer); 1638 data->change.referer_alloc = FALSE; 1639 } 1640 1641 data->change.referer = strdup(data->change.url); 1642 if(!data->change.referer) 1643 return CURLE_OUT_OF_MEMORY; 1644 data->change.referer_alloc = TRUE; /* yes, free this later */ 1645 } 1646 } 1647 1648 if(!is_absolute_url(newurl)) { 1649 /*** 1650 *DANG* this is an RFC 2068 violation. The URL is supposed 1651 to be absolute and this doesn't seem to be that! 1652 */ 1653 char *absolute = concat_url(data->change.url, newurl); 1654 if(!absolute) 1655 return CURLE_OUT_OF_MEMORY; 1656 free(newurl); 1657 newurl = absolute; 1658 } 1659 else { 1660 /* This is an absolute URL, don't allow the custom port number */ 1661 disallowport = TRUE; 1662 1663 if(strchr(newurl, ' ')) { 1664 /* This new URL contains at least one space, this is a mighty stupid 1665 redirect but we still make an effort to do "right". */ 1666 char *newest; 1667 size_t newlen = strlen_url(newurl); 1668 1669 newest = malloc(newlen+1); /* get memory for this */ 1670 if(!newest) 1671 return CURLE_OUT_OF_MEMORY; 1672 strcpy_url(newest, newurl); /* create a space-free URL */ 1673 1674 free(newurl); /* that was no good */ 1675 newurl = newest; /* use this instead now */ 1676 } 1677 1678 } 1679 1680 if(type == FOLLOW_FAKE) { 1681 /* we're only figuring out the new url if we would've followed locations 1682 but now we're done so we can get out! */ 1683 data->info.wouldredirect = newurl; 1684 return CURLE_OK; 1685 } 1686 1687 if(disallowport) 1688 data->state.allow_port = FALSE; 1689 1690 if(data->change.url_alloc) { 1691 Curl_safefree(data->change.url); 1692 data->change.url_alloc = FALSE; 1693 } 1694 1695 data->change.url = newurl; 1696 data->change.url_alloc = TRUE; 1697 newurl = NULL; /* don't free! */ 1698 1699 infof(data, "Issue another request to this URL: '%s'\n", data->change.url); 1700 1701 /* 1702 * We get here when the HTTP code is 300-399 (and 401). We need to perform 1703 * differently based on exactly what return code there was. 1704 * 1705 * News from 7.10.6: we can also get here on a 401 or 407, in case we act on 1706 * a HTTP (proxy-) authentication scheme other than Basic. 1707 */ 1708 switch(data->info.httpcode) { 1709 /* 401 - Act on a WWW-Authenticate, we keep on moving and do the 1710 Authorization: XXXX header in the HTTP request code snippet */ 1711 /* 407 - Act on a Proxy-Authenticate, we keep on moving and do the 1712 Proxy-Authorization: XXXX header in the HTTP request code snippet */ 1713 /* 300 - Multiple Choices */ 1714 /* 306 - Not used */ 1715 /* 307 - Temporary Redirect */ 1716 default: /* for all above (and the unknown ones) */ 1717 /* Some codes are explicitly mentioned since I've checked RFC2616 and they 1718 * seem to be OK to POST to. 1719 */ 1720 break; 1721 case 301: /* Moved Permanently */ 1722 /* (quote from RFC7231, section 6.4.2) 1723 * 1724 * Note: For historical reasons, a user agent MAY change the request 1725 * method from POST to GET for the subsequent request. If this 1726 * behavior is undesired, the 307 (Temporary Redirect) status code 1727 * can be used instead. 1728 * 1729 * ---- 1730 * 1731 * Many webservers expect this, so these servers often answers to a POST 1732 * request with an error page. To be sure that libcurl gets the page that 1733 * most user agents would get, libcurl has to force GET. 1734 * 1735 * This behaviour is forbidden by RFC1945 and the obsolete RFC2616, and 1736 * can be overridden with CURLOPT_POSTREDIR. 1737 */ 1738 if((data->set.httpreq == HTTPREQ_POST 1739 || data->set.httpreq == HTTPREQ_POST_FORM) 1740 && !(data->set.keep_post & CURL_REDIR_POST_301)) { 1741 infof(data, "Switch from POST to GET\n"); 1742 data->set.httpreq = HTTPREQ_GET; 1743 } 1744 break; 1745 case 302: /* Found */ 1746 /* (quote from RFC7231, section 6.4.3) 1747 * 1748 * Note: For historical reasons, a user agent MAY change the request 1749 * method from POST to GET for the subsequent request. If this 1750 * behavior is undesired, the 307 (Temporary Redirect) status code 1751 * can be used instead. 1752 * 1753 * ---- 1754 * 1755 * Many webservers expect this, so these servers often answers to a POST 1756 * request with an error page. To be sure that libcurl gets the page that 1757 * most user agents would get, libcurl has to force GET. 1758 * 1759 * This behaviour is forbidden by RFC1945 and the obsolete RFC2616, and 1760 * can be overridden with CURLOPT_POSTREDIR. 1761 */ 1762 if((data->set.httpreq == HTTPREQ_POST 1763 || data->set.httpreq == HTTPREQ_POST_FORM) 1764 && !(data->set.keep_post & CURL_REDIR_POST_302)) { 1765 infof(data, "Switch from POST to GET\n"); 1766 data->set.httpreq = HTTPREQ_GET; 1767 } 1768 break; 1769 1770 case 303: /* See Other */ 1771 /* Disable both types of POSTs, unless the user explicitely 1772 asks for POST after POST */ 1773 if(data->set.httpreq != HTTPREQ_GET 1774 && !(data->set.keep_post & CURL_REDIR_POST_303)) { 1775 data->set.httpreq = HTTPREQ_GET; /* enforce GET request */ 1776 infof(data, "Disables POST, goes with %s\n", 1777 data->set.opt_no_body?"HEAD":"GET"); 1778 } 1779 break; 1780 case 304: /* Not Modified */ 1781 /* 304 means we did a conditional request and it was "Not modified". 1782 * We shouldn't get any Location: header in this response! 1783 */ 1784 break; 1785 case 305: /* Use Proxy */ 1786 /* (quote from RFC2616, section 10.3.6): 1787 * "The requested resource MUST be accessed through the proxy given 1788 * by the Location field. The Location field gives the URI of the 1789 * proxy. The recipient is expected to repeat this single request 1790 * via the proxy. 305 responses MUST only be generated by origin 1791 * servers." 1792 */ 1793 break; 1794 } 1795 Curl_pgrsTime(data, TIMER_REDIRECT); 1796 Curl_pgrsResetTimesSizes(data); 1797 1798 return CURLE_OK; 1799#endif /* CURL_DISABLE_HTTP */ 1800} 1801 1802CURLcode 1803Curl_reconnect_request(struct connectdata **connp) 1804{ 1805 CURLcode result = CURLE_OK; 1806 struct connectdata *conn = *connp; 1807 struct SessionHandle *data = conn->data; 1808 1809 /* This was a re-use of a connection and we got a write error in the 1810 * DO-phase. Then we DISCONNECT this connection and have another attempt to 1811 * CONNECT and then DO again! The retry cannot possibly find another 1812 * connection to re-use, since we only keep one possible connection for 1813 * each. */ 1814 1815 infof(data, "Re-used connection seems dead, get a new one\n"); 1816 1817 connclose(conn, "Reconnect dead connection"); /* enforce close */ 1818 result = Curl_done(&conn, result, FALSE); /* we are so done with this */ 1819 1820 /* conn may no longer be a good pointer, clear it to avoid mistakes by 1821 parent functions */ 1822 *connp = NULL; 1823 1824 /* 1825 * According to bug report #1330310. We need to check for CURLE_SEND_ERROR 1826 * here as well. I figure this could happen when the request failed on a FTP 1827 * connection and thus Curl_done() itself tried to use the connection 1828 * (again). Slight Lack of feedback in the report, but I don't think this 1829 * extra check can do much harm. 1830 */ 1831 if((CURLE_OK == result) || (CURLE_SEND_ERROR == result)) { 1832 bool async; 1833 bool protocol_done = TRUE; 1834 1835 /* Now, redo the connect and get a new connection */ 1836 result = Curl_connect(data, connp, &async, &protocol_done); 1837 if(CURLE_OK == result) { 1838 /* We have connected or sent away a name resolve query fine */ 1839 1840 conn = *connp; /* setup conn to again point to something nice */ 1841 if(async) { 1842 /* Now, if async is TRUE here, we need to wait for the name 1843 to resolve */ 1844 result = Curl_resolver_wait_resolv(conn, NULL); 1845 if(result) 1846 return result; 1847 1848 /* Resolved, continue with the connection */ 1849 result = Curl_async_resolved(conn, &protocol_done); 1850 if(result) 1851 return result; 1852 } 1853 } 1854 } 1855 1856 return result; 1857} 1858 1859/* Returns CURLE_OK *and* sets '*url' if a request retry is wanted. 1860 1861 NOTE: that the *url is malloc()ed. */ 1862CURLcode Curl_retry_request(struct connectdata *conn, 1863 char **url) 1864{ 1865 struct SessionHandle *data = conn->data; 1866 1867 *url = NULL; 1868 1869 /* if we're talking upload, we can't do the checks below, unless the protocol 1870 is HTTP as when uploading over HTTP we will still get a response */ 1871 if(data->set.upload && 1872 !(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP))) 1873 return CURLE_OK; 1874 1875 if(/* workaround for broken TLS servers */ data->state.ssl_connect_retry || 1876 ((data->req.bytecount + 1877 data->req.headerbytecount == 0) && 1878 conn->bits.reuse && 1879 !data->set.opt_no_body && 1880 data->set.rtspreq != RTSPREQ_RECEIVE)) { 1881 /* We got no data, we attempted to re-use a connection and yet we want a 1882 "body". This might happen if the connection was left alive when we were 1883 done using it before, but that was closed when we wanted to read from 1884 it again. Bad luck. Retry the same request on a fresh connect! */ 1885 infof(conn->data, "Connection died, retrying a fresh connect\n"); 1886 *url = strdup(conn->data->change.url); 1887 if(!*url) 1888 return CURLE_OUT_OF_MEMORY; 1889 1890 connclose(conn, "retry"); /* close this connection */ 1891 conn->bits.retry = TRUE; /* mark this as a connection we're about 1892 to retry. Marking it this way should 1893 prevent i.e HTTP transfers to return 1894 error just because nothing has been 1895 transferred! */ 1896 1897 1898 if(conn->handler->protocol&PROTO_FAMILY_HTTP) { 1899 struct HTTP *http = data->req.protop; 1900 if(http->writebytecount) 1901 return Curl_readrewind(conn); 1902 } 1903 } 1904 return CURLE_OK; 1905} 1906 1907/* 1908 * Curl_setup_transfer() is called to setup some basic properties for the 1909 * upcoming transfer. 1910 */ 1911void 1912Curl_setup_transfer( 1913 struct connectdata *conn, /* connection data */ 1914 int sockindex, /* socket index to read from or -1 */ 1915 curl_off_t size, /* -1 if unknown at this point */ 1916 bool getheader, /* TRUE if header parsing is wanted */ 1917 curl_off_t *bytecountp, /* return number of bytes read or NULL */ 1918 int writesockindex, /* socket index to write to, it may very well be 1919 the same we read from. -1 disables */ 1920 curl_off_t *writecountp /* return number of bytes written or NULL */ 1921 ) 1922{ 1923 struct SessionHandle *data; 1924 struct SingleRequest *k; 1925 1926 DEBUGASSERT(conn != NULL); 1927 1928 data = conn->data; 1929 k = &data->req; 1930 1931 DEBUGASSERT((sockindex <= 1) && (sockindex >= -1)); 1932 1933 /* now copy all input parameters */ 1934 conn->sockfd = sockindex == -1 ? 1935 CURL_SOCKET_BAD : conn->sock[sockindex]; 1936 conn->writesockfd = writesockindex == -1 ? 1937 CURL_SOCKET_BAD:conn->sock[writesockindex]; 1938 k->getheader = getheader; 1939 1940 k->size = size; 1941 k->bytecountp = bytecountp; 1942 k->writebytecountp = writecountp; 1943 1944 /* The code sequence below is placed in this function just because all 1945 necessary input is not always known in do_complete() as this function may 1946 be called after that */ 1947 1948 if(!k->getheader) { 1949 k->header = FALSE; 1950 if(size > 0) 1951 Curl_pgrsSetDownloadSize(data, size); 1952 } 1953 /* we want header and/or body, if neither then don't do this! */ 1954 if(k->getheader || !data->set.opt_no_body) { 1955 1956 if(conn->sockfd != CURL_SOCKET_BAD) 1957 k->keepon |= KEEP_RECV; 1958 1959 if(conn->writesockfd != CURL_SOCKET_BAD) { 1960 struct HTTP *http = data->req.protop; 1961 /* HTTP 1.1 magic: 1962 1963 Even if we require a 100-return code before uploading data, we might 1964 need to write data before that since the REQUEST may not have been 1965 finished sent off just yet. 1966 1967 Thus, we must check if the request has been sent before we set the 1968 state info where we wait for the 100-return code 1969 */ 1970 if((data->state.expect100header) && 1971 (conn->handler->protocol&PROTO_FAMILY_HTTP) && 1972 (http->sending == HTTPSEND_BODY)) { 1973 /* wait with write until we either got 100-continue or a timeout */ 1974 k->exp100 = EXP100_AWAITING_CONTINUE; 1975 k->start100 = Curl_tvnow(); 1976 1977 /* Set a timeout for the multi interface. Add the inaccuracy margin so 1978 that we don't fire slightly too early and get denied to run. */ 1979 Curl_expire(data, data->set.expect_100_timeout); 1980 } 1981 else { 1982 if(data->state.expect100header) 1983 /* when we've sent off the rest of the headers, we must await a 1984 100-continue but first finish sending the request */ 1985 k->exp100 = EXP100_SENDING_REQUEST; 1986 1987 /* enable the write bit when we're not waiting for continue */ 1988 k->keepon |= KEEP_SEND; 1989 } 1990 } /* if(conn->writesockfd != CURL_SOCKET_BAD) */ 1991 } /* if(k->getheader || !data->set.opt_no_body) */ 1992 1993} 1994