1/* 2 * Copyright (C) 2004, 2006 Apple Inc. All rights reserved. 3 * Copyright (C) 2006 Michael Emmel mike.emmel@gmail.com 4 * Copyright (C) 2007 Alp Toker <alp@atoker.com> 5 * Copyright (C) 2007 Holger Hans Peter Freyther 6 * Copyright (C) 2008 Collabora Ltd. 7 * Copyright (C) 2008 Nuanti Ltd. 8 * Copyright (C) 2009 Appcelerator Inc. 9 * Copyright (C) 2009 Brent Fulgham <bfulgham@webkit.org> 10 * Copyright (C) 2013 Peter Gal <galpeter@inf.u-szeged.hu>, University of Szeged 11 * Copyright (C) 2013 Alex Christensen <achristensen@webkit.org> 12 * Copyright (C) 2013 University of Szeged 13 * All rights reserved. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY 25 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR 28 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 29 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 30 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 31 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 32 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37#include "config.h" 38#include "ResourceHandleManager.h" 39 40#if USE(CURL) 41 42#include "CredentialStorage.h" 43#include "CurlCacheManager.h" 44#include "DataURL.h" 45#include "HTTPHeaderNames.h" 46#include "HTTPParsers.h" 47#include "MIMETypeRegistry.h" 48#include "MultipartHandle.h" 49#include "ResourceError.h" 50#include "ResourceHandle.h" 51#include "ResourceHandleInternal.h" 52#include "SSLHandle.h" 53 54#if OS(WINDOWS) 55#include "WebCoreBundleWin.h" 56#include <shlobj.h> 57#include <shlwapi.h> 58#else 59#include <sys/param.h> 60#define MAX_PATH MAXPATHLEN 61#endif 62 63#include <errno.h> 64#include <stdio.h> 65#if ENABLE(WEB_TIMING) 66#include <wtf/CurrentTime.h> 67#endif 68#if USE(CF) 69#include <wtf/RetainPtr.h> 70#endif 71#include <wtf/Threading.h> 72#include <wtf/Vector.h> 73#include <wtf/text/CString.h> 74 75 76namespace WebCore { 77 78const int selectTimeoutMS = 5; 79const double pollTimeSeconds = 0.05; 80const int maxRunningJobs = 5; 81 82static const bool ignoreSSLErrors = getenv("WEBKIT_IGNORE_SSL_ERRORS"); 83 84static CString certificatePath() 85{ 86#if USE(CF) 87 CFBundleRef webKitBundleRef = webKitBundle(); 88 if (webKitBundleRef) { 89 RetainPtr<CFURLRef> certURLRef = adoptCF(CFBundleCopyResourceURL(webKitBundleRef, CFSTR("cacert"), CFSTR("pem"), CFSTR("certificates"))); 90 if (certURLRef) { 91 char path[MAX_PATH]; 92 CFURLGetFileSystemRepresentation(certURLRef.get(), false, reinterpret_cast<UInt8*>(path), MAX_PATH); 93 return path; 94 } 95 } 96#endif 97 char* envPath = getenv("CURL_CA_BUNDLE_PATH"); 98 if (envPath) 99 return envPath; 100 101 return CString(); 102} 103 104static char* cookieJarPath() 105{ 106 char* cookieJarPath = getenv("CURL_COOKIE_JAR_PATH"); 107 if (cookieJarPath) 108 return fastStrDup(cookieJarPath); 109 110#if OS(WINDOWS) 111 char executablePath[MAX_PATH]; 112 char appDataDirectory[MAX_PATH]; 113 char cookieJarFullPath[MAX_PATH]; 114 char cookieJarDirectory[MAX_PATH]; 115 116 if (FAILED(::SHGetFolderPathA(0, CSIDL_LOCAL_APPDATA | CSIDL_FLAG_CREATE, 0, 0, appDataDirectory)) 117 || FAILED(::GetModuleFileNameA(0, executablePath, MAX_PATH))) 118 return fastStrDup("cookies.dat"); 119 120 ::PathRemoveExtensionA(executablePath); 121 LPSTR executableName = ::PathFindFileNameA(executablePath); 122 sprintf_s(cookieJarDirectory, MAX_PATH, "%s/%s", appDataDirectory, executableName); 123 sprintf_s(cookieJarFullPath, MAX_PATH, "%s/cookies.dat", cookieJarDirectory); 124 125 if (::SHCreateDirectoryExA(0, cookieJarDirectory, 0) != ERROR_SUCCESS 126 && ::GetLastError() != ERROR_FILE_EXISTS 127 && ::GetLastError() != ERROR_ALREADY_EXISTS) 128 return fastStrDup("cookies.dat"); 129 130 return fastStrDup(cookieJarFullPath); 131#else 132 return fastStrDup("cookies.dat"); 133#endif 134} 135 136static Mutex* sharedResourceMutex(curl_lock_data data) { 137 DEPRECATED_DEFINE_STATIC_LOCAL(Mutex, cookieMutex, ()); 138 DEPRECATED_DEFINE_STATIC_LOCAL(Mutex, dnsMutex, ()); 139 DEPRECATED_DEFINE_STATIC_LOCAL(Mutex, shareMutex, ()); 140 141 switch (data) { 142 case CURL_LOCK_DATA_COOKIE: 143 return &cookieMutex; 144 case CURL_LOCK_DATA_DNS: 145 return &dnsMutex; 146 case CURL_LOCK_DATA_SHARE: 147 return &shareMutex; 148 default: 149 ASSERT_NOT_REACHED(); 150 return NULL; 151 } 152} 153 154#if ENABLE(WEB_TIMING) 155static int milisecondsSinceRequest(double requestTime) 156{ 157 return static_cast<int>((monotonicallyIncreasingTime() - requestTime) * 1000.0); 158} 159 160static void calculateWebTimingInformations(ResourceHandleInternal* d) 161{ 162 double startTransfertTime = 0; 163 double preTransferTime = 0; 164 double dnslookupTime = 0; 165 double connectTime = 0; 166 double appConnectTime = 0; 167 168 curl_easy_getinfo(d->m_handle, CURLINFO_NAMELOOKUP_TIME, &dnslookupTime); 169 curl_easy_getinfo(d->m_handle, CURLINFO_CONNECT_TIME, &connectTime); 170 curl_easy_getinfo(d->m_handle, CURLINFO_APPCONNECT_TIME, &appConnectTime); 171 curl_easy_getinfo(d->m_handle, CURLINFO_STARTTRANSFER_TIME, &startTransfertTime); 172 curl_easy_getinfo(d->m_handle, CURLINFO_PRETRANSFER_TIME, &preTransferTime); 173 174 d->m_response.resourceLoadTiming()->dnsStart = 0; 175 d->m_response.resourceLoadTiming()->dnsEnd = static_cast<int>(dnslookupTime * 1000); 176 177 d->m_response.resourceLoadTiming()->connectStart = static_cast<int>(dnslookupTime * 1000); 178 d->m_response.resourceLoadTiming()->connectEnd = static_cast<int>(connectTime * 1000); 179 180 d->m_response.resourceLoadTiming()->sendStart = static_cast<int>(connectTime *1000); 181 d->m_response.resourceLoadTiming()->sendEnd =static_cast<int>(preTransferTime * 1000); 182 183 if (appConnectTime) { 184 d->m_response.resourceLoadTiming()->sslStart = static_cast<int>(connectTime * 1000); 185 d->m_response.resourceLoadTiming()->sslEnd = static_cast<int>(appConnectTime *1000); 186 } 187} 188 189static int sockoptfunction(void* data, curl_socket_t /*curlfd*/, curlsocktype /*purpose*/) 190{ 191 ResourceHandle* job = static_cast<ResourceHandle*>(data); 192 ResourceHandleInternal* d = job->getInternal(); 193 194 if (d->m_response.resourceLoadTiming()) 195 d->m_response.resourceLoadTiming()->requestTime = monotonicallyIncreasingTime(); 196 197 return 0; 198} 199#endif 200 201// libcurl does not implement its own thread synchronization primitives. 202// these two functions provide mutexes for cookies, and for the global DNS 203// cache. 204static void curl_lock_callback(CURL* /* handle */, curl_lock_data data, curl_lock_access /* access */, void* /* userPtr */) 205{ 206 if (Mutex* mutex = sharedResourceMutex(data)) 207 mutex->lock(); 208} 209 210static void curl_unlock_callback(CURL* /* handle */, curl_lock_data data, void* /* userPtr */) 211{ 212 if (Mutex* mutex = sharedResourceMutex(data)) 213 mutex->unlock(); 214} 215 216inline static bool isHttpInfo(int statusCode) 217{ 218 return 100 <= statusCode && statusCode < 200; 219} 220 221inline static bool isHttpRedirect(int statusCode) 222{ 223 return 300 <= statusCode && statusCode < 400 && statusCode != 304; 224} 225 226inline static bool isHttpAuthentication(int statusCode) 227{ 228 return statusCode == 401; 229} 230 231ResourceHandleManager::ResourceHandleManager() 232 : m_downloadTimer(this, &ResourceHandleManager::downloadTimerCallback) 233 , m_cookieJarFileName(cookieJarPath()) 234 , m_certificatePath (certificatePath()) 235 , m_runningJobs(0) 236{ 237 curl_global_init(CURL_GLOBAL_ALL); 238 m_curlMultiHandle = curl_multi_init(); 239 m_curlShareHandle = curl_share_init(); 240 curl_share_setopt(m_curlShareHandle, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); 241 curl_share_setopt(m_curlShareHandle, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); 242 curl_share_setopt(m_curlShareHandle, CURLSHOPT_LOCKFUNC, curl_lock_callback); 243 curl_share_setopt(m_curlShareHandle, CURLSHOPT_UNLOCKFUNC, curl_unlock_callback); 244 245 initCookieSession(); 246} 247 248ResourceHandleManager::~ResourceHandleManager() 249{ 250 curl_multi_cleanup(m_curlMultiHandle); 251 curl_share_cleanup(m_curlShareHandle); 252 if (m_cookieJarFileName) 253 fastFree(m_cookieJarFileName); 254 curl_global_cleanup(); 255} 256 257CURLSH* ResourceHandleManager::getCurlShareHandle() const 258{ 259 return m_curlShareHandle; 260} 261 262void ResourceHandleManager::setCookieJarFileName(const char* cookieJarFileName) 263{ 264 m_cookieJarFileName = fastStrDup(cookieJarFileName); 265} 266 267const char* ResourceHandleManager::getCookieJarFileName() const 268{ 269 return m_cookieJarFileName; 270} 271 272ResourceHandleManager* ResourceHandleManager::sharedInstance() 273{ 274 static ResourceHandleManager* sharedInstance = 0; 275 if (!sharedInstance) 276 sharedInstance = new ResourceHandleManager(); 277 return sharedInstance; 278} 279 280static void handleLocalReceiveResponse (CURL* handle, ResourceHandle* job, ResourceHandleInternal* d) 281{ 282 // since the code in headerCallback will not have run for local files 283 // the code to set the URL and fire didReceiveResponse is never run, 284 // which means the ResourceLoader's response does not contain the URL. 285 // Run the code here for local files to resolve the issue. 286 // TODO: See if there is a better approach for handling this. 287 const char* hdr; 288 CURLcode err = curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &hdr); 289 ASSERT_UNUSED(err, CURLE_OK == err); 290 d->m_response.setURL(URL(ParsedURLString, hdr)); 291 if (d->client()) 292 d->client()->didReceiveResponse(job, d->m_response); 293 d->m_response.setResponseFired(true); 294} 295 296 297// called with data after all headers have been processed via headerCallback 298static size_t writeCallback(void* ptr, size_t size, size_t nmemb, void* data) 299{ 300 ResourceHandle* job = static_cast<ResourceHandle*>(data); 301 ResourceHandleInternal* d = job->getInternal(); 302 if (d->m_cancelled) 303 return 0; 304 305 // We should never be called when deferred loading is activated. 306 ASSERT(!d->m_defersLoading); 307 308 size_t totalSize = size * nmemb; 309 310 // this shouldn't be necessary but apparently is. CURL writes the data 311 // of html page even if it is a redirect that was handled internally 312 // can be observed e.g. on gmail.com 313 CURL* h = d->m_handle; 314 long httpCode = 0; 315 CURLcode err = curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode); 316 if (CURLE_OK == err && httpCode >= 300 && httpCode < 400) 317 return totalSize; 318 319 if (!d->m_response.responseFired()) { 320 handleLocalReceiveResponse(h, job, d); 321 if (d->m_cancelled) 322 return 0; 323 } 324 325 if (d->m_multipartHandle) 326 d->m_multipartHandle->contentReceived(static_cast<const char*>(ptr), totalSize); 327 else if (d->client()) { 328 d->client()->didReceiveData(job, static_cast<char*>(ptr), totalSize, 0); 329 CurlCacheManager::getInstance().didReceiveData(*job, static_cast<char*>(ptr), totalSize); 330 } 331 332 return totalSize; 333} 334 335static bool isAppendableHeader(const String &key) 336{ 337 static const char* appendableHeaders[] = { 338 "access-control-allow-headers", 339 "access-control-allow-methods", 340 "access-control-allow-origin", 341 "access-control-expose-headers", 342 "allow", 343 "cache-control", 344 "connection", 345 "content-encoding", 346 "content-language", 347 "if-match", 348 "if-none-match", 349 "keep-alive", 350 "pragma", 351 "proxy-authenticate", 352 "public", 353 "server", 354 "set-cookie", 355 "te", 356 "trailer", 357 "transfer-encoding", 358 "upgrade", 359 "user-agent", 360 "vary", 361 "via", 362 "warning", 363 "www-authenticate", 364 0 365 }; 366 367 // Custom headers start with 'X-', and need no further checking. 368 if (key.startsWith("x-", /* caseSensitive */ false)) 369 return true; 370 371 for (unsigned i = 0; appendableHeaders[i]; ++i) 372 if (equalIgnoringCase(key, appendableHeaders[i])) 373 return true; 374 375 return false; 376} 377 378static void removeLeadingAndTrailingQuotes(String& value) 379{ 380 unsigned length = value.length(); 381 if (value.startsWith('"') && value.endsWith('"') && length > 1) 382 value = value.substring(1, length-2); 383} 384 385static bool getProtectionSpace(CURL* h, const ResourceResponse& response, ProtectionSpace& protectionSpace) 386{ 387 CURLcode err; 388 389 long port = 0; 390 err = curl_easy_getinfo(h, CURLINFO_PRIMARY_PORT, &port); 391 if (err != CURLE_OK) 392 return false; 393 394 long availableAuth = CURLAUTH_NONE; 395 err = curl_easy_getinfo(h, CURLINFO_HTTPAUTH_AVAIL, &availableAuth); 396 if (err != CURLE_OK) 397 return false; 398 399 const char* effectiveUrl = 0; 400 err = curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &effectiveUrl); 401 if (err != CURLE_OK) 402 return false; 403 404 URL url(ParsedURLString, effectiveUrl); 405 406 String host = url.host(); 407 String protocol = url.protocol(); 408 409 String realm; 410 411 const String authHeader = response.httpHeaderField(HTTPHeaderName::Authorization); 412 const String realmString = "realm="; 413 int realmPos = authHeader.find(realmString); 414 if (realmPos > 0) { 415 realm = authHeader.substring(realmPos + realmString.length()); 416 realm = realm.left(realm.find(',')); 417 removeLeadingAndTrailingQuotes(realm); 418 } 419 420 ProtectionSpaceServerType serverType = ProtectionSpaceServerHTTP; 421 if (protocol == "https") 422 serverType = ProtectionSpaceServerHTTPS; 423 424 ProtectionSpaceAuthenticationScheme authScheme = ProtectionSpaceAuthenticationSchemeUnknown; 425 426 if (availableAuth & CURLAUTH_BASIC) 427 authScheme = ProtectionSpaceAuthenticationSchemeHTTPBasic; 428 if (availableAuth & CURLAUTH_DIGEST) 429 authScheme = ProtectionSpaceAuthenticationSchemeHTTPDigest; 430 if (availableAuth & CURLAUTH_GSSNEGOTIATE) 431 authScheme = ProtectionSpaceAuthenticationSchemeNegotiate; 432 if (availableAuth & CURLAUTH_NTLM) 433 authScheme = ProtectionSpaceAuthenticationSchemeNTLM; 434 435 protectionSpace = ProtectionSpace(host, port, serverType, realm, authScheme); 436 437 return true; 438} 439 440/* 441 * This is being called for each HTTP header in the response. This includes '\r\n' 442 * for the last line of the header. 443 * 444 * We will add each HTTP Header to the ResourceResponse and on the termination 445 * of the header (\r\n) we will parse Content-Type and Content-Disposition and 446 * update the ResourceResponse and then send it away. 447 * 448 */ 449static size_t headerCallback(char* ptr, size_t size, size_t nmemb, void* data) 450{ 451 ResourceHandle* job = static_cast<ResourceHandle*>(data); 452 ResourceHandleInternal* d = job->getInternal(); 453 if (d->m_cancelled) 454 return 0; 455 456 // We should never be called when deferred loading is activated. 457 ASSERT(!d->m_defersLoading); 458 459 size_t totalSize = size * nmemb; 460 ResourceHandleClient* client = d->client(); 461 462 String header = String::fromUTF8WithLatin1Fallback(static_cast<const char*>(ptr), totalSize); 463 464 /* 465 * a) We can finish and send the ResourceResponse 466 * b) We will add the current header to the HTTPHeaderMap of the ResourceResponse 467 * 468 * The HTTP standard requires to use \r\n but for compatibility it recommends to 469 * accept also \n. 470 */ 471 if (header == String("\r\n") || header == String("\n")) { 472 CURL* h = d->m_handle; 473 474 long httpCode = 0; 475 curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode); 476 477 if (isHttpInfo(httpCode)) { 478 // Just return when receiving http info, e.g. HTTP/1.1 100 Continue. 479 // If not, the request might be cancelled, because the MIME type will be empty for this response. 480 return totalSize; 481 } 482 483 double contentLength = 0; 484 curl_easy_getinfo(h, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength); 485 d->m_response.setExpectedContentLength(static_cast<long long int>(contentLength)); 486 487 const char* hdr; 488 curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &hdr); 489 d->m_response.setURL(URL(ParsedURLString, hdr)); 490 491 d->m_response.setHTTPStatusCode(httpCode); 492 d->m_response.setMimeType(extractMIMETypeFromMediaType(d->m_response.httpHeaderField(HTTPHeaderName::ContentType)).lower()); 493 d->m_response.setTextEncodingName(extractCharsetFromMediaType(d->m_response.httpHeaderField(HTTPHeaderName::ContentType))); 494 d->m_response.setSuggestedFilename(filenameFromHTTPContentDisposition(d->m_response.httpHeaderField(HTTPHeaderName::ContentDisposition))); 495 496 if (d->m_response.isMultipart()) { 497 String boundary; 498 bool parsed = MultipartHandle::extractBoundary(d->m_response.httpHeaderField(HTTPHeaderName::ContentType), boundary); 499 if (parsed) 500 d->m_multipartHandle = MultipartHandle::create(job, boundary); 501 } 502 503#if ENABLE(WEB_TIMING) 504 if (d->m_response.resourceLoadTiming() && d->m_response.resourceLoadTiming()->requestTime) 505 d->m_response.resourceLoadTiming()->receiveHeadersEnd = milisecondsSinceRequest(d->m_response.resourceLoadTiming()->requestTime); 506#endif 507 508 // HTTP redirection 509 if (isHttpRedirect(httpCode)) { 510 String location = d->m_response.httpHeaderField(HTTPHeaderName::Location); 511 if (!location.isEmpty()) { 512 URL newURL = URL(job->firstRequest().url(), location); 513 514 ResourceRequest redirectedRequest = job->firstRequest(); 515 redirectedRequest.setURL(newURL); 516 if (client) 517 client->willSendRequest(job, redirectedRequest, d->m_response); 518 519 d->m_firstRequest.setURL(newURL); 520 521 return totalSize; 522 } 523 } else if (isHttpAuthentication(httpCode)) { 524 ProtectionSpace protectionSpace; 525 if (getProtectionSpace(d->m_handle, d->m_response, protectionSpace)) { 526 Credential credential; 527 AuthenticationChallenge challenge(protectionSpace, credential, d->m_authFailureCount, d->m_response, ResourceError()); 528 challenge.setAuthenticationClient(job); 529 job->didReceiveAuthenticationChallenge(challenge); 530 d->m_authFailureCount++; 531 return totalSize; 532 } 533 } 534 535 if (client) { 536 if (httpCode == 304) { 537 const String& url = job->firstRequest().url().string(); 538 CurlCacheManager::getInstance().getCachedResponse(url, d->m_response); 539 } 540 client->didReceiveResponse(job, d->m_response); 541 CurlCacheManager::getInstance().didReceiveResponse(*job, d->m_response); 542 } 543 d->m_response.setResponseFired(true); 544 545 } else { 546 int splitPos = header.find(":"); 547 if (splitPos != -1) { 548 String key = header.left(splitPos).stripWhiteSpace(); 549 String value = header.substring(splitPos + 1).stripWhiteSpace(); 550 551 if (isAppendableHeader(key)) 552 d->m_response.addHTTPHeaderField(key, value); 553 else 554 d->m_response.setHTTPHeaderField(key, value); 555 } else if (header.startsWith("HTTP", false)) { 556 // This is the first line of the response. 557 // Extract the http status text from this. 558 // 559 // If the FOLLOWLOCATION option is enabled for the curl handle then 560 // curl will follow the redirections internally. Thus this header callback 561 // will be called more than one time with the line starting "HTTP" for one job. 562 long httpCode = 0; 563 curl_easy_getinfo(d->m_handle, CURLINFO_RESPONSE_CODE, &httpCode); 564 565 String httpCodeString = String::number(httpCode); 566 int statusCodePos = header.find(httpCodeString); 567 568 if (statusCodePos != -1) { 569 // The status text is after the status code. 570 String status = header.substring(statusCodePos + httpCodeString.length()); 571 d->m_response.setHTTPStatusText(status.stripWhiteSpace()); 572 } 573 574 } 575 } 576 577 return totalSize; 578} 579 580/* This is called to obtain HTTP POST or PUT data. 581 Iterate through FormData elements and upload files. 582 Carefully respect the given buffer size and fill the rest of the data at the next calls. 583*/ 584size_t readCallback(void* ptr, size_t size, size_t nmemb, void* data) 585{ 586 ResourceHandle* job = static_cast<ResourceHandle*>(data); 587 ResourceHandleInternal* d = job->getInternal(); 588 589 if (d->m_cancelled) 590 return 0; 591 592 // We should never be called when deferred loading is activated. 593 ASSERT(!d->m_defersLoading); 594 595 if (!size || !nmemb) 596 return 0; 597 598 if (!d->m_formDataStream.hasMoreElements()) 599 return 0; 600 601 size_t sent = d->m_formDataStream.read(ptr, size, nmemb); 602 603 // Something went wrong so cancel the job. 604 if (!sent) 605 job->cancel(); 606 607 return sent; 608} 609 610void ResourceHandleManager::downloadTimerCallback(Timer<ResourceHandleManager>* /* timer */) 611{ 612 startScheduledJobs(); 613 614 fd_set fdread; 615 fd_set fdwrite; 616 fd_set fdexcep; 617 int maxfd = 0; 618 619 struct timeval timeout; 620 timeout.tv_sec = 0; 621 timeout.tv_usec = selectTimeoutMS * 1000; // select waits microseconds 622 623 // Retry 'select' if it was interrupted by a process signal. 624 int rc = 0; 625 do { 626 FD_ZERO(&fdread); 627 FD_ZERO(&fdwrite); 628 FD_ZERO(&fdexcep); 629 curl_multi_fdset(m_curlMultiHandle, &fdread, &fdwrite, &fdexcep, &maxfd); 630 // When the 3 file descriptors are empty, winsock will return -1 631 // and bail out, stopping the file download. So make sure we 632 // have valid file descriptors before calling select. 633 if (maxfd >= 0) 634 rc = ::select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout); 635 } while (rc == -1 && errno == EINTR); 636 637 if (-1 == rc) { 638#ifndef NDEBUG 639 perror("bad: select() returned -1: "); 640#endif 641 return; 642 } 643 644 int runningHandles = 0; 645 while (curl_multi_perform(m_curlMultiHandle, &runningHandles) == CURLM_CALL_MULTI_PERFORM) { } 646 647 // check the curl messages indicating completed transfers 648 // and free their resources 649 while (true) { 650 int messagesInQueue; 651 CURLMsg* msg = curl_multi_info_read(m_curlMultiHandle, &messagesInQueue); 652 if (!msg) 653 break; 654 655 // find the node which has same d->m_handle as completed transfer 656 CURL* handle = msg->easy_handle; 657 ASSERT(handle); 658 ResourceHandle* job = 0; 659 CURLcode err = curl_easy_getinfo(handle, CURLINFO_PRIVATE, &job); 660 ASSERT_UNUSED(err, CURLE_OK == err); 661 ASSERT(job); 662 if (!job) 663 continue; 664 ResourceHandleInternal* d = job->getInternal(); 665 ASSERT(d->m_handle == handle); 666 667 if (d->m_cancelled) { 668 removeFromCurl(job); 669 continue; 670 } 671 672 if (CURLMSG_DONE != msg->msg) 673 continue; 674 675 676 if (CURLE_OK == msg->data.result) { 677#if ENABLE(WEB_TIMING) 678 calculateWebTimingInformations(d); 679#endif 680 if (!d->m_response.responseFired()) { 681 handleLocalReceiveResponse(d->m_handle, job, d); 682 if (d->m_cancelled) { 683 removeFromCurl(job); 684 continue; 685 } 686 } 687 688 if (d->m_multipartHandle) 689 d->m_multipartHandle->contentEnded(); 690 691 if (d->client()) { 692 d->client()->didFinishLoading(job, 0); 693 CurlCacheManager::getInstance().didFinishLoading(*job); 694 } 695 } else { 696 char* url = 0; 697 curl_easy_getinfo(d->m_handle, CURLINFO_EFFECTIVE_URL, &url); 698#ifndef NDEBUG 699 fprintf(stderr, "Curl ERROR for url='%s', error: '%s'\n", url, curl_easy_strerror(msg->data.result)); 700#endif 701 if (d->client()) { 702 ResourceError resourceError(String(), msg->data.result, String(url), String(curl_easy_strerror(msg->data.result))); 703 resourceError.setSSLErrors(d->m_sslErrors); 704 d->client()->didFail(job, resourceError); 705 CurlCacheManager::getInstance().didFail(*job); 706 } 707 } 708 709 removeFromCurl(job); 710 } 711 712 bool started = startScheduledJobs(); // new jobs might have been added in the meantime 713 714 if (!m_downloadTimer.isActive() && (started || (runningHandles > 0))) 715 m_downloadTimer.startOneShot(pollTimeSeconds); 716} 717 718void ResourceHandleManager::setProxyInfo(const String& host, 719 unsigned long port, 720 ProxyType type, 721 const String& username, 722 const String& password) 723{ 724 m_proxyType = type; 725 726 if (!host.length()) { 727 m_proxy = emptyString(); 728 } else { 729 String userPass; 730 if (username.length() || password.length()) 731 userPass = username + ":" + password + "@"; 732 733 m_proxy = String("http://") + userPass + host + ":" + String::number(port); 734 } 735} 736 737void ResourceHandleManager::removeFromCurl(ResourceHandle* job) 738{ 739 ResourceHandleInternal* d = job->getInternal(); 740 ASSERT(d->m_handle); 741 if (!d->m_handle) 742 return; 743 m_runningJobs--; 744 curl_multi_remove_handle(m_curlMultiHandle, d->m_handle); 745 curl_easy_cleanup(d->m_handle); 746 d->m_handle = 0; 747 job->deref(); 748} 749 750static inline size_t getFormElementsCount(ResourceHandle* job) 751{ 752 RefPtr<FormData> formData = job->firstRequest().httpBody(); 753 754 if (!formData) 755 return 0; 756 757 // Resolve the blob elements so the formData can correctly report it's size. 758 formData = formData->resolveBlobReferences(); 759 job->firstRequest().setHTTPBody(formData); 760 761 return formData->elements().size(); 762} 763 764static void setupFormData(ResourceHandle* job, CURLoption sizeOption, struct curl_slist** headers) 765{ 766 ResourceHandleInternal* d = job->getInternal(); 767 Vector<FormDataElement> elements = job->firstRequest().httpBody()->elements(); 768 size_t numElements = elements.size(); 769 770 // The size of a curl_off_t could be different in WebKit and in cURL depending on 771 // compilation flags of both. 772 static int expectedSizeOfCurlOffT = 0; 773 if (!expectedSizeOfCurlOffT) { 774 curl_version_info_data *infoData = curl_version_info(CURLVERSION_NOW); 775 if (infoData->features & CURL_VERSION_LARGEFILE) 776 expectedSizeOfCurlOffT = sizeof(long long); 777 else 778 expectedSizeOfCurlOffT = sizeof(int); 779 } 780 781 static const long long maxCurlOffT = (1LL << (expectedSizeOfCurlOffT * 8 - 1)) - 1; 782 // Obtain the total size of the form data 783 curl_off_t size = 0; 784 bool chunkedTransfer = false; 785 for (size_t i = 0; i < numElements; i++) { 786 FormDataElement element = elements[i]; 787 if (element.m_type == FormDataElement::Type::EncodedFile) { 788 long long fileSizeResult; 789 if (getFileSize(element.m_filename, fileSizeResult)) { 790 if (fileSizeResult > maxCurlOffT) { 791 // File size is too big for specifying it to cURL 792 chunkedTransfer = true; 793 break; 794 } 795 size += fileSizeResult; 796 } else { 797 chunkedTransfer = true; 798 break; 799 } 800 } else 801 size += elements[i].m_data.size(); 802 } 803 804 // cURL guesses that we want chunked encoding as long as we specify the header 805 if (chunkedTransfer) 806 *headers = curl_slist_append(*headers, "Transfer-Encoding: chunked"); 807 else { 808 if (sizeof(long long) == expectedSizeOfCurlOffT) 809 curl_easy_setopt(d->m_handle, sizeOption, (long long)size); 810 else 811 curl_easy_setopt(d->m_handle, sizeOption, (int)size); 812 } 813 814 curl_easy_setopt(d->m_handle, CURLOPT_READFUNCTION, readCallback); 815 curl_easy_setopt(d->m_handle, CURLOPT_READDATA, job); 816} 817 818void ResourceHandleManager::setupPUT(ResourceHandle* job, struct curl_slist** headers) 819{ 820 ResourceHandleInternal* d = job->getInternal(); 821 curl_easy_setopt(d->m_handle, CURLOPT_UPLOAD, TRUE); 822 curl_easy_setopt(d->m_handle, CURLOPT_INFILESIZE, 0); 823 824 // Disable the Expect: 100 continue header 825 *headers = curl_slist_append(*headers, "Expect:"); 826 827 size_t numElements = getFormElementsCount(job); 828 if (!numElements) 829 return; 830 831 setupFormData(job, CURLOPT_INFILESIZE_LARGE, headers); 832} 833 834void ResourceHandleManager::setupPOST(ResourceHandle* job, struct curl_slist** headers) 835{ 836 ResourceHandleInternal* d = job->getInternal(); 837 curl_easy_setopt(d->m_handle, CURLOPT_POST, TRUE); 838 curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDSIZE, 0); 839 840 size_t numElements = getFormElementsCount(job); 841 if (!numElements) 842 return; 843 844 // Do not stream for simple POST data 845 if (numElements == 1) { 846 job->firstRequest().httpBody()->flatten(d->m_postBytes); 847 if (d->m_postBytes.size()) { 848 curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDSIZE, d->m_postBytes.size()); 849 curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDS, d->m_postBytes.data()); 850 } 851 return; 852 } 853 854 setupFormData(job, CURLOPT_POSTFIELDSIZE_LARGE, headers); 855} 856 857void ResourceHandleManager::add(ResourceHandle* job) 858{ 859 // we can be called from within curl, so to avoid re-entrancy issues 860 // schedule this job to be added the next time we enter curl download loop 861 job->ref(); 862 m_resourceHandleList.append(job); 863 if (!m_downloadTimer.isActive()) 864 m_downloadTimer.startOneShot(pollTimeSeconds); 865} 866 867bool ResourceHandleManager::removeScheduledJob(ResourceHandle* job) 868{ 869 int size = m_resourceHandleList.size(); 870 for (int i = 0; i < size; i++) { 871 if (job == m_resourceHandleList[i]) { 872 m_resourceHandleList.remove(i); 873 job->deref(); 874 return true; 875 } 876 } 877 return false; 878} 879 880bool ResourceHandleManager::startScheduledJobs() 881{ 882 // TODO: Create a separate stack of jobs for each domain. 883 884 bool started = false; 885 while (!m_resourceHandleList.isEmpty() && m_runningJobs < maxRunningJobs) { 886 ResourceHandle* job = m_resourceHandleList[0]; 887 m_resourceHandleList.remove(0); 888 startJob(job); 889 started = true; 890 } 891 return started; 892} 893 894void ResourceHandleManager::dispatchSynchronousJob(ResourceHandle* job) 895{ 896 URL kurl = job->firstRequest().url(); 897 898 if (kurl.protocolIsData()) { 899 handleDataURL(job); 900 return; 901 } 902 903 ResourceHandleInternal* handle = job->getInternal(); 904 905 // If defersLoading is true and we call curl_easy_perform 906 // on a paused handle, libcURL would do the transfert anyway 907 // and we would assert so force defersLoading to be false. 908 handle->m_defersLoading = false; 909 910 initializeHandle(job); 911 912 // curl_easy_perform blocks until the transfert is finished. 913 CURLcode ret = curl_easy_perform(handle->m_handle); 914 915 if (ret != CURLE_OK) { 916 ResourceError error(String(handle->m_url), ret, String(handle->m_url), String(curl_easy_strerror(ret))); 917 error.setSSLErrors(handle->m_sslErrors); 918 handle->client()->didFail(job, error); 919 } else { 920 if (handle->client()) 921 handle->client()->didReceiveResponse(job, handle->m_response); 922 } 923 924#if ENABLE(WEB_TIMING) 925 calculateWebTimingInformations(handle); 926#endif 927 928 curl_easy_cleanup(handle->m_handle); 929} 930 931void ResourceHandleManager::startJob(ResourceHandle* job) 932{ 933 URL kurl = job->firstRequest().url(); 934 935 if (kurl.protocolIsData()) { 936 handleDataURL(job); 937 return; 938 } 939 940 initializeHandle(job); 941 942 m_runningJobs++; 943 CURLMcode ret = curl_multi_add_handle(m_curlMultiHandle, job->getInternal()->m_handle); 944 // don't call perform, because events must be async 945 // timeout will occur and do curl_multi_perform 946 if (ret && ret != CURLM_CALL_MULTI_PERFORM) { 947#ifndef NDEBUG 948 fprintf(stderr, "Error %d starting job %s\n", ret, encodeWithURLEscapeSequences(job->firstRequest().url().string()).latin1().data()); 949#endif 950 job->cancel(); 951 return; 952 } 953} 954 955void ResourceHandleManager::applyAuthenticationToRequest(ResourceHandle* handle, ResourceRequest& request) 956{ 957 // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open(). 958 ResourceHandleInternal* d = handle->getInternal(); 959 960 if (handle->shouldUseCredentialStorage()) { 961 if (d->m_user.isEmpty() && d->m_pass.isEmpty()) { 962 // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication, 963 // try and reuse the credential preemptively, as allowed by RFC 2617. 964 d->m_initialCredential = CredentialStorage::get(request.url()); 965 } else { 966 // If there is already a protection space known for the URL, update stored credentials 967 // before sending a request. This makes it possible to implement logout by sending an 968 // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that 969 // an authentication dialog doesn't pop up). 970 CredentialStorage::set(Credential(d->m_user, d->m_pass, CredentialPersistenceNone), request.url()); 971 } 972 } 973 974 String user = d->m_user; 975 String password = d->m_pass; 976 977 if (!d->m_initialCredential.isEmpty()) { 978 user = d->m_initialCredential.user(); 979 password = d->m_initialCredential.password(); 980 curl_easy_setopt(d->m_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); 981 } 982 983 // It seems we need to set CURLOPT_USERPWD even if username and password is empty. 984 // Otherwise cURL will not automatically continue with a new request after a 401 response. 985 986 // curl CURLOPT_USERPWD expects username:password 987 String userpass = user + ":" + password; 988 curl_easy_setopt(d->m_handle, CURLOPT_USERPWD, userpass.utf8().data()); 989} 990 991void ResourceHandleManager::initializeHandle(ResourceHandle* job) 992{ 993 static const int allowedProtocols = CURLPROTO_FILE | CURLPROTO_FTP | CURLPROTO_FTPS | CURLPROTO_HTTP | CURLPROTO_HTTPS; 994 URL url = job->firstRequest().url(); 995 996 // Remove any fragment part, otherwise curl will send it as part of the request. 997 url.removeFragmentIdentifier(); 998 999 ResourceHandleInternal* d = job->getInternal(); 1000 String urlString = url.string(); 1001 1002 if (url.isLocalFile()) { 1003 // Remove any query part sent to a local file. 1004 if (!url.query().isEmpty()) { 1005 // By setting the query to a null string it'll be removed. 1006 url.setQuery(String()); 1007 urlString = url.string(); 1008 } 1009 // Determine the MIME type based on the path. 1010 d->m_response.setMimeType(MIMETypeRegistry::getMIMETypeForPath(url)); 1011 } 1012 1013 d->m_handle = curl_easy_init(); 1014 1015 if (d->m_defersLoading) { 1016 CURLcode error = curl_easy_pause(d->m_handle, CURLPAUSE_ALL); 1017 // If we did not pause the handle, we would ASSERT in the 1018 // header callback. So just assert here. 1019 ASSERT_UNUSED(error, error == CURLE_OK); 1020 } 1021#ifndef NDEBUG 1022 if (getenv("DEBUG_CURL")) 1023 curl_easy_setopt(d->m_handle, CURLOPT_VERBOSE, 1); 1024#endif 1025 curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYPEER, 1L); 1026 curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYHOST, 2L); 1027 curl_easy_setopt(d->m_handle, CURLOPT_PRIVATE, job); 1028 curl_easy_setopt(d->m_handle, CURLOPT_ERRORBUFFER, m_curlErrorBuffer); 1029 curl_easy_setopt(d->m_handle, CURLOPT_WRITEFUNCTION, writeCallback); 1030 curl_easy_setopt(d->m_handle, CURLOPT_WRITEDATA, job); 1031 curl_easy_setopt(d->m_handle, CURLOPT_HEADERFUNCTION, headerCallback); 1032 curl_easy_setopt(d->m_handle, CURLOPT_WRITEHEADER, job); 1033 curl_easy_setopt(d->m_handle, CURLOPT_AUTOREFERER, 1); 1034 curl_easy_setopt(d->m_handle, CURLOPT_FOLLOWLOCATION, 1); 1035 curl_easy_setopt(d->m_handle, CURLOPT_MAXREDIRS, 10); 1036 curl_easy_setopt(d->m_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY); 1037 curl_easy_setopt(d->m_handle, CURLOPT_SHARE, m_curlShareHandle); 1038 curl_easy_setopt(d->m_handle, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 5); // 5 minutes 1039 curl_easy_setopt(d->m_handle, CURLOPT_PROTOCOLS, allowedProtocols); 1040 curl_easy_setopt(d->m_handle, CURLOPT_REDIR_PROTOCOLS, allowedProtocols); 1041 setSSLClientCertificate(job); 1042 1043 if (ignoreSSLErrors) 1044 curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYPEER, false); 1045 else 1046 setSSLVerifyOptions(job); 1047 1048 if (!m_certificatePath.isNull()) 1049 curl_easy_setopt(d->m_handle, CURLOPT_CAINFO, m_certificatePath.data()); 1050 1051 // enable gzip and deflate through Accept-Encoding: 1052 curl_easy_setopt(d->m_handle, CURLOPT_ENCODING, ""); 1053 1054 // url must remain valid through the request 1055 ASSERT(!d->m_url); 1056 1057 // url is in ASCII so latin1() will only convert it to char* without character translation. 1058 d->m_url = fastStrDup(urlString.latin1().data()); 1059 curl_easy_setopt(d->m_handle, CURLOPT_URL, d->m_url); 1060 1061 if (m_cookieJarFileName) 1062 curl_easy_setopt(d->m_handle, CURLOPT_COOKIEJAR, m_cookieJarFileName); 1063 1064 struct curl_slist* headers = 0; 1065 if (job->firstRequest().httpHeaderFields().size() > 0) { 1066 HTTPHeaderMap customHeaders = job->firstRequest().httpHeaderFields(); 1067 1068 if (CurlCacheManager::getInstance().isCached(url)) { 1069 HTTPHeaderMap& requestHeaders = CurlCacheManager::getInstance().requestHeaders(url); 1070 1071 // append additional cache information 1072 HTTPHeaderMap::const_iterator it = requestHeaders.begin(); 1073 HTTPHeaderMap::const_iterator end = requestHeaders.end(); 1074 while (it != end) { 1075 customHeaders.set(it->key, it->value); 1076 ++it; 1077 } 1078 } 1079 1080 HTTPHeaderMap::const_iterator end = customHeaders.end(); 1081 for (HTTPHeaderMap::const_iterator it = customHeaders.begin(); it != end; ++it) { 1082 String key = it->key; 1083 String value = it->value; 1084 String headerString(key); 1085 if (value.isEmpty()) 1086 // Insert the ; to tell curl that this header has an empty value. 1087 headerString.append(";"); 1088 else { 1089 headerString.append(": "); 1090 headerString.append(value); 1091 } 1092 CString headerLatin1 = headerString.latin1(); 1093 headers = curl_slist_append(headers, headerLatin1.data()); 1094 } 1095 } 1096 1097 String method = job->firstRequest().httpMethod(); 1098 if ("GET" == method) 1099 curl_easy_setopt(d->m_handle, CURLOPT_HTTPGET, TRUE); 1100 else if ("POST" == method) 1101 setupPOST(job, &headers); 1102 else if ("PUT" == method) 1103 setupPUT(job, &headers); 1104 else if ("HEAD" == method) 1105 curl_easy_setopt(d->m_handle, CURLOPT_NOBODY, TRUE); 1106 else { 1107 curl_easy_setopt(d->m_handle, CURLOPT_CUSTOMREQUEST, method.latin1().data()); 1108 setupPUT(job, &headers); 1109 } 1110 1111 if (headers) { 1112 curl_easy_setopt(d->m_handle, CURLOPT_HTTPHEADER, headers); 1113 d->m_customHeaders = headers; 1114 } 1115 1116 applyAuthenticationToRequest(job, job->firstRequest()); 1117 1118 // Set proxy options if we have them. 1119 if (m_proxy.length()) { 1120 curl_easy_setopt(d->m_handle, CURLOPT_PROXY, m_proxy.utf8().data()); 1121 curl_easy_setopt(d->m_handle, CURLOPT_PROXYTYPE, m_proxyType); 1122 } 1123#if ENABLE(WEB_TIMING) 1124 curl_easy_setopt(d->m_handle, CURLOPT_SOCKOPTFUNCTION, sockoptfunction); 1125 curl_easy_setopt(d->m_handle, CURLOPT_SOCKOPTDATA, job); 1126 d->m_response.setResourceLoadTiming(ResourceLoadTiming::create()); 1127#endif 1128} 1129 1130void ResourceHandleManager::initCookieSession() 1131{ 1132 // Curl saves both persistent cookies, and session cookies to the cookie file. 1133 // The session cookies should be deleted before starting a new session. 1134 1135 CURL* curl = curl_easy_init(); 1136 1137 if (!curl) 1138 return; 1139 1140 curl_easy_setopt(curl, CURLOPT_SHARE, m_curlShareHandle); 1141 1142 if (m_cookieJarFileName) { 1143 curl_easy_setopt(curl, CURLOPT_COOKIEFILE, m_cookieJarFileName); 1144 curl_easy_setopt(curl, CURLOPT_COOKIEJAR, m_cookieJarFileName); 1145 } 1146 1147 curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1); 1148 1149 curl_easy_cleanup(curl); 1150} 1151 1152void ResourceHandleManager::cancel(ResourceHandle* job) 1153{ 1154 if (removeScheduledJob(job)) 1155 return; 1156 1157 ResourceHandleInternal* d = job->getInternal(); 1158 d->m_cancelled = true; 1159 if (!m_downloadTimer.isActive()) 1160 m_downloadTimer.startOneShot(pollTimeSeconds); 1161} 1162 1163} // namespace WebCore 1164 1165#endif 1166