1/*
2 * Copyright (C) 2004, 2006 Apple Inc.  All rights reserved.
3 * Copyright (C) 2006 Michael Emmel mike.emmel@gmail.com
4 * Copyright (C) 2007 Alp Toker <alp@atoker.com>
5 * Copyright (C) 2007 Holger Hans Peter Freyther
6 * Copyright (C) 2008 Collabora Ltd.
7 * Copyright (C) 2008 Nuanti Ltd.
8 * Copyright (C) 2009 Appcelerator Inc.
9 * Copyright (C) 2009 Brent Fulgham <bfulgham@webkit.org>
10 * Copyright (C) 2013 Peter Gal <galpeter@inf.u-szeged.hu>, University of Szeged
11 * Copyright (C) 2013 Alex Christensen <achristensen@webkit.org>
12 * Copyright (C) 2013 University of Szeged
13 * All rights reserved.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 *    notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 *    notice, this list of conditions and the following disclaimer in the
22 *    documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
25 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
28 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
29 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
30 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
31 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
32 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 */
36
37#include "config.h"
38#include "ResourceHandleManager.h"
39
40#if USE(CURL)
41
42#include "CredentialStorage.h"
43#include "CurlCacheManager.h"
44#include "DataURL.h"
45#include "HTTPHeaderNames.h"
46#include "HTTPParsers.h"
47#include "MIMETypeRegistry.h"
48#include "MultipartHandle.h"
49#include "ResourceError.h"
50#include "ResourceHandle.h"
51#include "ResourceHandleInternal.h"
52#include "SSLHandle.h"
53
54#if OS(WINDOWS)
55#include "WebCoreBundleWin.h"
56#include <shlobj.h>
57#include <shlwapi.h>
58#else
59#include <sys/param.h>
60#define MAX_PATH MAXPATHLEN
61#endif
62
63#include <errno.h>
64#include <stdio.h>
65#if ENABLE(WEB_TIMING)
66#include <wtf/CurrentTime.h>
67#endif
68#if USE(CF)
69#include <wtf/RetainPtr.h>
70#endif
71#include <wtf/Threading.h>
72#include <wtf/Vector.h>
73#include <wtf/text/CString.h>
74
75
76namespace WebCore {
77
78const int selectTimeoutMS = 5;
79const double pollTimeSeconds = 0.05;
80const int maxRunningJobs = 5;
81
82static const bool ignoreSSLErrors = getenv("WEBKIT_IGNORE_SSL_ERRORS");
83
84static CString certificatePath()
85{
86#if USE(CF)
87    CFBundleRef webKitBundleRef = webKitBundle();
88    if (webKitBundleRef) {
89        RetainPtr<CFURLRef> certURLRef = adoptCF(CFBundleCopyResourceURL(webKitBundleRef, CFSTR("cacert"), CFSTR("pem"), CFSTR("certificates")));
90        if (certURLRef) {
91            char path[MAX_PATH];
92            CFURLGetFileSystemRepresentation(certURLRef.get(), false, reinterpret_cast<UInt8*>(path), MAX_PATH);
93            return path;
94        }
95    }
96#endif
97    char* envPath = getenv("CURL_CA_BUNDLE_PATH");
98    if (envPath)
99       return envPath;
100
101    return CString();
102}
103
104static char* cookieJarPath()
105{
106    char* cookieJarPath = getenv("CURL_COOKIE_JAR_PATH");
107    if (cookieJarPath)
108        return fastStrDup(cookieJarPath);
109
110#if OS(WINDOWS)
111    char executablePath[MAX_PATH];
112    char appDataDirectory[MAX_PATH];
113    char cookieJarFullPath[MAX_PATH];
114    char cookieJarDirectory[MAX_PATH];
115
116    if (FAILED(::SHGetFolderPathA(0, CSIDL_LOCAL_APPDATA | CSIDL_FLAG_CREATE, 0, 0, appDataDirectory))
117        || FAILED(::GetModuleFileNameA(0, executablePath, MAX_PATH)))
118        return fastStrDup("cookies.dat");
119
120    ::PathRemoveExtensionA(executablePath);
121    LPSTR executableName = ::PathFindFileNameA(executablePath);
122    sprintf_s(cookieJarDirectory, MAX_PATH, "%s/%s", appDataDirectory, executableName);
123    sprintf_s(cookieJarFullPath, MAX_PATH, "%s/cookies.dat", cookieJarDirectory);
124
125    if (::SHCreateDirectoryExA(0, cookieJarDirectory, 0) != ERROR_SUCCESS
126        && ::GetLastError() != ERROR_FILE_EXISTS
127        && ::GetLastError() != ERROR_ALREADY_EXISTS)
128        return fastStrDup("cookies.dat");
129
130    return fastStrDup(cookieJarFullPath);
131#else
132    return fastStrDup("cookies.dat");
133#endif
134}
135
136static Mutex* sharedResourceMutex(curl_lock_data data) {
137    DEPRECATED_DEFINE_STATIC_LOCAL(Mutex, cookieMutex, ());
138    DEPRECATED_DEFINE_STATIC_LOCAL(Mutex, dnsMutex, ());
139    DEPRECATED_DEFINE_STATIC_LOCAL(Mutex, shareMutex, ());
140
141    switch (data) {
142        case CURL_LOCK_DATA_COOKIE:
143            return &cookieMutex;
144        case CURL_LOCK_DATA_DNS:
145            return &dnsMutex;
146        case CURL_LOCK_DATA_SHARE:
147            return &shareMutex;
148        default:
149            ASSERT_NOT_REACHED();
150            return NULL;
151    }
152}
153
154#if ENABLE(WEB_TIMING)
155static int milisecondsSinceRequest(double requestTime)
156{
157    return static_cast<int>((monotonicallyIncreasingTime() - requestTime) * 1000.0);
158}
159
160static void calculateWebTimingInformations(ResourceHandleInternal* d)
161{
162    double startTransfertTime = 0;
163    double preTransferTime = 0;
164    double dnslookupTime = 0;
165    double connectTime = 0;
166    double appConnectTime = 0;
167
168    curl_easy_getinfo(d->m_handle, CURLINFO_NAMELOOKUP_TIME, &dnslookupTime);
169    curl_easy_getinfo(d->m_handle, CURLINFO_CONNECT_TIME, &connectTime);
170    curl_easy_getinfo(d->m_handle, CURLINFO_APPCONNECT_TIME, &appConnectTime);
171    curl_easy_getinfo(d->m_handle, CURLINFO_STARTTRANSFER_TIME, &startTransfertTime);
172    curl_easy_getinfo(d->m_handle, CURLINFO_PRETRANSFER_TIME, &preTransferTime);
173
174    d->m_response.resourceLoadTiming()->dnsStart = 0;
175    d->m_response.resourceLoadTiming()->dnsEnd = static_cast<int>(dnslookupTime * 1000);
176
177    d->m_response.resourceLoadTiming()->connectStart = static_cast<int>(dnslookupTime * 1000);
178    d->m_response.resourceLoadTiming()->connectEnd = static_cast<int>(connectTime * 1000);
179
180    d->m_response.resourceLoadTiming()->sendStart = static_cast<int>(connectTime *1000);
181    d->m_response.resourceLoadTiming()->sendEnd =static_cast<int>(preTransferTime * 1000);
182
183    if (appConnectTime) {
184        d->m_response.resourceLoadTiming()->sslStart = static_cast<int>(connectTime * 1000);
185        d->m_response.resourceLoadTiming()->sslEnd = static_cast<int>(appConnectTime *1000);
186    }
187}
188
189static int sockoptfunction(void* data, curl_socket_t /*curlfd*/, curlsocktype /*purpose*/)
190{
191    ResourceHandle* job = static_cast<ResourceHandle*>(data);
192    ResourceHandleInternal* d = job->getInternal();
193
194    if (d->m_response.resourceLoadTiming())
195        d->m_response.resourceLoadTiming()->requestTime = monotonicallyIncreasingTime();
196
197    return 0;
198}
199#endif
200
201// libcurl does not implement its own thread synchronization primitives.
202// these two functions provide mutexes for cookies, and for the global DNS
203// cache.
204static void curl_lock_callback(CURL* /* handle */, curl_lock_data data, curl_lock_access /* access */, void* /* userPtr */)
205{
206    if (Mutex* mutex = sharedResourceMutex(data))
207        mutex->lock();
208}
209
210static void curl_unlock_callback(CURL* /* handle */, curl_lock_data data, void* /* userPtr */)
211{
212    if (Mutex* mutex = sharedResourceMutex(data))
213        mutex->unlock();
214}
215
216inline static bool isHttpInfo(int statusCode)
217{
218    return 100 <= statusCode && statusCode < 200;
219}
220
221inline static bool isHttpRedirect(int statusCode)
222{
223    return 300 <= statusCode && statusCode < 400 && statusCode != 304;
224}
225
226inline static bool isHttpAuthentication(int statusCode)
227{
228    return statusCode == 401;
229}
230
231ResourceHandleManager::ResourceHandleManager()
232    : m_downloadTimer(this, &ResourceHandleManager::downloadTimerCallback)
233    , m_cookieJarFileName(cookieJarPath())
234    , m_certificatePath (certificatePath())
235    , m_runningJobs(0)
236{
237    curl_global_init(CURL_GLOBAL_ALL);
238    m_curlMultiHandle = curl_multi_init();
239    m_curlShareHandle = curl_share_init();
240    curl_share_setopt(m_curlShareHandle, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
241    curl_share_setopt(m_curlShareHandle, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
242    curl_share_setopt(m_curlShareHandle, CURLSHOPT_LOCKFUNC, curl_lock_callback);
243    curl_share_setopt(m_curlShareHandle, CURLSHOPT_UNLOCKFUNC, curl_unlock_callback);
244
245    initCookieSession();
246}
247
248ResourceHandleManager::~ResourceHandleManager()
249{
250    curl_multi_cleanup(m_curlMultiHandle);
251    curl_share_cleanup(m_curlShareHandle);
252    if (m_cookieJarFileName)
253        fastFree(m_cookieJarFileName);
254    curl_global_cleanup();
255}
256
257CURLSH* ResourceHandleManager::getCurlShareHandle() const
258{
259    return m_curlShareHandle;
260}
261
262void ResourceHandleManager::setCookieJarFileName(const char* cookieJarFileName)
263{
264    m_cookieJarFileName = fastStrDup(cookieJarFileName);
265}
266
267const char* ResourceHandleManager::getCookieJarFileName() const
268{
269    return m_cookieJarFileName;
270}
271
272ResourceHandleManager* ResourceHandleManager::sharedInstance()
273{
274    static ResourceHandleManager* sharedInstance = 0;
275    if (!sharedInstance)
276        sharedInstance = new ResourceHandleManager();
277    return sharedInstance;
278}
279
280static void handleLocalReceiveResponse (CURL* handle, ResourceHandle* job, ResourceHandleInternal* d)
281{
282    // since the code in headerCallback will not have run for local files
283    // the code to set the URL and fire didReceiveResponse is never run,
284    // which means the ResourceLoader's response does not contain the URL.
285    // Run the code here for local files to resolve the issue.
286    // TODO: See if there is a better approach for handling this.
287     const char* hdr;
288     CURLcode err = curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &hdr);
289     ASSERT_UNUSED(err, CURLE_OK == err);
290     d->m_response.setURL(URL(ParsedURLString, hdr));
291     if (d->client())
292         d->client()->didReceiveResponse(job, d->m_response);
293     d->m_response.setResponseFired(true);
294}
295
296
297// called with data after all headers have been processed via headerCallback
298static size_t writeCallback(void* ptr, size_t size, size_t nmemb, void* data)
299{
300    ResourceHandle* job = static_cast<ResourceHandle*>(data);
301    ResourceHandleInternal* d = job->getInternal();
302    if (d->m_cancelled)
303        return 0;
304
305    // We should never be called when deferred loading is activated.
306    ASSERT(!d->m_defersLoading);
307
308    size_t totalSize = size * nmemb;
309
310    // this shouldn't be necessary but apparently is. CURL writes the data
311    // of html page even if it is a redirect that was handled internally
312    // can be observed e.g. on gmail.com
313    CURL* h = d->m_handle;
314    long httpCode = 0;
315    CURLcode err = curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
316    if (CURLE_OK == err && httpCode >= 300 && httpCode < 400)
317        return totalSize;
318
319    if (!d->m_response.responseFired()) {
320        handleLocalReceiveResponse(h, job, d);
321        if (d->m_cancelled)
322            return 0;
323    }
324
325    if (d->m_multipartHandle)
326        d->m_multipartHandle->contentReceived(static_cast<const char*>(ptr), totalSize);
327    else if (d->client()) {
328        d->client()->didReceiveData(job, static_cast<char*>(ptr), totalSize, 0);
329        CurlCacheManager::getInstance().didReceiveData(*job, static_cast<char*>(ptr), totalSize);
330    }
331
332    return totalSize;
333}
334
335static bool isAppendableHeader(const String &key)
336{
337    static const char* appendableHeaders[] = {
338        "access-control-allow-headers",
339        "access-control-allow-methods",
340        "access-control-allow-origin",
341        "access-control-expose-headers",
342        "allow",
343        "cache-control",
344        "connection",
345        "content-encoding",
346        "content-language",
347        "if-match",
348        "if-none-match",
349        "keep-alive",
350        "pragma",
351        "proxy-authenticate",
352        "public",
353        "server",
354        "set-cookie",
355        "te",
356        "trailer",
357        "transfer-encoding",
358        "upgrade",
359        "user-agent",
360        "vary",
361        "via",
362        "warning",
363        "www-authenticate",
364        0
365    };
366
367    // Custom headers start with 'X-', and need no further checking.
368    if (key.startsWith("x-", /* caseSensitive */ false))
369        return true;
370
371    for (unsigned i = 0; appendableHeaders[i]; ++i)
372        if (equalIgnoringCase(key, appendableHeaders[i]))
373            return true;
374
375    return false;
376}
377
378static void removeLeadingAndTrailingQuotes(String& value)
379{
380    unsigned length = value.length();
381    if (value.startsWith('"') && value.endsWith('"') && length > 1)
382        value = value.substring(1, length-2);
383}
384
385static bool getProtectionSpace(CURL* h, const ResourceResponse& response, ProtectionSpace& protectionSpace)
386{
387    CURLcode err;
388
389    long port = 0;
390    err = curl_easy_getinfo(h, CURLINFO_PRIMARY_PORT, &port);
391    if (err != CURLE_OK)
392        return false;
393
394    long availableAuth = CURLAUTH_NONE;
395    err = curl_easy_getinfo(h, CURLINFO_HTTPAUTH_AVAIL, &availableAuth);
396    if (err != CURLE_OK)
397        return false;
398
399    const char* effectiveUrl = 0;
400    err = curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &effectiveUrl);
401    if (err != CURLE_OK)
402        return false;
403
404    URL url(ParsedURLString, effectiveUrl);
405
406    String host = url.host();
407    String protocol = url.protocol();
408
409    String realm;
410
411    const String authHeader = response.httpHeaderField(HTTPHeaderName::Authorization);
412    const String realmString = "realm=";
413    int realmPos = authHeader.find(realmString);
414    if (realmPos > 0) {
415        realm = authHeader.substring(realmPos + realmString.length());
416        realm = realm.left(realm.find(','));
417        removeLeadingAndTrailingQuotes(realm);
418    }
419
420    ProtectionSpaceServerType serverType = ProtectionSpaceServerHTTP;
421    if (protocol == "https")
422        serverType = ProtectionSpaceServerHTTPS;
423
424    ProtectionSpaceAuthenticationScheme authScheme = ProtectionSpaceAuthenticationSchemeUnknown;
425
426    if (availableAuth & CURLAUTH_BASIC)
427        authScheme = ProtectionSpaceAuthenticationSchemeHTTPBasic;
428    if (availableAuth & CURLAUTH_DIGEST)
429        authScheme = ProtectionSpaceAuthenticationSchemeHTTPDigest;
430    if (availableAuth & CURLAUTH_GSSNEGOTIATE)
431        authScheme = ProtectionSpaceAuthenticationSchemeNegotiate;
432    if (availableAuth & CURLAUTH_NTLM)
433        authScheme = ProtectionSpaceAuthenticationSchemeNTLM;
434
435    protectionSpace = ProtectionSpace(host, port, serverType, realm, authScheme);
436
437    return true;
438}
439
440/*
441 * This is being called for each HTTP header in the response. This includes '\r\n'
442 * for the last line of the header.
443 *
444 * We will add each HTTP Header to the ResourceResponse and on the termination
445 * of the header (\r\n) we will parse Content-Type and Content-Disposition and
446 * update the ResourceResponse and then send it away.
447 *
448 */
449static size_t headerCallback(char* ptr, size_t size, size_t nmemb, void* data)
450{
451    ResourceHandle* job = static_cast<ResourceHandle*>(data);
452    ResourceHandleInternal* d = job->getInternal();
453    if (d->m_cancelled)
454        return 0;
455
456    // We should never be called when deferred loading is activated.
457    ASSERT(!d->m_defersLoading);
458
459    size_t totalSize = size * nmemb;
460    ResourceHandleClient* client = d->client();
461
462    String header = String::fromUTF8WithLatin1Fallback(static_cast<const char*>(ptr), totalSize);
463
464    /*
465     * a) We can finish and send the ResourceResponse
466     * b) We will add the current header to the HTTPHeaderMap of the ResourceResponse
467     *
468     * The HTTP standard requires to use \r\n but for compatibility it recommends to
469     * accept also \n.
470     */
471    if (header == String("\r\n") || header == String("\n")) {
472        CURL* h = d->m_handle;
473
474        long httpCode = 0;
475        curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
476
477        if (isHttpInfo(httpCode)) {
478            // Just return when receiving http info, e.g. HTTP/1.1 100 Continue.
479            // If not, the request might be cancelled, because the MIME type will be empty for this response.
480            return totalSize;
481        }
482
483        double contentLength = 0;
484        curl_easy_getinfo(h, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength);
485        d->m_response.setExpectedContentLength(static_cast<long long int>(contentLength));
486
487        const char* hdr;
488        curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &hdr);
489        d->m_response.setURL(URL(ParsedURLString, hdr));
490
491        d->m_response.setHTTPStatusCode(httpCode);
492        d->m_response.setMimeType(extractMIMETypeFromMediaType(d->m_response.httpHeaderField(HTTPHeaderName::ContentType)).lower());
493        d->m_response.setTextEncodingName(extractCharsetFromMediaType(d->m_response.httpHeaderField(HTTPHeaderName::ContentType)));
494        d->m_response.setSuggestedFilename(filenameFromHTTPContentDisposition(d->m_response.httpHeaderField(HTTPHeaderName::ContentDisposition)));
495
496        if (d->m_response.isMultipart()) {
497            String boundary;
498            bool parsed = MultipartHandle::extractBoundary(d->m_response.httpHeaderField(HTTPHeaderName::ContentType), boundary);
499            if (parsed)
500                d->m_multipartHandle = MultipartHandle::create(job, boundary);
501        }
502
503#if ENABLE(WEB_TIMING)
504        if (d->m_response.resourceLoadTiming() && d->m_response.resourceLoadTiming()->requestTime)
505            d->m_response.resourceLoadTiming()->receiveHeadersEnd = milisecondsSinceRequest(d->m_response.resourceLoadTiming()->requestTime);
506#endif
507
508        // HTTP redirection
509        if (isHttpRedirect(httpCode)) {
510            String location = d->m_response.httpHeaderField(HTTPHeaderName::Location);
511            if (!location.isEmpty()) {
512                URL newURL = URL(job->firstRequest().url(), location);
513
514                ResourceRequest redirectedRequest = job->firstRequest();
515                redirectedRequest.setURL(newURL);
516                if (client)
517                    client->willSendRequest(job, redirectedRequest, d->m_response);
518
519                d->m_firstRequest.setURL(newURL);
520
521                return totalSize;
522            }
523        } else if (isHttpAuthentication(httpCode)) {
524            ProtectionSpace protectionSpace;
525            if (getProtectionSpace(d->m_handle, d->m_response, protectionSpace)) {
526                Credential credential;
527                AuthenticationChallenge challenge(protectionSpace, credential, d->m_authFailureCount, d->m_response, ResourceError());
528                challenge.setAuthenticationClient(job);
529                job->didReceiveAuthenticationChallenge(challenge);
530                d->m_authFailureCount++;
531                return totalSize;
532            }
533        }
534
535        if (client) {
536            if (httpCode == 304) {
537                const String& url = job->firstRequest().url().string();
538                CurlCacheManager::getInstance().getCachedResponse(url, d->m_response);
539            }
540            client->didReceiveResponse(job, d->m_response);
541            CurlCacheManager::getInstance().didReceiveResponse(*job, d->m_response);
542        }
543        d->m_response.setResponseFired(true);
544
545    } else {
546        int splitPos = header.find(":");
547        if (splitPos != -1) {
548            String key = header.left(splitPos).stripWhiteSpace();
549            String value = header.substring(splitPos + 1).stripWhiteSpace();
550
551            if (isAppendableHeader(key))
552                d->m_response.addHTTPHeaderField(key, value);
553            else
554                d->m_response.setHTTPHeaderField(key, value);
555        } else if (header.startsWith("HTTP", false)) {
556            // This is the first line of the response.
557            // Extract the http status text from this.
558            //
559            // If the FOLLOWLOCATION option is enabled for the curl handle then
560            // curl will follow the redirections internally. Thus this header callback
561            // will be called more than one time with the line starting "HTTP" for one job.
562            long httpCode = 0;
563            curl_easy_getinfo(d->m_handle, CURLINFO_RESPONSE_CODE, &httpCode);
564
565            String httpCodeString = String::number(httpCode);
566            int statusCodePos = header.find(httpCodeString);
567
568            if (statusCodePos != -1) {
569                // The status text is after the status code.
570                String status = header.substring(statusCodePos + httpCodeString.length());
571                d->m_response.setHTTPStatusText(status.stripWhiteSpace());
572            }
573
574        }
575    }
576
577    return totalSize;
578}
579
580/* This is called to obtain HTTP POST or PUT data.
581   Iterate through FormData elements and upload files.
582   Carefully respect the given buffer size and fill the rest of the data at the next calls.
583*/
584size_t readCallback(void* ptr, size_t size, size_t nmemb, void* data)
585{
586    ResourceHandle* job = static_cast<ResourceHandle*>(data);
587    ResourceHandleInternal* d = job->getInternal();
588
589    if (d->m_cancelled)
590        return 0;
591
592    // We should never be called when deferred loading is activated.
593    ASSERT(!d->m_defersLoading);
594
595    if (!size || !nmemb)
596        return 0;
597
598    if (!d->m_formDataStream.hasMoreElements())
599        return 0;
600
601    size_t sent = d->m_formDataStream.read(ptr, size, nmemb);
602
603    // Something went wrong so cancel the job.
604    if (!sent)
605        job->cancel();
606
607    return sent;
608}
609
610void ResourceHandleManager::downloadTimerCallback(Timer<ResourceHandleManager>* /* timer */)
611{
612    startScheduledJobs();
613
614    fd_set fdread;
615    fd_set fdwrite;
616    fd_set fdexcep;
617    int maxfd = 0;
618
619    struct timeval timeout;
620    timeout.tv_sec = 0;
621    timeout.tv_usec = selectTimeoutMS * 1000;       // select waits microseconds
622
623    // Retry 'select' if it was interrupted by a process signal.
624    int rc = 0;
625    do {
626        FD_ZERO(&fdread);
627        FD_ZERO(&fdwrite);
628        FD_ZERO(&fdexcep);
629        curl_multi_fdset(m_curlMultiHandle, &fdread, &fdwrite, &fdexcep, &maxfd);
630        // When the 3 file descriptors are empty, winsock will return -1
631        // and bail out, stopping the file download. So make sure we
632        // have valid file descriptors before calling select.
633        if (maxfd >= 0)
634            rc = ::select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
635    } while (rc == -1 && errno == EINTR);
636
637    if (-1 == rc) {
638#ifndef NDEBUG
639        perror("bad: select() returned -1: ");
640#endif
641        return;
642    }
643
644    int runningHandles = 0;
645    while (curl_multi_perform(m_curlMultiHandle, &runningHandles) == CURLM_CALL_MULTI_PERFORM) { }
646
647    // check the curl messages indicating completed transfers
648    // and free their resources
649    while (true) {
650        int messagesInQueue;
651        CURLMsg* msg = curl_multi_info_read(m_curlMultiHandle, &messagesInQueue);
652        if (!msg)
653            break;
654
655        // find the node which has same d->m_handle as completed transfer
656        CURL* handle = msg->easy_handle;
657        ASSERT(handle);
658        ResourceHandle* job = 0;
659        CURLcode err = curl_easy_getinfo(handle, CURLINFO_PRIVATE, &job);
660        ASSERT_UNUSED(err, CURLE_OK == err);
661        ASSERT(job);
662        if (!job)
663            continue;
664        ResourceHandleInternal* d = job->getInternal();
665        ASSERT(d->m_handle == handle);
666
667        if (d->m_cancelled) {
668            removeFromCurl(job);
669            continue;
670        }
671
672        if (CURLMSG_DONE != msg->msg)
673            continue;
674
675
676        if (CURLE_OK == msg->data.result) {
677#if ENABLE(WEB_TIMING)
678            calculateWebTimingInformations(d);
679#endif
680            if (!d->m_response.responseFired()) {
681                handleLocalReceiveResponse(d->m_handle, job, d);
682                if (d->m_cancelled) {
683                    removeFromCurl(job);
684                    continue;
685                }
686            }
687
688            if (d->m_multipartHandle)
689                d->m_multipartHandle->contentEnded();
690
691            if (d->client()) {
692                d->client()->didFinishLoading(job, 0);
693                CurlCacheManager::getInstance().didFinishLoading(*job);
694            }
695        } else {
696            char* url = 0;
697            curl_easy_getinfo(d->m_handle, CURLINFO_EFFECTIVE_URL, &url);
698#ifndef NDEBUG
699            fprintf(stderr, "Curl ERROR for url='%s', error: '%s'\n", url, curl_easy_strerror(msg->data.result));
700#endif
701            if (d->client()) {
702                ResourceError resourceError(String(), msg->data.result, String(url), String(curl_easy_strerror(msg->data.result)));
703                resourceError.setSSLErrors(d->m_sslErrors);
704                d->client()->didFail(job, resourceError);
705                CurlCacheManager::getInstance().didFail(*job);
706            }
707        }
708
709        removeFromCurl(job);
710    }
711
712    bool started = startScheduledJobs(); // new jobs might have been added in the meantime
713
714    if (!m_downloadTimer.isActive() && (started || (runningHandles > 0)))
715        m_downloadTimer.startOneShot(pollTimeSeconds);
716}
717
718void ResourceHandleManager::setProxyInfo(const String& host,
719                                         unsigned long port,
720                                         ProxyType type,
721                                         const String& username,
722                                         const String& password)
723{
724    m_proxyType = type;
725
726    if (!host.length()) {
727        m_proxy = emptyString();
728    } else {
729        String userPass;
730        if (username.length() || password.length())
731            userPass = username + ":" + password + "@";
732
733        m_proxy = String("http://") + userPass + host + ":" + String::number(port);
734    }
735}
736
737void ResourceHandleManager::removeFromCurl(ResourceHandle* job)
738{
739    ResourceHandleInternal* d = job->getInternal();
740    ASSERT(d->m_handle);
741    if (!d->m_handle)
742        return;
743    m_runningJobs--;
744    curl_multi_remove_handle(m_curlMultiHandle, d->m_handle);
745    curl_easy_cleanup(d->m_handle);
746    d->m_handle = 0;
747    job->deref();
748}
749
750static inline size_t getFormElementsCount(ResourceHandle* job)
751{
752    RefPtr<FormData> formData = job->firstRequest().httpBody();
753
754    if (!formData)
755        return 0;
756
757    // Resolve the blob elements so the formData can correctly report it's size.
758    formData = formData->resolveBlobReferences();
759    job->firstRequest().setHTTPBody(formData);
760
761    return formData->elements().size();
762}
763
764static void setupFormData(ResourceHandle* job, CURLoption sizeOption, struct curl_slist** headers)
765{
766    ResourceHandleInternal* d = job->getInternal();
767    Vector<FormDataElement> elements = job->firstRequest().httpBody()->elements();
768    size_t numElements = elements.size();
769
770    // The size of a curl_off_t could be different in WebKit and in cURL depending on
771    // compilation flags of both.
772    static int expectedSizeOfCurlOffT = 0;
773    if (!expectedSizeOfCurlOffT) {
774        curl_version_info_data *infoData = curl_version_info(CURLVERSION_NOW);
775        if (infoData->features & CURL_VERSION_LARGEFILE)
776            expectedSizeOfCurlOffT = sizeof(long long);
777        else
778            expectedSizeOfCurlOffT = sizeof(int);
779    }
780
781    static const long long maxCurlOffT = (1LL << (expectedSizeOfCurlOffT * 8 - 1)) - 1;
782    // Obtain the total size of the form data
783    curl_off_t size = 0;
784    bool chunkedTransfer = false;
785    for (size_t i = 0; i < numElements; i++) {
786        FormDataElement element = elements[i];
787        if (element.m_type == FormDataElement::Type::EncodedFile) {
788            long long fileSizeResult;
789            if (getFileSize(element.m_filename, fileSizeResult)) {
790                if (fileSizeResult > maxCurlOffT) {
791                    // File size is too big for specifying it to cURL
792                    chunkedTransfer = true;
793                    break;
794                }
795                size += fileSizeResult;
796            } else {
797                chunkedTransfer = true;
798                break;
799            }
800        } else
801            size += elements[i].m_data.size();
802    }
803
804    // cURL guesses that we want chunked encoding as long as we specify the header
805    if (chunkedTransfer)
806        *headers = curl_slist_append(*headers, "Transfer-Encoding: chunked");
807    else {
808        if (sizeof(long long) == expectedSizeOfCurlOffT)
809            curl_easy_setopt(d->m_handle, sizeOption, (long long)size);
810        else
811            curl_easy_setopt(d->m_handle, sizeOption, (int)size);
812    }
813
814    curl_easy_setopt(d->m_handle, CURLOPT_READFUNCTION, readCallback);
815    curl_easy_setopt(d->m_handle, CURLOPT_READDATA, job);
816}
817
818void ResourceHandleManager::setupPUT(ResourceHandle* job, struct curl_slist** headers)
819{
820    ResourceHandleInternal* d = job->getInternal();
821    curl_easy_setopt(d->m_handle, CURLOPT_UPLOAD, TRUE);
822    curl_easy_setopt(d->m_handle, CURLOPT_INFILESIZE, 0);
823
824    // Disable the Expect: 100 continue header
825    *headers = curl_slist_append(*headers, "Expect:");
826
827    size_t numElements = getFormElementsCount(job);
828    if (!numElements)
829        return;
830
831    setupFormData(job, CURLOPT_INFILESIZE_LARGE, headers);
832}
833
834void ResourceHandleManager::setupPOST(ResourceHandle* job, struct curl_slist** headers)
835{
836    ResourceHandleInternal* d = job->getInternal();
837    curl_easy_setopt(d->m_handle, CURLOPT_POST, TRUE);
838    curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDSIZE, 0);
839
840    size_t numElements = getFormElementsCount(job);
841    if (!numElements)
842        return;
843
844    // Do not stream for simple POST data
845    if (numElements == 1) {
846        job->firstRequest().httpBody()->flatten(d->m_postBytes);
847        if (d->m_postBytes.size()) {
848            curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDSIZE, d->m_postBytes.size());
849            curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDS, d->m_postBytes.data());
850        }
851        return;
852    }
853
854    setupFormData(job, CURLOPT_POSTFIELDSIZE_LARGE, headers);
855}
856
857void ResourceHandleManager::add(ResourceHandle* job)
858{
859    // we can be called from within curl, so to avoid re-entrancy issues
860    // schedule this job to be added the next time we enter curl download loop
861    job->ref();
862    m_resourceHandleList.append(job);
863    if (!m_downloadTimer.isActive())
864        m_downloadTimer.startOneShot(pollTimeSeconds);
865}
866
867bool ResourceHandleManager::removeScheduledJob(ResourceHandle* job)
868{
869    int size = m_resourceHandleList.size();
870    for (int i = 0; i < size; i++) {
871        if (job == m_resourceHandleList[i]) {
872            m_resourceHandleList.remove(i);
873            job->deref();
874            return true;
875        }
876    }
877    return false;
878}
879
880bool ResourceHandleManager::startScheduledJobs()
881{
882    // TODO: Create a separate stack of jobs for each domain.
883
884    bool started = false;
885    while (!m_resourceHandleList.isEmpty() && m_runningJobs < maxRunningJobs) {
886        ResourceHandle* job = m_resourceHandleList[0];
887        m_resourceHandleList.remove(0);
888        startJob(job);
889        started = true;
890    }
891    return started;
892}
893
894void ResourceHandleManager::dispatchSynchronousJob(ResourceHandle* job)
895{
896    URL kurl = job->firstRequest().url();
897
898    if (kurl.protocolIsData()) {
899        handleDataURL(job);
900        return;
901    }
902
903    ResourceHandleInternal* handle = job->getInternal();
904
905    // If defersLoading is true and we call curl_easy_perform
906    // on a paused handle, libcURL would do the transfert anyway
907    // and we would assert so force defersLoading to be false.
908    handle->m_defersLoading = false;
909
910    initializeHandle(job);
911
912    // curl_easy_perform blocks until the transfert is finished.
913    CURLcode ret =  curl_easy_perform(handle->m_handle);
914
915    if (ret != CURLE_OK) {
916        ResourceError error(String(handle->m_url), ret, String(handle->m_url), String(curl_easy_strerror(ret)));
917        error.setSSLErrors(handle->m_sslErrors);
918        handle->client()->didFail(job, error);
919    } else {
920        if (handle->client())
921            handle->client()->didReceiveResponse(job, handle->m_response);
922    }
923
924#if ENABLE(WEB_TIMING)
925    calculateWebTimingInformations(handle);
926#endif
927
928    curl_easy_cleanup(handle->m_handle);
929}
930
931void ResourceHandleManager::startJob(ResourceHandle* job)
932{
933    URL kurl = job->firstRequest().url();
934
935    if (kurl.protocolIsData()) {
936        handleDataURL(job);
937        return;
938    }
939
940    initializeHandle(job);
941
942    m_runningJobs++;
943    CURLMcode ret = curl_multi_add_handle(m_curlMultiHandle, job->getInternal()->m_handle);
944    // don't call perform, because events must be async
945    // timeout will occur and do curl_multi_perform
946    if (ret && ret != CURLM_CALL_MULTI_PERFORM) {
947#ifndef NDEBUG
948        fprintf(stderr, "Error %d starting job %s\n", ret, encodeWithURLEscapeSequences(job->firstRequest().url().string()).latin1().data());
949#endif
950        job->cancel();
951        return;
952    }
953}
954
955void ResourceHandleManager::applyAuthenticationToRequest(ResourceHandle* handle, ResourceRequest& request)
956{
957    // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open().
958    ResourceHandleInternal* d = handle->getInternal();
959
960    if (handle->shouldUseCredentialStorage()) {
961        if (d->m_user.isEmpty() && d->m_pass.isEmpty()) {
962            // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication,
963            // try and reuse the credential preemptively, as allowed by RFC 2617.
964            d->m_initialCredential = CredentialStorage::get(request.url());
965        } else {
966            // If there is already a protection space known for the URL, update stored credentials
967            // before sending a request. This makes it possible to implement logout by sending an
968            // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that
969            // an authentication dialog doesn't pop up).
970            CredentialStorage::set(Credential(d->m_user, d->m_pass, CredentialPersistenceNone), request.url());
971        }
972    }
973
974    String user = d->m_user;
975    String password = d->m_pass;
976
977    if (!d->m_initialCredential.isEmpty()) {
978        user = d->m_initialCredential.user();
979        password = d->m_initialCredential.password();
980        curl_easy_setopt(d->m_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
981    }
982
983    // It seems we need to set CURLOPT_USERPWD even if username and password is empty.
984    // Otherwise cURL will not automatically continue with a new request after a 401 response.
985
986    // curl CURLOPT_USERPWD expects username:password
987    String userpass = user + ":" + password;
988    curl_easy_setopt(d->m_handle, CURLOPT_USERPWD, userpass.utf8().data());
989}
990
991void ResourceHandleManager::initializeHandle(ResourceHandle* job)
992{
993    static const int allowedProtocols = CURLPROTO_FILE | CURLPROTO_FTP | CURLPROTO_FTPS | CURLPROTO_HTTP | CURLPROTO_HTTPS;
994    URL url = job->firstRequest().url();
995
996    // Remove any fragment part, otherwise curl will send it as part of the request.
997    url.removeFragmentIdentifier();
998
999    ResourceHandleInternal* d = job->getInternal();
1000    String urlString = url.string();
1001
1002    if (url.isLocalFile()) {
1003        // Remove any query part sent to a local file.
1004        if (!url.query().isEmpty()) {
1005            // By setting the query to a null string it'll be removed.
1006            url.setQuery(String());
1007            urlString = url.string();
1008        }
1009        // Determine the MIME type based on the path.
1010        d->m_response.setMimeType(MIMETypeRegistry::getMIMETypeForPath(url));
1011    }
1012
1013    d->m_handle = curl_easy_init();
1014
1015    if (d->m_defersLoading) {
1016        CURLcode error = curl_easy_pause(d->m_handle, CURLPAUSE_ALL);
1017        // If we did not pause the handle, we would ASSERT in the
1018        // header callback. So just assert here.
1019        ASSERT_UNUSED(error, error == CURLE_OK);
1020    }
1021#ifndef NDEBUG
1022    if (getenv("DEBUG_CURL"))
1023        curl_easy_setopt(d->m_handle, CURLOPT_VERBOSE, 1);
1024#endif
1025    curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1026    curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYHOST, 2L);
1027    curl_easy_setopt(d->m_handle, CURLOPT_PRIVATE, job);
1028    curl_easy_setopt(d->m_handle, CURLOPT_ERRORBUFFER, m_curlErrorBuffer);
1029    curl_easy_setopt(d->m_handle, CURLOPT_WRITEFUNCTION, writeCallback);
1030    curl_easy_setopt(d->m_handle, CURLOPT_WRITEDATA, job);
1031    curl_easy_setopt(d->m_handle, CURLOPT_HEADERFUNCTION, headerCallback);
1032    curl_easy_setopt(d->m_handle, CURLOPT_WRITEHEADER, job);
1033    curl_easy_setopt(d->m_handle, CURLOPT_AUTOREFERER, 1);
1034    curl_easy_setopt(d->m_handle, CURLOPT_FOLLOWLOCATION, 1);
1035    curl_easy_setopt(d->m_handle, CURLOPT_MAXREDIRS, 10);
1036    curl_easy_setopt(d->m_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
1037    curl_easy_setopt(d->m_handle, CURLOPT_SHARE, m_curlShareHandle);
1038    curl_easy_setopt(d->m_handle, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 5); // 5 minutes
1039    curl_easy_setopt(d->m_handle, CURLOPT_PROTOCOLS, allowedProtocols);
1040    curl_easy_setopt(d->m_handle, CURLOPT_REDIR_PROTOCOLS, allowedProtocols);
1041    setSSLClientCertificate(job);
1042
1043    if (ignoreSSLErrors)
1044        curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYPEER, false);
1045    else
1046        setSSLVerifyOptions(job);
1047
1048    if (!m_certificatePath.isNull())
1049       curl_easy_setopt(d->m_handle, CURLOPT_CAINFO, m_certificatePath.data());
1050
1051    // enable gzip and deflate through Accept-Encoding:
1052    curl_easy_setopt(d->m_handle, CURLOPT_ENCODING, "");
1053
1054    // url must remain valid through the request
1055    ASSERT(!d->m_url);
1056
1057    // url is in ASCII so latin1() will only convert it to char* without character translation.
1058    d->m_url = fastStrDup(urlString.latin1().data());
1059    curl_easy_setopt(d->m_handle, CURLOPT_URL, d->m_url);
1060
1061    if (m_cookieJarFileName)
1062        curl_easy_setopt(d->m_handle, CURLOPT_COOKIEJAR, m_cookieJarFileName);
1063
1064    struct curl_slist* headers = 0;
1065    if (job->firstRequest().httpHeaderFields().size() > 0) {
1066        HTTPHeaderMap customHeaders = job->firstRequest().httpHeaderFields();
1067
1068        if (CurlCacheManager::getInstance().isCached(url)) {
1069            HTTPHeaderMap& requestHeaders = CurlCacheManager::getInstance().requestHeaders(url);
1070
1071            // append additional cache information
1072            HTTPHeaderMap::const_iterator it = requestHeaders.begin();
1073            HTTPHeaderMap::const_iterator end = requestHeaders.end();
1074            while (it != end) {
1075                customHeaders.set(it->key, it->value);
1076                ++it;
1077            }
1078        }
1079
1080        HTTPHeaderMap::const_iterator end = customHeaders.end();
1081        for (HTTPHeaderMap::const_iterator it = customHeaders.begin(); it != end; ++it) {
1082            String key = it->key;
1083            String value = it->value;
1084            String headerString(key);
1085            if (value.isEmpty())
1086                // Insert the ; to tell curl that this header has an empty value.
1087                headerString.append(";");
1088            else {
1089                headerString.append(": ");
1090                headerString.append(value);
1091            }
1092            CString headerLatin1 = headerString.latin1();
1093            headers = curl_slist_append(headers, headerLatin1.data());
1094        }
1095    }
1096
1097    String method = job->firstRequest().httpMethod();
1098    if ("GET" == method)
1099        curl_easy_setopt(d->m_handle, CURLOPT_HTTPGET, TRUE);
1100    else if ("POST" == method)
1101        setupPOST(job, &headers);
1102    else if ("PUT" == method)
1103        setupPUT(job, &headers);
1104    else if ("HEAD" == method)
1105        curl_easy_setopt(d->m_handle, CURLOPT_NOBODY, TRUE);
1106    else {
1107        curl_easy_setopt(d->m_handle, CURLOPT_CUSTOMREQUEST, method.latin1().data());
1108        setupPUT(job, &headers);
1109    }
1110
1111    if (headers) {
1112        curl_easy_setopt(d->m_handle, CURLOPT_HTTPHEADER, headers);
1113        d->m_customHeaders = headers;
1114    }
1115
1116    applyAuthenticationToRequest(job, job->firstRequest());
1117
1118    // Set proxy options if we have them.
1119    if (m_proxy.length()) {
1120        curl_easy_setopt(d->m_handle, CURLOPT_PROXY, m_proxy.utf8().data());
1121        curl_easy_setopt(d->m_handle, CURLOPT_PROXYTYPE, m_proxyType);
1122    }
1123#if ENABLE(WEB_TIMING)
1124    curl_easy_setopt(d->m_handle, CURLOPT_SOCKOPTFUNCTION, sockoptfunction);
1125    curl_easy_setopt(d->m_handle, CURLOPT_SOCKOPTDATA, job);
1126    d->m_response.setResourceLoadTiming(ResourceLoadTiming::create());
1127#endif
1128}
1129
1130void ResourceHandleManager::initCookieSession()
1131{
1132    // Curl saves both persistent cookies, and session cookies to the cookie file.
1133    // The session cookies should be deleted before starting a new session.
1134
1135    CURL* curl = curl_easy_init();
1136
1137    if (!curl)
1138        return;
1139
1140    curl_easy_setopt(curl, CURLOPT_SHARE, m_curlShareHandle);
1141
1142    if (m_cookieJarFileName) {
1143        curl_easy_setopt(curl, CURLOPT_COOKIEFILE, m_cookieJarFileName);
1144        curl_easy_setopt(curl, CURLOPT_COOKIEJAR, m_cookieJarFileName);
1145    }
1146
1147    curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1);
1148
1149    curl_easy_cleanup(curl);
1150}
1151
1152void ResourceHandleManager::cancel(ResourceHandle* job)
1153{
1154    if (removeScheduledJob(job))
1155        return;
1156
1157    ResourceHandleInternal* d = job->getInternal();
1158    d->m_cancelled = true;
1159    if (!m_downloadTimer.isActive())
1160        m_downloadTimer.startOneShot(pollTimeSeconds);
1161}
1162
1163} // namespace WebCore
1164
1165#endif
1166