ResourceHandleManager.cpp [plain text]
#include "config.h"
#include "ResourceHandleManager.h"
#if USE(CURL)
#include "CredentialStorage.h"
#include "CurlCacheManager.h"
#include "HTTPHeaderNames.h"
#include "HTTPParsers.h"
#include "MIMETypeRegistry.h"
#include "MultipartHandle.h"
#include "ResourceError.h"
#include "ResourceHandle.h"
#include "ResourceHandleClient.h"
#include "ResourceHandleInternal.h"
#include "ResourceRequest.h"
#include "ResourceResponse.h"
#include "SSLHandle.h"
#include "TextEncoding.h"
#include <wtf/text/Base64.h>
#include <wtf/text/CString.h>
#include <wtf/text/StringView.h>
#if OS(WINDOWS)
#include "WebCoreBundleWin.h"
#include <shlobj.h>
#include <shlwapi.h>
#else
#include <sys/param.h>
#define MAX_PATH MAXPATHLEN
#endif
#include <errno.h>
#include <stdio.h>
#if ENABLE(WEB_TIMING)
#include <wtf/CurrentTime.h>
#endif
#if USE(CF)
#include <wtf/RetainPtr.h>
#endif
#include <wtf/Lock.h>
#include <wtf/Threading.h>
#include <wtf/Vector.h>
#include <wtf/text/CString.h>
namespace WebCore {
const int selectTimeoutMS = 5;
const double pollTimeSeconds = 0.05;
const int maxRunningJobs = 128;
URL getCurlEffectiveURL(CURL* handle)
{
const char* url;
CURLcode err = curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &url);
if (CURLE_OK != err)
return URL();
return URL(URL(), url);
}
static const bool ignoreSSLErrors = getenv("WEBKIT_IGNORE_SSL_ERRORS");
static CString certificatePath()
{
#if USE(CF)
CFBundleRef webKitBundleRef = webKitBundle();
if (webKitBundleRef) {
RetainPtr<CFURLRef> certURLRef = adoptCF(CFBundleCopyResourceURL(webKitBundleRef, CFSTR("cacert"), CFSTR("pem"), CFSTR("certificates")));
if (certURLRef) {
char path[MAX_PATH];
CFURLGetFileSystemRepresentation(certURLRef.get(), false, reinterpret_cast<UInt8*>(path), MAX_PATH);
return path;
}
}
#endif
char* envPath = getenv("CURL_CA_BUNDLE_PATH");
if (envPath)
return envPath;
return CString();
}
static char* cookieJarPath()
{
char* cookieJarPath = getenv("CURL_COOKIE_JAR_PATH");
if (cookieJarPath)
return fastStrDup(cookieJarPath);
#if OS(WINDOWS)
char executablePath[MAX_PATH];
char appDataDirectory[MAX_PATH];
char cookieJarFullPath[MAX_PATH];
char cookieJarDirectory[MAX_PATH];
if (FAILED(::SHGetFolderPathA(0, CSIDL_LOCAL_APPDATA | CSIDL_FLAG_CREATE, 0, 0, appDataDirectory))
|| FAILED(::GetModuleFileNameA(0, executablePath, MAX_PATH)))
return fastStrDup("cookies.dat");
::PathRemoveExtensionA(executablePath);
LPSTR executableName = ::PathFindFileNameA(executablePath);
sprintf_s(cookieJarDirectory, MAX_PATH, "%s/%s", appDataDirectory, executableName);
sprintf_s(cookieJarFullPath, MAX_PATH, "%s/cookies.dat", cookieJarDirectory);
if (::SHCreateDirectoryExA(0, cookieJarDirectory, 0) != ERROR_SUCCESS
&& ::GetLastError() != ERROR_FILE_EXISTS
&& ::GetLastError() != ERROR_ALREADY_EXISTS)
return fastStrDup("cookies.dat");
return fastStrDup(cookieJarFullPath);
#else
return fastStrDup("cookies.dat");
#endif
}
static Lock* sharedResourceMutex(curl_lock_data data)
{
DEPRECATED_DEFINE_STATIC_LOCAL(Lock, cookieMutex, ());
DEPRECATED_DEFINE_STATIC_LOCAL(Lock, dnsMutex, ());
DEPRECATED_DEFINE_STATIC_LOCAL(Lock, shareMutex, ());
switch (data) {
case CURL_LOCK_DATA_COOKIE:
return &cookieMutex;
case CURL_LOCK_DATA_DNS:
return &dnsMutex;
case CURL_LOCK_DATA_SHARE:
return &shareMutex;
default:
ASSERT_NOT_REACHED();
return NULL;
}
}
#if ENABLE(WEB_TIMING)
static int milisecondsSinceRequest(double requestTime)
{
return static_cast<int>((monotonicallyIncreasingTime() - requestTime) * 1000.0);
}
static void calculateWebTimingInformations(ResourceHandleInternal* d)
{
double startTransfertTime = 0;
double preTransferTime = 0;
double dnslookupTime = 0;
double connectTime = 0;
double appConnectTime = 0;
curl_easy_getinfo(d->m_handle, CURLINFO_NAMELOOKUP_TIME, &dnslookupTime);
curl_easy_getinfo(d->m_handle, CURLINFO_CONNECT_TIME, &connectTime);
curl_easy_getinfo(d->m_handle, CURLINFO_APPCONNECT_TIME, &appConnectTime);
curl_easy_getinfo(d->m_handle, CURLINFO_STARTTRANSFER_TIME, &startTransfertTime);
curl_easy_getinfo(d->m_handle, CURLINFO_PRETRANSFER_TIME, &preTransferTime);
d->m_response.resourceLoadTiming().domainLookupStart = 0;
d->m_response.resourceLoadTiming().domainLookupEnd = static_cast<int>(dnslookupTime * 1000);
d->m_response.resourceLoadTiming().connectStart = static_cast<int>(dnslookupTime * 1000);
d->m_response.resourceLoadTiming().connectEnd = static_cast<int>(connectTime * 1000);
d->m_response.resourceLoadTiming().requestStart = static_cast<int>(connectTime *1000);
d->m_response.resourceLoadTiming().responseStart =static_cast<int>(preTransferTime * 1000);
if (appConnectTime)
d->m_response.resourceLoadTiming().secureConnectionStart = static_cast<int>(connectTime * 1000);
}
#endif
static void curl_lock_callback(CURL* , curl_lock_data data, curl_lock_access , void* )
{
if (Lock* mutex = sharedResourceMutex(data))
mutex->lock();
}
static void curl_unlock_callback(CURL* , curl_lock_data data, void* )
{
if (Lock* mutex = sharedResourceMutex(data))
mutex->unlock();
}
inline static bool isHttpInfo(int statusCode)
{
return 100 <= statusCode && statusCode < 200;
}
inline static bool isHttpRedirect(int statusCode)
{
return 300 <= statusCode && statusCode < 400 && statusCode != 304;
}
inline static bool isHttpAuthentication(int statusCode)
{
return statusCode == 401;
}
inline static bool isHttpNotModified(int statusCode)
{
return statusCode == 304;
}
ResourceHandleManager::ResourceHandleManager()
: m_downloadTimer(*this, &ResourceHandleManager::downloadTimerCallback)
, m_cookieJarFileName(cookieJarPath())
, m_certificatePath (certificatePath())
, m_runningJobs(0)
#ifndef NDEBUG
, m_logFile(nullptr)
#endif
{
curl_global_init(CURL_GLOBAL_ALL);
m_curlMultiHandle = curl_multi_init();
m_curlShareHandle = curl_share_init();
curl_share_setopt(m_curlShareHandle, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
curl_share_setopt(m_curlShareHandle, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
curl_share_setopt(m_curlShareHandle, CURLSHOPT_LOCKFUNC, curl_lock_callback);
curl_share_setopt(m_curlShareHandle, CURLSHOPT_UNLOCKFUNC, curl_unlock_callback);
initCookieSession();
#ifndef NDEBUG
char* logFile = getenv("CURL_LOG_FILE");
if (logFile)
m_logFile = fopen(logFile, "a");
#endif
}
ResourceHandleManager::~ResourceHandleManager()
{
curl_multi_cleanup(m_curlMultiHandle);
curl_share_cleanup(m_curlShareHandle);
if (m_cookieJarFileName)
fastFree(m_cookieJarFileName);
curl_global_cleanup();
#ifndef NDEBUG
if (m_logFile)
fclose(m_logFile);
#endif
}
CURLSH* ResourceHandleManager::getCurlShareHandle() const
{
return m_curlShareHandle;
}
void ResourceHandleManager::setCookieJarFileName(const char* cookieJarFileName)
{
m_cookieJarFileName = fastStrDup(cookieJarFileName);
}
const char* ResourceHandleManager::getCookieJarFileName() const
{
return m_cookieJarFileName;
}
ResourceHandleManager* ResourceHandleManager::sharedInstance()
{
static ResourceHandleManager* sharedInstance = 0;
if (!sharedInstance)
sharedInstance = new ResourceHandleManager();
return sharedInstance;
}
static void handleLocalReceiveResponse (CURL* handle, ResourceHandle* job, ResourceHandleInternal* d)
{
URL url = getCurlEffectiveURL(handle);
ASSERT(url.isValid());
d->m_response.setURL(url);
if (d->client())
d->client()->didReceiveResponse(job, ResourceResponse(d->m_response));
d->m_response.setResponseFired(true);
}
static size_t writeCallback(void* ptr, size_t size, size_t nmemb, void* data)
{
ResourceHandle* job = static_cast<ResourceHandle*>(data);
ResourceHandleInternal* d = job->getInternal();
if (d->m_cancelled)
return 0;
ASSERT(!d->m_defersLoading);
size_t totalSize = size * nmemb;
CURL* h = d->m_handle;
long httpCode = 0;
CURLcode err = curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
if (CURLE_OK == err && httpCode >= 300 && httpCode < 400)
return totalSize;
if (!d->m_response.responseFired()) {
handleLocalReceiveResponse(h, job, d);
if (d->m_cancelled)
return 0;
}
if (d->m_multipartHandle)
d->m_multipartHandle->contentReceived(static_cast<const char*>(ptr), totalSize);
else if (d->client()) {
d->client()->didReceiveData(job, static_cast<char*>(ptr), totalSize, 0);
CurlCacheManager::getInstance().didReceiveData(*job, static_cast<char*>(ptr), totalSize);
}
return totalSize;
}
static bool isAppendableHeader(const String &key)
{
static const char* appendableHeaders[] = {
"access-control-allow-headers",
"access-control-allow-methods",
"access-control-allow-origin",
"access-control-expose-headers",
"allow",
"cache-control",
"connection",
"content-encoding",
"content-language",
"if-match",
"if-none-match",
"keep-alive",
"pragma",
"proxy-authenticate",
"public",
"server",
"set-cookie",
"te",
"trailer",
"transfer-encoding",
"upgrade",
"user-agent",
"vary",
"via",
"warning",
"www-authenticate"
};
if (key.startsWith("x-", false))
return true;
for (auto& header : appendableHeaders) {
if (equalIgnoringASCIICase(key, header))
return true;
}
return false;
}
static void removeLeadingAndTrailingQuotes(String& value)
{
unsigned length = value.length();
if (value.startsWith('"') && value.endsWith('"') && length > 1)
value = value.substring(1, length-2);
}
static bool getProtectionSpace(CURL* h, const ResourceResponse& response, ProtectionSpace& protectionSpace)
{
CURLcode err;
long port = 0;
err = curl_easy_getinfo(h, CURLINFO_PRIMARY_PORT, &port);
if (err != CURLE_OK)
return false;
long availableAuth = CURLAUTH_NONE;
err = curl_easy_getinfo(h, CURLINFO_HTTPAUTH_AVAIL, &availableAuth);
if (err != CURLE_OK)
return false;
URL url = getCurlEffectiveURL(h);
if (!url.isValid())
return false;
String host = url.host();
String protocol = url.protocol();
String realm;
const String authHeader = response.httpHeaderField(HTTPHeaderName::Authorization);
const String realmString = "realm=";
int realmPos = authHeader.find(realmString);
if (realmPos > 0) {
realm = authHeader.substring(realmPos + realmString.length());
realm = realm.left(realm.find(','));
removeLeadingAndTrailingQuotes(realm);
}
ProtectionSpaceServerType serverType = ProtectionSpaceServerHTTP;
if (protocol == "https")
serverType = ProtectionSpaceServerHTTPS;
ProtectionSpaceAuthenticationScheme authScheme = ProtectionSpaceAuthenticationSchemeUnknown;
if (availableAuth & CURLAUTH_BASIC)
authScheme = ProtectionSpaceAuthenticationSchemeHTTPBasic;
if (availableAuth & CURLAUTH_DIGEST)
authScheme = ProtectionSpaceAuthenticationSchemeHTTPDigest;
if (availableAuth & CURLAUTH_GSSNEGOTIATE)
authScheme = ProtectionSpaceAuthenticationSchemeNegotiate;
if (availableAuth & CURLAUTH_NTLM)
authScheme = ProtectionSpaceAuthenticationSchemeNTLM;
protectionSpace = ProtectionSpace(host, port, serverType, realm, authScheme);
return true;
}
static size_t headerCallback(char* ptr, size_t size, size_t nmemb, void* data)
{
ResourceHandle* job = static_cast<ResourceHandle*>(data);
ResourceHandleInternal* d = job->getInternal();
if (d->m_cancelled)
return 0;
ASSERT(!d->m_defersLoading);
size_t totalSize = size * nmemb;
ResourceHandleClient* client = d->client();
String header(static_cast<const char*>(ptr), totalSize);
if (header == String("\r\n") || header == String("\n")) {
CURL* h = d->m_handle;
long httpCode = 0;
curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
if (!httpCode) {
return totalSize;
}
if (isHttpInfo(httpCode)) {
return totalSize;
}
double contentLength = 0;
curl_easy_getinfo(h, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength);
d->m_response.setExpectedContentLength(static_cast<long long int>(contentLength));
d->m_response.setURL(getCurlEffectiveURL(h));
d->m_response.setHTTPStatusCode(httpCode);
d->m_response.setMimeType(extractMIMETypeFromMediaType(d->m_response.httpHeaderField(HTTPHeaderName::ContentType)).convertToASCIILowercase());
d->m_response.setTextEncodingName(extractCharsetFromMediaType(d->m_response.httpHeaderField(HTTPHeaderName::ContentType)));
if (d->m_response.isMultipart()) {
String boundary;
bool parsed = MultipartHandle::extractBoundary(d->m_response.httpHeaderField(HTTPHeaderName::ContentType), boundary);
if (parsed)
d->m_multipartHandle = std::make_unique<MultipartHandle>(job, boundary);
}
if (isHttpRedirect(httpCode)) {
String location = d->m_response.httpHeaderField(HTTPHeaderName::Location);
if (!location.isEmpty()) {
URL newURL = URL(job->firstRequest().url(), location);
ResourceRequest redirectedRequest = job->firstRequest();
redirectedRequest.setURL(newURL);
ResourceResponse response = d->m_response;
if (client)
client->willSendRequest(job, WTFMove(redirectedRequest), WTFMove(response));
d->m_firstRequest.setURL(newURL);
return totalSize;
}
} else if (isHttpAuthentication(httpCode)) {
ProtectionSpace protectionSpace;
if (getProtectionSpace(d->m_handle, d->m_response, protectionSpace)) {
Credential credential;
AuthenticationChallenge challenge(protectionSpace, credential, d->m_authFailureCount, d->m_response, ResourceError());
challenge.setAuthenticationClient(job);
job->didReceiveAuthenticationChallenge(challenge);
d->m_authFailureCount++;
return totalSize;
}
}
if (client) {
if (isHttpNotModified(httpCode)) {
const String& url = job->firstRequest().url().string();
if (CurlCacheManager::getInstance().getCachedResponse(url, d->m_response)) {
if (d->m_addedCacheValidationHeaders) {
d->m_response.setHTTPStatusCode(200);
d->m_response.setHTTPStatusText("OK");
}
}
}
client->didReceiveResponse(job, ResourceResponse(d->m_response));
CurlCacheManager::getInstance().didReceiveResponse(*job, d->m_response);
}
d->m_response.setResponseFired(true);
} else {
int splitPos = header.find(":");
if (splitPos != -1) {
String key = header.left(splitPos).stripWhiteSpace();
String value = header.substring(splitPos + 1).stripWhiteSpace();
if (isAppendableHeader(key))
d->m_response.addHTTPHeaderField(key, value);
else
d->m_response.setHTTPHeaderField(key, value);
} else if (header.startsWith("HTTP", false)) {
long httpCode = 0;
curl_easy_getinfo(d->m_handle, CURLINFO_RESPONSE_CODE, &httpCode);
String httpCodeString = String::number(httpCode);
int statusCodePos = header.find(httpCodeString);
if (statusCodePos != -1) {
String status = header.substring(statusCodePos + httpCodeString.length());
d->m_response.setHTTPStatusText(status.stripWhiteSpace());
}
}
}
return totalSize;
}
size_t readCallback(void* ptr, size_t size, size_t nmemb, void* data)
{
ResourceHandle* job = static_cast<ResourceHandle*>(data);
ResourceHandleInternal* d = job->getInternal();
if (d->m_cancelled)
return 0;
ASSERT(!d->m_defersLoading);
if (!size || !nmemb)
return 0;
if (!d->m_formDataStream.hasMoreElements())
return 0;
size_t sent = d->m_formDataStream.read(ptr, size, nmemb);
if (!sent)
job->cancel();
return sent;
}
void ResourceHandleManager::downloadTimerCallback()
{
startScheduledJobs();
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd = 0;
struct timeval timeout;
timeout.tv_sec = 0;
timeout.tv_usec = selectTimeoutMS * 1000;
int rc = 0;
do {
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
curl_multi_fdset(m_curlMultiHandle, &fdread, &fdwrite, &fdexcep, &maxfd);
if (maxfd >= 0)
rc = ::select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
} while (rc == -1 && errno == EINTR);
if (-1 == rc) {
#ifndef NDEBUG
perror("bad: select() returned -1: ");
#endif
return;
}
int runningHandles = 0;
while (curl_multi_perform(m_curlMultiHandle, &runningHandles) == CURLM_CALL_MULTI_PERFORM) { }
while (true) {
int messagesInQueue;
CURLMsg* msg = curl_multi_info_read(m_curlMultiHandle, &messagesInQueue);
if (!msg)
break;
CURL* handle = msg->easy_handle;
ASSERT(handle);
ResourceHandle* job = 0;
CURLcode err = curl_easy_getinfo(handle, CURLINFO_PRIVATE, &job);
ASSERT_UNUSED(err, CURLE_OK == err);
ASSERT(job);
if (!job)
continue;
ResourceHandleInternal* d = job->getInternal();
ASSERT(d->m_handle == handle);
if (d->m_cancelled) {
removeFromCurl(job);
continue;
}
if (CURLMSG_DONE != msg->msg)
continue;
if (CURLE_OK == msg->data.result) {
#if ENABLE(WEB_TIMING)
calculateWebTimingInformations(d);
#endif
if (!d->m_response.responseFired()) {
handleLocalReceiveResponse(d->m_handle, job, d);
if (d->m_cancelled) {
removeFromCurl(job);
continue;
}
}
if (d->m_multipartHandle)
d->m_multipartHandle->contentEnded();
if (d->client()) {
d->client()->didFinishLoading(job, 0);
CurlCacheManager::getInstance().didFinishLoading(*job);
}
} else {
URL url = getCurlEffectiveURL(d->m_handle);
#ifndef NDEBUG
fprintf(stderr, "Curl ERROR for url='%s', error: '%s'\n", url.string().utf8().data(), curl_easy_strerror(msg->data.result));
#endif
if (d->client()) {
ResourceError resourceError(String(), msg->data.result, url, String(curl_easy_strerror(msg->data.result)));
resourceError.setSSLErrors(d->m_sslErrors);
d->client()->didFail(job, resourceError);
CurlCacheManager::getInstance().didFail(*job);
}
}
removeFromCurl(job);
}
bool started = startScheduledJobs();
if (!m_downloadTimer.isActive() && (started || (runningHandles > 0)))
m_downloadTimer.startOneShot(pollTimeSeconds);
}
void ResourceHandleManager::setProxyInfo(const String& host,
unsigned long port,
ProxyType type,
const String& username,
const String& password)
{
m_proxyType = type;
if (!host.length()) {
m_proxy = emptyString();
} else {
String userPass;
if (username.length() || password.length())
userPass = username + ":" + password + "@";
m_proxy = String("http://") + userPass + host + ":" + String::number(port);
}
}
void ResourceHandleManager::removeFromCurl(ResourceHandle* job)
{
ResourceHandleInternal* d = job->getInternal();
ASSERT(d->m_handle);
if (!d->m_handle)
return;
m_runningJobs--;
curl_multi_remove_handle(m_curlMultiHandle, d->m_handle);
curl_easy_cleanup(d->m_handle);
d->m_handle = 0;
job->deref();
}
static inline size_t getFormElementsCount(ResourceHandle* job)
{
RefPtr<FormData> formData = job->firstRequest().httpBody();
if (!formData)
return 0;
formData = formData->resolveBlobReferences();
size_t size = formData->elements().size();
job->firstRequest().setHTTPBody(WTFMove(formData));
return size;
}
static void setupFormData(ResourceHandle* job, CURLoption sizeOption, struct curl_slist** headers)
{
ResourceHandleInternal* d = job->getInternal();
Vector<FormDataElement> elements = job->firstRequest().httpBody()->elements();
size_t numElements = elements.size();
static int expectedSizeOfCurlOffT = 0;
if (!expectedSizeOfCurlOffT) {
curl_version_info_data *infoData = curl_version_info(CURLVERSION_NOW);
if (infoData->features & CURL_VERSION_LARGEFILE)
expectedSizeOfCurlOffT = sizeof(long long);
else
expectedSizeOfCurlOffT = sizeof(int);
}
static const long long maxCurlOffT = (1LL << (expectedSizeOfCurlOffT * 8 - 1)) - 1;
curl_off_t size = 0;
bool chunkedTransfer = false;
for (size_t i = 0; i < numElements; i++) {
FormDataElement element = elements[i];
if (element.m_type == FormDataElement::Type::EncodedFile) {
long long fileSizeResult;
if (getFileSize(element.m_filename, fileSizeResult)) {
if (fileSizeResult > maxCurlOffT) {
chunkedTransfer = true;
break;
}
size += fileSizeResult;
} else {
chunkedTransfer = true;
break;
}
} else
size += elements[i].m_data.size();
}
if (chunkedTransfer)
*headers = curl_slist_append(*headers, "Transfer-Encoding: chunked");
else {
if (sizeof(long long) == expectedSizeOfCurlOffT)
curl_easy_setopt(d->m_handle, sizeOption, (long long)size);
else
curl_easy_setopt(d->m_handle, sizeOption, (int)size);
}
curl_easy_setopt(d->m_handle, CURLOPT_READFUNCTION, readCallback);
curl_easy_setopt(d->m_handle, CURLOPT_READDATA, job);
}
void ResourceHandleManager::setupPUT(ResourceHandle* job, struct curl_slist** headers)
{
ResourceHandleInternal* d = job->getInternal();
curl_easy_setopt(d->m_handle, CURLOPT_UPLOAD, TRUE);
curl_easy_setopt(d->m_handle, CURLOPT_INFILESIZE, 0);
*headers = curl_slist_append(*headers, "Expect:");
size_t numElements = getFormElementsCount(job);
if (!numElements)
return;
setupFormData(job, CURLOPT_INFILESIZE_LARGE, headers);
}
void ResourceHandleManager::setupPOST(ResourceHandle* job, struct curl_slist** headers)
{
ResourceHandleInternal* d = job->getInternal();
curl_easy_setopt(d->m_handle, CURLOPT_POST, TRUE);
curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDSIZE, 0);
size_t numElements = getFormElementsCount(job);
if (!numElements)
return;
if (numElements == 1) {
job->firstRequest().httpBody()->flatten(d->m_postBytes);
if (d->m_postBytes.size()) {
curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDSIZE, d->m_postBytes.size());
curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDS, d->m_postBytes.data());
}
return;
}
setupFormData(job, CURLOPT_POSTFIELDSIZE_LARGE, headers);
}
void ResourceHandleManager::add(ResourceHandle* job)
{
job->ref();
m_resourceHandleList.append(job);
if (!m_downloadTimer.isActive())
m_downloadTimer.startOneShot(pollTimeSeconds);
}
bool ResourceHandleManager::removeScheduledJob(ResourceHandle* job)
{
int size = m_resourceHandleList.size();
for (int i = 0; i < size; i++) {
if (job == m_resourceHandleList[i]) {
m_resourceHandleList.remove(i);
job->deref();
return true;
}
}
return false;
}
bool ResourceHandleManager::startScheduledJobs()
{
bool started = false;
while (!m_resourceHandleList.isEmpty() && m_runningJobs < maxRunningJobs) {
ResourceHandle* job = m_resourceHandleList[0];
m_resourceHandleList.remove(0);
startJob(job);
started = true;
}
return started;
}
static void handleDataURL(ResourceHandle* handle)
{
ASSERT(handle->firstRequest().url().protocolIsData());
String url = handle->firstRequest().url().string();
ASSERT(handle);
ASSERT(handle->client());
int index = url.find(',');
if (index == -1) {
handle->client()->cannotShowURL(handle);
return;
}
String mediaType = url.substring(5, index - 5);
String data = url.substring(index + 1);
bool base64 = mediaType.endsWith(";base64", false);
if (base64)
mediaType = mediaType.left(mediaType.length() - 7);
if (mediaType.isEmpty())
mediaType = "text/plain";
String mimeType = extractMIMETypeFromMediaType(mediaType);
String charset = extractCharsetFromMediaType(mediaType);
if (charset.isEmpty())
charset = "US-ASCII";
ResourceResponse response;
response.setMimeType(mimeType);
response.setTextEncodingName(charset);
response.setURL(handle->firstRequest().url());
if (base64) {
data = decodeURLEscapeSequences(data);
handle->client()->didReceiveResponse(handle, WTFMove(response));
if (handle->client()) {
Vector<char> out;
if (base64Decode(data, out, Base64IgnoreSpacesAndNewLines) && out.size() > 0)
handle->client()->didReceiveData(handle, out.data(), out.size(), 0);
}
} else {
TextEncoding encoding(charset);
data = decodeURLEscapeSequences(data, encoding);
handle->client()->didReceiveResponse(handle, WTFMove(response));
if (handle->client()) {
CString encodedData = encoding.encode(data, URLEncodedEntitiesForUnencodables);
if (encodedData.length())
handle->client()->didReceiveData(handle, encodedData.data(), encodedData.length(), 0);
}
}
if (handle->client())
handle->client()->didFinishLoading(handle, 0);
}
void ResourceHandleManager::dispatchSynchronousJob(ResourceHandle* job)
{
URL kurl = job->firstRequest().url();
if (kurl.protocolIsData()) {
handleDataURL(job);
return;
}
ResourceHandleInternal* handle = job->getInternal();
handle->m_defersLoading = false;
initializeHandle(job);
CURLcode ret = curl_easy_perform(handle->m_handle);
if (ret != CURLE_OK) {
ResourceError error(String(handle->m_url), ret, kurl, String(curl_easy_strerror(ret)));
error.setSSLErrors(handle->m_sslErrors);
handle->client()->didFail(job, error);
} else {
if (handle->client())
handle->client()->didReceiveResponse(job, ResourceResponse(handle->m_response));
}
#if ENABLE(WEB_TIMING)
calculateWebTimingInformations(handle);
#endif
curl_easy_cleanup(handle->m_handle);
}
void ResourceHandleManager::startJob(ResourceHandle* job)
{
URL url = job->firstRequest().url();
if (url.protocolIsData()) {
handleDataURL(job);
job->deref();
return;
}
initializeHandle(job);
m_runningJobs++;
CURLMcode ret = curl_multi_add_handle(m_curlMultiHandle, job->getInternal()->m_handle);
if (ret && ret != CURLM_CALL_MULTI_PERFORM) {
#ifndef NDEBUG
fprintf(stderr, "Error %d starting job %s\n", ret, encodeWithURLEscapeSequences(job->firstRequest().url().string()).latin1().data());
#endif
job->cancel();
return;
}
}
void ResourceHandleManager::applyAuthenticationToRequest(ResourceHandle* handle, ResourceRequest& request)
{
ResourceHandleInternal* d = handle->getInternal();
if (handle->shouldUseCredentialStorage()) {
if (d->m_user.isEmpty() && d->m_pass.isEmpty()) {
d->m_initialCredential = CredentialStorage::defaultCredentialStorage().get(request.url());
} else {
CredentialStorage::defaultCredentialStorage().set(Credential(d->m_user, d->m_pass, CredentialPersistenceNone), request.url());
}
}
String user = d->m_user;
String password = d->m_pass;
if (!d->m_initialCredential.isEmpty()) {
user = d->m_initialCredential.user();
password = d->m_initialCredential.password();
curl_easy_setopt(d->m_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
}
String userpass = user + ":" + password;
curl_easy_setopt(d->m_handle, CURLOPT_USERPWD, userpass.utf8().data());
}
void ResourceHandleManager::initializeHandle(ResourceHandle* job)
{
static const int allowedProtocols = CURLPROTO_FILE | CURLPROTO_FTP | CURLPROTO_FTPS | CURLPROTO_HTTP | CURLPROTO_HTTPS;
URL url = job->firstRequest().url();
url.removeFragmentIdentifier();
ResourceHandleInternal* d = job->getInternal();
String urlString = url.string();
if (url.isLocalFile()) {
if (!url.query().isEmpty()) {
url.setQuery(String());
urlString = url.string();
}
d->m_response.setMimeType(MIMETypeRegistry::getMIMETypeForPath(url));
}
d->m_handle = curl_easy_init();
if (d->m_defersLoading) {
CURLcode error = curl_easy_pause(d->m_handle, CURLPAUSE_ALL);
ASSERT_UNUSED(error, error == CURLE_OK);
}
#ifndef NDEBUG
if (getenv("DEBUG_CURL"))
curl_easy_setopt(d->m_handle, CURLOPT_VERBOSE, 1);
if (m_logFile)
curl_easy_setopt(d->m_handle, CURLOPT_STDERR, m_logFile);
#endif
curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYPEER, 1L);
curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYHOST, 2L);
curl_easy_setopt(d->m_handle, CURLOPT_PRIVATE, job);
curl_easy_setopt(d->m_handle, CURLOPT_ERRORBUFFER, m_curlErrorBuffer);
curl_easy_setopt(d->m_handle, CURLOPT_WRITEFUNCTION, writeCallback);
curl_easy_setopt(d->m_handle, CURLOPT_WRITEDATA, job);
curl_easy_setopt(d->m_handle, CURLOPT_HEADERFUNCTION, headerCallback);
curl_easy_setopt(d->m_handle, CURLOPT_WRITEHEADER, job);
curl_easy_setopt(d->m_handle, CURLOPT_AUTOREFERER, 1);
curl_easy_setopt(d->m_handle, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(d->m_handle, CURLOPT_MAXREDIRS, 10);
curl_easy_setopt(d->m_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
curl_easy_setopt(d->m_handle, CURLOPT_SHARE, m_curlShareHandle);
curl_easy_setopt(d->m_handle, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 5); curl_easy_setopt(d->m_handle, CURLOPT_PROTOCOLS, allowedProtocols);
curl_easy_setopt(d->m_handle, CURLOPT_REDIR_PROTOCOLS, allowedProtocols);
setSSLClientCertificate(job);
if (ignoreSSLErrors)
curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYPEER, false);
else
setSSLVerifyOptions(job);
if (!m_certificatePath.isNull())
curl_easy_setopt(d->m_handle, CURLOPT_CAINFO, m_certificatePath.data());
curl_easy_setopt(d->m_handle, CURLOPT_ENCODING, "");
ASSERT(!d->m_url);
d->m_url = fastStrDup(urlString.latin1().data());
curl_easy_setopt(d->m_handle, CURLOPT_URL, d->m_url);
if (m_cookieJarFileName)
curl_easy_setopt(d->m_handle, CURLOPT_COOKIEJAR, m_cookieJarFileName);
struct curl_slist* headers = 0;
if (job->firstRequest().httpHeaderFields().size() > 0) {
HTTPHeaderMap customHeaders = job->firstRequest().httpHeaderFields();
bool hasCacheHeaders = customHeaders.contains(HTTPHeaderName::IfModifiedSince) || customHeaders.contains(HTTPHeaderName::IfNoneMatch);
if (!hasCacheHeaders && CurlCacheManager::getInstance().isCached(url)) {
CurlCacheManager::getInstance().addCacheEntryClient(url, job);
HTTPHeaderMap& requestHeaders = CurlCacheManager::getInstance().requestHeaders(url);
HTTPHeaderMap::const_iterator it = requestHeaders.begin();
HTTPHeaderMap::const_iterator end = requestHeaders.end();
while (it != end) {
customHeaders.set(it->key, it->value);
++it;
}
d->m_addedCacheValidationHeaders = true;
}
HTTPHeaderMap::const_iterator end = customHeaders.end();
for (HTTPHeaderMap::const_iterator it = customHeaders.begin(); it != end; ++it) {
String key = it->key;
String value = it->value;
String headerString(key);
if (value.isEmpty())
headerString.append(";");
else {
headerString.append(": ");
headerString.append(value);
}
CString headerLatin1 = headerString.latin1();
headers = curl_slist_append(headers, headerLatin1.data());
}
}
String method = job->firstRequest().httpMethod();
if ("GET" == method)
curl_easy_setopt(d->m_handle, CURLOPT_HTTPGET, TRUE);
else if ("POST" == method)
setupPOST(job, &headers);
else if ("PUT" == method)
setupPUT(job, &headers);
else if ("HEAD" == method)
curl_easy_setopt(d->m_handle, CURLOPT_NOBODY, TRUE);
else {
curl_easy_setopt(d->m_handle, CURLOPT_CUSTOMREQUEST, method.ascii().data());
setupPUT(job, &headers);
}
if (headers) {
curl_easy_setopt(d->m_handle, CURLOPT_HTTPHEADER, headers);
d->m_customHeaders = headers;
}
applyAuthenticationToRequest(job, job->firstRequest());
if (m_proxy.length()) {
curl_easy_setopt(d->m_handle, CURLOPT_PROXY, m_proxy.utf8().data());
curl_easy_setopt(d->m_handle, CURLOPT_PROXYTYPE, m_proxyType);
}
}
void ResourceHandleManager::initCookieSession()
{
CURL* curl = curl_easy_init();
if (!curl)
return;
curl_easy_setopt(curl, CURLOPT_SHARE, m_curlShareHandle);
if (m_cookieJarFileName) {
curl_easy_setopt(curl, CURLOPT_COOKIEFILE, m_cookieJarFileName);
curl_easy_setopt(curl, CURLOPT_COOKIEJAR, m_cookieJarFileName);
}
curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1);
curl_easy_cleanup(curl);
}
void ResourceHandleManager::cancel(ResourceHandle* job)
{
if (removeScheduledJob(job))
return;
ResourceHandleInternal* d = job->getInternal();
d->m_cancelled = true;
if (!m_downloadTimer.isActive())
m_downloadTimer.startOneShot(pollTimeSeconds);
}
}
#endif