NetworkResourceLoader.cpp [plain text]
#include "config.h"
#include "NetworkResourceLoader.h"
#if ENABLE(NETWORK_PROCESS)
#include "AuthenticationManager.h"
#include "DataReference.h"
#include "Logging.h"
#include "NetworkBlobRegistry.h"
#include "NetworkCache.h"
#include "NetworkConnectionToWebProcess.h"
#include "NetworkProcess.h"
#include "NetworkProcessConnectionMessages.h"
#include "NetworkResourceLoadParameters.h"
#include "RemoteNetworkingContext.h"
#include "ShareableResource.h"
#include "SharedMemory.h"
#include "WebCoreArgumentCoders.h"
#include "WebErrors.h"
#include "WebResourceLoaderMessages.h"
#include <WebCore/BlobDataFileReference.h>
#include <WebCore/CertificateInfo.h>
#include <WebCore/HTTPHeaderNames.h>
#include <WebCore/NotImplemented.h>
#include <WebCore/ResourceHandle.h>
#include <WebCore/SharedBuffer.h>
#include <WebCore/SynchronousLoaderClient.h>
#include <wtf/CurrentTime.h>
#include <wtf/MainThread.h>
using namespace WebCore;
namespace WebKit {
struct NetworkResourceLoader::SynchronousLoadData {
SynchronousLoadData(PassRefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply> reply)
: delayedReply(reply)
{
ASSERT(delayedReply);
}
WebCore::ResourceRequest currentRequest;
RefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply> delayedReply;
WebCore::ResourceResponse response;
WebCore::ResourceError error;
};
static void sendReplyToSynchronousRequest(NetworkResourceLoader::SynchronousLoadData& data, const WebCore::SharedBuffer* buffer)
{
ASSERT(data.delayedReply);
ASSERT(!data.response.isNull() || !data.error.isNull());
Vector<char> responseBuffer;
if (buffer && buffer->size())
responseBuffer.append(buffer->data(), buffer->size());
data.delayedReply->send(data.error, data.response, responseBuffer);
data.delayedReply = nullptr;
}
NetworkResourceLoader::NetworkResourceLoader(const NetworkResourceLoadParameters& parameters, NetworkConnectionToWebProcess* connection, PassRefPtr<Messages::NetworkConnectionToWebProcess::PerformSynchronousLoad::DelayedReply> synchronousReply)
: m_parameters(parameters)
, m_connection(connection)
, m_bytesReceived(0)
, m_bufferedDataEncodedDataLength(0)
, m_didConvertHandleToDownload(false)
, m_didConsumeSandboxExtensions(false)
, m_defersLoading(parameters.defersLoading)
, m_bufferingTimer(*this, &NetworkResourceLoader::bufferingTimerFired)
{
ASSERT(RunLoop::isMain());
ASSERT((m_parameters.webPageID && m_parameters.webFrameID) || m_parameters.clientCredentialPolicy == DoNotAskClientForAnyCredentials);
if (originalRequest().httpBody()) {
for (const FormDataElement& element : originalRequest().httpBody()->elements()) {
if (element.m_type == FormDataElement::Type::EncodedBlob)
m_fileReferences.appendVector(NetworkBlobRegistry::singleton().filesInBlob(connection, element.m_url));
}
}
if (originalRequest().url().protocolIs("blob")) {
ASSERT(!m_parameters.resourceSandboxExtension);
m_fileReferences.appendVector(NetworkBlobRegistry::singleton().filesInBlob(connection, originalRequest().url()));
}
if (synchronousReply)
m_synchronousLoadData = std::make_unique<SynchronousLoadData>(synchronousReply);
}
NetworkResourceLoader::~NetworkResourceLoader()
{
ASSERT(RunLoop::isMain());
ASSERT(!m_handle);
ASSERT(!isSynchronous() || !m_synchronousLoadData->delayedReply);
}
bool NetworkResourceLoader::isSynchronous() const
{
return !!m_synchronousLoadData;
}
void NetworkResourceLoader::start()
{
ASSERT(RunLoop::isMain());
if (m_defersLoading)
return;
m_currentRequest = originalRequest();
#if ENABLE(NETWORK_CACHE)
if (!NetworkCache::singleton().isEnabled() || sessionID().isEphemeral() || !originalRequest().url().protocolIsInHTTPFamily()) {
startNetworkLoad();
return;
}
RefPtr<NetworkResourceLoader> loader(this);
NetworkCache::singleton().retrieve(originalRequest(), m_parameters.webPageID, [loader](std::unique_ptr<NetworkCache::Entry> entry) {
if (loader->hasOneRef()) {
return;
}
if (!entry) {
loader->startNetworkLoad();
return;
}
if (loader->m_parameters.needsCertificateInfo && !entry->response().containsCertificateInfo()) {
loader->startNetworkLoad();
return;
}
if (entry->needsValidation()) {
loader->validateCacheEntry(WTF::move(entry));
return;
}
loader->didRetrieveCacheEntry(WTF::move(entry));
});
#else
startNetworkLoad();
#endif
}
void NetworkResourceLoader::startNetworkLoad()
{
m_networkingContext = RemoteNetworkingContext::create(sessionID(), m_parameters.shouldClearReferrerOnHTTPSToHTTPRedirect);
consumeSandboxExtensions();
if (isSynchronous() || m_parameters.maximumBufferingTime > 0_ms)
m_bufferedData = WebCore::SharedBuffer::create();
#if ENABLE(NETWORK_CACHE)
if (NetworkCache::singleton().isEnabled())
m_bufferedDataForCache = WebCore::SharedBuffer::create();
#endif
bool shouldSniff = m_parameters.contentSniffingPolicy == SniffContent;
m_handle = ResourceHandle::create(m_networkingContext.get(), m_currentRequest, this, m_defersLoading, shouldSniff);
}
void NetworkResourceLoader::setDefersLoading(bool defers)
{
if (m_defersLoading == defers)
return;
m_defersLoading = defers;
if (m_handle) {
m_handle->setDefersLoading(defers);
return;
}
if (!m_defersLoading)
start();
}
void NetworkResourceLoader::cleanup()
{
ASSERT(RunLoop::isMain());
m_bufferingTimer.stop();
invalidateSandboxExtensions();
if (m_handle) {
m_handle->setClient(nullptr);
m_handle = nullptr;
}
m_connection->didCleanupResourceLoader(*this);
}
void NetworkResourceLoader::didConvertHandleToDownload()
{
ASSERT(m_handle);
m_didConvertHandleToDownload = true;
}
void NetworkResourceLoader::abort()
{
ASSERT(RunLoop::isMain());
if (m_handle && !m_didConvertHandleToDownload)
m_handle->cancel();
cleanup();
}
void NetworkResourceLoader::didReceiveResponseAsync(ResourceHandle* handle, const ResourceResponse& receivedResponse)
{
ASSERT_UNUSED(handle, handle == m_handle);
m_response = receivedResponse;
m_response.setSource(ResourceResponse::Source::Network);
if (m_parameters.needsCertificateInfo)
m_response.includeCertificateInfo();
if (!isSynchronous() && m_response.isMultipart())
m_bufferedData = nullptr;
bool shouldSendDidReceiveResponse = true;
#if ENABLE(NETWORK_CACHE)
if (m_response.isMultipart())
m_bufferedDataForCache = nullptr;
if (m_cacheEntryForValidation) {
bool validationSucceeded = m_response.httpStatusCode() == 304; if (validationSucceeded)
NetworkCache::singleton().update(originalRequest(), m_parameters.webPageID, *m_cacheEntryForValidation, m_response);
else
m_cacheEntryForValidation = nullptr;
}
shouldSendDidReceiveResponse = !m_cacheEntryForValidation;
#endif
bool shouldWaitContinueDidReceiveResponse = originalRequest().requester() == ResourceRequest::Requester::Main;
if (shouldSendDidReceiveResponse) {
if (isSynchronous())
m_synchronousLoadData->response = m_response;
else {
if (!sendAbortingOnFailure(Messages::WebResourceLoader::DidReceiveResponse(m_response, shouldWaitContinueDidReceiveResponse)))
return;
}
}
bool shouldContinueDidReceiveResponse = !shouldWaitContinueDidReceiveResponse;
#if ENABLE(NETWORK_CACHE)
shouldContinueDidReceiveResponse = shouldContinueDidReceiveResponse || m_cacheEntryForValidation;
#endif
if (!shouldContinueDidReceiveResponse)
return;
m_handle->continueDidReceiveResponse();
}
void NetworkResourceLoader::didReceiveData(ResourceHandle*, const char* , unsigned , int )
{
ASSERT_NOT_REACHED();
}
void NetworkResourceLoader::didReceiveBuffer(ResourceHandle* handle, PassRefPtr<SharedBuffer> buffer, int reportedEncodedDataLength)
{
ASSERT_UNUSED(handle, handle == m_handle);
#if ENABLE(NETWORK_CACHE)
ASSERT(!m_cacheEntryForValidation);
if (m_bufferedDataForCache) {
const size_t maximumCacheBufferSize = 10 * 1024 * 1024;
if (m_bufferedDataForCache->size() + buffer->size() <= maximumCacheBufferSize)
m_bufferedDataForCache->append(buffer.get());
else
m_bufferedDataForCache = nullptr;
}
#endif
unsigned encodedDataLength = reportedEncodedDataLength >= 0 ? reportedEncodedDataLength : buffer->size();
m_bytesReceived += buffer->size();
if (m_bufferedData) {
m_bufferedData->append(buffer.get());
m_bufferedDataEncodedDataLength += encodedDataLength;
startBufferingTimerIfNeeded();
return;
}
sendBufferMaybeAborting(*buffer, encodedDataLength);
}
void NetworkResourceLoader::didFinishLoading(ResourceHandle* handle, double finishTime)
{
ASSERT_UNUSED(handle, handle == m_handle);
#if ENABLE(NETWORK_CACHE)
if (NetworkCache::singleton().isEnabled()) {
if (m_cacheEntryForValidation) {
ASSERT(m_response.httpStatusCode() == 304);
LOG(NetworkCache, "(NetworkProcess) revalidated");
didRetrieveCacheEntry(WTF::move(m_cacheEntryForValidation));
return;
}
bool allowStale = originalRequest().cachePolicy() >= ReturnCacheDataElseLoad;
bool hasCacheableRedirect = m_response.isHTTP() && WebCore::redirectChainAllowsReuse(m_redirectChainCacheStatus, allowStale ? WebCore::ReuseExpiredRedirection : WebCore::DoNotReuseExpiredRedirection);
if (hasCacheableRedirect && m_redirectChainCacheStatus.status == RedirectChainCacheStatus::CachedRedirection) {
auto now = std::chrono::system_clock::now();
auto responseEndOfValidity = now + WebCore::computeFreshnessLifetimeForHTTPFamily(m_response, now) - WebCore::computeCurrentAge(m_response, now);
hasCacheableRedirect = responseEndOfValidity <= m_redirectChainCacheStatus.endOfValidity;
}
bool isPrivate = sessionID().isEphemeral();
if (m_bufferedDataForCache && hasCacheableRedirect && !isPrivate) {
RefPtr<NetworkConnectionToWebProcess> connection(connectionToWebProcess());
RefPtr<NetworkResourceLoader> loader(this);
NetworkCache::singleton().store(originalRequest(), m_response, WTF::move(m_bufferedDataForCache), [loader, connection](NetworkCache::MappedBody& mappedBody) {
#if ENABLE(SHAREABLE_RESOURCE)
if (mappedBody.shareableResourceHandle.isNull())
return;
LOG(NetworkCache, "(NetworkProcess) sending DidCacheResource");
loader->send(Messages::NetworkProcessConnection::DidCacheResource(loader->originalRequest(), mappedBody.shareableResourceHandle, loader->sessionID()));
#endif
});
} else if (!hasCacheableRedirect) {
NetworkCache::singleton().remove(originalRequest());
}
}
#endif
if (isSynchronous())
sendReplyToSynchronousRequest(*m_synchronousLoadData, m_bufferedData.get());
else {
if (m_bufferedData && m_bufferedData->size()) {
bool shouldContinue = sendBufferMaybeAborting(*m_bufferedData, -1);
if (!shouldContinue)
return;
}
send(Messages::WebResourceLoader::DidFinishResourceLoad(finishTime));
}
cleanup();
}
void NetworkResourceLoader::didFail(ResourceHandle* handle, const ResourceError& error)
{
ASSERT_UNUSED(handle, !handle || handle == m_handle);
#if ENABLE(NETWORK_CACHE)
m_cacheEntryForValidation = nullptr;
#endif
if (isSynchronous()) {
m_synchronousLoadData->error = error;
sendReplyToSynchronousRequest(*m_synchronousLoadData, nullptr);
} else
send(Messages::WebResourceLoader::DidFailResourceLoad(error));
cleanup();
}
void NetworkResourceLoader::willSendRequestAsync(ResourceHandle* handle, const ResourceRequest& request, const ResourceResponse& redirectResponse)
{
ASSERT_UNUSED(handle, handle == m_handle);
ASSERT(!redirectResponse.isNull());
ASSERT(RunLoop::isMain());
m_currentRequest = request;
#if ENABLE(NETWORK_CACHE)
WebCore::updateRedirectChainStatus(m_redirectChainCacheStatus, redirectResponse);
#endif
if (isSynchronous()) {
if (!protocolHostAndPortAreEqual(originalRequest().url(), m_currentRequest.url())) {
ASSERT(m_synchronousLoadData->error.isNull());
m_synchronousLoadData->error = SynchronousLoaderClient::platformBadResponseError();
m_currentRequest = ResourceRequest();
}
continueWillSendRequest(m_currentRequest);
return;
}
sendAbortingOnFailure(Messages::WebResourceLoader::WillSendRequest(m_currentRequest, redirectResponse));
}
void NetworkResourceLoader::continueWillSendRequest(const ResourceRequest& newRequest)
{
#if PLATFORM(COCOA)
m_currentRequest.updateFromDelegatePreservingOldProperties(newRequest.nsURLRequest(DoNotUpdateHTTPBody));
#elif USE(SOUP)
m_currentRequest.updateFromDelegatePreservingOldProperties(newRequest);
#endif
if (m_currentRequest.isNull()) {
m_handle->cancel();
didFail(m_handle.get(), cancelledError(m_currentRequest));
return;
}
m_handle->continueWillSendRequest(m_currentRequest);
}
void NetworkResourceLoader::continueDidReceiveResponse()
{
if (!m_handle)
return;
m_handle->continueDidReceiveResponse();
}
void NetworkResourceLoader::didSendData(ResourceHandle* handle, unsigned long long bytesSent, unsigned long long totalBytesToBeSent)
{
ASSERT_UNUSED(handle, handle == m_handle);
if (!isSynchronous())
send(Messages::WebResourceLoader::DidSendData(bytesSent, totalBytesToBeSent));
}
void NetworkResourceLoader::wasBlocked(ResourceHandle* handle)
{
ASSERT_UNUSED(handle, handle == m_handle);
didFail(handle, WebKit::blockedError(m_currentRequest));
}
void NetworkResourceLoader::cannotShowURL(ResourceHandle* handle)
{
ASSERT_UNUSED(handle, handle == m_handle);
didFail(handle, WebKit::cannotShowURLError(m_currentRequest));
}
bool NetworkResourceLoader::shouldUseCredentialStorage(ResourceHandle* handle)
{
ASSERT_UNUSED(handle, handle == m_handle || !m_handle);
return m_parameters.allowStoredCredentials == AllowStoredCredentials;
}
void NetworkResourceLoader::didReceiveAuthenticationChallenge(ResourceHandle* handle, const AuthenticationChallenge& challenge)
{
ASSERT_UNUSED(handle, handle == m_handle);
ASSERT(m_parameters.clientCredentialPolicy != DoNotAskClientForCrossOriginCredentials);
if (m_parameters.clientCredentialPolicy == DoNotAskClientForAnyCredentials) {
challenge.authenticationClient()->receivedRequestToContinueWithoutCredential(challenge);
return;
}
NetworkProcess::singleton().authenticationManager().didReceiveAuthenticationChallenge(m_parameters.webPageID, m_parameters.webFrameID, challenge);
}
void NetworkResourceLoader::didCancelAuthenticationChallenge(ResourceHandle* handle, const AuthenticationChallenge&)
{
ASSERT_UNUSED(handle, handle == m_handle);
notImplemented();
}
void NetworkResourceLoader::receivedCancellation(ResourceHandle* handle, const AuthenticationChallenge&)
{
ASSERT_UNUSED(handle, handle == m_handle);
m_handle->cancel();
didFail(m_handle.get(), cancelledError(m_currentRequest));
}
void NetworkResourceLoader::startBufferingTimerIfNeeded()
{
if (isSynchronous())
return;
if (m_bufferingTimer.isActive())
return;
m_bufferingTimer.startOneShot(m_parameters.maximumBufferingTime);
}
void NetworkResourceLoader::bufferingTimerFired()
{
ASSERT(m_bufferedData);
ASSERT(m_handle);
if (!m_bufferedData->size())
return;
IPC::SharedBufferDataReference dataReference(m_bufferedData.get());
size_t encodedLength = m_bufferedDataEncodedDataLength;
m_bufferedData = WebCore::SharedBuffer::create();
m_bufferedDataEncodedDataLength = 0;
sendAbortingOnFailure(Messages::WebResourceLoader::DidReceiveData(dataReference, encodedLength));
}
bool NetworkResourceLoader::sendBufferMaybeAborting(const WebCore::SharedBuffer& buffer, size_t encodedDataLength)
{
ASSERT(!isSynchronous());
#if PLATFORM(COCOA)
ShareableResource::Handle shareableResourceHandle;
NetworkResourceLoader::tryGetShareableHandleFromSharedBuffer(shareableResourceHandle, const_cast<WebCore::SharedBuffer&>(buffer));
if (!shareableResourceHandle.isNull()) {
send(Messages::WebResourceLoader::DidReceiveResource(shareableResourceHandle, currentTime()));
abort();
return false;
}
#endif
IPC::SharedBufferDataReference dataReference(&const_cast<WebCore::SharedBuffer&>(buffer));
return sendAbortingOnFailure(Messages::WebResourceLoader::DidReceiveData(dataReference, encodedDataLength));
}
#if ENABLE(NETWORK_CACHE)
void NetworkResourceLoader::didRetrieveCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
{
if (isSynchronous()) {
m_synchronousLoadData->response = entry->response();
sendReplyToSynchronousRequest(*m_synchronousLoadData, entry->buffer());
} else {
if (entry->response().url() != originalRequest().url()) {
ResourceRequest syntheticRedirectRequest(entry->response().url());
ResourceResponse syntheticRedirectResponse(originalRequest().url(), { }, 0, { });
sendAbortingOnFailure(Messages::WebResourceLoader::WillSendRequest(syntheticRedirectRequest, syntheticRedirectResponse));
}
bool needsContinueDidReceiveResponseMessage = originalRequest().requester() == ResourceRequest::Requester::Main;
sendAbortingOnFailure(Messages::WebResourceLoader::DidReceiveResponse(entry->response(), needsContinueDidReceiveResponseMessage));
#if ENABLE(SHAREABLE_RESOURCE)
if (!entry->shareableResourceHandle().isNull())
send(Messages::WebResourceLoader::DidReceiveResource(entry->shareableResourceHandle(), currentTime()));
else {
#endif
bool shouldContinue = sendBufferMaybeAborting(*entry->buffer(), entry->buffer()->size());
if (!shouldContinue)
return;
send(Messages::WebResourceLoader::DidFinishResourceLoad(currentTime()));
#if ENABLE(SHAREABLE_RESOURCE)
}
#endif
}
cleanup();
}
void NetworkResourceLoader::validateCacheEntry(std::unique_ptr<NetworkCache::Entry> entry)
{
ASSERT(!m_handle);
String eTag = entry->response().httpHeaderField(WebCore::HTTPHeaderName::ETag);
String lastModified = entry->response().httpHeaderField(WebCore::HTTPHeaderName::LastModified);
if (!eTag.isEmpty())
m_currentRequest.setHTTPHeaderField(WebCore::HTTPHeaderName::IfNoneMatch, eTag);
if (!lastModified.isEmpty())
m_currentRequest.setHTTPHeaderField(WebCore::HTTPHeaderName::IfModifiedSince, lastModified);
m_cacheEntryForValidation = WTF::move(entry);
startNetworkLoad();
}
#endif
IPC::Connection* NetworkResourceLoader::messageSenderConnection()
{
return connectionToWebProcess()->connection();
}
void NetworkResourceLoader::consumeSandboxExtensions()
{
for (auto& extension : m_parameters.requestBodySandboxExtensions)
extension->consume();
if (auto& extension = m_parameters.resourceSandboxExtension)
extension->consume();
for (auto& fileReference : m_fileReferences)
fileReference->prepareForFileAccess();
m_didConsumeSandboxExtensions = true;
}
void NetworkResourceLoader::invalidateSandboxExtensions()
{
if (m_didConsumeSandboxExtensions) {
for (auto& extension : m_parameters.requestBodySandboxExtensions)
extension->revoke();
if (auto& extension = m_parameters.resourceSandboxExtension)
extension->revoke();
for (auto& fileReference : m_fileReferences)
fileReference->revokeFileAccess();
}
m_fileReferences.clear();
m_didConsumeSandboxExtensions = false;
}
template<typename T>
bool NetworkResourceLoader::sendAbortingOnFailure(T&& message, unsigned messageSendFlags)
{
bool result = messageSenderConnection()->send(std::forward<T>(message), messageSenderDestinationID(), messageSendFlags);
if (!result)
abort();
return result;
}
#if USE(PROTECTION_SPACE_AUTH_CALLBACK)
void NetworkResourceLoader::canAuthenticateAgainstProtectionSpaceAsync(ResourceHandle* handle, const ProtectionSpace& protectionSpace)
{
ASSERT(RunLoop::isMain());
ASSERT_UNUSED(handle, handle == m_handle);
if (protectionSpace.authenticationScheme() == ProtectionSpaceAuthenticationSchemeServerTrustEvaluationRequested
&& !NetworkProcess::singleton().canHandleHTTPSServerTrustEvaluation()) {
continueCanAuthenticateAgainstProtectionSpace(false);
return;
}
if (isSynchronous()) {
continueCanAuthenticateAgainstProtectionSpace(true);
return;
}
sendAbortingOnFailure(Messages::WebResourceLoader::CanAuthenticateAgainstProtectionSpace(protectionSpace));
}
void NetworkResourceLoader::continueCanAuthenticateAgainstProtectionSpace(bool result)
{
m_handle->continueCanAuthenticateAgainstProtectionSpace(result);
}
#endif
#if USE(NETWORK_CFDATA_ARRAY_CALLBACK)
bool NetworkResourceLoader::supportsDataArray()
{
notImplemented();
return false;
}
void NetworkResourceLoader::didReceiveDataArray(ResourceHandle*, CFArrayRef)
{
ASSERT_NOT_REACHED();
notImplemented();
}
#endif
}
#endif // ENABLE(NETWORK_PROCESS)