diff options
Diffstat (limited to 'dom/cache')
117 files changed, 19885 insertions, 0 deletions
diff --git a/dom/cache/Action.cpp b/dom/cache/Action.cpp new file mode 100644 index 0000000000..a71f8f8dff --- /dev/null +++ b/dom/cache/Action.cpp @@ -0,0 +1,40 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/Action.h" + +namespace mozilla { +namespace dom { +namespace cache { + +void +Action::CancelOnInitiatingThread() +{ + NS_ASSERT_OWNINGTHREAD(Action); + // It is possible for cancellation to be duplicated. For example, an + // individual Cache could have its Actions canceled and then shutdown + // could trigger a second action. + mCanceled = true; +} + +Action::Action() + : mCanceled(false) +{ +} + +Action::~Action() +{ +} + +bool +Action::IsCanceled() const +{ + return mCanceled; +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/Action.h b/dom/cache/Action.h new file mode 100644 index 0000000000..da6b3fd499 --- /dev/null +++ b/dom/cache/Action.h @@ -0,0 +1,105 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Action_h +#define mozilla_dom_cache_Action_h + +#include "mozilla/Atomics.h" +#include "mozilla/dom/cache/Types.h" +#include "nsISupportsImpl.h" + +class mozIStorageConnection; + +namespace mozilla { +namespace dom { +namespace cache { + +class Action +{ +public: + class Resolver + { + public: + // Note: Action must drop Resolver ref after calling Resolve()! + // Note: Must be called on the same thread used to execute + // Action::RunOnTarget(). + virtual void Resolve(nsresult aRv) = 0; + + NS_IMETHOD_(MozExternalRefCountType) + AddRef(void) = 0; + + NS_IMETHOD_(MozExternalRefCountType) + Release(void) = 0; + }; + + // Class containing data that can be opportunistically shared between + // multiple Actions running on the same thread/Context. In theory + // this could be abstracted to a generic key/value map, but for now + // just explicitly provide accessors for the data we need. + class Data + { + public: + virtual mozIStorageConnection* + GetConnection() const = 0; + + virtual void + SetConnection(mozIStorageConnection* aConn) = 0; + }; + + // Execute operations on the target thread. Once complete call + // Resolver::Resolve(). This can be done sync or async. + // Note: Action should hold Resolver ref until its ready to call Resolve(). + // Note: The "target" thread is determined when the Action is scheduled on + // Context. The Action should not assume any particular thread is used. + virtual void RunOnTarget(Resolver* aResolver, const QuotaInfo& aQuotaInfo, + Data* aOptionalData) = 0; + + // Called on initiating thread when the Action is canceled. The Action is + // responsible for calling Resolver::Resolve() as normal; either with a + // normal error code or NS_ERROR_ABORT. If CancelOnInitiatingThread() is + // called after Resolve() has already occurred, then the cancel can be + // ignored. + // + // Cancellation is a best effort to stop processing as soon as possible, but + // does not guarantee the Action will not run. + // + // CancelOnInitiatingThread() may be called more than once. Subsequent + // calls should have no effect. + // + // Default implementation sets an internal cancellation flag that can be + // queried with IsCanceled(). + virtual void CancelOnInitiatingThread(); + + // Executed on the initiating thread and is passed the nsresult given to + // Resolver::Resolve(). + virtual void CompleteOnInitiatingThread(nsresult aRv) { } + + // Executed on the initiating thread. If this Action will operate on the + // given cache ID then override this to return true. + virtual bool MatchesCacheId(CacheId aCacheId) const { return false; } + + NS_INLINE_DECL_REFCOUNTING(cache::Action) + +protected: + Action(); + + // virtual because deleted through base class pointer + virtual ~Action(); + + // Check if this Action has been canceled. May be called from any thread, + // but typically used from the target thread. + bool IsCanceled() const; + +private: + // Accessible from any thread. + Atomic<bool> mCanceled; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_Action_h diff --git a/dom/cache/ActorChild.cpp b/dom/cache/ActorChild.cpp new file mode 100644 index 0000000000..d3bd0553c8 --- /dev/null +++ b/dom/cache/ActorChild.cpp @@ -0,0 +1,66 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/ActorChild.h" + +#include "mozilla/dom/cache/CacheWorkerHolder.h" +#include "nsThreadUtils.h" + +namespace mozilla { +namespace dom { +namespace cache { + +void +ActorChild::SetWorkerHolder(CacheWorkerHolder* aWorkerHolder) +{ + // Some of the Cache actors can have multiple DOM objects associated with + // them. In this case the workerHolder will be added multiple times. This is + // permitted, but the workerHolder should be the same each time. + if (mWorkerHolder) { + MOZ_DIAGNOSTIC_ASSERT(mWorkerHolder == aWorkerHolder); + return; + } + + mWorkerHolder = aWorkerHolder; + if (mWorkerHolder) { + mWorkerHolder->AddActor(this); + } +} + +void +ActorChild::RemoveWorkerHolder() +{ + MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerHolder); + if (mWorkerHolder) { + mWorkerHolder->RemoveActor(this); + mWorkerHolder = nullptr; + } +} + +CacheWorkerHolder* +ActorChild::GetWorkerHolder() const +{ + return mWorkerHolder; +} + +bool +ActorChild::WorkerHolderNotified() const +{ + return mWorkerHolder && mWorkerHolder->Notified(); +} + +ActorChild::ActorChild() +{ +} + +ActorChild::~ActorChild() +{ + MOZ_DIAGNOSTIC_ASSERT(!mWorkerHolder); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/ActorChild.h b/dom/cache/ActorChild.h new file mode 100644 index 0000000000..b56397e588 --- /dev/null +++ b/dom/cache/ActorChild.h @@ -0,0 +1,48 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_ActioChild_h +#define mozilla_dom_cache_ActioChild_h + +#include "mozilla/RefPtr.h" + +namespace mozilla { +namespace dom { +namespace cache { + +class CacheWorkerHolder; + +class ActorChild +{ +public: + virtual void + StartDestroy() = 0; + + void + SetWorkerHolder(CacheWorkerHolder* aWorkerHolder); + + void + RemoveWorkerHolder(); + + CacheWorkerHolder* + GetWorkerHolder() const; + + bool + WorkerHolderNotified() const; + +protected: + ActorChild(); + ~ActorChild(); + +private: + RefPtr<CacheWorkerHolder> mWorkerHolder; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_ActioChild_h diff --git a/dom/cache/ActorUtils.h b/dom/cache/ActorUtils.h new file mode 100644 index 0000000000..7528821c9f --- /dev/null +++ b/dom/cache/ActorUtils.h @@ -0,0 +1,65 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_ActorUtils_h +#define mozilla_dom_cache_ActorUtils_h + +#include "mozilla/dom/cache/Types.h" + +namespace mozilla { + +namespace ipc { +class PBackgroundParent; +class PrincipalInfo; +} // namespace ipc + +namespace dom { +namespace cache { + +class PCacheChild; +class PCacheParent; +class PCacheStreamControlChild; +class PCacheStreamControlParent; +class PCacheStorageChild; +class PCacheStorageParent; + +// Factory methods for use in ipc/glue methods. Implemented in individual actor +// cpp files. + +PCacheChild* +AllocPCacheChild(); + +void +DeallocPCacheChild(PCacheChild* aActor); + +void +DeallocPCacheParent(PCacheParent* aActor); + +PCacheStreamControlChild* +AllocPCacheStreamControlChild(); + +void +DeallocPCacheStreamControlChild(PCacheStreamControlChild* aActor); + +void +DeallocPCacheStreamControlParent(PCacheStreamControlParent* aActor); + +PCacheStorageParent* +AllocPCacheStorageParent(mozilla::ipc::PBackgroundParent* aManagingActor, + Namespace aNamespace, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo); + +void +DeallocPCacheStorageChild(PCacheStorageChild* aActor); + +void +DeallocPCacheStorageParent(PCacheStorageParent* aActor); + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_ActorUtils_h diff --git a/dom/cache/AutoUtils.cpp b/dom/cache/AutoUtils.cpp new file mode 100644 index 0000000000..c64b47f7a1 --- /dev/null +++ b/dom/cache/AutoUtils.cpp @@ -0,0 +1,565 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/AutoUtils.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/InternalHeaders.h" +#include "mozilla/dom/InternalRequest.h" +#include "mozilla/dom/cache/CacheParent.h" +#include "mozilla/dom/cache/CacheStreamControlParent.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/dom/cache/SavedTypes.h" +#include "mozilla/dom/cache/StreamList.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "mozilla/ipc/IPCStreamUtils.h" +#include "mozilla/ipc/PBackgroundParent.h" +#include "nsCRT.h" +#include "nsHttp.h" + +using mozilla::Unused; +using mozilla::dom::cache::CacheReadStream; +using mozilla::dom::cache::CacheReadStreamOrVoid; +using mozilla::ipc::AutoIPCStream; +using mozilla::ipc::PBackgroundParent; + +namespace { + +enum CleanupAction +{ + Forget, + Delete +}; + +void +CleanupChild(CacheReadStream& aReadStream, CleanupAction aAction) +{ + // fds cleaned up by mStreamCleanupList + // PSendStream actors cleaned up by mStreamCleanupList +} + +void +CleanupChild(CacheReadStreamOrVoid& aReadStreamOrVoid, CleanupAction aAction) +{ + if (aReadStreamOrVoid.type() == CacheReadStreamOrVoid::Tvoid_t) { + return; + } + + CleanupChild(aReadStreamOrVoid.get_CacheReadStream(), aAction); +} + +} // namespace + +namespace mozilla { +namespace dom { +namespace cache { + +// -------------------------------------------- + +AutoChildOpArgs::AutoChildOpArgs(TypeUtils* aTypeUtils, + const CacheOpArgs& aOpArgs, + uint32_t aEntryCount) + : mTypeUtils(aTypeUtils) + , mOpArgs(aOpArgs) + , mSent(false) +{ + MOZ_DIAGNOSTIC_ASSERT(mTypeUtils); + MOZ_RELEASE_ASSERT(aEntryCount != 0); + // We are using AutoIPCStream objects to cleanup target IPCStream + // structures embedded in our CacheOpArgs. These IPCStream structs + // must not move once we attach our AutoIPCStream to them. Therefore, + // its important that any arrays containing streams are pre-sized for + // the number of entries we have in order to avoid realloc moving + // things around on us. + if (mOpArgs.type() == CacheOpArgs::TCachePutAllArgs) { + CachePutAllArgs& args = mOpArgs.get_CachePutAllArgs(); + args.requestResponseList().SetCapacity(aEntryCount); + } else { + MOZ_DIAGNOSTIC_ASSERT(aEntryCount == 1); + } +} + +AutoChildOpArgs::~AutoChildOpArgs() +{ + CleanupAction action = mSent ? Forget : Delete; + + switch(mOpArgs.type()) { + case CacheOpArgs::TCacheMatchArgs: + { + CacheMatchArgs& args = mOpArgs.get_CacheMatchArgs(); + CleanupChild(args.request().body(), action); + break; + } + case CacheOpArgs::TCacheMatchAllArgs: + { + CacheMatchAllArgs& args = mOpArgs.get_CacheMatchAllArgs(); + if (args.requestOrVoid().type() == CacheRequestOrVoid::Tvoid_t) { + break; + } + CleanupChild(args.requestOrVoid().get_CacheRequest().body(), action); + break; + } + case CacheOpArgs::TCachePutAllArgs: + { + CachePutAllArgs& args = mOpArgs.get_CachePutAllArgs(); + auto& list = args.requestResponseList(); + for (uint32_t i = 0; i < list.Length(); ++i) { + CleanupChild(list[i].request().body(), action); + CleanupChild(list[i].response().body(), action); + } + break; + } + case CacheOpArgs::TCacheDeleteArgs: + { + CacheDeleteArgs& args = mOpArgs.get_CacheDeleteArgs(); + CleanupChild(args.request().body(), action); + break; + } + case CacheOpArgs::TCacheKeysArgs: + { + CacheKeysArgs& args = mOpArgs.get_CacheKeysArgs(); + if (args.requestOrVoid().type() == CacheRequestOrVoid::Tvoid_t) { + break; + } + CleanupChild(args.requestOrVoid().get_CacheRequest().body(), action); + break; + } + case CacheOpArgs::TStorageMatchArgs: + { + StorageMatchArgs& args = mOpArgs.get_StorageMatchArgs(); + CleanupChild(args.request().body(), action); + break; + } + default: + // Other types do not need cleanup + break; + } + + mStreamCleanupList.Clear(); +} + +void +AutoChildOpArgs::Add(InternalRequest* aRequest, BodyAction aBodyAction, + SchemeAction aSchemeAction, ErrorResult& aRv) +{ + MOZ_DIAGNOSTIC_ASSERT(!mSent); + + switch(mOpArgs.type()) { + case CacheOpArgs::TCacheMatchArgs: + { + CacheMatchArgs& args = mOpArgs.get_CacheMatchArgs(); + mTypeUtils->ToCacheRequest(args.request(), aRequest, aBodyAction, + aSchemeAction, mStreamCleanupList, aRv); + break; + } + case CacheOpArgs::TCacheMatchAllArgs: + { + CacheMatchAllArgs& args = mOpArgs.get_CacheMatchAllArgs(); + MOZ_DIAGNOSTIC_ASSERT(args.requestOrVoid().type() == CacheRequestOrVoid::Tvoid_t); + args.requestOrVoid() = CacheRequest(); + mTypeUtils->ToCacheRequest(args.requestOrVoid().get_CacheRequest(), + aRequest, aBodyAction, aSchemeAction, + mStreamCleanupList, aRv); + break; + } + case CacheOpArgs::TCacheDeleteArgs: + { + CacheDeleteArgs& args = mOpArgs.get_CacheDeleteArgs(); + mTypeUtils->ToCacheRequest(args.request(), aRequest, aBodyAction, + aSchemeAction, mStreamCleanupList, aRv); + break; + } + case CacheOpArgs::TCacheKeysArgs: + { + CacheKeysArgs& args = mOpArgs.get_CacheKeysArgs(); + MOZ_DIAGNOSTIC_ASSERT(args.requestOrVoid().type() == CacheRequestOrVoid::Tvoid_t); + args.requestOrVoid() = CacheRequest(); + mTypeUtils->ToCacheRequest(args.requestOrVoid().get_CacheRequest(), + aRequest, aBodyAction, aSchemeAction, + mStreamCleanupList, aRv); + break; + } + case CacheOpArgs::TStorageMatchArgs: + { + StorageMatchArgs& args = mOpArgs.get_StorageMatchArgs(); + mTypeUtils->ToCacheRequest(args.request(), aRequest, aBodyAction, + aSchemeAction, mStreamCleanupList, aRv); + break; + } + default: + MOZ_CRASH("Cache args type cannot send a Request!"); + } +} + +namespace { + +bool +MatchInPutList(InternalRequest* aRequest, + const nsTArray<CacheRequestResponse>& aPutList) +{ + MOZ_DIAGNOSTIC_ASSERT(aRequest); + + // This method implements the SW spec QueryCache algorithm against an + // in memory array of Request/Response objects. This essentially the + // same algorithm that is implemented in DBSchema.cpp. Unfortunately + // we cannot unify them because when operating against the real database + // we don't want to load all request/response objects into memory. + + // Note, we can skip the check for a invalid request method because + // Cache should only call into here with a GET or HEAD. +#ifdef DEBUG + nsAutoCString method; + aRequest->GetMethod(method); + MOZ_ASSERT(method.LowerCaseEqualsLiteral("get") || + method.LowerCaseEqualsLiteral("head")); +#endif + + RefPtr<InternalHeaders> requestHeaders = aRequest->Headers(); + + for (uint32_t i = 0; i < aPutList.Length(); ++i) { + const CacheRequest& cachedRequest = aPutList[i].request(); + const CacheResponse& cachedResponse = aPutList[i].response(); + + nsAutoCString url; + aRequest->GetURL(url); + + nsAutoCString requestUrl(cachedRequest.urlWithoutQuery()); + requestUrl.Append(cachedRequest.urlQuery()); + + // If the URLs don't match, then just skip to the next entry. + if (url != requestUrl) { + continue; + } + + RefPtr<InternalHeaders> cachedRequestHeaders = + TypeUtils::ToInternalHeaders(cachedRequest.headers()); + + RefPtr<InternalHeaders> cachedResponseHeaders = + TypeUtils::ToInternalHeaders(cachedResponse.headers()); + + nsCString varyHeaders; + ErrorResult rv; + cachedResponseHeaders->Get(NS_LITERAL_CSTRING("vary"), varyHeaders, rv); + MOZ_ALWAYS_TRUE(!rv.Failed()); + + // Assume the vary headers match until we find a conflict + bool varyHeadersMatch = true; + + char* rawBuffer = varyHeaders.BeginWriting(); + char* token = nsCRT::strtok(rawBuffer, NS_HTTP_HEADER_SEPS, &rawBuffer); + for (; token; + token = nsCRT::strtok(rawBuffer, NS_HTTP_HEADER_SEPS, &rawBuffer)) { + nsDependentCString header(token); + MOZ_DIAGNOSTIC_ASSERT(!header.EqualsLiteral("*"), + "We should have already caught this in " + "TypeUtils::ToPCacheResponseWithoutBody()"); + + ErrorResult headerRv; + nsAutoCString value; + requestHeaders->Get(header, value, headerRv); + if (NS_WARN_IF(headerRv.Failed())) { + headerRv.SuppressException(); + MOZ_DIAGNOSTIC_ASSERT(value.IsEmpty()); + } + + nsAutoCString cachedValue; + cachedRequestHeaders->Get(header, cachedValue, headerRv); + if (NS_WARN_IF(headerRv.Failed())) { + headerRv.SuppressException(); + MOZ_DIAGNOSTIC_ASSERT(cachedValue.IsEmpty()); + } + + if (value != cachedValue) { + varyHeadersMatch = false; + break; + } + } + + // URL was equal and all vary headers match! + if (varyHeadersMatch) { + return true; + } + } + + return false; +} + +} // namespace + +void +AutoChildOpArgs::Add(InternalRequest* aRequest, BodyAction aBodyAction, + SchemeAction aSchemeAction, Response& aResponse, + ErrorResult& aRv) +{ + MOZ_DIAGNOSTIC_ASSERT(!mSent); + + switch(mOpArgs.type()) { + case CacheOpArgs::TCachePutAllArgs: + { + CachePutAllArgs& args = mOpArgs.get_CachePutAllArgs(); + + // Throw an error if a request/response pair would mask another + // request/response pair in the same PutAll operation. This is + // step 2.3.2.3 from the "Batch Cache Operations" spec algorithm. + if (MatchInPutList(aRequest, args.requestResponseList())) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + return; + } + + // Ensure that we don't realloc the array since this can result + // in our AutoIPCStream objects to reference the wrong memory + // location. This should never happen and is a UAF if it does. + // Therefore make this a release assertion. + MOZ_RELEASE_ASSERT(args.requestResponseList().Length() < + args.requestResponseList().Capacity()); + + // The FileDescriptorSetChild asserts in its destructor that all fds have + // been removed. The copy constructor, however, simply duplicates the + // fds without removing any. This means each temporary and copy must be + // explicitly cleaned up. + // + // Avoid a lot of this hassle by making sure we only create one here. On + // error we remove it. + CacheRequestResponse& pair = *args.requestResponseList().AppendElement(); + pair.request().body() = void_t(); + pair.response().body() = void_t(); + + mTypeUtils->ToCacheRequest(pair.request(), aRequest, aBodyAction, + aSchemeAction, mStreamCleanupList, aRv); + if (!aRv.Failed()) { + mTypeUtils->ToCacheResponse(pair.response(), aResponse, + mStreamCleanupList, aRv); + } + + if (aRv.Failed()) { + CleanupChild(pair.request().body(), Delete); + args.requestResponseList().RemoveElementAt( + args.requestResponseList().Length() - 1); + } + + break; + } + default: + MOZ_CRASH("Cache args type cannot send a Request/Response pair!"); + } +} + +const CacheOpArgs& +AutoChildOpArgs::SendAsOpArgs() +{ + MOZ_DIAGNOSTIC_ASSERT(!mSent); + mSent = true; + for (UniquePtr<AutoIPCStream>& autoStream : mStreamCleanupList) { + autoStream->TakeValue(); + } + return mOpArgs; +} + +// -------------------------------------------- + +AutoParentOpResult::AutoParentOpResult(mozilla::ipc::PBackgroundParent* aManager, + const CacheOpResult& aOpResult, + uint32_t aEntryCount) + : mManager(aManager) + , mOpResult(aOpResult) + , mStreamControl(nullptr) + , mSent(false) +{ + MOZ_DIAGNOSTIC_ASSERT(mManager); + MOZ_RELEASE_ASSERT(aEntryCount != 0); + // We are using AutoIPCStream objects to cleanup target IPCStream + // structures embedded in our CacheOpArgs. These IPCStream structs + // must not move once we attach our AutoIPCStream to them. Therefore, + // its important that any arrays containing streams are pre-sized for + // the number of entries we have in order to avoid realloc moving + // things around on us. + if (mOpResult.type() == CacheOpResult::TCacheMatchAllResult) { + CacheMatchAllResult& result = mOpResult.get_CacheMatchAllResult(); + result.responseList().SetCapacity(aEntryCount); + } else if (mOpResult.type() == CacheOpResult::TCacheKeysResult) { + CacheKeysResult& result = mOpResult.get_CacheKeysResult(); + result.requestList().SetCapacity(aEntryCount); + } else { + MOZ_DIAGNOSTIC_ASSERT(aEntryCount == 1); + } +} + +AutoParentOpResult::~AutoParentOpResult() +{ + CleanupAction action = mSent ? Forget : Delete; + + switch (mOpResult.type()) { + case CacheOpResult::TStorageOpenResult: + { + StorageOpenResult& result = mOpResult.get_StorageOpenResult(); + if (action == Forget || result.actorParent() == nullptr) { + break; + } + Unused << PCacheParent::Send__delete__(result.actorParent()); + break; + } + default: + // other types do not need additional clean up + break; + } + + if (action == Delete && mStreamControl) { + Unused << PCacheStreamControlParent::Send__delete__(mStreamControl); + } + + mStreamCleanupList.Clear(); +} + +void +AutoParentOpResult::Add(CacheId aOpenedCacheId, Manager* aManager) +{ + MOZ_DIAGNOSTIC_ASSERT(mOpResult.type() == CacheOpResult::TStorageOpenResult); + MOZ_DIAGNOSTIC_ASSERT(mOpResult.get_StorageOpenResult().actorParent() == nullptr); + mOpResult.get_StorageOpenResult().actorParent() = + mManager->SendPCacheConstructor(new CacheParent(aManager, aOpenedCacheId)); +} + +void +AutoParentOpResult::Add(const SavedResponse& aSavedResponse, + StreamList* aStreamList) +{ + MOZ_DIAGNOSTIC_ASSERT(!mSent); + + switch (mOpResult.type()) { + case CacheOpResult::TCacheMatchResult: + { + CacheMatchResult& result = mOpResult.get_CacheMatchResult(); + MOZ_DIAGNOSTIC_ASSERT(result.responseOrVoid().type() == CacheResponseOrVoid::Tvoid_t); + result.responseOrVoid() = aSavedResponse.mValue; + SerializeResponseBody(aSavedResponse, aStreamList, + &result.responseOrVoid().get_CacheResponse()); + break; + } + case CacheOpResult::TCacheMatchAllResult: + { + CacheMatchAllResult& result = mOpResult.get_CacheMatchAllResult(); + // Ensure that we don't realloc the array since this can result + // in our AutoIPCStream objects to reference the wrong memory + // location. This should never happen and is a UAF if it does. + // Therefore make this a release assertion. + MOZ_RELEASE_ASSERT(result.responseList().Length() < + result.responseList().Capacity()); + result.responseList().AppendElement(aSavedResponse.mValue); + SerializeResponseBody(aSavedResponse, aStreamList, + &result.responseList().LastElement()); + break; + } + case CacheOpResult::TStorageMatchResult: + { + StorageMatchResult& result = mOpResult.get_StorageMatchResult(); + MOZ_DIAGNOSTIC_ASSERT(result.responseOrVoid().type() == CacheResponseOrVoid::Tvoid_t); + result.responseOrVoid() = aSavedResponse.mValue; + SerializeResponseBody(aSavedResponse, aStreamList, + &result.responseOrVoid().get_CacheResponse()); + break; + } + default: + MOZ_CRASH("Cache result type cannot handle returning a Response!"); + } +} + +void +AutoParentOpResult::Add(const SavedRequest& aSavedRequest, + StreamList* aStreamList) +{ + MOZ_DIAGNOSTIC_ASSERT(!mSent); + + switch (mOpResult.type()) { + case CacheOpResult::TCacheKeysResult: + { + CacheKeysResult& result = mOpResult.get_CacheKeysResult(); + // Ensure that we don't realloc the array since this can result + // in our AutoIPCStream objects to reference the wrong memory + // location. This should never happen and is a UAF if it does. + // Therefore make this a release assertion. + MOZ_RELEASE_ASSERT(result.requestList().Length() < + result.requestList().Capacity()); + result.requestList().AppendElement(aSavedRequest.mValue); + CacheRequest& request = result.requestList().LastElement(); + + if (!aSavedRequest.mHasBodyId) { + request.body() = void_t(); + break; + } + + request.body() = CacheReadStream(); + SerializeReadStream(aSavedRequest.mBodyId, aStreamList, + &request.body().get_CacheReadStream()); + break; + } + default: + MOZ_CRASH("Cache result type cannot handle returning a Request!"); + } +} + +const CacheOpResult& +AutoParentOpResult::SendAsOpResult() +{ + MOZ_DIAGNOSTIC_ASSERT(!mSent); + mSent = true; + for (UniquePtr<AutoIPCStream>& autoStream : mStreamCleanupList) { + autoStream->TakeValue(); + } + return mOpResult; +} + +void +AutoParentOpResult::SerializeResponseBody(const SavedResponse& aSavedResponse, + StreamList* aStreamList, + CacheResponse* aResponseOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aResponseOut); + + if (!aSavedResponse.mHasBodyId) { + aResponseOut->body() = void_t(); + return; + } + + aResponseOut->body() = CacheReadStream(); + SerializeReadStream(aSavedResponse.mBodyId, aStreamList, + &aResponseOut->body().get_CacheReadStream()); +} + +void +AutoParentOpResult::SerializeReadStream(const nsID& aId, StreamList* aStreamList, + CacheReadStream* aReadStreamOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aStreamList); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + MOZ_DIAGNOSTIC_ASSERT(!mSent); + + nsCOMPtr<nsIInputStream> stream = aStreamList->Extract(aId); + MOZ_DIAGNOSTIC_ASSERT(stream); + + if (!mStreamControl) { + mStreamControl = static_cast<CacheStreamControlParent*>( + mManager->SendPCacheStreamControlConstructor(new CacheStreamControlParent())); + + // If this failed, then the child process is gone. Warn and allow actor + // cleanup to proceed as normal. + if (!mStreamControl) { + NS_WARNING("Cache failed to create stream control actor."); + return; + } + } + + aStreamList->SetStreamControl(mStreamControl); + + RefPtr<ReadStream> readStream = ReadStream::Create(mStreamControl, + aId, stream); + ErrorResult rv; + readStream->Serialize(aReadStreamOut, mStreamCleanupList, rv); + MOZ_DIAGNOSTIC_ASSERT(!rv.Failed()); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/AutoUtils.h b/dom/cache/AutoUtils.h new file mode 100644 index 0000000000..595a48c780 --- /dev/null +++ b/dom/cache/AutoUtils.h @@ -0,0 +1,104 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_AutoUtils_h +#define mozilla_dom_cache_AutoUtils_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "nsTArray.h" + +struct nsID; + +namespace mozilla { + +class ErrorResult; + +namespace ipc { +class PBackgroundParent; +class AutoIPCStream; +} // namespace ipc + +namespace dom { + +class InternalRequest; + +namespace cache { + +class CacheStreamControlParent; +class Manager; +struct SavedRequest; +struct SavedResponse; +class StreamList; + +// A collection of RAII-style helper classes to ensure that IPC +// FileDescriptorSet actors are properly cleaned up. The user of these actors +// must manually either Forget() the Fds or Send__delete__() the actor +// depending on if the descriptors were actually sent. +// +// Note, these should only be used when *sending* streams across IPC. The +// deserialization case is handled by creating a ReadStream object. + +class MOZ_STACK_CLASS AutoChildOpArgs final +{ +public: + typedef TypeUtils::BodyAction BodyAction; + typedef TypeUtils::SchemeAction SchemeAction; + + AutoChildOpArgs(TypeUtils* aTypeUtils, const CacheOpArgs& aOpArgs, + uint32_t aEntryCount); + ~AutoChildOpArgs(); + + void Add(InternalRequest* aRequest, BodyAction aBodyAction, + SchemeAction aSchemeAction, ErrorResult& aRv); + void Add(InternalRequest* aRequest, BodyAction aBodyAction, + SchemeAction aSchemeAction, Response& aResponse, ErrorResult& aRv); + + const CacheOpArgs& SendAsOpArgs(); + +private: + TypeUtils* mTypeUtils; + CacheOpArgs mOpArgs; + nsTArray<UniquePtr<mozilla::ipc::AutoIPCStream>> mStreamCleanupList; + bool mSent; +}; + +class MOZ_STACK_CLASS AutoParentOpResult final +{ +public: + AutoParentOpResult(mozilla::ipc::PBackgroundParent* aManager, + const CacheOpResult& aOpResult, + uint32_t aEntryCount); + ~AutoParentOpResult(); + + void Add(CacheId aOpenedCacheId, Manager* aManager); + void Add(const SavedResponse& aSavedResponse, StreamList* aStreamList); + void Add(const SavedRequest& aSavedRequest, StreamList* aStreamList); + + const CacheOpResult& SendAsOpResult(); + +private: + void SerializeResponseBody(const SavedResponse& aSavedResponse, + StreamList* aStreamList, + CacheResponse* aResponseOut); + + void SerializeReadStream(const nsID& aId, StreamList* aStreamList, + CacheReadStream* aReadStreamOut); + + mozilla::ipc::PBackgroundParent* mManager; + CacheOpResult mOpResult; + CacheStreamControlParent* mStreamControl; + nsTArray<UniquePtr<mozilla::ipc::AutoIPCStream>> mStreamCleanupList; + bool mSent; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_AutoUtils_h diff --git a/dom/cache/Cache.cpp b/dom/cache/Cache.cpp new file mode 100644 index 0000000000..0d5815edbf --- /dev/null +++ b/dom/cache/Cache.cpp @@ -0,0 +1,659 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/Cache.h" + +#include "mozilla/dom/Headers.h" +#include "mozilla/dom/InternalResponse.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/PromiseNativeHandler.h" +#include "mozilla/dom/Response.h" +#include "mozilla/dom/WorkerPrivate.h" +#include "mozilla/dom/CacheBinding.h" +#include "mozilla/dom/cache/AutoUtils.h" +#include "mozilla/dom/cache/CacheChild.h" +#include "mozilla/dom/cache/CacheWorkerHolder.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Preferences.h" +#include "mozilla/Unused.h" +#include "nsIGlobalObject.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::dom::workers::GetCurrentThreadWorkerPrivate; +using mozilla::dom::workers::WorkerPrivate; +using mozilla::ipc::PBackgroundChild; + +namespace { + +bool +IsValidPutRequestURL(const nsAString& aUrl, ErrorResult& aRv) +{ + bool validScheme = false; + + // make a copy because ProcessURL strips the fragmet + NS_ConvertUTF16toUTF8 url(aUrl); + + TypeUtils::ProcessURL(url, &validScheme, nullptr, nullptr, aRv); + if (aRv.Failed()) { + return false; + } + + if (!validScheme) { + aRv.ThrowTypeError<MSG_INVALID_URL_SCHEME>(NS_LITERAL_STRING("Request"), + aUrl); + return false; + } + + return true; +} + +static bool +IsValidPutRequestMethod(const Request& aRequest, ErrorResult& aRv) +{ + nsAutoCString method; + aRequest.GetMethod(method); + if (!method.LowerCaseEqualsLiteral("get")) { + NS_ConvertASCIItoUTF16 label(method); + aRv.ThrowTypeError<MSG_INVALID_REQUEST_METHOD>(label); + return false; + } + + return true; +} + +static bool +IsValidPutRequestMethod(const RequestOrUSVString& aRequest, ErrorResult& aRv) +{ + // If the provided request is a string URL, then it will default to + // a valid http method automatically. + if (!aRequest.IsRequest()) { + return true; + } + return IsValidPutRequestMethod(aRequest.GetAsRequest(), aRv); +} + +} // namespace + +// Helper class to wait for Add()/AddAll() fetch requests to complete and +// then perform a PutAll() with the responses. This class holds a WorkerHolder +// to keep the Worker thread alive. This is mainly to ensure that Add/AddAll +// act the same as other Cache operations that directly create a CacheOpChild +// actor. +class Cache::FetchHandler final : public PromiseNativeHandler +{ +public: + FetchHandler(CacheWorkerHolder* aWorkerHolder, Cache* aCache, + nsTArray<RefPtr<Request>>&& aRequestList, Promise* aPromise) + : mWorkerHolder(aWorkerHolder) + , mCache(aCache) + , mRequestList(Move(aRequestList)) + , mPromise(aPromise) + { + MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerHolder); + MOZ_DIAGNOSTIC_ASSERT(mCache); + MOZ_DIAGNOSTIC_ASSERT(mPromise); + } + + virtual void + ResolvedCallback(JSContext* aCx, JS::Handle<JS::Value> aValue) override + { + NS_ASSERT_OWNINGTHREAD(FetchHandler); + + // Stop holding the worker alive when we leave this method. + RefPtr<CacheWorkerHolder> workerHolder; + workerHolder.swap(mWorkerHolder); + + // Promise::All() passed an array of fetch() Promises should give us + // an Array of Response objects. The following code unwraps these + // JS values back to an nsTArray<RefPtr<Response>>. + + AutoTArray<RefPtr<Response>, 256> responseList; + responseList.SetCapacity(mRequestList.Length()); + + bool isArray; + if (NS_WARN_IF(!JS_IsArrayObject(aCx, aValue, &isArray) || !isArray)) { + Fail(); + return; + } + + JS::Rooted<JSObject*> obj(aCx, &aValue.toObject()); + + uint32_t length; + if (NS_WARN_IF(!JS_GetArrayLength(aCx, obj, &length))) { + Fail(); + return; + } + + for (uint32_t i = 0; i < length; ++i) { + JS::Rooted<JS::Value> value(aCx); + + if (NS_WARN_IF(!JS_GetElement(aCx, obj, i, &value))) { + Fail(); + return; + } + + if (NS_WARN_IF(!value.isObject())) { + Fail(); + return; + } + + JS::Rooted<JSObject*> responseObj(aCx, &value.toObject()); + + RefPtr<Response> response; + nsresult rv = UNWRAP_OBJECT(Response, responseObj, response); + if (NS_WARN_IF(NS_FAILED(rv))) { + Fail(); + return; + } + + if (NS_WARN_IF(response->Type() == ResponseType::Error)) { + Fail(); + return; + } + + // Do not allow the convenience methods .add()/.addAll() to store failed + // responses. A consequence of this is that these methods cannot be + // used to store opaque or opaqueredirect responses since they always + // expose a 0 status value. + if (!response->Ok()) { + uint32_t t = static_cast<uint32_t>(response->Type()); + NS_ConvertASCIItoUTF16 type(ResponseTypeValues::strings[t].value, + ResponseTypeValues::strings[t].length); + nsAutoString status; + status.AppendInt(response->Status()); + nsAutoString url; + mRequestList[i]->GetUrl(url); + ErrorResult rv; + rv.ThrowTypeError<MSG_CACHE_ADD_FAILED_RESPONSE>(type, status, url); + + // TODO: abort the fetch requests we have running (bug 1157434) + mPromise->MaybeReject(rv); + return; + } + + responseList.AppendElement(Move(response)); + } + + MOZ_DIAGNOSTIC_ASSERT(mRequestList.Length() == responseList.Length()); + + // Now store the unwrapped Response list in the Cache. + ErrorResult result; + RefPtr<Promise> put = mCache->PutAll(mRequestList, responseList, result); + if (NS_WARN_IF(result.Failed())) { + // TODO: abort the fetch requests we have running (bug 1157434) + mPromise->MaybeReject(result); + return; + } + + // Chain the Cache::Put() promise to the original promise returned to + // the content script. + mPromise->MaybeResolve(put); + } + + virtual void + RejectedCallback(JSContext* aCx, JS::Handle<JS::Value> aValue) override + { + NS_ASSERT_OWNINGTHREAD(FetchHandler); + Fail(); + } + +private: + ~FetchHandler() + { + } + + void + Fail() + { + ErrorResult rv; + rv.ThrowTypeError<MSG_FETCH_FAILED>(); + mPromise->MaybeReject(rv); + } + + RefPtr<CacheWorkerHolder> mWorkerHolder; + RefPtr<Cache> mCache; + nsTArray<RefPtr<Request>> mRequestList; + RefPtr<Promise> mPromise; + + NS_DECL_ISUPPORTS +}; + +NS_IMPL_ISUPPORTS0(Cache::FetchHandler) + +NS_IMPL_CYCLE_COLLECTING_ADDREF(mozilla::dom::cache::Cache); +NS_IMPL_CYCLE_COLLECTING_RELEASE(mozilla::dom::cache::Cache); +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(mozilla::dom::cache::Cache, mGlobal); + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(Cache) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +Cache::Cache(nsIGlobalObject* aGlobal, CacheChild* aActor) + : mGlobal(aGlobal) + , mActor(aActor) +{ + MOZ_DIAGNOSTIC_ASSERT(mGlobal); + MOZ_DIAGNOSTIC_ASSERT(mActor); + mActor->SetListener(this); +} + +already_AddRefed<Promise> +Cache::Match(const RequestOrUSVString& aRequest, + const CacheQueryOptions& aOptions, ErrorResult& aRv) +{ + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(mActor); + + RefPtr<InternalRequest> ir = ToInternalRequest(aRequest, IgnoreBody, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + CacheQueryParams params; + ToCacheQueryParams(params, aOptions); + + AutoChildOpArgs args(this, CacheMatchArgs(CacheRequest(), params), 1); + + args.Add(ir, IgnoreBody, IgnoreInvalidScheme, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + return ExecuteOp(args, aRv); +} + +already_AddRefed<Promise> +Cache::MatchAll(const Optional<RequestOrUSVString>& aRequest, + const CacheQueryOptions& aOptions, ErrorResult& aRv) +{ + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(mActor); + + CacheQueryParams params; + ToCacheQueryParams(params, aOptions); + + AutoChildOpArgs args(this, CacheMatchAllArgs(void_t(), params), 1); + + if (aRequest.WasPassed()) { + RefPtr<InternalRequest> ir = ToInternalRequest(aRequest.Value(), + IgnoreBody, aRv); + if (aRv.Failed()) { + return nullptr; + } + + args.Add(ir, IgnoreBody, IgnoreInvalidScheme, aRv); + if (aRv.Failed()) { + return nullptr; + } + } + + return ExecuteOp(args, aRv); +} + +already_AddRefed<Promise> +Cache::Add(JSContext* aContext, const RequestOrUSVString& aRequest, + ErrorResult& aRv) +{ + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(mActor); + + if (!IsValidPutRequestMethod(aRequest, aRv)) { + return nullptr; + } + + GlobalObject global(aContext, mGlobal->GetGlobalJSObject()); + MOZ_DIAGNOSTIC_ASSERT(!global.Failed()); + + nsTArray<RefPtr<Request>> requestList(1); + RefPtr<Request> request = Request::Constructor(global, aRequest, + RequestInit(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + nsAutoString url; + request->GetUrl(url); + if (NS_WARN_IF(!IsValidPutRequestURL(url, aRv))) { + return nullptr; + } + + requestList.AppendElement(Move(request)); + return AddAll(global, Move(requestList), aRv); +} + +already_AddRefed<Promise> +Cache::AddAll(JSContext* aContext, + const Sequence<OwningRequestOrUSVString>& aRequestList, + ErrorResult& aRv) +{ + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(mActor); + + GlobalObject global(aContext, mGlobal->GetGlobalJSObject()); + MOZ_DIAGNOSTIC_ASSERT(!global.Failed()); + + nsTArray<RefPtr<Request>> requestList(aRequestList.Length()); + for (uint32_t i = 0; i < aRequestList.Length(); ++i) { + RequestOrUSVString requestOrString; + + if (aRequestList[i].IsRequest()) { + requestOrString.SetAsRequest() = aRequestList[i].GetAsRequest(); + if (NS_WARN_IF(!IsValidPutRequestMethod(requestOrString.GetAsRequest(), + aRv))) { + return nullptr; + } + } else { + requestOrString.SetAsUSVString().Rebind( + aRequestList[i].GetAsUSVString().Data(), + aRequestList[i].GetAsUSVString().Length()); + } + + RefPtr<Request> request = Request::Constructor(global, requestOrString, + RequestInit(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + nsAutoString url; + request->GetUrl(url); + if (NS_WARN_IF(!IsValidPutRequestURL(url, aRv))) { + return nullptr; + } + + requestList.AppendElement(Move(request)); + } + + return AddAll(global, Move(requestList), aRv); +} + +already_AddRefed<Promise> +Cache::Put(const RequestOrUSVString& aRequest, Response& aResponse, + ErrorResult& aRv) +{ + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(mActor); + + if (NS_WARN_IF(!IsValidPutRequestMethod(aRequest, aRv))) { + return nullptr; + } + + RefPtr<InternalRequest> ir = ToInternalRequest(aRequest, ReadBody, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + AutoChildOpArgs args(this, CachePutAllArgs(), 1); + + args.Add(ir, ReadBody, TypeErrorOnInvalidScheme, + aResponse, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + return ExecuteOp(args, aRv); +} + +already_AddRefed<Promise> +Cache::Delete(const RequestOrUSVString& aRequest, + const CacheQueryOptions& aOptions, ErrorResult& aRv) +{ + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(mActor); + + RefPtr<InternalRequest> ir = ToInternalRequest(aRequest, IgnoreBody, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + CacheQueryParams params; + ToCacheQueryParams(params, aOptions); + + AutoChildOpArgs args(this, CacheDeleteArgs(CacheRequest(), params), 1); + + args.Add(ir, IgnoreBody, IgnoreInvalidScheme, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + return ExecuteOp(args, aRv); +} + +already_AddRefed<Promise> +Cache::Keys(const Optional<RequestOrUSVString>& aRequest, + const CacheQueryOptions& aOptions, ErrorResult& aRv) +{ + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(mActor); + + CacheQueryParams params; + ToCacheQueryParams(params, aOptions); + + AutoChildOpArgs args(this, CacheKeysArgs(void_t(), params), 1); + + if (aRequest.WasPassed()) { + RefPtr<InternalRequest> ir = ToInternalRequest(aRequest.Value(), + IgnoreBody, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + args.Add(ir, IgnoreBody, IgnoreInvalidScheme, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + } + + return ExecuteOp(args, aRv); +} + +// static +bool +Cache::PrefEnabled(JSContext* aCx, JSObject* aObj) +{ + using mozilla::dom::workers::WorkerPrivate; + using mozilla::dom::workers::GetWorkerPrivateFromContext; + + // If we're on the main thread, then check the pref directly. + if (NS_IsMainThread()) { + bool enabled = false; + Preferences::GetBool("dom.caches.enabled", &enabled); + return enabled; + } + + // Otherwise check the pref via the work private helper + WorkerPrivate* workerPrivate = GetWorkerPrivateFromContext(aCx); + if (!workerPrivate) { + return false; + } + + return workerPrivate->DOMCachesEnabled(); +} + +nsISupports* +Cache::GetParentObject() const +{ + return mGlobal; +} + +JSObject* +Cache::WrapObject(JSContext* aContext, JS::Handle<JSObject*> aGivenProto) +{ + return CacheBinding::Wrap(aContext, this, aGivenProto); +} + +void +Cache::DestroyInternal(CacheChild* aActor) +{ + MOZ_DIAGNOSTIC_ASSERT(mActor); + MOZ_DIAGNOSTIC_ASSERT(mActor == aActor); + mActor->ClearListener(); + mActor = nullptr; +} + +nsIGlobalObject* +Cache::GetGlobalObject() const +{ + return mGlobal; +} + +#ifdef DEBUG +void +Cache::AssertOwningThread() const +{ + NS_ASSERT_OWNINGTHREAD(Cache); +} +#endif + +PBackgroundChild* +Cache::GetIPCManager() +{ + NS_ASSERT_OWNINGTHREAD(Cache); + MOZ_DIAGNOSTIC_ASSERT(mActor); + return mActor->Manager(); +} + +Cache::~Cache() +{ + NS_ASSERT_OWNINGTHREAD(Cache); + if (mActor) { + mActor->StartDestroyFromListener(); + // DestroyInternal() is called synchronously by StartDestroyFromListener(). + // So we should have already cleared the mActor. + MOZ_DIAGNOSTIC_ASSERT(!mActor); + } +} + +already_AddRefed<Promise> +Cache::ExecuteOp(AutoChildOpArgs& aOpArgs, ErrorResult& aRv) +{ + MOZ_DIAGNOSTIC_ASSERT(mActor); + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + mActor->ExecuteOp(mGlobal, promise, this, aOpArgs.SendAsOpArgs()); + return promise.forget(); +} + +already_AddRefed<Promise> +Cache::AddAll(const GlobalObject& aGlobal, + nsTArray<RefPtr<Request>>&& aRequestList, ErrorResult& aRv) +{ + MOZ_DIAGNOSTIC_ASSERT(mActor); + + // If there is no work to do, then resolve immediately + if (aRequestList.IsEmpty()) { + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + promise->MaybeResolveWithUndefined(); + return promise.forget(); + } + + AutoTArray<RefPtr<Promise>, 256> fetchList; + fetchList.SetCapacity(aRequestList.Length()); + + // Begin fetching each request in parallel. For now, if an error occurs just + // abandon our previous fetch calls. In theory we could cancel them in the + // future once fetch supports it. + + for (uint32_t i = 0; i < aRequestList.Length(); ++i) { + RequestOrUSVString requestOrString; + requestOrString.SetAsRequest() = aRequestList[i]; + RefPtr<Promise> fetch = FetchRequest(mGlobal, requestOrString, + RequestInit(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + fetchList.AppendElement(Move(fetch)); + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + RefPtr<FetchHandler> handler = + new FetchHandler(mActor->GetWorkerHolder(), this, + Move(aRequestList), promise); + + RefPtr<Promise> fetchPromise = Promise::All(aGlobal, fetchList, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + fetchPromise->AppendNativeHandler(handler); + + return promise.forget(); +} + +already_AddRefed<Promise> +Cache::PutAll(const nsTArray<RefPtr<Request>>& aRequestList, + const nsTArray<RefPtr<Response>>& aResponseList, + ErrorResult& aRv) +{ + MOZ_DIAGNOSTIC_ASSERT(aRequestList.Length() == aResponseList.Length()); + + if (NS_WARN_IF(!mActor)) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + + CacheChild::AutoLock actorLock(mActor); + + AutoChildOpArgs args(this, CachePutAllArgs(), aRequestList.Length()); + + for (uint32_t i = 0; i < aRequestList.Length(); ++i) { + RefPtr<InternalRequest> ir = aRequestList[i]->GetInternalRequest(); + args.Add(ir, ReadBody, TypeErrorOnInvalidScheme, *aResponseList[i], aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + } + + return ExecuteOp(args, aRv); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/Cache.h b/dom/cache/Cache.h new file mode 100644 index 0000000000..ba11cda520 --- /dev/null +++ b/dom/cache/Cache.h @@ -0,0 +1,120 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Cache_h +#define mozilla_dom_cache_Cache_h + +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "nsCOMPtr.h" +#include "nsISupportsImpl.h" +#include "nsString.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { + +class ErrorResult; + +namespace dom { + +class OwningRequestOrUSVString; +class Promise; +struct CacheQueryOptions; +class RequestOrUSVString; +class Response; +template<typename T> class Optional; +template<typename T> class Sequence; + +namespace cache { + +class AutoChildOpArgs; +class CacheChild; + +class Cache final : public nsISupports + , public nsWrapperCache + , public TypeUtils +{ +public: + Cache(nsIGlobalObject* aGlobal, CacheChild* aActor); + + // webidl interface methods + already_AddRefed<Promise> + Match(const RequestOrUSVString& aRequest, const CacheQueryOptions& aOptions, + ErrorResult& aRv); + already_AddRefed<Promise> + MatchAll(const Optional<RequestOrUSVString>& aRequest, + const CacheQueryOptions& aOptions, ErrorResult& aRv); + already_AddRefed<Promise> + Add(JSContext* aContext, const RequestOrUSVString& aRequest, + ErrorResult& aRv); + already_AddRefed<Promise> + AddAll(JSContext* aContext, + const Sequence<OwningRequestOrUSVString>& aRequests, ErrorResult& aRv); + already_AddRefed<Promise> + Put(const RequestOrUSVString& aRequest, Response& aResponse, + ErrorResult& aRv); + already_AddRefed<Promise> + Delete(const RequestOrUSVString& aRequest, const CacheQueryOptions& aOptions, + ErrorResult& aRv); + already_AddRefed<Promise> + Keys(const Optional<RequestOrUSVString>& aRequest, + const CacheQueryOptions& aParams, ErrorResult& aRv); + + // binding methods + static bool PrefEnabled(JSContext* aCx, JSObject* aObj); + + nsISupports* GetParentObject() const; + virtual JSObject* WrapObject(JSContext* aContext, JS::Handle<JSObject*> aGivenProto) override; + + // Called when CacheChild actor is being destroyed + void DestroyInternal(CacheChild* aActor); + + // TypeUtils methods + virtual nsIGlobalObject* + GetGlobalObject() const override; + +#ifdef DEBUG + virtual void AssertOwningThread() const override; +#endif + + virtual mozilla::ipc::PBackgroundChild* + GetIPCManager() override; + +private: + class FetchHandler; + + ~Cache(); + + // Called when we're destroyed or CCed. + void DisconnectFromActor(); + + already_AddRefed<Promise> + ExecuteOp(AutoChildOpArgs& aOpArgs, ErrorResult& aRv); + + already_AddRefed<Promise> + AddAll(const GlobalObject& aGlobal, nsTArray<RefPtr<Request>>&& aRequestList, + ErrorResult& aRv); + + already_AddRefed<Promise> + PutAll(const nsTArray<RefPtr<Request>>& aRequestList, + const nsTArray<RefPtr<Response>>& aResponseList, + ErrorResult& aRv); + + nsCOMPtr<nsIGlobalObject> mGlobal; + CacheChild* mActor; + +public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS(Cache) +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_Cache_h diff --git a/dom/cache/CacheChild.cpp b/dom/cache/CacheChild.cpp new file mode 100644 index 0000000000..58902552f4 --- /dev/null +++ b/dom/cache/CacheChild.cpp @@ -0,0 +1,185 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheChild.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/ActorUtils.h" +#include "mozilla/dom/cache/Cache.h" +#include "mozilla/dom/cache/CacheOpChild.h" + +namespace mozilla { +namespace dom { +namespace cache { + +// Declared in ActorUtils.h +PCacheChild* +AllocPCacheChild() +{ + return new CacheChild(); +} + +// Declared in ActorUtils.h +void +DeallocPCacheChild(PCacheChild* aActor) +{ + delete aActor; +} + +CacheChild::CacheChild() + : mListener(nullptr) + , mNumChildActors(0) + , mDelayedDestroy(false) + , mLocked(false) +{ + MOZ_COUNT_CTOR(cache::CacheChild); +} + +CacheChild::~CacheChild() +{ + MOZ_COUNT_DTOR(cache::CacheChild); + NS_ASSERT_OWNINGTHREAD(CacheChild); + MOZ_DIAGNOSTIC_ASSERT(!mListener); + MOZ_DIAGNOSTIC_ASSERT(!mNumChildActors); + MOZ_DIAGNOSTIC_ASSERT(!mLocked); +} + +void +CacheChild::SetListener(Cache* aListener) +{ + NS_ASSERT_OWNINGTHREAD(CacheChild); + MOZ_DIAGNOSTIC_ASSERT(!mListener); + mListener = aListener; + MOZ_DIAGNOSTIC_ASSERT(mListener); +} + +void +CacheChild::ClearListener() +{ + NS_ASSERT_OWNINGTHREAD(CacheChild); + MOZ_DIAGNOSTIC_ASSERT(mListener); + mListener = nullptr; +} + +void +CacheChild::ExecuteOp(nsIGlobalObject* aGlobal, Promise* aPromise, + nsISupports* aParent, const CacheOpArgs& aArgs) +{ + mNumChildActors += 1; + MOZ_ALWAYS_TRUE(SendPCacheOpConstructor( + new CacheOpChild(GetWorkerHolder(), aGlobal, aParent, aPromise), aArgs)); +} + +void +CacheChild::StartDestroyFromListener() +{ + NS_ASSERT_OWNINGTHREAD(CacheChild); + + // The listener should be held alive by any async operations, so if it + // is going away then there must not be any child actors. This in turn + // ensures that StartDestroy() will not trigger the delayed path. + MOZ_DIAGNOSTIC_ASSERT(!mNumChildActors); + + StartDestroy(); +} + +void +CacheChild::StartDestroy() +{ + NS_ASSERT_OWNINGTHREAD(CacheChild); + + // If we have outstanding child actors, then don't destroy ourself yet. + // The child actors should be short lived and we should allow them to complete + // if possible. NoteDeletedActor() will call back into this Shutdown() + // method when the last child actor is gone. Also, delay destruction if we + // have been explicitly locked by someone using us on the stack. + if (mNumChildActors || mLocked) { + mDelayedDestroy = true; + return; + } + + RefPtr<Cache> listener = mListener; + + // StartDestroy() can get called from either Cache or the WorkerHolder. + // Theoretically we can get double called if the right race happens. Handle + // that by just ignoring the second StartDestroy() call. + if (!listener) { + return; + } + + listener->DestroyInternal(this); + + // Cache listener should call ClearListener() in DestroyInternal() + MOZ_DIAGNOSTIC_ASSERT(!mListener); + + // Start actor destruction from parent process + Unused << SendTeardown(); +} + +void +CacheChild::ActorDestroy(ActorDestroyReason aReason) +{ + NS_ASSERT_OWNINGTHREAD(CacheChild); + RefPtr<Cache> listener = mListener; + if (listener) { + listener->DestroyInternal(this); + // Cache listener should call ClearListener() in DestroyInternal() + MOZ_DIAGNOSTIC_ASSERT(!mListener); + } + + RemoveWorkerHolder(); +} + +PCacheOpChild* +CacheChild::AllocPCacheOpChild(const CacheOpArgs& aOpArgs) +{ + MOZ_CRASH("CacheOpChild should be manually constructed."); + return nullptr; +} + +bool +CacheChild::DeallocPCacheOpChild(PCacheOpChild* aActor) +{ + delete aActor; + NoteDeletedActor(); + return true; +} + +void +CacheChild::NoteDeletedActor() +{ + mNumChildActors -= 1; + MaybeFlushDelayedDestroy(); +} + +void +CacheChild::MaybeFlushDelayedDestroy() +{ + if (!mNumChildActors && !mLocked && mDelayedDestroy) { + StartDestroy(); + } +} + +void +CacheChild::Lock() +{ + NS_ASSERT_OWNINGTHREAD(CacheChild); + MOZ_DIAGNOSTIC_ASSERT(!mLocked); + mLocked = true; +} + +void +CacheChild::Unlock() +{ + NS_ASSERT_OWNINGTHREAD(CacheChild); + MOZ_DIAGNOSTIC_ASSERT(mLocked); + mLocked = false; + MaybeFlushDelayedDestroy(); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheChild.h b/dom/cache/CacheChild.h new file mode 100644 index 0000000000..8888007b06 --- /dev/null +++ b/dom/cache/CacheChild.h @@ -0,0 +1,111 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheChild_h +#define mozilla_dom_cache_CacheChild_h + +#include "mozilla/dom/cache/ActorChild.h" +#include "mozilla/dom/cache/PCacheChild.h" + +class nsIAsyncInputStream; +class nsIGlobalObject; + +namespace mozilla { +namespace dom { + +class Promise; + +namespace cache { + +class Cache; +class CacheOpArgs; + +class CacheChild final : public PCacheChild + , public ActorChild +{ +public: + class MOZ_RAII AutoLock final + { + CacheChild* mActor; + + public: + explicit AutoLock(CacheChild* aActor) + : mActor(aActor) + { + MOZ_DIAGNOSTIC_ASSERT(mActor); + mActor->Lock(); + } + + ~AutoLock() + { + mActor->Unlock(); + } + }; + + CacheChild(); + ~CacheChild(); + + void SetListener(Cache* aListener); + + // Must be called by the associated Cache listener in its DestroyInternal() + // method. Also, Cache must call StartDestroyFromListener() on the actor in + // its destructor to trigger ActorDestroy() if it has not been called yet. + void ClearListener(); + + void + ExecuteOp(nsIGlobalObject* aGlobal, Promise* aPromise, + nsISupports* aParent, const CacheOpArgs& aArgs); + + // Our parent Listener object has gone out of scope and is being destroyed. + void StartDestroyFromListener(); + +private: + // ActorChild methods + + // WorkerHolder is trying to destroy due to worker shutdown. + virtual void StartDestroy() override; + + // PCacheChild methods + virtual void + ActorDestroy(ActorDestroyReason aReason) override; + + virtual PCacheOpChild* + AllocPCacheOpChild(const CacheOpArgs& aOpArgs) override; + + virtual bool + DeallocPCacheOpChild(PCacheOpChild* aActor) override; + + // utility methods + void + NoteDeletedActor(); + + void + MaybeFlushDelayedDestroy(); + + // Methods used to temporarily force the actor alive. Only called from + // AutoLock. + void + Lock(); + + void + Unlock(); + + // Use a weak ref so actor does not hold DOM object alive past content use. + // The Cache object must call ClearListener() to null this before its + // destroyed. + Cache* MOZ_NON_OWNING_REF mListener; + uint32_t mNumChildActors; + bool mDelayedDestroy; + bool mLocked; + + NS_DECL_OWNINGTHREAD +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheChild_h diff --git a/dom/cache/CacheOpChild.cpp b/dom/cache/CacheOpChild.cpp new file mode 100644 index 0000000000..fecac83070 --- /dev/null +++ b/dom/cache/CacheOpChild.cpp @@ -0,0 +1,268 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheOpChild.h" + +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/Request.h" +#include "mozilla/dom/Response.h" +#include "mozilla/dom/cache/Cache.h" +#include "mozilla/dom/cache/CacheChild.h" +#include "mozilla/dom/cache/CacheStreamControlChild.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::ipc::PBackgroundChild; + +namespace { + +void +AddWorkerHolderToStreamChild(const CacheReadStream& aReadStream, + CacheWorkerHolder* aWorkerHolder) +{ + MOZ_ASSERT_IF(!NS_IsMainThread(), aWorkerHolder); + CacheStreamControlChild* cacheControl = + static_cast<CacheStreamControlChild*>(aReadStream.controlChild()); + if (cacheControl) { + cacheControl->SetWorkerHolder(aWorkerHolder); + } +} + +void +AddWorkerHolderToStreamChild(const CacheResponse& aResponse, + CacheWorkerHolder* aWorkerHolder) +{ + MOZ_ASSERT_IF(!NS_IsMainThread(), aWorkerHolder); + + if (aResponse.body().type() == CacheReadStreamOrVoid::Tvoid_t) { + return; + } + + AddWorkerHolderToStreamChild(aResponse.body().get_CacheReadStream(), + aWorkerHolder); +} + +void +AddWorkerHolderToStreamChild(const CacheRequest& aRequest, + CacheWorkerHolder* aWorkerHolder) +{ + MOZ_ASSERT_IF(!NS_IsMainThread(), aWorkerHolder); + + if (aRequest.body().type() == CacheReadStreamOrVoid::Tvoid_t) { + return; + } + + AddWorkerHolderToStreamChild(aRequest.body().get_CacheReadStream(), + aWorkerHolder); +} + +} // namespace + +CacheOpChild::CacheOpChild(CacheWorkerHolder* aWorkerHolder, + nsIGlobalObject* aGlobal, + nsISupports* aParent, Promise* aPromise) + : mGlobal(aGlobal) + , mParent(aParent) + , mPromise(aPromise) +{ + MOZ_DIAGNOSTIC_ASSERT(mGlobal); + MOZ_DIAGNOSTIC_ASSERT(mParent); + MOZ_DIAGNOSTIC_ASSERT(mPromise); + + MOZ_ASSERT_IF(!NS_IsMainThread(), aWorkerHolder); + SetWorkerHolder(aWorkerHolder); +} + +CacheOpChild::~CacheOpChild() +{ + NS_ASSERT_OWNINGTHREAD(CacheOpChild); + MOZ_DIAGNOSTIC_ASSERT(!mPromise); +} + +void +CacheOpChild::ActorDestroy(ActorDestroyReason aReason) +{ + NS_ASSERT_OWNINGTHREAD(CacheOpChild); + + // If the actor was terminated for some unknown reason, then indicate the + // operation is dead. + if (mPromise) { + mPromise->MaybeReject(NS_ERROR_FAILURE); + mPromise = nullptr; + } + + RemoveWorkerHolder(); +} + +bool +CacheOpChild::Recv__delete__(const ErrorResult& aRv, + const CacheOpResult& aResult) +{ + NS_ASSERT_OWNINGTHREAD(CacheOpChild); + + if (NS_WARN_IF(aRv.Failed())) { + MOZ_DIAGNOSTIC_ASSERT(aResult.type() == CacheOpResult::Tvoid_t); + // TODO: Remove this const_cast (bug 1152078). + // It is safe for now since this ErrorResult is handed off to us by IPDL + // and is thrown into the trash afterwards. + mPromise->MaybeReject(const_cast<ErrorResult&>(aRv)); + mPromise = nullptr; + return true; + } + + switch (aResult.type()) { + case CacheOpResult::TCacheMatchResult: + { + HandleResponse(aResult.get_CacheMatchResult().responseOrVoid()); + break; + } + case CacheOpResult::TCacheMatchAllResult: + { + HandleResponseList(aResult.get_CacheMatchAllResult().responseList()); + break; + } + case CacheOpResult::TCachePutAllResult: + { + mPromise->MaybeResolveWithUndefined(); + break; + } + case CacheOpResult::TCacheDeleteResult: + { + mPromise->MaybeResolve(aResult.get_CacheDeleteResult().success()); + break; + } + case CacheOpResult::TCacheKeysResult: + { + HandleRequestList(aResult.get_CacheKeysResult().requestList()); + break; + } + case CacheOpResult::TStorageMatchResult: + { + HandleResponse(aResult.get_StorageMatchResult().responseOrVoid()); + break; + } + case CacheOpResult::TStorageHasResult: + { + mPromise->MaybeResolve(aResult.get_StorageHasResult().success()); + break; + } + case CacheOpResult::TStorageOpenResult: + { + auto actor = static_cast<CacheChild*>( + aResult.get_StorageOpenResult().actorChild()); + + // If we have a success status then we should have an actor. Gracefully + // reject instead of crashing, though, if we get a nullptr here. + MOZ_DIAGNOSTIC_ASSERT(actor); + if (!actor) { + ErrorResult status; + status.ThrowTypeError<MSG_CACHE_OPEN_FAILED>(); + mPromise->MaybeReject(status); + break; + } + + actor->SetWorkerHolder(GetWorkerHolder()); + RefPtr<Cache> cache = new Cache(mGlobal, actor); + mPromise->MaybeResolve(cache); + break; + } + case CacheOpResult::TStorageDeleteResult: + { + mPromise->MaybeResolve(aResult.get_StorageDeleteResult().success()); + break; + } + case CacheOpResult::TStorageKeysResult: + { + mPromise->MaybeResolve(aResult.get_StorageKeysResult().keyList()); + break; + } + default: + MOZ_CRASH("Unknown Cache op result type!"); + } + + mPromise = nullptr; + + return true; +} + +void +CacheOpChild::StartDestroy() +{ + NS_ASSERT_OWNINGTHREAD(CacheOpChild); + + // Do not cancel on-going operations when WorkerHolder calls this. Instead, + // keep the Worker alive until we are done. +} + +nsIGlobalObject* +CacheOpChild::GetGlobalObject() const +{ + return mGlobal; +} + +#ifdef DEBUG +void +CacheOpChild::AssertOwningThread() const +{ + NS_ASSERT_OWNINGTHREAD(CacheOpChild); +} +#endif + +PBackgroundChild* +CacheOpChild::GetIPCManager() +{ + MOZ_CRASH("CacheOpChild does not implement TypeUtils::GetIPCManager()"); +} + +void +CacheOpChild::HandleResponse(const CacheResponseOrVoid& aResponseOrVoid) +{ + if (aResponseOrVoid.type() == CacheResponseOrVoid::Tvoid_t) { + mPromise->MaybeResolveWithUndefined(); + return; + } + + const CacheResponse& cacheResponse = aResponseOrVoid.get_CacheResponse(); + + AddWorkerHolderToStreamChild(cacheResponse, GetWorkerHolder()); + RefPtr<Response> response = ToResponse(cacheResponse); + + mPromise->MaybeResolve(response); +} + +void +CacheOpChild::HandleResponseList(const nsTArray<CacheResponse>& aResponseList) +{ + AutoTArray<RefPtr<Response>, 256> responses; + responses.SetCapacity(aResponseList.Length()); + + for (uint32_t i = 0; i < aResponseList.Length(); ++i) { + AddWorkerHolderToStreamChild(aResponseList[i], GetWorkerHolder()); + responses.AppendElement(ToResponse(aResponseList[i])); + } + + mPromise->MaybeResolve(responses); +} + +void +CacheOpChild::HandleRequestList(const nsTArray<CacheRequest>& aRequestList) +{ + AutoTArray<RefPtr<Request>, 256> requests; + requests.SetCapacity(aRequestList.Length()); + + for (uint32_t i = 0; i < aRequestList.Length(); ++i) { + AddWorkerHolderToStreamChild(aRequestList[i], GetWorkerHolder()); + requests.AppendElement(ToRequest(aRequestList[i])); + } + + mPromise->MaybeResolve(requests); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheOpChild.h b/dom/cache/CacheOpChild.h new file mode 100644 index 0000000000..4beeb33767 --- /dev/null +++ b/dom/cache/CacheOpChild.h @@ -0,0 +1,84 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheOpChild_h +#define mozilla_dom_cache_CacheOpChild_h + +#include "mozilla/dom/cache/ActorChild.h" +#include "mozilla/dom/cache/PCacheOpChild.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "mozilla/RefPtr.h" + +class nsIGlobalObject; + +namespace mozilla { +namespace dom { + +class Promise; + +namespace cache { + +class CacheOpChild final : public PCacheOpChild + , public ActorChild + , public TypeUtils +{ + friend class CacheChild; + friend class CacheStorageChild; + +private: + // This class must be constructed by CacheChild or CacheStorageChild using + // their ExecuteOp() factory method. + CacheOpChild(CacheWorkerHolder* aWorkerHolder, nsIGlobalObject* aGlobal, + nsISupports* aParent, Promise* aPromise); + ~CacheOpChild(); + + // PCacheOpChild methods + virtual void + ActorDestroy(ActorDestroyReason aReason) override; + + virtual bool + Recv__delete__(const ErrorResult& aRv, const CacheOpResult& aResult) override; + + // ActorChild methods + virtual void + StartDestroy() override; + + // TypeUtils methods + virtual nsIGlobalObject* + GetGlobalObject() const override; + +#ifdef DEBUG + virtual void + AssertOwningThread() const override; +#endif + + virtual mozilla::ipc::PBackgroundChild* + GetIPCManager() override; + + // Utility methods + void + HandleResponse(const CacheResponseOrVoid& aResponseOrVoid); + + void + HandleResponseList(const nsTArray<CacheResponse>& aResponseList); + + void + HandleRequestList(const nsTArray<CacheRequest>& aRequestList); + + nsCOMPtr<nsIGlobalObject> mGlobal; + // Hold the parent Cache or CacheStorage object alive until this async + // operation completes. + nsCOMPtr<nsISupports> mParent; + RefPtr<Promise> mPromise; + + NS_DECL_OWNINGTHREAD +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheOpChild_h diff --git a/dom/cache/CacheOpParent.cpp b/dom/cache/CacheOpParent.cpp new file mode 100644 index 0000000000..992ee82a76 --- /dev/null +++ b/dom/cache/CacheOpParent.cpp @@ -0,0 +1,230 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheOpParent.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/AutoUtils.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/dom/cache/SavedTypes.h" +#include "mozilla/ipc/FileDescriptorSetParent.h" +#include "mozilla/ipc/InputStreamUtils.h" +#include "mozilla/ipc/SendStream.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::ipc::FileDescriptorSetParent; +using mozilla::ipc::PBackgroundParent; +using mozilla::ipc::SendStreamParent; + +CacheOpParent::CacheOpParent(PBackgroundParent* aIpcManager, CacheId aCacheId, + const CacheOpArgs& aOpArgs) + : mIpcManager(aIpcManager) + , mCacheId(aCacheId) + , mNamespace(INVALID_NAMESPACE) + , mOpArgs(aOpArgs) +{ + MOZ_DIAGNOSTIC_ASSERT(mIpcManager); +} + +CacheOpParent::CacheOpParent(PBackgroundParent* aIpcManager, + Namespace aNamespace, const CacheOpArgs& aOpArgs) + : mIpcManager(aIpcManager) + , mCacheId(INVALID_CACHE_ID) + , mNamespace(aNamespace) + , mOpArgs(aOpArgs) +{ + MOZ_DIAGNOSTIC_ASSERT(mIpcManager); +} + +CacheOpParent::~CacheOpParent() +{ + NS_ASSERT_OWNINGTHREAD(CacheOpParent); +} + +void +CacheOpParent::Execute(ManagerId* aManagerId) +{ + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + MOZ_DIAGNOSTIC_ASSERT(!mManager); + MOZ_DIAGNOSTIC_ASSERT(!mVerifier); + + RefPtr<cache::Manager> manager; + nsresult rv = cache::Manager::GetOrCreate(aManagerId, getter_AddRefs(manager)); + if (NS_WARN_IF(NS_FAILED(rv))) { + ErrorResult result(rv); + Unused << Send__delete__(this, result, void_t()); + result.SuppressException(); + return; + } + + Execute(manager); +} + +void +CacheOpParent::Execute(cache::Manager* aManager) +{ + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + MOZ_DIAGNOSTIC_ASSERT(!mManager); + MOZ_DIAGNOSTIC_ASSERT(!mVerifier); + + mManager = aManager; + + // Handle put op + if (mOpArgs.type() == CacheOpArgs::TCachePutAllArgs) { + MOZ_DIAGNOSTIC_ASSERT(mCacheId != INVALID_CACHE_ID); + + const CachePutAllArgs& args = mOpArgs.get_CachePutAllArgs(); + const nsTArray<CacheRequestResponse>& list = args.requestResponseList(); + + AutoTArray<nsCOMPtr<nsIInputStream>, 256> requestStreamList; + AutoTArray<nsCOMPtr<nsIInputStream>, 256> responseStreamList; + + for (uint32_t i = 0; i < list.Length(); ++i) { + requestStreamList.AppendElement( + DeserializeCacheStream(list[i].request().body())); + responseStreamList.AppendElement( + DeserializeCacheStream(list[i].response().body())); + } + + mManager->ExecutePutAll(this, mCacheId, args.requestResponseList(), + requestStreamList, responseStreamList); + return; + } + + // Handle all other cache ops + if (mCacheId != INVALID_CACHE_ID) { + MOZ_DIAGNOSTIC_ASSERT(mNamespace == INVALID_NAMESPACE); + mManager->ExecuteCacheOp(this, mCacheId, mOpArgs); + return; + } + + // Handle all storage ops + MOZ_DIAGNOSTIC_ASSERT(mNamespace != INVALID_NAMESPACE); + mManager->ExecuteStorageOp(this, mNamespace, mOpArgs); +} + +void +CacheOpParent::WaitForVerification(PrincipalVerifier* aVerifier) +{ + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + MOZ_DIAGNOSTIC_ASSERT(!mManager); + MOZ_DIAGNOSTIC_ASSERT(!mVerifier); + + mVerifier = aVerifier; + mVerifier->AddListener(this); +} + +void +CacheOpParent::ActorDestroy(ActorDestroyReason aReason) +{ + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + + if (mVerifier) { + mVerifier->RemoveListener(this); + mVerifier = nullptr; + } + + if (mManager) { + mManager->RemoveListener(this); + mManager = nullptr; + } + + mIpcManager = nullptr; +} + +void +CacheOpParent::OnPrincipalVerified(nsresult aRv, ManagerId* aManagerId) +{ + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + + mVerifier->RemoveListener(this); + mVerifier = nullptr; + + if (NS_WARN_IF(NS_FAILED(aRv))) { + ErrorResult result(aRv); + Unused << Send__delete__(this, result, void_t()); + result.SuppressException(); + return; + } + + Execute(aManagerId); +} + +void +CacheOpParent::OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + CacheId aOpenedCacheId, + const nsTArray<SavedResponse>& aSavedResponseList, + const nsTArray<SavedRequest>& aSavedRequestList, + StreamList* aStreamList) +{ + NS_ASSERT_OWNINGTHREAD(CacheOpParent); + MOZ_DIAGNOSTIC_ASSERT(mIpcManager); + MOZ_DIAGNOSTIC_ASSERT(mManager); + + // Never send an op-specific result if we have an error. Instead, send + // void_t() to ensure that we don't leak actors on the child side. + if (NS_WARN_IF(aRv.Failed())) { + Unused << Send__delete__(this, aRv, void_t()); + aRv.SuppressException(); // We serialiazed it, as best we could. + return; + } + + uint32_t entryCount = std::max(1lu, static_cast<unsigned long>( + std::max(aSavedResponseList.Length(), + aSavedRequestList.Length()))); + + // The result must contain the appropriate type at this point. It may + // or may not contain the additional result data yet. For types that + // do not need special processing, it should already be set. If the + // result requires actor-specific operations, then we do that below. + // If the type and data types don't match, then we will trigger an + // assertion in AutoParentOpResult::Add(). + AutoParentOpResult result(mIpcManager, aResult, entryCount); + + if (aOpenedCacheId != INVALID_CACHE_ID) { + result.Add(aOpenedCacheId, mManager); + } + + for (uint32_t i = 0; i < aSavedResponseList.Length(); ++i) { + result.Add(aSavedResponseList[i], aStreamList); + } + + for (uint32_t i = 0; i < aSavedRequestList.Length(); ++i) { + result.Add(aSavedRequestList[i], aStreamList); + } + + Unused << Send__delete__(this, aRv, result.SendAsOpResult()); +} + +already_AddRefed<nsIInputStream> +CacheOpParent::DeserializeCacheStream(const CacheReadStreamOrVoid& aStreamOrVoid) +{ + if (aStreamOrVoid.type() == CacheReadStreamOrVoid::Tvoid_t) { + return nullptr; + } + + nsCOMPtr<nsIInputStream> stream; + const CacheReadStream& readStream = aStreamOrVoid.get_CacheReadStream(); + + // Option 1: One of our own ReadStreams was passed back to us with a stream + // control actor. + stream = ReadStream::Create(readStream); + if (stream) { + return stream.forget(); + } + + // Option 2: A stream was serialized using normal methods or passed + // as a PSendStream actor. Use the standard method for + // extracting the resulting stream. + return DeserializeIPCStream(readStream.stream()); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheOpParent.h b/dom/cache/CacheOpParent.h new file mode 100644 index 0000000000..6e53d4a22e --- /dev/null +++ b/dom/cache/CacheOpParent.h @@ -0,0 +1,80 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheOpParent_h +#define mozilla_dom_cache_CacheOpParent_h + +#include "mozilla/dom/cache/Manager.h" +#include "mozilla/dom/cache/PCacheOpParent.h" +#include "mozilla/dom/cache/PrincipalVerifier.h" +#include "nsTArray.h" + +namespace mozilla { +namespace ipc { +class PBackgroundParent; +} // namespace ipc +namespace dom { +namespace cache { + +class CacheOpParent final : public PCacheOpParent + , public PrincipalVerifier::Listener + , public Manager::Listener +{ + // to allow use of convenience overrides + using Manager::Listener::OnOpComplete; + +public: + CacheOpParent(mozilla::ipc::PBackgroundParent* aIpcManager, CacheId aCacheId, + const CacheOpArgs& aOpArgs); + CacheOpParent(mozilla::ipc::PBackgroundParent* aIpcManager, + Namespace aNamespace, const CacheOpArgs& aOpArgs); + ~CacheOpParent(); + + void + Execute(ManagerId* aManagerId); + + void + Execute(cache::Manager* aManager); + + void + WaitForVerification(PrincipalVerifier* aVerifier); + +private: + // PCacheOpParent methods + virtual void + ActorDestroy(ActorDestroyReason aReason) override; + + // PrincipalVerifier::Listener methods + virtual void + OnPrincipalVerified(nsresult aRv, ManagerId* aManagerId) override; + + // Manager::Listener methods + virtual void + OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + CacheId aOpenedCacheId, + const nsTArray<SavedResponse>& aSavedResponseList, + const nsTArray<SavedRequest>& aSavedRequestList, + StreamList* aStreamList) override; + + // utility methods + already_AddRefed<nsIInputStream> + DeserializeCacheStream(const CacheReadStreamOrVoid& aStreamOrVoid); + + mozilla::ipc::PBackgroundParent* mIpcManager; + const CacheId mCacheId; + const Namespace mNamespace; + const CacheOpArgs mOpArgs; + RefPtr<cache::Manager> mManager; + RefPtr<PrincipalVerifier> mVerifier; + + NS_DECL_OWNINGTHREAD +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheOpParent_h diff --git a/dom/cache/CacheParent.cpp b/dom/cache/CacheParent.cpp new file mode 100644 index 0000000000..05f6c4975c --- /dev/null +++ b/dom/cache/CacheParent.cpp @@ -0,0 +1,89 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheParent.h" + +#include "mozilla/dom/cache/CacheOpParent.h" +#include "nsCOMPtr.h" + +namespace mozilla { +namespace dom { +namespace cache { + +// Declared in ActorUtils.h +void +DeallocPCacheParent(PCacheParent* aActor) +{ + delete aActor; +} + +CacheParent::CacheParent(cache::Manager* aManager, CacheId aCacheId) + : mManager(aManager) + , mCacheId(aCacheId) +{ + MOZ_COUNT_CTOR(cache::CacheParent); + MOZ_DIAGNOSTIC_ASSERT(mManager); + mManager->AddRefCacheId(mCacheId); +} + +CacheParent::~CacheParent() +{ + MOZ_COUNT_DTOR(cache::CacheParent); + MOZ_DIAGNOSTIC_ASSERT(!mManager); +} + +void +CacheParent::ActorDestroy(ActorDestroyReason aReason) +{ + MOZ_DIAGNOSTIC_ASSERT(mManager); + mManager->ReleaseCacheId(mCacheId); + mManager = nullptr; +} + +PCacheOpParent* +CacheParent::AllocPCacheOpParent(const CacheOpArgs& aOpArgs) +{ + if (aOpArgs.type() != CacheOpArgs::TCacheMatchArgs && + aOpArgs.type() != CacheOpArgs::TCacheMatchAllArgs && + aOpArgs.type() != CacheOpArgs::TCachePutAllArgs && + aOpArgs.type() != CacheOpArgs::TCacheDeleteArgs && + aOpArgs.type() != CacheOpArgs::TCacheKeysArgs) + { + MOZ_CRASH("Invalid operation sent to Cache actor!"); + } + + return new CacheOpParent(Manager(), mCacheId, aOpArgs); +} + +bool +CacheParent::DeallocPCacheOpParent(PCacheOpParent* aActor) +{ + delete aActor; + return true; +} + +bool +CacheParent::RecvPCacheOpConstructor(PCacheOpParent* aActor, + const CacheOpArgs& aOpArgs) +{ + auto actor = static_cast<CacheOpParent*>(aActor); + actor->Execute(mManager); + return true; +} + +bool +CacheParent::RecvTeardown() +{ + if (!Send__delete__(this)) { + // child process is gone, warn and allow actor to clean up normally + NS_WARNING("Cache failed to send delete."); + } + return true; +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheParent.h b/dom/cache/CacheParent.h new file mode 100644 index 0000000000..3a8a542da3 --- /dev/null +++ b/dom/cache/CacheParent.h @@ -0,0 +1,50 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheParent_h +#define mozilla_dom_cache_CacheParent_h + +#include "mozilla/dom/cache/PCacheParent.h" +#include "mozilla/dom/cache/Types.h" + +namespace mozilla { +namespace dom { +namespace cache { + +class Manager; + +class CacheParent final : public PCacheParent +{ +public: + CacheParent(cache::Manager* aManager, CacheId aCacheId); + virtual ~CacheParent(); + +private: + // PCacheParent methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + + virtual PCacheOpParent* + AllocPCacheOpParent(const CacheOpArgs& aOpArgs) override; + + virtual bool + DeallocPCacheOpParent(PCacheOpParent* aActor) override; + + virtual bool + RecvPCacheOpConstructor(PCacheOpParent* actor, + const CacheOpArgs& aOpArgs) override; + + virtual bool + RecvTeardown() override; + + RefPtr<cache::Manager> mManager; + const CacheId mCacheId; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheParent_h diff --git a/dom/cache/CacheStorage.cpp b/dom/cache/CacheStorage.cpp new file mode 100644 index 0000000000..937e3826f2 --- /dev/null +++ b/dom/cache/CacheStorage.cpp @@ -0,0 +1,628 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheStorage.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/CacheStorageBinding.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/Response.h" +#include "mozilla/dom/cache/AutoUtils.h" +#include "mozilla/dom/cache/Cache.h" +#include "mozilla/dom/cache/CacheChild.h" +#include "mozilla/dom/cache/CacheStorageChild.h" +#include "mozilla/dom/cache/CacheWorkerHolder.h" +#include "mozilla/dom/cache/PCacheChild.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "mozilla/ipc/BackgroundChild.h" +#include "mozilla/ipc/BackgroundUtils.h" +#include "mozilla/ipc/PBackgroundChild.h" +#include "mozilla/ipc/PBackgroundSharedTypes.h" +#include "nsContentUtils.h" +#include "nsIDocument.h" +#include "nsIGlobalObject.h" +#include "nsIScriptSecurityManager.h" +#include "nsURLParsers.h" +#include "WorkerPrivate.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::Unused; +using mozilla::ErrorResult; +using mozilla::dom::workers::WorkerPrivate; +using mozilla::ipc::BackgroundChild; +using mozilla::ipc::PBackgroundChild; +using mozilla::ipc::IProtocol; +using mozilla::ipc::PrincipalInfo; +using mozilla::ipc::PrincipalToPrincipalInfo; + +NS_IMPL_CYCLE_COLLECTING_ADDREF(mozilla::dom::cache::CacheStorage); +NS_IMPL_CYCLE_COLLECTING_RELEASE(mozilla::dom::cache::CacheStorage); +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(mozilla::dom::cache::CacheStorage, + mGlobal); + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(CacheStorage) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) + NS_INTERFACE_MAP_ENTRY(nsIIPCBackgroundChildCreateCallback) +NS_INTERFACE_MAP_END + +// We cannot reference IPC types in a webidl binding implementation header. So +// define this in the .cpp and use heap storage in the mPendingRequests list. +struct CacheStorage::Entry final +{ + RefPtr<Promise> mPromise; + CacheOpArgs mArgs; + // We cannot add the requests until after the actor is present. So store + // the request data separately for now. + RefPtr<InternalRequest> mRequest; +}; + +namespace { + +bool +IsTrusted(const PrincipalInfo& aPrincipalInfo, bool aTestingPrefEnabled) +{ + // Can happen on main thread or worker thread + + if (aPrincipalInfo.type() == PrincipalInfo::TSystemPrincipalInfo) { + return true; + } + + // Require a ContentPrincipal to avoid null principal, etc. + // + // Also, an unknown appId means that this principal was created for the + // codebase without all the security information from the end document or + // worker. We require exact knowledge of this information before allowing + // the caller to touch the disk using the Cache API. + if (NS_WARN_IF(aPrincipalInfo.type() != PrincipalInfo::TContentPrincipalInfo || + aPrincipalInfo.get_ContentPrincipalInfo().attrs().mAppId == + nsIScriptSecurityManager::UNKNOWN_APP_ID)) { + return false; + } + + // If we're in testing mode, then don't do any more work to determing if + // the origin is trusted. We have to run some tests as http. + if (aTestingPrefEnabled) { + return true; + } + + // Now parse the scheme of the principal's origin. This is a short term + // method for determining "trust". In the long term we need to implement + // the full algorithm here: + // + // https://w3c.github.io/webappsec/specs/powerfulfeatures/#settings-secure + // + // TODO: Implement full secure setting algorithm. (bug 1177856) + + const nsCString& flatURL = aPrincipalInfo.get_ContentPrincipalInfo().spec(); + const char* url = flatURL.get(); + + // off the main thread URL parsing using nsStdURLParser. + nsCOMPtr<nsIURLParser> urlParser = new nsStdURLParser(); + + uint32_t schemePos; + int32_t schemeLen; + uint32_t authPos; + int32_t authLen; + nsresult rv = urlParser->ParseURL(url, flatURL.Length(), + &schemePos, &schemeLen, + &authPos, &authLen, + nullptr, nullptr); // ignore path + if (NS_WARN_IF(NS_FAILED(rv))) { return false; } + + nsAutoCString scheme(Substring(flatURL, schemePos, schemeLen)); + if (scheme.LowerCaseEqualsLiteral("https") || + scheme.LowerCaseEqualsLiteral("file")) { + return true; + } + + uint32_t hostPos; + int32_t hostLen; + + rv = urlParser->ParseAuthority(url + authPos, authLen, + nullptr, nullptr, // ignore username + nullptr, nullptr, // ignore password + &hostPos, &hostLen, + nullptr); // ignore port + if (NS_WARN_IF(NS_FAILED(rv))) { return false; } + + nsDependentCSubstring hostname(url + authPos + hostPos, hostLen); + + return hostname.EqualsLiteral("localhost") || + hostname.EqualsLiteral("127.0.0.1") || + hostname.EqualsLiteral("::1"); +} + +} // namespace + +// static +already_AddRefed<CacheStorage> +CacheStorage::CreateOnMainThread(Namespace aNamespace, nsIGlobalObject* aGlobal, + nsIPrincipal* aPrincipal, bool aStorageDisabled, + bool aForceTrustedOrigin, ErrorResult& aRv) +{ + MOZ_DIAGNOSTIC_ASSERT(aGlobal); + MOZ_DIAGNOSTIC_ASSERT(aPrincipal); + MOZ_ASSERT(NS_IsMainThread()); + + if (aStorageDisabled) { + NS_WARNING("CacheStorage has been disabled."); + RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR); + return ref.forget(); + } + + PrincipalInfo principalInfo; + nsresult rv = PrincipalToPrincipalInfo(aPrincipal, &principalInfo); + if (NS_WARN_IF(NS_FAILED(rv))) { + aRv.Throw(rv); + return nullptr; + } + + bool testingEnabled = aForceTrustedOrigin || + Preferences::GetBool("dom.caches.testing.enabled", false) || + Preferences::GetBool("dom.serviceWorkers.testing.enabled", false); + + if (!IsTrusted(principalInfo, testingEnabled)) { + NS_WARNING("CacheStorage not supported on untrusted origins."); + RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR); + return ref.forget(); + } + + RefPtr<CacheStorage> ref = new CacheStorage(aNamespace, aGlobal, + principalInfo, nullptr); + return ref.forget(); +} + +// static +already_AddRefed<CacheStorage> +CacheStorage::CreateOnWorker(Namespace aNamespace, nsIGlobalObject* aGlobal, + WorkerPrivate* aWorkerPrivate, ErrorResult& aRv) +{ + MOZ_DIAGNOSTIC_ASSERT(aGlobal); + MOZ_DIAGNOSTIC_ASSERT(aWorkerPrivate); + aWorkerPrivate->AssertIsOnWorkerThread(); + + if (!aWorkerPrivate->IsStorageAllowed()) { + NS_WARNING("CacheStorage is not allowed."); + RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR); + return ref.forget(); + } + + if (aWorkerPrivate->GetOriginAttributes().mPrivateBrowsingId > 0) { + NS_WARNING("CacheStorage not supported during private browsing."); + RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR); + return ref.forget(); + } + + RefPtr<CacheWorkerHolder> workerHolder = + CacheWorkerHolder::Create(aWorkerPrivate); + if (!workerHolder) { + NS_WARNING("Worker thread is shutting down."); + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + const PrincipalInfo& principalInfo = aWorkerPrivate->GetPrincipalInfo(); + + // We have a number of cases where we want to skip the https scheme + // validation: + // + // 1) Any worker when dom.caches.testing.enabled pref is true. + // 2) Any worker when dom.serviceWorkers.testing.enabled pref is true. This + // is mainly because most sites using SWs will expect Cache to work if + // SWs are enabled. + // 3) If the window that created this worker has the devtools SW testing + // option enabled. Same reasoning as (2). + // 4) If the worker itself is a ServiceWorker, then we always skip the + // origin checks. The ServiceWorker has its own trusted origin checks + // that are better than ours. In addition, we don't have information + // about the window any more, so we can't do our own checks. + bool testingEnabled = aWorkerPrivate->DOMCachesTestingEnabled() || + aWorkerPrivate->ServiceWorkersTestingEnabled() || + aWorkerPrivate->ServiceWorkersTestingInWindow() || + aWorkerPrivate->IsServiceWorker(); + + if (!IsTrusted(principalInfo, testingEnabled)) { + NS_WARNING("CacheStorage not supported on untrusted origins."); + RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR); + return ref.forget(); + } + + RefPtr<CacheStorage> ref = new CacheStorage(aNamespace, aGlobal, + principalInfo, workerHolder); + return ref.forget(); +} + +// static +bool +CacheStorage::DefineCaches(JSContext* aCx, JS::Handle<JSObject*> aGlobal) +{ + MOZ_ASSERT(NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(js::GetObjectClass(aGlobal)->flags & JSCLASS_DOM_GLOBAL, + "Passed object is not a global object!"); + js::AssertSameCompartment(aCx, aGlobal); + + if (NS_WARN_IF(!CacheStorageBinding::GetConstructorObject(aCx) || + !CacheBinding::GetConstructorObject(aCx))) { + return false; + } + + nsIPrincipal* principal = nsContentUtils::ObjectPrincipal(aGlobal); + MOZ_DIAGNOSTIC_ASSERT(principal); + + ErrorResult rv; + RefPtr<CacheStorage> storage = + CreateOnMainThread(DEFAULT_NAMESPACE, xpc::NativeGlobal(aGlobal), principal, + false, /* private browsing */ + true, /* force trusted */ + rv); + if (NS_WARN_IF(rv.MaybeSetPendingException(aCx))) { + return false; + } + + JS::Rooted<JS::Value> caches(aCx); + if (NS_WARN_IF(!ToJSValue(aCx, storage, &caches))) { + return false; + } + + return JS_DefineProperty(aCx, aGlobal, "caches", caches, JSPROP_ENUMERATE); +} + +CacheStorage::CacheStorage(Namespace aNamespace, nsIGlobalObject* aGlobal, + const PrincipalInfo& aPrincipalInfo, + CacheWorkerHolder* aWorkerHolder) + : mNamespace(aNamespace) + , mGlobal(aGlobal) + , mPrincipalInfo(MakeUnique<PrincipalInfo>(aPrincipalInfo)) + , mWorkerHolder(aWorkerHolder) + , mActor(nullptr) + , mStatus(NS_OK) +{ + MOZ_DIAGNOSTIC_ASSERT(mGlobal); + + // If the PBackground actor is already initialized then we can + // immediately use it + PBackgroundChild* actor = BackgroundChild::GetForCurrentThread(); + if (actor) { + ActorCreated(actor); + return; + } + + // Otherwise we must begin the PBackground initialization process and + // wait for the async ActorCreated() callback. + MOZ_ASSERT(NS_IsMainThread()); + bool ok = BackgroundChild::GetOrCreateForCurrentThread(this); + if (NS_WARN_IF(!ok)) { + ActorFailed(); + } +} + +CacheStorage::CacheStorage(nsresult aFailureResult) + : mNamespace(INVALID_NAMESPACE) + , mActor(nullptr) + , mStatus(aFailureResult) +{ + MOZ_DIAGNOSTIC_ASSERT(NS_FAILED(mStatus)); +} + +already_AddRefed<Promise> +CacheStorage::Match(const RequestOrUSVString& aRequest, + const CacheQueryOptions& aOptions, ErrorResult& aRv) +{ + NS_ASSERT_OWNINGTHREAD(CacheStorage); + + if (NS_WARN_IF(NS_FAILED(mStatus))) { + aRv.Throw(mStatus); + return nullptr; + } + + RefPtr<InternalRequest> request = ToInternalRequest(aRequest, IgnoreBody, + aRv); + if (NS_WARN_IF(aRv.Failed())) { + return nullptr; + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + CacheQueryParams params; + ToCacheQueryParams(params, aOptions); + + nsAutoPtr<Entry> entry(new Entry()); + entry->mPromise = promise; + entry->mArgs = StorageMatchArgs(CacheRequest(), params); + entry->mRequest = request; + + mPendingRequests.AppendElement(entry.forget()); + MaybeRunPendingRequests(); + + return promise.forget(); +} + +already_AddRefed<Promise> +CacheStorage::Has(const nsAString& aKey, ErrorResult& aRv) +{ + NS_ASSERT_OWNINGTHREAD(CacheStorage); + + if (NS_WARN_IF(NS_FAILED(mStatus))) { + aRv.Throw(mStatus); + return nullptr; + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + nsAutoPtr<Entry> entry(new Entry()); + entry->mPromise = promise; + entry->mArgs = StorageHasArgs(nsString(aKey)); + + mPendingRequests.AppendElement(entry.forget()); + MaybeRunPendingRequests(); + + return promise.forget(); +} + +already_AddRefed<Promise> +CacheStorage::Open(const nsAString& aKey, ErrorResult& aRv) +{ + NS_ASSERT_OWNINGTHREAD(CacheStorage); + + if (NS_WARN_IF(NS_FAILED(mStatus))) { + aRv.Throw(mStatus); + return nullptr; + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + nsAutoPtr<Entry> entry(new Entry()); + entry->mPromise = promise; + entry->mArgs = StorageOpenArgs(nsString(aKey)); + + mPendingRequests.AppendElement(entry.forget()); + MaybeRunPendingRequests(); + + return promise.forget(); +} + +already_AddRefed<Promise> +CacheStorage::Delete(const nsAString& aKey, ErrorResult& aRv) +{ + NS_ASSERT_OWNINGTHREAD(CacheStorage); + + if (NS_WARN_IF(NS_FAILED(mStatus))) { + aRv.Throw(mStatus); + return nullptr; + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + nsAutoPtr<Entry> entry(new Entry()); + entry->mPromise = promise; + entry->mArgs = StorageDeleteArgs(nsString(aKey)); + + mPendingRequests.AppendElement(entry.forget()); + MaybeRunPendingRequests(); + + return promise.forget(); +} + +already_AddRefed<Promise> +CacheStorage::Keys(ErrorResult& aRv) +{ + NS_ASSERT_OWNINGTHREAD(CacheStorage); + + if (NS_WARN_IF(NS_FAILED(mStatus))) { + aRv.Throw(mStatus); + return nullptr; + } + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (NS_WARN_IF(!promise)) { + return nullptr; + } + + nsAutoPtr<Entry> entry(new Entry()); + entry->mPromise = promise; + entry->mArgs = StorageKeysArgs(); + + mPendingRequests.AppendElement(entry.forget()); + MaybeRunPendingRequests(); + + return promise.forget(); +} + +// static +bool +CacheStorage::PrefEnabled(JSContext* aCx, JSObject* aObj) +{ + return Cache::PrefEnabled(aCx, aObj); +} + +// static +already_AddRefed<CacheStorage> +CacheStorage::Constructor(const GlobalObject& aGlobal, + CacheStorageNamespace aNamespace, + nsIPrincipal* aPrincipal, ErrorResult& aRv) +{ + if (NS_WARN_IF(!NS_IsMainThread())) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + // TODO: remove Namespace in favor of CacheStorageNamespace + static_assert(DEFAULT_NAMESPACE == (uint32_t)CacheStorageNamespace::Content, + "Default namespace should match webidl Content enum"); + static_assert(CHROME_ONLY_NAMESPACE == (uint32_t)CacheStorageNamespace::Chrome, + "Chrome namespace should match webidl Chrome enum"); + static_assert(NUMBER_OF_NAMESPACES == (uint32_t)CacheStorageNamespace::EndGuard_, + "Number of namespace should match webidl endguard enum"); + + Namespace ns = static_cast<Namespace>(aNamespace); + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + + bool privateBrowsing = false; + if (nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(global)) { + nsCOMPtr<nsIDocument> doc = window->GetExtantDoc(); + if (doc) { + nsCOMPtr<nsILoadContext> loadContext = doc->GetLoadContext(); + privateBrowsing = loadContext && loadContext->UsePrivateBrowsing(); + } + } + + // Create a CacheStorage object bypassing the trusted origin checks + // since this is a chrome-only constructor. + return CreateOnMainThread(ns, global, aPrincipal, privateBrowsing, + true /* force trusted origin */, aRv); +} + +nsISupports* +CacheStorage::GetParentObject() const +{ + return mGlobal; +} + +JSObject* +CacheStorage::WrapObject(JSContext* aContext, JS::Handle<JSObject*> aGivenProto) +{ + return mozilla::dom::CacheStorageBinding::Wrap(aContext, this, aGivenProto); +} + +void +CacheStorage::ActorCreated(PBackgroundChild* aActor) +{ + NS_ASSERT_OWNINGTHREAD(CacheStorage); + MOZ_DIAGNOSTIC_ASSERT(aActor); + + if (NS_WARN_IF(mWorkerHolder && mWorkerHolder->Notified())) { + ActorFailed(); + return; + } + + // WorkerHolder ownership is passed to the CacheStorageChild actor and any + // actors it may create. The WorkerHolder will keep the worker thread alive + // until the actors can gracefully shutdown. + CacheStorageChild* newActor = new CacheStorageChild(this, mWorkerHolder); + PCacheStorageChild* constructedActor = + aActor->SendPCacheStorageConstructor(newActor, mNamespace, *mPrincipalInfo); + + if (NS_WARN_IF(!constructedActor)) { + ActorFailed(); + return; + } + + mWorkerHolder = nullptr; + + MOZ_DIAGNOSTIC_ASSERT(constructedActor == newActor); + mActor = newActor; + + MaybeRunPendingRequests(); + MOZ_DIAGNOSTIC_ASSERT(mPendingRequests.IsEmpty()); +} + +void +CacheStorage::ActorFailed() +{ + NS_ASSERT_OWNINGTHREAD(CacheStorage); + MOZ_DIAGNOSTIC_ASSERT(!NS_FAILED(mStatus)); + + mStatus = NS_ERROR_UNEXPECTED; + mWorkerHolder = nullptr; + + for (uint32_t i = 0; i < mPendingRequests.Length(); ++i) { + nsAutoPtr<Entry> entry(mPendingRequests[i].forget()); + entry->mPromise->MaybeReject(NS_ERROR_UNEXPECTED); + } + mPendingRequests.Clear(); +} + +void +CacheStorage::DestroyInternal(CacheStorageChild* aActor) +{ + NS_ASSERT_OWNINGTHREAD(CacheStorage); + MOZ_DIAGNOSTIC_ASSERT(mActor); + MOZ_DIAGNOSTIC_ASSERT(mActor == aActor); + mActor->ClearListener(); + mActor = nullptr; + + // Note that we will never get an actor again in case another request is + // made before this object is destructed. + ActorFailed(); +} + +nsIGlobalObject* +CacheStorage::GetGlobalObject() const +{ + return mGlobal; +} + +#ifdef DEBUG +void +CacheStorage::AssertOwningThread() const +{ + NS_ASSERT_OWNINGTHREAD(CacheStorage); +} +#endif + +PBackgroundChild* +CacheStorage::GetIPCManager() +{ + // This is true because CacheStorage always uses IgnoreBody for requests. + // So we should never need to get the IPC manager during Request or + // Response serialization. + MOZ_CRASH("CacheStorage does not implement TypeUtils::GetIPCManager()"); +} + +CacheStorage::~CacheStorage() +{ + NS_ASSERT_OWNINGTHREAD(CacheStorage); + if (mActor) { + mActor->StartDestroyFromListener(); + // DestroyInternal() is called synchronously by StartDestroyFromListener(). + // So we should have already cleared the mActor. + MOZ_DIAGNOSTIC_ASSERT(!mActor); + } +} + +void +CacheStorage::MaybeRunPendingRequests() +{ + if (!mActor) { + return; + } + + for (uint32_t i = 0; i < mPendingRequests.Length(); ++i) { + ErrorResult rv; + nsAutoPtr<Entry> entry(mPendingRequests[i].forget()); + AutoChildOpArgs args(this, entry->mArgs, 1); + if (entry->mRequest) { + args.Add(entry->mRequest, IgnoreBody, IgnoreInvalidScheme, rv); + } + if (NS_WARN_IF(rv.Failed())) { + entry->mPromise->MaybeReject(rv); + continue; + } + mActor->ExecuteOp(mGlobal, entry->mPromise, this, args.SendAsOpArgs()); + } + mPendingRequests.Clear(); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheStorage.h b/dom/cache/CacheStorage.h new file mode 100644 index 0000000000..10310f6681 --- /dev/null +++ b/dom/cache/CacheStorage.h @@ -0,0 +1,130 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheStorage_h +#define mozilla_dom_cache_CacheStorage_h + +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "nsAutoPtr.h" +#include "nsCOMPtr.h" +#include "nsISupportsImpl.h" +#include "nsTArray.h" +#include "nsWrapperCache.h" +#include "nsIIPCBackgroundChildCreateCallback.h" + +class nsIGlobalObject; + +namespace mozilla { + +class ErrorResult; + +namespace ipc { + class PrincipalInfo; +} // namespace ipc + +namespace dom { + +enum class CacheStorageNamespace : uint32_t; +class Promise; + +namespace workers { + class WorkerPrivate; +} // namespace workers + +namespace cache { + +class CacheStorageChild; +class CacheWorkerHolder; + +class CacheStorage final : public nsIIPCBackgroundChildCreateCallback + , public nsWrapperCache + , public TypeUtils +{ + typedef mozilla::ipc::PBackgroundChild PBackgroundChild; + +public: + static already_AddRefed<CacheStorage> + CreateOnMainThread(Namespace aNamespace, nsIGlobalObject* aGlobal, + nsIPrincipal* aPrincipal, bool aStorageDisabled, + bool aForceTrustedOrigin, ErrorResult& aRv); + + static already_AddRefed<CacheStorage> + CreateOnWorker(Namespace aNamespace, nsIGlobalObject* aGlobal, + workers::WorkerPrivate* aWorkerPrivate, ErrorResult& aRv); + + static bool + DefineCaches(JSContext* aCx, JS::Handle<JSObject*> aGlobal); + + // webidl interface methods + already_AddRefed<Promise> Match(const RequestOrUSVString& aRequest, + const CacheQueryOptions& aOptions, + ErrorResult& aRv); + already_AddRefed<Promise> Has(const nsAString& aKey, ErrorResult& aRv); + already_AddRefed<Promise> Open(const nsAString& aKey, ErrorResult& aRv); + already_AddRefed<Promise> Delete(const nsAString& aKey, ErrorResult& aRv); + already_AddRefed<Promise> Keys(ErrorResult& aRv); + + // chrome-only webidl interface methods + static already_AddRefed<CacheStorage> + Constructor(const GlobalObject& aGlobal, CacheStorageNamespace aNamespace, + nsIPrincipal* aPrincipal, ErrorResult& aRv); + + // binding methods + static bool PrefEnabled(JSContext* aCx, JSObject* aObj); + + nsISupports* GetParentObject() const; + virtual JSObject* WrapObject(JSContext* aContext, JS::Handle<JSObject*> aGivenProto) override; + + // nsIIPCbackgroundChildCreateCallback methods + virtual void ActorCreated(PBackgroundChild* aActor) override; + virtual void ActorFailed() override; + + // Called when CacheStorageChild actor is being destroyed + void DestroyInternal(CacheStorageChild* aActor); + + // TypeUtils methods + virtual nsIGlobalObject* GetGlobalObject() const override; +#ifdef DEBUG + virtual void AssertOwningThread() const override; +#endif + + virtual mozilla::ipc::PBackgroundChild* + GetIPCManager() override; + +private: + CacheStorage(Namespace aNamespace, nsIGlobalObject* aGlobal, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo, + CacheWorkerHolder* aWorkerHolder); + explicit CacheStorage(nsresult aFailureResult); + ~CacheStorage(); + + void MaybeRunPendingRequests(); + + const Namespace mNamespace; + nsCOMPtr<nsIGlobalObject> mGlobal; + UniquePtr<mozilla::ipc::PrincipalInfo> mPrincipalInfo; + RefPtr<CacheWorkerHolder> mWorkerHolder; + + // weak ref cleared in DestroyInternal + CacheStorageChild* mActor; + + struct Entry; + nsTArray<nsAutoPtr<Entry>> mPendingRequests; + + nsresult mStatus; + +public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS_AMBIGUOUS(CacheStorage, + nsIIPCBackgroundChildCreateCallback) +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheStorage_h diff --git a/dom/cache/CacheStorageChild.cpp b/dom/cache/CacheStorageChild.cpp new file mode 100644 index 0000000000..aa8bb7b90c --- /dev/null +++ b/dom/cache/CacheStorageChild.cpp @@ -0,0 +1,148 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheStorageChild.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/CacheChild.h" +#include "mozilla/dom/cache/CacheOpChild.h" +#include "mozilla/dom/cache/CacheStorage.h" + +namespace mozilla { +namespace dom { +namespace cache { + +// declared in ActorUtils.h +void +DeallocPCacheStorageChild(PCacheStorageChild* aActor) +{ + delete aActor; +} + +CacheStorageChild::CacheStorageChild(CacheStorage* aListener, + CacheWorkerHolder* aWorkerHolder) + : mListener(aListener) + , mNumChildActors(0) + , mDelayedDestroy(false) +{ + MOZ_COUNT_CTOR(cache::CacheStorageChild); + MOZ_DIAGNOSTIC_ASSERT(mListener); + + SetWorkerHolder(aWorkerHolder); +} + +CacheStorageChild::~CacheStorageChild() +{ + MOZ_COUNT_DTOR(cache::CacheStorageChild); + NS_ASSERT_OWNINGTHREAD(CacheStorageChild); + MOZ_DIAGNOSTIC_ASSERT(!mListener); +} + +void +CacheStorageChild::ClearListener() +{ + NS_ASSERT_OWNINGTHREAD(CacheStorageChild); + MOZ_DIAGNOSTIC_ASSERT(mListener); + mListener = nullptr; +} + +void +CacheStorageChild::ExecuteOp(nsIGlobalObject* aGlobal, Promise* aPromise, + nsISupports* aParent, const CacheOpArgs& aArgs) +{ + mNumChildActors += 1; + Unused << SendPCacheOpConstructor( + new CacheOpChild(GetWorkerHolder(), aGlobal, aParent, aPromise), aArgs); +} + +void +CacheStorageChild::StartDestroyFromListener() +{ + NS_ASSERT_OWNINGTHREAD(CacheStorageChild); + + // The listener should be held alive by any async operations, so if it + // is going away then there must not be any child actors. This in turn + // ensures that StartDestroy() will not trigger the delayed path. + MOZ_DIAGNOSTIC_ASSERT(!mNumChildActors); + + StartDestroy(); +} + +void +CacheStorageChild::StartDestroy() +{ + NS_ASSERT_OWNINGTHREAD(CacheStorageChild); + + // If we have outstanding child actors, then don't destroy ourself yet. + // The child actors should be short lived and we should allow them to complete + // if possible. NoteDeletedActor() will call back into this Shutdown() + // method when the last child actor is gone. + if (mNumChildActors) { + mDelayedDestroy = true; + return; + } + + RefPtr<CacheStorage> listener = mListener; + + // StartDestroy() can get called from either CacheStorage or the + // CacheWorkerHolder. + // Theoretically we can get double called if the right race happens. Handle + // that by just ignoring the second StartDestroy() call. + if (!listener) { + return; + } + + listener->DestroyInternal(this); + + // CacheStorage listener should call ClearListener() in DestroyInternal() + MOZ_DIAGNOSTIC_ASSERT(!mListener); + + // Start actor destruction from parent process + Unused << SendTeardown(); +} + +void +CacheStorageChild::ActorDestroy(ActorDestroyReason aReason) +{ + NS_ASSERT_OWNINGTHREAD(CacheStorageChild); + RefPtr<CacheStorage> listener = mListener; + if (listener) { + listener->DestroyInternal(this); + // CacheStorage listener should call ClearListener() in DestroyInternal() + MOZ_DIAGNOSTIC_ASSERT(!mListener); + } + + RemoveWorkerHolder(); +} + +PCacheOpChild* +CacheStorageChild::AllocPCacheOpChild(const CacheOpArgs& aOpArgs) +{ + MOZ_CRASH("CacheOpChild should be manually constructed."); + return nullptr; +} + +bool +CacheStorageChild::DeallocPCacheOpChild(PCacheOpChild* aActor) +{ + delete aActor; + NoteDeletedActor(); + return true; +} + +void +CacheStorageChild::NoteDeletedActor() +{ + MOZ_DIAGNOSTIC_ASSERT(mNumChildActors); + mNumChildActors -= 1; + if (!mNumChildActors && mDelayedDestroy) { + StartDestroy(); + } +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheStorageChild.h b/dom/cache/CacheStorageChild.h new file mode 100644 index 0000000000..71ab1f47de --- /dev/null +++ b/dom/cache/CacheStorageChild.h @@ -0,0 +1,81 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheStorageChild_h +#define mozilla_dom_cache_CacheStorageChild_h + +#include "mozilla/dom/cache/ActorChild.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/cache/PCacheStorageChild.h" + +class nsIGlobalObject; + +namespace mozilla { +namespace dom { + +class Promise; + +namespace cache { + +class CacheOpArgs; +class CacheStorage; +class CacheWorkerHolder; +class PCacheChild; + +class CacheStorageChild final : public PCacheStorageChild + , public ActorChild +{ +public: + CacheStorageChild(CacheStorage* aListener, CacheWorkerHolder* aWorkerHolder); + ~CacheStorageChild(); + + // Must be called by the associated CacheStorage listener in its + // DestroyInternal() method. Also, CacheStorage must call + // SendDestroyFromListener() on the actor in its destructor to trigger + // ActorDestroy() if it has not been called yet. + void ClearListener(); + + void + ExecuteOp(nsIGlobalObject* aGlobal, Promise* aPromise, + nsISupports* aParent, const CacheOpArgs& aArgs); + + // Our parent Listener object has gone out of scope and is being destroyed. + void StartDestroyFromListener(); + +private: + // ActorChild methods + + // CacheWorkerHolder is trying to destroy due to worker shutdown. + virtual void StartDestroy() override; + + // PCacheStorageChild methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + + virtual PCacheOpChild* + AllocPCacheOpChild(const CacheOpArgs& aOpArgs) override; + + virtual bool + DeallocPCacheOpChild(PCacheOpChild* aActor) override; + + // utility methods + void + NoteDeletedActor(); + + // Use a weak ref so actor does not hold DOM object alive past content use. + // The CacheStorage object must call ClearListener() to null this before its + // destroyed. + CacheStorage* MOZ_NON_OWNING_REF mListener; + uint32_t mNumChildActors; + bool mDelayedDestroy; + + NS_DECL_OWNINGTHREAD +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheStorageChild_h diff --git a/dom/cache/CacheStorageParent.cpp b/dom/cache/CacheStorageParent.cpp new file mode 100644 index 0000000000..a8ede7d3d9 --- /dev/null +++ b/dom/cache/CacheStorageParent.cpp @@ -0,0 +1,143 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheStorageParent.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/ContentParent.h" +#include "mozilla/dom/cache/ActorUtils.h" +#include "mozilla/dom/cache/CacheOpParent.h" +#include "mozilla/dom/cache/ManagerId.h" +#include "mozilla/ipc/PBackgroundParent.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::ipc::PBackgroundParent; +using mozilla::ipc::PrincipalInfo; + +// declared in ActorUtils.h +PCacheStorageParent* +AllocPCacheStorageParent(PBackgroundParent* aManagingActor, + Namespace aNamespace, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo) +{ + return new CacheStorageParent(aManagingActor, aNamespace, aPrincipalInfo); +} + +// declared in ActorUtils.h +void +DeallocPCacheStorageParent(PCacheStorageParent* aActor) +{ + delete aActor; +} + +CacheStorageParent::CacheStorageParent(PBackgroundParent* aManagingActor, + Namespace aNamespace, + const PrincipalInfo& aPrincipalInfo) + : mNamespace(aNamespace) + , mVerifiedStatus(NS_OK) +{ + MOZ_COUNT_CTOR(cache::CacheStorageParent); + MOZ_DIAGNOSTIC_ASSERT(aManagingActor); + + // Start the async principal verification process immediately. + mVerifier = PrincipalVerifier::CreateAndDispatch(this, aManagingActor, + aPrincipalInfo); + MOZ_DIAGNOSTIC_ASSERT(mVerifier); +} + +CacheStorageParent::~CacheStorageParent() +{ + MOZ_COUNT_DTOR(cache::CacheStorageParent); + MOZ_DIAGNOSTIC_ASSERT(!mVerifier); +} + +void +CacheStorageParent::ActorDestroy(ActorDestroyReason aReason) +{ + if (mVerifier) { + mVerifier->RemoveListener(this); + mVerifier = nullptr; + } +} + +PCacheOpParent* +CacheStorageParent::AllocPCacheOpParent(const CacheOpArgs& aOpArgs) +{ + if (aOpArgs.type() != CacheOpArgs::TStorageMatchArgs && + aOpArgs.type() != CacheOpArgs::TStorageHasArgs && + aOpArgs.type() != CacheOpArgs::TStorageOpenArgs && + aOpArgs.type() != CacheOpArgs::TStorageDeleteArgs && + aOpArgs.type() != CacheOpArgs::TStorageKeysArgs) + { + MOZ_CRASH("Invalid operation sent to CacheStorage actor!"); + } + + return new CacheOpParent(Manager(), mNamespace, aOpArgs); +} + +bool +CacheStorageParent::DeallocPCacheOpParent(PCacheOpParent* aActor) +{ + delete aActor; + return true; +} + +bool +CacheStorageParent::RecvPCacheOpConstructor(PCacheOpParent* aActor, + const CacheOpArgs& aOpArgs) +{ + auto actor = static_cast<CacheOpParent*>(aActor); + + if (mVerifier) { + MOZ_DIAGNOSTIC_ASSERT(!mManagerId); + actor->WaitForVerification(mVerifier); + return true; + } + + if (NS_WARN_IF(NS_FAILED(mVerifiedStatus))) { + ErrorResult result(mVerifiedStatus); + Unused << CacheOpParent::Send__delete__(actor, result, void_t()); + result.SuppressException(); + return true; + } + + MOZ_DIAGNOSTIC_ASSERT(mManagerId); + actor->Execute(mManagerId); + return true; +} + +bool +CacheStorageParent::RecvTeardown() +{ + if (!Send__delete__(this)) { + // child process is gone, warn and allow actor to clean up normally + NS_WARNING("CacheStorage failed to delete actor."); + } + return true; +} + +void +CacheStorageParent::OnPrincipalVerified(nsresult aRv, ManagerId* aManagerId) +{ + MOZ_DIAGNOSTIC_ASSERT(mVerifier); + MOZ_DIAGNOSTIC_ASSERT(!mManagerId); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(mVerifiedStatus)); + + if (NS_WARN_IF(NS_FAILED(aRv))) { + mVerifiedStatus = aRv; + } + + mManagerId = aManagerId; + mVerifier->RemoveListener(this); + mVerifier = nullptr; +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheStorageParent.h b/dom/cache/CacheStorageParent.h new file mode 100644 index 0000000000..9aa431f64c --- /dev/null +++ b/dom/cache/CacheStorageParent.h @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheStorageParent_h +#define mozilla_dom_cache_CacheStorageParent_h + +#include "mozilla/dom/cache/PCacheStorageParent.h" +#include "mozilla/dom/cache/PrincipalVerifier.h" +#include "mozilla/dom/cache/Types.h" + +namespace mozilla { +namespace dom { +namespace cache { + +class ManagerId; + +class CacheStorageParent final : public PCacheStorageParent + , public PrincipalVerifier::Listener +{ +public: + CacheStorageParent(PBackgroundParent* aManagingActor, Namespace aNamespace, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo); + virtual ~CacheStorageParent(); + +private: + // PCacheStorageParent methods + virtual void + ActorDestroy(ActorDestroyReason aReason) override; + + virtual PCacheOpParent* + AllocPCacheOpParent(const CacheOpArgs& aOpArgs) override; + + virtual bool + DeallocPCacheOpParent(PCacheOpParent* aActor) override; + + virtual bool + RecvPCacheOpConstructor(PCacheOpParent* actor, + const CacheOpArgs& aOpArgs) override; + + virtual bool + RecvTeardown() override; + + // PrincipalVerifier::Listener methods + virtual void OnPrincipalVerified(nsresult aRv, + ManagerId* aManagerId) override; + + const Namespace mNamespace; + RefPtr<PrincipalVerifier> mVerifier; + nsresult mVerifiedStatus; + RefPtr<ManagerId> mManagerId; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheStorageParent_h diff --git a/dom/cache/CacheStreamControlChild.cpp b/dom/cache/CacheStreamControlChild.cpp new file mode 100644 index 0000000000..b7ca3a0e71 --- /dev/null +++ b/dom/cache/CacheStreamControlChild.cpp @@ -0,0 +1,159 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheStreamControlChild.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/ActorUtils.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/ipc/FileDescriptorSetChild.h" +#include "mozilla/ipc/PBackgroundChild.h" +#include "mozilla/ipc/PFileDescriptorSetChild.h" +#include "nsISupportsImpl.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::dom::OptionalFileDescriptorSet; +using mozilla::ipc::AutoIPCStream; +using mozilla::ipc::FileDescriptor; +using mozilla::ipc::FileDescriptorSetChild; +using mozilla::ipc::PFileDescriptorSetChild; + +// declared in ActorUtils.h +PCacheStreamControlChild* +AllocPCacheStreamControlChild() +{ + return new CacheStreamControlChild(); +} + +// declared in ActorUtils.h +void +DeallocPCacheStreamControlChild(PCacheStreamControlChild* aActor) +{ + delete aActor; +} + +CacheStreamControlChild::CacheStreamControlChild() + : mDestroyStarted(false) + , mDestroyDelayed(false) +{ + MOZ_COUNT_CTOR(cache::CacheStreamControlChild); +} + +CacheStreamControlChild::~CacheStreamControlChild() +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + MOZ_COUNT_DTOR(cache::CacheStreamControlChild); +} + +void +CacheStreamControlChild::StartDestroy() +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + // This can get called twice under some circumstances. For example, if the + // actor is added to a CacheWorkerHolder that has already been notified and + // the Cache actor has no mListener. + if (mDestroyStarted) { + return; + } + mDestroyStarted = true; + + // If any of the streams have started to be read, then wait for them to close + // naturally. + if (HasEverBeenRead()) { + // Note that we are delaying so that we can re-check for active streams + // in NoteClosedAfterForget(). + mDestroyDelayed = true; + return; + } + + // Otherwise, if the streams have not been touched then just pre-emptively + // close them now. This handles the case where someone retrieves a Response + // from the Cache, but never accesses the body. We should not keep the + // Worker alive until that Response is GC'd just because of its ignored + // body stream. + + // Begin shutting down all streams. This is the same as if the parent had + // asked us to shutdown. So simulate the CloseAll IPC message. + RecvCloseAll(); +} + +void +CacheStreamControlChild::SerializeControl(CacheReadStream* aReadStreamOut) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + aReadStreamOut->controlParent() = nullptr; + aReadStreamOut->controlChild() = this; +} + +void +CacheStreamControlChild::SerializeStream(CacheReadStream* aReadStreamOut, + nsIInputStream* aStream, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + MOZ_DIAGNOSTIC_ASSERT(aStream); + UniquePtr<AutoIPCStream> autoStream(new AutoIPCStream(aReadStreamOut->stream())); + autoStream->Serialize(aStream, Manager()); + aStreamCleanupList.AppendElement(Move(autoStream)); +} + +void +CacheStreamControlChild::NoteClosedAfterForget(const nsID& aId) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + Unused << SendNoteClosed(aId); + + // A stream has closed. If we delayed StartDestry() due to this stream + // being read, then we should check to see if any of the remaining streams + // are active. If none of our other streams have been read, then we can + // proceed with the shutdown now. + if (mDestroyDelayed && !HasEverBeenRead()) { + mDestroyDelayed = false; + RecvCloseAll(); + } +} + +#ifdef DEBUG +void +CacheStreamControlChild::AssertOwningThread() +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); +} +#endif + +void +CacheStreamControlChild::ActorDestroy(ActorDestroyReason aReason) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + CloseAllReadStreamsWithoutReporting(); + RemoveWorkerHolder(); +} + +bool +CacheStreamControlChild::RecvClose(const nsID& aId) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + CloseReadStreams(aId); + return true; +} + +bool +CacheStreamControlChild::RecvCloseAll() +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlChild); + CloseAllReadStreams(); + return true; +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheStreamControlChild.h b/dom/cache/CacheStreamControlChild.h new file mode 100644 index 0000000000..20c1d054ba --- /dev/null +++ b/dom/cache/CacheStreamControlChild.h @@ -0,0 +1,67 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheStreamControlChild_h +#define mozilla_dom_cache_CacheStreamControlChild_h + +#include "mozilla/dom/cache/ActorChild.h" +#include "mozilla/dom/cache/PCacheStreamControlChild.h" +#include "mozilla/dom/cache/StreamControl.h" +#include "nsTObserverArray.h" + +namespace mozilla { +namespace ipc { +class AutoIPCStream; +} // namespace ipc +namespace dom { +namespace cache { + +class ReadStream; + +class CacheStreamControlChild final : public PCacheStreamControlChild + , public StreamControl + , public ActorChild +{ +public: + CacheStreamControlChild(); + ~CacheStreamControlChild(); + + // ActorChild methods + virtual void StartDestroy() override; + + // StreamControl methods + virtual void + SerializeControl(CacheReadStream* aReadStreamOut) override; + + virtual void + SerializeStream(CacheReadStream* aReadStreamOut, nsIInputStream* aStream, + nsTArray<UniquePtr<mozilla::ipc::AutoIPCStream>>& aStreamCleanupList) override; + +private: + virtual void + NoteClosedAfterForget(const nsID& aId) override; + +#ifdef DEBUG + virtual void + AssertOwningThread() override; +#endif + + // PCacheStreamControlChild methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + virtual bool RecvClose(const nsID& aId) override; + virtual bool RecvCloseAll() override; + + bool mDestroyStarted; + bool mDestroyDelayed; + + NS_DECL_OWNINGTHREAD +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheStreamControlChild_h diff --git a/dom/cache/CacheStreamControlParent.cpp b/dom/cache/CacheStreamControlParent.cpp new file mode 100644 index 0000000000..54eef01d85 --- /dev/null +++ b/dom/cache/CacheStreamControlParent.cpp @@ -0,0 +1,158 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheStreamControlParent.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/dom/cache/StreamList.h" +#include "mozilla/ipc/FileDescriptorSetParent.h" +#include "mozilla/ipc/PBackgroundParent.h" +#include "mozilla/ipc/PFileDescriptorSetParent.h" +#include "nsISupportsImpl.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::dom::OptionalFileDescriptorSet; +using mozilla::ipc::FileDescriptor; +using mozilla::ipc::FileDescriptorSetParent; +using mozilla::ipc::PFileDescriptorSetParent; + +// declared in ActorUtils.h +void +DeallocPCacheStreamControlParent(PCacheStreamControlParent* aActor) +{ + delete aActor; +} + +CacheStreamControlParent::CacheStreamControlParent() +{ + MOZ_COUNT_CTOR(cache::CacheStreamControlParent); +} + +CacheStreamControlParent::~CacheStreamControlParent() +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(!mStreamList); + MOZ_COUNT_DTOR(cache::CacheStreamControlParent); +} + +void +CacheStreamControlParent::SerializeControl(CacheReadStream* aReadStreamOut) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + aReadStreamOut->controlChild() = nullptr; + aReadStreamOut->controlParent() = this; +} + +void +CacheStreamControlParent::SerializeStream(CacheReadStream* aReadStreamOut, + nsIInputStream* aStream, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + MOZ_DIAGNOSTIC_ASSERT(aStream); + UniquePtr<AutoIPCStream> autoStream(new AutoIPCStream(aReadStreamOut->stream())); + autoStream->Serialize(aStream, Manager()); + aStreamCleanupList.AppendElement(Move(autoStream)); +} + +void +CacheStreamControlParent::NoteClosedAfterForget(const nsID& aId) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + RecvNoteClosed(aId); +} + +#ifdef DEBUG +void +CacheStreamControlParent::AssertOwningThread() +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); +} +#endif + +void +CacheStreamControlParent::ActorDestroy(ActorDestroyReason aReason) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + CloseAllReadStreamsWithoutReporting(); + // If the initial SendPStreamControlConstructor() fails we will + // be called before mStreamList is set. + if (!mStreamList) { + return; + } + mStreamList->RemoveStreamControl(this); + mStreamList->NoteClosedAll(); + mStreamList = nullptr; +} + +bool +CacheStreamControlParent::RecvNoteClosed(const nsID& aId) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(mStreamList); + mStreamList->NoteClosed(aId); + return true; +} + +void +CacheStreamControlParent::SetStreamList(StreamList* aStreamList) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + MOZ_DIAGNOSTIC_ASSERT(!mStreamList); + mStreamList = aStreamList; +} + +void +CacheStreamControlParent::Close(const nsID& aId) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + NotifyClose(aId); + Unused << SendClose(aId); +} + +void +CacheStreamControlParent::CloseAll() +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + NotifyCloseAll(); + Unused << SendCloseAll(); +} + +void +CacheStreamControlParent::Shutdown() +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + if (!Send__delete__(this)) { + // child process is gone, allow actor to be destroyed normally + NS_WARNING("Cache failed to delete stream actor."); + return; + } +} + +void +CacheStreamControlParent::NotifyClose(const nsID& aId) +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + CloseReadStreams(aId); +} + +void +CacheStreamControlParent::NotifyCloseAll() +{ + NS_ASSERT_OWNINGTHREAD(CacheStreamControlParent); + CloseAllReadStreams(); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheStreamControlParent.h b/dom/cache/CacheStreamControlParent.h new file mode 100644 index 0000000000..c1d373176d --- /dev/null +++ b/dom/cache/CacheStreamControlParent.h @@ -0,0 +1,72 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheStreamControlParent_h +#define mozilla_dom_cache_CacheStreamControlParent_h + +#include "mozilla/dom/cache/PCacheStreamControlParent.h" +#include "mozilla/dom/cache/StreamControl.h" +#include "nsTObserverArray.h" + +namespace mozilla { +namespace ipc { +class AutoIPCStream; +} // namespace ipc +namespace dom { +namespace cache { + +class ReadStream; +class StreamList; + +class CacheStreamControlParent final : public PCacheStreamControlParent + , public StreamControl +{ +public: + CacheStreamControlParent(); + ~CacheStreamControlParent(); + + void SetStreamList(StreamList* aStreamList); + void Close(const nsID& aId); + void CloseAll(); + void Shutdown(); + + // StreamControl methods + virtual void + SerializeControl(CacheReadStream* aReadStreamOut) override; + + virtual void + SerializeStream(CacheReadStream* aReadStreamOut, nsIInputStream* aStream, + nsTArray<UniquePtr<mozilla::ipc::AutoIPCStream>>& aStreamCleanupList) override; + +private: + virtual void + NoteClosedAfterForget(const nsID& aId) override; + +#ifdef DEBUG + virtual void + AssertOwningThread() override; +#endif + + // PCacheStreamControlParent methods + virtual void ActorDestroy(ActorDestroyReason aReason) override; + virtual bool RecvNoteClosed(const nsID& aId) override; + + void NotifyClose(const nsID& aId); + void NotifyCloseAll(); + + // Cycle with StreamList via a weak-ref to us. Cleanup occurs when the actor + // is deleted by the PBackground manager. ActorDestroy() then calls + // StreamList::RemoveStreamControl() to clear the weak ref. + RefPtr<StreamList> mStreamList; + + NS_DECL_OWNINGTHREAD +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheStreamControlParent_h diff --git a/dom/cache/CacheTypes.ipdlh b/dom/cache/CacheTypes.ipdlh new file mode 100644 index 0000000000..0c51d8eb4e --- /dev/null +++ b/dom/cache/CacheTypes.ipdlh @@ -0,0 +1,237 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PCache; +include protocol PCacheStreamControl; +include protocol PSendStream; +include IPCStream; +include ChannelInfo; +include PBackgroundSharedTypes; + +using HeadersGuardEnum from "mozilla/dom/FetchIPCTypes.h"; +using ReferrerPolicy from "mozilla/dom/FetchIPCTypes.h"; +using RequestCredentials from "mozilla/dom/FetchIPCTypes.h"; +using RequestMode from "mozilla/dom/FetchIPCTypes.h"; +using RequestCache from "mozilla/dom/FetchIPCTypes.h"; +using RequestRedirect from "mozilla/dom/FetchIPCTypes.h"; +using ResponseType from "mozilla/dom/FetchIPCTypes.h"; +using mozilla::void_t from "ipc/IPCMessageUtils.h"; +using struct nsID from "nsID.h"; + +namespace mozilla { +namespace dom { +namespace cache { + +struct CacheQueryParams +{ + bool ignoreSearch; + bool ignoreMethod; + bool ignoreVary; + bool cacheNameSet; + nsString cacheName; +}; + +struct CacheReadStream +{ + nsID id; + nullable PCacheStreamControl control; + IPCStream stream; +}; + +union CacheReadStreamOrVoid +{ + void_t; + CacheReadStream; +}; + +struct HeadersEntry +{ + nsCString name; + nsCString value; +}; +struct CacheRequest +{ + nsCString method; + nsCString urlWithoutQuery; + nsCString urlQuery; + nsCString urlFragment; + HeadersEntry[] headers; + HeadersGuardEnum headersGuard; + nsString referrer; + ReferrerPolicy referrerPolicy; + RequestMode mode; + RequestCredentials credentials; + CacheReadStreamOrVoid body; + uint32_t contentPolicyType; + RequestCache requestCache; + RequestRedirect requestRedirect; + nsString integrity; +}; + +union CacheRequestOrVoid +{ + void_t; + CacheRequest; +}; + +struct CacheResponse +{ + ResponseType type; + nsCString[] urlList; + uint32_t status; + nsCString statusText; + HeadersEntry[] headers; + HeadersGuardEnum headersGuard; + CacheReadStreamOrVoid body; + IPCChannelInfo channelInfo; + OptionalPrincipalInfo principalInfo; +}; + +union CacheResponseOrVoid +{ + void_t; + CacheResponse; +}; + +struct CacheRequestResponse +{ + CacheRequest request; + CacheResponse response; +}; + +struct CacheMatchArgs +{ + CacheRequest request; + CacheQueryParams params; +}; + +struct CacheMatchAllArgs +{ + CacheRequestOrVoid requestOrVoid; + CacheQueryParams params; +}; + +struct CachePutAllArgs +{ + CacheRequestResponse[] requestResponseList; +}; + +struct CacheDeleteArgs +{ + CacheRequest request; + CacheQueryParams params; +}; + +struct CacheKeysArgs +{ + CacheRequestOrVoid requestOrVoid; + CacheQueryParams params; +}; + +struct StorageMatchArgs +{ + CacheRequest request; + CacheQueryParams params; +}; + +struct StorageHasArgs +{ + nsString key; +}; + +struct StorageOpenArgs +{ + nsString key; +}; + +struct StorageDeleteArgs +{ + nsString key; +}; + +struct StorageKeysArgs +{ +}; + +union CacheOpArgs +{ + CacheMatchArgs; + CacheMatchAllArgs; + CachePutAllArgs; + CacheDeleteArgs; + CacheKeysArgs; + StorageMatchArgs; + StorageHasArgs; + StorageOpenArgs; + StorageDeleteArgs; + StorageKeysArgs; +}; + +struct CacheMatchResult +{ + CacheResponseOrVoid responseOrVoid; +}; + +struct CacheMatchAllResult +{ + CacheResponse[] responseList; +}; + +struct CachePutAllResult +{ +}; + +struct CacheDeleteResult +{ + bool success; +}; + +struct CacheKeysResult +{ + CacheRequest[] requestList; +}; + +struct StorageMatchResult +{ + CacheResponseOrVoid responseOrVoid; +}; + +struct StorageHasResult +{ + bool success; +}; + +struct StorageOpenResult +{ + nullable PCache actor; +}; + +struct StorageDeleteResult +{ + bool success; +}; + +struct StorageKeysResult +{ + nsString[] keyList; +}; + +union CacheOpResult +{ + void_t; + CacheMatchResult; + CacheMatchAllResult; + CachePutAllResult; + CacheDeleteResult; + CacheKeysResult; + StorageMatchResult; + StorageHasResult; + StorageOpenResult; + StorageDeleteResult; + StorageKeysResult; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheWorkerHolder.cpp b/dom/cache/CacheWorkerHolder.cpp new file mode 100644 index 0000000000..3879e1521d --- /dev/null +++ b/dom/cache/CacheWorkerHolder.cpp @@ -0,0 +1,109 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/CacheWorkerHolder.h" + +#include "mozilla/dom/cache/ActorChild.h" +#include "WorkerPrivate.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::dom::workers::Terminating; +using mozilla::dom::workers::Status; +using mozilla::dom::workers::WorkerPrivate; + +// static +already_AddRefed<CacheWorkerHolder> +CacheWorkerHolder::Create(WorkerPrivate* aWorkerPrivate) +{ + MOZ_DIAGNOSTIC_ASSERT(aWorkerPrivate); + + RefPtr<CacheWorkerHolder> workerHolder = new CacheWorkerHolder(); + if (NS_WARN_IF(!workerHolder->HoldWorker(aWorkerPrivate, Terminating))) { + return nullptr; + } + + return workerHolder.forget(); +} + +void +CacheWorkerHolder::AddActor(ActorChild* aActor) +{ + NS_ASSERT_OWNINGTHREAD(CacheWorkerHolder); + MOZ_DIAGNOSTIC_ASSERT(aActor); + MOZ_ASSERT(!mActorList.Contains(aActor)); + + mActorList.AppendElement(aActor); + + // Allow an actor to be added after we've entered the Notifying case. We + // can't stop the actor creation from racing with out destruction of the + // other actors and we need to wait for this extra one to close as well. + // Signal it should destroy itself right away. + if (mNotified) { + aActor->StartDestroy(); + } +} + +void +CacheWorkerHolder::RemoveActor(ActorChild* aActor) +{ + NS_ASSERT_OWNINGTHREAD(CacheWorkerHolder); + MOZ_DIAGNOSTIC_ASSERT(aActor); + +#if defined(RELEASE_OR_BETA) + mActorList.RemoveElement(aActor); +#else + MOZ_DIAGNOSTIC_ASSERT(mActorList.RemoveElement(aActor)); +#endif + + MOZ_ASSERT(!mActorList.Contains(aActor)); +} + +bool +CacheWorkerHolder::Notified() const +{ + return mNotified; +} + +bool +CacheWorkerHolder::Notify(Status aStatus) +{ + NS_ASSERT_OWNINGTHREAD(CacheWorkerHolder); + + // When the service worker thread is stopped we will get Terminating, + // but nothing higher than that. We must shut things down at Terminating. + if (aStatus < Terminating || mNotified) { + return true; + } + + mNotified = true; + + // Start the asynchronous destruction of our actors. These will call back + // into RemoveActor() once the actor is destroyed. + for (uint32_t i = 0; i < mActorList.Length(); ++i) { + MOZ_DIAGNOSTIC_ASSERT(mActorList[i]); + mActorList[i]->StartDestroy(); + } + + return true; +} + +CacheWorkerHolder::CacheWorkerHolder() + : mNotified(false) +{ +} + +CacheWorkerHolder::~CacheWorkerHolder() +{ + NS_ASSERT_OWNINGTHREAD(CacheWorkerHolder); + MOZ_DIAGNOSTIC_ASSERT(mActorList.IsEmpty()); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/CacheWorkerHolder.h b/dom/cache/CacheWorkerHolder.h new file mode 100644 index 0000000000..513cc0e43d --- /dev/null +++ b/dom/cache/CacheWorkerHolder.h @@ -0,0 +1,54 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_CacheWorkerHolder_h +#define mozilla_dom_cache_CacheWorkerHolder_h + +#include "nsISupportsImpl.h" +#include "nsTArray.h" +#include "WorkerHolder.h" + +namespace mozilla { + +namespace workers { +class WorkerPrivate; +} // namespace workers + +namespace dom { +namespace cache { + +class ActorChild; + +class CacheWorkerHolder final : public workers::WorkerHolder +{ +public: + static already_AddRefed<CacheWorkerHolder> + Create(workers::WorkerPrivate* aWorkerPrivate); + + void AddActor(ActorChild* aActor); + void RemoveActor(ActorChild* aActor); + + bool Notified() const; + + // WorkerHolder methods + virtual bool Notify(workers::Status aStatus) override; + +private: + CacheWorkerHolder(); + ~CacheWorkerHolder(); + + nsTArray<ActorChild*> mActorList; + bool mNotified; + +public: + NS_INLINE_DECL_REFCOUNTING(mozilla::dom::cache::CacheWorkerHolder) +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_CacheWorkerHolder_h diff --git a/dom/cache/Connection.cpp b/dom/cache/Connection.cpp new file mode 100644 index 0000000000..91f4a4154d --- /dev/null +++ b/dom/cache/Connection.cpp @@ -0,0 +1,286 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/Connection.h" + +#include "mozilla/dom/cache/DBSchema.h" +#include "mozStorageHelper.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::dom::quota::QuotaObject; + +NS_IMPL_ISUPPORTS(cache::Connection, mozIStorageAsyncConnection, + mozIStorageConnection); + +Connection::Connection(mozIStorageConnection* aBase) + : mBase(aBase) + , mClosed(false) +{ + MOZ_DIAGNOSTIC_ASSERT(mBase); +} + +Connection::~Connection() +{ + NS_ASSERT_OWNINGTHREAD(Connection); + MOZ_ALWAYS_SUCCEEDS(Close()); +} + +NS_IMETHODIMP +Connection::Close() +{ + NS_ASSERT_OWNINGTHREAD(Connection); + + if (mClosed) { + return NS_OK; + } + mClosed = true; + + // If we are closing here, then Cache must not have a transaction + // open anywhere else. This should be guaranteed to succeed. + MOZ_ALWAYS_SUCCEEDS(db::IncrementalVacuum(this)); + + return mBase->Close(); +} + +// The following methods are all boilerplate that either forward to the +// base connection or block the method. All the async execution methods +// are blocked because Cache does not use them and they would require more +// work to wrap properly. + +// mozIStorageAsyncConnection methods + +NS_IMETHODIMP +Connection::AsyncClose(mozIStorageCompletionCallback*) +{ + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::AsyncClone(bool, mozIStorageCompletionCallback*) +{ + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::GetDatabaseFile(nsIFile** aFileOut) +{ + return mBase->GetDatabaseFile(aFileOut); +} + +NS_IMETHODIMP +Connection::CreateAsyncStatement(const nsACString&, mozIStorageAsyncStatement**) +{ + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::ExecuteAsync(mozIStorageBaseStatement**, uint32_t, + mozIStorageStatementCallback*, + mozIStoragePendingStatement**) +{ + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::ExecuteSimpleSQLAsync(const nsACString&, + mozIStorageStatementCallback*, + mozIStoragePendingStatement**) +{ + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::CreateFunction(const nsACString& aFunctionName, + int32_t aNumArguments, + mozIStorageFunction* aFunction) +{ + // async methods are not supported + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +Connection::CreateAggregateFunction(const nsACString& aFunctionName, + int32_t aNumArguments, + mozIStorageAggregateFunction* aFunction) +{ + return mBase->CreateAggregateFunction(aFunctionName, aNumArguments, + aFunction); +} + +NS_IMETHODIMP +Connection::RemoveFunction(const nsACString& aFunctionName) +{ + return mBase->RemoveFunction(aFunctionName); +} + +NS_IMETHODIMP +Connection::SetProgressHandler(int32_t aGranularity, + mozIStorageProgressHandler* aHandler, + mozIStorageProgressHandler** aHandlerOut) +{ + return mBase->SetProgressHandler(aGranularity, aHandler, aHandlerOut); +} + +NS_IMETHODIMP +Connection::RemoveProgressHandler(mozIStorageProgressHandler** aHandlerOut) +{ + return mBase->RemoveProgressHandler(aHandlerOut); +} + +// mozIStorageConnection methods + +NS_IMETHODIMP +Connection::Clone(bool aReadOnly, mozIStorageConnection** aConnectionOut) +{ + nsCOMPtr<mozIStorageConnection> conn; + nsresult rv = mBase->Clone(aReadOnly, getter_AddRefs(conn)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<mozIStorageConnection> wrapped = new Connection(conn); + wrapped.forget(aConnectionOut); + + return rv; +} + +NS_IMETHODIMP +Connection::GetDefaultPageSize(int32_t* aSizeOut) +{ + return mBase->GetDefaultPageSize(aSizeOut); +} + +NS_IMETHODIMP +Connection::GetConnectionReady(bool* aReadyOut) +{ + return mBase->GetConnectionReady(aReadyOut); +} + +NS_IMETHODIMP +Connection::GetLastInsertRowID(int64_t* aRowIdOut) +{ + return mBase->GetLastInsertRowID(aRowIdOut); +} + +NS_IMETHODIMP +Connection::GetAffectedRows(int32_t* aCountOut) +{ + return mBase->GetAffectedRows(aCountOut); +} + +NS_IMETHODIMP +Connection::GetLastError(int32_t* aErrorOut) +{ + return mBase->GetLastError(aErrorOut); +} + +NS_IMETHODIMP +Connection::GetLastErrorString(nsACString& aErrorOut) +{ + return mBase->GetLastErrorString(aErrorOut); +} + +NS_IMETHODIMP +Connection::GetSchemaVersion(int32_t* aVersionOut) +{ + return mBase->GetSchemaVersion(aVersionOut); +} + +NS_IMETHODIMP +Connection::SetSchemaVersion(int32_t aVersion) +{ + return mBase->SetSchemaVersion(aVersion); +} + +NS_IMETHODIMP +Connection::CreateStatement(const nsACString& aQuery, + mozIStorageStatement** aStatementOut) +{ + return mBase->CreateStatement(aQuery, aStatementOut); +} + +NS_IMETHODIMP +Connection::ExecuteSimpleSQL(const nsACString& aQuery) +{ + return mBase->ExecuteSimpleSQL(aQuery); +} + +NS_IMETHODIMP +Connection::TableExists(const nsACString& aTableName, bool* aExistsOut) +{ + return mBase->TableExists(aTableName, aExistsOut); +} + +NS_IMETHODIMP +Connection::IndexExists(const nsACString& aIndexName, bool* aExistsOut) +{ + return mBase->IndexExists(aIndexName, aExistsOut); +} + +NS_IMETHODIMP +Connection::GetTransactionInProgress(bool* aResultOut) +{ + return mBase->GetTransactionInProgress(aResultOut); +} + +NS_IMETHODIMP +Connection::BeginTransaction() +{ + return mBase->BeginTransaction(); +} + +NS_IMETHODIMP +Connection::BeginTransactionAs(int32_t aType) +{ + return mBase->BeginTransactionAs(aType); +} + +NS_IMETHODIMP +Connection::CommitTransaction() +{ + return mBase->CommitTransaction(); +} + +NS_IMETHODIMP +Connection::RollbackTransaction() +{ + return mBase->RollbackTransaction(); +} + +NS_IMETHODIMP +Connection::CreateTable(const char* aTable, const char* aSchema) +{ + return mBase->CreateTable(aTable, aSchema); +} + +NS_IMETHODIMP +Connection::SetGrowthIncrement(int32_t aIncrement, const nsACString& aDatabase) +{ + return mBase->SetGrowthIncrement(aIncrement, aDatabase); +} + +NS_IMETHODIMP +Connection::EnableModule(const nsACString& aModule) +{ + return mBase->EnableModule(aModule); +} + +NS_IMETHODIMP +Connection::GetQuotaObjects(QuotaObject** aDatabaseQuotaObject, + QuotaObject** aJournalQuotaObject) +{ + return mBase->GetQuotaObjects(aDatabaseQuotaObject, aJournalQuotaObject); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/Connection.h b/dom/cache/Connection.h new file mode 100644 index 0000000000..e69c50892d --- /dev/null +++ b/dom/cache/Connection.h @@ -0,0 +1,37 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Connection_h +#define mozilla_dom_cache_Connection_h + +#include "mozIStorageConnection.h" +#include "nsCOMPtr.h" + +namespace mozilla { +namespace dom { +namespace cache { + +class Connection final : public mozIStorageConnection +{ +public: + explicit Connection(mozIStorageConnection* aBase); + +private: + ~Connection(); + + nsCOMPtr<mozIStorageConnection> mBase; + bool mClosed; + + NS_DECL_ISUPPORTS + NS_DECL_MOZISTORAGEASYNCCONNECTION + NS_DECL_MOZISTORAGECONNECTION +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_Connection_h diff --git a/dom/cache/Context.cpp b/dom/cache/Context.cpp new file mode 100644 index 0000000000..db66ae90e0 --- /dev/null +++ b/dom/cache/Context.cpp @@ -0,0 +1,1149 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/Context.h" + +#include "mozilla/AutoRestore.h" +#include "mozilla/dom/cache/Action.h" +#include "mozilla/dom/cache/FileUtils.h" +#include "mozilla/dom/cache/Manager.h" +#include "mozilla/dom/cache/ManagerId.h" +#include "mozilla/dom/quota/QuotaManager.h" +#include "mozIStorageConnection.h" +#include "nsIFile.h" +#include "nsIPrincipal.h" +#include "nsIRunnable.h" +#include "nsThreadUtils.h" + +namespace { + +using mozilla::dom::cache::Action; +using mozilla::dom::cache::QuotaInfo; + +class NullAction final : public Action +{ +public: + NullAction() + { + } + + virtual void + RunOnTarget(Resolver* aResolver, const QuotaInfo&, Data*) override + { + // Resolve success immediately. This Action does no actual work. + MOZ_DIAGNOSTIC_ASSERT(aResolver); + aResolver->Resolve(NS_OK); + } +}; + +} // namespace + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::dom::quota::AssertIsOnIOThread; +using mozilla::dom::quota::OpenDirectoryListener; +using mozilla::dom::quota::QuotaManager; +using mozilla::dom::quota::PERSISTENCE_TYPE_DEFAULT; +using mozilla::dom::quota::PersistenceType; + +class Context::Data final : public Action::Data +{ +public: + explicit Data(nsIThread* aTarget) + : mTarget(aTarget) + { + MOZ_DIAGNOSTIC_ASSERT(mTarget); + } + + virtual mozIStorageConnection* + GetConnection() const override + { + MOZ_ASSERT(mTarget == NS_GetCurrentThread()); + return mConnection; + } + + virtual void + SetConnection(mozIStorageConnection* aConn) override + { + MOZ_ASSERT(mTarget == NS_GetCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(!mConnection); + mConnection = aConn; + MOZ_DIAGNOSTIC_ASSERT(mConnection); + } + +private: + ~Data() + { + // We could proxy release our data here, but instead just assert. The + // Context code should guarantee that we are destroyed on the target + // thread once the connection is initialized. If we're not, then + // QuotaManager might race and try to clear the origin out from under us. + MOZ_ASSERT_IF(mConnection, mTarget == NS_GetCurrentThread()); + } + + nsCOMPtr<nsIThread> mTarget; + nsCOMPtr<mozIStorageConnection> mConnection; + + // Threadsafe counting because we're created on the PBackground thread + // and destroyed on the target IO thread. + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Context::Data) +}; + +// Executed to perform the complicated dance of steps necessary to initialize +// the QuotaManager. This must be performed for each origin before any disk +// IO occurrs. +class Context::QuotaInitRunnable final : public nsIRunnable + , public OpenDirectoryListener +{ +public: + QuotaInitRunnable(Context* aContext, + Manager* aManager, + Data* aData, + nsIThread* aTarget, + Action* aInitAction) + : mContext(aContext) + , mThreadsafeHandle(aContext->CreateThreadsafeHandle()) + , mManager(aManager) + , mData(aData) + , mTarget(aTarget) + , mInitAction(aInitAction) + , mInitiatingThread(NS_GetCurrentThread()) + , mResult(NS_OK) + , mState(STATE_INIT) + , mCanceled(false) + { + MOZ_DIAGNOSTIC_ASSERT(mContext); + MOZ_DIAGNOSTIC_ASSERT(mManager); + MOZ_DIAGNOSTIC_ASSERT(mData); + MOZ_DIAGNOSTIC_ASSERT(mTarget); + MOZ_DIAGNOSTIC_ASSERT(mInitiatingThread); + MOZ_DIAGNOSTIC_ASSERT(mInitAction); + } + + nsresult Dispatch() + { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_INIT); + + mState = STATE_GET_INFO; + nsresult rv = NS_DispatchToMainThread(this, nsIThread::DISPATCH_NORMAL); + if (NS_WARN_IF(NS_FAILED(rv))) { + mState = STATE_COMPLETE; + Clear(); + } + return rv; + } + + void Cancel() + { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(!mCanceled); + mCanceled = true; + mInitAction->CancelOnInitiatingThread(); + } + + void OpenDirectory(); + + // OpenDirectoryListener methods + virtual void + DirectoryLockAcquired(DirectoryLock* aLock) override; + + virtual void + DirectoryLockFailed() override; + +private: + class SyncResolver final : public Action::Resolver + { + public: + SyncResolver() + : mResolved(false) + , mResult(NS_OK) + { } + + virtual void + Resolve(nsresult aRv) override + { + MOZ_DIAGNOSTIC_ASSERT(!mResolved); + mResolved = true; + mResult = aRv; + }; + + bool Resolved() const { return mResolved; } + nsresult Result() const { return mResult; } + + private: + ~SyncResolver() { } + + bool mResolved; + nsresult mResult; + + NS_INLINE_DECL_REFCOUNTING(Context::QuotaInitRunnable::SyncResolver, override) + }; + + ~QuotaInitRunnable() + { + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_COMPLETE); + MOZ_DIAGNOSTIC_ASSERT(!mContext); + MOZ_DIAGNOSTIC_ASSERT(!mInitAction); + } + + enum State + { + STATE_INIT, + STATE_GET_INFO, + STATE_CREATE_QUOTA_MANAGER, + STATE_OPEN_DIRECTORY, + STATE_WAIT_FOR_DIRECTORY_LOCK, + STATE_ENSURE_ORIGIN_INITIALIZED, + STATE_RUN_ON_TARGET, + STATE_RUNNING, + STATE_COMPLETING, + STATE_COMPLETE + }; + + void Complete(nsresult aResult) + { + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_RUNNING || NS_FAILED(aResult)); + + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(mResult)); + mResult = aResult; + + mState = STATE_COMPLETING; + MOZ_ALWAYS_SUCCEEDS( + mInitiatingThread->Dispatch(this, nsIThread::DISPATCH_NORMAL)); + } + + void Clear() + { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(mContext); + mContext = nullptr; + mManager = nullptr; + mInitAction = nullptr; + } + + RefPtr<Context> mContext; + RefPtr<ThreadsafeHandle> mThreadsafeHandle; + RefPtr<Manager> mManager; + RefPtr<Data> mData; + nsCOMPtr<nsIThread> mTarget; + RefPtr<Action> mInitAction; + nsCOMPtr<nsIThread> mInitiatingThread; + nsresult mResult; + QuotaInfo mQuotaInfo; + RefPtr<DirectoryLock> mDirectoryLock; + State mState; + Atomic<bool> mCanceled; + +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIRUNNABLE +}; + +void +Context::QuotaInitRunnable::OpenDirectory() +{ + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_CREATE_QUOTA_MANAGER || + mState == STATE_OPEN_DIRECTORY); + MOZ_DIAGNOSTIC_ASSERT(QuotaManager::Get()); + + // QuotaManager::OpenDirectory() will hold a reference to us as + // a listener. We will then get DirectoryLockAcquired() on the owning + // thread when it is safe to access our storage directory. + mState = STATE_WAIT_FOR_DIRECTORY_LOCK; + QuotaManager::Get()->OpenDirectory(PERSISTENCE_TYPE_DEFAULT, + mQuotaInfo.mGroup, + mQuotaInfo.mOrigin, + mQuotaInfo.mIsApp, + quota::Client::DOMCACHE, + /* aExclusive */ false, + this); +} + +void +Context::QuotaInitRunnable::DirectoryLockAcquired(DirectoryLock* aLock) +{ + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_WAIT_FOR_DIRECTORY_LOCK); + MOZ_DIAGNOSTIC_ASSERT(!mDirectoryLock); + + mDirectoryLock = aLock; + + if (mCanceled) { + Complete(NS_ERROR_ABORT); + return; + } + + QuotaManager* qm = QuotaManager::Get(); + MOZ_DIAGNOSTIC_ASSERT(qm); + + mState = STATE_ENSURE_ORIGIN_INITIALIZED; + nsresult rv = qm->IOThread()->Dispatch(this, nsIThread::DISPATCH_NORMAL); + if (NS_WARN_IF(NS_FAILED(rv))) { + Complete(rv); + return; + } +} + +void +Context::QuotaInitRunnable::DirectoryLockFailed() +{ + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_WAIT_FOR_DIRECTORY_LOCK); + MOZ_DIAGNOSTIC_ASSERT(!mDirectoryLock); + + NS_WARNING("Failed to acquire a directory lock!"); + + Complete(NS_ERROR_FAILURE); +} + +NS_IMPL_ISUPPORTS(mozilla::dom::cache::Context::QuotaInitRunnable, nsIRunnable); + +// The QuotaManager init state machine is represented in the following diagram: +// +// +---------------+ +// | Start | Resolve(error) +// | (Orig Thread) +---------------------+ +// +-------+-------+ | +// | | +// +----------v-----------+ | +// | GetInfo | Resolve(error) | +// | (Main Thread) +-----------------+ +// +----------+-----------+ | +// | | +// +----------v-----------+ | +// | CreateQuotaManager | Resolve(error) | +// | (Orig Thread) +-----------------+ +// +----------+-----------+ | +// | | +// +----------v-----------+ | +// | OpenDirectory | Resolve(error) | +// | (Orig Thread) +-----------------+ +// +----------+-----------+ | +// | | +// +----------v-----------+ | +// | WaitForDirectoryLock | Resolve(error) | +// | (Orig Thread) +-----------------+ +// +----------+-----------+ | +// | | +// +----------v------------+ | +// |EnsureOriginInitialized| Resolve(error) | +// | (Quota IO Thread) +----------------+ +// +----------+------------+ | +// | | +// +----------v------------+ | +// | RunOnTarget | Resolve(error) | +// | (Target Thread) +----------------+ +// +----------+------------+ | +// | | +// +---------v---------+ +------v------+ +// | Running | | Completing | +// | (Target Thread) +------------>(Orig Thread)| +// +-------------------+ +------+------+ +// | +// +-----v----+ +// | Complete | +// +----------+ +// +// The initialization process proceeds through the main states. If an error +// occurs, then we transition to Completing state back on the original thread. +NS_IMETHODIMP +Context::QuotaInitRunnable::Run() +{ + // May run on different threads depending on the state. See individual + // state cases for thread assertions. + + RefPtr<SyncResolver> resolver = new SyncResolver(); + + switch(mState) { + // ----------------------------------- + case STATE_GET_INFO: + { + MOZ_ASSERT(NS_IsMainThread()); + + if (mCanceled) { + resolver->Resolve(NS_ERROR_ABORT); + break; + } + + RefPtr<ManagerId> managerId = mManager->GetManagerId(); + nsCOMPtr<nsIPrincipal> principal = managerId->Principal(); + nsresult rv = QuotaManager::GetInfoFromPrincipal(principal, + &mQuotaInfo.mSuffix, + &mQuotaInfo.mGroup, + &mQuotaInfo.mOrigin, + &mQuotaInfo.mIsApp); + if (NS_WARN_IF(NS_FAILED(rv))) { + resolver->Resolve(rv); + break; + } + + mState = STATE_CREATE_QUOTA_MANAGER; + MOZ_ALWAYS_SUCCEEDS( + mInitiatingThread->Dispatch(this, nsIThread::DISPATCH_NORMAL)); + break; + } + // ---------------------------------- + case STATE_CREATE_QUOTA_MANAGER: + { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + + if (mCanceled || QuotaManager::IsShuttingDown()) { + resolver->Resolve(NS_ERROR_ABORT); + break; + } + + if (QuotaManager::Get()) { + OpenDirectory(); + return NS_OK; + } + + mState = STATE_OPEN_DIRECTORY; + QuotaManager::GetOrCreate(this); + break; + } + // ---------------------------------- + case STATE_OPEN_DIRECTORY: + { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + + if (NS_WARN_IF(!QuotaManager::Get())) { + resolver->Resolve(NS_ERROR_FAILURE); + break; + } + + OpenDirectory(); + break; + } + // ---------------------------------- + case STATE_ENSURE_ORIGIN_INITIALIZED: + { + AssertIsOnIOThread(); + + if (mCanceled) { + resolver->Resolve(NS_ERROR_ABORT); + break; + } + + QuotaManager* qm = QuotaManager::Get(); + MOZ_DIAGNOSTIC_ASSERT(qm); + nsresult rv = qm->EnsureOriginIsInitialized(PERSISTENCE_TYPE_DEFAULT, + mQuotaInfo.mSuffix, + mQuotaInfo.mGroup, + mQuotaInfo.mOrigin, + mQuotaInfo.mIsApp, + getter_AddRefs(mQuotaInfo.mDir)); + if (NS_FAILED(rv)) { + resolver->Resolve(rv); + break; + } + + mState = STATE_RUN_ON_TARGET; + + MOZ_ALWAYS_SUCCEEDS( + mTarget->Dispatch(this, nsIThread::DISPATCH_NORMAL)); + break; + } + // ------------------- + case STATE_RUN_ON_TARGET: + { + MOZ_ASSERT(NS_GetCurrentThread() == mTarget); + + mState = STATE_RUNNING; + + // Execute the provided initialization Action. The Action must Resolve() + // before returning. + mInitAction->RunOnTarget(resolver, mQuotaInfo, mData); + MOZ_DIAGNOSTIC_ASSERT(resolver->Resolved()); + + mData = nullptr; + + // If the database was opened, then we should always succeed when creating + // the marker file. If it wasn't opened successfully, then no need to + // create a marker file anyway. + if (NS_SUCCEEDED(resolver->Result())) { + MOZ_ALWAYS_SUCCEEDS(CreateMarkerFile(mQuotaInfo)); + } + + break; + } + // ------------------- + case STATE_COMPLETING: + { + NS_ASSERT_OWNINGTHREAD(QuotaInitRunnable); + mInitAction->CompleteOnInitiatingThread(mResult); + mContext->OnQuotaInit(mResult, mQuotaInfo, mDirectoryLock.forget()); + mState = STATE_COMPLETE; + + // Explicitly cleanup here as the destructor could fire on any of + // the threads we have bounced through. + Clear(); + break; + } + // ----- + case STATE_WAIT_FOR_DIRECTORY_LOCK: + default: + { + MOZ_CRASH("unexpected state in QuotaInitRunnable"); + } + } + + if (resolver->Resolved()) { + Complete(resolver->Result()); + } + + return NS_OK; +} + +// Runnable wrapper around Action objects dispatched on the Context. This +// runnable executes the Action on the appropriate threads while the Context +// is initialized. +class Context::ActionRunnable final : public nsIRunnable + , public Action::Resolver + , public Context::Activity +{ +public: + ActionRunnable(Context* aContext, Data* aData, nsIEventTarget* aTarget, + Action* aAction, const QuotaInfo& aQuotaInfo) + : mContext(aContext) + , mData(aData) + , mTarget(aTarget) + , mAction(aAction) + , mQuotaInfo(aQuotaInfo) + , mInitiatingThread(NS_GetCurrentThread()) + , mState(STATE_INIT) + , mResult(NS_OK) + , mExecutingRunOnTarget(false) + { + MOZ_DIAGNOSTIC_ASSERT(mContext); + // mData may be nullptr + MOZ_DIAGNOSTIC_ASSERT(mTarget); + MOZ_DIAGNOSTIC_ASSERT(mAction); + // mQuotaInfo.mDir may be nullptr if QuotaInitRunnable failed + MOZ_DIAGNOSTIC_ASSERT(mInitiatingThread); + } + + nsresult Dispatch() + { + NS_ASSERT_OWNINGTHREAD(ActionRunnable); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_INIT); + + mState = STATE_RUN_ON_TARGET; + nsresult rv = mTarget->Dispatch(this, nsIEventTarget::DISPATCH_NORMAL); + if (NS_WARN_IF(NS_FAILED(rv))) { + mState = STATE_COMPLETE; + Clear(); + } + return rv; + } + + virtual bool + MatchesCacheId(CacheId aCacheId) const override + { + NS_ASSERT_OWNINGTHREAD(ActionRunnable); + return mAction->MatchesCacheId(aCacheId); + } + + virtual void + Cancel() override + { + NS_ASSERT_OWNINGTHREAD(ActionRunnable); + mAction->CancelOnInitiatingThread(); + } + + virtual void Resolve(nsresult aRv) override + { + MOZ_ASSERT(mTarget == NS_GetCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_RUNNING); + + mResult = aRv; + + // We ultimately must complete on the initiating thread, but bounce through + // the current thread again to ensure that we don't destroy objects and + // state out from under the currently running action's stack. + mState = STATE_RESOLVING; + + // If we were resolved synchronously within Action::RunOnTarget() then we + // can avoid a thread bounce and just resolve once RunOnTarget() returns. + // The Run() method will handle this by looking at mState after + // RunOnTarget() returns. + if (mExecutingRunOnTarget) { + return; + } + + // Otherwise we are in an asynchronous resolve. And must perform a thread + // bounce to run on the target thread again. + MOZ_ALWAYS_SUCCEEDS( + mTarget->Dispatch(this, nsIThread::DISPATCH_NORMAL)); + } + +private: + ~ActionRunnable() + { + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_COMPLETE); + MOZ_DIAGNOSTIC_ASSERT(!mContext); + MOZ_DIAGNOSTIC_ASSERT(!mAction); + } + + void Clear() + { + NS_ASSERT_OWNINGTHREAD(ActionRunnable); + MOZ_DIAGNOSTIC_ASSERT(mContext); + MOZ_DIAGNOSTIC_ASSERT(mAction); + mContext->RemoveActivity(this); + mContext = nullptr; + mAction = nullptr; + } + + enum State + { + STATE_INIT, + STATE_RUN_ON_TARGET, + STATE_RUNNING, + STATE_RESOLVING, + STATE_COMPLETING, + STATE_COMPLETE + }; + + RefPtr<Context> mContext; + RefPtr<Data> mData; + nsCOMPtr<nsIEventTarget> mTarget; + RefPtr<Action> mAction; + const QuotaInfo mQuotaInfo; + nsCOMPtr<nsIThread> mInitiatingThread; + State mState; + nsresult mResult; + + // Only accessible on target thread; + bool mExecutingRunOnTarget; + +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIRUNNABLE +}; + +NS_IMPL_ISUPPORTS(mozilla::dom::cache::Context::ActionRunnable, nsIRunnable); + +// The ActionRunnable has a simpler state machine. It basically needs to run +// the action on the target thread and then complete on the original thread. +// +// +-------------+ +// | Start | +// |(Orig Thread)| +// +-----+-------+ +// | +// +-------v---------+ +// | RunOnTarget | +// |Target IO Thread)+---+ Resolve() +// +-------+---------+ | +// | | +// +-------v----------+ | +// | Running | | +// |(Target IO Thread)| | +// +------------------+ | +// | Resolve() | +// +-------v----------+ | +// | Resolving <--+ +-------------+ +// | | | Completing | +// |(Target IO Thread)+---------------------->(Orig Thread)| +// +------------------+ +-------+-----+ +// | +// | +// +----v---+ +// |Complete| +// +--------+ +// +// Its important to note that synchronous actions will effectively Resolve() +// out of the Running state immediately. Asynchronous Actions may remain +// in the Running state for some time, but normally the ActionRunnable itself +// does not see any execution there. Its all handled internal to the Action. +NS_IMETHODIMP +Context::ActionRunnable::Run() +{ + switch(mState) { + // ---------------------- + case STATE_RUN_ON_TARGET: + { + MOZ_ASSERT(NS_GetCurrentThread() == mTarget); + MOZ_DIAGNOSTIC_ASSERT(!mExecutingRunOnTarget); + + // Note that we are calling RunOnTarget(). This lets us detect + // if Resolve() is called synchronously. + AutoRestore<bool> executingRunOnTarget(mExecutingRunOnTarget); + mExecutingRunOnTarget = true; + + mState = STATE_RUNNING; + mAction->RunOnTarget(this, mQuotaInfo, mData); + + mData = nullptr; + + // Resolve was called synchronously from RunOnTarget(). We can + // immediately move to completing now since we are sure RunOnTarget() + // completed. + if (mState == STATE_RESOLVING) { + // Use recursion instead of switch case fall-through... Seems slightly + // easier to understand. + Run(); + } + + break; + } + // ----------------- + case STATE_RESOLVING: + { + MOZ_ASSERT(NS_GetCurrentThread() == mTarget); + // The call to Action::RunOnTarget() must have returned now if we + // are running on the target thread again. We may now proceed + // with completion. + mState = STATE_COMPLETING; + // Shutdown must be delayed until all Contexts are destroyed. Crash + // for this invariant violation. + MOZ_ALWAYS_SUCCEEDS( + mInitiatingThread->Dispatch(this, nsIThread::DISPATCH_NORMAL)); + break; + } + // ------------------- + case STATE_COMPLETING: + { + NS_ASSERT_OWNINGTHREAD(ActionRunnable); + mAction->CompleteOnInitiatingThread(mResult); + mState = STATE_COMPLETE; + // Explicitly cleanup here as the destructor could fire on any of + // the threads we have bounced through. + Clear(); + break; + } + // ----------------- + default: + { + MOZ_CRASH("unexpected state in ActionRunnable"); + break; + } + } + return NS_OK; +} + +void +Context::ThreadsafeHandle::AllowToClose() +{ + if (mOwningThread == NS_GetCurrentThread()) { + AllowToCloseOnOwningThread(); + return; + } + + // Dispatch is guaranteed to succeed here because we block shutdown until + // all Contexts have been destroyed. + nsCOMPtr<nsIRunnable> runnable = + NewRunnableMethod(this, &ThreadsafeHandle::AllowToCloseOnOwningThread); + MOZ_ALWAYS_SUCCEEDS( + mOwningThread->Dispatch(runnable, nsIThread::DISPATCH_NORMAL)); +} + +void +Context::ThreadsafeHandle::InvalidateAndAllowToClose() +{ + if (mOwningThread == NS_GetCurrentThread()) { + InvalidateAndAllowToCloseOnOwningThread(); + return; + } + + // Dispatch is guaranteed to succeed here because we block shutdown until + // all Contexts have been destroyed. + nsCOMPtr<nsIRunnable> runnable = + NewRunnableMethod(this, &ThreadsafeHandle::InvalidateAndAllowToCloseOnOwningThread); + MOZ_ALWAYS_SUCCEEDS( + mOwningThread->Dispatch(runnable, nsIThread::DISPATCH_NORMAL)); +} + +Context::ThreadsafeHandle::ThreadsafeHandle(Context* aContext) + : mStrongRef(aContext) + , mWeakRef(aContext) + , mOwningThread(NS_GetCurrentThread()) +{ +} + +Context::ThreadsafeHandle::~ThreadsafeHandle() +{ + // Normally we only touch mStrongRef on the owning thread. This is safe, + // however, because when we do use mStrongRef on the owning thread we are + // always holding a strong ref to the ThreadsafeHandle via the owning + // runnable. So we cannot run the ThreadsafeHandle destructor simultaneously. + if (!mStrongRef || mOwningThread == NS_GetCurrentThread()) { + return; + } + + // Dispatch is guaranteed to succeed here because we block shutdown until + // all Contexts have been destroyed. + NS_ProxyRelease(mOwningThread, mStrongRef.forget()); +} + +void +Context::ThreadsafeHandle::AllowToCloseOnOwningThread() +{ + MOZ_ASSERT(mOwningThread == NS_GetCurrentThread()); + + // A Context "closes" when its ref count drops to zero. Dropping this + // strong ref is necessary, but not sufficient for the close to occur. + // Any outstanding IO will continue and keep the Context alive. Once + // the Context is idle, it will be destroyed. + + // First, tell the context to flush any target thread shared data. This + // data must be released on the target thread prior to running the Context + // destructor. This will schedule an Action which ensures that the + // ~Context() is not immediately executed when we drop the strong ref. + if (mStrongRef) { + mStrongRef->DoomTargetData(); + } + + // Now drop our strong ref and let Context finish running any outstanding + // Actions. + mStrongRef = nullptr; +} + +void +Context::ThreadsafeHandle::InvalidateAndAllowToCloseOnOwningThread() +{ + MOZ_ASSERT(mOwningThread == NS_GetCurrentThread()); + // Cancel the Context through the weak reference. This means we can + // allow the Context to close by dropping the strong ref, but then + // still cancel ongoing IO if necessary. + if (mWeakRef) { + mWeakRef->Invalidate(); + } + // We should synchronously have AllowToCloseOnOwningThread called when + // the Context is canceled. + MOZ_DIAGNOSTIC_ASSERT(!mStrongRef); +} + +void +Context::ThreadsafeHandle::ContextDestroyed(Context* aContext) +{ + MOZ_ASSERT(mOwningThread == NS_GetCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(!mStrongRef); + MOZ_DIAGNOSTIC_ASSERT(mWeakRef); + MOZ_DIAGNOSTIC_ASSERT(mWeakRef == aContext); + mWeakRef = nullptr; +} + +// static +already_AddRefed<Context> +Context::Create(Manager* aManager, nsIThread* aTarget, + Action* aInitAction, Context* aOldContext) +{ + RefPtr<Context> context = new Context(aManager, aTarget, aInitAction); + context->Init(aOldContext); + return context.forget(); +} + +Context::Context(Manager* aManager, nsIThread* aTarget, Action* aInitAction) + : mManager(aManager) + , mTarget(aTarget) + , mData(new Data(aTarget)) + , mState(STATE_CONTEXT_PREINIT) + , mOrphanedData(false) + , mInitAction(aInitAction) +{ + MOZ_DIAGNOSTIC_ASSERT(mManager); + MOZ_DIAGNOSTIC_ASSERT(mTarget); +} + +void +Context::Dispatch(Action* aAction) +{ + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_DIAGNOSTIC_ASSERT(aAction); + + MOZ_DIAGNOSTIC_ASSERT(mState != STATE_CONTEXT_CANCELED); + if (mState == STATE_CONTEXT_CANCELED) { + return; + } else if (mState == STATE_CONTEXT_INIT || + mState == STATE_CONTEXT_PREINIT) { + PendingAction* pending = mPendingActions.AppendElement(); + pending->mAction = aAction; + return; + } + + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_CONTEXT_READY); + DispatchAction(aAction); +} + +void +Context::CancelAll() +{ + NS_ASSERT_OWNINGTHREAD(Context); + + // In PREINIT state we have not dispatch the init action yet. Just + // forget it. + if (mState == STATE_CONTEXT_PREINIT) { + MOZ_DIAGNOSTIC_ASSERT(!mInitRunnable); + mInitAction = nullptr; + + // In INIT state we have dispatched the runnable, but not received the + // async completion yet. Cancel the runnable, but don't forget about it + // until we get OnQuotaInit() callback. + } else if (mState == STATE_CONTEXT_INIT) { + mInitRunnable->Cancel(); + } + + mState = STATE_CONTEXT_CANCELED; + mPendingActions.Clear(); + { + ActivityList::ForwardIterator iter(mActivityList); + while (iter.HasMore()) { + iter.GetNext()->Cancel(); + } + } + AllowToClose(); +} + +bool +Context::IsCanceled() const +{ + NS_ASSERT_OWNINGTHREAD(Context); + return mState == STATE_CONTEXT_CANCELED; +} + +void +Context::Invalidate() +{ + NS_ASSERT_OWNINGTHREAD(Context); + mManager->NoteClosing(); + CancelAll(); +} + +void +Context::AllowToClose() +{ + NS_ASSERT_OWNINGTHREAD(Context); + if (mThreadsafeHandle) { + mThreadsafeHandle->AllowToClose(); + } +} + +void +Context::CancelForCacheId(CacheId aCacheId) +{ + NS_ASSERT_OWNINGTHREAD(Context); + + // Remove matching pending actions + for (int32_t i = mPendingActions.Length() - 1; i >= 0; --i) { + if (mPendingActions[i].mAction->MatchesCacheId(aCacheId)) { + mPendingActions.RemoveElementAt(i); + } + } + + // Cancel activities and let them remove themselves + ActivityList::ForwardIterator iter(mActivityList); + while (iter.HasMore()) { + Activity* activity = iter.GetNext(); + if (activity->MatchesCacheId(aCacheId)) { + activity->Cancel(); + } + } +} + +Context::~Context() +{ + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_DIAGNOSTIC_ASSERT(mManager); + MOZ_DIAGNOSTIC_ASSERT(!mData); + + if (mThreadsafeHandle) { + mThreadsafeHandle->ContextDestroyed(this); + } + + // Note, this may set the mOrphanedData flag. + mManager->RemoveContext(this); + + if (mQuotaInfo.mDir && !mOrphanedData) { + MOZ_ALWAYS_SUCCEEDS(DeleteMarkerFile(mQuotaInfo)); + } + + if (mNextContext) { + mNextContext->Start(); + } +} + +void +Context::Init(Context* aOldContext) +{ + NS_ASSERT_OWNINGTHREAD(Context); + + if (aOldContext) { + aOldContext->SetNextContext(this); + return; + } + + Start(); +} + +void +Context::Start() +{ + NS_ASSERT_OWNINGTHREAD(Context); + + // Previous context closing delayed our start, but then we were canceled. + // In this case, just do nothing here. + if (mState == STATE_CONTEXT_CANCELED) { + MOZ_DIAGNOSTIC_ASSERT(!mInitRunnable); + MOZ_DIAGNOSTIC_ASSERT(!mInitAction); + return; + } + + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_CONTEXT_PREINIT); + MOZ_DIAGNOSTIC_ASSERT(!mInitRunnable); + + mInitRunnable = new QuotaInitRunnable(this, mManager, mData, mTarget, + mInitAction); + mInitAction = nullptr; + + mState = STATE_CONTEXT_INIT; + + nsresult rv = mInitRunnable->Dispatch(); + if (NS_FAILED(rv)) { + // Shutdown must be delayed until all Contexts are destroyed. Shutdown + // must also prevent any new Contexts from being constructed. Crash + // for this invariant violation. + MOZ_CRASH("Failed to dispatch QuotaInitRunnable."); + } +} + +void +Context::DispatchAction(Action* aAction, bool aDoomData) +{ + NS_ASSERT_OWNINGTHREAD(Context); + + RefPtr<ActionRunnable> runnable = + new ActionRunnable(this, mData, mTarget, aAction, mQuotaInfo); + + if (aDoomData) { + mData = nullptr; + } + + nsresult rv = runnable->Dispatch(); + if (NS_FAILED(rv)) { + // Shutdown must be delayed until all Contexts are destroyed. Crash + // for this invariant violation. + MOZ_CRASH("Failed to dispatch ActionRunnable to target thread."); + } + AddActivity(runnable); +} + +void +Context::OnQuotaInit(nsresult aRv, const QuotaInfo& aQuotaInfo, + already_AddRefed<DirectoryLock> aDirectoryLock) +{ + NS_ASSERT_OWNINGTHREAD(Context); + + MOZ_DIAGNOSTIC_ASSERT(mInitRunnable); + mInitRunnable = nullptr; + + mQuotaInfo = aQuotaInfo; + + // Always save the directory lock to ensure QuotaManager does not shutdown + // before the Context has gone away. + MOZ_DIAGNOSTIC_ASSERT(!mDirectoryLock); + mDirectoryLock = aDirectoryLock; + + // If we opening the context failed, but we were not explicitly canceled, + // still treat the entire context as canceled. We don't want to allow + // new actions to be dispatched. We also cannot leave the context in + // the INIT state after failing to open. + if (NS_FAILED(aRv)) { + mState = STATE_CONTEXT_CANCELED; + } + + if (mState == STATE_CONTEXT_CANCELED) { + for (uint32_t i = 0; i < mPendingActions.Length(); ++i) { + mPendingActions[i].mAction->CompleteOnInitiatingThread(aRv); + } + mPendingActions.Clear(); + mThreadsafeHandle->AllowToClose(); + // Context will destruct after return here and last ref is released. + return; + } + + MOZ_DIAGNOSTIC_ASSERT(mState == STATE_CONTEXT_INIT); + mState = STATE_CONTEXT_READY; + + for (uint32_t i = 0; i < mPendingActions.Length(); ++i) { + DispatchAction(mPendingActions[i].mAction); + } + mPendingActions.Clear(); +} + +void +Context::AddActivity(Activity* aActivity) +{ + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_DIAGNOSTIC_ASSERT(aActivity); + MOZ_ASSERT(!mActivityList.Contains(aActivity)); + mActivityList.AppendElement(aActivity); +} + +void +Context::RemoveActivity(Activity* aActivity) +{ + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_DIAGNOSTIC_ASSERT(aActivity); + MOZ_ALWAYS_TRUE(mActivityList.RemoveElement(aActivity)); + MOZ_ASSERT(!mActivityList.Contains(aActivity)); +} + +void +Context::NoteOrphanedData() +{ + NS_ASSERT_OWNINGTHREAD(Context); + // This may be called more than once + mOrphanedData = true; +} + +already_AddRefed<Context::ThreadsafeHandle> +Context::CreateThreadsafeHandle() +{ + NS_ASSERT_OWNINGTHREAD(Context); + if (!mThreadsafeHandle) { + mThreadsafeHandle = new ThreadsafeHandle(this); + } + RefPtr<ThreadsafeHandle> ref = mThreadsafeHandle; + return ref.forget(); +} + +void +Context::SetNextContext(Context* aNextContext) +{ + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_DIAGNOSTIC_ASSERT(aNextContext); + MOZ_DIAGNOSTIC_ASSERT(!mNextContext); + mNextContext = aNextContext; +} + +void +Context::DoomTargetData() +{ + NS_ASSERT_OWNINGTHREAD(Context); + MOZ_DIAGNOSTIC_ASSERT(mData); + + // We are about to drop our reference to the Data. We need to ensure that + // the ~Context() destructor does not run until contents of Data have been + // released on the Target thread. + + // Dispatch a no-op Action. This will hold the Context alive through a + // roundtrip to the target thread and back to the owning thread. The + // ref to the Data object is cleared on the owning thread after creating + // the ActionRunnable, but before dispatching it. + RefPtr<Action> action = new NullAction(); + DispatchAction(action, true /* doomed data */); + + MOZ_DIAGNOSTIC_ASSERT(!mData); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/Context.h b/dom/cache/Context.h new file mode 100644 index 0000000000..278302bf6d --- /dev/null +++ b/dom/cache/Context.h @@ -0,0 +1,235 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Context_h +#define mozilla_dom_cache_Context_h + +#include "mozilla/dom/cache/Types.h" +#include "nsCOMPtr.h" +#include "nsISupportsImpl.h" +#include "nsProxyRelease.h" +#include "nsString.h" +#include "nsTArray.h" +#include "nsTObserverArray.h" + +class nsIEventTarget; +class nsIThread; + +namespace mozilla { +namespace dom { + +namespace quota { + +class DirectoryLock; + +} // namespace quota + +namespace cache { + +class Action; +class Manager; + +// The Context class is RAII-style class for managing IO operations within the +// Cache. +// +// When a Context is created it performs the complicated steps necessary to +// initialize the QuotaManager. Action objects dispatched on the Context are +// delayed until this initialization is complete. They are then allow to +// execute on any specified thread. Once all references to the Context are +// gone, then the steps necessary to release the QuotaManager are performed. +// After initialization the Context holds a self reference, so it will stay +// alive until one of three conditions occur: +// +// 1) The Manager will call Context::AllowToClose() when all of the actors +// have removed themselves as listener. This means an idle context with +// no active DOM objects will close gracefully. +// 2) The QuotaManager aborts all operations so it can delete the files. +// In this case the QuotaManager calls Client::AbortOperations() which +// in turn cancels all existing Action objects and then marks the Manager +// as invalid. +// 3) Browser shutdown occurs and the Manager calls Context::CancelAll(). +// +// In either case, though, the Action objects must be destroyed first to +// allow the Context to be destroyed. +// +// While the Context performs operations asynchronously on threads, all of +// methods in its public interface must be called on the same thread +// originally used to create the Context. +// +// As an invariant, all Context objects must be destroyed before permitting +// the "profile-before-change" shutdown event to complete. This is ensured +// via the code in ShutdownObserver.cpp. +class Context final +{ + typedef mozilla::dom::quota::DirectoryLock DirectoryLock; + +public: + // Define a class allowing other threads to hold the Context alive. This also + // allows these other threads to safely close or cancel the Context. + class ThreadsafeHandle final + { + friend class Context; + public: + void AllowToClose(); + void InvalidateAndAllowToClose(); + private: + explicit ThreadsafeHandle(Context* aContext); + ~ThreadsafeHandle(); + + // disallow copying + ThreadsafeHandle(const ThreadsafeHandle&) = delete; + ThreadsafeHandle& operator=(const ThreadsafeHandle&) = delete; + + void AllowToCloseOnOwningThread(); + void InvalidateAndAllowToCloseOnOwningThread(); + + void ContextDestroyed(Context* aContext); + + // Cleared to allow the Context to close. Only safe to access on + // owning thread. + RefPtr<Context> mStrongRef; + + // Used to support cancelation even while the Context is already allowed + // to close. Cleared by ~Context() calling ContextDestroyed(). Only + // safe to access on owning thread. + Context* mWeakRef; + + nsCOMPtr<nsIThread> mOwningThread; + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(cache::Context::ThreadsafeHandle) + }; + + // Different objects hold references to the Context while some work is being + // performed asynchronously. These objects must implement the Activity + // interface and register themselves with the AddActivity(). When they are + // destroyed they must call RemoveActivity(). This allows the Context to + // cancel any outstanding Activity work when the Context is cancelled. + class Activity + { + public: + virtual void Cancel() = 0; + virtual bool MatchesCacheId(CacheId aCacheId) const = 0; + }; + + // Create a Context attached to the given Manager. The given Action + // will run on the QuotaManager IO thread. Note, this Action must + // be execute synchronously. + static already_AddRefed<Context> + Create(Manager* aManager, nsIThread* aTarget, + Action* aInitAction, Context* aOldContext); + + // Execute given action on the target once the quota manager has been + // initialized. + // + // Only callable from the thread that created the Context. + void Dispatch(Action* aAction); + + // Cancel any Actions running or waiting to run. This should allow the + // Context to be released and Listener::RemoveContext() will be called + // when complete. + // + // Only callable from the thread that created the Context. + void CancelAll(); + + // True if CancelAll() has been called. + bool IsCanceled() const; + + // Like CancelAll(), but also marks the Manager as "invalid". + void Invalidate(); + + // Remove any self references and allow the Context to be released when + // there are no more Actions to process. + void AllowToClose(); + + // Cancel any Actions running or waiting to run that operate on the given + // cache ID. + // + // Only callable from the thread that created the Context. + void CancelForCacheId(CacheId aCacheId); + + void AddActivity(Activity* aActivity); + void RemoveActivity(Activity* aActivity); + + const QuotaInfo& + GetQuotaInfo() const + { + return mQuotaInfo; + } + + // Tell the Context that some state information has been orphaned in the + // data store and won't be cleaned up. The Context will leave the marker + // in place to trigger cleanup the next times its opened. + void NoteOrphanedData(); + +private: + class Data; + class QuotaInitRunnable; + class ActionRunnable; + + enum State + { + STATE_CONTEXT_PREINIT, + STATE_CONTEXT_INIT, + STATE_CONTEXT_READY, + STATE_CONTEXT_CANCELED + }; + + struct PendingAction + { + nsCOMPtr<nsIEventTarget> mTarget; + RefPtr<Action> mAction; + }; + + Context(Manager* aManager, nsIThread* aTarget, Action* aInitAction); + ~Context(); + void Init(Context* aOldContext); + void Start(); + void DispatchAction(Action* aAction, bool aDoomData = false); + void OnQuotaInit(nsresult aRv, const QuotaInfo& aQuotaInfo, + already_AddRefed<DirectoryLock> aDirectoryLock); + + + already_AddRefed<ThreadsafeHandle> + CreateThreadsafeHandle(); + + void + SetNextContext(Context* aNextContext); + + void + DoomTargetData(); + + RefPtr<Manager> mManager; + nsCOMPtr<nsIThread> mTarget; + RefPtr<Data> mData; + State mState; + bool mOrphanedData; + QuotaInfo mQuotaInfo; + RefPtr<QuotaInitRunnable> mInitRunnable; + RefPtr<Action> mInitAction; + nsTArray<PendingAction> mPendingActions; + + // Weak refs since activites must remove themselves from this list before + // being destroyed by calling RemoveActivity(). + typedef nsTObserverArray<Activity*> ActivityList; + ActivityList mActivityList; + + // The ThreadsafeHandle may have a strong ref back to us. This creates + // a ref-cycle that keeps the Context alive. The ref-cycle is broken + // when ThreadsafeHandle::AllowToClose() is called. + RefPtr<ThreadsafeHandle> mThreadsafeHandle; + + RefPtr<DirectoryLock> mDirectoryLock; + RefPtr<Context> mNextContext; + +public: + NS_INLINE_DECL_REFCOUNTING(cache::Context) +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_Context_h diff --git a/dom/cache/DBAction.cpp b/dom/cache/DBAction.cpp new file mode 100644 index 0000000000..ae009dc105 --- /dev/null +++ b/dom/cache/DBAction.cpp @@ -0,0 +1,231 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/DBAction.h" + +#include "mozilla/dom/cache/Connection.h" +#include "mozilla/dom/cache/DBSchema.h" +#include "mozilla/dom/cache/FileUtils.h" +#include "mozilla/dom/quota/PersistenceType.h" +#include "mozilla/net/nsFileProtocolHandler.h" +#include "mozIStorageConnection.h" +#include "mozIStorageService.h" +#include "mozStorageCID.h" +#include "nsIFile.h" +#include "nsIURI.h" +#include "nsIFileURL.h" +#include "nsThreadUtils.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::dom::quota::PERSISTENCE_TYPE_DEFAULT; +using mozilla::dom::quota::PersistenceType; + +DBAction::DBAction(Mode aMode) + : mMode(aMode) +{ +} + +DBAction::~DBAction() +{ +} + +void +DBAction::RunOnTarget(Resolver* aResolver, const QuotaInfo& aQuotaInfo, + Data* aOptionalData) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aResolver); + MOZ_DIAGNOSTIC_ASSERT(aQuotaInfo.mDir); + + if (IsCanceled()) { + aResolver->Resolve(NS_ERROR_ABORT); + return; + } + + nsCOMPtr<nsIFile> dbDir; + nsresult rv = aQuotaInfo.mDir->Clone(getter_AddRefs(dbDir)); + if (NS_WARN_IF(NS_FAILED(rv))) { + aResolver->Resolve(rv); + return; + } + + rv = dbDir->Append(NS_LITERAL_STRING("cache")); + if (NS_WARN_IF(NS_FAILED(rv))) { + aResolver->Resolve(rv); + return; + } + + nsCOMPtr<mozIStorageConnection> conn; + + // Attempt to reuse the connection opened by a previous Action. + if (aOptionalData) { + conn = aOptionalData->GetConnection(); + } + + // If there is no previous Action, then we must open one. + if (!conn) { + rv = OpenConnection(aQuotaInfo, dbDir, getter_AddRefs(conn)); + if (NS_WARN_IF(NS_FAILED(rv))) { + aResolver->Resolve(rv); + return; + } + MOZ_DIAGNOSTIC_ASSERT(conn); + + // Save this connection in the shared Data object so later Actions can + // use it. This avoids opening a new connection for every Action. + if (aOptionalData) { + // Since we know this connection will be around for as long as the + // Cache is open, use our special wrapped connection class. This + // will let us perform certain operations once the Cache origin + // is closed. + nsCOMPtr<mozIStorageConnection> wrapped = new Connection(conn); + aOptionalData->SetConnection(wrapped); + } + } + + RunWithDBOnTarget(aResolver, aQuotaInfo, dbDir, conn); +} + +nsresult +DBAction::OpenConnection(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection** aConnOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + MOZ_DIAGNOSTIC_ASSERT(aConnOut); + + nsCOMPtr<mozIStorageConnection> conn; + + bool exists; + nsresult rv = aDBDir->Exists(&exists); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!exists) { + if (NS_WARN_IF(mMode != Create)) { return NS_ERROR_FILE_NOT_FOUND; } + rv = aDBDir->Create(nsIFile::DIRECTORY_TYPE, 0755); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + nsCOMPtr<nsIFile> dbFile; + rv = aDBDir->Clone(getter_AddRefs(dbFile)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = dbFile->Append(NS_LITERAL_STRING("caches.sqlite")); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = dbFile->Exists(&exists); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Use our default file:// protocol handler directly to construct the database + // URL. This avoids any problems if a plugin registers a custom file:// + // handler. If such a custom handler used javascript, then we would have a + // bad time running off the main thread here. + RefPtr<nsFileProtocolHandler> handler = new nsFileProtocolHandler(); + rv = handler->Init(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsIURI> uri; + rv = handler->NewFileURI(dbFile, getter_AddRefs(uri)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsIFileURL> dbFileUrl = do_QueryInterface(uri); + if (NS_WARN_IF(!dbFileUrl)) { return NS_ERROR_UNEXPECTED; } + + nsAutoCString type; + PersistenceTypeToText(PERSISTENCE_TYPE_DEFAULT, type); + + rv = dbFileUrl->SetQuery( + NS_LITERAL_CSTRING("persistenceType=") + type + + NS_LITERAL_CSTRING("&group=") + aQuotaInfo.mGroup + + NS_LITERAL_CSTRING("&origin=") + aQuotaInfo.mOrigin + + NS_LITERAL_CSTRING("&cache=private")); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<mozIStorageService> ss = + do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID); + if (NS_WARN_IF(!ss)) { return NS_ERROR_UNEXPECTED; } + + rv = ss->OpenDatabaseWithFileURL(dbFileUrl, getter_AddRefs(conn)); + if (rv == NS_ERROR_FILE_CORRUPTED) { + NS_WARNING("Cache database corrupted. Recreating empty database."); + + conn = nullptr; + + // There is nothing else we can do to recover. Also, this data can + // be deleted by QuotaManager at any time anyways. + rv = WipeDatabase(dbFile, aDBDir); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = ss->OpenDatabaseWithFileURL(dbFileUrl, getter_AddRefs(conn)); + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Check the schema to make sure it is not too old. + int32_t schemaVersion = 0; + rv = conn->GetSchemaVersion(&schemaVersion); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (schemaVersion > 0 && schemaVersion < db::kFirstShippedSchemaVersion) { + conn = nullptr; + rv = WipeDatabase(dbFile, aDBDir); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = ss->OpenDatabaseWithFileURL(dbFileUrl, getter_AddRefs(conn)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + rv = db::InitializeConnection(conn); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + conn.forget(aConnOut); + + return rv; +} + +nsresult +DBAction::WipeDatabase(nsIFile* aDBFile, nsIFile* aDBDir) +{ + nsresult rv = aDBFile->Remove(false); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Note, the -wal journal file will be automatically deleted by sqlite when + // the new database is created. No need to explicitly delete it here. + + // Delete the morgue as well. + rv = BodyDeleteDir(aDBDir); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +SyncDBAction::SyncDBAction(Mode aMode) + : DBAction(aMode) +{ +} + +SyncDBAction::~SyncDBAction() +{ +} + +void +SyncDBAction::RunWithDBOnTarget(Resolver* aResolver, + const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aResolver); + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + nsresult rv = RunSyncWithDBOnTarget(aQuotaInfo, aDBDir, aConn); + aResolver->Resolve(rv); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/DBAction.h b/dom/cache/DBAction.h new file mode 100644 index 0000000000..d0bbedbbc9 --- /dev/null +++ b/dom/cache/DBAction.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_DBAction_h +#define mozilla_dom_cache_DBAction_h + +#include "mozilla/dom/cache/Action.h" +#include "mozilla/RefPtr.h" +#include "nsString.h" + +class mozIStorageConnection; +class nsIFile; + +namespace mozilla { +namespace dom { +namespace cache { + +class DBAction : public Action +{ +protected: + // The mode specifies whether the database should already exist or if its + // ok to create a new database. + enum Mode + { + Existing, + Create + }; + + explicit DBAction(Mode aMode); + + // Action objects are deleted through their base pointer + virtual ~DBAction(); + + // Just as the resolver must be ref'd until resolve, you may also + // ref the DB connection. The connection can only be referenced from the + // target thread and must be released upon resolve. + virtual void + RunWithDBOnTarget(Resolver* aResolver, const QuotaInfo& aQuotaInfo, + nsIFile* aDBDir, mozIStorageConnection* aConn) = 0; + +private: + virtual void + RunOnTarget(Resolver* aResolver, const QuotaInfo& aQuotaInfo, + Data* aOptionalData) override; + + nsresult OpenConnection(const QuotaInfo& aQuotaInfo, nsIFile* aQuotaDir, + mozIStorageConnection** aConnOut); + + nsresult WipeDatabase(nsIFile* aDBFile, nsIFile* aDBDir); + + const Mode mMode; +}; + +class SyncDBAction : public DBAction +{ +protected: + explicit SyncDBAction(Mode aMode); + + // Action objects are deleted through their base pointer + virtual ~SyncDBAction(); + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) = 0; + +private: + virtual void + RunWithDBOnTarget(Resolver* aResolver, const QuotaInfo& aQuotaInfo, + nsIFile* aDBDir, mozIStorageConnection* aConn) override; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_DBAction_h diff --git a/dom/cache/DBSchema.cpp b/dom/cache/DBSchema.cpp new file mode 100644 index 0000000000..d16ba2d6ab --- /dev/null +++ b/dom/cache/DBSchema.cpp @@ -0,0 +1,3018 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/DBSchema.h" + +#include "ipc/IPCMessageUtils.h" +#include "mozilla/BasePrincipal.h" +#include "mozilla/dom/HeadersBinding.h" +#include "mozilla/dom/InternalHeaders.h" +#include "mozilla/dom/RequestBinding.h" +#include "mozilla/dom/ResponseBinding.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/SavedTypes.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/dom/cache/TypeUtils.h" +#include "mozIStorageConnection.h" +#include "mozIStorageStatement.h" +#include "mozStorageHelper.h" +#include "nsCOMPtr.h" +#include "nsCRT.h" +#include "nsHttp.h" +#include "nsIContentPolicy.h" +#include "nsICryptoHash.h" +#include "nsNetCID.h" +#include "nsPrintfCString.h" +#include "nsTArray.h" + +namespace mozilla { +namespace dom { +namespace cache { +namespace db { +const int32_t kFirstShippedSchemaVersion = 15; +namespace { +// Update this whenever the DB schema is changed. +const int32_t kLatestSchemaVersion = 24; +// --------- +// The following constants define the SQL schema. These are defined in the +// same order the SQL should be executed in CreateOrMigrateSchema(). They are +// broken out as constants for convenient use in validation and migration. +// --------- +// The caches table is the single source of truth about what Cache +// objects exist for the origin. The contents of the Cache are stored +// in the entries table that references back to caches. +// +// The caches table is also referenced from storage. Rows in storage +// represent named Cache objects. There are cases, however, where +// a Cache can still exist, but not be in a named Storage. For example, +// when content is still using the Cache after CacheStorage::Delete() +// has been run. +// +// For now, the caches table mainly exists for data integrity with +// foreign keys, but could be expanded to contain additional cache object +// information. +// +// AUTOINCREMENT is necessary to prevent CacheId values from being reused. +const char* const kTableCaches = + "CREATE TABLE caches (" + "id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT " + ")"; + +// Security blobs are quite large and duplicated for every Response from +// the same https origin. This table is used to de-duplicate this data. +const char* const kTableSecurityInfo = + "CREATE TABLE security_info (" + "id INTEGER NOT NULL PRIMARY KEY, " + "hash BLOB NOT NULL, " // first 8-bytes of the sha1 hash of data column + "data BLOB NOT NULL, " // full security info data, usually a few KB + "refcount INTEGER NOT NULL" + ")"; + +// Index the smaller hash value instead of the large security data blob. +const char* const kIndexSecurityInfoHash = + "CREATE INDEX security_info_hash_index ON security_info (hash)"; + +const char* const kTableEntries = + "CREATE TABLE entries (" + "id INTEGER NOT NULL PRIMARY KEY, " + "request_method TEXT NOT NULL, " + "request_url_no_query TEXT NOT NULL, " + "request_url_no_query_hash BLOB NOT NULL, " // first 8-bytes of sha1 hash + "request_url_query TEXT NOT NULL, " + "request_url_query_hash BLOB NOT NULL, " // first 8-bytes of sha1 hash + "request_referrer TEXT NOT NULL, " + "request_headers_guard INTEGER NOT NULL, " + "request_mode INTEGER NOT NULL, " + "request_credentials INTEGER NOT NULL, " + "request_contentpolicytype INTEGER NOT NULL, " + "request_cache INTEGER NOT NULL, " + "request_body_id TEXT NULL, " + "response_type INTEGER NOT NULL, " + "response_status INTEGER NOT NULL, " + "response_status_text TEXT NOT NULL, " + "response_headers_guard INTEGER NOT NULL, " + "response_body_id TEXT NULL, " + "response_security_info_id INTEGER NULL REFERENCES security_info(id), " + "response_principal_info TEXT NOT NULL, " + "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, " + "request_redirect INTEGER NOT NULL, " + "request_referrer_policy INTEGER NOT NULL, " + "request_integrity TEXT NOT NULL, " + "request_url_fragment TEXT NOT NULL" + // New columns must be added at the end of table to migrate and + // validate properly. + ")"; +// Create an index to support the QueryCache() matching algorithm. This +// needs to quickly find entries in a given Cache that match the request +// URL. The url query is separated in order to support the ignoreSearch +// option. Finally, we index hashes of the URL values instead of the +// actual strings to avoid excessive disk bloat. The index will duplicate +// the contents of the columsn in the index. The hash index will prune +// the vast majority of values from the query result so that normal +// scanning only has to be done on a few values to find an exact URL match. +const char* const kIndexEntriesRequest = + "CREATE INDEX entries_request_match_index " + "ON entries (cache_id, request_url_no_query_hash, " + "request_url_query_hash)"; + +const char* const kTableRequestHeaders = + "CREATE TABLE request_headers (" + "name TEXT NOT NULL, " + "value TEXT NOT NULL, " + "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE" + ")"; + +const char* const kTableResponseHeaders = + "CREATE TABLE response_headers (" + "name TEXT NOT NULL, " + "value TEXT NOT NULL, " + "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE" + ")"; + +// We need an index on response_headers, but not on request_headers, +// because we quickly need to determine if a VARY header is present. +const char* const kIndexResponseHeadersName = + "CREATE INDEX response_headers_name_index " + "ON response_headers (name)"; + +const char* const kTableResponseUrlList = + "CREATE TABLE response_url_list (" + "url TEXT NOT NULL, " + "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE" + ")"; + +// NOTE: key allows NULL below since that is how "" is represented +// in a BLOB column. We use BLOB to avoid encoding issues +// with storing DOMStrings. +const char* const kTableStorage = + "CREATE TABLE storage (" + "namespace INTEGER NOT NULL, " + "key BLOB NULL, " + "cache_id INTEGER NOT NULL REFERENCES caches(id), " + "PRIMARY KEY(namespace, key) " + ")"; + +// --------- +// End schema definition +// --------- + +const int32_t kMaxEntriesPerStatement = 255; + +const uint32_t kPageSize = 4 * 1024; + +// Grow the database in chunks to reduce fragmentation +const uint32_t kGrowthSize = 32 * 1024; +const uint32_t kGrowthPages = kGrowthSize / kPageSize; +static_assert(kGrowthSize % kPageSize == 0, + "Growth size must be multiple of page size"); + +// Only release free pages when we have more than this limit +const int32_t kMaxFreePages = kGrowthPages; + +// Limit WAL journal to a reasonable size +const uint32_t kWalAutoCheckpointSize = 512 * 1024; +const uint32_t kWalAutoCheckpointPages = kWalAutoCheckpointSize / kPageSize; +static_assert(kWalAutoCheckpointSize % kPageSize == 0, + "WAL checkpoint size must be multiple of page size"); + +} // namespace + +// If any of the static_asserts below fail, it means that you have changed +// the corresponding WebIDL enum in a way that may be incompatible with the +// existing data stored in the DOM Cache. You would need to update the Cache +// database schema accordingly and adjust the failing static_assert. +static_assert(int(HeadersGuardEnum::None) == 0 && + int(HeadersGuardEnum::Request) == 1 && + int(HeadersGuardEnum::Request_no_cors) == 2 && + int(HeadersGuardEnum::Response) == 3 && + int(HeadersGuardEnum::Immutable) == 4 && + int(HeadersGuardEnum::EndGuard_) == 5, + "HeadersGuardEnum values are as expected"); +static_assert(int(ReferrerPolicy::_empty) == 0 && + int(ReferrerPolicy::No_referrer) == 1 && + int(ReferrerPolicy::No_referrer_when_downgrade) == 2 && + int(ReferrerPolicy::Origin) == 3 && + int(ReferrerPolicy::Origin_when_cross_origin) == 4 && + int(ReferrerPolicy::Unsafe_url) == 5 && + int(ReferrerPolicy::EndGuard_) == 6, + "ReferrerPolicy values are as expected"); +static_assert(int(RequestMode::Same_origin) == 0 && + int(RequestMode::No_cors) == 1 && + int(RequestMode::Cors) == 2 && + int(RequestMode::Navigate) == 3 && + int(RequestMode::EndGuard_) == 4, + "RequestMode values are as expected"); +static_assert(int(RequestCredentials::Omit) == 0 && + int(RequestCredentials::Same_origin) == 1 && + int(RequestCredentials::Include) == 2 && + int(RequestCredentials::EndGuard_) == 3, + "RequestCredentials values are as expected"); +static_assert(int(RequestCache::Default) == 0 && + int(RequestCache::No_store) == 1 && + int(RequestCache::Reload) == 2 && + int(RequestCache::No_cache) == 3 && + int(RequestCache::Force_cache) == 4 && + int(RequestCache::Only_if_cached) == 5 && + int(RequestCache::EndGuard_) == 6, + "RequestCache values are as expected"); +static_assert(int(RequestRedirect::Follow) == 0 && + int(RequestRedirect::Error) == 1 && + int(RequestRedirect::Manual) == 2 && + int(RequestRedirect::EndGuard_) == 3, + "RequestRedirect values are as expected"); +static_assert(int(ResponseType::Basic) == 0 && + int(ResponseType::Cors) == 1 && + int(ResponseType::Default) == 2 && + int(ResponseType::Error) == 3 && + int(ResponseType::Opaque) == 4 && + int(ResponseType::Opaqueredirect) == 5 && + int(ResponseType::EndGuard_) == 6, + "ResponseType values are as expected"); + +// If the static_asserts below fails, it means that you have changed the +// Namespace enum in a way that may be incompatible with the existing data +// stored in the DOM Cache. You would need to update the Cache database schema +// accordingly and adjust the failing static_assert. +static_assert(DEFAULT_NAMESPACE == 0 && + CHROME_ONLY_NAMESPACE == 1 && + NUMBER_OF_NAMESPACES == 2, + "Namespace values are as expected"); + +// If the static_asserts below fails, it means that you have changed the +// nsContentPolicy enum in a way that may be incompatible with the existing data +// stored in the DOM Cache. You would need to update the Cache database schema +// accordingly and adjust the failing static_assert. +static_assert(nsIContentPolicy::TYPE_INVALID == 0 && + nsIContentPolicy::TYPE_OTHER == 1 && + nsIContentPolicy::TYPE_SCRIPT == 2 && + nsIContentPolicy::TYPE_IMAGE == 3 && + nsIContentPolicy::TYPE_STYLESHEET == 4 && + nsIContentPolicy::TYPE_OBJECT == 5 && + nsIContentPolicy::TYPE_DOCUMENT == 6 && + nsIContentPolicy::TYPE_SUBDOCUMENT == 7 && + nsIContentPolicy::TYPE_REFRESH == 8 && + nsIContentPolicy::TYPE_XBL == 9 && + nsIContentPolicy::TYPE_PING == 10 && + nsIContentPolicy::TYPE_XMLHTTPREQUEST == 11 && + nsIContentPolicy::TYPE_DATAREQUEST == 11 && + nsIContentPolicy::TYPE_OBJECT_SUBREQUEST == 12 && + nsIContentPolicy::TYPE_DTD == 13 && + nsIContentPolicy::TYPE_FONT == 14 && + nsIContentPolicy::TYPE_MEDIA == 15 && + nsIContentPolicy::TYPE_WEBSOCKET == 16 && + nsIContentPolicy::TYPE_CSP_REPORT == 17 && + nsIContentPolicy::TYPE_XSLT == 18 && + nsIContentPolicy::TYPE_BEACON == 19 && + nsIContentPolicy::TYPE_FETCH == 20 && + nsIContentPolicy::TYPE_IMAGESET == 21 && + nsIContentPolicy::TYPE_WEB_MANIFEST == 22 && + nsIContentPolicy::TYPE_INTERNAL_SCRIPT == 23 && + nsIContentPolicy::TYPE_INTERNAL_WORKER == 24 && + nsIContentPolicy::TYPE_INTERNAL_SHARED_WORKER == 25 && + nsIContentPolicy::TYPE_INTERNAL_EMBED == 26 && + nsIContentPolicy::TYPE_INTERNAL_OBJECT == 27 && + nsIContentPolicy::TYPE_INTERNAL_FRAME == 28 && + nsIContentPolicy::TYPE_INTERNAL_IFRAME == 29 && + nsIContentPolicy::TYPE_INTERNAL_AUDIO == 30 && + nsIContentPolicy::TYPE_INTERNAL_VIDEO == 31 && + nsIContentPolicy::TYPE_INTERNAL_TRACK == 32 && + nsIContentPolicy::TYPE_INTERNAL_XMLHTTPREQUEST == 33 && + nsIContentPolicy::TYPE_INTERNAL_EVENTSOURCE == 34 && + nsIContentPolicy::TYPE_INTERNAL_SERVICE_WORKER == 35 && + nsIContentPolicy::TYPE_INTERNAL_SCRIPT_PRELOAD == 36 && + nsIContentPolicy::TYPE_INTERNAL_IMAGE == 37 && + nsIContentPolicy::TYPE_INTERNAL_IMAGE_PRELOAD == 38 && + nsIContentPolicy::TYPE_INTERNAL_STYLESHEET == 39 && + nsIContentPolicy::TYPE_INTERNAL_STYLESHEET_PRELOAD == 40 && + nsIContentPolicy::TYPE_INTERNAL_IMAGE_FAVICON == 41, + "nsContentPolicyType values are as expected"); + +namespace { + +typedef int32_t EntryId; + +struct IdCount +{ + IdCount() : mId(-1), mCount(0) { } + explicit IdCount(int32_t aId) : mId(aId), mCount(1) { } + int32_t mId; + int32_t mCount; +}; + +static nsresult QueryAll(mozIStorageConnection* aConn, CacheId aCacheId, + nsTArray<EntryId>& aEntryIdListOut); +static nsresult QueryCache(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequest& aRequest, + const CacheQueryParams& aParams, + nsTArray<EntryId>& aEntryIdListOut, + uint32_t aMaxResults = UINT32_MAX); +static nsresult MatchByVaryHeader(mozIStorageConnection* aConn, + const CacheRequest& aRequest, + EntryId entryId, bool* aSuccessOut); +static nsresult DeleteEntries(mozIStorageConnection* aConn, + const nsTArray<EntryId>& aEntryIdList, + nsTArray<nsID>& aDeletedBodyIdListOut, + nsTArray<IdCount>& aDeletedSecurityIdListOut, + uint32_t aPos=0, int32_t aLen=-1); +static nsresult InsertSecurityInfo(mozIStorageConnection* aConn, + nsICryptoHash* aCrypto, + const nsACString& aData, int32_t *aIdOut); +static nsresult DeleteSecurityInfo(mozIStorageConnection* aConn, int32_t aId, + int32_t aCount); +static nsresult DeleteSecurityInfoList(mozIStorageConnection* aConn, + const nsTArray<IdCount>& aDeletedStorageIdList); +static nsresult InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequest& aRequest, + const nsID* aRequestBodyId, + const CacheResponse& aResponse, + const nsID* aResponseBodyId); +static nsresult ReadResponse(mozIStorageConnection* aConn, EntryId aEntryId, + SavedResponse* aSavedResponseOut); +static nsresult ReadRequest(mozIStorageConnection* aConn, EntryId aEntryId, + SavedRequest* aSavedRequestOut); + +static void AppendListParamsToQuery(nsACString& aQuery, + const nsTArray<EntryId>& aEntryIdList, + uint32_t aPos, int32_t aLen); +static nsresult BindListParamsToQuery(mozIStorageStatement* aState, + const nsTArray<EntryId>& aEntryIdList, + uint32_t aPos, int32_t aLen); +static nsresult BindId(mozIStorageStatement* aState, const nsACString& aName, + const nsID* aId); +static nsresult ExtractId(mozIStorageStatement* aState, uint32_t aPos, + nsID* aIdOut); +static nsresult CreateAndBindKeyStatement(mozIStorageConnection* aConn, + const char* aQueryFormat, + const nsAString& aKey, + mozIStorageStatement** aStateOut); +static nsresult HashCString(nsICryptoHash* aCrypto, const nsACString& aIn, + nsACString& aOut); +nsresult Validate(mozIStorageConnection* aConn); +nsresult Migrate(mozIStorageConnection* aConn); +} // namespace + +class MOZ_RAII AutoDisableForeignKeyChecking +{ +public: + explicit AutoDisableForeignKeyChecking(mozIStorageConnection* aConn) + : mConn(aConn) + , mForeignKeyCheckingDisabled(false) + { + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = mConn->CreateStatement(NS_LITERAL_CSTRING( + "PRAGMA foreign_keys;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return; } + + int32_t mode; + rv = state->GetInt32(0, &mode); + if (NS_WARN_IF(NS_FAILED(rv))) { return; } + + if (mode) { + nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "PRAGMA foreign_keys = OFF;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return; } + mForeignKeyCheckingDisabled = true; + } + } + + ~AutoDisableForeignKeyChecking() + { + if (mForeignKeyCheckingDisabled) { + nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "PRAGMA foreign_keys = ON;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return; } + } + } + +private: + nsCOMPtr<mozIStorageConnection> mConn; + bool mForeignKeyCheckingDisabled; +}; + +nsresult +CreateOrMigrateSchema(mozIStorageConnection* aConn) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + int32_t schemaVersion; + nsresult rv = aConn->GetSchemaVersion(&schemaVersion); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (schemaVersion == kLatestSchemaVersion) { + // We already have the correct schema version. Validate it matches + // our expected schema and then proceed. + rv = Validate(aConn); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; + } + + // Turn off checking foreign keys before starting a transaction, and restore + // it once we're done. + AutoDisableForeignKeyChecking restoreForeignKeyChecking(aConn); + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + bool needVacuum = false; + + if (schemaVersion) { + // A schema exists, but its not the current version. Attempt to + // migrate it to our new schema. + rv = Migrate(aConn); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Migrations happen infrequently and reflect a chance in DB structure. + // This is a good time to rebuild the database. It also helps catch + // if a new migration is incorrect by fast failing on the corruption. + needVacuum = true; + } else { + // There is no schema installed. Create the database from scratch. + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableCaches)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableSecurityInfo)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kIndexSecurityInfoHash)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableEntries)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kIndexEntriesRequest)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableRequestHeaders)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableResponseHeaders)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kIndexResponseHeadersName)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableResponseUrlList)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kTableStorage)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->SetSchemaVersion(kLatestSchemaVersion); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->GetSchemaVersion(&schemaVersion); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + rv = Validate(aConn); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = trans.Commit(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (needVacuum) { + // Unfortunately, this must be performed outside of the transaction. + aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("VACUUM")); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + return rv; +} + +nsresult +InitializeConnection(mozIStorageConnection* aConn) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // This function needs to perform per-connection initialization tasks that + // need to happen regardless of the schema. + + nsPrintfCString pragmas( + // Use a smaller page size to improve perf/footprint; default is too large + "PRAGMA page_size = %u; " + // Enable auto_vacuum; this must happen after page_size and before WAL + "PRAGMA auto_vacuum = INCREMENTAL; " + "PRAGMA foreign_keys = ON; ", + kPageSize + ); + + // Note, the default encoding of UTF-8 is preferred. mozStorage does all + // the work necessary to convert UTF-16 nsString values for us. We don't + // need ordering and the binary equality operations are correct. So, do + // NOT set PRAGMA encoding to UTF-16. + + nsresult rv = aConn->ExecuteSimpleSQL(pragmas); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Limit fragmentation by growing the database by many pages at once. + rv = aConn->SetGrowthIncrement(kGrowthSize, EmptyCString()); + if (rv == NS_ERROR_FILE_TOO_BIG) { + NS_WARNING("Not enough disk space to set sqlite growth increment."); + rv = NS_OK; + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Enable WAL journaling. This must be performed in a separate transaction + // after changing the page_size and enabling auto_vacuum. + nsPrintfCString wal( + // WAL journal can grow to given number of *pages* + "PRAGMA wal_autocheckpoint = %u; " + // Always truncate the journal back to given number of *bytes* + "PRAGMA journal_size_limit = %u; " + // WAL must be enabled at the end to allow page size to be changed, etc. + "PRAGMA journal_mode = WAL; ", + kWalAutoCheckpointPages, + kWalAutoCheckpointSize + ); + + rv = aConn->ExecuteSimpleSQL(wal); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Verify that we successfully set the vacuum mode to incremental. It + // is very easy to put the database in a state where the auto_vacuum + // pragma above fails silently. +#ifdef DEBUG + nsCOMPtr<mozIStorageStatement> state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "PRAGMA auto_vacuum;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t mode; + rv = state->GetInt32(0, &mode); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // integer value 2 is incremental mode + if (NS_WARN_IF(mode != 2)) { return NS_ERROR_UNEXPECTED; } +#endif + + return NS_OK; +} + +nsresult +CreateCacheId(mozIStorageConnection* aConn, CacheId* aCacheIdOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aCacheIdOut); + + nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "INSERT INTO caches DEFAULT VALUES;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<mozIStorageStatement> state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT last_insert_rowid()" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (NS_WARN_IF(!hasMoreData)) { return NS_ERROR_UNEXPECTED; } + + rv = state->GetInt64(0, aCacheIdOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult +DeleteCacheId(mozIStorageConnection* aConn, CacheId aCacheId, + nsTArray<nsID>& aDeletedBodyIdListOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // Delete the bodies explicitly as we need to read out the body IDs + // anyway. These body IDs must be deleted one-by-one as content may + // still be referencing them invidivually. + AutoTArray<EntryId, 256> matches; + nsresult rv = QueryAll(aConn, aCacheId, matches); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + AutoTArray<IdCount, 16> deletedSecurityIdList; + rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut, + deletedSecurityIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Delete the remainder of the cache using cascade semantics. + nsCOMPtr<mozIStorageStatement> state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "DELETE FROM caches WHERE id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt64ByName(NS_LITERAL_CSTRING("id"), aCacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult +IsCacheOrphaned(mozIStorageConnection* aConn, CacheId aCacheId, + bool* aOrphanedOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aOrphanedOut); + + // err on the side of not deleting user data + *aOrphanedOut = false; + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT COUNT(*) FROM storage WHERE cache_id=:cache_id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt64ByName(NS_LITERAL_CSTRING("cache_id"), aCacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + MOZ_DIAGNOSTIC_ASSERT(hasMoreData); + + int32_t refCount; + rv = state->GetInt32(0, &refCount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + *aOrphanedOut = refCount == 0; + + return rv; +} + +nsresult +FindOrphanedCacheIds(mozIStorageConnection* aConn, + nsTArray<CacheId>& aOrphanedListOut) +{ + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT id FROM caches " + "WHERE id NOT IN (SELECT cache_id from storage);" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + CacheId cacheId = INVALID_CACHE_ID; + rv = state->GetInt64(0, &cacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aOrphanedListOut.AppendElement(cacheId); + } + + return rv; +} + +nsresult +GetKnownBodyIds(mozIStorageConnection* aConn, nsTArray<nsID>& aBodyIdListOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT request_body_id, response_body_id FROM entries;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + // extract 0 to 2 nsID structs per row + for (uint32_t i = 0; i < 2; ++i) { + bool isNull = false; + + rv = state->GetIsNull(i, &isNull); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!isNull) { + nsID id; + rv = ExtractId(state, i, &id); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aBodyIdListOut.AppendElement(id); + } + } + } + + return rv; +} + +nsresult +CacheMatch(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequest& aRequest, + const CacheQueryParams& aParams, + bool* aFoundResponseOut, + SavedResponse* aSavedResponseOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aFoundResponseOut); + MOZ_DIAGNOSTIC_ASSERT(aSavedResponseOut); + + *aFoundResponseOut = false; + + AutoTArray<EntryId, 1> matches; + nsresult rv = QueryCache(aConn, aCacheId, aRequest, aParams, matches, 1); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (matches.IsEmpty()) { + return rv; + } + + rv = ReadResponse(aConn, matches[0], aSavedResponseOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aSavedResponseOut->mCacheId = aCacheId; + *aFoundResponseOut = true; + + return rv; +} + +nsresult +CacheMatchAll(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequestOrVoid& aRequestOrVoid, + const CacheQueryParams& aParams, + nsTArray<SavedResponse>& aSavedResponsesOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + nsresult rv; + + AutoTArray<EntryId, 256> matches; + if (aRequestOrVoid.type() == CacheRequestOrVoid::Tvoid_t) { + rv = QueryAll(aConn, aCacheId, matches); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } else { + rv = QueryCache(aConn, aCacheId, aRequestOrVoid, aParams, matches); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + // TODO: replace this with a bulk load using SQL IN clause (bug 1110458) + for (uint32_t i = 0; i < matches.Length(); ++i) { + SavedResponse savedResponse; + rv = ReadResponse(aConn, matches[i], &savedResponse); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + savedResponse.mCacheId = aCacheId; + aSavedResponsesOut.AppendElement(savedResponse); + } + + return rv; +} + +nsresult +CachePut(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequest& aRequest, + const nsID* aRequestBodyId, + const CacheResponse& aResponse, + const nsID* aResponseBodyId, + nsTArray<nsID>& aDeletedBodyIdListOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + CacheQueryParams params(false, false, false, false, + NS_LITERAL_STRING("")); + AutoTArray<EntryId, 256> matches; + nsresult rv = QueryCache(aConn, aCacheId, aRequest, params, matches); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + AutoTArray<IdCount, 16> deletedSecurityIdList; + rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut, + deletedSecurityIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = InsertEntry(aConn, aCacheId, aRequest, aRequestBodyId, aResponse, + aResponseBodyId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Delete the security values after doing the insert to avoid churning + // the security table when its not necessary. + rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult +CacheDelete(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequest& aRequest, + const CacheQueryParams& aParams, + nsTArray<nsID>& aDeletedBodyIdListOut, bool* aSuccessOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aSuccessOut); + + *aSuccessOut = false; + + AutoTArray<EntryId, 256> matches; + nsresult rv = QueryCache(aConn, aCacheId, aRequest, aParams, matches); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (matches.IsEmpty()) { + return rv; + } + + AutoTArray<IdCount, 16> deletedSecurityIdList; + rv = DeleteEntries(aConn, matches, aDeletedBodyIdListOut, + deletedSecurityIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = DeleteSecurityInfoList(aConn, deletedSecurityIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + *aSuccessOut = true; + + return rv; +} + +nsresult +CacheKeys(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequestOrVoid& aRequestOrVoid, + const CacheQueryParams& aParams, + nsTArray<SavedRequest>& aSavedRequestsOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + nsresult rv; + + AutoTArray<EntryId, 256> matches; + if (aRequestOrVoid.type() == CacheRequestOrVoid::Tvoid_t) { + rv = QueryAll(aConn, aCacheId, matches); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } else { + rv = QueryCache(aConn, aCacheId, aRequestOrVoid, aParams, matches); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + // TODO: replace this with a bulk load using SQL IN clause (bug 1110458) + for (uint32_t i = 0; i < matches.Length(); ++i) { + SavedRequest savedRequest; + rv = ReadRequest(aConn, matches[i], &savedRequest); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + savedRequest.mCacheId = aCacheId; + aSavedRequestsOut.AppendElement(savedRequest); + } + + return rv; +} + +nsresult +StorageMatch(mozIStorageConnection* aConn, + Namespace aNamespace, + const CacheRequest& aRequest, + const CacheQueryParams& aParams, + bool* aFoundResponseOut, + SavedResponse* aSavedResponseOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aFoundResponseOut); + MOZ_DIAGNOSTIC_ASSERT(aSavedResponseOut); + + *aFoundResponseOut = false; + + nsresult rv; + + // If we are given a cache to check, then simply find its cache ID + // and perform the match. + if (!aParams.cacheName().EqualsLiteral("")) { + bool foundCache = false; + // no invalid CacheId, init to least likely real value + CacheId cacheId = INVALID_CACHE_ID; + rv = StorageGetCacheId(aConn, aNamespace, aParams.cacheName(), &foundCache, + &cacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (!foundCache) { return NS_OK; } + + rv = CacheMatch(aConn, cacheId, aRequest, aParams, aFoundResponseOut, + aSavedResponseOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; + } + + // Otherwise we need to get a list of all the cache IDs in this namespace. + + nsCOMPtr<mozIStorageStatement> state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT cache_id FROM storage WHERE namespace=:namespace ORDER BY rowid;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("namespace"), aNamespace); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + AutoTArray<CacheId, 32> cacheIdList; + + bool hasMoreData = false; + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + CacheId cacheId = INVALID_CACHE_ID; + rv = state->GetInt64(0, &cacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + cacheIdList.AppendElement(cacheId); + } + + // Now try to find a match in each cache in order + for (uint32_t i = 0; i < cacheIdList.Length(); ++i) { + rv = CacheMatch(aConn, cacheIdList[i], aRequest, aParams, aFoundResponseOut, + aSavedResponseOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (*aFoundResponseOut) { + aSavedResponseOut->mCacheId = cacheIdList[i]; + return rv; + } + } + + return NS_OK; +} + +nsresult +StorageGetCacheId(mozIStorageConnection* aConn, Namespace aNamespace, + const nsAString& aKey, bool* aFoundCacheOut, + CacheId* aCacheIdOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aFoundCacheOut); + MOZ_DIAGNOSTIC_ASSERT(aCacheIdOut); + + *aFoundCacheOut = false; + + // How we constrain the key column depends on the value of our key. Use + // a format string for the query and let CreateAndBindKeyStatement() fill + // it in for us. + const char* query = "SELECT cache_id FROM storage " + "WHERE namespace=:namespace AND %s " + "ORDER BY rowid;"; + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = CreateAndBindKeyStatement(aConn, query, aKey, + getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("namespace"), aNamespace); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!hasMoreData) { + return rv; + } + + rv = state->GetInt64(0, aCacheIdOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + *aFoundCacheOut = true; + return rv; +} + +nsresult +StoragePutCache(mozIStorageConnection* aConn, Namespace aNamespace, + const nsAString& aKey, CacheId aCacheId) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "INSERT INTO storage (namespace, key, cache_id) " + "VALUES (:namespace, :key, :cache_id);" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("namespace"), aNamespace); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindStringAsBlobByName(NS_LITERAL_CSTRING("key"), aKey); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt64ByName(NS_LITERAL_CSTRING("cache_id"), aCacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult +StorageForgetCache(mozIStorageConnection* aConn, Namespace aNamespace, + const nsAString& aKey) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // How we constrain the key column depends on the value of our key. Use + // a format string for the query and let CreateAndBindKeyStatement() fill + // it in for us. + const char *query = "DELETE FROM storage WHERE namespace=:namespace AND %s;"; + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = CreateAndBindKeyStatement(aConn, query, aKey, + getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("namespace"), aNamespace); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult +StorageGetKeys(mozIStorageConnection* aConn, Namespace aNamespace, + nsTArray<nsString>& aKeysOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT key FROM storage WHERE namespace=:namespace ORDER BY rowid;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("namespace"), aNamespace); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + nsAutoString key; + rv = state->GetBlobAsString(0, key); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aKeysOut.AppendElement(key); + } + + return rv; +} + +namespace { + +nsresult +QueryAll(mozIStorageConnection* aConn, CacheId aCacheId, + nsTArray<EntryId>& aEntryIdListOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT id FROM entries WHERE cache_id=:cache_id ORDER BY id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt64ByName(NS_LITERAL_CSTRING("cache_id"), aCacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + EntryId entryId = INT32_MAX; + rv = state->GetInt32(0, &entryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aEntryIdListOut.AppendElement(entryId); + } + + return rv; +} + +nsresult +QueryCache(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequest& aRequest, + const CacheQueryParams& aParams, + nsTArray<EntryId>& aEntryIdListOut, + uint32_t aMaxResults) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aMaxResults > 0); + + if (!aParams.ignoreMethod() && !aRequest.method().LowerCaseEqualsLiteral("get") + && !aRequest.method().LowerCaseEqualsLiteral("head")) + { + return NS_OK; + } + + nsAutoCString query( + "SELECT id, COUNT(response_headers.name) AS vary_count " + "FROM entries " + "LEFT OUTER JOIN response_headers ON entries.id=response_headers.entry_id " + "AND response_headers.name='vary' " + "WHERE entries.cache_id=:cache_id " + "AND entries.request_url_no_query_hash=:url_no_query_hash " + ); + + if (!aParams.ignoreSearch()) { + query.AppendLiteral("AND entries.request_url_query_hash=:url_query_hash "); + } + + query.AppendLiteral("AND entries.request_url_no_query=:url_no_query "); + + if (!aParams.ignoreSearch()) { + query.AppendLiteral("AND entries.request_url_query=:url_query "); + } + + query.AppendLiteral("GROUP BY entries.id ORDER BY entries.id;"); + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(query, getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt64ByName(NS_LITERAL_CSTRING("cache_id"), aCacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsICryptoHash> crypto = + do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsAutoCString urlWithoutQueryHash; + rv = HashCString(crypto, aRequest.urlWithoutQuery(), urlWithoutQueryHash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("url_no_query_hash"), + urlWithoutQueryHash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!aParams.ignoreSearch()) { + nsAutoCString urlQueryHash; + rv = HashCString(crypto, aRequest.urlQuery(), urlQueryHash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("url_query_hash"), + urlQueryHash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("url_no_query"), + aRequest.urlWithoutQuery()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!aParams.ignoreSearch()) { + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("url_query"), + aRequest.urlQuery()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + bool hasMoreData = false; + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + // no invalid EntryId, init to least likely real value + EntryId entryId = INT32_MAX; + rv = state->GetInt32(0, &entryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t varyCount; + rv = state->GetInt32(1, &varyCount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!aParams.ignoreVary() && varyCount > 0) { + bool matchedByVary = false; + rv = MatchByVaryHeader(aConn, aRequest, entryId, &matchedByVary); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (!matchedByVary) { + continue; + } + } + + aEntryIdListOut.AppendElement(entryId); + + if (aEntryIdListOut.Length() == aMaxResults) { + return NS_OK; + } + } + + return rv; +} + +nsresult +MatchByVaryHeader(mozIStorageConnection* aConn, + const CacheRequest& aRequest, + EntryId entryId, bool* aSuccessOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + *aSuccessOut = false; + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT value FROM response_headers " + "WHERE name='vary' AND entry_id=:entry_id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("entry_id"), entryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + AutoTArray<nsCString, 8> varyValues; + + bool hasMoreData = false; + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + nsAutoCString value; + rv = state->GetUTF8String(0, value); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + varyValues.AppendElement(value); + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Should not have called this function if this was not the case + MOZ_DIAGNOSTIC_ASSERT(!varyValues.IsEmpty()); + + state->Reset(); + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT name, value FROM request_headers " + "WHERE entry_id=:entry_id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("entry_id"), entryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + RefPtr<InternalHeaders> cachedHeaders = + new InternalHeaders(HeadersGuardEnum::None); + + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + nsAutoCString name; + nsAutoCString value; + rv = state->GetUTF8String(0, name); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + rv = state->GetUTF8String(1, value); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + ErrorResult errorResult; + + cachedHeaders->Append(name, value, errorResult); + if (errorResult.Failed()) { return errorResult.StealNSResult(); } + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + RefPtr<InternalHeaders> queryHeaders = + TypeUtils::ToInternalHeaders(aRequest.headers()); + + // Assume the vary headers match until we find a conflict + bool varyHeadersMatch = true; + + for (uint32_t i = 0; i < varyValues.Length(); ++i) { + // Extract the header names inside the Vary header value. + nsAutoCString varyValue(varyValues[i]); + char* rawBuffer = varyValue.BeginWriting(); + char* token = nsCRT::strtok(rawBuffer, NS_HTTP_HEADER_SEPS, &rawBuffer); + bool bailOut = false; + for (; token; + token = nsCRT::strtok(rawBuffer, NS_HTTP_HEADER_SEPS, &rawBuffer)) { + nsDependentCString header(token); + MOZ_DIAGNOSTIC_ASSERT(!header.EqualsLiteral("*"), + "We should have already caught this in " + "TypeUtils::ToPCacheResponseWithoutBody()"); + + ErrorResult errorResult; + nsAutoCString queryValue; + queryHeaders->Get(header, queryValue, errorResult); + if (errorResult.Failed()) { + errorResult.SuppressException(); + MOZ_DIAGNOSTIC_ASSERT(queryValue.IsEmpty()); + } + + nsAutoCString cachedValue; + cachedHeaders->Get(header, cachedValue, errorResult); + if (errorResult.Failed()) { + errorResult.SuppressException(); + MOZ_DIAGNOSTIC_ASSERT(cachedValue.IsEmpty()); + } + + if (queryValue != cachedValue) { + varyHeadersMatch = false; + bailOut = true; + break; + } + } + + if (bailOut) { + break; + } + } + + *aSuccessOut = varyHeadersMatch; + return rv; +} + +nsresult +DeleteEntries(mozIStorageConnection* aConn, + const nsTArray<EntryId>& aEntryIdList, + nsTArray<nsID>& aDeletedBodyIdListOut, + nsTArray<IdCount>& aDeletedSecurityIdListOut, + uint32_t aPos, int32_t aLen) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + if (aEntryIdList.IsEmpty()) { + return NS_OK; + } + + MOZ_DIAGNOSTIC_ASSERT(aPos < aEntryIdList.Length()); + + if (aLen < 0) { + aLen = aEntryIdList.Length() - aPos; + } + + // Sqlite limits the number of entries allowed for an IN clause, + // so split up larger operations. + if (aLen > kMaxEntriesPerStatement) { + uint32_t curPos = aPos; + int32_t remaining = aLen; + while (remaining > 0) { + int32_t max = kMaxEntriesPerStatement; + int32_t curLen = std::min(max, remaining); + nsresult rv = DeleteEntries(aConn, aEntryIdList, aDeletedBodyIdListOut, + aDeletedSecurityIdListOut, curPos, curLen); + if (NS_FAILED(rv)) { return rv; } + + curPos += curLen; + remaining -= curLen; + } + return NS_OK; + } + + nsCOMPtr<mozIStorageStatement> state; + nsAutoCString query( + "SELECT request_body_id, response_body_id, response_security_info_id " + "FROM entries WHERE id IN (" + ); + AppendListParamsToQuery(query, aEntryIdList, aPos, aLen); + query.AppendLiteral(")"); + + nsresult rv = aConn->CreateStatement(query, getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = BindListParamsToQuery(state, aEntryIdList, aPos, aLen); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + // extract 0 to 2 nsID structs per row + for (uint32_t i = 0; i < 2; ++i) { + bool isNull = false; + + rv = state->GetIsNull(i, &isNull); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!isNull) { + nsID id; + rv = ExtractId(state, i, &id); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aDeletedBodyIdListOut.AppendElement(id); + } + } + + // and then a possible third entry for the security id + bool isNull = false; + rv = state->GetIsNull(2, &isNull); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!isNull) { + int32_t securityId = -1; + rv = state->GetInt32(2, &securityId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // First try to increment the count for this ID if we're already + // seen it + bool found = false; + for (uint32_t i = 0; i < aDeletedSecurityIdListOut.Length(); ++i) { + if (aDeletedSecurityIdListOut[i].mId == securityId) { + found = true; + aDeletedSecurityIdListOut[i].mCount += 1; + break; + } + } + + // Otherwise add a new entry for this ID with a count of 1 + if (!found) { + aDeletedSecurityIdListOut.AppendElement(IdCount(securityId)); + } + } + } + + // Dependent records removed via ON DELETE CASCADE + + query = NS_LITERAL_CSTRING( + "DELETE FROM entries WHERE id IN (" + ); + AppendListParamsToQuery(query, aEntryIdList, aPos, aLen); + query.AppendLiteral(")"); + + rv = aConn->CreateStatement(query, getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = BindListParamsToQuery(state, aEntryIdList, aPos, aLen); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult +InsertSecurityInfo(mozIStorageConnection* aConn, nsICryptoHash* aCrypto, + const nsACString& aData, int32_t *aIdOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aCrypto); + MOZ_DIAGNOSTIC_ASSERT(aIdOut); + MOZ_DIAGNOSTIC_ASSERT(!aData.IsEmpty()); + + // We want to use an index to find existing security blobs, but indexing + // the full blob would be quite expensive. Instead, we index a small + // hash value. Calculate this hash as the first 8 bytes of the SHA1 of + // the full data. + nsAutoCString hash; + nsresult rv = HashCString(aCrypto, aData, hash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Next, search for an existing entry for this blob by comparing the hash + // value first and then the full data. SQLite is smart enough to use + // the index on the hash to search the table before doing the expensive + // comparison of the large data column. (This was verified with EXPLAIN.) + nsCOMPtr<mozIStorageStatement> state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + // Note that hash and data are blobs, but we can use = here since the + // columns are NOT NULL. + "SELECT id, refcount FROM security_info WHERE hash=:hash AND data=:data;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("hash"), hash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("data"), aData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // This security info blob is already in the database + if (hasMoreData) { + // get the existing security blob id to return + rv = state->GetInt32(0, aIdOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t refcount = -1; + rv = state->GetInt32(1, &refcount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // But first, update the refcount in the database. + refcount += 1; + + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "UPDATE security_info SET refcount=:refcount WHERE id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("refcount"), refcount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), *aIdOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return NS_OK; + } + + // This is a new security info blob. Create a new row in the security table + // with an initial refcount of 1. + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "INSERT INTO security_info (hash, data, refcount) VALUES (:hash, :data, 1);" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("hash"), hash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName(NS_LITERAL_CSTRING("data"), aData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT last_insert_rowid()" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->GetInt32(0, aIdOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return NS_OK; +} + +nsresult +DeleteSecurityInfo(mozIStorageConnection* aConn, int32_t aId, int32_t aCount) +{ + // First, we need to determine the current refcount for this security blob. + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT refcount FROM security_info WHERE id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t refcount = -1; + rv = state->GetInt32(0, &refcount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + MOZ_DIAGNOSTIC_ASSERT(refcount >= aCount); + + // Next, calculate the new refcount + int32_t newCount = refcount - aCount; + + // If the last reference to this security blob was removed we can + // just remove the entire row. + if (newCount == 0) { + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "DELETE FROM security_info WHERE id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return NS_OK; + } + + // Otherwise update the refcount in the table to reflect the reduced + // number of references to the security blob. + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "UPDATE security_info SET refcount=:refcount WHERE id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("refcount"), newCount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return NS_OK; +} + +nsresult +DeleteSecurityInfoList(mozIStorageConnection* aConn, + const nsTArray<IdCount>& aDeletedStorageIdList) +{ + for (uint32_t i = 0; i < aDeletedStorageIdList.Length(); ++i) { + nsresult rv = DeleteSecurityInfo(aConn, aDeletedStorageIdList[i].mId, + aDeletedStorageIdList[i].mCount); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + return NS_OK; +} + +nsresult +InsertEntry(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequest& aRequest, + const nsID* aRequestBodyId, + const CacheResponse& aResponse, + const nsID* aResponseBodyId) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + nsresult rv = NS_OK; + + nsCOMPtr<nsICryptoHash> crypto = + do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t securityId = -1; + if (!aResponse.channelInfo().securityInfo().IsEmpty()) { + rv = InsertSecurityInfo(aConn, crypto, + aResponse.channelInfo().securityInfo(), + &securityId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + nsCOMPtr<mozIStorageStatement> state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "INSERT INTO entries (" + "request_method, " + "request_url_no_query, " + "request_url_no_query_hash, " + "request_url_query, " + "request_url_query_hash, " + "request_url_fragment, " + "request_referrer, " + "request_referrer_policy, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_integrity, " + "request_body_id, " + "response_type, " + "response_status, " + "response_status_text, " + "response_headers_guard, " + "response_body_id, " + "response_security_info_id, " + "response_principal_info, " + "cache_id " + ") VALUES (" + ":request_method, " + ":request_url_no_query, " + ":request_url_no_query_hash, " + ":request_url_query, " + ":request_url_query_hash, " + ":request_url_fragment, " + ":request_referrer, " + ":request_referrer_policy, " + ":request_headers_guard, " + ":request_mode, " + ":request_credentials, " + ":request_contentpolicytype, " + ":request_cache, " + ":request_redirect, " + ":request_integrity, " + ":request_body_id, " + ":response_type, " + ":response_status, " + ":response_status_text, " + ":response_headers_guard, " + ":response_body_id, " + ":response_security_info_id, " + ":response_principal_info, " + ":cache_id " + ");" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("request_method"), + aRequest.method()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("request_url_no_query"), + aRequest.urlWithoutQuery()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsAutoCString urlWithoutQueryHash; + rv = HashCString(crypto, aRequest.urlWithoutQuery(), urlWithoutQueryHash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringAsBlobByName( + NS_LITERAL_CSTRING("request_url_no_query_hash"), urlWithoutQueryHash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("request_url_query"), + aRequest.urlQuery()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsAutoCString urlQueryHash; + rv = HashCString(crypto, aRequest.urlQuery(), urlQueryHash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + rv = state->BindUTF8StringAsBlobByName( + NS_LITERAL_CSTRING("request_url_query_hash"), urlQueryHash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("request_url_fragment"), + aRequest.urlFragment()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindStringByName(NS_LITERAL_CSTRING("request_referrer"), + aRequest.referrer()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("request_referrer_policy"), + static_cast<int32_t>(aRequest.referrerPolicy())); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("request_headers_guard"), + static_cast<int32_t>(aRequest.headersGuard())); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("request_mode"), + static_cast<int32_t>(aRequest.mode())); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("request_credentials"), + static_cast<int32_t>(aRequest.credentials())); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("request_contentpolicytype"), + static_cast<int32_t>(aRequest.contentPolicyType())); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("request_cache"), + static_cast<int32_t>(aRequest.requestCache())); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("request_redirect"), + static_cast<int32_t>(aRequest.requestRedirect())); + + rv = state->BindStringByName(NS_LITERAL_CSTRING("request_integrity"), + aRequest.integrity()); + + rv = BindId(state, NS_LITERAL_CSTRING("request_body_id"), aRequestBodyId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("response_type"), + static_cast<int32_t>(aResponse.type())); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("response_status"), + aResponse.status()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("response_status_text"), + aResponse.statusText()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("response_headers_guard"), + static_cast<int32_t>(aResponse.headersGuard())); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = BindId(state, NS_LITERAL_CSTRING("response_body_id"), aResponseBodyId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (aResponse.channelInfo().securityInfo().IsEmpty()) { + rv = state->BindNullByName(NS_LITERAL_CSTRING("response_security_info_id")); + } else { + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("response_security_info_id"), + securityId); + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsAutoCString serializedInfo; + // We only allow content serviceworkers right now. + if (aResponse.principalInfo().type() == mozilla::ipc::OptionalPrincipalInfo::TPrincipalInfo) { + const mozilla::ipc::PrincipalInfo& principalInfo = + aResponse.principalInfo().get_PrincipalInfo(); + MOZ_DIAGNOSTIC_ASSERT(principalInfo.type() == mozilla::ipc::PrincipalInfo::TContentPrincipalInfo); + const mozilla::ipc::ContentPrincipalInfo& cInfo = + principalInfo.get_ContentPrincipalInfo(); + + serializedInfo.Append(cInfo.spec()); + + nsAutoCString suffix; + cInfo.attrs().CreateSuffix(suffix); + serializedInfo.Append(suffix); + } + + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("response_principal_info"), + serializedInfo); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt64ByName(NS_LITERAL_CSTRING("cache_id"), aCacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT last_insert_rowid()" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t entryId; + rv = state->GetInt32(0, &entryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "INSERT INTO request_headers (" + "name, " + "value, " + "entry_id " + ") VALUES (:name, :value, :entry_id)" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + const nsTArray<HeadersEntry>& requestHeaders = aRequest.headers(); + for (uint32_t i = 0; i < requestHeaders.Length(); ++i) { + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("name"), + requestHeaders[i].name()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("value"), + requestHeaders[i].value()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("entry_id"), entryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "INSERT INTO response_headers (" + "name, " + "value, " + "entry_id " + ") VALUES (:name, :value, :entry_id)" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + const nsTArray<HeadersEntry>& responseHeaders = aResponse.headers(); + for (uint32_t i = 0; i < responseHeaders.Length(); ++i) { + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("name"), + responseHeaders[i].name()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("value"), + responseHeaders[i].value()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("entry_id"), entryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "INSERT INTO response_url_list (" + "url, " + "entry_id " + ") VALUES (:url, :entry_id)" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + const nsTArray<nsCString>& responseUrlList = aResponse.urlList(); + for (uint32_t i = 0; i < responseUrlList.Length(); ++i) { + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("url"), + responseUrlList[i]); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt64ByName(NS_LITERAL_CSTRING("entry_id"), entryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + return rv; +} + +nsresult +ReadResponse(mozIStorageConnection* aConn, EntryId aEntryId, + SavedResponse* aSavedResponseOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aSavedResponseOut); + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT " + "entries.response_type, " + "entries.response_status, " + "entries.response_status_text, " + "entries.response_headers_guard, " + "entries.response_body_id, " + "entries.response_principal_info, " + "security_info.data " + "FROM entries " + "LEFT OUTER JOIN security_info " + "ON entries.response_security_info_id=security_info.id " + "WHERE entries.id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aEntryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t type; + rv = state->GetInt32(0, &type); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedResponseOut->mValue.type() = static_cast<ResponseType>(type); + + int32_t status; + rv = state->GetInt32(1, &status); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedResponseOut->mValue.status() = status; + + rv = state->GetUTF8String(2, aSavedResponseOut->mValue.statusText()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t guard; + rv = state->GetInt32(3, &guard); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedResponseOut->mValue.headersGuard() = + static_cast<HeadersGuardEnum>(guard); + + bool nullBody = false; + rv = state->GetIsNull(4, &nullBody); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedResponseOut->mHasBodyId = !nullBody; + + if (aSavedResponseOut->mHasBodyId) { + rv = ExtractId(state, 4, &aSavedResponseOut->mBodyId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + nsAutoCString serializedInfo; + rv = state->GetUTF8String(5, serializedInfo); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aSavedResponseOut->mValue.principalInfo() = void_t(); + if (!serializedInfo.IsEmpty()) { + nsAutoCString specNoSuffix; + PrincipalOriginAttributes attrs; + if (!attrs.PopulateFromOrigin(serializedInfo, specNoSuffix)) { + NS_WARNING("Something went wrong parsing a serialized principal!"); + return NS_ERROR_FAILURE; + } + + aSavedResponseOut->mValue.principalInfo() = + mozilla::ipc::ContentPrincipalInfo(attrs, void_t(), specNoSuffix); + } + + rv = state->GetBlobAsUTF8String(6, aSavedResponseOut->mValue.channelInfo().securityInfo()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT " + "name, " + "value " + "FROM response_headers " + "WHERE entry_id=:entry_id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("entry_id"), aEntryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + HeadersEntry header; + + rv = state->GetUTF8String(0, header.name()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->GetUTF8String(1, header.value()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aSavedResponseOut->mValue.headers().AppendElement(header); + } + + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT " + "url " + "FROM response_url_list " + "WHERE entry_id=:entry_id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("entry_id"), aEntryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + nsCString url; + + rv = state->GetUTF8String(0, url); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aSavedResponseOut->mValue.urlList().AppendElement(url); + } + + return rv; +} + +nsresult +ReadRequest(mozIStorageConnection* aConn, EntryId aEntryId, + SavedRequest* aSavedRequestOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aSavedRequestOut); + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT " + "request_method, " + "request_url_no_query, " + "request_url_query, " + "request_url_fragment, " + "request_referrer, " + "request_referrer_policy, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_integrity, " + "request_body_id " + "FROM entries " + "WHERE id=:id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("id"), aEntryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->GetUTF8String(0, aSavedRequestOut->mValue.method()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + rv = state->GetUTF8String(1, aSavedRequestOut->mValue.urlWithoutQuery()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + rv = state->GetUTF8String(2, aSavedRequestOut->mValue.urlQuery()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + rv = state->GetUTF8String(3, aSavedRequestOut->mValue.urlFragment()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + rv = state->GetString(4, aSavedRequestOut->mValue.referrer()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t referrerPolicy; + rv = state->GetInt32(5, &referrerPolicy); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedRequestOut->mValue.referrerPolicy() = + static_cast<ReferrerPolicy>(referrerPolicy); + int32_t guard; + rv = state->GetInt32(6, &guard); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedRequestOut->mValue.headersGuard() = + static_cast<HeadersGuardEnum>(guard); + int32_t mode; + rv = state->GetInt32(7, &mode); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedRequestOut->mValue.mode() = static_cast<RequestMode>(mode); + int32_t credentials; + rv = state->GetInt32(8, &credentials); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedRequestOut->mValue.credentials() = + static_cast<RequestCredentials>(credentials); + int32_t requestContentPolicyType; + rv = state->GetInt32(9, &requestContentPolicyType); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedRequestOut->mValue.contentPolicyType() = + static_cast<nsContentPolicyType>(requestContentPolicyType); + int32_t requestCache; + rv = state->GetInt32(10, &requestCache); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedRequestOut->mValue.requestCache() = + static_cast<RequestCache>(requestCache); + int32_t requestRedirect; + rv = state->GetInt32(11, &requestRedirect); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedRequestOut->mValue.requestRedirect() = + static_cast<RequestRedirect>(requestRedirect); + rv = state->GetString(12, aSavedRequestOut->mValue.integrity()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + bool nullBody = false; + rv = state->GetIsNull(13, &nullBody); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + aSavedRequestOut->mHasBodyId = !nullBody; + if (aSavedRequestOut->mHasBodyId) { + rv = ExtractId(state, 13, &aSavedRequestOut->mBodyId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT " + "name, " + "value " + "FROM request_headers " + "WHERE entry_id=:entry_id;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindInt32ByName(NS_LITERAL_CSTRING("entry_id"), aEntryId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + HeadersEntry header; + + rv = state->GetUTF8String(0, header.name()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->GetUTF8String(1, header.value()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aSavedRequestOut->mValue.headers().AppendElement(header); + } + + return rv; +} + +void +AppendListParamsToQuery(nsACString& aQuery, + const nsTArray<EntryId>& aEntryIdList, + uint32_t aPos, int32_t aLen) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT((aPos + aLen) <= aEntryIdList.Length()); + for (int32_t i = aPos; i < aLen; ++i) { + if (i == 0) { + aQuery.AppendLiteral("?"); + } else { + aQuery.AppendLiteral(",?"); + } + } +} + +nsresult +BindListParamsToQuery(mozIStorageStatement* aState, + const nsTArray<EntryId>& aEntryIdList, + uint32_t aPos, int32_t aLen) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT((aPos + aLen) <= aEntryIdList.Length()); + for (int32_t i = aPos; i < aLen; ++i) { + nsresult rv = aState->BindInt32ByIndex(i, aEntryIdList[i]); + NS_ENSURE_SUCCESS(rv, rv); + } + return NS_OK; +} + +nsresult +BindId(mozIStorageStatement* aState, const nsACString& aName, const nsID* aId) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aState); + nsresult rv; + + if (!aId) { + rv = aState->BindNullByName(aName); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + return rv; + } + + char idBuf[NSID_LENGTH]; + aId->ToProvidedString(idBuf); + rv = aState->BindUTF8StringByName(aName, nsDependentCString(idBuf)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult +ExtractId(mozIStorageStatement* aState, uint32_t aPos, nsID* aIdOut) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aState); + MOZ_DIAGNOSTIC_ASSERT(aIdOut); + + nsAutoCString idString; + nsresult rv = aState->GetUTF8String(aPos, idString); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool success = aIdOut->Parse(idString.get()); + if (NS_WARN_IF(!success)) { return NS_ERROR_UNEXPECTED; } + + return rv; +} + +nsresult +CreateAndBindKeyStatement(mozIStorageConnection* aConn, + const char* aQueryFormat, + const nsAString& aKey, + mozIStorageStatement** aStateOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(aQueryFormat); + MOZ_DIAGNOSTIC_ASSERT(aStateOut); + + // The key is stored as a blob to avoid encoding issues. An empty string + // is mapped to NULL for blobs. Normally we would just write the query + // as "key IS :key" to do the proper NULL checking, but that prevents + // sqlite from using the key index. Therefore use "IS NULL" explicitly + // if the key is empty, otherwise use "=:key" so that sqlite uses the + // index. + const char* constraint = nullptr; + if (aKey.IsEmpty()) { + constraint = "key IS NULL"; + } else { + constraint = "key=:key"; + } + + nsPrintfCString query(aQueryFormat, constraint); + + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(query, getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!aKey.IsEmpty()) { + rv = state->BindStringAsBlobByName(NS_LITERAL_CSTRING("key"), aKey); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + state.forget(aStateOut); + + return rv; +} + +nsresult +HashCString(nsICryptoHash* aCrypto, const nsACString& aIn, nsACString& aOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aCrypto); + + nsresult rv = aCrypto->Init(nsICryptoHash::SHA1); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aCrypto->Update(reinterpret_cast<const uint8_t*>(aIn.BeginReading()), + aIn.Length()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsAutoCString fullHash; + rv = aCrypto->Finish(false /* based64 result */, fullHash); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aOut = Substring(fullHash, 0, 8); + return rv; +} + +} // namespace + +nsresult +IncrementalVacuum(mozIStorageConnection* aConn) +{ + // Determine how much free space is in the database. + nsCOMPtr<mozIStorageStatement> state; + nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "PRAGMA freelist_count;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + int32_t freePages = 0; + rv = state->GetInt32(0, &freePages); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // We have a relatively small page size, so we want to be careful to avoid + // fragmentation. We already use a growth incremental which will cause + // sqlite to allocate and release multiple pages at the same time. We can + // further reduce fragmentation by making our allocated chunks a bit + // "sticky". This is done by creating some hysteresis where we allocate + // pages/chunks as soon as we need them, but we only release pages/chunks + // when we have a large amount of free space. This helps with the case + // where a page is adding and remove resources causing it to dip back and + // forth across a chunk boundary. + // + // So only proceed with releasing pages if we have more than our constant + // threshold. + if (freePages <= kMaxFreePages) { + return NS_OK; + } + + // Release the excess pages back to the sqlite VFS. This may also release + // chunks of multiple pages back to the OS. + int32_t pagesToRelease = freePages - kMaxFreePages; + + rv = aConn->ExecuteSimpleSQL(nsPrintfCString( + "PRAGMA incremental_vacuum(%d);", pagesToRelease + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Verify that our incremental vacuum actually did something +#ifdef DEBUG + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "PRAGMA freelist_count;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + freePages = 0; + rv = state->GetInt32(0, &freePages); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + MOZ_ASSERT(freePages <= kMaxFreePages); +#endif + + return NS_OK; +} + +namespace { + +#ifdef DEBUG +struct Expect +{ + // Expect exact SQL + Expect(const char* aName, const char* aType, const char* aSql) + : mName(aName) + , mType(aType) + , mSql(aSql) + , mIgnoreSql(false) + { } + + // Ignore SQL + Expect(const char* aName, const char* aType) + : mName(aName) + , mType(aType) + , mIgnoreSql(true) + { } + + const nsCString mName; + const nsCString mType; + const nsCString mSql; + const bool mIgnoreSql; +}; +#endif + +nsresult +Validate(mozIStorageConnection* aConn) +{ + int32_t schemaVersion; + nsresult rv = aConn->GetSchemaVersion(&schemaVersion); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (NS_WARN_IF(schemaVersion != kLatestSchemaVersion)) { + return NS_ERROR_FAILURE; + } + +#ifdef DEBUG + // This is the schema we expect the database at the latest version to + // contain. Update this list if you add a new table or index. + Expect expect[] = { + Expect("caches", "table", kTableCaches), + Expect("sqlite_sequence", "table"), // auto-gen by sqlite + Expect("security_info", "table", kTableSecurityInfo), + Expect("security_info_hash_index", "index", kIndexSecurityInfoHash), + Expect("entries", "table", kTableEntries), + Expect("entries_request_match_index", "index", kIndexEntriesRequest), + Expect("request_headers", "table", kTableRequestHeaders), + Expect("response_headers", "table", kTableResponseHeaders), + Expect("response_headers_name_index", "index", kIndexResponseHeadersName), + Expect("response_url_list", "table", kTableResponseUrlList), + Expect("storage", "table", kTableStorage), + Expect("sqlite_autoindex_storage_1", "index"), // auto-gen by sqlite + }; + const uint32_t expectLength = sizeof(expect) / sizeof(Expect); + + // Read the schema from the sqlite_master table and compare. + nsCOMPtr<mozIStorageStatement> state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "SELECT name, type, sql FROM sqlite_master;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) { + nsAutoCString name; + rv = state->GetUTF8String(0, name); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsAutoCString type; + rv = state->GetUTF8String(1, type); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsAutoCString sql; + rv = state->GetUTF8String(2, sql); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool foundMatch = false; + for (uint32_t i = 0; i < expectLength; ++i) { + if (name == expect[i].mName) { + if (type != expect[i].mType) { + NS_WARNING(nsPrintfCString("Unexpected type for Cache schema entry %s", + name.get()).get()); + return NS_ERROR_FAILURE; + } + + if (!expect[i].mIgnoreSql && sql != expect[i].mSql) { + NS_WARNING(nsPrintfCString("Unexpected SQL for Cache schema entry %s", + name.get()).get()); + return NS_ERROR_FAILURE; + } + + foundMatch = true; + break; + } + } + + if (NS_WARN_IF(!foundMatch)) { + NS_WARNING(nsPrintfCString("Unexpected schema entry %s in Cache database", + name.get()).get()); + return NS_ERROR_FAILURE; + } + } +#endif + + return rv; +} + +// ----- +// Schema migration code +// ----- + +typedef nsresult (*MigrationFunc)(mozIStorageConnection*, bool&); +struct Migration +{ + constexpr Migration(int32_t aFromVersion, MigrationFunc aFunc) + : mFromVersion(aFromVersion) + , mFunc(aFunc) + { } + int32_t mFromVersion; + MigrationFunc mFunc; +}; + +// Declare migration functions here. Each function should upgrade +// the version by a single increment. Don't skip versions. +nsresult MigrateFrom15To16(mozIStorageConnection* aConn, bool& aRewriteSchema); +nsresult MigrateFrom16To17(mozIStorageConnection* aConn, bool& aRewriteSchema); +nsresult MigrateFrom17To18(mozIStorageConnection* aConn, bool& aRewriteSchema); +nsresult MigrateFrom18To19(mozIStorageConnection* aConn, bool& aRewriteSchema); +nsresult MigrateFrom19To20(mozIStorageConnection* aConn, bool& aRewriteSchema); +nsresult MigrateFrom20To21(mozIStorageConnection* aConn, bool& aRewriteSchema); +nsresult MigrateFrom21To22(mozIStorageConnection* aConn, bool& aRewriteSchema); +nsresult MigrateFrom22To23(mozIStorageConnection* aConn, bool& aRewriteSchema); +nsresult MigrateFrom23To24(mozIStorageConnection* aConn, bool& aRewriteSchema); +// Configure migration functions to run for the given starting version. +Migration sMigrationList[] = { + Migration(15, MigrateFrom15To16), + Migration(16, MigrateFrom16To17), + Migration(17, MigrateFrom17To18), + Migration(18, MigrateFrom18To19), + Migration(19, MigrateFrom19To20), + Migration(20, MigrateFrom20To21), + Migration(21, MigrateFrom21To22), + Migration(22, MigrateFrom22To23), + Migration(23, MigrateFrom23To24), +}; +uint32_t sMigrationListLength = sizeof(sMigrationList) / sizeof(Migration); +nsresult +RewriteEntriesSchema(mozIStorageConnection* aConn) +{ + nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "PRAGMA writable_schema = ON" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<mozIStorageStatement> state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "UPDATE sqlite_master SET sql=:sql WHERE name='entries'" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("sql"), + nsDependentCString(kTableEntries)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = state->Execute(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "PRAGMA writable_schema = OFF" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult +Migrate(mozIStorageConnection* aConn) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + int32_t currentVersion = 0; + nsresult rv = aConn->GetSchemaVersion(¤tVersion); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool rewriteSchema = false; + + while (currentVersion < kLatestSchemaVersion) { + // Wiping old databases is handled in DBAction because it requires + // making a whole new mozIStorageConnection. Make sure we don't + // accidentally get here for one of those old databases. + MOZ_DIAGNOSTIC_ASSERT(currentVersion >= kFirstShippedSchemaVersion); + + for (uint32_t i = 0; i < sMigrationListLength; ++i) { + if (sMigrationList[i].mFromVersion == currentVersion) { + bool shouldRewrite = false; + rv = sMigrationList[i].mFunc(aConn, shouldRewrite); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (shouldRewrite) { + rewriteSchema = true; + } + break; + } + } + +#if defined(DEBUG) || !defined(RELEASE_OR_BETA) + int32_t lastVersion = currentVersion; +#endif + rv = aConn->GetSchemaVersion(¤tVersion); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + MOZ_DIAGNOSTIC_ASSERT(currentVersion > lastVersion); + } + + MOZ_DIAGNOSTIC_ASSERT(currentVersion == kLatestSchemaVersion); + + if (rewriteSchema) { + // Now overwrite the master SQL for the entries table to remove the column + // default value. This is also necessary for our Validate() method to + // pass on this database. + rv = RewriteEntriesSchema(aConn); + } + + return rv; +} + +nsresult MigrateFrom15To16(mozIStorageConnection* aConn, bool& aRewriteSchema) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // Add the request_redirect column with a default value of "follow". Note, + // we only use a default value here because its required by ALTER TABLE and + // we need to apply the default "follow" to existing records in the table. + // We don't actually want to keep the default in the schema for future + // INSERTs. + nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "ALTER TABLE entries " + "ADD COLUMN request_redirect INTEGER NOT NULL DEFAULT 0" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->SetSchemaVersion(16); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aRewriteSchema = true; + + return rv; +} + +nsresult +MigrateFrom16To17(mozIStorageConnection* aConn, bool& aRewriteSchema) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // This migration path removes the response_redirected and + // response_redirected_url columns from the entries table. sqlite doesn't + // support removing a column from a table using ALTER TABLE, so we need to + // create a new table without those columns, fill it up with the existing + // data, and then drop the original table and rename the new one to the old + // one. + + // Create a new_entries table with the new fields as of version 17. + nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "CREATE TABLE new_entries (" + "id INTEGER NOT NULL PRIMARY KEY, " + "request_method TEXT NOT NULL, " + "request_url_no_query TEXT NOT NULL, " + "request_url_no_query_hash BLOB NOT NULL, " + "request_url_query TEXT NOT NULL, " + "request_url_query_hash BLOB NOT NULL, " + "request_referrer TEXT NOT NULL, " + "request_headers_guard INTEGER NOT NULL, " + "request_mode INTEGER NOT NULL, " + "request_credentials INTEGER NOT NULL, " + "request_contentpolicytype INTEGER NOT NULL, " + "request_cache INTEGER NOT NULL, " + "request_body_id TEXT NULL, " + "response_type INTEGER NOT NULL, " + "response_url TEXT NOT NULL, " + "response_status INTEGER NOT NULL, " + "response_status_text TEXT NOT NULL, " + "response_headers_guard INTEGER NOT NULL, " + "response_body_id TEXT NULL, " + "response_security_info_id INTEGER NULL REFERENCES security_info(id), " + "response_principal_info TEXT NOT NULL, " + "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, " + "request_redirect INTEGER NOT NULL" + ")" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Copy all of the data to the newly created table. + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "INSERT INTO new_entries (" + "id, " + "request_method, " + "request_url_no_query, " + "request_url_no_query_hash, " + "request_url_query, " + "request_url_query_hash, " + "request_referrer, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_body_id, " + "response_type, " + "response_url, " + "response_status, " + "response_status_text, " + "response_headers_guard, " + "response_body_id, " + "response_security_info_id, " + "response_principal_info, " + "cache_id " + ") SELECT " + "id, " + "request_method, " + "request_url_no_query, " + "request_url_no_query_hash, " + "request_url_query, " + "request_url_query_hash, " + "request_referrer, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_body_id, " + "response_type, " + "response_url, " + "response_status, " + "response_status_text, " + "response_headers_guard, " + "response_body_id, " + "response_security_info_id, " + "response_principal_info, " + "cache_id " + "FROM entries;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Remove the old table. + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "DROP TABLE entries;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Rename new_entries to entries. + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "ALTER TABLE new_entries RENAME to entries;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Now, recreate our indices. + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kIndexEntriesRequest)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Revalidate the foreign key constraints, and ensure that there are no + // violations. + nsCOMPtr<mozIStorageStatement> state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "PRAGMA foreign_key_check;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (NS_WARN_IF(hasMoreData)) { return NS_ERROR_FAILURE; } + + rv = aConn->SetSchemaVersion(17); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult +MigrateFrom17To18(mozIStorageConnection* aConn, bool& aRewriteSchema) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // This migration is needed in order to remove "only-if-cached" RequestCache + // values from the database. This enum value was removed from the spec in + // https://github.com/whatwg/fetch/issues/39 but we unfortunately happily + // accepted this value in the Request constructor. + // + // There is no good value to upgrade this to, so we just stick to "default". + + static_assert(int(RequestCache::Default) == 0, + "This is where the 0 below comes from!"); + nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "UPDATE entries SET request_cache = 0 " + "WHERE request_cache = 5;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->SetSchemaVersion(18); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult +MigrateFrom18To19(mozIStorageConnection* aConn, bool& aRewriteSchema) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // This migration is needed in order to update the RequestMode values for + // Request objects corresponding to a navigation content policy type to + // "navigate". + + static_assert(int(nsIContentPolicy::TYPE_DOCUMENT) == 6 && + int(nsIContentPolicy::TYPE_SUBDOCUMENT) == 7 && + int(nsIContentPolicy::TYPE_INTERNAL_FRAME) == 28 && + int(nsIContentPolicy::TYPE_INTERNAL_IFRAME) == 29 && + int(nsIContentPolicy::TYPE_REFRESH) == 8 && + int(RequestMode::Navigate) == 3, + "This is where the numbers below come from!"); + nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "UPDATE entries SET request_mode = 3 " + "WHERE request_contentpolicytype IN (6, 7, 28, 29, 8);" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->SetSchemaVersion(19); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +nsresult MigrateFrom19To20(mozIStorageConnection* aConn, bool& aRewriteSchema) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // Add the request_referrer_policy column with a default value of + // "no-referrer-when-downgrade". Note, we only use a default value here + // because its required by ALTER TABLE and we need to apply the default + // "no-referrer-when-downgrade" to existing records in the table. We don't + // actually want to keep the default in the schema for future INSERTs. + nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "ALTER TABLE entries " + "ADD COLUMN request_referrer_policy INTEGER NOT NULL DEFAULT 2" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->SetSchemaVersion(20); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aRewriteSchema = true; + + return rv; +} + +nsresult MigrateFrom20To21(mozIStorageConnection* aConn, bool& aRewriteSchema) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // This migration creates response_url_list table to store response_url and + // removes the response_url column from the entries table. + // sqlite doesn't support removing a column from a table using ALTER TABLE, + // so we need to create a new table without those columns, fill it up with the + // existing data, and then drop the original table and rename the new one to + // the old one. + + // Create a new_entries table with the new fields as of version 21. + nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "CREATE TABLE new_entries (" + "id INTEGER NOT NULL PRIMARY KEY, " + "request_method TEXT NOT NULL, " + "request_url_no_query TEXT NOT NULL, " + "request_url_no_query_hash BLOB NOT NULL, " + "request_url_query TEXT NOT NULL, " + "request_url_query_hash BLOB NOT NULL, " + "request_referrer TEXT NOT NULL, " + "request_headers_guard INTEGER NOT NULL, " + "request_mode INTEGER NOT NULL, " + "request_credentials INTEGER NOT NULL, " + "request_contentpolicytype INTEGER NOT NULL, " + "request_cache INTEGER NOT NULL, " + "request_body_id TEXT NULL, " + "response_type INTEGER NOT NULL, " + "response_status INTEGER NOT NULL, " + "response_status_text TEXT NOT NULL, " + "response_headers_guard INTEGER NOT NULL, " + "response_body_id TEXT NULL, " + "response_security_info_id INTEGER NULL REFERENCES security_info(id), " + "response_principal_info TEXT NOT NULL, " + "cache_id INTEGER NOT NULL REFERENCES caches(id) ON DELETE CASCADE, " + "request_redirect INTEGER NOT NULL, " + "request_referrer_policy INTEGER NOT NULL" + ")" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Create a response_url_list table with the new fields as of version 21. + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "CREATE TABLE response_url_list (" + "url TEXT NOT NULL, " + "entry_id INTEGER NOT NULL REFERENCES entries(id) ON DELETE CASCADE" + ")" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Copy all of the data to the newly created entries table. + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "INSERT INTO new_entries (" + "id, " + "request_method, " + "request_url_no_query, " + "request_url_no_query_hash, " + "request_url_query, " + "request_url_query_hash, " + "request_referrer, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_referrer_policy, " + "request_body_id, " + "response_type, " + "response_status, " + "response_status_text, " + "response_headers_guard, " + "response_body_id, " + "response_security_info_id, " + "response_principal_info, " + "cache_id " + ") SELECT " + "id, " + "request_method, " + "request_url_no_query, " + "request_url_no_query_hash, " + "request_url_query, " + "request_url_query_hash, " + "request_referrer, " + "request_headers_guard, " + "request_mode, " + "request_credentials, " + "request_contentpolicytype, " + "request_cache, " + "request_redirect, " + "request_referrer_policy, " + "request_body_id, " + "response_type, " + "response_status, " + "response_status_text, " + "response_headers_guard, " + "response_body_id, " + "response_security_info_id, " + "response_principal_info, " + "cache_id " + "FROM entries;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Copy reponse_url to the newly created response_url_list table. + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "INSERT INTO response_url_list (" + "url, " + "entry_id " + ") SELECT " + "response_url, " + "id " + "FROM entries;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Remove the old table. + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "DROP TABLE entries;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Rename new_entries to entries. + rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "ALTER TABLE new_entries RENAME to entries;" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Now, recreate our indices. + rv = aConn->ExecuteSimpleSQL(nsDependentCString(kIndexEntriesRequest)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Revalidate the foreign key constraints, and ensure that there are no + // violations. + nsCOMPtr<mozIStorageStatement> state; + rv = aConn->CreateStatement(NS_LITERAL_CSTRING( + "PRAGMA foreign_key_check;" + ), getter_AddRefs(state)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMoreData = false; + rv = state->ExecuteStep(&hasMoreData); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (NS_WARN_IF(hasMoreData)) { return NS_ERROR_FAILURE; } + + rv = aConn->SetSchemaVersion(21); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aRewriteSchema = true; + + return rv; +} + +nsresult MigrateFrom21To22(mozIStorageConnection* aConn, bool& aRewriteSchema) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // Add the request_integrity column. + nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "ALTER TABLE entries " + "ADD COLUMN request_integrity TEXT NULL" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->SetSchemaVersion(22); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aRewriteSchema = true; + + return rv; +} + +nsresult MigrateFrom22To23(mozIStorageConnection* aConn, bool& aRewriteSchema) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // The only change between 22 and 23 was a different snappy compression + // format, but it's backwards-compatible. + nsresult rv = aConn->SetSchemaVersion(23); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + return rv; +} +nsresult MigrateFrom23To24(mozIStorageConnection* aConn, bool& aRewriteSchema) +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_DIAGNOSTIC_ASSERT(aConn); + + // Add the request_url_fragment column. + nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "ALTER TABLE entries " + "ADD COLUMN request_url_fragment TEXT NOT NULL DEFAULT ''" + )); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aConn->SetSchemaVersion(24); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + aRewriteSchema = true; + + return rv; +} + +} // anonymous namespace +} // namespace db +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/DBSchema.h b/dom/cache/DBSchema.h new file mode 100644 index 0000000000..cc23f77434 --- /dev/null +++ b/dom/cache/DBSchema.h @@ -0,0 +1,129 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_DBSchema_h +#define mozilla_dom_cache_DBSchema_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/cache/Types.h" +#include "nsError.h" +#include "nsString.h" +#include "nsTArrayForwardDeclare.h" + +class mozIStorageConnection; +struct nsID; + +namespace mozilla { +namespace dom { +namespace cache { + +class CacheQueryParams; +class CacheRequest; +class CacheRequestOrVoid; +class CacheResponse; +struct SavedRequest; +struct SavedResponse; + +namespace db { + +// Note, this cannot be executed within a transaction. +nsresult +CreateOrMigrateSchema(mozIStorageConnection* aConn); + +// Note, this cannot be executed within a transaction. +nsresult +InitializeConnection(mozIStorageConnection* aConn); + +nsresult +CreateCacheId(mozIStorageConnection* aConn, CacheId* aCacheIdOut); + +nsresult +DeleteCacheId(mozIStorageConnection* aConn, CacheId aCacheId, + nsTArray<nsID>& aDeletedBodyIdListOut); + +// TODO: Consider removing unused IsCacheOrphaned after writing cleanup code. (bug 1110446) +nsresult +IsCacheOrphaned(mozIStorageConnection* aConn, CacheId aCacheId, + bool* aOrphanedOut); + +nsresult +FindOrphanedCacheIds(mozIStorageConnection* aConn, + nsTArray<CacheId>& aOrphanedListOut); + +nsresult +GetKnownBodyIds(mozIStorageConnection* aConn, nsTArray<nsID>& aBodyIdListOut); + +nsresult +CacheMatch(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequest& aRequest, const CacheQueryParams& aParams, + bool* aFoundResponseOut, SavedResponse* aSavedResponseOut); + +nsresult +CacheMatchAll(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequestOrVoid& aRequestOrVoid, + const CacheQueryParams& aParams, + nsTArray<SavedResponse>& aSavedResponsesOut); + +nsresult +CachePut(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequest& aRequest, + const nsID* aRequestBodyId, + const CacheResponse& aResponse, + const nsID* aResponseBodyId, + nsTArray<nsID>& aDeletedBodyIdListOut); + +nsresult +CacheDelete(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequest& aRequest, + const CacheQueryParams& aParams, + nsTArray<nsID>& aDeletedBodyIdListOut, + bool* aSuccessOut); + +nsresult +CacheKeys(mozIStorageConnection* aConn, CacheId aCacheId, + const CacheRequestOrVoid& aRequestOrVoid, + const CacheQueryParams& aParams, + nsTArray<SavedRequest>& aSavedRequestsOut); + +nsresult +StorageMatch(mozIStorageConnection* aConn, + Namespace aNamespace, + const CacheRequest& aRequest, + const CacheQueryParams& aParams, + bool* aFoundResponseOut, + SavedResponse* aSavedResponseOut); + +nsresult +StorageGetCacheId(mozIStorageConnection* aConn, Namespace aNamespace, + const nsAString& aKey, bool* aFoundCacheOut, + CacheId* aCacheIdOut); + +nsresult +StoragePutCache(mozIStorageConnection* aConn, Namespace aNamespace, + const nsAString& aKey, CacheId aCacheId); + +nsresult +StorageForgetCache(mozIStorageConnection* aConn, Namespace aNamespace, + const nsAString& aKey); + +nsresult +StorageGetKeys(mozIStorageConnection* aConn, Namespace aNamespace, + nsTArray<nsString>& aKeysOut); + +// Note, this works best when its NOT executed within a transaction. +nsresult +IncrementalVacuum(mozIStorageConnection* aConn); + +// We will wipe out databases with a schema versions less than this. Newer +// versions will be migrated on open to the latest schema version. +extern const int32_t kFirstShippedSchemaVersion; + +} // namespace db +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_DBSchema_h diff --git a/dom/cache/FileUtils.cpp b/dom/cache/FileUtils.cpp new file mode 100644 index 0000000000..dce98ac1c4 --- /dev/null +++ b/dom/cache/FileUtils.cpp @@ -0,0 +1,501 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/FileUtils.h" + +#include "mozilla/dom/quota/FileStreams.h" +#include "mozilla/SnappyCompressOutputStream.h" +#include "mozilla/Unused.h" +#include "nsIFile.h" +#include "nsIUUIDGenerator.h" +#include "nsNetCID.h" +#include "nsISimpleEnumerator.h" +#include "nsServiceManagerUtils.h" +#include "nsString.h" +#include "nsThreadUtils.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::dom::quota::FileInputStream; +using mozilla::dom::quota::FileOutputStream; +using mozilla::dom::quota::PERSISTENCE_TYPE_DEFAULT; + +namespace { + +enum BodyFileType +{ + BODY_FILE_FINAL, + BODY_FILE_TMP +}; + +nsresult +BodyIdToFile(nsIFile* aBaseDir, const nsID& aId, BodyFileType aType, + nsIFile** aBodyFileOut); + +} // namespace + +// static +nsresult +BodyCreateDir(nsIFile* aBaseDir) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + nsCOMPtr<nsIFile> aBodyDir; + nsresult rv = aBaseDir->Clone(getter_AddRefs(aBodyDir)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aBodyDir->Append(NS_LITERAL_STRING("morgue")); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aBodyDir->Create(nsIFile::DIRECTORY_TYPE, 0755); + if (rv == NS_ERROR_FILE_ALREADY_EXISTS) { + return NS_OK; + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +// static +nsresult +BodyDeleteDir(nsIFile* aBaseDir) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + nsCOMPtr<nsIFile> aBodyDir; + nsresult rv = aBaseDir->Clone(getter_AddRefs(aBodyDir)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aBodyDir->Append(NS_LITERAL_STRING("morgue")); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = aBodyDir->Remove(/* recursive = */ true); + if (rv == NS_ERROR_FILE_NOT_FOUND || + rv == NS_ERROR_FILE_TARGET_DOES_NOT_EXIST) { + rv = NS_OK; + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +// static +nsresult +BodyGetCacheDir(nsIFile* aBaseDir, const nsID& aId, nsIFile** aCacheDirOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aCacheDirOut); + + *aCacheDirOut = nullptr; + + nsresult rv = aBaseDir->Clone(aCacheDirOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + MOZ_DIAGNOSTIC_ASSERT(*aCacheDirOut); + + rv = (*aCacheDirOut)->Append(NS_LITERAL_STRING("morgue")); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Some file systems have poor performance when there are too many files + // in a single directory. Mitigate this issue by spreading the body + // files out into sub-directories. We use the last byte of the ID for + // the name of the sub-directory. + nsAutoString subDirName; + subDirName.AppendInt(aId.m3[7]); + rv = (*aCacheDirOut)->Append(subDirName); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = (*aCacheDirOut)->Create(nsIFile::DIRECTORY_TYPE, 0755); + if (rv == NS_ERROR_FILE_ALREADY_EXISTS) { + return NS_OK; + } + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +// static +nsresult +BodyStartWriteStream(const QuotaInfo& aQuotaInfo, + nsIFile* aBaseDir, nsIInputStream* aSource, + void* aClosure, + nsAsyncCopyCallbackFun aCallback, nsID* aIdOut, + nsISupports** aCopyContextOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aSource); + MOZ_DIAGNOSTIC_ASSERT(aClosure); + MOZ_DIAGNOSTIC_ASSERT(aCallback); + MOZ_DIAGNOSTIC_ASSERT(aIdOut); + MOZ_DIAGNOSTIC_ASSERT(aCopyContextOut); + + nsresult rv; + nsCOMPtr<nsIUUIDGenerator> idGen = + do_GetService("@mozilla.org/uuid-generator;1", &rv); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = idGen->GenerateUUIDInPlace(aIdOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsIFile> finalFile; + rv = BodyIdToFile(aBaseDir, *aIdOut, BODY_FILE_FINAL, + getter_AddRefs(finalFile)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool exists; + rv = finalFile->Exists(&exists); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (NS_WARN_IF(exists)) { return NS_ERROR_FILE_ALREADY_EXISTS; } + + nsCOMPtr<nsIFile> tmpFile; + rv = BodyIdToFile(aBaseDir, *aIdOut, BODY_FILE_TMP, getter_AddRefs(tmpFile)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = tmpFile->Exists(&exists); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (NS_WARN_IF(exists)) { return NS_ERROR_FILE_ALREADY_EXISTS; } + + nsCOMPtr<nsIOutputStream> fileStream = + FileOutputStream::Create(PERSISTENCE_TYPE_DEFAULT, aQuotaInfo.mGroup, + aQuotaInfo.mOrigin, tmpFile); + if (NS_WARN_IF(!fileStream)) { return NS_ERROR_UNEXPECTED; } + + RefPtr<SnappyCompressOutputStream> compressed = + new SnappyCompressOutputStream(fileStream); + + nsCOMPtr<nsIEventTarget> target = + do_GetService(NS_STREAMTRANSPORTSERVICE_CONTRACTID); + + rv = NS_AsyncCopy(aSource, compressed, target, NS_ASYNCCOPY_VIA_WRITESEGMENTS, + compressed->BlockSize(), aCallback, aClosure, + true, true, // close streams + aCopyContextOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +// static +void +BodyCancelWrite(nsIFile* aBaseDir, nsISupports* aCopyContext) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aCopyContext); + + nsresult rv = NS_CancelAsyncCopy(aCopyContext, NS_ERROR_ABORT); + Unused << NS_WARN_IF(NS_FAILED(rv)); + + // The partially written file must be cleaned up after the async copy + // makes its callback. +} + +// static +nsresult +BodyFinalizeWrite(nsIFile* aBaseDir, const nsID& aId) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + nsCOMPtr<nsIFile> tmpFile; + nsresult rv = BodyIdToFile(aBaseDir, aId, BODY_FILE_TMP, getter_AddRefs(tmpFile)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsIFile> finalFile; + rv = BodyIdToFile(aBaseDir, aId, BODY_FILE_FINAL, getter_AddRefs(finalFile)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsAutoString finalFileName; + rv = finalFile->GetLeafName(finalFileName); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = tmpFile->RenameTo(nullptr, finalFileName); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +// static +nsresult +BodyOpen(const QuotaInfo& aQuotaInfo, nsIFile* aBaseDir, const nsID& aId, + nsIInputStream** aStreamOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aStreamOut); + + nsCOMPtr<nsIFile> finalFile; + nsresult rv = BodyIdToFile(aBaseDir, aId, BODY_FILE_FINAL, + getter_AddRefs(finalFile)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool exists; + rv = finalFile->Exists(&exists); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (NS_WARN_IF(!exists)) { return NS_ERROR_FILE_NOT_FOUND; } + + nsCOMPtr<nsIInputStream> fileStream = + FileInputStream::Create(PERSISTENCE_TYPE_DEFAULT, aQuotaInfo.mGroup, + aQuotaInfo.mOrigin, finalFile); + if (NS_WARN_IF(!fileStream)) { return NS_ERROR_UNEXPECTED; } + + fileStream.forget(aStreamOut); + + return rv; +} + +// static +nsresult +BodyDeleteFiles(nsIFile* aBaseDir, const nsTArray<nsID>& aIdList) +{ + nsresult rv = NS_OK; + + for (uint32_t i = 0; i < aIdList.Length(); ++i) { + nsCOMPtr<nsIFile> tmpFile; + rv = BodyIdToFile(aBaseDir, aIdList[i], BODY_FILE_TMP, + getter_AddRefs(tmpFile)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = tmpFile->Remove(false /* recursive */); + if (rv == NS_ERROR_FILE_NOT_FOUND || + rv == NS_ERROR_FILE_TARGET_DOES_NOT_EXIST) { + rv = NS_OK; + } + + // Only treat file deletion as a hard failure in DEBUG builds. Users + // can unfortunately hit this on windows if anti-virus is scanning files, + // etc. + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + nsCOMPtr<nsIFile> finalFile; + rv = BodyIdToFile(aBaseDir, aIdList[i], BODY_FILE_FINAL, + getter_AddRefs(finalFile)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = finalFile->Remove(false /* recursive */); + if (rv == NS_ERROR_FILE_NOT_FOUND || + rv == NS_ERROR_FILE_TARGET_DOES_NOT_EXIST) { + rv = NS_OK; + } + + // Again, only treat removal as hard failure in debug build. + MOZ_ASSERT(NS_SUCCEEDED(rv)); + } + + return NS_OK; +} + +namespace { + +nsresult +BodyIdToFile(nsIFile* aBaseDir, const nsID& aId, BodyFileType aType, + nsIFile** aBodyFileOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + MOZ_DIAGNOSTIC_ASSERT(aBodyFileOut); + + *aBodyFileOut = nullptr; + + nsresult rv = BodyGetCacheDir(aBaseDir, aId, aBodyFileOut); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + MOZ_DIAGNOSTIC_ASSERT(*aBodyFileOut); + + char idString[NSID_LENGTH]; + aId.ToProvidedString(idString); + + NS_ConvertASCIItoUTF16 fileName(idString); + + if (aType == BODY_FILE_FINAL) { + fileName.AppendLiteral(".final"); + } else { + fileName.AppendLiteral(".tmp"); + } + + rv = (*aBodyFileOut)->Append(fileName); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; +} + +} // namespace + +nsresult +BodyDeleteOrphanedFiles(nsIFile* aBaseDir, nsTArray<nsID>& aKnownBodyIdList) +{ + MOZ_DIAGNOSTIC_ASSERT(aBaseDir); + + // body files are stored in a directory structure like: + // + // /morgue/01/{01fdddb2-884d-4c3d-95ba-0c8062f6c325}.final + // /morgue/02/{02fdddb2-884d-4c3d-95ba-0c8062f6c325}.tmp + + nsCOMPtr<nsIFile> dir; + nsresult rv = aBaseDir->Clone(getter_AddRefs(dir)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Add the root morgue directory + rv = dir->Append(NS_LITERAL_STRING("morgue")); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsISimpleEnumerator> entries; + rv = dir->GetDirectoryEntries(getter_AddRefs(entries)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Iterate over all the intermediate morgue subdirs + bool hasMore = false; + while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) { + nsCOMPtr<nsISupports> entry; + rv = entries->GetNext(getter_AddRefs(entry)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsIFile> subdir = do_QueryInterface(entry); + + bool isDir = false; + rv = subdir->IsDirectory(&isDir); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // If a file got in here somehow, try to remove it and move on + if (NS_WARN_IF(!isDir)) { + rv = subdir->Remove(false /* recursive */); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + continue; + } + + nsCOMPtr<nsISimpleEnumerator> subEntries; + rv = subdir->GetDirectoryEntries(getter_AddRefs(subEntries)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Now iterate over all the files in the subdir + bool subHasMore = false; + while(NS_SUCCEEDED(rv = subEntries->HasMoreElements(&subHasMore)) && + subHasMore) { + nsCOMPtr<nsISupports> subEntry; + rv = subEntries->GetNext(getter_AddRefs(subEntry)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsIFile> file = do_QueryInterface(subEntry); + + nsAutoCString leafName; + rv = file->GetNativeLeafName(leafName); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // Delete all tmp files regardless of known bodies. These are + // all considered orphans. + if (StringEndsWith(leafName, NS_LITERAL_CSTRING(".tmp"))) { + // remove recursively in case its somehow a directory + rv = file->Remove(true /* recursive */); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + continue; + } + + nsCString suffix(NS_LITERAL_CSTRING(".final")); + + // Otherwise, it must be a .final file. If its not, then just + // skip it. + if (NS_WARN_IF(!StringEndsWith(leafName, suffix) || + leafName.Length() != NSID_LENGTH - 1 + suffix.Length())) { + continue; + } + + // Finally, parse the uuid out of the name. If its fails to parse, + // the ignore the file. + nsID id; + if (NS_WARN_IF(!id.Parse(leafName.BeginReading()))) { + continue; + } + + if (!aKnownBodyIdList.Contains(id)) { + // remove recursively in case its somehow a directory + rv = file->Remove(true /* recursive */); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + } + } + } + + return rv; +} + +namespace { + +nsresult +GetMarkerFileHandle(const QuotaInfo& aQuotaInfo, nsIFile** aFileOut) +{ + MOZ_DIAGNOSTIC_ASSERT(aFileOut); + + nsCOMPtr<nsIFile> marker; + nsresult rv = aQuotaInfo.mDir->Clone(getter_AddRefs(marker)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = marker->Append(NS_LITERAL_STRING("cache")); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = marker->Append(NS_LITERAL_STRING("context_open.marker")); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + marker.forget(aFileOut); + + return rv; +} + +} // namespace + +nsresult +CreateMarkerFile(const QuotaInfo& aQuotaInfo) +{ + nsCOMPtr<nsIFile> marker; + nsresult rv = GetMarkerFileHandle(aQuotaInfo, getter_AddRefs(marker)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = marker->Create(nsIFile::NORMAL_FILE_TYPE, 0644); + if (rv == NS_ERROR_FILE_ALREADY_EXISTS) { + rv = NS_OK; + } + + // Note, we don't need to fsync here. We only care about actually + // writing the marker if later modifications to the Cache are + // actually flushed to the disk. If the OS crashes before the marker + // is written then we are ensured no other changes to the Cache were + // flushed either. + + return rv; +} + +nsresult +DeleteMarkerFile(const QuotaInfo& aQuotaInfo) +{ + nsCOMPtr<nsIFile> marker; + nsresult rv = GetMarkerFileHandle(aQuotaInfo, getter_AddRefs(marker)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = marker->Remove(/* recursive = */ false); + if (rv == NS_ERROR_FILE_NOT_FOUND || + rv == NS_ERROR_FILE_TARGET_DOES_NOT_EXIST) { + rv = NS_OK; + } + + // Again, no fsync is necessary. If the OS crashes before the file + // removal is flushed, then the Cache will search for stale data on + // startup. This will cause the next Cache access to be a bit slow, but + // it seems appropriate after an OS crash. + + return NS_OK; +} + +bool +MarkerFileExists(const QuotaInfo& aQuotaInfo) +{ + nsCOMPtr<nsIFile> marker; + nsresult rv = GetMarkerFileHandle(aQuotaInfo, getter_AddRefs(marker)); + if (NS_WARN_IF(NS_FAILED(rv))) { return false; } + + bool exists = false; + rv = marker->Exists(&exists); + if (NS_WARN_IF(NS_FAILED(rv))) { return false; } + + return exists; +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/FileUtils.h b/dom/cache/FileUtils.h new file mode 100644 index 0000000000..e7389abeae --- /dev/null +++ b/dom/cache/FileUtils.h @@ -0,0 +1,69 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_FileUtils_h +#define mozilla_dom_cache_FileUtils_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/cache/Types.h" +#include "nsStreamUtils.h" +#include "nsTArrayForwardDeclare.h" + +struct nsID; +class nsIFile; + +namespace mozilla { +namespace dom { +namespace cache { + +nsresult +BodyCreateDir(nsIFile* aBaseDir); + +// Note that this function can only be used during the initialization of the +// database. We're unlikely to be able to delete the DB successfully past +// that point due to the file being in use. +nsresult +BodyDeleteDir(nsIFile* aBaseDir); + +nsresult +BodyGetCacheDir(nsIFile* aBaseDir, const nsID& aId, nsIFile** aCacheDirOut); + +nsresult +BodyStartWriteStream(const QuotaInfo& aQuotaInfo, nsIFile* aBaseDir, + nsIInputStream* aSource, void* aClosure, + nsAsyncCopyCallbackFun aCallback, nsID* aIdOut, + nsISupports** aCopyContextOut); + +void +BodyCancelWrite(nsIFile* aBaseDir, nsISupports* aCopyContext); + +nsresult +BodyFinalizeWrite(nsIFile* aBaseDir, const nsID& aId); + +nsresult +BodyOpen(const QuotaInfo& aQuotaInfo, nsIFile* aBaseDir, const nsID& aId, + nsIInputStream** aStreamOut); + +nsresult +BodyDeleteFiles(nsIFile* aBaseDir, const nsTArray<nsID>& aIdList); + +nsresult +BodyDeleteOrphanedFiles(nsIFile* aBaseDir, nsTArray<nsID>& aKnownBodyIdList); + +nsresult +CreateMarkerFile(const QuotaInfo& aQuotaInfo); + +nsresult +DeleteMarkerFile(const QuotaInfo& aQuotaInfo); + +bool +MarkerFileExists(const QuotaInfo& aQuotaInfo); + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_FileUtils_h diff --git a/dom/cache/IPCUtils.h b/dom/cache/IPCUtils.h new file mode 100644 index 0000000000..143e4b4dbd --- /dev/null +++ b/dom/cache/IPCUtils.h @@ -0,0 +1,23 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_IPCUtils_h +#define mozilla_dom_cache_IPCUtils_h + +#include "ipc/IPCMessageUtils.h" + +#include "mozilla/dom/cache/Types.h" + +namespace IPC { + template<> + struct ParamTraits<mozilla::dom::cache::Namespace> : + public ContiguousEnumSerializer<mozilla::dom::cache::Namespace, + mozilla::dom::cache::DEFAULT_NAMESPACE, + mozilla::dom::cache::NUMBER_OF_NAMESPACES> + {}; +} // namespace IPC + +#endif // mozilla_dom_cache_IPCUtils_h diff --git a/dom/cache/Manager.cpp b/dom/cache/Manager.cpp new file mode 100644 index 0000000000..ee7cc51ac4 --- /dev/null +++ b/dom/cache/Manager.cpp @@ -0,0 +1,1952 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/Manager.h" + +#include "mozilla/AutoRestore.h" +#include "mozilla/Mutex.h" +#include "mozilla/StaticMutex.h" +#include "mozilla/StaticPtr.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/Context.h" +#include "mozilla/dom/cache/DBAction.h" +#include "mozilla/dom/cache/DBSchema.h" +#include "mozilla/dom/cache/FileUtils.h" +#include "mozilla/dom/cache/ManagerId.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/SavedTypes.h" +#include "mozilla/dom/cache/StreamList.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/ipc/BackgroundParent.h" +#include "mozStorageHelper.h" +#include "nsIInputStream.h" +#include "nsID.h" +#include "nsIFile.h" +#include "nsIThread.h" +#include "nsThreadUtils.h" +#include "nsTObserverArray.h" + + +namespace mozilla { +namespace dom { +namespace cache { + +namespace { + +// An Action that is executed when a Context is first created. It ensures that +// the directory and database are setup properly. This lets other actions +// not worry about these details. +class SetupAction final : public SyncDBAction +{ +public: + SetupAction() + : SyncDBAction(DBAction::Create) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + nsresult rv = BodyCreateDir(aDBDir); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // executes in its own transaction + rv = db::CreateOrMigrateSchema(aConn); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + // If the Context marker file exists, then the last session was + // not cleanly shutdown. In these cases sqlite will ensure that + // the database is valid, but we might still orphan data. Both + // Cache objects and body files can be referenced by DOM objects + // after they are "removed" from their parent. So we need to + // look and see if any of these late access objects have been + // orphaned. + // + // Note, this must be done after any schema version updates to + // ensure our DBSchema methods work correctly. + if (MarkerFileExists(aQuotaInfo)) { + NS_WARNING("Cache not shutdown cleanly! Cleaning up stale data..."); + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + // Clean up orphaned Cache objects + AutoTArray<CacheId, 8> orphanedCacheIdList; + nsresult rv = db::FindOrphanedCacheIds(aConn, orphanedCacheIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + for (uint32_t i = 0; i < orphanedCacheIdList.Length(); ++i) { + AutoTArray<nsID, 16> deletedBodyIdList; + rv = db::DeleteCacheId(aConn, orphanedCacheIdList[i], deletedBodyIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = BodyDeleteFiles(aDBDir, deletedBodyIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + // Clean up orphaned body objects + AutoTArray<nsID, 64> knownBodyIdList; + rv = db::GetKnownBodyIds(aConn, knownBodyIdList); + + rv = BodyDeleteOrphanedFiles(aDBDir, knownBodyIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } + + return rv; + } +}; + +// ---------------------------------------------------------------------------- + +// Action that is executed when we determine that content has stopped using +// a body file that has been orphaned. +class DeleteOrphanedBodyAction final : public Action +{ +public: + explicit DeleteOrphanedBodyAction(const nsTArray<nsID>& aDeletedBodyIdList) + : mDeletedBodyIdList(aDeletedBodyIdList) + { } + + explicit DeleteOrphanedBodyAction(const nsID& aBodyId) + { + mDeletedBodyIdList.AppendElement(aBodyId); + } + + virtual void + RunOnTarget(Resolver* aResolver, const QuotaInfo& aQuotaInfo, Data*) override + { + MOZ_DIAGNOSTIC_ASSERT(aResolver); + MOZ_DIAGNOSTIC_ASSERT(aQuotaInfo.mDir); + + // Note that since DeleteOrphanedBodyAction isn't used while the context is + // being initialized, we don't need to check for cancellation here. + + nsCOMPtr<nsIFile> dbDir; + nsresult rv = aQuotaInfo.mDir->Clone(getter_AddRefs(dbDir)); + if (NS_WARN_IF(NS_FAILED(rv))) { + aResolver->Resolve(rv); + return; + } + + rv = dbDir->Append(NS_LITERAL_STRING("cache")); + if (NS_WARN_IF(NS_FAILED(rv))) { + aResolver->Resolve(rv); + return; + } + + rv = BodyDeleteFiles(dbDir, mDeletedBodyIdList); + Unused << NS_WARN_IF(NS_FAILED(rv)); + + aResolver->Resolve(rv); + } + +private: + nsTArray<nsID> mDeletedBodyIdList; +}; + +bool IsHeadRequest(const CacheRequest& aRequest, const CacheQueryParams& aParams) +{ + return !aParams.ignoreMethod() && aRequest.method().LowerCaseEqualsLiteral("head"); +} + +bool IsHeadRequest(const CacheRequestOrVoid& aRequest, const CacheQueryParams& aParams) +{ + if (aRequest.type() == CacheRequestOrVoid::TCacheRequest) { + return !aParams.ignoreMethod() && + aRequest.get_CacheRequest().method().LowerCaseEqualsLiteral("head"); + } + return false; +} + +} // namespace + +// ---------------------------------------------------------------------------- + +// Singleton class to track Manager instances and ensure there is only +// one for each unique ManagerId. +class Manager::Factory +{ +public: + friend class StaticAutoPtr<Manager::Factory>; + + static nsresult + GetOrCreate(ManagerId* aManagerId, Manager** aManagerOut) + { + mozilla::ipc::AssertIsOnBackgroundThread(); + + // Ensure there is a factory instance. This forces the Get() call + // below to use the same factory. + nsresult rv = MaybeCreateInstance(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + RefPtr<Manager> ref = Get(aManagerId); + if (!ref) { + // TODO: replace this with a thread pool (bug 1119864) + nsCOMPtr<nsIThread> ioThread; + rv = NS_NewNamedThread("DOMCacheThread", getter_AddRefs(ioThread)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + ref = new Manager(aManagerId, ioThread); + + // There may be an old manager for this origin in the process of + // cleaning up. We need to tell the new manager about this so + // that it won't actually start until the old manager is done. + RefPtr<Manager> oldManager = Get(aManagerId, Closing); + ref->Init(oldManager); + + MOZ_ASSERT(!sFactory->mManagerList.Contains(ref)); + sFactory->mManagerList.AppendElement(ref); + } + + ref.forget(aManagerOut); + + return NS_OK; + } + + static already_AddRefed<Manager> + Get(ManagerId* aManagerId, State aState = Open) + { + mozilla::ipc::AssertIsOnBackgroundThread(); + + nsresult rv = MaybeCreateInstance(); + if (NS_WARN_IF(NS_FAILED(rv))) { return nullptr; } + + // Iterate in reverse to find the most recent, matching Manager. This + // is important when looking for a Closing Manager. If a new Manager + // chains to an old Manager we want it to be the most recent one. + ManagerList::BackwardIterator iter(sFactory->mManagerList); + while (iter.HasMore()) { + RefPtr<Manager> manager = iter.GetNext(); + if (aState == manager->GetState() && *manager->mManagerId == *aManagerId) { + return manager.forget(); + } + } + + return nullptr; + } + + static void + Remove(Manager* aManager) + { + mozilla::ipc::AssertIsOnBackgroundThread(); + MOZ_DIAGNOSTIC_ASSERT(aManager); + MOZ_DIAGNOSTIC_ASSERT(sFactory); + + MOZ_ALWAYS_TRUE(sFactory->mManagerList.RemoveElement(aManager)); + + // clean up the factory singleton if there are no more managers + MaybeDestroyInstance(); + } + + static void + Abort(const nsACString& aOrigin) + { + mozilla::ipc::AssertIsOnBackgroundThread(); + + if (!sFactory) { + return; + } + + MOZ_DIAGNOSTIC_ASSERT(!sFactory->mManagerList.IsEmpty()); + + { + ManagerList::ForwardIterator iter(sFactory->mManagerList); + while (iter.HasMore()) { + RefPtr<Manager> manager = iter.GetNext(); + if (aOrigin.IsVoid() || + manager->mManagerId->QuotaOrigin() == aOrigin) { + manager->Abort(); + } + } + } + } + + static void + ShutdownAll() + { + mozilla::ipc::AssertIsOnBackgroundThread(); + + if (!sFactory) { + return; + } + + MOZ_DIAGNOSTIC_ASSERT(!sFactory->mManagerList.IsEmpty()); + + { + // Note that we are synchronously calling shutdown code here. If any + // of the shutdown code synchronously decides to delete the Factory + // we need to delay that delete until the end of this method. + AutoRestore<bool> restore(sFactory->mInSyncShutdown); + sFactory->mInSyncShutdown = true; + + ManagerList::ForwardIterator iter(sFactory->mManagerList); + while (iter.HasMore()) { + RefPtr<Manager> manager = iter.GetNext(); + manager->Shutdown(); + } + } + + MaybeDestroyInstance(); + } + + static bool + IsShutdownAllComplete() + { + mozilla::ipc::AssertIsOnBackgroundThread(); + return !sFactory; + } + +private: + Factory() + : mInSyncShutdown(false) + { + MOZ_COUNT_CTOR(cache::Manager::Factory); + } + + ~Factory() + { + MOZ_COUNT_DTOR(cache::Manager::Factory); + MOZ_DIAGNOSTIC_ASSERT(mManagerList.IsEmpty()); + MOZ_DIAGNOSTIC_ASSERT(!mInSyncShutdown); + } + + static nsresult + MaybeCreateInstance() + { + mozilla::ipc::AssertIsOnBackgroundThread(); + + if (!sFactory) { + // Be clear about what we are locking. sFactory is bg thread only, so + // we don't need to lock it here. Just protect sFactoryShutdown and + // sBackgroundThread. + { + StaticMutexAutoLock lock(sMutex); + + if (sFactoryShutdown) { + return NS_ERROR_ILLEGAL_DURING_SHUTDOWN; + } + } + + // We cannot use ClearOnShutdown() here because we're not on the main + // thread. Instead, we delete sFactory in Factory::Remove() after the + // last manager is removed. ShutdownObserver ensures this happens + // before shutdown. + sFactory = new Factory(); + } + + // Never return sFactory to code outside Factory. We need to delete it + // out from under ourselves just before we return from Remove(). This + // would be (even more) dangerous if other code had a pointer to the + // factory itself. + + return NS_OK; + } + + static void + MaybeDestroyInstance() + { + mozilla::ipc::AssertIsOnBackgroundThread(); + MOZ_DIAGNOSTIC_ASSERT(sFactory); + + // If the factory is is still in use then we cannot delete yet. This + // could be due to managers still existing or because we are in the + // middle of shutting down. We need to be careful not to delete ourself + // synchronously during shutdown. + if (!sFactory->mManagerList.IsEmpty() || sFactory->mInSyncShutdown) { + return; + } + + sFactory = nullptr; + } + + // Singleton created on demand and deleted when last Manager is cleared + // in Remove(). + // PBackground thread only. + static StaticAutoPtr<Factory> sFactory; + + // protects following static attribute + static StaticMutex sMutex; + + // Indicate if shutdown has occurred to block re-creation of sFactory. + // Must hold sMutex to access. + static bool sFactoryShutdown; + + // Weak references as we don't want to keep Manager objects alive forever. + // When a Manager is destroyed it calls Factory::Remove() to clear itself. + // PBackground thread only. + typedef nsTObserverArray<Manager*> ManagerList; + ManagerList mManagerList; + + // This flag is set when we are looping through the list and calling + // Shutdown() on each Manager. We need to be careful not to synchronously + // trigger the deletion of the factory while still executing this loop. + bool mInSyncShutdown; +}; + +// static +StaticAutoPtr<Manager::Factory> Manager::Factory::sFactory; + +// static +StaticMutex Manager::Factory::sMutex; + +// static +bool Manager::Factory::sFactoryShutdown = false; + +// ---------------------------------------------------------------------------- + +// Abstract class to help implement the various Actions. The vast majority +// of Actions are synchronous and need to report back to a Listener on the +// Manager. +class Manager::BaseAction : public SyncDBAction +{ +protected: + BaseAction(Manager* aManager, ListenerId aListenerId) + : SyncDBAction(DBAction::Existing) + , mManager(aManager) + , mListenerId(aListenerId) + { + } + + virtual void + Complete(Listener* aListener, ErrorResult&& aRv) = 0; + + virtual void + CompleteOnInitiatingThread(nsresult aRv) override + { + NS_ASSERT_OWNINGTHREAD(Manager::BaseAction); + Listener* listener = mManager->GetListener(mListenerId); + if (listener) { + Complete(listener, ErrorResult(aRv)); + } + + // ensure we release the manager on the initiating thread + mManager = nullptr; + } + + RefPtr<Manager> mManager; + const ListenerId mListenerId; +}; + +// ---------------------------------------------------------------------------- + +// Action that is executed when we determine that content has stopped using +// a Cache object that has been orphaned. +class Manager::DeleteOrphanedCacheAction final : public SyncDBAction +{ +public: + DeleteOrphanedCacheAction(Manager* aManager, CacheId aCacheId) + : SyncDBAction(DBAction::Existing) + , mManager(aManager) + , mCacheId(aCacheId) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + nsresult rv = db::DeleteCacheId(aConn, mCacheId, mDeletedBodyIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = trans.Commit(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + return rv; + } + + virtual void + CompleteOnInitiatingThread(nsresult aRv) override + { + mManager->NoteOrphanedBodyIdList(mDeletedBodyIdList); + + // ensure we release the manager on the initiating thread + mManager = nullptr; + } + +private: + RefPtr<Manager> mManager; + const CacheId mCacheId; + nsTArray<nsID> mDeletedBodyIdList; +}; + +// ---------------------------------------------------------------------------- + +class Manager::CacheMatchAction final : public Manager::BaseAction +{ +public: + CacheMatchAction(Manager* aManager, ListenerId aListenerId, + CacheId aCacheId, const CacheMatchArgs& aArgs, + StreamList* aStreamList) + : BaseAction(aManager, aListenerId) + , mCacheId(aCacheId) + , mArgs(aArgs) + , mStreamList(aStreamList) + , mFoundResponse(false) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + nsresult rv = db::CacheMatch(aConn, mCacheId, mArgs.request(), + mArgs.params(), &mFoundResponse, &mResponse); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!mFoundResponse || !mResponse.mHasBodyId + || IsHeadRequest(mArgs.request(), mArgs.params())) { + mResponse.mHasBodyId = false; + return rv; + } + + nsCOMPtr<nsIInputStream> stream; + rv = BodyOpen(aQuotaInfo, aDBDir, mResponse.mBodyId, getter_AddRefs(stream)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (NS_WARN_IF(!stream)) { return NS_ERROR_FILE_NOT_FOUND; } + + mStreamList->Add(mResponse.mBodyId, stream); + + return rv; + } + + virtual void + Complete(Listener* aListener, ErrorResult&& aRv) override + { + if (!mFoundResponse) { + aListener->OnOpComplete(Move(aRv), CacheMatchResult(void_t())); + } else { + mStreamList->Activate(mCacheId); + aListener->OnOpComplete(Move(aRv), CacheMatchResult(void_t()), mResponse, + mStreamList); + } + mStreamList = nullptr; + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override + { + return aCacheId == mCacheId; + } + +private: + const CacheId mCacheId; + const CacheMatchArgs mArgs; + RefPtr<StreamList> mStreamList; + bool mFoundResponse; + SavedResponse mResponse; +}; + +// ---------------------------------------------------------------------------- + +class Manager::CacheMatchAllAction final : public Manager::BaseAction +{ +public: + CacheMatchAllAction(Manager* aManager, ListenerId aListenerId, + CacheId aCacheId, const CacheMatchAllArgs& aArgs, + StreamList* aStreamList) + : BaseAction(aManager, aListenerId) + , mCacheId(aCacheId) + , mArgs(aArgs) + , mStreamList(aStreamList) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + nsresult rv = db::CacheMatchAll(aConn, mCacheId, mArgs.requestOrVoid(), + mArgs.params(), mSavedResponses); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + for (uint32_t i = 0; i < mSavedResponses.Length(); ++i) { + if (!mSavedResponses[i].mHasBodyId + || IsHeadRequest(mArgs.requestOrVoid(), mArgs.params())) { + mSavedResponses[i].mHasBodyId = false; + continue; + } + + nsCOMPtr<nsIInputStream> stream; + rv = BodyOpen(aQuotaInfo, aDBDir, mSavedResponses[i].mBodyId, + getter_AddRefs(stream)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (NS_WARN_IF(!stream)) { return NS_ERROR_FILE_NOT_FOUND; } + + mStreamList->Add(mSavedResponses[i].mBodyId, stream); + } + + return rv; + } + + virtual void + Complete(Listener* aListener, ErrorResult&& aRv) override + { + mStreamList->Activate(mCacheId); + aListener->OnOpComplete(Move(aRv), CacheMatchAllResult(), mSavedResponses, + mStreamList); + mStreamList = nullptr; + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override + { + return aCacheId == mCacheId; + } + +private: + const CacheId mCacheId; + const CacheMatchAllArgs mArgs; + RefPtr<StreamList> mStreamList; + nsTArray<SavedResponse> mSavedResponses; +}; + +// ---------------------------------------------------------------------------- + +// This is the most complex Action. It puts a request/response pair into the +// Cache. It does not complete until all of the body data has been saved to +// disk. This means its an asynchronous Action. +class Manager::CachePutAllAction final : public DBAction +{ +public: + CachePutAllAction(Manager* aManager, ListenerId aListenerId, + CacheId aCacheId, + const nsTArray<CacheRequestResponse>& aPutList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aRequestStreamList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aResponseStreamList) + : DBAction(DBAction::Existing) + , mManager(aManager) + , mListenerId(aListenerId) + , mCacheId(aCacheId) + , mList(aPutList.Length()) + , mExpectedAsyncCopyCompletions(1) + , mAsyncResult(NS_OK) + , mMutex("cache::Manager::CachePutAllAction") + { + MOZ_DIAGNOSTIC_ASSERT(!aPutList.IsEmpty()); + MOZ_DIAGNOSTIC_ASSERT(aPutList.Length() == aRequestStreamList.Length()); + MOZ_DIAGNOSTIC_ASSERT(aPutList.Length() == aResponseStreamList.Length()); + + for (uint32_t i = 0; i < aPutList.Length(); ++i) { + Entry* entry = mList.AppendElement(); + entry->mRequest = aPutList[i].request(); + entry->mRequestStream = aRequestStreamList[i]; + entry->mResponse = aPutList[i].response(); + entry->mResponseStream = aResponseStreamList[i]; + } + } + +private: + ~CachePutAllAction() { } + + virtual void + RunWithDBOnTarget(Resolver* aResolver, const QuotaInfo& aQuotaInfo, + nsIFile* aDBDir, mozIStorageConnection* aConn) override + { + MOZ_DIAGNOSTIC_ASSERT(aResolver); + MOZ_DIAGNOSTIC_ASSERT(aDBDir); + MOZ_DIAGNOSTIC_ASSERT(aConn); + MOZ_DIAGNOSTIC_ASSERT(!mResolver); + MOZ_DIAGNOSTIC_ASSERT(!mDBDir); + MOZ_DIAGNOSTIC_ASSERT(!mConn); + + MOZ_DIAGNOSTIC_ASSERT(!mTargetThread); + mTargetThread = NS_GetCurrentThread(); + MOZ_DIAGNOSTIC_ASSERT(mTargetThread); + + // We should be pre-initialized to expect one async completion. This is + // the "manual" completion we call at the end of this method in all + // cases. + MOZ_DIAGNOSTIC_ASSERT(mExpectedAsyncCopyCompletions == 1); + + mResolver = aResolver; + mDBDir = aDBDir; + mConn = aConn; + + // File bodies are streamed to disk via asynchronous copying. Start + // this copying now. Each copy will eventually result in a call + // to OnAsyncCopyComplete(). + nsresult rv = NS_OK; + for (uint32_t i = 0; i < mList.Length(); ++i) { + rv = StartStreamCopy(aQuotaInfo, mList[i], RequestStream, + &mExpectedAsyncCopyCompletions); + if (NS_WARN_IF(NS_FAILED(rv))) { + break; + } + + rv = StartStreamCopy(aQuotaInfo, mList[i], ResponseStream, + &mExpectedAsyncCopyCompletions); + if (NS_WARN_IF(NS_FAILED(rv))) { + break; + } + } + + + // Always call OnAsyncCopyComplete() manually here. This covers the + // case where there is no async copying and also reports any startup + // errors correctly. If we hit an error, then OnAsyncCopyComplete() + // will cancel any async copying. + OnAsyncCopyComplete(rv); + } + + // Called once for each asynchronous file copy whether it succeeds or + // fails. If a file copy is canceled, it still calls this method with + // an error code. + void + OnAsyncCopyComplete(nsresult aRv) + { + MOZ_ASSERT(mTargetThread == NS_GetCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(mConn); + MOZ_DIAGNOSTIC_ASSERT(mResolver); + MOZ_DIAGNOSTIC_ASSERT(mExpectedAsyncCopyCompletions > 0); + + // Explicitly check for cancellation here to catch a race condition. + // Consider: + // + // 1) NS_AsyncCopy() executes on IO thread, but has not saved its + // copy context yet. + // 2) CancelAllStreamCopying() occurs on PBackground thread + // 3) Copy context from (1) is saved on IO thread. + // + // Checking for cancellation here catches this condition when we + // first call OnAsyncCopyComplete() manually from RunWithDBOnTarget(). + // + // This explicit cancellation check also handles the case where we + // are canceled just after all stream copying completes. We should + // abort the synchronous DB operations in this case if we have not + // started them yet. + if (NS_SUCCEEDED(aRv) && IsCanceled()) { + aRv = NS_ERROR_ABORT; + } + + // If any of the async copies fail, we need to still wait for them all to + // complete. Cancel any other streams still working and remember the + // error. All canceled streams will call OnAsyncCopyComplete(). + if (NS_FAILED(aRv) && NS_SUCCEEDED(mAsyncResult)) { + CancelAllStreamCopying(); + mAsyncResult = aRv; + } + + // Check to see if async copying is still on-going. If so, then simply + // return for now. We must wait for a later OnAsyncCopyComplete() call. + mExpectedAsyncCopyCompletions -= 1; + if (mExpectedAsyncCopyCompletions > 0) { + return; + } + + // We have finished with all async copying. Indicate this by clearing all + // our copy contexts. + { + MutexAutoLock lock(mMutex); + mCopyContextList.Clear(); + } + + // An error occurred while async copying. Terminate the Action. + // DoResolve() will clean up any files we may have written. + if (NS_FAILED(mAsyncResult)) { + DoResolve(mAsyncResult); + return; + } + + mozStorageTransaction trans(mConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + nsresult rv = NS_OK; + for (uint32_t i = 0; i < mList.Length(); ++i) { + Entry& e = mList[i]; + if (e.mRequestStream) { + rv = BodyFinalizeWrite(mDBDir, e.mRequestBodyId); + if (NS_WARN_IF(NS_FAILED(rv))) { + DoResolve(rv); + return; + } + } + if (e.mResponseStream) { + rv = BodyFinalizeWrite(mDBDir, e.mResponseBodyId); + if (NS_WARN_IF(NS_FAILED(rv))) { + DoResolve(rv); + return; + } + } + + rv = db::CachePut(mConn, mCacheId, e.mRequest, + e.mRequestStream ? &e.mRequestBodyId : nullptr, + e.mResponse, + e.mResponseStream ? &e.mResponseBodyId : nullptr, + mDeletedBodyIdList); + if (NS_WARN_IF(NS_FAILED(rv))) { + DoResolve(rv); + return; + } + } + + rv = trans.Commit(); + Unused << NS_WARN_IF(NS_FAILED(rv)); + + DoResolve(rv); + } + + virtual void + CompleteOnInitiatingThread(nsresult aRv) override + { + NS_ASSERT_OWNINGTHREAD(Action); + + for (uint32_t i = 0; i < mList.Length(); ++i) { + mList[i].mRequestStream = nullptr; + mList[i].mResponseStream = nullptr; + } + + mManager->NoteOrphanedBodyIdList(mDeletedBodyIdList); + + Listener* listener = mManager->GetListener(mListenerId); + mManager = nullptr; + if (listener) { + listener->OnOpComplete(ErrorResult(aRv), CachePutAllResult()); + } + } + + virtual void + CancelOnInitiatingThread() override + { + NS_ASSERT_OWNINGTHREAD(Action); + Action::CancelOnInitiatingThread(); + CancelAllStreamCopying(); + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override + { + NS_ASSERT_OWNINGTHREAD(Action); + return aCacheId == mCacheId; + } + + struct Entry + { + CacheRequest mRequest; + nsCOMPtr<nsIInputStream> mRequestStream; + nsID mRequestBodyId; + nsCOMPtr<nsISupports> mRequestCopyContext; + + CacheResponse mResponse; + nsCOMPtr<nsIInputStream> mResponseStream; + nsID mResponseBodyId; + nsCOMPtr<nsISupports> mResponseCopyContext; + }; + + enum StreamId + { + RequestStream, + ResponseStream + }; + + nsresult + StartStreamCopy(const QuotaInfo& aQuotaInfo, Entry& aEntry, + StreamId aStreamId, uint32_t* aCopyCountOut) + { + MOZ_ASSERT(mTargetThread == NS_GetCurrentThread()); + MOZ_DIAGNOSTIC_ASSERT(aCopyCountOut); + + if (IsCanceled()) { + return NS_ERROR_ABORT; + } + + nsCOMPtr<nsIInputStream> source; + nsID* bodyId; + + if (aStreamId == RequestStream) { + source = aEntry.mRequestStream; + bodyId = &aEntry.mRequestBodyId; + } else { + MOZ_DIAGNOSTIC_ASSERT(aStreamId == ResponseStream); + source = aEntry.mResponseStream; + bodyId = &aEntry.mResponseBodyId; + } + + if (!source) { + return NS_OK; + } + + nsCOMPtr<nsISupports> copyContext; + + nsresult rv = BodyStartWriteStream(aQuotaInfo, mDBDir, source, this, + AsyncCopyCompleteFunc, bodyId, + getter_AddRefs(copyContext)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + mBodyIdWrittenList.AppendElement(*bodyId); + + if (copyContext) { + MutexAutoLock lock(mMutex); + mCopyContextList.AppendElement(copyContext); + } + + *aCopyCountOut += 1; + + return rv; + } + + void + CancelAllStreamCopying() + { + // May occur on either owning thread or target thread + MutexAutoLock lock(mMutex); + for (uint32_t i = 0; i < mCopyContextList.Length(); ++i) { + BodyCancelWrite(mDBDir, mCopyContextList[i]); + } + mCopyContextList.Clear(); + } + + static void + AsyncCopyCompleteFunc(void* aClosure, nsresult aRv) + { + // May be on any thread, including STS event target. + MOZ_DIAGNOSTIC_ASSERT(aClosure); + // Weak ref as we are guaranteed to the action is alive until + // CompleteOnInitiatingThread is called. + CachePutAllAction* action = static_cast<CachePutAllAction*>(aClosure); + action->CallOnAsyncCopyCompleteOnTargetThread(aRv); + } + + void + CallOnAsyncCopyCompleteOnTargetThread(nsresult aRv) + { + // May be on any thread, including STS event target. Non-owning runnable + // here since we are guaranteed the Action will survive until + // CompleteOnInitiatingThread is called. + nsCOMPtr<nsIRunnable> runnable = NewNonOwningRunnableMethod<nsresult>( + this, &CachePutAllAction::OnAsyncCopyComplete, aRv); + MOZ_ALWAYS_SUCCEEDS( + mTargetThread->Dispatch(runnable, nsIThread::DISPATCH_NORMAL)); + } + + void + DoResolve(nsresult aRv) + { + MOZ_ASSERT(mTargetThread == NS_GetCurrentThread()); + + // DoResolve() must not be called until all async copying has completed. +#ifdef DEBUG + { + MutexAutoLock lock(mMutex); + MOZ_ASSERT(mCopyContextList.IsEmpty()); + } +#endif + + // Clean up any files we might have written before hitting the error. + if (NS_FAILED(aRv)) { + BodyDeleteFiles(mDBDir, mBodyIdWrittenList); + } + + // Must be released on the target thread where it was opened. + mConn = nullptr; + + // Drop our ref to the target thread as we are done with this thread. + // Also makes our thread assertions catch any incorrect method calls + // after resolve. + mTargetThread = nullptr; + + // Make sure to de-ref the resolver per the Action API contract. + RefPtr<Action::Resolver> resolver; + mResolver.swap(resolver); + resolver->Resolve(aRv); + } + + // initiating thread only + RefPtr<Manager> mManager; + const ListenerId mListenerId; + + // Set on initiating thread, read on target thread. State machine guarantees + // these are not modified while being read by the target thread. + const CacheId mCacheId; + nsTArray<Entry> mList; + uint32_t mExpectedAsyncCopyCompletions; + + // target thread only + RefPtr<Resolver> mResolver; + nsCOMPtr<nsIFile> mDBDir; + nsCOMPtr<mozIStorageConnection> mConn; + nsCOMPtr<nsIThread> mTargetThread; + nsresult mAsyncResult; + nsTArray<nsID> mBodyIdWrittenList; + + // Written to on target thread, accessed on initiating thread after target + // thread activity is guaranteed complete + nsTArray<nsID> mDeletedBodyIdList; + + // accessed from any thread while mMutex locked + Mutex mMutex; + nsTArray<nsCOMPtr<nsISupports>> mCopyContextList; +}; + +// ---------------------------------------------------------------------------- + +class Manager::CacheDeleteAction final : public Manager::BaseAction +{ +public: + CacheDeleteAction(Manager* aManager, ListenerId aListenerId, + CacheId aCacheId, const CacheDeleteArgs& aArgs) + : BaseAction(aManager, aListenerId) + , mCacheId(aCacheId) + , mArgs(aArgs) + , mSuccess(false) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + nsresult rv = db::CacheDelete(aConn, mCacheId, mArgs.request(), + mArgs.params(), mDeletedBodyIdList, + &mSuccess); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = trans.Commit(); + if (NS_WARN_IF(NS_FAILED(rv))) { + mSuccess = false; + return rv; + } + + return rv; + } + + virtual void + Complete(Listener* aListener, ErrorResult&& aRv) override + { + mManager->NoteOrphanedBodyIdList(mDeletedBodyIdList); + aListener->OnOpComplete(Move(aRv), CacheDeleteResult(mSuccess)); + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override + { + return aCacheId == mCacheId; + } + +private: + const CacheId mCacheId; + const CacheDeleteArgs mArgs; + bool mSuccess; + nsTArray<nsID> mDeletedBodyIdList; +}; + +// ---------------------------------------------------------------------------- + +class Manager::CacheKeysAction final : public Manager::BaseAction +{ +public: + CacheKeysAction(Manager* aManager, ListenerId aListenerId, + CacheId aCacheId, const CacheKeysArgs& aArgs, + StreamList* aStreamList) + : BaseAction(aManager, aListenerId) + , mCacheId(aCacheId) + , mArgs(aArgs) + , mStreamList(aStreamList) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + nsresult rv = db::CacheKeys(aConn, mCacheId, mArgs.requestOrVoid(), + mArgs.params(), mSavedRequests); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + for (uint32_t i = 0; i < mSavedRequests.Length(); ++i) { + if (!mSavedRequests[i].mHasBodyId + || IsHeadRequest(mArgs.requestOrVoid(), mArgs.params())) { + mSavedRequests[i].mHasBodyId = false; + continue; + } + + nsCOMPtr<nsIInputStream> stream; + rv = BodyOpen(aQuotaInfo, aDBDir, mSavedRequests[i].mBodyId, + getter_AddRefs(stream)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (NS_WARN_IF(!stream)) { return NS_ERROR_FILE_NOT_FOUND; } + + mStreamList->Add(mSavedRequests[i].mBodyId, stream); + } + + return rv; + } + + virtual void + Complete(Listener* aListener, ErrorResult&& aRv) override + { + mStreamList->Activate(mCacheId); + aListener->OnOpComplete(Move(aRv), CacheKeysResult(), mSavedRequests, + mStreamList); + mStreamList = nullptr; + } + + virtual bool MatchesCacheId(CacheId aCacheId) const override + { + return aCacheId == mCacheId; + } + +private: + const CacheId mCacheId; + const CacheKeysArgs mArgs; + RefPtr<StreamList> mStreamList; + nsTArray<SavedRequest> mSavedRequests; +}; + +// ---------------------------------------------------------------------------- + +class Manager::StorageMatchAction final : public Manager::BaseAction +{ +public: + StorageMatchAction(Manager* aManager, ListenerId aListenerId, + Namespace aNamespace, + const StorageMatchArgs& aArgs, + StreamList* aStreamList) + : BaseAction(aManager, aListenerId) + , mNamespace(aNamespace) + , mArgs(aArgs) + , mStreamList(aStreamList) + , mFoundResponse(false) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + nsresult rv = db::StorageMatch(aConn, mNamespace, mArgs.request(), + mArgs.params(), &mFoundResponse, + &mSavedResponse); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!mFoundResponse || !mSavedResponse.mHasBodyId + || IsHeadRequest(mArgs.request(), mArgs.params())) { + mSavedResponse.mHasBodyId = false; + return rv; + } + + nsCOMPtr<nsIInputStream> stream; + rv = BodyOpen(aQuotaInfo, aDBDir, mSavedResponse.mBodyId, + getter_AddRefs(stream)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (NS_WARN_IF(!stream)) { return NS_ERROR_FILE_NOT_FOUND; } + + mStreamList->Add(mSavedResponse.mBodyId, stream); + + return rv; + } + + virtual void + Complete(Listener* aListener, ErrorResult&& aRv) override + { + if (!mFoundResponse) { + aListener->OnOpComplete(Move(aRv), StorageMatchResult(void_t())); + } else { + mStreamList->Activate(mSavedResponse.mCacheId); + aListener->OnOpComplete(Move(aRv), StorageMatchResult(void_t()), mSavedResponse, + mStreamList); + } + mStreamList = nullptr; + } + +private: + const Namespace mNamespace; + const StorageMatchArgs mArgs; + RefPtr<StreamList> mStreamList; + bool mFoundResponse; + SavedResponse mSavedResponse; +}; + +// ---------------------------------------------------------------------------- + +class Manager::StorageHasAction final : public Manager::BaseAction +{ +public: + StorageHasAction(Manager* aManager, ListenerId aListenerId, + Namespace aNamespace, const StorageHasArgs& aArgs) + : BaseAction(aManager, aListenerId) + , mNamespace(aNamespace) + , mArgs(aArgs) + , mCacheFound(false) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + CacheId cacheId; + return db::StorageGetCacheId(aConn, mNamespace, mArgs.key(), + &mCacheFound, &cacheId); + } + + virtual void + Complete(Listener* aListener, ErrorResult&& aRv) override + { + aListener->OnOpComplete(Move(aRv), StorageHasResult(mCacheFound)); + } + +private: + const Namespace mNamespace; + const StorageHasArgs mArgs; + bool mCacheFound; +}; + +// ---------------------------------------------------------------------------- + +class Manager::StorageOpenAction final : public Manager::BaseAction +{ +public: + StorageOpenAction(Manager* aManager, ListenerId aListenerId, + Namespace aNamespace, const StorageOpenArgs& aArgs) + : BaseAction(aManager, aListenerId) + , mNamespace(aNamespace) + , mArgs(aArgs) + , mCacheId(INVALID_CACHE_ID) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + // Cache does not exist, create it instead + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + // Look for existing cache + bool cacheFound; + nsresult rv = db::StorageGetCacheId(aConn, mNamespace, mArgs.key(), + &cacheFound, &mCacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + if (cacheFound) { + MOZ_DIAGNOSTIC_ASSERT(mCacheId != INVALID_CACHE_ID); + return rv; + } + + rv = db::CreateCacheId(aConn, &mCacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = db::StoragePutCache(aConn, mNamespace, mArgs.key(), mCacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = trans.Commit(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + MOZ_DIAGNOSTIC_ASSERT(mCacheId != INVALID_CACHE_ID); + return rv; + } + + virtual void + Complete(Listener* aListener, ErrorResult&& aRv) override + { + MOZ_DIAGNOSTIC_ASSERT(aRv.Failed() || mCacheId != INVALID_CACHE_ID); + aListener->OnOpComplete(Move(aRv), StorageOpenResult(), mCacheId); + } + +private: + const Namespace mNamespace; + const StorageOpenArgs mArgs; + CacheId mCacheId; +}; + +// ---------------------------------------------------------------------------- + +class Manager::StorageDeleteAction final : public Manager::BaseAction +{ +public: + StorageDeleteAction(Manager* aManager, ListenerId aListenerId, + Namespace aNamespace, const StorageDeleteArgs& aArgs) + : BaseAction(aManager, aListenerId) + , mNamespace(aNamespace) + , mArgs(aArgs) + , mCacheDeleted(false) + , mCacheId(INVALID_CACHE_ID) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + mozStorageTransaction trans(aConn, false, + mozIStorageConnection::TRANSACTION_IMMEDIATE); + + bool exists; + nsresult rv = db::StorageGetCacheId(aConn, mNamespace, mArgs.key(), + &exists, &mCacheId); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (!exists) { + mCacheDeleted = false; + return NS_OK; + } + + rv = db::StorageForgetCache(aConn, mNamespace, mArgs.key()); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = trans.Commit(); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + mCacheDeleted = true; + return rv; + } + + virtual void + Complete(Listener* aListener, ErrorResult&& aRv) override + { + if (mCacheDeleted) { + // If content is referencing this cache, mark it orphaned to be + // deleted later. + if (!mManager->SetCacheIdOrphanedIfRefed(mCacheId)) { + + // no outstanding references, delete immediately + RefPtr<Context> context = mManager->mContext; + + if (context->IsCanceled()) { + context->NoteOrphanedData(); + } else { + context->CancelForCacheId(mCacheId); + RefPtr<Action> action = + new DeleteOrphanedCacheAction(mManager, mCacheId); + context->Dispatch(action); + } + } + } + + aListener->OnOpComplete(Move(aRv), StorageDeleteResult(mCacheDeleted)); + } + +private: + const Namespace mNamespace; + const StorageDeleteArgs mArgs; + bool mCacheDeleted; + CacheId mCacheId; +}; + +// ---------------------------------------------------------------------------- + +class Manager::StorageKeysAction final : public Manager::BaseAction +{ +public: + StorageKeysAction(Manager* aManager, ListenerId aListenerId, + Namespace aNamespace) + : BaseAction(aManager, aListenerId) + , mNamespace(aNamespace) + { } + + virtual nsresult + RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir, + mozIStorageConnection* aConn) override + { + return db::StorageGetKeys(aConn, mNamespace, mKeys); + } + + virtual void + Complete(Listener* aListener, ErrorResult&& aRv) override + { + if (aRv.Failed()) { + mKeys.Clear(); + } + aListener->OnOpComplete(Move(aRv), StorageKeysResult(mKeys)); + } + +private: + const Namespace mNamespace; + nsTArray<nsString> mKeys; +}; + +// ---------------------------------------------------------------------------- + +//static +Manager::ListenerId Manager::sNextListenerId = 0; + +void +Manager::Listener::OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult) +{ + OnOpComplete(Move(aRv), aResult, INVALID_CACHE_ID, nsTArray<SavedResponse>(), + nsTArray<SavedRequest>(), nullptr); +} + +void +Manager::Listener::OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + CacheId aOpenedCacheId) +{ + OnOpComplete(Move(aRv), aResult, aOpenedCacheId, nsTArray<SavedResponse>(), + nsTArray<SavedRequest>(), nullptr); +} + +void +Manager::Listener::OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + const SavedResponse& aSavedResponse, + StreamList* aStreamList) +{ + AutoTArray<SavedResponse, 1> responseList; + responseList.AppendElement(aSavedResponse); + OnOpComplete(Move(aRv), aResult, INVALID_CACHE_ID, responseList, + nsTArray<SavedRequest>(), aStreamList); +} + +void +Manager::Listener::OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + const nsTArray<SavedResponse>& aSavedResponseList, + StreamList* aStreamList) +{ + OnOpComplete(Move(aRv), aResult, INVALID_CACHE_ID, aSavedResponseList, + nsTArray<SavedRequest>(), aStreamList); +} + +void +Manager::Listener::OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + const nsTArray<SavedRequest>& aSavedRequestList, + StreamList* aStreamList) +{ + OnOpComplete(Move(aRv), aResult, INVALID_CACHE_ID, nsTArray<SavedResponse>(), + aSavedRequestList, aStreamList); +} + +// static +nsresult +Manager::GetOrCreate(ManagerId* aManagerId, Manager** aManagerOut) +{ + mozilla::ipc::AssertIsOnBackgroundThread(); + return Factory::GetOrCreate(aManagerId, aManagerOut); +} + +// static +already_AddRefed<Manager> +Manager::Get(ManagerId* aManagerId) +{ + mozilla::ipc::AssertIsOnBackgroundThread(); + return Factory::Get(aManagerId); +} + +// static +void +Manager::ShutdownAll() +{ + mozilla::ipc::AssertIsOnBackgroundThread(); + + Factory::ShutdownAll(); + + while (!Factory::IsShutdownAllComplete()) { + if (!NS_ProcessNextEvent()) { + NS_WARNING("Something bad happened!"); + break; + } + } +} + +// static +void +Manager::Abort(const nsACString& aOrigin) +{ + mozilla::ipc::AssertIsOnBackgroundThread(); + + Factory::Abort(aOrigin); +} + +void +Manager::RemoveListener(Listener* aListener) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + // There may not be a listener here in the case where an actor is killed + // before it can perform any actual async requests on Manager. + mListeners.RemoveElement(aListener, ListenerEntryListenerComparator()); + MOZ_ASSERT(!mListeners.Contains(aListener, + ListenerEntryListenerComparator())); + MaybeAllowContextToClose(); +} + +void +Manager::RemoveContext(Context* aContext) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(mContext); + MOZ_DIAGNOSTIC_ASSERT(mContext == aContext); + + // Whether the Context destruction was triggered from the Manager going + // idle or the underlying storage being invalidated, we should know we + // are closing before the Context is destroyed. + MOZ_DIAGNOSTIC_ASSERT(mState == Closing); + + // Before forgetting the Context, check to see if we have any outstanding + // cache or body objects waiting for deletion. If so, note that we've + // orphaned data so it will be cleaned up on the next open. + for (uint32_t i = 0; i < mCacheIdRefs.Length(); ++i) { + if (mCacheIdRefs[i].mOrphaned) { + aContext->NoteOrphanedData(); + break; + } + } + + for (uint32_t i = 0; i < mBodyIdRefs.Length(); ++i) { + if (mBodyIdRefs[i].mOrphaned) { + aContext->NoteOrphanedData(); + break; + } + } + + mContext = nullptr; + + // Once the context is gone, we can immediately remove ourself from the + // Factory list. We don't need to block shutdown by staying in the list + // any more. + Factory::Remove(this); +} + +void +Manager::NoteClosing() +{ + NS_ASSERT_OWNINGTHREAD(Manager); + // This can be called more than once legitimately through different paths. + mState = Closing; +} + +Manager::State +Manager::GetState() const +{ + NS_ASSERT_OWNINGTHREAD(Manager); + return mState; +} + +void +Manager::AddRefCacheId(CacheId aCacheId) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + for (uint32_t i = 0; i < mCacheIdRefs.Length(); ++i) { + if (mCacheIdRefs[i].mCacheId == aCacheId) { + mCacheIdRefs[i].mCount += 1; + return; + } + } + CacheIdRefCounter* entry = mCacheIdRefs.AppendElement(); + entry->mCacheId = aCacheId; + entry->mCount = 1; + entry->mOrphaned = false; +} + +void +Manager::ReleaseCacheId(CacheId aCacheId) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + for (uint32_t i = 0; i < mCacheIdRefs.Length(); ++i) { + if (mCacheIdRefs[i].mCacheId == aCacheId) { +#if defined(DEBUG) || !defined(RELEASE_OR_BETA) + uint32_t oldRef = mCacheIdRefs[i].mCount; +#endif + mCacheIdRefs[i].mCount -= 1; + MOZ_DIAGNOSTIC_ASSERT(mCacheIdRefs[i].mCount < oldRef); + if (mCacheIdRefs[i].mCount == 0) { + bool orphaned = mCacheIdRefs[i].mOrphaned; + mCacheIdRefs.RemoveElementAt(i); + RefPtr<Context> context = mContext; + // If the context is already gone, then orphan flag should have been + // set in RemoveContext(). + if (orphaned && context) { + if (context->IsCanceled()) { + context->NoteOrphanedData(); + } else { + context->CancelForCacheId(aCacheId); + RefPtr<Action> action = new DeleteOrphanedCacheAction(this, + aCacheId); + context->Dispatch(action); + } + } + } + MaybeAllowContextToClose(); + return; + } + } + MOZ_ASSERT_UNREACHABLE("Attempt to release CacheId that is not referenced!"); +} + +void +Manager::AddRefBodyId(const nsID& aBodyId) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + for (uint32_t i = 0; i < mBodyIdRefs.Length(); ++i) { + if (mBodyIdRefs[i].mBodyId == aBodyId) { + mBodyIdRefs[i].mCount += 1; + return; + } + } + BodyIdRefCounter* entry = mBodyIdRefs.AppendElement(); + entry->mBodyId = aBodyId; + entry->mCount = 1; + entry->mOrphaned = false; +} + +void +Manager::ReleaseBodyId(const nsID& aBodyId) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + for (uint32_t i = 0; i < mBodyIdRefs.Length(); ++i) { + if (mBodyIdRefs[i].mBodyId == aBodyId) { +#if defined(DEBUG) || !defined(RELEASE_OR_BETA) + uint32_t oldRef = mBodyIdRefs[i].mCount; +#endif + mBodyIdRefs[i].mCount -= 1; + MOZ_DIAGNOSTIC_ASSERT(mBodyIdRefs[i].mCount < oldRef); + if (mBodyIdRefs[i].mCount < 1) { + bool orphaned = mBodyIdRefs[i].mOrphaned; + mBodyIdRefs.RemoveElementAt(i); + RefPtr<Context> context = mContext; + // If the context is already gone, then orphan flag should have been + // set in RemoveContext(). + if (orphaned && context) { + if (context->IsCanceled()) { + context->NoteOrphanedData(); + } else { + RefPtr<Action> action = new DeleteOrphanedBodyAction(aBodyId); + context->Dispatch(action); + } + } + } + MaybeAllowContextToClose(); + return; + } + } + MOZ_ASSERT_UNREACHABLE("Attempt to release BodyId that is not referenced!"); +} + +already_AddRefed<ManagerId> +Manager::GetManagerId() const +{ + RefPtr<ManagerId> ref = mManagerId; + return ref.forget(); +} + +void +Manager::AddStreamList(StreamList* aStreamList) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(aStreamList); + mStreamLists.AppendElement(aStreamList); +} + +void +Manager::RemoveStreamList(StreamList* aStreamList) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(aStreamList); + mStreamLists.RemoveElement(aStreamList); +} + +void +Manager::ExecuteCacheOp(Listener* aListener, CacheId aCacheId, + const CacheOpArgs& aOpArgs) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(aListener); + MOZ_DIAGNOSTIC_ASSERT(aOpArgs.type() != CacheOpArgs::TCachePutAllArgs); + + if (NS_WARN_IF(mState == Closing)) { + aListener->OnOpComplete(ErrorResult(NS_ERROR_FAILURE), void_t()); + return; + } + + RefPtr<Context> context = mContext; + MOZ_DIAGNOSTIC_ASSERT(!context->IsCanceled()); + + RefPtr<StreamList> streamList = new StreamList(this, context); + ListenerId listenerId = SaveListener(aListener); + + RefPtr<Action> action; + switch(aOpArgs.type()) { + case CacheOpArgs::TCacheMatchArgs: + action = new CacheMatchAction(this, listenerId, aCacheId, + aOpArgs.get_CacheMatchArgs(), streamList); + break; + case CacheOpArgs::TCacheMatchAllArgs: + action = new CacheMatchAllAction(this, listenerId, aCacheId, + aOpArgs.get_CacheMatchAllArgs(), + streamList); + break; + case CacheOpArgs::TCacheDeleteArgs: + action = new CacheDeleteAction(this, listenerId, aCacheId, + aOpArgs.get_CacheDeleteArgs()); + break; + case CacheOpArgs::TCacheKeysArgs: + action = new CacheKeysAction(this, listenerId, aCacheId, + aOpArgs.get_CacheKeysArgs(), streamList); + break; + default: + MOZ_CRASH("Unknown Cache operation!"); + } + + context->Dispatch(action); +} + +void +Manager::ExecuteStorageOp(Listener* aListener, Namespace aNamespace, + const CacheOpArgs& aOpArgs) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(aListener); + + if (NS_WARN_IF(mState == Closing)) { + aListener->OnOpComplete(ErrorResult(NS_ERROR_FAILURE), void_t()); + return; + } + + RefPtr<Context> context = mContext; + MOZ_DIAGNOSTIC_ASSERT(!context->IsCanceled()); + + RefPtr<StreamList> streamList = new StreamList(this, context); + ListenerId listenerId = SaveListener(aListener); + + RefPtr<Action> action; + switch(aOpArgs.type()) { + case CacheOpArgs::TStorageMatchArgs: + action = new StorageMatchAction(this, listenerId, aNamespace, + aOpArgs.get_StorageMatchArgs(), + streamList); + break; + case CacheOpArgs::TStorageHasArgs: + action = new StorageHasAction(this, listenerId, aNamespace, + aOpArgs.get_StorageHasArgs()); + break; + case CacheOpArgs::TStorageOpenArgs: + action = new StorageOpenAction(this, listenerId, aNamespace, + aOpArgs.get_StorageOpenArgs()); + break; + case CacheOpArgs::TStorageDeleteArgs: + action = new StorageDeleteAction(this, listenerId, aNamespace, + aOpArgs.get_StorageDeleteArgs()); + break; + case CacheOpArgs::TStorageKeysArgs: + action = new StorageKeysAction(this, listenerId, aNamespace); + break; + default: + MOZ_CRASH("Unknown CacheStorage operation!"); + } + + context->Dispatch(action); +} + +void +Manager::ExecutePutAll(Listener* aListener, CacheId aCacheId, + const nsTArray<CacheRequestResponse>& aPutList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aRequestStreamList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aResponseStreamList) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(aListener); + + if (NS_WARN_IF(mState == Closing)) { + aListener->OnOpComplete(ErrorResult(NS_ERROR_FAILURE), CachePutAllResult()); + return; + } + + RefPtr<Context> context = mContext; + MOZ_DIAGNOSTIC_ASSERT(!context->IsCanceled()); + + ListenerId listenerId = SaveListener(aListener); + + RefPtr<Action> action = new CachePutAllAction(this, listenerId, aCacheId, + aPutList, aRequestStreamList, + aResponseStreamList); + + context->Dispatch(action); +} + +Manager::Manager(ManagerId* aManagerId, nsIThread* aIOThread) + : mManagerId(aManagerId) + , mIOThread(aIOThread) + , mContext(nullptr) + , mShuttingDown(false) + , mState(Open) +{ + MOZ_DIAGNOSTIC_ASSERT(mManagerId); + MOZ_DIAGNOSTIC_ASSERT(mIOThread); +} + +Manager::~Manager() +{ + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(mState == Closing); + MOZ_DIAGNOSTIC_ASSERT(!mContext); + + nsCOMPtr<nsIThread> ioThread; + mIOThread.swap(ioThread); + + // Don't spin the event loop in the destructor waiting for the thread to + // shutdown. Defer this to the main thread, instead. + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(NewRunnableMethod(ioThread, &nsIThread::Shutdown))); +} + +void +Manager::Init(Manager* aOldManager) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + + RefPtr<Context> oldContext; + if (aOldManager) { + oldContext = aOldManager->mContext; + } + + // Create the context immediately. Since there can at most be one Context + // per Manager now, this lets us cleanly call Factory::Remove() once the + // Context goes away. + RefPtr<Action> setupAction = new SetupAction(); + RefPtr<Context> ref = Context::Create(this, mIOThread, setupAction, + oldContext); + mContext = ref; +} + +void +Manager::Shutdown() +{ + NS_ASSERT_OWNINGTHREAD(Manager); + + // Ignore duplicate attempts to shutdown. This can occur when we start + // a browser initiated shutdown and then run ~Manager() which also + // calls Shutdown(). + if (mShuttingDown) { + return; + } + + mShuttingDown = true; + + // Note that we are closing to prevent any new requests from coming in and + // creating a new Context. We must ensure all Contexts and IO operations are + // complete before shutdown proceeds. + NoteClosing(); + + // If there is a context, then cancel and only note that we are done after + // its cleaned up. + if (mContext) { + RefPtr<Context> context = mContext; + context->CancelAll(); + return; + } +} + +void +Manager::Abort() +{ + NS_ASSERT_OWNINGTHREAD(Manager); + MOZ_DIAGNOSTIC_ASSERT(mContext); + + // Note that we are closing to prevent any new requests from coming in and + // creating a new Context. We must ensure all Contexts and IO operations are + // complete before origin clear proceeds. + NoteClosing(); + + // Cancel and only note that we are done after the context is cleaned up. + RefPtr<Context> context = mContext; + context->CancelAll(); +} + +Manager::ListenerId +Manager::SaveListener(Listener* aListener) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + + // Once a Listener is added, we keep a reference to it until its + // removed. Since the same Listener might make multiple requests, + // ensure we only have a single reference in our list. + ListenerList::index_type index = + mListeners.IndexOf(aListener, 0, ListenerEntryListenerComparator()); + if (index != ListenerList::NoIndex) { + return mListeners[index].mId; + } + + ListenerId id = sNextListenerId; + sNextListenerId += 1; + + mListeners.AppendElement(ListenerEntry(id, aListener)); + return id; +} + +Manager::Listener* +Manager::GetListener(ListenerId aListenerId) const +{ + NS_ASSERT_OWNINGTHREAD(Manager); + ListenerList::index_type index = + mListeners.IndexOf(aListenerId, 0, ListenerEntryIdComparator()); + if (index != ListenerList::NoIndex) { + return mListeners[index].mListener; + } + + // This can legitimately happen if the actor is deleted while a request is + // in process. For example, the child process OOMs. + return nullptr; +} + +bool +Manager::SetCacheIdOrphanedIfRefed(CacheId aCacheId) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + for (uint32_t i = 0; i < mCacheIdRefs.Length(); ++i) { + if (mCacheIdRefs[i].mCacheId == aCacheId) { + MOZ_DIAGNOSTIC_ASSERT(mCacheIdRefs[i].mCount > 0); + MOZ_DIAGNOSTIC_ASSERT(!mCacheIdRefs[i].mOrphaned); + mCacheIdRefs[i].mOrphaned = true; + return true; + } + } + return false; +} + +// TODO: provide way to set body non-orphaned if its added back to a cache (bug 1110479) + +bool +Manager::SetBodyIdOrphanedIfRefed(const nsID& aBodyId) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + for (uint32_t i = 0; i < mBodyIdRefs.Length(); ++i) { + if (mBodyIdRefs[i].mBodyId == aBodyId) { + MOZ_DIAGNOSTIC_ASSERT(mBodyIdRefs[i].mCount > 0); + MOZ_DIAGNOSTIC_ASSERT(!mBodyIdRefs[i].mOrphaned); + mBodyIdRefs[i].mOrphaned = true; + return true; + } + } + return false; +} + +void +Manager::NoteOrphanedBodyIdList(const nsTArray<nsID>& aDeletedBodyIdList) +{ + NS_ASSERT_OWNINGTHREAD(Manager); + + AutoTArray<nsID, 64> deleteNowList; + deleteNowList.SetCapacity(aDeletedBodyIdList.Length()); + + for (uint32_t i = 0; i < aDeletedBodyIdList.Length(); ++i) { + if (!SetBodyIdOrphanedIfRefed(aDeletedBodyIdList[i])) { + deleteNowList.AppendElement(aDeletedBodyIdList[i]); + } + } + + // TODO: note that we need to check these bodies for staleness on startup (bug 1110446) + RefPtr<Context> context = mContext; + if (!deleteNowList.IsEmpty() && context && !context->IsCanceled()) { + RefPtr<Action> action = new DeleteOrphanedBodyAction(deleteNowList); + context->Dispatch(action); + } +} + +void +Manager::MaybeAllowContextToClose() +{ + NS_ASSERT_OWNINGTHREAD(Manager); + + // If we have an active context, but we have no more users of the Manager, + // then let it shut itself down. We must wait for all possible users of + // Cache state information to complete before doing this. Once we allow + // the Context to close we may not reliably get notified of storage + // invalidation. + RefPtr<Context> context = mContext; + if (context && mListeners.IsEmpty() + && mCacheIdRefs.IsEmpty() + && mBodyIdRefs.IsEmpty()) { + + // Mark this Manager as invalid so that it won't get used again. We don't + // want to start any new operations once we allow the Context to close since + // it may race with the underlying storage getting invalidated. + NoteClosing(); + + context->AllowToClose(); + } +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/Manager.h b/dom/cache/Manager.h new file mode 100644 index 0000000000..20392dad85 --- /dev/null +++ b/dom/cache/Manager.h @@ -0,0 +1,292 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Manager_h +#define mozilla_dom_cache_Manager_h + +#include "mozilla/dom/cache/Types.h" +#include "nsCOMPtr.h" +#include "nsISupportsImpl.h" +#include "mozilla/RefPtr.h" +#include "nsString.h" +#include "nsTArray.h" + +class nsIInputStream; +class nsIThread; + +namespace mozilla { + +class ErrorResult; + +namespace dom { +namespace cache { + +class CacheOpArgs; +class CacheOpResult; +class CacheRequestResponse; +class Context; +class ManagerId; +struct SavedRequest; +struct SavedResponse; +class StreamList; + +// The Manager is class is responsible for performing all of the underlying +// work for a Cache or CacheStorage operation. The DOM objects and IPC actors +// are basically just plumbing to get the request to the right Manager object +// running in the parent process. +// +// There should be exactly one Manager object for each origin or app using the +// Cache API. This uniqueness is defined by the ManagerId equality operator. +// The uniqueness is enforced by the Manager GetOrCreate() factory method. +// +// The life cycle of Manager objects is somewhat complex. While code may +// hold a strong reference to the Manager, it will invalidate itself once it +// believes it has become completely idle. This is currently determined when +// all of the following conditions occur: +// +// 1) There are no more Manager::Listener objects registered with the Manager +// by performing a Cache or Storage operation. +// 2) There are no more CacheId references noted via Manager::AddRefCacheId(). +// 3) There are no more BodyId references noted via Manager::AddRefBodyId(). +// +// In order to keep your Manager alive you should perform an operation to set +// a Listener, call AddRefCacheId(), or call AddRefBodyId(). +// +// Even once a Manager becomes invalid, however, it may still continue to +// exist. This is allowed so that any in-progress Actions can gracefully +// complete. +// +// As an invariant, all Manager objects must cease all IO before shutdown. This +// is enforced by the Manager::Factory. If content still holds references to +// Cache DOM objects during shutdown, then all operations will begin rejecting. +class Manager final +{ +public: + // Callback interface implemented by clients of Manager, such as CacheParent + // and CacheStorageParent. In general, if you call a Manager method you + // should expect to receive exactly one On*() callback. For example, if + // you call Manager::CacheMatch(), then you should expect to receive + // OnCacheMatch() back in response. + // + // Listener objects are set on a per-operation basis. So you pass the + // Listener to a call like Manager::CacheMatch(). Once set in this way, + // the Manager will continue to reference the Listener until RemoveListener() + // is called. This is done to allow the same listener to be used for + // multiple operations simultaneously without having to maintain an exact + // count of operations-in-flight. + // + // Note, the Manager only holds weak references to Listener objects. + // Listeners must call Manager::RemoveListener() before they are destroyed + // to clear these weak references. + // + // All public methods should be invoked on the same thread used to create + // the Manager. + class Listener + { + public: + // convenience routines + void + OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult); + + void + OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + CacheId aOpenedCacheId); + + void + OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + const SavedResponse& aSavedResponse, + StreamList* aStreamList); + + void + OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + const nsTArray<SavedResponse>& aSavedResponseList, + StreamList* aStreamList); + + void + OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + const nsTArray<SavedRequest>& aSavedRequestList, + StreamList* aStreamList); + + // interface to be implemented + virtual void + OnOpComplete(ErrorResult&& aRv, const CacheOpResult& aResult, + CacheId aOpenedCacheId, + const nsTArray<SavedResponse>& aSavedResponseList, + const nsTArray<SavedRequest>& aSavedRequestList, + StreamList* aStreamList) { } + + protected: + ~Listener() { } + }; + + enum State + { + Open, + Closing + }; + + static nsresult GetOrCreate(ManagerId* aManagerId, Manager** aManagerOut); + static already_AddRefed<Manager> Get(ManagerId* aManagerId); + + // Synchronously shutdown. This spins the event loop. + static void ShutdownAll(); + + // Cancel actions for given origin or all actions if passed string is null. + static void Abort(const nsACString& aOrigin); + + // Must be called by Listener objects before they are destroyed. + void RemoveListener(Listener* aListener); + + // Must be called by Context objects before they are destroyed. + void RemoveContext(Context* aContext); + + // Marks the Manager "invalid". Once the Context completes no new operations + // will be permitted with this Manager. New actors will get a new Manager. + void NoteClosing(); + + State GetState() const; + + // If an actor represents a long term reference to a cache or body stream, + // then they must call AddRefCacheId() or AddRefBodyId(). This will + // cause the Manager to keep the backing data store alive for the given + // object. The actor must then call ReleaseCacheId() or ReleaseBodyId() + // exactly once for every AddRef*() call it made. Any delayed deletion + // will then be performed. + void AddRefCacheId(CacheId aCacheId); + void ReleaseCacheId(CacheId aCacheId); + void AddRefBodyId(const nsID& aBodyId); + void ReleaseBodyId(const nsID& aBodyId); + + already_AddRefed<ManagerId> GetManagerId() const; + + // Methods to allow a StreamList to register themselves with the Manager. + // StreamList objects must call RemoveStreamList() before they are destroyed. + void AddStreamList(StreamList* aStreamList); + void RemoveStreamList(StreamList* aStreamList); + + void ExecuteCacheOp(Listener* aListener, CacheId aCacheId, + const CacheOpArgs& aOpArgs); + void ExecutePutAll(Listener* aListener, CacheId aCacheId, + const nsTArray<CacheRequestResponse>& aPutList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aRequestStreamList, + const nsTArray<nsCOMPtr<nsIInputStream>>& aResponseStreamList); + + void ExecuteStorageOp(Listener* aListener, Namespace aNamespace, + const CacheOpArgs& aOpArgs); + +private: + class Factory; + class BaseAction; + class DeleteOrphanedCacheAction; + + class CacheMatchAction; + class CacheMatchAllAction; + class CachePutAllAction; + class CacheDeleteAction; + class CacheKeysAction; + + class StorageMatchAction; + class StorageHasAction; + class StorageOpenAction; + class StorageDeleteAction; + class StorageKeysAction; + + typedef uint64_t ListenerId; + + Manager(ManagerId* aManagerId, nsIThread* aIOThread); + ~Manager(); + void Init(Manager* aOldManager); + void Shutdown(); + + void Abort(); + + ListenerId SaveListener(Listener* aListener); + Listener* GetListener(ListenerId aListenerId) const; + + bool SetCacheIdOrphanedIfRefed(CacheId aCacheId); + bool SetBodyIdOrphanedIfRefed(const nsID& aBodyId); + void NoteOrphanedBodyIdList(const nsTArray<nsID>& aDeletedBodyIdList); + + void MaybeAllowContextToClose(); + + RefPtr<ManagerId> mManagerId; + nsCOMPtr<nsIThread> mIOThread; + + // Weak reference cleared by RemoveContext() in Context destructor. + Context* MOZ_NON_OWNING_REF mContext; + + // Weak references cleared by RemoveListener() in Listener destructors. + struct ListenerEntry + { + ListenerEntry() + : mId(UINT64_MAX) + , mListener(nullptr) + { + } + + ListenerEntry(ListenerId aId, Listener* aListener) + : mId(aId) + , mListener(aListener) + { + } + + ListenerId mId; + Listener* mListener; + }; + + class ListenerEntryIdComparator + { + public: + bool Equals(const ListenerEntry& aA, const ListenerId& aB) const + { + return aA.mId == aB; + } + }; + + class ListenerEntryListenerComparator + { + public: + bool Equals(const ListenerEntry& aA, const Listener* aB) const + { + return aA.mListener == aB; + } + }; + + typedef nsTArray<ListenerEntry> ListenerList; + ListenerList mListeners; + static ListenerId sNextListenerId; + + // Weak references cleared by RemoveStreamList() in StreamList destructors. + nsTArray<StreamList*> mStreamLists; + + bool mShuttingDown; + State mState; + + struct CacheIdRefCounter + { + CacheId mCacheId; + MozRefCountType mCount; + bool mOrphaned; + }; + nsTArray<CacheIdRefCounter> mCacheIdRefs; + + struct BodyIdRefCounter + { + nsID mBodyId; + MozRefCountType mCount; + bool mOrphaned; + }; + nsTArray<BodyIdRefCounter> mBodyIdRefs; + +public: + NS_INLINE_DECL_REFCOUNTING(cache::Manager) +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_Manager_h diff --git a/dom/cache/ManagerId.cpp b/dom/cache/ManagerId.cpp new file mode 100644 index 0000000000..43b46410c1 --- /dev/null +++ b/dom/cache/ManagerId.cpp @@ -0,0 +1,74 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/ManagerId.h" +#include "mozilla/dom/quota/QuotaManager.h" +#include "nsIPrincipal.h" +#include "nsProxyRelease.h" +#include "mozilla/RefPtr.h" +#include "nsThreadUtils.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::dom::quota::QuotaManager; + +// static +nsresult +ManagerId::Create(nsIPrincipal* aPrincipal, ManagerId** aManagerIdOut) +{ + MOZ_ASSERT(NS_IsMainThread()); + + // The QuotaManager::GetInfoFromPrincipal() has special logic for system + // and about: principals. We need to use the same modified origin in + // order to interpret calls from QM correctly. + nsCString quotaOrigin; + nsresult rv = QuotaManager::GetInfoFromPrincipal(aPrincipal, + nullptr, // suffix + nullptr, // group + "aOrigin, + nullptr); // is app + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + RefPtr<ManagerId> ref = new ManagerId(aPrincipal, quotaOrigin); + ref.forget(aManagerIdOut); + + return NS_OK; +} + +already_AddRefed<nsIPrincipal> +ManagerId::Principal() const +{ + MOZ_ASSERT(NS_IsMainThread()); + nsCOMPtr<nsIPrincipal> ref = mPrincipal; + return ref.forget(); +} + +ManagerId::ManagerId(nsIPrincipal* aPrincipal, const nsACString& aQuotaOrigin) + : mPrincipal(aPrincipal) + , mQuotaOrigin(aQuotaOrigin) +{ + MOZ_DIAGNOSTIC_ASSERT(mPrincipal); +} + +ManagerId::~ManagerId() +{ + // If we're already on the main thread, then default destruction is fine + if (NS_IsMainThread()) { + return; + } + + // Otherwise we need to proxy to main thread to do the release + + // The PBackground worker thread shouldn't be running after the main thread + // is stopped. So main thread is guaranteed to exist here. + NS_ReleaseOnMainThread(mPrincipal.forget()); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/ManagerId.h b/dom/cache/ManagerId.h new file mode 100644 index 0000000000..445520bb72 --- /dev/null +++ b/dom/cache/ManagerId.h @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_ManagerId_h +#define mozilla_dom_cache_ManagerId_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/cache/Types.h" +#include "nsCOMPtr.h" +#include "nsError.h" +#include "nsISupportsImpl.h" +#include "nsString.h" + +class nsIPrincipal; + +namespace mozilla { +namespace dom { +namespace cache { + +class ManagerId final +{ +public: + // Main thread only + static nsresult Create(nsIPrincipal* aPrincipal, ManagerId** aManagerIdOut); + + // Main thread only + already_AddRefed<nsIPrincipal> Principal() const; + + const nsACString& QuotaOrigin() const { return mQuotaOrigin; } + + bool operator==(const ManagerId& aOther) const + { + return mQuotaOrigin == aOther.mQuotaOrigin; + } + +private: + ManagerId(nsIPrincipal* aPrincipal, const nsACString& aOrigin); + ~ManagerId(); + + ManagerId(const ManagerId&) = delete; + ManagerId& operator=(const ManagerId&) = delete; + + // only accessible on main thread + nsCOMPtr<nsIPrincipal> mPrincipal; + + // immutable to allow threadsfe access + const nsCString mQuotaOrigin; + +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(mozilla::dom::cache::ManagerId) +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_ManagerId_h diff --git a/dom/cache/PCache.ipdl b/dom/cache/PCache.ipdl new file mode 100644 index 0000000000..f9040009be --- /dev/null +++ b/dom/cache/PCache.ipdl @@ -0,0 +1,33 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PBackground; +include protocol PBlob; // FIXME: bug 792908 +include protocol PCacheOp; +include protocol PCacheStreamControl; +include protocol PFileDescriptorSet; +include protocol PSendStream; + +include CacheTypes; + +namespace mozilla { +namespace dom { +namespace cache { + +protocol PCache +{ + manager PBackground; + manages PCacheOp; + +parent: + async PCacheOp(CacheOpArgs aOpArgs); + async Teardown(); + +child: + async __delete__(); +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/PCacheOp.ipdl b/dom/cache/PCacheOp.ipdl new file mode 100644 index 0000000000..cc5aa93575 --- /dev/null +++ b/dom/cache/PCacheOp.ipdl @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PCache; +include protocol PCacheStorage; +include protocol PCacheStreamControl; +include protocol PFileDescriptorSet; +include protocol PSendStream; + +include CacheTypes; + +using mozilla::ErrorResult from "ipc/ErrorIPCUtils.h"; + +namespace mozilla { +namespace dom { +namespace cache { + +protocol PCacheOp +{ + manager PCache or PCacheStorage; + +child: + async __delete__(ErrorResult aRv, CacheOpResult aResult); +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/PCacheStorage.ipdl b/dom/cache/PCacheStorage.ipdl new file mode 100644 index 0000000000..4a1345393f --- /dev/null +++ b/dom/cache/PCacheStorage.ipdl @@ -0,0 +1,34 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PBackground; +include protocol PBlob; // FIXME: bug 792908 +include protocol PCache; +include protocol PCacheOp; +include protocol PCacheStreamControl; +include protocol PFileDescriptorSet; +include protocol PSendStream; + +include CacheTypes; + +namespace mozilla { +namespace dom { +namespace cache { + +protocol PCacheStorage +{ + manager PBackground; + manages PCacheOp; + +parent: + async PCacheOp(CacheOpArgs aOpArgs); + async Teardown(); + +child: + async __delete__(); +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/PCacheStreamControl.ipdl b/dom/cache/PCacheStreamControl.ipdl new file mode 100644 index 0000000000..f42b8f9e4f --- /dev/null +++ b/dom/cache/PCacheStreamControl.ipdl @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PBackground; + +using struct nsID from "nsID.h"; + +namespace mozilla { +namespace dom { +namespace cache { + +protocol PCacheStreamControl +{ + manager PBackground; + +parent: + async NoteClosed(nsID aStreamId); + +child: + async Close(nsID aStreamId); + async CloseAll(); + async __delete__(); +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/PrincipalVerifier.cpp b/dom/cache/PrincipalVerifier.cpp new file mode 100644 index 0000000000..c9b410a923 --- /dev/null +++ b/dom/cache/PrincipalVerifier.cpp @@ -0,0 +1,222 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/PrincipalVerifier.h" + +#include "mozilla/AppProcessChecker.h" +#include "mozilla/dom/ContentParent.h" +#include "mozilla/dom/cache/ManagerId.h" +#include "mozilla/ipc/BackgroundParent.h" +#include "mozilla/ipc/PBackgroundParent.h" +#include "mozilla/ipc/BackgroundUtils.h" +#include "nsContentUtils.h" +#include "nsIPrincipal.h" +#include "nsIScriptSecurityManager.h" +#include "nsNetUtil.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::ipc::AssertIsOnBackgroundThread; +using mozilla::ipc::BackgroundParent; +using mozilla::ipc::PBackgroundParent; +using mozilla::ipc::PrincipalInfo; +using mozilla::ipc::PrincipalInfoToPrincipal; + +// static +already_AddRefed<PrincipalVerifier> +PrincipalVerifier::CreateAndDispatch(Listener* aListener, + PBackgroundParent* aActor, + const PrincipalInfo& aPrincipalInfo) +{ + // We must get the ContentParent actor from the PBackgroundParent. This + // only works on the PBackground thread. + AssertIsOnBackgroundThread(); + + RefPtr<PrincipalVerifier> verifier = new PrincipalVerifier(aListener, + aActor, + aPrincipalInfo); + + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(verifier)); + + return verifier.forget(); +} + +void +PrincipalVerifier::AddListener(Listener* aListener) +{ + AssertIsOnBackgroundThread(); + MOZ_DIAGNOSTIC_ASSERT(aListener); + MOZ_ASSERT(!mListenerList.Contains(aListener)); + mListenerList.AppendElement(aListener); +} + +void +PrincipalVerifier::RemoveListener(Listener* aListener) +{ + AssertIsOnBackgroundThread(); + MOZ_DIAGNOSTIC_ASSERT(aListener); + MOZ_ALWAYS_TRUE(mListenerList.RemoveElement(aListener)); +} + +PrincipalVerifier::PrincipalVerifier(Listener* aListener, + PBackgroundParent* aActor, + const PrincipalInfo& aPrincipalInfo) + : mActor(BackgroundParent::GetContentParent(aActor)) + , mPrincipalInfo(aPrincipalInfo) + , mInitiatingThread(NS_GetCurrentThread()) + , mResult(NS_OK) +{ + AssertIsOnBackgroundThread(); + MOZ_DIAGNOSTIC_ASSERT(mInitiatingThread); + MOZ_DIAGNOSTIC_ASSERT(aListener); + + mListenerList.AppendElement(aListener); +} + +PrincipalVerifier::~PrincipalVerifier() +{ + // Since the PrincipalVerifier is a Runnable that executes on multiple + // threads, its a race to see which thread de-refs us last. Therefore + // we cannot guarantee which thread we destruct on. + + MOZ_DIAGNOSTIC_ASSERT(mListenerList.IsEmpty()); + + // We should always be able to explicitly release the actor on the main + // thread. + MOZ_DIAGNOSTIC_ASSERT(!mActor); +} + +NS_IMETHODIMP +PrincipalVerifier::Run() +{ + // Executed twice. First, on the main thread and then back on the + // originating thread. + + if (NS_IsMainThread()) { + VerifyOnMainThread(); + return NS_OK; + } + + CompleteOnInitiatingThread(); + return NS_OK; +} + +void +PrincipalVerifier::VerifyOnMainThread() +{ + MOZ_ASSERT(NS_IsMainThread()); + + // No matter what happens, we need to release the actor before leaving + // this method. + RefPtr<ContentParent> actor; + actor.swap(mActor); + + nsresult rv; + RefPtr<nsIPrincipal> principal = PrincipalInfoToPrincipal(mPrincipalInfo, + &rv); + if (NS_WARN_IF(NS_FAILED(rv))) { + DispatchToInitiatingThread(rv); + return; + } + + // We disallow null principal and unknown app IDs on the client side, but + // double-check here. + if (NS_WARN_IF(principal->GetIsNullPrincipal() || + principal->GetUnknownAppId())) { + DispatchToInitiatingThread(NS_ERROR_FAILURE); + return; + } + + nsCOMPtr<nsIScriptSecurityManager> ssm = nsContentUtils::GetSecurityManager(); + if (NS_WARN_IF(!ssm)) { + DispatchToInitiatingThread(NS_ERROR_ILLEGAL_DURING_SHUTDOWN); + return; + } + + // Verify if a child process uses system principal, which is not allowed + // to prevent system principal is spoofed. + if (NS_WARN_IF(actor && ssm->IsSystemPrincipal(principal))) { + DispatchToInitiatingThread(NS_ERROR_FAILURE); + return; + } + + // Verify that a child process claims to own the app for this principal + if (NS_WARN_IF(actor && !AssertAppPrincipal(actor, principal))) { + DispatchToInitiatingThread(NS_ERROR_FAILURE); + return; + } + actor = nullptr; + +#ifdef DEBUG + // Sanity check principal origin by using it to construct a URI and security + // checking it. Don't do this for the system principal, though, as its origin + // is a synthetic [System Principal] string. + if (!ssm->IsSystemPrincipal(principal)) { + nsAutoCString origin; + rv = principal->GetOriginNoSuffix(origin); + if (NS_WARN_IF(NS_FAILED(rv))) { + DispatchToInitiatingThread(rv); + return; + } + nsCOMPtr<nsIURI> uri; + rv = NS_NewURI(getter_AddRefs(uri), origin); + if (NS_WARN_IF(NS_FAILED(rv))) { + DispatchToInitiatingThread(rv); + return; + } + rv = principal->CheckMayLoad(uri, false, false); + if (NS_WARN_IF(NS_FAILED(rv))) { + DispatchToInitiatingThread(rv); + return; + } + } +#endif + + rv = ManagerId::Create(principal, getter_AddRefs(mManagerId)); + if (NS_WARN_IF(NS_FAILED(rv))) { + DispatchToInitiatingThread(rv); + return; + } + + DispatchToInitiatingThread(NS_OK); +} + +void +PrincipalVerifier::CompleteOnInitiatingThread() +{ + AssertIsOnBackgroundThread(); + ListenerList::ForwardIterator iter(mListenerList); + while (iter.HasMore()) { + iter.GetNext()->OnPrincipalVerified(mResult, mManagerId); + } + + // The listener must clear its reference in OnPrincipalVerified() + MOZ_DIAGNOSTIC_ASSERT(mListenerList.IsEmpty()); +} + +void +PrincipalVerifier::DispatchToInitiatingThread(nsresult aRv) +{ + MOZ_ASSERT(NS_IsMainThread()); + + mResult = aRv; + + // The Cache ShutdownObserver does not track all principal verifiers, so we + // cannot ensure this always succeeds. Instead, simply warn on failures. + // This will result in a new CacheStorage object delaying operations until + // shutdown completes and the browser goes away. This is as graceful as + // we can get here. + nsresult rv = mInitiatingThread->Dispatch(this, nsIThread::DISPATCH_NORMAL); + if (NS_FAILED(rv)) { + NS_WARNING("Cache unable to complete principal verification due to shutdown."); + } +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/PrincipalVerifier.h b/dom/cache/PrincipalVerifier.h new file mode 100644 index 0000000000..d9bc980054 --- /dev/null +++ b/dom/cache/PrincipalVerifier.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_PrincipalVerifier_h +#define mozilla_dom_cache_PrincipalVerifier_h + +#include "mozilla/ipc/PBackgroundSharedTypes.h" +#include "nsThreadUtils.h" +#include "nsTObserverArray.h" + +namespace mozilla { + +namespace ipc { + class PBackgroundParent; +} // namespace ipc + +namespace dom { +namespace cache { + +class ManagerId; + +class PrincipalVerifier final : public Runnable +{ +public: + // An interface to be implemented by code wishing to use the + // PrincipalVerifier. Note, the Listener implementation is responsible + // for calling RemoveListener() on the PrincipalVerifier to clear the + // weak reference. + class Listener + { + public: + virtual void OnPrincipalVerified(nsresult aRv, ManagerId* aManagerId) = 0; + }; + + static already_AddRefed<PrincipalVerifier> + CreateAndDispatch(Listener* aListener, mozilla::ipc::PBackgroundParent* aActor, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo); + + void AddListener(Listener* aListener); + + // The Listener must call RemoveListener() when OnPrincipalVerified() is + // called or when the Listener is destroyed. + void RemoveListener(Listener* aListener); + +private: + PrincipalVerifier(Listener* aListener, mozilla::ipc::PBackgroundParent* aActor, + const mozilla::ipc::PrincipalInfo& aPrincipalInfo); + virtual ~PrincipalVerifier(); + + void VerifyOnMainThread(); + void CompleteOnInitiatingThread(); + + void DispatchToInitiatingThread(nsresult aRv); + + // Weak reference cleared by RemoveListener() + typedef nsTObserverArray<Listener*> ListenerList; + ListenerList mListenerList; + + // set in originating thread at construction, but must be accessed and + // released on main thread + RefPtr<ContentParent> mActor; + + const mozilla::ipc::PrincipalInfo mPrincipalInfo; + nsCOMPtr<nsIThread> mInitiatingThread; + nsresult mResult; + RefPtr<ManagerId> mManagerId; + +public: + NS_DECL_NSIRUNNABLE +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_PrincipalVerifier_h diff --git a/dom/cache/QuotaClient.cpp b/dom/cache/QuotaClient.cpp new file mode 100644 index 0000000000..5641c953c6 --- /dev/null +++ b/dom/cache/QuotaClient.cpp @@ -0,0 +1,243 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/QuotaClient.h" + +#include "mozilla/dom/cache/Manager.h" +#include "mozilla/dom/quota/QuotaManager.h" +#include "mozilla/dom/quota/UsageInfo.h" +#include "mozilla/ipc/BackgroundParent.h" +#include "nsIFile.h" +#include "nsISimpleEnumerator.h" +#include "nsThreadUtils.h" + +namespace { + +using mozilla::Atomic; +using mozilla::dom::ContentParentId; +using mozilla::dom::cache::Manager; +using mozilla::dom::quota::Client; +using mozilla::dom::quota::PersistenceType; +using mozilla::dom::quota::QuotaManager; +using mozilla::dom::quota::UsageInfo; +using mozilla::ipc::AssertIsOnBackgroundThread; + +static nsresult +GetBodyUsage(nsIFile* aDir, const Atomic<bool>& aCanceled, + UsageInfo* aUsageInfo) +{ + nsCOMPtr<nsISimpleEnumerator> entries; + nsresult rv = aDir->GetDirectoryEntries(getter_AddRefs(entries)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMore; + while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore && + !aCanceled) { + nsCOMPtr<nsISupports> entry; + rv = entries->GetNext(getter_AddRefs(entry)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsIFile> file = do_QueryInterface(entry); + + bool isDir; + rv = file->IsDirectory(&isDir); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (isDir) { + rv = GetBodyUsage(file, aCanceled, aUsageInfo); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + continue; + } + + int64_t fileSize; + rv = file->GetFileSize(&fileSize); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + MOZ_DIAGNOSTIC_ASSERT(fileSize >= 0); + + aUsageInfo->AppendToFileUsage(fileSize); + } + + return NS_OK; +} + +class CacheQuotaClient final : public Client +{ +public: + virtual Type + GetType() override + { + return DOMCACHE; + } + + virtual nsresult + InitOrigin(PersistenceType aPersistenceType, const nsACString& aGroup, + const nsACString& aOrigin, const AtomicBool& aCanceled, + UsageInfo* aUsageInfo) override + { + // The QuotaManager passes a nullptr UsageInfo if there is no quota being + // enforced against the origin. + if (!aUsageInfo) { + return NS_OK; + } + + return GetUsageForOrigin(aPersistenceType, aGroup, aOrigin, aCanceled, + aUsageInfo); + } + + virtual nsresult + GetUsageForOrigin(PersistenceType aPersistenceType, const nsACString& aGroup, + const nsACString& aOrigin, const AtomicBool& aCanceled, + UsageInfo* aUsageInfo) override + { + MOZ_DIAGNOSTIC_ASSERT(aUsageInfo); + + QuotaManager* qm = QuotaManager::Get(); + MOZ_DIAGNOSTIC_ASSERT(qm); + + nsCOMPtr<nsIFile> dir; + nsresult rv = qm->GetDirectoryForOrigin(aPersistenceType, aOrigin, + getter_AddRefs(dir)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + rv = dir->Append(NS_LITERAL_STRING(DOMCACHE_DIRECTORY_NAME)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsISimpleEnumerator> entries; + rv = dir->GetDirectoryEntries(getter_AddRefs(entries)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool hasMore; + while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore && + !aCanceled) { + nsCOMPtr<nsISupports> entry; + rv = entries->GetNext(getter_AddRefs(entry)); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + nsCOMPtr<nsIFile> file = do_QueryInterface(entry); + + nsAutoString leafName; + rv = file->GetLeafName(leafName); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + bool isDir; + rv = file->IsDirectory(&isDir); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + + if (isDir) { + if (leafName.EqualsLiteral("morgue")) { + rv = GetBodyUsage(file, aCanceled, aUsageInfo); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + } else { + NS_WARNING("Unknown Cache directory found!"); + } + + continue; + } + + // Ignore transient sqlite files and marker files + if (leafName.EqualsLiteral("caches.sqlite-journal") || + leafName.EqualsLiteral("caches.sqlite-shm") || + leafName.Find(NS_LITERAL_CSTRING("caches.sqlite-mj"), false, 0, 0) == 0 || + leafName.EqualsLiteral("context_open.marker")) { + continue; + } + + if (leafName.EqualsLiteral("caches.sqlite") || + leafName.EqualsLiteral("caches.sqlite-wal")) { + int64_t fileSize; + rv = file->GetFileSize(&fileSize); + if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } + MOZ_DIAGNOSTIC_ASSERT(fileSize >= 0); + + aUsageInfo->AppendToDatabaseUsage(fileSize); + continue; + } + + NS_WARNING("Unknown Cache file found!"); + } + + return NS_OK; + } + + virtual void + OnOriginClearCompleted(PersistenceType aPersistenceType, + const nsACString& aOrigin) override + { + // Nothing to do here. + } + + virtual void + ReleaseIOThreadObjects() override + { + // Nothing to do here as the Context handles cleaning everything up + // automatically. + } + + virtual void + AbortOperations(const nsACString& aOrigin) override + { + AssertIsOnBackgroundThread(); + + Manager::Abort(aOrigin); + } + + virtual void + AbortOperationsForProcess(ContentParentId aContentParentId) override + { + // The Cache and Context can be shared by multiple client processes. They + // are not exclusively owned by a single process. + // + // As far as I can tell this is used by QuotaManager to abort operations + // when a particular process goes away. We definitely don't want this + // since we are shared. Also, the Cache actor code already properly + // handles asynchronous actor destruction when the child process dies. + // + // Therefore, do nothing here. + } + + virtual void + StartIdleMaintenance() override + { } + + virtual void + StopIdleMaintenance() override + { } + + virtual void + ShutdownWorkThreads() override + { + AssertIsOnBackgroundThread(); + + // spins the event loop and synchronously shuts down all Managers + Manager::ShutdownAll(); + } + +private: + ~CacheQuotaClient() + { + AssertIsOnBackgroundThread(); + } + + NS_INLINE_DECL_REFCOUNTING(CacheQuotaClient, override) +}; + +} // namespace + +namespace mozilla { +namespace dom { +namespace cache { + +already_AddRefed<quota::Client> CreateQuotaClient() +{ + AssertIsOnBackgroundThread(); + + RefPtr<CacheQuotaClient> ref = new CacheQuotaClient(); + return ref.forget(); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/QuotaClient.h b/dom/cache/QuotaClient.h new file mode 100644 index 0000000000..25b9c1c2db --- /dev/null +++ b/dom/cache/QuotaClient.h @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_QuotaClient_h +#define mozilla_dom_cache_QuotaClient_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/quota/Client.h" + +namespace mozilla { +namespace dom { +namespace cache { + +already_AddRefed<quota::Client> +CreateQuotaClient(); + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_QuotaClient_h diff --git a/dom/cache/ReadStream.cpp b/dom/cache/ReadStream.cpp new file mode 100644 index 0000000000..efce027a19 --- /dev/null +++ b/dom/cache/ReadStream.cpp @@ -0,0 +1,578 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/ReadStream.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/cache/CacheStreamControlChild.h" +#include "mozilla/dom/cache/CacheStreamControlParent.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/ipc/IPCStreamUtils.h" +#include "mozilla/SnappyUncompressInputStream.h" +#include "nsIAsyncInputStream.h" +#include "nsTArray.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::Unused; +using mozilla::ipc::AutoIPCStream; +using mozilla::ipc::IPCStream; + +// ---------------------------------------------------------------------------- + +// The inner stream class. This is where all of the real work is done. As +// an invariant Inner::Close() must be called before ~Inner(). This is +// guaranteed by our outer ReadStream class. +class ReadStream::Inner final : public ReadStream::Controllable +{ +public: + Inner(StreamControl* aControl, const nsID& aId, + nsIInputStream* aStream); + + void + Serialize(CacheReadStreamOrVoid* aReadStreamOut, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv); + + void + Serialize(CacheReadStream* aReadStreamOut, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv); + + // ReadStream::Controllable methods + virtual void + CloseStream() override; + + virtual void + CloseStreamWithoutReporting() override; + + virtual bool + MatchId(const nsID& aId) const override; + + virtual bool + HasEverBeenRead() const override; + + // Simulate nsIInputStream methods, but we don't actually inherit from it + nsresult + Close(); + + nsresult + Available(uint64_t *aNumAvailableOut); + + nsresult + Read(char *aBuf, uint32_t aCount, uint32_t *aNumReadOut); + + nsresult + ReadSegments(nsWriteSegmentFun aWriter, void *aClosure, uint32_t aCount, + uint32_t *aNumReadOut); + + nsresult + IsNonBlocking(bool *aNonBlockingOut); + +private: + class NoteClosedRunnable; + class ForgetRunnable; + + ~Inner(); + + void + NoteClosed(); + + void + Forget(); + + void + NoteClosedOnOwningThread(); + + void + ForgetOnOwningThread(); + + // Weak ref to the stream control actor. The actor will always call either + // CloseStream() or CloseStreamWithoutReporting() before it's destroyed. The + // weak ref is cleared in the resulting NoteClosedOnOwningThread() or + // ForgetOnOwningThread() method call. + StreamControl* mControl; + + const nsID mId; + nsCOMPtr<nsIThread> mOwningThread; + + enum State + { + Open, + Closed, + NumStates + }; + Atomic<State> mState; + Atomic<bool> mHasEverBeenRead; + + + // The wrapped stream objects may not be threadsafe. We need to be able + // to close a stream on our owning thread while an IO thread is simultaneously + // reading the same stream. Therefore, protect all access to these stream + // objects with a mutex. + Mutex mMutex; + nsCOMPtr<nsIInputStream> mStream; + nsCOMPtr<nsIInputStream> mSnappyStream; + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(cache::ReadStream::Inner, override) +}; + +// ---------------------------------------------------------------------------- + +// Runnable to notify actors that the ReadStream has closed. This must +// be done on the thread associated with the PBackground actor. Must be +// cancelable to execute on Worker threads (which can occur when the +// ReadStream is constructed on a child process Worker thread). +class ReadStream::Inner::NoteClosedRunnable final : public CancelableRunnable +{ +public: + explicit NoteClosedRunnable(ReadStream::Inner* aStream) + : mStream(aStream) + { } + + NS_IMETHOD Run() override + { + mStream->NoteClosedOnOwningThread(); + mStream = nullptr; + return NS_OK; + } + + // Note, we must proceed with the Run() method since our actor will not + // clean itself up until we note that the stream is closed. + nsresult Cancel() override + { + Run(); + return NS_OK; + } + +private: + ~NoteClosedRunnable() { } + + RefPtr<ReadStream::Inner> mStream; +}; + +// ---------------------------------------------------------------------------- + +// Runnable to clear actors without reporting that the ReadStream has +// closed. Since this can trigger actor destruction, we need to do +// it on the thread associated with the PBackground actor. Must be +// cancelable to execute on Worker threads (which can occur when the +// ReadStream is constructed on a child process Worker thread). +class ReadStream::Inner::ForgetRunnable final : public CancelableRunnable +{ +public: + explicit ForgetRunnable(ReadStream::Inner* aStream) + : mStream(aStream) + { } + + NS_IMETHOD Run() override + { + mStream->ForgetOnOwningThread(); + mStream = nullptr; + return NS_OK; + } + + // Note, we must proceed with the Run() method so that we properly + // call RemoveListener on the actor. + nsresult Cancel() override + { + Run(); + return NS_OK; + } + +private: + ~ForgetRunnable() { } + + RefPtr<ReadStream::Inner> mStream; +}; + +// ---------------------------------------------------------------------------- + +ReadStream::Inner::Inner(StreamControl* aControl, const nsID& aId, + nsIInputStream* aStream) + : mControl(aControl) + , mId(aId) + , mOwningThread(NS_GetCurrentThread()) + , mState(Open) + , mHasEverBeenRead(false) + , mMutex("dom::cache::ReadStream") + , mStream(aStream) + , mSnappyStream(new SnappyUncompressInputStream(aStream)) +{ + MOZ_DIAGNOSTIC_ASSERT(mStream); + MOZ_DIAGNOSTIC_ASSERT(mControl); + mControl->AddReadStream(this); +} + +void +ReadStream::Inner::Serialize(CacheReadStreamOrVoid* aReadStreamOut, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv) +{ + MOZ_ASSERT(NS_GetCurrentThread() == mOwningThread); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + *aReadStreamOut = CacheReadStream(); + Serialize(&aReadStreamOut->get_CacheReadStream(), aStreamCleanupList, aRv); +} + +void +ReadStream::Inner::Serialize(CacheReadStream* aReadStreamOut, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv) +{ + MOZ_ASSERT(NS_GetCurrentThread() == mOwningThread); + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut); + + if (mState != Open) { + aRv.ThrowTypeError<MSG_CACHE_STREAM_CLOSED>(); + return; + } + + MOZ_DIAGNOSTIC_ASSERT(mControl); + + aReadStreamOut->id() = mId; + mControl->SerializeControl(aReadStreamOut); + + { + MutexAutoLock lock(mMutex); + mControl->SerializeStream(aReadStreamOut, mStream, aStreamCleanupList); + } + + MOZ_DIAGNOSTIC_ASSERT(aReadStreamOut->stream().type() == + IPCStream::TInputStreamParamsWithFds); + + // We're passing ownership across the IPC barrier with the control, so + // do not signal that the stream is closed here. + Forget(); +} + +void +ReadStream::Inner::CloseStream() +{ + MOZ_ASSERT(NS_GetCurrentThread() == mOwningThread); + Close(); +} + +void +ReadStream::Inner::CloseStreamWithoutReporting() +{ + MOZ_ASSERT(NS_GetCurrentThread() == mOwningThread); + Forget(); +} + +bool +ReadStream::Inner::MatchId(const nsID& aId) const +{ + MOZ_ASSERT(NS_GetCurrentThread() == mOwningThread); + return mId.Equals(aId); +} + +bool +ReadStream::Inner::HasEverBeenRead() const +{ + MOZ_ASSERT(NS_GetCurrentThread() == mOwningThread); + return mHasEverBeenRead; +} + +nsresult +ReadStream::Inner::Close() +{ + // stream ops can happen on any thread + nsresult rv = NS_OK; + { + MutexAutoLock lock(mMutex); + rv = mSnappyStream->Close(); + } + NoteClosed(); + return rv; +} + +nsresult +ReadStream::Inner::Available(uint64_t* aNumAvailableOut) +{ + // stream ops can happen on any thread + nsresult rv = NS_OK; + { + MutexAutoLock lock(mMutex); + rv = mSnappyStream->Available(aNumAvailableOut); + } + + if (NS_FAILED(rv)) { + Close(); + } + + return rv; +} + +nsresult +ReadStream::Inner::Read(char* aBuf, uint32_t aCount, uint32_t* aNumReadOut) +{ + // stream ops can happen on any thread + MOZ_DIAGNOSTIC_ASSERT(aNumReadOut); + + nsresult rv = NS_OK; + { + MutexAutoLock lock(mMutex); + rv = mSnappyStream->Read(aBuf, aCount, aNumReadOut); + } + + if ((NS_FAILED(rv) && rv != NS_BASE_STREAM_WOULD_BLOCK) || + *aNumReadOut == 0) { + Close(); + } + + mHasEverBeenRead = true; + + return rv; +} + +nsresult +ReadStream::Inner::ReadSegments(nsWriteSegmentFun aWriter, void* aClosure, + uint32_t aCount, uint32_t* aNumReadOut) +{ + // stream ops can happen on any thread + MOZ_DIAGNOSTIC_ASSERT(aNumReadOut); + + if (aCount) { + mHasEverBeenRead = true; + } + + + nsresult rv = NS_OK; + { + MutexAutoLock lock(mMutex); + rv = mSnappyStream->ReadSegments(aWriter, aClosure, aCount, aNumReadOut); + } + + if ((NS_FAILED(rv) && rv != NS_BASE_STREAM_WOULD_BLOCK && + rv != NS_ERROR_NOT_IMPLEMENTED) || *aNumReadOut == 0) { + Close(); + } + + // Verify bytes were actually read before marking as being ever read. For + // example, code can test if the stream supports ReadSegments() by calling + // this method with a dummy callback which doesn't read anything. We don't + // want to trigger on that. + if (*aNumReadOut) { + mHasEverBeenRead = true; + } + + return rv; +} + +nsresult +ReadStream::Inner::IsNonBlocking(bool* aNonBlockingOut) +{ + // stream ops can happen on any thread + MutexAutoLock lock(mMutex); + return mSnappyStream->IsNonBlocking(aNonBlockingOut); +} + +ReadStream::Inner::~Inner() +{ + // Any thread + MOZ_DIAGNOSTIC_ASSERT(mState == Closed); + MOZ_DIAGNOSTIC_ASSERT(!mControl); +} + +void +ReadStream::Inner::NoteClosed() +{ + // Any thread + if (mState == Closed) { + return; + } + + if (NS_GetCurrentThread() == mOwningThread) { + NoteClosedOnOwningThread(); + return; + } + + nsCOMPtr<nsIRunnable> runnable = new NoteClosedRunnable(this); + MOZ_ALWAYS_SUCCEEDS( + mOwningThread->Dispatch(runnable, nsIThread::DISPATCH_NORMAL)); +} + +void +ReadStream::Inner::Forget() +{ + // Any thread + if (mState == Closed) { + return; + } + + if (NS_GetCurrentThread() == mOwningThread) { + ForgetOnOwningThread(); + return; + } + + nsCOMPtr<nsIRunnable> runnable = new ForgetRunnable(this); + MOZ_ALWAYS_SUCCEEDS( + mOwningThread->Dispatch(runnable, nsIThread::DISPATCH_NORMAL)); +} + +void +ReadStream::Inner::NoteClosedOnOwningThread() +{ + MOZ_ASSERT(NS_GetCurrentThread() == mOwningThread); + + // Mark closed and do nothing if we were already closed + if (!mState.compareExchange(Open, Closed)) { + return; + } + + MOZ_DIAGNOSTIC_ASSERT(mControl); + mControl->NoteClosed(this, mId); + mControl = nullptr; +} + +void +ReadStream::Inner::ForgetOnOwningThread() +{ + MOZ_ASSERT(NS_GetCurrentThread() == mOwningThread); + + // Mark closed and do nothing if we were already closed + if (!mState.compareExchange(Open, Closed)) { + return; + } + + MOZ_DIAGNOSTIC_ASSERT(mControl); + mControl->ForgetReadStream(this); + mControl = nullptr; +} + +// ---------------------------------------------------------------------------- + +NS_IMPL_ISUPPORTS(cache::ReadStream, nsIInputStream, ReadStream); + +// static +already_AddRefed<ReadStream> +ReadStream::Create(const CacheReadStreamOrVoid& aReadStreamOrVoid) +{ + if (aReadStreamOrVoid.type() == CacheReadStreamOrVoid::Tvoid_t) { + return nullptr; + } + + return Create(aReadStreamOrVoid.get_CacheReadStream()); +} + +// static +already_AddRefed<ReadStream> +ReadStream::Create(const CacheReadStream& aReadStream) +{ + // The parameter may or may not be for a Cache created stream. The way we + // tell is by looking at the stream control actor. If the actor exists, + // then we know the Cache created it. + if (!aReadStream.controlChild() && !aReadStream.controlParent()) { + return nullptr; + } + + MOZ_DIAGNOSTIC_ASSERT(aReadStream.stream().type() == + IPCStream::TInputStreamParamsWithFds); + + // Control is guaranteed to survive this method as ActorDestroy() cannot + // run on this thread until we complete. + StreamControl* control; + if (aReadStream.controlChild()) { + auto actor = static_cast<CacheStreamControlChild*>(aReadStream.controlChild()); + control = actor; + } else { + auto actor = static_cast<CacheStreamControlParent*>(aReadStream.controlParent()); + control = actor; + } + MOZ_DIAGNOSTIC_ASSERT(control); + + nsCOMPtr<nsIInputStream> stream = DeserializeIPCStream(aReadStream.stream()); + MOZ_DIAGNOSTIC_ASSERT(stream); + + // Currently we expect all cache read streams to be blocking file streams. +#if !defined(RELEASE_OR_BETA) + nsCOMPtr<nsIAsyncInputStream> asyncStream = do_QueryInterface(stream); + MOZ_DIAGNOSTIC_ASSERT(!asyncStream); +#endif + + RefPtr<Inner> inner = new Inner(control, aReadStream.id(), stream); + RefPtr<ReadStream> ref = new ReadStream(inner); + return ref.forget(); +} + +// static +already_AddRefed<ReadStream> +ReadStream::Create(PCacheStreamControlParent* aControl, const nsID& aId, + nsIInputStream* aStream) +{ + MOZ_DIAGNOSTIC_ASSERT(aControl); + auto actor = static_cast<CacheStreamControlParent*>(aControl); + RefPtr<Inner> inner = new Inner(actor, aId, aStream); + RefPtr<ReadStream> ref = new ReadStream(inner); + return ref.forget(); +} + +void +ReadStream::Serialize(CacheReadStreamOrVoid* aReadStreamOut, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv) +{ + mInner->Serialize(aReadStreamOut, aStreamCleanupList, aRv); +} + +void +ReadStream::Serialize(CacheReadStream* aReadStreamOut, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv) +{ + mInner->Serialize(aReadStreamOut, aStreamCleanupList, aRv); +} + +ReadStream::ReadStream(ReadStream::Inner* aInner) + : mInner(aInner) +{ + MOZ_DIAGNOSTIC_ASSERT(mInner); +} + +ReadStream::~ReadStream() +{ + // Explicitly close the inner stream so that it does not have to + // deal with implicitly closing at destruction time. + mInner->Close(); +} + +NS_IMETHODIMP +ReadStream::Close() +{ + return mInner->Close(); +} + +NS_IMETHODIMP +ReadStream::Available(uint64_t* aNumAvailableOut) +{ + return mInner->Available(aNumAvailableOut); +} + +NS_IMETHODIMP +ReadStream::Read(char* aBuf, uint32_t aCount, uint32_t* aNumReadOut) +{ + return mInner->Read(aBuf, aCount, aNumReadOut); +} + +NS_IMETHODIMP +ReadStream::ReadSegments(nsWriteSegmentFun aWriter, void* aClosure, + uint32_t aCount, uint32_t* aNumReadOut) +{ + return mInner->ReadSegments(aWriter, aClosure, aCount, aNumReadOut); +} + +NS_IMETHODIMP +ReadStream::IsNonBlocking(bool* aNonBlockingOut) +{ + return mInner->IsNonBlocking(aNonBlockingOut); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/ReadStream.h b/dom/cache/ReadStream.h new file mode 100644 index 0000000000..824cedb029 --- /dev/null +++ b/dom/cache/ReadStream.h @@ -0,0 +1,119 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_ReadStream_h +#define mozilla_dom_cache_ReadStream_h + +#include "mozilla/ipc/FileDescriptor.h" +#include "nsCOMPtr.h" +#include "nsID.h" +#include "nsIInputStream.h" +#include "nsISupportsImpl.h" +#include "mozilla/RefPtr.h" +#include "nsTArrayForwardDeclare.h" + +namespace mozilla { +namespace ipc { +class AutoIPCStream; +} // namespace ipc +namespace dom { +namespace cache { + +class CacheReadStream; +class CacheReadStreamOrVoid; +class PCacheStreamControlParent; + +// IID for the dom::cache::ReadStream interface +#define NS_DOM_CACHE_READSTREAM_IID \ +{0x8e5da7c9, 0x0940, 0x4f1d, \ + {0x97, 0x25, 0x5c, 0x59, 0x38, 0xdd, 0xb9, 0x9f}} + + +// Custom stream class for Request and Response bodies being read from +// a Cache. The main purpose of this class is to report back to the +// Cache's Manager when the stream is closed. This allows the Cache to +// accurately determine when the underlying body file can be deleted, +// etc. +// +// The ReadStream class also provides us with a convenient QI'able +// interface that we can use to pass additional meta-data with the +// stream channel. For example, Cache.put() can detect that the content +// script is passing a Cache-originated-stream back into the Cache +// again. This enables certain optimizations. +class ReadStream final : public nsIInputStream +{ +public: + // Interface that lets the StreamControl classes interact with + // our private inner stream. + class Controllable + { + public: + // Closes the stream, notifies the stream control, and then forgets + // the stream control. + virtual void + CloseStream() = 0; + + // Closes the stream and then forgets the stream control. Does not + // notify. + virtual void + CloseStreamWithoutReporting() = 0; + + virtual bool + MatchId(const nsID& aId) const = 0; + + virtual bool + HasEverBeenRead() const = 0; + + NS_IMETHOD_(MozExternalRefCountType) + AddRef(void) = 0; + + NS_IMETHOD_(MozExternalRefCountType) + Release(void) = 0; + }; + + static already_AddRefed<ReadStream> + Create(const CacheReadStreamOrVoid& aReadStreamOrVoid); + + static already_AddRefed<ReadStream> + Create(const CacheReadStream& aReadStream); + + static already_AddRefed<ReadStream> + Create(PCacheStreamControlParent* aControl, const nsID& aId, + nsIInputStream* aStream); + + void Serialize(CacheReadStreamOrVoid* aReadStreamOut, + nsTArray<UniquePtr<mozilla::ipc::AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv); + void Serialize(CacheReadStream* aReadStreamOut, + nsTArray<UniquePtr<mozilla::ipc::AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv); + +private: + class Inner; + + explicit ReadStream(Inner* aInner); + ~ReadStream(); + + // Hold a strong ref to an inner class that actually implements the + // majority of the stream logic. Before releasing this ref the outer + // ReadStream guarantees it will call Close() on the inner stream. + // This is essential for the inner stream to avoid dealing with the + // implicit close that can happen when a stream is destroyed. + RefPtr<Inner> mInner; + +public: + NS_DECLARE_STATIC_IID_ACCESSOR(NS_DOM_CACHE_READSTREAM_IID); + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIINPUTSTREAM +}; + +NS_DEFINE_STATIC_IID_ACCESSOR(ReadStream, NS_DOM_CACHE_READSTREAM_IID); + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_ReadStream_h diff --git a/dom/cache/SavedTypes.h b/dom/cache/SavedTypes.h new file mode 100644 index 0000000000..9e1f686b4a --- /dev/null +++ b/dom/cache/SavedTypes.h @@ -0,0 +1,45 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_SavedTypes_h +#define mozilla_dom_cache_SavedTypes_h + +// NOTE: This cannot be rolled into Types.h because the IPC dependency. +// breaks webidl unified builds. + +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/Types.h" +#include "nsCOMPtr.h" +#include "nsID.h" +#include "nsIOutputStream.h" + +namespace mozilla { +namespace dom { +namespace cache { + +struct SavedRequest +{ + SavedRequest() : mHasBodyId(false) { mValue.body() = void_t(); } + CacheRequest mValue; + bool mHasBodyId; + nsID mBodyId; + CacheId mCacheId; +}; + +struct SavedResponse +{ + SavedResponse() : mHasBodyId(false) { mValue.body() = void_t(); } + CacheResponse mValue; + bool mHasBodyId; + nsID mBodyId; + CacheId mCacheId; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_SavedTypes_h diff --git a/dom/cache/StreamControl.cpp b/dom/cache/StreamControl.cpp new file mode 100644 index 0000000000..aab1766662 --- /dev/null +++ b/dom/cache/StreamControl.cpp @@ -0,0 +1,105 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/StreamControl.h" + +namespace mozilla { +namespace dom { +namespace cache { + +void +StreamControl::AddReadStream(ReadStream::Controllable* aReadStream) +{ + AssertOwningThread(); + MOZ_DIAGNOSTIC_ASSERT(aReadStream); + MOZ_ASSERT(!mReadStreamList.Contains(aReadStream)); + mReadStreamList.AppendElement(aReadStream); +} + +void +StreamControl::ForgetReadStream(ReadStream::Controllable* aReadStream) +{ + AssertOwningThread(); + MOZ_ALWAYS_TRUE(mReadStreamList.RemoveElement(aReadStream)); +} + +void +StreamControl::NoteClosed(ReadStream::Controllable* aReadStream, + const nsID& aId) +{ + AssertOwningThread(); + ForgetReadStream(aReadStream); + NoteClosedAfterForget(aId); +} + +StreamControl::~StreamControl() +{ + // owning thread only, but can't call virtual AssertOwningThread in destructor + MOZ_DIAGNOSTIC_ASSERT(mReadStreamList.IsEmpty()); +} + +void +StreamControl::CloseReadStreams(const nsID& aId) +{ + AssertOwningThread(); +#if defined(DEBUG) || !defined(RELEASE_OR_BETA) + uint32_t closedCount = 0; +#endif + + ReadStreamList::ForwardIterator iter(mReadStreamList); + while (iter.HasMore()) { + RefPtr<ReadStream::Controllable> stream = iter.GetNext(); + if (stream->MatchId(aId)) { + stream->CloseStream(); +#if defined(DEBUG) || !defined(RELEASE_OR_BETA) + closedCount += 1; +#endif + } + } + + MOZ_DIAGNOSTIC_ASSERT(closedCount > 0); +} + +void +StreamControl::CloseAllReadStreams() +{ + AssertOwningThread(); + + ReadStreamList::ForwardIterator iter(mReadStreamList); + while (iter.HasMore()) { + iter.GetNext()->CloseStream(); + } +} + +void +StreamControl::CloseAllReadStreamsWithoutReporting() +{ + AssertOwningThread(); + + ReadStreamList::ForwardIterator iter(mReadStreamList); + while (iter.HasMore()) { + RefPtr<ReadStream::Controllable> stream = iter.GetNext(); + // Note, we cannot trigger IPC traffic here. So use + // CloseStreamWithoutReporting(). + stream->CloseStreamWithoutReporting(); + } +} + +bool +StreamControl::HasEverBeenRead() const +{ + ReadStreamList::ForwardIterator iter(mReadStreamList); + while (iter.HasMore()) { + if (iter.GetNext()->HasEverBeenRead()) { + return true; + } + } + return false; +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/StreamControl.h b/dom/cache/StreamControl.h new file mode 100644 index 0000000000..c68d91ff4c --- /dev/null +++ b/dom/cache/StreamControl.h @@ -0,0 +1,92 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_StreamControl_h +#define mozilla_dom_cache_StreamControl_h + +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/RefPtr.h" +#include "nsTObserverArray.h" + +struct nsID; + +namespace mozilla { +namespace ipc { +class AutoIPCStream; +} // namespace ipc +namespace dom { +namespace cache { + +class CacheReadStream; + +// Abstract class to help implement the stream control Child and Parent actors. +// This provides an interface to partly help with serialization of IPC types, +// but also an implementation for tracking ReadStream objects. +class StreamControl +{ +public: + // abstract interface that must be implemented by child class + virtual void + SerializeControl(CacheReadStream* aReadStreamOut) = 0; + + virtual void + SerializeStream(CacheReadStream* aReadStreamOut, nsIInputStream* aStream, + nsTArray<UniquePtr<mozilla::ipc::AutoIPCStream>>& aStreamCleanupList) = 0; + + // inherited implementation of the ReadStream::Controllable list + + // Begin controlling the given ReadStream. This causes a strong ref to + // be held by the control. The ReadStream must call NoteClosed() or + // ForgetReadStream() to release this ref. + void + AddReadStream(ReadStream::Controllable* aReadStream); + + // Forget the ReadStream without notifying the actor. + void + ForgetReadStream(ReadStream::Controllable* aReadStream); + + // Forget the ReadStream and then notify the actor the stream is closed. + void + NoteClosed(ReadStream::Controllable* aReadStream, const nsID& aId); + +protected: + ~StreamControl(); + + void + CloseReadStreams(const nsID& aId); + + void + CloseAllReadStreams(); + + void + CloseAllReadStreamsWithoutReporting(); + + bool + HasEverBeenRead() const; + + // protected parts of the abstract interface + virtual void + NoteClosedAfterForget(const nsID& aId) = 0; + +#ifdef DEBUG + virtual void + AssertOwningThread() = 0; +#else + void AssertOwningThread() { } +#endif + +private: + // Hold strong references to ReadStream object. When the stream is closed + // it should call NoteClosed() or ForgetReadStream() to release this ref. + typedef nsTObserverArray<RefPtr<ReadStream::Controllable>> ReadStreamList; + ReadStreamList mReadStreamList; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_StreamControl_h diff --git a/dom/cache/StreamList.cpp b/dom/cache/StreamList.cpp new file mode 100644 index 0000000000..991563a3b5 --- /dev/null +++ b/dom/cache/StreamList.cpp @@ -0,0 +1,175 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/StreamList.h" + +#include "mozilla/dom/cache/CacheStreamControlParent.h" +#include "mozilla/dom/cache/Context.h" +#include "mozilla/dom/cache/Manager.h" +#include "nsIInputStream.h" + +namespace mozilla { +namespace dom { +namespace cache { + +StreamList::StreamList(Manager* aManager, Context* aContext) + : mManager(aManager) + , mContext(aContext) + , mCacheId(INVALID_CACHE_ID) + , mStreamControl(nullptr) + , mActivated(false) +{ + MOZ_DIAGNOSTIC_ASSERT(mManager); + mContext->AddActivity(this); +} + +void +StreamList::SetStreamControl(CacheStreamControlParent* aStreamControl) +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + MOZ_DIAGNOSTIC_ASSERT(aStreamControl); + + // For cases where multiple streams are serialized for a single list + // then the control will get passed multiple times. This is ok, but + // it should be the same control each time. + if (mStreamControl) { + MOZ_DIAGNOSTIC_ASSERT(aStreamControl == mStreamControl); + return; + } + + mStreamControl = aStreamControl; + mStreamControl->SetStreamList(this); +} + +void +StreamList::RemoveStreamControl(CacheStreamControlParent* aStreamControl) +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + MOZ_DIAGNOSTIC_ASSERT(mStreamControl); + MOZ_DIAGNOSTIC_ASSERT(mStreamControl == aStreamControl); + mStreamControl = nullptr; +} + +void +StreamList::Activate(CacheId aCacheId) +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + MOZ_DIAGNOSTIC_ASSERT(!mActivated); + MOZ_DIAGNOSTIC_ASSERT(mCacheId == INVALID_CACHE_ID); + mActivated = true; + mCacheId = aCacheId; + mManager->AddRefCacheId(mCacheId); + mManager->AddStreamList(this); + + for (uint32_t i = 0; i < mList.Length(); ++i) { + mManager->AddRefBodyId(mList[i].mId); + } +} + +void +StreamList::Add(const nsID& aId, nsIInputStream* aStream) +{ + // All streams should be added on IO thread before we set the stream + // control on the owning IPC thread. + MOZ_DIAGNOSTIC_ASSERT(!mStreamControl); + MOZ_DIAGNOSTIC_ASSERT(aStream); + Entry* entry = mList.AppendElement(); + entry->mId = aId; + entry->mStream = aStream; +} + +already_AddRefed<nsIInputStream> +StreamList::Extract(const nsID& aId) +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + for (uint32_t i = 0; i < mList.Length(); ++i) { + if (mList[i].mId == aId) { + return mList[i].mStream.forget(); + } + } + return nullptr; +} + +void +StreamList::NoteClosed(const nsID& aId) +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + for (uint32_t i = 0; i < mList.Length(); ++i) { + if (mList[i].mId == aId) { + mList.RemoveElementAt(i); + mManager->ReleaseBodyId(aId); + break; + } + } + + if (mList.IsEmpty() && mStreamControl) { + mStreamControl->Shutdown(); + } +} + +void +StreamList::NoteClosedAll() +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + for (uint32_t i = 0; i < mList.Length(); ++i) { + mManager->ReleaseBodyId(mList[i].mId); + } + mList.Clear(); + + if (mStreamControl) { + mStreamControl->Shutdown(); + } +} + +void +StreamList::Close(const nsID& aId) +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + if (mStreamControl) { + mStreamControl->Close(aId); + } +} + +void +StreamList::CloseAll() +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + if (mStreamControl) { + mStreamControl->CloseAll(); + } +} + +void +StreamList::Cancel() +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + CloseAll(); +} + +bool +StreamList::MatchesCacheId(CacheId aCacheId) const +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + return aCacheId == mCacheId; +} + +StreamList::~StreamList() +{ + NS_ASSERT_OWNINGTHREAD(StreamList); + MOZ_DIAGNOSTIC_ASSERT(!mStreamControl); + if (mActivated) { + mManager->RemoveStreamList(this); + for (uint32_t i = 0; i < mList.Length(); ++i) { + mManager->ReleaseBodyId(mList[i].mId); + } + mManager->ReleaseCacheId(mCacheId); + } + mContext->RemoveActivity(this); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/StreamList.h b/dom/cache/StreamList.h new file mode 100644 index 0000000000..9ff65d20ba --- /dev/null +++ b/dom/cache/StreamList.h @@ -0,0 +1,68 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_StreamList_h +#define mozilla_dom_cache_StreamList_h + +#include "mozilla/dom/cache/Context.h" +#include "mozilla/dom/cache/Types.h" +#include "mozilla/RefPtr.h" +#include "nsTArray.h" + +class nsIInputStream; + +namespace mozilla { +namespace dom { +namespace cache { + +class CacheStreamControlParent; +class Manager; + +class StreamList final : public Context::Activity +{ +public: + StreamList(Manager* aManager, Context* aContext); + + void SetStreamControl(CacheStreamControlParent* aStreamControl); + void RemoveStreamControl(CacheStreamControlParent* aStreamControl); + + void Activate(CacheId aCacheId); + + void Add(const nsID& aId, nsIInputStream* aStream); + already_AddRefed<nsIInputStream> Extract(const nsID& aId); + + void NoteClosed(const nsID& aId); + void NoteClosedAll(); + void Close(const nsID& aId); + void CloseAll(); + + // Context::Activity methods + virtual void Cancel() override; + virtual bool MatchesCacheId(CacheId aCacheId) const override; + +private: + ~StreamList(); + struct Entry + { + nsID mId; + nsCOMPtr<nsIInputStream> mStream; + }; + RefPtr<Manager> mManager; + RefPtr<Context> mContext; + CacheId mCacheId; + CacheStreamControlParent* mStreamControl; + nsTArray<Entry> mList; + bool mActivated; + +public: + NS_INLINE_DECL_REFCOUNTING(cache::StreamList) +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_StreamList_h diff --git a/dom/cache/TypeUtils.cpp b/dom/cache/TypeUtils.cpp new file mode 100644 index 0000000000..2d4cb30eb2 --- /dev/null +++ b/dom/cache/TypeUtils.cpp @@ -0,0 +1,504 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/cache/TypeUtils.h" + +#include "mozilla/Unused.h" +#include "mozilla/dom/CacheBinding.h" +#include "mozilla/dom/InternalRequest.h" +#include "mozilla/dom/Request.h" +#include "mozilla/dom/Response.h" +#include "mozilla/dom/cache/CacheTypes.h" +#include "mozilla/dom/cache/ReadStream.h" +#include "mozilla/ipc/BackgroundChild.h" +#include "mozilla/ipc/PBackgroundChild.h" +#include "mozilla/ipc/PFileDescriptorSetChild.h" +#include "mozilla/ipc/InputStreamUtils.h" +#include "mozilla/ipc/SendStream.h" +#include "nsCOMPtr.h" +#include "nsIAsyncInputStream.h" +#include "nsIAsyncOutputStream.h" +#include "nsIIPCSerializableInputStream.h" +#include "nsQueryObject.h" +#include "nsPromiseFlatString.h" +#include "nsStreamUtils.h" +#include "nsString.h" +#include "nsURLParsers.h" +#include "nsCRT.h" +#include "nsHttp.h" + +namespace mozilla { +namespace dom { +namespace cache { + +using mozilla::ipc::AutoIPCStream; +using mozilla::ipc::BackgroundChild; +using mozilla::ipc::FileDescriptor; +using mozilla::ipc::PBackgroundChild; +using mozilla::ipc::PFileDescriptorSetChild; + +namespace { + +static bool +HasVaryStar(mozilla::dom::InternalHeaders* aHeaders) +{ + nsCString varyHeaders; + ErrorResult rv; + aHeaders->Get(NS_LITERAL_CSTRING("vary"), varyHeaders, rv); + MOZ_ALWAYS_TRUE(!rv.Failed()); + + char* rawBuffer = varyHeaders.BeginWriting(); + char* token = nsCRT::strtok(rawBuffer, NS_HTTP_HEADER_SEPS, &rawBuffer); + for (; token; + token = nsCRT::strtok(rawBuffer, NS_HTTP_HEADER_SEPS, &rawBuffer)) { + nsDependentCString header(token); + if (header.EqualsLiteral("*")) { + return true; + } + } + return false; +} + +void +ToHeadersEntryList(nsTArray<HeadersEntry>& aOut, InternalHeaders* aHeaders) +{ + MOZ_DIAGNOSTIC_ASSERT(aHeaders); + + AutoTArray<InternalHeaders::Entry, 16> entryList; + aHeaders->GetEntries(entryList); + + for (uint32_t i = 0; i < entryList.Length(); ++i) { + InternalHeaders::Entry& entry = entryList[i]; + aOut.AppendElement(HeadersEntry(entry.mName, entry.mValue)); + } +} + +} // namespace + +already_AddRefed<InternalRequest> +TypeUtils::ToInternalRequest(const RequestOrUSVString& aIn, + BodyAction aBodyAction, ErrorResult& aRv) +{ + if (aIn.IsRequest()) { + Request& request = aIn.GetAsRequest(); + + // Check and set bodyUsed flag immediately because its on Request + // instead of InternalRequest. + CheckAndSetBodyUsed(&request, aBodyAction, aRv); + if (aRv.Failed()) { return nullptr; } + + return request.GetInternalRequest(); + } + + return ToInternalRequest(aIn.GetAsUSVString(), aRv); +} + +already_AddRefed<InternalRequest> +TypeUtils::ToInternalRequest(const OwningRequestOrUSVString& aIn, + BodyAction aBodyAction, ErrorResult& aRv) +{ + + if (aIn.IsRequest()) { + RefPtr<Request> request = aIn.GetAsRequest().get(); + + // Check and set bodyUsed flag immediately because its on Request + // instead of InternalRequest. + CheckAndSetBodyUsed(request, aBodyAction, aRv); + if (aRv.Failed()) { return nullptr; } + + return request->GetInternalRequest(); + } + + return ToInternalRequest(aIn.GetAsUSVString(), aRv); +} + +void +TypeUtils::ToCacheRequest(CacheRequest& aOut, InternalRequest* aIn, + BodyAction aBodyAction, SchemeAction aSchemeAction, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv) +{ + MOZ_DIAGNOSTIC_ASSERT(aIn); + aIn->GetMethod(aOut.method()); + nsCString url(aIn->GetURLWithoutFragment()); + bool schemeValid; + ProcessURL(url, &schemeValid, &aOut.urlWithoutQuery(), &aOut.urlQuery(), aRv); + if (aRv.Failed()) { + return; + } + if (!schemeValid) { + if (aSchemeAction == TypeErrorOnInvalidScheme) { + NS_ConvertUTF8toUTF16 urlUTF16(url); + aRv.ThrowTypeError<MSG_INVALID_URL_SCHEME>(NS_LITERAL_STRING("Request"), + urlUTF16); + return; + } + } + aOut.urlFragment() = aIn->GetFragment(); + + aIn->GetReferrer(aOut.referrer()); + aOut.referrerPolicy() = aIn->ReferrerPolicy_(); + RefPtr<InternalHeaders> headers = aIn->Headers(); + MOZ_DIAGNOSTIC_ASSERT(headers); + ToHeadersEntryList(aOut.headers(), headers); + aOut.headersGuard() = headers->Guard(); + aOut.mode() = aIn->Mode(); + aOut.credentials() = aIn->GetCredentialsMode(); + aOut.contentPolicyType() = aIn->ContentPolicyType(); + aOut.requestCache() = aIn->GetCacheMode(); + aOut.requestRedirect() = aIn->GetRedirectMode(); + + aOut.integrity() = aIn->GetIntegrity(); + + if (aBodyAction == IgnoreBody) { + aOut.body() = void_t(); + return; + } + + // BodyUsed flag is checked and set previously in ToInternalRequest() + + nsCOMPtr<nsIInputStream> stream; + aIn->GetBody(getter_AddRefs(stream)); + SerializeCacheStream(stream, &aOut.body(), aStreamCleanupList, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } +} + +void +TypeUtils::ToCacheResponseWithoutBody(CacheResponse& aOut, + InternalResponse& aIn, ErrorResult& aRv) +{ + aOut.type() = aIn.Type(); + + aIn.GetUnfilteredURLList(aOut.urlList()); + AutoTArray<nsCString, 4> urlList; + aIn.GetURLList(urlList); + + for (uint32_t i = 0; i < aOut.urlList().Length(); i++) { + MOZ_DIAGNOSTIC_ASSERT(!aOut.urlList()[i].IsEmpty()); + // Pass all Response URL schemes through... The spec only requires we take + // action on invalid schemes for Request objects. + ProcessURL(aOut.urlList()[i], nullptr, nullptr, nullptr, aRv); + } + + aOut.status() = aIn.GetUnfilteredStatus(); + aOut.statusText() = aIn.GetUnfilteredStatusText(); + RefPtr<InternalHeaders> headers = aIn.UnfilteredHeaders(); + MOZ_DIAGNOSTIC_ASSERT(headers); + if (HasVaryStar(headers)) { + aRv.ThrowTypeError<MSG_RESPONSE_HAS_VARY_STAR>(); + return; + } + ToHeadersEntryList(aOut.headers(), headers); + aOut.headersGuard() = headers->Guard(); + aOut.channelInfo() = aIn.GetChannelInfo().AsIPCChannelInfo(); + if (aIn.GetPrincipalInfo()) { + aOut.principalInfo() = *aIn.GetPrincipalInfo(); + } else { + aOut.principalInfo() = void_t(); + } +} + +void +TypeUtils::ToCacheResponse(CacheResponse& aOut, Response& aIn, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv) +{ + if (aIn.BodyUsed()) { + aRv.ThrowTypeError<MSG_FETCH_BODY_CONSUMED_ERROR>(); + return; + } + + RefPtr<InternalResponse> ir = aIn.GetInternalResponse(); + ToCacheResponseWithoutBody(aOut, *ir, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + + nsCOMPtr<nsIInputStream> stream; + ir->GetUnfilteredBody(getter_AddRefs(stream)); + if (stream) { + aIn.SetBodyUsed(); + } + + SerializeCacheStream(stream, &aOut.body(), aStreamCleanupList, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } +} + +// static +void +TypeUtils::ToCacheQueryParams(CacheQueryParams& aOut, + const CacheQueryOptions& aIn) +{ + aOut.ignoreSearch() = aIn.mIgnoreSearch; + aOut.ignoreMethod() = aIn.mIgnoreMethod; + aOut.ignoreVary() = aIn.mIgnoreVary; + aOut.cacheNameSet() = aIn.mCacheName.WasPassed(); + if (aOut.cacheNameSet()) { + aOut.cacheName() = aIn.mCacheName.Value(); + } else { + aOut.cacheName() = NS_LITERAL_STRING(""); + } +} + +already_AddRefed<Response> +TypeUtils::ToResponse(const CacheResponse& aIn) +{ + if (aIn.type() == ResponseType::Error) { + RefPtr<InternalResponse> error = InternalResponse::NetworkError(); + RefPtr<Response> r = new Response(GetGlobalObject(), error); + return r.forget(); + } + + RefPtr<InternalResponse> ir = new InternalResponse(aIn.status(), + aIn.statusText()); + ir->SetURLList(aIn.urlList()); + + RefPtr<InternalHeaders> internalHeaders = + ToInternalHeaders(aIn.headers(), aIn.headersGuard()); + ErrorResult result; + + // Be careful to fill the headers before setting the guard in order to + // correctly re-create the original headers. + ir->Headers()->Fill(*internalHeaders, result); + MOZ_DIAGNOSTIC_ASSERT(!result.Failed()); + ir->Headers()->SetGuard(aIn.headersGuard(), result); + MOZ_DIAGNOSTIC_ASSERT(!result.Failed()); + + ir->InitChannelInfo(aIn.channelInfo()); + if (aIn.principalInfo().type() == mozilla::ipc::OptionalPrincipalInfo::TPrincipalInfo) { + UniquePtr<mozilla::ipc::PrincipalInfo> info(new mozilla::ipc::PrincipalInfo(aIn.principalInfo().get_PrincipalInfo())); + ir->SetPrincipalInfo(Move(info)); + } + + nsCOMPtr<nsIInputStream> stream = ReadStream::Create(aIn.body()); + ir->SetBody(stream, InternalResponse::UNKNOWN_BODY_SIZE); + + switch (aIn.type()) + { + case ResponseType::Basic: + ir = ir->BasicResponse(); + break; + case ResponseType::Cors: + ir = ir->CORSResponse(); + break; + case ResponseType::Default: + break; + case ResponseType::Opaque: + ir = ir->OpaqueResponse(); + break; + case ResponseType::Opaqueredirect: + ir = ir->OpaqueRedirectResponse(); + break; + default: + MOZ_CRASH("Unexpected ResponseType!"); + } + MOZ_DIAGNOSTIC_ASSERT(ir); + + RefPtr<Response> ref = new Response(GetGlobalObject(), ir); + return ref.forget(); +} +already_AddRefed<InternalRequest> +TypeUtils::ToInternalRequest(const CacheRequest& aIn) +{ + nsAutoCString url(aIn.urlWithoutQuery()); + url.Append(aIn.urlQuery()); + RefPtr<InternalRequest> internalRequest = + new InternalRequest(url, aIn.urlFragment()); + internalRequest->SetMethod(aIn.method()); + internalRequest->SetReferrer(aIn.referrer()); + internalRequest->SetReferrerPolicy(aIn.referrerPolicy()); + internalRequest->SetMode(aIn.mode()); + internalRequest->SetCredentialsMode(aIn.credentials()); + internalRequest->SetContentPolicyType(aIn.contentPolicyType()); + internalRequest->SetCacheMode(aIn.requestCache()); + internalRequest->SetRedirectMode(aIn.requestRedirect()); + internalRequest->SetIntegrity(aIn.integrity()); + + RefPtr<InternalHeaders> internalHeaders = + ToInternalHeaders(aIn.headers(), aIn.headersGuard()); + ErrorResult result; + + // Be careful to fill the headers before setting the guard in order to + // correctly re-create the original headers. + internalRequest->Headers()->Fill(*internalHeaders, result); + MOZ_DIAGNOSTIC_ASSERT(!result.Failed()); + + internalRequest->Headers()->SetGuard(aIn.headersGuard(), result); + MOZ_DIAGNOSTIC_ASSERT(!result.Failed()); + + nsCOMPtr<nsIInputStream> stream = ReadStream::Create(aIn.body()); + + internalRequest->SetBody(stream); + + return internalRequest.forget(); +} + +already_AddRefed<Request> +TypeUtils::ToRequest(const CacheRequest& aIn) +{ + RefPtr<InternalRequest> internalRequest = ToInternalRequest(aIn); + RefPtr<Request> request = new Request(GetGlobalObject(), internalRequest); + return request.forget(); +} + +// static +already_AddRefed<InternalHeaders> +TypeUtils::ToInternalHeaders(const nsTArray<HeadersEntry>& aHeadersEntryList, + HeadersGuardEnum aGuard) +{ + nsTArray<InternalHeaders::Entry> entryList(aHeadersEntryList.Length()); + + for (uint32_t i = 0; i < aHeadersEntryList.Length(); ++i) { + const HeadersEntry& headersEntry = aHeadersEntryList[i]; + entryList.AppendElement(InternalHeaders::Entry(headersEntry.name(), + headersEntry.value())); + } + + RefPtr<InternalHeaders> ref = new InternalHeaders(Move(entryList), aGuard); + return ref.forget(); +} + +// Utility function to remove the fragment from a URL, check its scheme, and optionally +// provide a URL without the query. We're not using nsIURL or URL to do this because +// they require going to the main thread. +// static +void +TypeUtils::ProcessURL(nsACString& aUrl, bool* aSchemeValidOut, + nsACString* aUrlWithoutQueryOut,nsACString* aUrlQueryOut, + ErrorResult& aRv) +{ + const nsAFlatCString& flatURL = PromiseFlatCString(aUrl); + const char* url = flatURL.get(); + + // off the main thread URL parsing using nsStdURLParser. + nsCOMPtr<nsIURLParser> urlParser = new nsStdURLParser(); + + uint32_t pathPos; + int32_t pathLen; + uint32_t schemePos; + int32_t schemeLen; + aRv = urlParser->ParseURL(url, flatURL.Length(), &schemePos, &schemeLen, + nullptr, nullptr, // ignore authority + &pathPos, &pathLen); + if (NS_WARN_IF(aRv.Failed())) { return; } + + if (aSchemeValidOut) { + nsAutoCString scheme(Substring(flatURL, schemePos, schemeLen)); + *aSchemeValidOut = scheme.LowerCaseEqualsLiteral("http") || + scheme.LowerCaseEqualsLiteral("https"); + } + + uint32_t queryPos; + int32_t queryLen; + + aRv = urlParser->ParsePath(url + pathPos, flatURL.Length() - pathPos, + nullptr, nullptr, // ignore filepath + &queryPos, &queryLen, + nullptr, nullptr); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + + if (!aUrlWithoutQueryOut) { + return; + } + + MOZ_DIAGNOSTIC_ASSERT(aUrlQueryOut); + + if (queryLen < 0) { + *aUrlWithoutQueryOut = aUrl; + *aUrlQueryOut = EmptyCString(); + return; + } + + // ParsePath gives us query position relative to the start of the path + queryPos += pathPos; + + *aUrlWithoutQueryOut = Substring(aUrl, 0, queryPos - 1); + *aUrlQueryOut = Substring(aUrl, queryPos - 1, queryLen + 1); +} + +void +TypeUtils::CheckAndSetBodyUsed(Request* aRequest, BodyAction aBodyAction, + ErrorResult& aRv) +{ + MOZ_DIAGNOSTIC_ASSERT(aRequest); + + if (aBodyAction == IgnoreBody) { + return; + } + + if (aRequest->BodyUsed()) { + aRv.ThrowTypeError<MSG_FETCH_BODY_CONSUMED_ERROR>(); + return; + } + + nsCOMPtr<nsIInputStream> stream; + aRequest->GetBody(getter_AddRefs(stream)); + if (stream) { + aRequest->SetBodyUsed(); + } +} + +already_AddRefed<InternalRequest> +TypeUtils::ToInternalRequest(const nsAString& aIn, ErrorResult& aRv) +{ + RequestOrUSVString requestOrString; + requestOrString.SetAsUSVString().Rebind(aIn.Data(), aIn.Length()); + + // Re-create a GlobalObject stack object so we can use webidl Constructors. + AutoJSAPI jsapi; + if (NS_WARN_IF(!jsapi.Init(GetGlobalObject()))) { + aRv.Throw(NS_ERROR_UNEXPECTED); + return nullptr; + } + JSContext* cx = jsapi.cx(); + GlobalObject global(cx, GetGlobalObject()->GetGlobalJSObject()); + MOZ_DIAGNOSTIC_ASSERT(!global.Failed()); + + RefPtr<Request> request = Request::Constructor(global, requestOrString, + RequestInit(), aRv); + if (NS_WARN_IF(aRv.Failed())) { return nullptr; } + + return request->GetInternalRequest(); +} + +void +TypeUtils::SerializeCacheStream(nsIInputStream* aStream, + CacheReadStreamOrVoid* aStreamOut, + nsTArray<UniquePtr<AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv) +{ + *aStreamOut = void_t(); + if (!aStream) { + return; + } + + RefPtr<ReadStream> controlled = do_QueryObject(aStream); + if (controlled) { + controlled->Serialize(aStreamOut, aStreamCleanupList, aRv); + return; + } + + *aStreamOut = CacheReadStream(); + CacheReadStream& cacheStream = aStreamOut->get_CacheReadStream(); + + cacheStream.controlChild() = nullptr; + cacheStream.controlParent() = nullptr; + + UniquePtr<AutoIPCStream> autoStream(new AutoIPCStream(cacheStream.stream())); + autoStream->Serialize(aStream, GetIPCManager()); + + aStreamCleanupList.AppendElement(Move(autoStream)); +} + +} // namespace cache +} // namespace dom +} // namespace mozilla diff --git a/dom/cache/TypeUtils.h b/dom/cache/TypeUtils.h new file mode 100644 index 0000000000..731ef9506b --- /dev/null +++ b/dom/cache/TypeUtils.h @@ -0,0 +1,157 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_TypesUtils_h +#define mozilla_dom_cache_TypesUtils_h + +#include "mozilla/Attributes.h" +#include "mozilla/dom/BindingUtils.h" +#include "mozilla/dom/InternalHeaders.h" +#include "nsError.h" + +class nsIGlobalObject; +class nsIAsyncInputStream; +class nsIInputStream; + +namespace mozilla { + +namespace ipc { +class PBackgroundChild; +class SendStreamChild; +class AutoIPCStream; +} + +namespace dom { + +struct CacheQueryOptions; +class InternalRequest; +class InternalResponse; +class OwningRequestOrUSVString; +class Request; +class RequestOrUSVString; +class Response; + +namespace cache { + +class CacheQueryParams; +class CacheReadStream; +class CacheReadStreamOrVoid; +class CacheRequest; +class CacheResponse; +class HeadersEntry; + +class TypeUtils +{ +public: + enum BodyAction + { + IgnoreBody, + ReadBody + }; + + enum SchemeAction + { + IgnoreInvalidScheme, + TypeErrorOnInvalidScheme + }; + + ~TypeUtils() { } + virtual nsIGlobalObject* GetGlobalObject() const = 0; +#ifdef DEBUG + virtual void AssertOwningThread() const = 0; +#else + inline void AssertOwningThread() const { } +#endif + + // This is mainly declared to support serializing body streams. Some + // TypeUtils implementations do not expect to be used for this kind of + // serialization. These classes will MOZ_CRASH() if you try to call + // GetIPCManager(). + virtual mozilla::ipc::PBackgroundChild* + GetIPCManager() = 0; + + already_AddRefed<InternalRequest> + ToInternalRequest(const RequestOrUSVString& aIn, BodyAction aBodyAction, + ErrorResult& aRv); + + already_AddRefed<InternalRequest> + ToInternalRequest(const OwningRequestOrUSVString& aIn, BodyAction aBodyAction, + ErrorResult& aRv); + + void + ToCacheRequest(CacheRequest& aOut, InternalRequest* aIn, + BodyAction aBodyAction, SchemeAction aSchemeAction, + nsTArray<UniquePtr<mozilla::ipc::AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv); + + void + ToCacheResponseWithoutBody(CacheResponse& aOut, InternalResponse& aIn, + ErrorResult& aRv); + + void + ToCacheResponse(CacheResponse& aOut, Response& aIn, + nsTArray<UniquePtr<mozilla::ipc::AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv); + + void + ToCacheQueryParams(CacheQueryParams& aOut, const CacheQueryOptions& aIn); + + already_AddRefed<Response> + ToResponse(const CacheResponse& aIn); + + already_AddRefed<InternalRequest> + ToInternalRequest(const CacheRequest& aIn); + + already_AddRefed<Request> + ToRequest(const CacheRequest& aIn); + + // static methods + static already_AddRefed<InternalHeaders> + ToInternalHeaders(const nsTArray<HeadersEntry>& aHeadersEntryList, + HeadersGuardEnum aGuard = HeadersGuardEnum::None); + + // Utility method for parsing a URL and doing associated operations. A mix + // of things are done in this one method to avoid duplicated parsing: + // + // 1) The aUrl argument is modified to strip the fragment + // 2) If aSchemaValidOut is set, then a boolean value is set indicating + // if the aUrl's scheme is valid or not for storing in the cache. + // 3) If aUrlWithoutQueryOut is set, then a url string is provided without + // the search section. + // 4) If aUrlQueryOut is set then its populated with the search section + // of the URL. Note, this parameter must be set if aUrlWithoutQueryOut + // is set. They must either both be nullptr or set to valid string + // pointers. + // + // Any errors are thrown on ErrorResult. + static void + ProcessURL(nsACString& aUrl, bool* aSchemeValidOut, + nsACString* aUrlWithoutQueryOut, nsACString* aUrlQueryOut, + ErrorResult& aRv); + +private: + void + CheckAndSetBodyUsed(Request* aRequest, BodyAction aBodyAction, + ErrorResult& aRv); + + already_AddRefed<InternalRequest> + ToInternalRequest(const nsAString& aIn, ErrorResult& aRv); + + void + SerializeCacheStream(nsIInputStream* aStream, CacheReadStreamOrVoid* aStreamOut, + nsTArray<UniquePtr<mozilla::ipc::AutoIPCStream>>& aStreamCleanupList, + ErrorResult& aRv); + + void + SerializeSendStream(nsIInputStream* aStream, CacheReadStream& aReadStreamOut, + ErrorResult& aRv); +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_TypesUtils_h diff --git a/dom/cache/Types.h b/dom/cache/Types.h new file mode 100644 index 0000000000..1fc791a022 --- /dev/null +++ b/dom/cache/Types.h @@ -0,0 +1,44 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_cache_Types_h +#define mozilla_dom_cache_Types_h + +#include <stdint.h> +#include "nsCOMPtr.h" +#include "nsIFile.h" +#include "nsString.h" + +namespace mozilla { +namespace dom { +namespace cache { + +enum Namespace +{ + DEFAULT_NAMESPACE, + CHROME_ONLY_NAMESPACE, + NUMBER_OF_NAMESPACES +}; +static const Namespace INVALID_NAMESPACE = NUMBER_OF_NAMESPACES; + +typedef int64_t CacheId; +static const CacheId INVALID_CACHE_ID = -1; + +struct QuotaInfo +{ + QuotaInfo() : mIsApp(false) { } + nsCOMPtr<nsIFile> mDir; + nsCString mSuffix; + nsCString mGroup; + nsCString mOrigin; + bool mIsApp; +}; + +} // namespace cache +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_cache_Types_h diff --git a/dom/cache/moz.build b/dom/cache/moz.build new file mode 100644 index 0000000000..66f9f30fd0 --- /dev/null +++ b/dom/cache/moz.build @@ -0,0 +1,100 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS.mozilla.dom.cache += [ + 'Action.h', + 'ActorChild.h', + 'ActorUtils.h', + 'AutoUtils.h', + 'Cache.h', + 'CacheChild.h', + 'CacheOpChild.h', + 'CacheOpParent.h', + 'CacheParent.h', + 'CacheStorage.h', + 'CacheStorageChild.h', + 'CacheStorageParent.h', + 'CacheStreamControlChild.h', + 'CacheStreamControlParent.h', + 'CacheWorkerHolder.h', + 'Connection.h', + 'Context.h', + 'DBAction.h', + 'DBSchema.h', + 'FileUtils.h', + 'IPCUtils.h', + 'Manager.h', + 'ManagerId.h', + 'PrincipalVerifier.h', + 'QuotaClient.h', + 'ReadStream.h', + 'SavedTypes.h', + 'StreamControl.h', + 'StreamList.h', + 'Types.h', + 'TypeUtils.h', +] + +UNIFIED_SOURCES += [ + 'Action.cpp', + 'ActorChild.cpp', + 'AutoUtils.cpp', + 'Cache.cpp', + 'CacheChild.cpp', + 'CacheOpChild.cpp', + 'CacheOpParent.cpp', + 'CacheParent.cpp', + 'CacheStorage.cpp', + 'CacheStorageChild.cpp', + 'CacheStorageParent.cpp', + 'CacheStreamControlChild.cpp', + 'CacheStreamControlParent.cpp', + 'CacheWorkerHolder.cpp', + 'Connection.cpp', + 'Context.cpp', + 'DBAction.cpp', + 'DBSchema.cpp', + 'FileUtils.cpp', + 'Manager.cpp', + 'ManagerId.cpp', + 'PrincipalVerifier.cpp', + 'QuotaClient.cpp', + 'ReadStream.cpp', + 'StreamControl.cpp', + 'StreamList.cpp', + 'TypeUtils.cpp', +] + +IPDL_SOURCES += [ + 'CacheTypes.ipdlh', + 'PCache.ipdl', + 'PCacheOp.ipdl', + 'PCacheStorage.ipdl', + 'PCacheStreamControl.ipdl', +] + +include('/ipc/chromium/chromium-config.mozbuild') + +LOCAL_INCLUDES += [ + '../workers', +] + +FINAL_LIBRARY = 'xul' + +MOCHITEST_MANIFESTS += [ + 'test/mochitest/mochitest.ini', +] + +BROWSER_CHROME_MANIFESTS += [ + 'test/mochitest/browser.ini', +] + +XPCSHELL_TESTS_MANIFESTS += [ + 'test/xpcshell/xpcshell.ini', +] + +if CONFIG['GNU_CXX']: + CXXFLAGS += ['-Wno-error=shadow'] diff --git a/dom/cache/test/mochitest/browser.ini b/dom/cache/test/mochitest/browser.ini new file mode 100644 index 0000000000..90a7a53a3a --- /dev/null +++ b/dom/cache/test/mochitest/browser.ini @@ -0,0 +1 @@ +[browser_cache_pb_window.js] diff --git a/dom/cache/test/mochitest/browser_cache_pb_window.js b/dom/cache/test/mochitest/browser_cache_pb_window.js new file mode 100644 index 0000000000..05a0aa22b1 --- /dev/null +++ b/dom/cache/test/mochitest/browser_cache_pb_window.js @@ -0,0 +1,81 @@ +var name = 'pb-window-cache'; + +function testMatch(win) { + return new Promise(function(resolve, reject) { + win.caches.match('http://foo.com').then(function(response) { + ok(false, 'caches.match() should not return success'); + reject(); + }).catch(function(err) { + is('SecurityError', err.name, 'caches.match() should throw SecurityError'); + resolve(); + }); + }); +} + +function testHas(win) { + return new Promise(function(resolve, reject) { + win.caches.has(name).then(function(result) { + ok(false, 'caches.has() should not return success'); + reject(); + }).catch(function(err) { + is('SecurityError', err.name, 'caches.has() should throw SecurityError'); + resolve(); + }); + }); +} + +function testOpen(win) { + return new Promise(function(resolve, reject) { + win.caches.open(name).then(function(c) { + ok(false, 'caches.open() should not return success'); + reject(); + }).catch(function(err) { + is('SecurityError', err.name, 'caches.open() should throw SecurityError'); + resolve(); + }); + }); +} + +function testDelete(win) { + return new Promise(function(resolve, reject) { + win.caches.delete(name).then(function(result) { + ok(false, 'caches.delete() should not return success'); + reject(); + }).catch(function(err) { + is('SecurityError', err.name, 'caches.delete() should throw SecurityError'); + resolve(); + }); + }); +} + +function testKeys(win) { + return new Promise(function(resolve, reject) { + win.caches.keys().then(function(names) { + ok(false, 'caches.keys() should not return success'); + reject(); + }).catch(function(err) { + is('SecurityError', err.name, 'caches.keys() should throw SecurityError'); + resolve(); + }); + }); +} + +function test() { + waitForExplicitFinish(); + SpecialPowers.pushPrefEnv({'set': [['dom.caches.enabled', true], + ['dom.caches.testing.enabled', true]]}, + function() { + var privateWin = OpenBrowserWindow({private: true}); + privateWin.addEventListener('load', function() { + Promise.all([ + testMatch(privateWin), + testHas(privateWin), + testOpen(privateWin), + testDelete(privateWin), + testKeys(privateWin) + ]).then(function() { + BrowserTestUtils.closeWindow(privateWin).then(finish); + }); + }); + }); +} diff --git a/dom/cache/test/mochitest/chrome.ini b/dom/cache/test/mochitest/chrome.ini new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/dom/cache/test/mochitest/chrome.ini diff --git a/dom/cache/test/mochitest/driver.js b/dom/cache/test/mochitest/driver.js new file mode 100644 index 0000000000..108e32deb4 --- /dev/null +++ b/dom/cache/test/mochitest/driver.js @@ -0,0 +1,132 @@ +// Any copyright is dedicated to the Public Domain. +// http://creativecommons.org/publicdomain/zero/1.0/ +// +// This helper script exposes a runTests function that takes the name of a +// test script as its input argument and runs the test in three different +// contexts: +// 1. Regular Worker context +// 2. Service Worker context +// 3. Window context +// The function returns a promise which will get resolved once all tests +// finish. The testFile argument is the name of the test file to be run +// in the different contexts, and the optional order argument can be set +// to either "parallel" or "sequential" depending on how the caller wants +// the tests to be run. If this argument is not provided, the default is +// "both", which runs the tests in both modes. +// The caller of this function is responsible to call SimpleTest.finish +// when the returned promise is resolved. + +function runTests(testFile, order) { + function setupPrefs() { + return new Promise(function(resolve, reject) { + SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.serviceWorkers.enabled", true], + ["dom.serviceWorkers.testing.enabled", true], + ["dom.serviceWorkers.exemptFromPerDomainMax", true]] + }, function() { + resolve(); + }); + }); + } + + // adapted from dom/indexedDB/test/helpers.js + function clearStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); + } + + function loadScript(script) { + return new Promise(function(resolve, reject) { + var s = document.createElement("script"); + s.src = script; + s.onerror = reject; + s.onload = resolve; + document.body.appendChild(s); + }); + } + + function importDrivers() { + return Promise.all([loadScript("worker_driver.js"), + loadScript("serviceworker_driver.js")]); + } + + function runWorkerTest() { + return workerTestExec(testFile); + } + + function runServiceWorkerTest() { + return serviceWorkerTestExec(testFile); + } + + function runFrameTest() { + return new Promise(function(resolve, reject) { + var iframe = document.createElement("iframe"); + iframe.src = "frame.html"; + iframe.onload = function() { + var doc = iframe.contentDocument; + var s = doc.createElement("script"); + s.src = testFile; + window.addEventListener("message", function onMessage(event) { + if (event.data.context != "Window") { + return; + } + if (event.data.type == 'finish') { + window.removeEventListener("message", onMessage); + resolve(); + } else if (event.data.type == 'status') { + ok(event.data.status, event.data.context + ": " + event.data.msg); + } + }, false); + doc.body.appendChild(s); + }; + document.body.appendChild(iframe); + }); + } + + SimpleTest.waitForExplicitFinish(); + + if (typeof order == "undefined") { + order = "sequential"; // sequential by default, see bug 1143222. + // TODO: Make this "both" again. + } + + ok(order == "parallel" || order == "sequential" || order == "both", + "order argument should be valid"); + + if (order == "both") { + info("Running tests in both modes; first: sequential"); + return runTests(testFile, "sequential") + .then(function() { + info("Running tests in parallel mode"); + return runTests(testFile, "parallel"); + }); + } + if (order == "sequential") { + return setupPrefs() + .then(importDrivers) + .then(runWorkerTest) + .then(clearStorage) + .then(runServiceWorkerTest) + .then(clearStorage) + .then(runFrameTest) + .then(clearStorage) + .catch(function(e) { + ok(false, "A promise was rejected during test execution: " + e); + }); + } + return setupPrefs() + .then(importDrivers) + .then(() => Promise.all([runWorkerTest(), runServiceWorkerTest(), runFrameTest()])) + .then(clearStorage) + .catch(function(e) { + ok(false, "A promise was rejected during test execution: " + e); + }); +} + diff --git a/dom/cache/test/mochitest/empty.html b/dom/cache/test/mochitest/empty.html new file mode 100644 index 0000000000..5b56631db7 --- /dev/null +++ b/dom/cache/test/mochitest/empty.html @@ -0,0 +1,2 @@ +<!DOCTYPE html> +<!-- This is only used to give us access to the caches global after setting the pref. --> diff --git a/dom/cache/test/mochitest/frame.html b/dom/cache/test/mochitest/frame.html new file mode 100644 index 0000000000..8d471161b9 --- /dev/null +++ b/dom/cache/test/mochitest/frame.html @@ -0,0 +1,17 @@ +<!DOCTYPE html> +<script> + var context = "Window"; + function ok(a, msg) { + parent.postMessage({type: 'status', status: !!a, + msg: a + ": " + msg, context: context}, "*"); + } + + function is(a, b, msg) { + parent.postMessage({type: 'status', status: a === b, + msg: a + " === " + b + ": " + msg, context: context}, "*"); + } + + function testDone() { + parent.postMessage({type: 'finish', context: context}, "*"); + } +</script> diff --git a/dom/cache/test/mochitest/large_url_list.js b/dom/cache/test/mochitest/large_url_list.js new file mode 100644 index 0000000000..40fb7acfb2 --- /dev/null +++ b/dom/cache/test/mochitest/large_url_list.js @@ -0,0 +1,1002 @@ +var largeUrlList = [ + 'http://example.com/cia5rherm0000hkjx7r8of8b1/cia5rhern0001hkjxes0mptxq/cia5rhern0002hkjx0ze9t1oy/cia5rhern0003hkjxa07rei71', + 'http://example.com/cia5rhern0004hkjx15dh2a23/cia5rhern0005hkjxukeg547n/cia5rhern0006hkjxn78qlfk0/cia5rhern0007hkjxx7pkqem5', + 'http://example.com/cia5rhern0008hkjx6xh95dcp/cia5rhern0009hkjxu3212vkx/cia5rhern000ahkjxlhu69sv7/cia5rhern000bhkjxwffaxb8c', + 'http://example.com/cia5rhern000chkjxylxzrc0l/cia5rhern000dhkjxoe8p7jap/cia5rhern000ehkjx49ssgz0i/cia5rhern000fhkjxlqcm6jq9', + 'http://example.com/cia5rhern000ghkjxjfag4uqk/cia5rhern000hhkjxxjmvakfj/cia5rhern000ihkjxp5j7d9ic/cia5rhern000jhkjxdog5rxfb', + 'http://example.com/cia5rhern000khkjxrdrwsflr/cia5rhern000lhkjxjqj2ydrd/cia5rhern000mhkjxujiqkc1c/cia5rhern000nhkjxw0nk2nvg', + 'http://example.com/cia5rhern000ohkjxpglj3lmb/cia5rhern000phkjxjekhrrqd/cia5rhern000qhkjxalsyb73l/cia5rhern000rhkjxyjbnn6er', + 'http://example.com/cia5rhero000shkjxmhknjjrv/cia5rhero000thkjx8xtfdrgh/cia5rhero000uhkjxpe3dwxk6/cia5rhero000vhkjxint5ohhn', + 'http://example.com/cia5rhero000whkjxvaiypkdj/cia5rhero000xhkjxaqm599g5/cia5rhero000yhkjxivrhnj8f/cia5rhero000zhkjxvmd4t8h6', + 'http://example.com/cia5rhero0010hkjxsx3ke38t/cia5rhero0011hkjx8xnw9tqf/cia5rhero0012hkjxl0lf3iup/cia5rhero0013hkjxbvhj1jj9', + 'http://example.com/cia5rhero0014hkjx4fmpoeu0/cia5rhero0015hkjxwpptyghv/cia5rhero0016hkjxva5k7lbw/cia5rhero0017hkjx138e6bmj', + 'http://example.com/cia5rhero0018hkjxx7j78vei/cia5rhero0019hkjx6xrih33b/cia5rhero001ahkjxd4zgngrg/cia5rhero001bhkjx1tloxb1q', + 'http://example.com/cia5rhero001chkjxrt8zdtde/cia5rhero001dhkjxm9mbqci7/cia5rhero001ehkjxowk5e4q6/cia5rhero001fhkjxh6qmsl1v', + 'http://example.com/cia5rhero001ghkjxfadnveq2/cia5rhero001hhkjx4edkaq54/cia5rhero001ihkjx9fqoqxga/cia5rhero001jhkjx58vqa1zr', + 'http://example.com/cia5rhero001khkjxdrmypltg/cia5rhero001lhkjxhqdzis67/cia5rhero001mhkjxcggzuqdx/cia5rhero001nhkjxctfvi81n', + 'http://example.com/cia5rhero001ohkjxe8pk4cgt/cia5rhero001phkjx6sm4qi4x/cia5rhero001qhkjxm9lqhc6e/cia5rhero001rhkjxy9k6yacj', + 'http://example.com/cia5rhero001shkjx0zfh0f80/cia5rhero001thkjx06fdx9h9/cia5rherp001uhkjxqdmh6jtq/cia5rherp001vhkjxywng3c85', + 'http://example.com/cia5rherp001whkjxm2qihavv/cia5rherp001xhkjxw9v508hp/cia5rherp001yhkjx3jtn1gc8/cia5rherp001zhkjxsmvesh8h', + 'http://example.com/cia5rherp0020hkjxz8yaovlg/cia5rherp0021hkjx66mvfalu/cia5rherp0022hkjxqpilypyo/cia5rherp0023hkjxsp9txbui', + 'http://example.com/cia5rherp0024hkjxcc9ni9ai/cia5rherp0025hkjxslerqaui/cia5rherp0026hkjxe0jl3k05/cia5rherp0027hkjxr0hf5tjn', + 'http://example.com/cia5rherp0028hkjxsg9l2j7w/cia5rherp0029hkjx8zs585su/cia5rherp002ahkjxuwrjr76m/cia5rherp002bhkjxh5nsz7nf', + 'http://example.com/cia5rherp002chkjxzph0v3u0/cia5rherp002dhkjxdayhh6lg/cia5rherp002ehkjxrkvcxz41/cia5rherp002fhkjxlqa9ul9r', + 'http://example.com/cia5rherp002ghkjxf6egbyrh/cia5rherp002hhkjxaw3isyif/cia5rherp002ihkjxboruj7fc/cia5rherp002jhkjx6fgq322u', + 'http://example.com/cia5rherp002khkjx5adgzcsd/cia5rherp002lhkjxo84dwluf/cia5rherp002mhkjxijwq9esb/cia5rherp002nhkjx59ky0n1a', + 'http://example.com/cia5rherp002ohkjx8r9ldy53/cia5rherp002phkjxn5vwv7yi/cia5rherp002qhkjxdsg26179/cia5rherp002rhkjxxvufen34', + 'http://example.com/cia5rherp002shkjxj17ade31/cia5rherp002thkjx37xtqdld/cia5rherp002uhkjxbl9a3b96/cia5rherp002vhkjxrq82quxi', + 'http://example.com/cia5rherp002whkjxj0kiekos/cia5rherp002xhkjxltksjia2/cia5rherp002yhkjx9f6j16y4/cia5rherp002zhkjxub535mh5', + 'http://example.com/cia5rherp0030hkjxsd01uqhn/cia5rherp0031hkjxxy7v0enj/cia5rherp0032hkjxm7afimyn/cia5rherp0033hkjxu9rm6wnw', + 'http://example.com/cia5rherp0034hkjxoe7lub90/cia5rherp0035hkjxdqfkfwe5/cia5rherp0036hkjx7bydg1o0/cia5rherp0037hkjxwiayjj4h', + 'http://example.com/cia5rherp0038hkjxrxyltlgt/cia5rherp0039hkjxq8m8gzd9/cia5rherp003ahkjxfzlwk84z/cia5rherp003bhkjxhzsd9psq', + 'http://example.com/cia5rherp003chkjx8k4d0ev6/cia5rherp003dhkjxe5rhd9bk/cia5rherp003ehkjxhdb9pe1z/cia5rherp003fhkjxnwfpch8g', + 'http://example.com/cia5rherp003ghkjxv07j52h2/cia5rherp003hhkjx1i0x37cm/cia5rherp003ihkjxey8otr6v/cia5rherp003jhkjxk1np7zo5', + 'http://example.com/cia5rherp003khkjxscsb30qa/cia5rherp003lhkjxcuap7ls3/cia5rherq003mhkjxsdoqgpxu/cia5rherq003nhkjxz21dqdpc', + 'http://example.com/cia5rherq003ohkjx66ms0nn0/cia5rherq003phkjxb8hblxdd/cia5rherq003qhkjxmr2bkt0b/cia5rherq003rhkjxkla58f3d', + 'http://example.com/cia5rherr003shkjxutmljflc/cia5rherr003thkjx9sglm092/cia5rherr003uhkjx6ttae5q1/cia5rherr003vhkjx2i22wbci', + 'http://example.com/cia5rherr003whkjx0gmqk0qu/cia5rherr003xhkjxnnopsaqa/cia5rherr003yhkjxhicji4pa/cia5rherr003zhkjxg7dm7usj', + 'http://example.com/cia5rherr0040hkjxqymjv2aj/cia5rherr0041hkjxgjublrsc/cia5rherr0042hkjxu34zz1y2/cia5rherr0043hkjx0v1t6s9s', + 'http://example.com/cia5rherr0044hkjxourvl7pc/cia5rherr0045hkjxem1bv16o/cia5rherr0046hkjxlza5reyz/cia5rherr0047hkjxc0hqmpn8', + 'http://example.com/cia5rherr0048hkjxab7n8nos/cia5rherr0049hkjxjwltgxvq/cia5rherr004ahkjxpskjnyvt/cia5rherr004bhkjx9t1zafr8', + 'http://example.com/cia5rherr004chkjxos007jlr/cia5rherr004dhkjxside72xk/cia5rherr004ehkjxmsb8r01v/cia5rherr004fhkjx1c1v6ypg', + 'http://example.com/cia5rherr004ghkjxl9hin1sd/cia5rherr004hhkjx1ktxr1zz/cia5rherr004ihkjxjnq6wq6b/cia5rherr004jhkjx6z9ffji0', + 'http://example.com/cia5rherr004khkjxxu7r4mzw/cia5rherr004lhkjxj9vru5i2/cia5rherr004mhkjxowwrqrzg/cia5rherr004nhkjxejliyhz4', + 'http://example.com/cia5rherr004ohkjx23jh1sos/cia5rherr004phkjxrq8hqfpy/cia5rherr004qhkjx4232owb6/cia5rherr004rhkjx6dbg83qw', + 'http://example.com/cia5rherr004shkjx6t43ggcp/cia5rherr004thkjxmf2k2w6n/cia5rherr004uhkjxwhhmhvx5/cia5rherr004vhkjxt4qahjbz', + 'http://example.com/cia5rherr004whkjx0bkl7a31/cia5rherr004xhkjxk34mm030/cia5rherr004yhkjxoundtp7u/cia5rherr004zhkjxr1htazrx', + 'http://example.com/cia5rherr0050hkjxfm9e7rcy/cia5rherr0051hkjxvp7y4api/cia5rherr0052hkjxdairex18/cia5rherr0053hkjxoqt8of5n', + 'http://example.com/cia5rherr0054hkjxbg8to9br/cia5rherr0055hkjxymwr54ie/cia5rherr0056hkjxez64qj8r/cia5rherr0057hkjxann7kwhj', + 'http://example.com/cia5rherr0058hkjxov69tz76/cia5rherr0059hkjxxh6rrvs8/cia5rherr005ahkjxwsmsljoc/cia5rherr005bhkjxi5o005me', + 'http://example.com/cia5rherr005chkjxu6w2wby8/cia5rherr005dhkjxvcmga3pp/cia5rherr005ehkjxcr90eaeq/cia5rherr005fhkjxon6fcg5h', + 'http://example.com/cia5rherr005ghkjxqo4vucke/cia5rherr005hhkjxxvdv1j8f/cia5rherr005ihkjxo2vl23qb/cia5rherr005jhkjx3tbu637k', + 'http://example.com/cia5rherr005khkjx8fnhdy4n/cia5rherr005lhkjxvgvc27dd/cia5rherr005mhkjxydbxzvrw/cia5rherr005nhkjx0ev5rnx0', + 'http://example.com/cia5rherr005ohkjxymk6ffpy/cia5rherr005phkjx79eg202o/cia5rherr005qhkjxloql9sm9/cia5rherr005rhkjxf0fcwa88', + 'http://example.com/cia5rherr005shkjxkclvai2n/cia5rherr005thkjx6lhii1h5/cia5rherr005uhkjx21imwf9f/cia5rherr005vhkjxct3akix6', + 'http://example.com/cia5rherr005whkjxam3tkrf9/cia5rherr005xhkjx3ooh8d77/cia5rherr005yhkjx3ih1052q/cia5rherr005zhkjxjn08ve87', + 'http://example.com/cia5rherr0060hkjxbovht1zk/cia5rherr0061hkjx3mq7yzke/cia5rherr0062hkjxc3jyhls3/cia5rherr0063hkjxw5y2kgtl', + 'http://example.com/cia5rherr0064hkjxlzchg2lf/cia5rherr0065hkjxjf4qz7h7/cia5rherr0066hkjxhzxfzly2/cia5rherr0067hkjxxpsm77n2', + 'http://example.com/cia5rherr0068hkjxqakagjms/cia5rherr0069hkjxahy5cahd/cia5rherr006ahkjxlx3eo0d8/cia5rherr006bhkjxqhufagp0', + 'http://example.com/cia5rherr006chkjxs6l4reuh/cia5rherr006dhkjx22dhomat/cia5rherr006ehkjxv8bhjw6s/cia5rherr006fhkjxiembar5h', + 'http://example.com/cia5rherr006ghkjx7uk2ahcv/cia5rherr006hhkjxrptkdzpc/cia5rherr006ihkjxoelimxey/cia5rherr006jhkjxh00sc9q0', + 'http://example.com/cia5rherr006khkjx7bjaq4p8/cia5rherr006lhkjx0t17bwqu/cia5rherr006mhkjx63m6nx0x/cia5rherr006nhkjx4aylco20', + 'http://example.com/cia5rherr006ohkjxqncx2bfn/cia5rherr006phkjxcs9lhj4k/cia5rherr006qhkjxltiaiox0/cia5rherr006rhkjxpxhk6ceh', + 'http://example.com/cia5rherr006shkjxkjpkluh5/cia5rherr006thkjxksjhi0rp/cia5rherr006uhkjxuluxeq56/cia5rherr006vhkjx2i5mkp5o', + 'http://example.com/cia5rherr006whkjx94ph66s7/cia5rherr006xhkjx963ey6z0/cia5rherr006yhkjxklr3zey7/cia5rherr006zhkjxhi9ckzcj', + 'http://example.com/cia5rherr0070hkjx8qu91dki/cia5rherr0071hkjxnsj72h7a/cia5rherr0072hkjxpcch22pw/cia5rherr0073hkjxr88cl6td', + 'http://example.com/cia5rherr0074hkjx33v1pc9p/cia5rherr0075hkjxbijzax59/cia5rherr0076hkjxdatwas40/cia5rherr0077hkjxp92fa2bh', + 'http://example.com/cia5rherr0078hkjxids7sgwp/cia5rherr0079hkjxptr6nbvk/cia5rherr007ahkjx51bynaee/cia5rherr007bhkjx2f0oimkg', + 'http://example.com/cia5rherr007chkjxj243t6fq/cia5rherr007dhkjxx0z54s2z/cia5rherr007ehkjxvtk1saqr/cia5rherr007fhkjxc99ot3di', + 'http://example.com/cia5rherr007ghkjxcdyrgbe5/cia5rherr007hhkjx4g2zda93/cia5rherr007ihkjx3dhwfh5w/cia5rherr007jhkjxnucjju1k', + 'http://example.com/cia5rherr007khkjxtucv4xam/cia5rherr007lhkjxzb52dzam/cia5rherr007mhkjxe5ytefsg/cia5rherr007nhkjxhn1htqim', + 'http://example.com/cia5rherr007ohkjxyhvqunje/cia5rherr007phkjxg1rmr8ia/cia5rherr007qhkjxvne6tsg2/cia5rherr007rhkjx92plnv33', + 'http://example.com/cia5rherr007shkjxedxbud9y/cia5rherr007thkjxsfcv846z/cia5rherr007uhkjxrtoyvp1k/cia5rherr007vhkjxwkkvb63p', + 'http://example.com/cia5rherr007whkjxhom33u72/cia5rherr007xhkjx00i52itn/cia5rherr007yhkjxlbwyrfbh/cia5rherr007zhkjx7hqxgotj', + 'http://example.com/cia5rherr0080hkjxm6fr03t9/cia5rherr0081hkjxsskkldqg/cia5rherr0082hkjx6agt52v9/cia5rherr0083hkjxmw1qpljq', + 'http://example.com/cia5rherr0084hkjx66lbczid/cia5rherr0085hkjx1yhf4qqz/cia5rherr0086hkjx156ah7x1/cia5rhers0087hkjxf7zmqrz3', + 'http://example.com/cia5rhers0088hkjxlvoy1vtq/cia5rhers0089hkjxwopzmrl8/cia5rhers008ahkjxa76skipt/cia5rhers008bhkjxchpeggii', + 'http://example.com/cia5rhers008chkjx77jxzrj5/cia5rhers008dhkjxwdtj4fxt/cia5rhers008ehkjxyk1rbqs4/cia5rhers008fhkjxi6c9h7on', + 'http://example.com/cia5rhers008ghkjxcjbq4qio/cia5rhers008hhkjx46uf4z78/cia5rhers008ihkjxiwjs7rs8/cia5rhers008jhkjx6gow2oc8', + 'http://example.com/cia5rhers008khkjxam5doybe/cia5rhers008lhkjx33h8n79o/cia5rhers008mhkjxch0uhhqa/cia5rhers008nhkjxvkh7mio2', + 'http://example.com/cia5rhers008ohkjxfixhe4vq/cia5rhers008phkjxrw2g2zi3/cia5rhers008qhkjxoynia1m2/cia5rhers008rhkjxoh1yvlac', + 'http://example.com/cia5rhers008shkjxewzwesdt/cia5rhers008thkjx2yx5o3rs/cia5rhers008uhkjxm75q9exl/cia5rhers008vhkjx4jogh9q0', + 'http://example.com/cia5rhers008whkjxdo6vglei/cia5rhers008xhkjx0bx8rrph/cia5rhers008yhkjx7sotv4z4/cia5rhers008zhkjxid6dq8hl', + 'http://example.com/cia5rhers0090hkjxpmid4zyj/cia5rhers0091hkjx7swur7ci/cia5rhers0092hkjxd4mc5j7l/cia5rhers0093hkjxri89p7jy', + 'http://example.com/cia5rhers0094hkjxirrewas9/cia5rhers0095hkjx1wkqq1g1/cia5rhers0096hkjxvpc3hp0t/cia5rhers0097hkjx1ugphwhs', + 'http://example.com/cia5rhers0098hkjxxptvggpn/cia5rhers0099hkjx0hgy6tpl/cia5rhers009ahkjxrgk4cwx1/cia5rhers009bhkjxhff7ocag', + 'http://example.com/cia5rhers009chkjxt2zanh56/cia5rhers009dhkjxeh17c4wg/cia5rhers009ehkjxkhdljhm4/cia5rhers009fhkjxa78k166d', + 'http://example.com/cia5rhers009ghkjxblshruiu/cia5rhers009hhkjx6bpkys70/cia5rhers009ihkjx097xi5jr/cia5rhers009jhkjxl1v2ym6h', + 'http://example.com/cia5rhers009khkjxvciqe48d/cia5rhers009lhkjxxgfzrzqn/cia5rhers009mhkjx99k29erp/cia5rhers009nhkjxnmdineg9', + 'http://example.com/cia5rhers009ohkjx0rr58cbk/cia5rhers009phkjxo0wwxbtm/cia5rhers009qhkjx8j0n0ta4/cia5rhers009rhkjx8jok0mo9', + 'http://example.com/cia5rhers009shkjxo1trhdsu/cia5rhers009thkjxtukv1mwy/cia5rhers009uhkjx0e82f0we/cia5rhers009vhkjx8j0ysbxu', + 'http://example.com/cia5rhers009whkjxdl5recmi/cia5rhers009xhkjx801pcdzp/cia5rhers009yhkjx2wnyv52l/cia5rhers009zhkjxdmjnipn0', + 'http://example.com/cia5rhers00a0hkjxj46kgklk/cia5rhers00a1hkjxmaxskno1/cia5rhers00a2hkjxrv6jtia4/cia5rhers00a3hkjx524js0bd', + 'http://example.com/cia5rhers00a4hkjx6y55uml1/cia5rhers00a5hkjxahhtmal1/cia5rhers00a6hkjx0pt7rbot/cia5rhers00a7hkjxtboirxmu', + 'http://example.com/cia5rhers00a8hkjxqmrjjens/cia5rhers00a9hkjxxbmuxaqn/cia5rhers00aahkjx2fdz4j2q/cia5rhers00abhkjxflwb8nml', + 'http://example.com/cia5rhers00achkjxc7n5xjqb/cia5rhers00adhkjx98mur9hy/cia5rhers00aehkjxrmv4miio/cia5rhers00afhkjxm1wmuk24', + 'http://example.com/cia5rhers00aghkjxhh9nyajy/cia5rhers00ahhkjxypuj93ni/cia5rhers00aihkjxqhvpkhvu/cia5rhers00ajhkjxcebhpi2g', + 'http://example.com/cia5rhers00akhkjxpq2z92bg/cia5rhers00alhkjx6ztkugxa/cia5rhers00amhkjxys8qr2ae/cia5rhers00anhkjxugpiwy4o', + 'http://example.com/cia5rhers00aohkjx5lptk7ll/cia5rhert00aphkjx3m0tdlo6/cia5rhert00aqhkjxtun34qda/cia5rhert00arhkjxgn2cukgo', + 'http://example.com/cia5rhert00ashkjxgndi0t1w/cia5rhert00athkjxngdtw2ac/cia5rhert00auhkjxtlqdm9tk/cia5rhert00avhkjxpmv39lb6', + 'http://example.com/cia5rhert00awhkjxsm1i5606/cia5rhert00axhkjxiini4u7n/cia5rhert00ayhkjxawcdih8y/cia5rhert00azhkjx2uwcskyo', + 'http://example.com/cia5rhert00b0hkjxjosiu610/cia5rhert00b1hkjx0inj6sis/cia5rhert00b2hkjx687j5ca5/cia5rhert00b3hkjxaaepyb37', + 'http://example.com/cia5rhert00b4hkjxccpt8awt/cia5rhert00b5hkjxb4kuj419/cia5rhert00b6hkjxq8bi4zga/cia5rhert00b7hkjxrgomql9g', + 'http://example.com/cia5rhert00b8hkjxjxy3fhb0/cia5rhert00b9hkjxftkoktwh/cia5rhert00bahkjxjx5o3cnn/cia5rhert00bbhkjxa6frtobg', + 'http://example.com/cia5rhert00bchkjxe5tkaifo/cia5rhert00bdhkjx5jppppvo/cia5rhert00behkjxdmsqdq9p/cia5rhert00bfhkjxnw4tk86n', + 'http://example.com/cia5rhert00bghkjx49wz4l5r/cia5rhert00bhhkjx2mfza2xe/cia5rhert00bihkjxbsd9ovfr/cia5rhert00bjhkjx3vye5ep3', + 'http://example.com/cia5rhert00bkhkjxwtdsj589/cia5rhert00blhkjxmjchgsmf/cia5rhert00bmhkjxm5fxzqpq/cia5rhert00bnhkjxyf65cmel', + 'http://example.com/cia5rhert00bohkjx7c22ja5m/cia5rhert00bphkjxhe6e895b/cia5rhert00bqhkjxj6kge909/cia5rhert00brhkjx65rkq4ma', + 'http://example.com/cia5rhert00bshkjx1cn2s8w0/cia5rhert00bthkjxlbbqak7z/cia5rhert00buhkjxfy9yreib/cia5rhert00bvhkjxaivuz0fw', + 'http://example.com/cia5rhert00bwhkjxinogvtl2/cia5rhert00bxhkjxw62nijla/cia5rhert00byhkjxl5nlc2eo/cia5rhert00bzhkjxhlmdfgi7', + 'http://example.com/cia5rhert00c0hkjx4n6b40nr/cia5rhert00c1hkjxsnmkujr5/cia5rhert00c2hkjxv6w0h717/cia5rhert00c3hkjxrcvb5qs6', + 'http://example.com/cia5rhert00c4hkjxgd95z86n/cia5rhert00c5hkjxv1tvmf9c/cia5rhert00c6hkjx4cv3ix8y/cia5rhert00c7hkjxli2to7w7', + 'http://example.com/cia5rhert00c8hkjxgaiuvkul/cia5rhert00c9hkjxkcpdyj68/cia5rhert00cahkjxjxrfjfl0/cia5rhert00cbhkjxk64v5mhq', + 'http://example.com/cia5rhert00cchkjxzkjnwkn1/cia5rhert00cdhkjx1cdlk3ms/cia5rhert00cehkjxaw4lnp43/cia5rhert00cfhkjxz4hk5gsw', + 'http://example.com/cia5rhert00cghkjxpil3za50/cia5rhert00chhkjxp7t5jz46/cia5rhert00cihkjx5cwk8th0/cia5rhert00cjhkjxaxp4y2o0', + 'http://example.com/cia5rhert00ckhkjxcylxiiem/cia5rhert00clhkjxzkx50uda/cia5rhert00cmhkjxs5uv5t4e/cia5rhert00cnhkjxxz9rplpv', + 'http://example.com/cia5rhert00cohkjxa1r51z2k/cia5rhert00cphkjx1zw5bwvd/cia5rhert00cqhkjxkmcxldzz/cia5rhert00crhkjx8xm9oeny', + 'http://example.com/cia5rhert00cshkjxuoqaoftt/cia5rhert00cthkjx1x01dmec/cia5rhert00cuhkjx9sixehxp/cia5rhert00cvhkjxomyssiza', + 'http://example.com/cia5rhert00cwhkjx7et34wux/cia5rhert00cxhkjxncva74m3/cia5rhert00cyhkjxfkizvrik/cia5rhert00czhkjxgbzmphlh', + 'http://example.com/cia5rhert00d0hkjx5amei299/cia5rhert00d1hkjxumygde64/cia5rhert00d2hkjxpnuisg9m/cia5rhert00d3hkjxakoawt1r', + 'http://example.com/cia5rhert00d4hkjx1h1sd1xk/cia5rhert00d5hkjxiszqt7gt/cia5rhert00d6hkjxrgly5qmw/cia5rhert00d7hkjx0khb7is3', + 'http://example.com/cia5rhert00d8hkjx8q3rh7mm/cia5rhert00d9hkjxsc3r3d6p/cia5rhert00dahkjxi4ka0kza/cia5rhert00dbhkjxzvnrcsv0', + 'http://example.com/cia5rhert00dchkjxqlqg4rat/cia5rhert00ddhkjxtjjv0jz2/cia5rhert00dehkjxf7uayrst/cia5rhert00dfhkjxzjqnclyl', + 'http://example.com/cia5rhert00dghkjxdq5ojgf7/cia5rhert00dhhkjxs3fjff88/cia5rheru00dihkjxo2axpgvn/cia5rheru00djhkjx8lrs9xha', + 'http://example.com/cia5rheru00dkhkjx45m33mct/cia5rheru00dlhkjxzrjk6adw/cia5rheru00dmhkjxhhgorfhb/cia5rheru00dnhkjxwadtejpl', + 'http://example.com/cia5rheru00dohkjxj8rdgozp/cia5rheru00dphkjxsqktwr0d/cia5rheru00dqhkjxlodva5ul/cia5rheru00drhkjxy79ve13k', + 'http://example.com/cia5rheru00dshkjxwawhyuhd/cia5rheru00dthkjxovwtz7zd/cia5rheru00duhkjxnhebnm70/cia5rheru00dvhkjxt28pubcm', + 'http://example.com/cia5rheru00dwhkjxasdto0h0/cia5rheru00dxhkjxvoxohbir/cia5rheru00dyhkjxr50yvvwt/cia5rheru00dzhkjxiffhdptt', + 'http://example.com/cia5rheru00e0hkjx9htn1puz/cia5rheru00e1hkjxqvm483lo/cia5rheru00e2hkjxbr3v8ysf/cia5rheru00e3hkjx1xxys4u4', + 'http://example.com/cia5rheru00e4hkjx0pdbwiqd/cia5rheru00e5hkjxcjyvelv6/cia5rheru00e6hkjxfar4jska/cia5rheru00e7hkjxfkw7rkrq', + 'http://example.com/cia5rheru00e8hkjx1rtfronr/cia5rheru00e9hkjxpgfyuqxd/cia5rheru00eahkjx9hw9j0h3/cia5rheru00ebhkjx83acges9', + 'http://example.com/cia5rheru00echkjxbrhqzezc/cia5rheru00edhkjxu3iycpdz/cia5rheru00eehkjxs91i2o4s/cia5rheru00efhkjxgsawfxoa', + 'http://example.com/cia5rheru00eghkjxdhzfetw8/cia5rheru00ehhkjx1wnw1va3/cia5rheru00eihkjxo0vft0jh/cia5rheru00ejhkjxa8vgdyzn', + 'http://example.com/cia5rheru00ekhkjxfo36jiae/cia5rheru00elhkjxim0qkqcu/cia5rheru00emhkjxyv3cm8s8/cia5rheru00enhkjx4zsm4g1k', + 'http://example.com/cia5rherv00eohkjxl16e03j1/cia5rherv00ephkjxtfqgb09e/cia5rherv00eqhkjxxnuib8vx/cia5rherv00erhkjx3c2eh7rw', + 'http://example.com/cia5rherv00eshkjxk2ugn862/cia5rherv00ethkjx4a4i5bu4/cia5rherv00euhkjx4x1ll1hu/cia5rherv00evhkjx4a90801w', + 'http://example.com/cia5rherv00ewhkjx16ofuvju/cia5rherv00exhkjxbjicg8ee/cia5rherv00eyhkjxi37ly2xd/cia5rherv00ezhkjxme2gnc8n', + 'http://example.com/cia5rherv00f0hkjxupxvqy6f/cia5rherv00f1hkjxif6y2ebt/cia5rherv00f2hkjx2hxktwi1/cia5rherv00f3hkjxboq2udk4', + 'http://example.com/cia5rherv00f4hkjxe5t1s9vm/cia5rherv00f5hkjx2p1bkk8q/cia5rherv00f6hkjx3ugeare1/cia5rherv00f7hkjxw5xpdje2', + 'http://example.com/cia5rherv00f8hkjxr347e4en/cia5rherv00f9hkjxjmjvxubi/cia5rherv00fahkjxhgdld04j/cia5rherv00fbhkjxury8qhpe', + 'http://example.com/cia5rherv00fchkjx36kklfip/cia5rherv00fdhkjx51q4kxdz/cia5rherv00fehkjxm1xs1n6p/cia5rherv00ffhkjxzrms1ho0', + 'http://example.com/cia5rherv00fghkjxq54a40bh/cia5rherv00fhhkjxfjd33rlp/cia5rherv00fihkjx8ydqzwn4/cia5rherv00fjhkjx800mxgbz', + 'http://example.com/cia5rherv00fkhkjxr5ktedsn/cia5rherv00flhkjxkh6ip88y/cia5rherv00fmhkjxx824akqe/cia5rherv00fnhkjxn8b9z0y4', + 'http://example.com/cia5rherv00fohkjximtuc4oo/cia5rherv00fphkjxo8xnj3a1/cia5rherv00fqhkjx3frdvz0l/cia5rherv00frhkjxnt4ip06p', + 'http://example.com/cia5rherv00fshkjxebbqlsj4/cia5rherv00fthkjxhejhs2en/cia5rherv00fuhkjxww4v962t/cia5rherv00fvhkjx5n0tpk2r', + 'http://example.com/cia5rherv00fwhkjxomhb4741/cia5rherv00fxhkjxkgs095w0/cia5rherv00fyhkjxtve4rfnk/cia5rherv00fzhkjxkl9m13ke', + 'http://example.com/cia5rherv00g0hkjxwfqm1g52/cia5rherv00g1hkjx0sajw56m/cia5rherv00g2hkjxcs30a0m9/cia5rherv00g3hkjxuq2edyvx', + 'http://example.com/cia5rherv00g4hkjxpmffchhu/cia5rherv00g5hkjxkhh7fc92/cia5rherv00g6hkjxsak4bwp4/cia5rherv00g7hkjxemsqgdmx', + 'http://example.com/cia5rherv00g8hkjxxd4iaucx/cia5rherv00g9hkjx82yt525c/cia5rherv00gahkjx9higv8wz/cia5rherv00gbhkjx0bxq8ejn', + 'http://example.com/cia5rherv00gchkjxolbgqpvx/cia5rherv00gdhkjxh1q1mlra/cia5rherv00gehkjx6rmpv7a6/cia5rherv00gfhkjx7v471pja', + 'http://example.com/cia5rherv00gghkjx4swi554w/cia5rherv00ghhkjxrbcdqi5x/cia5rherv00gihkjxrzkje2z3/cia5rherv00gjhkjx23xefzn2', + 'http://example.com/cia5rherv00gkhkjx7om372u4/cia5rherv00glhkjx7t4qbrj9/cia5rherv00gmhkjxm4021iiz/cia5rherv00gnhkjxgxmosagw', + 'http://example.com/cia5rherv00gohkjx92hhv2g9/cia5rherv00gphkjx7bf3eo7t/cia5rherv00gqhkjx6wj13cjb/cia5rherv00grhkjx9xdd6xqg', + 'http://example.com/cia5rherv00gshkjxapw92pmq/cia5rherv00gthkjx7uqh0m79/cia5rherv00guhkjx9uxnghi5/cia5rherv00gvhkjx9dk24e4x', + 'http://example.com/cia5rherv00gwhkjxm8spj2z7/cia5rherv00gxhkjx1bzjkoj5/cia5rherv00gyhkjxj4n460vb/cia5rherv00gzhkjx6824pani', + 'http://example.com/cia5rherv00h0hkjx8wfm03s5/cia5rherv00h1hkjxboapbcoy/cia5rherv00h2hkjxqgzz6g4s/cia5rherv00h3hkjxcbpuhpk2', + 'http://example.com/cia5rherv00h4hkjx0eho0z4b/cia5rherv00h5hkjx1z59ioaa/cia5rherv00h6hkjxzqmxtasv/cia5rherv00h7hkjxflt86e5f', + 'http://example.com/cia5rherv00h8hkjxxlr7008t/cia5rherv00h9hkjxz8kctoc9/cia5rherv00hahkjxq60p0kx0/cia5rherv00hbhkjxd0zud58a', + 'http://example.com/cia5rherv00hchkjx4o8rphb9/cia5rherv00hdhkjxs3n7yko7/cia5rherv00hehkjxnxwntfg5/cia5rherv00hfhkjxb39xgejv', + 'http://example.com/cia5rherv00hghkjxb1yco8y1/cia5rherv00hhhkjx2v6pz42w/cia5rherv00hihkjx400aow1n/cia5rherv00hjhkjxtwik0d1s', + 'http://example.com/cia5rherv00hkhkjxwnaoc3oc/cia5rherv00hlhkjxsmntvjhm/cia5rherv00hmhkjx4ag0j8cq/cia5rherv00hnhkjxbiigrngm', + 'http://example.com/cia5rherv00hohkjx2s4opprd/cia5rherv00hphkjx8pkc39p2/cia5rherv00hqhkjxif9qfvct/cia5rherv00hrhkjxe4vtvd6z', + 'http://example.com/cia5rherv00hshkjxampz8fwm/cia5rherv00hthkjxf48ybrar/cia5rherv00huhkjx3asrrys5/cia5rherv00hvhkjxc570882r', + 'http://example.com/cia5rherv00hwhkjxfooadgv0/cia5rherv00hxhkjxnmtttd03/cia5rherv00hyhkjx0deqal5b/cia5rherv00hzhkjxleqe94v1', + 'http://example.com/cia5rherv00i0hkjxs3ps41wq/cia5rherv00i1hkjxw46qft2r/cia5rherv00i2hkjxfmfjpssy/cia5rherv00i3hkjxayjpzx43', + 'http://example.com/cia5rherv00i4hkjxpxotlyhp/cia5rherv00i5hkjxhz77s1t6/cia5rherv00i6hkjxwgw9wpq7/cia5rherv00i7hkjxy724la5w', + 'http://example.com/cia5rherv00i8hkjx6wko56vc/cia5rherv00i9hkjxiqs85dqr/cia5rherv00iahkjx32pvc8lh/cia5rherv00ibhkjxouu7jcs7', + 'http://example.com/cia5rherv00ichkjxub141io8/cia5rherv00idhkjx4xp7yqzz/cia5rherv00iehkjx4l9xkah3/cia5rherv00ifhkjx3qco4ypm', + 'http://example.com/cia5rherv00ighkjx9sp14gne/cia5rherv00ihhkjx713r0569/cia5rherv00iihkjxorq3d5yp/cia5rherv00ijhkjxtdo94p17', + 'http://example.com/cia5rherv00ikhkjx4j5jbzre/cia5rherv00ilhkjxcgsn6xlu/cia5rherv00imhkjxx8la8kf5/cia5rherv00inhkjx3hz5opiw', + 'http://example.com/cia5rherv00iohkjx3dgmyl4e/cia5rherv00iphkjx8glt32u3/cia5rherv00iqhkjx6cb6wlxs/cia5rherv00irhkjxza76xuqt', + 'http://example.com/cia5rherv00ishkjxo51i49wr/cia5rherv00ithkjxxa546aho/cia5rherv00iuhkjx50q0dsyb/cia5rherv00ivhkjxcq50j6b5', + 'http://example.com/cia5rherv00iwhkjxaklfq2jh/cia5rherv00ixhkjxq47wm5d8/cia5rherv00iyhkjx93fsh773/cia5rherv00izhkjx05qw5ls6', + 'http://example.com/cia5rherv00j0hkjxcpk80dqg/cia5rherv00j1hkjxe5fze7ph/cia5rherv00j2hkjx0b4x6jj2/cia5rherv00j3hkjxslnfjk8f', + 'http://example.com/cia5rherv00j4hkjxn02a7r4w/cia5rherv00j5hkjxqbq2abzk/cia5rherv00j6hkjx046gf9gh/cia5rherv00j7hkjxw8y6aybl', + 'http://example.com/cia5rherv00j8hkjxua9fh4f2/cia5rherv00j9hkjx0n7j7tf0/cia5rherv00jahkjx1h8p0yzt/cia5rherv00jbhkjx6isout34', + 'http://example.com/cia5rherv00jchkjxscxpp7mk/cia5rherv00jdhkjxx7314ajy/cia5rherv00jehkjx2wej8wjh/cia5rherv00jfhkjxs4i5eiho', + 'http://example.com/cia5rherv00jghkjxcrkt8o3t/cia5rherv00jhhkjx2u0sm2r0/cia5rherv00jihkjxtpjif9k6/cia5rherv00jjhkjxaagcsohi', + 'http://example.com/cia5rherv00jkhkjx8gq5x6ov/cia5rherv00jlhkjxev9zq161/cia5rherv00jmhkjx2sjcp2px/cia5rherv00jnhkjxn8jpfipq', + 'http://example.com/cia5rherv00johkjxbx8744i5/cia5rherv00jphkjxi2eza54a/cia5rherv00jqhkjx1q5y2n4p/cia5rherv00jrhkjxvl0soujd', + 'http://example.com/cia5rherv00jshkjxs7ziy4vs/cia5rherv00jthkjx3ewtvj26/cia5rherv00juhkjxkwhid4b7/cia5rherv00jvhkjxx7nurgrx', + 'http://example.com/cia5rherv00jwhkjxxlcfjhj3/cia5rherv00jxhkjxoresrx2f/cia5rherv00jyhkjxtsmt6apo/cia5rherv00jzhkjxghybzsqs', + 'http://example.com/cia5rherv00k0hkjx7o0bc0o5/cia5rherv00k1hkjxmzwu9w7x/cia5rherv00k2hkjxw6z2g76g/cia5rherv00k3hkjx8htly8zt', + 'http://example.com/cia5rherv00k4hkjx59yd941i/cia5rherv00k5hkjx0j5ccesw/cia5rherv00k6hkjxduxlqh65/cia5rherv00k7hkjx5et0n6t0', + 'http://example.com/cia5rherv00k8hkjxj72kcc3p/cia5rherv00k9hkjx0zy8vr32/cia5rherv00kahkjx6kmi5wtf/cia5rherv00kbhkjxh2ut5f4w', + 'http://example.com/cia5rherv00kchkjxq9ki0lgr/cia5rherv00kdhkjx8mafuept/cia5rherv00kehkjx3kffqdj5/cia5rherv00kfhkjxk30tt2a0', + 'http://example.com/cia5rherv00kghkjxaw98qyx2/cia5rherv00khhkjxieg0bycx/cia5rherw00kihkjxdmdkp4x9/cia5rherw00kjhkjxijtzapeq', + 'http://example.com/cia5rherw00kkhkjxh1oh0kl4/cia5rherw00klhkjx6ffidsax/cia5rherw00kmhkjxqtgyxcy0/cia5rherw00knhkjx2xyipie3', + 'http://example.com/cia5rherw00kohkjxdq8vr7d0/cia5rherw00kphkjxes36xbks/cia5rherw00kqhkjx4ektym79/cia5rherw00krhkjxib4btsp0', + 'http://example.com/cia5rherw00kshkjxpc4m0m9e/cia5rherw00kthkjxgdi6nnzj/cia5rherw00kuhkjxcwj7gk86/cia5rherw00kvhkjx7cj949ld', + 'http://example.com/cia5rherw00kwhkjxmwj8brl1/cia5rherw00kxhkjxae8yeb7p/cia5rherw00kyhkjxqze289bb/cia5rherw00kzhkjx2jy2h9ji', + 'http://example.com/cia5rherw00l0hkjxxadfzs3n/cia5rherw00l1hkjxhngih2c7/cia5rherw00l2hkjxn5zwgxrx/cia5rherw00l3hkjxz3mp1gb9', + 'http://example.com/cia5rherw00l4hkjxhcw0erdu/cia5rherw00l5hkjxj6j3l2zh/cia5rherw00l6hkjx533r5z4r/cia5rherw00l7hkjxvjjbvgo6', + 'http://example.com/cia5rherw00l8hkjxx61zlkek/cia5rherw00l9hkjxb7xj73l2/cia5rherw00lahkjxrpt6fk4m/cia5rherw00lbhkjxlron4prb', + 'http://example.com/cia5rherw00lchkjxtq99vutt/cia5rherw00ldhkjxon95rdss/cia5rherw00lehkjxblctcrfw/cia5rherw00lfhkjxc3mffk2x', + 'http://example.com/cia5rherw00lghkjx4v9dujig/cia5rherw00lhhkjx734cxan0/cia5rherw00lihkjxlxx3bx3y/cia5rherw00ljhkjxa5te3vm6', + 'http://example.com/cia5rherw00lkhkjxy4akb3lo/cia5rherw00llhkjxxg2t6ecs/cia5rherw00lmhkjx5v0ur1zi/cia5rherw00lnhkjxuw4grqgg', + 'http://example.com/cia5rherw00lohkjx1tx7bq0f/cia5rherw00lphkjx118uu54q/cia5rherw00lqhkjxo63inpim/cia5rherw00lrhkjx60iprrlh', + 'http://example.com/cia5rherw00lshkjxq67z9znh/cia5rherw00lthkjxknfimvv6/cia5rherw00luhkjxln059pxx/cia5rherw00lvhkjx0b5u1wvh', + 'http://example.com/cia5rherw00lwhkjxdnbo7o8x/cia5rherw00lxhkjxialw39ph/cia5rherw00lyhkjxq44794oz/cia5rherw00lzhkjx063oj379', + 'http://example.com/cia5rherw00m0hkjxa51hcx2d/cia5rherw00m1hkjx5udmt5pw/cia5rherw00m2hkjxh2zenkxy/cia5rherw00m3hkjxaa4b1ukf', + 'http://example.com/cia5rherw00m4hkjxu0txyp2a/cia5rherw00m5hkjxku5t6nia/cia5rherw00m6hkjxpuvkxgxe/cia5rherw00m7hkjx7mgdnt4i', + 'http://example.com/cia5rherw00m8hkjx9tjak8nq/cia5rherw00m9hkjx6tjbc4c1/cia5rherw00mahkjxhrlq09fa/cia5rherw00mbhkjxncngvvyl', + 'http://example.com/cia5rherw00mchkjxqjijeepd/cia5rherw00mdhkjx5n3u57g1/cia5rherw00mehkjxlhfhx86m/cia5rherw00mfhkjxhtz36qq5', + 'http://example.com/cia5rherw00mghkjxiknnl4br/cia5rherw00mhhkjxl1kv5f98/cia5rherw00mihkjxzo6yez34/cia5rherw00mjhkjx2ffeb3lh', + 'http://example.com/cia5rherw00mkhkjxziimpsbk/cia5rherw00mlhkjxp3bcocf5/cia5rherw00mmhkjxbwztbp6k/cia5rherw00mnhkjxyc9eixhz', + 'http://example.com/cia5rherw00mohkjxm8kai7n5/cia5rherw00mphkjx65gvhaf2/cia5rherw00mqhkjxfh4vu8eq/cia5rherw00mrhkjxfv2gj9bt', + 'http://example.com/cia5rherw00mshkjxmnuhrdj4/cia5rherw00mthkjxjj5iizh6/cia5rherw00muhkjxiw62tfl7/cia5rherw00mvhkjxhzemezd3', + 'http://example.com/cia5rherw00mwhkjxlml9rs61/cia5rherw00mxhkjxl2mbwrnp/cia5rherw00myhkjxj96ye1zv/cia5rherw00mzhkjxodju2h5o', + 'http://example.com/cia5rherw00n0hkjxuivmtahu/cia5rherw00n1hkjx8gbwu61v/cia5rherw00n2hkjx1j9gzlce/cia5rherw00n3hkjxaz67uu2l', + 'http://example.com/cia5rherw00n4hkjxoz2v47cp/cia5rherw00n5hkjxc2m0xq1d/cia5rherw00n6hkjxs5pej7ym/cia5rherw00n7hkjxrj28b2tt', + 'http://example.com/cia5rherw00n8hkjxikm4mbwi/cia5rherw00n9hkjxcfsu9f9d/cia5rherw00nahkjxawec3u0q/cia5rherw00nbhkjx38j8uycv', + 'http://example.com/cia5rherw00nchkjxpxks6dv1/cia5rherw00ndhkjxme2uh9cm/cia5rherw00nehkjxiu6s9qet/cia5rherw00nfhkjxjeycmmn7', + 'http://example.com/cia5rherw00nghkjx66hdwbtg/cia5rherw00nhhkjxki3i81i4/cia5rherw00nihkjxvve944gl/cia5rherw00njhkjx3jwe46il', + 'http://example.com/cia5rherw00nkhkjx8znncv54/cia5rherw00nlhkjxsyqrjo5g/cia5rherw00nmhkjxkir8br45/cia5rherw00nnhkjxexumdwa2', + 'http://example.com/cia5rherw00nohkjxzs49c8vj/cia5rherw00nphkjx5eq7mllr/cia5rherw00nqhkjxxqc7wn0n/cia5rherw00nrhkjx3xnlk87f', + 'http://example.com/cia5rherw00nshkjxis89wcyt/cia5rherw00nthkjxmo962d98/cia5rherw00nuhkjx5102qsyd/cia5rherw00nvhkjx4dhdkq7d', + 'http://example.com/cia5rherw00nwhkjx6o2era0y/cia5rherw00nxhkjxibwv5nl3/cia5rherw00nyhkjxnobkkmmt/cia5rherw00nzhkjx2rfj3yw8', + 'http://example.com/cia5rherw00o0hkjxys1f6cpt/cia5rherw00o1hkjxv7y1wk16/cia5rherw00o2hkjx72rwd3p7/cia5rherw00o3hkjxdpw2fjf4', + 'http://example.com/cia5rherw00o4hkjxj0qwjpur/cia5rherw00o5hkjxr0vma4yn/cia5rherw00o6hkjxjdio6m66/cia5rherw00o7hkjx6tah5wos', + 'http://example.com/cia5rherw00o8hkjxtseqnpix/cia5rherw00o9hkjxcll29qxs/cia5rherw00oahkjxy6j4514j/cia5rherw00obhkjxyt12uhto', + 'http://example.com/cia5rherw00ochkjxv94azfo3/cia5rherw00odhkjxvvb9a4cf/cia5rherw00oehkjxro8j0dau/cia5rherw00ofhkjxazh1bnkv', + 'http://example.com/cia5rherw00oghkjxaiqblgaz/cia5rherw00ohhkjxq0jg2ekb/cia5rherw00oihkjxhxp787hh/cia5rherw00ojhkjxwn2bygkc', + 'http://example.com/cia5rherw00okhkjxrvuzh590/cia5rherw00olhkjxukov5sdm/cia5rherw00omhkjxprnz66kz/cia5rherw00onhkjxd19rb8es', + 'http://example.com/cia5rherw00oohkjxdp73iojj/cia5rherw00ophkjx0ejclbnc/cia5rherw00oqhkjxzbrto5j9/cia5rherw00orhkjx64ybndrk', + 'http://example.com/cia5rherw00oshkjxto8g8o9d/cia5rherw00othkjx0siti4zs/cia5rherw00ouhkjxn4k2rxwd/cia5rherw00ovhkjxlfh7kchr', + 'http://example.com/cia5rherw00owhkjx5u0y9ly7/cia5rherw00oxhkjxuc2947n2/cia5rherw00oyhkjxclqlf4cz/cia5rherw00ozhkjxw4ljr3n2', + 'http://example.com/cia5rherw00p0hkjxprwdryfg/cia5rherw00p1hkjxnm9mqxjr/cia5rherw00p2hkjxo97f8u6g/cia5rherw00p3hkjxu3v7vxny', + 'http://example.com/cia5rherw00p4hkjx52wsz26a/cia5rherw00p5hkjxbcznr0do/cia5rherw00p6hkjxytrdjjnt/cia5rherw00p7hkjx6l5uvkab', + 'http://example.com/cia5rherw00p8hkjxm2zue659/cia5rherx00p9hkjxbkzeky4l/cia5rherx00pahkjxrqp8ljqj/cia5rherx00pbhkjxuxod17kg', + 'http://example.com/cia5rherx00pchkjxa26ekxyy/cia5rherx00pdhkjx0kxle2w1/cia5rherx00pehkjxxugq6n6r/cia5rherx00pfhkjx2ucx69kc', + 'http://example.com/cia5rherx00pghkjxnqt22yl7/cia5rherx00phhkjxuxv53gmq/cia5rherx00pihkjxcb70hxmq/cia5rherx00pjhkjxlvjdb6xl', + 'http://example.com/cia5rherx00pkhkjxjc8qkw9h/cia5rherx00plhkjxcbs6t9k7/cia5rherx00pmhkjxh6cijkuv/cia5rherx00pnhkjxpf14evbg', + 'http://example.com/cia5rherx00pohkjxfgxqyo6d/cia5rherx00pphkjxtpilb91o/cia5rherx00pqhkjxny7abx9y/cia5rherx00prhkjx0l5g83bc', + 'http://example.com/cia5rherx00pshkjxxuxfrdbr/cia5rherx00pthkjxxnj6u6sk/cia5rherx00puhkjxpubm3g6s/cia5rherx00pvhkjxyj7u9c6t', + 'http://example.com/cia5rherx00pwhkjx6ppgibkl/cia5rherx00pxhkjxvv4p4kry/cia5rherx00pyhkjxpkbho8g0/cia5rherx00pzhkjxivjo0784', + 'http://example.com/cia5rherx00q0hkjxffxtz7i5/cia5rherx00q1hkjxygfowug9/cia5rherx00q2hkjxvsau87zx/cia5rherx00q3hkjx6z1kw9b2', + 'http://example.com/cia5rherx00q4hkjx4auglr08/cia5rherx00q5hkjxno848f23/cia5rherx00q6hkjxy6y8cv6y/cia5rherx00q7hkjxzzoogxhg', + 'http://example.com/cia5rherx00q8hkjx70m64of5/cia5rherx00q9hkjxg5c1aukr/cia5rherx00qahkjxqn1h5a85/cia5rherx00qbhkjxg2cf36ig', + 'http://example.com/cia5rherx00qchkjxlwf1o9ji/cia5rherx00qdhkjx1gdhjcsr/cia5rherx00qehkjx172b5dpn/cia5rherx00qfhkjxe3uruk25', + 'http://example.com/cia5rherx00qghkjx2ptty8ex/cia5rherx00qhhkjx5latps1q/cia5rherx00qihkjx9bdo19z2/cia5rherx00qjhkjxfj5a849t', + 'http://example.com/cia5rherx00qkhkjxataa91qp/cia5rherx00qlhkjxkos37rp2/cia5rherx00qmhkjxnr52z1ck/cia5rherx00qnhkjxg2wv4j3b', + 'http://example.com/cia5rherx00qohkjx5zybfy6z/cia5rherx00qphkjx3jotkzjk/cia5rherx00qqhkjxzqnuoxc1/cia5rherx00qrhkjxjqx7n6dd', + 'http://example.com/cia5rherx00qshkjxfusl4u8i/cia5rherx00qthkjx7nax9k3j/cia5rherx00quhkjxce6sda7o/cia5rherx00qvhkjxvjw9krhf', + 'http://example.com/cia5rherx00qwhkjx3myek18h/cia5rherx00qxhkjxye1vc3g5/cia5rherx00qyhkjx0qnkwhv8/cia5rherx00qzhkjx7xbpfb2g', + 'http://example.com/cia5rherx00r0hkjxr05rkysp/cia5rherx00r1hkjx5skxve27/cia5rherx00r2hkjxm42ww2wl/cia5rherx00r3hkjxvgaok3d7', + 'http://example.com/cia5rherx00r4hkjxhqn73qfk/cia5rherx00r5hkjx0vhqzi3y/cia5rherx00r6hkjxzo83uw13/cia5rherx00r7hkjxshtbkjap', + 'http://example.com/cia5rherx00r8hkjxkolk05tr/cia5rherx00r9hkjx87txftcu/cia5rherx00rahkjx7zt1mxfl/cia5rherx00rbhkjxj3225obu', + 'http://example.com/cia5rherx00rchkjxec3j4cbw/cia5rherx00rdhkjx2o60bre2/cia5rherx00rehkjx1gza5jo1/cia5rherx00rfhkjx6i15t347', + 'http://example.com/cia5rherx00rghkjxqsx1tilb/cia5rherx00rhhkjxld7q3ees/cia5rherx00rihkjxywwmwh7a/cia5rherx00rjhkjxa7lzkj0b', + 'http://example.com/cia5rherx00rkhkjxo2uekt9y/cia5rherx00rlhkjxz1y92fdx/cia5rherx00rmhkjxrls010bq/cia5rherx00rnhkjx5phg7y9q', + 'http://example.com/cia5rherx00rohkjxt69rlmpb/cia5rherx00rphkjxxdo9vbof/cia5rherx00rqhkjxgnvs2gxf/cia5rherx00rrhkjx5vv7kk7v', + 'http://example.com/cia5rherx00rshkjxfn0v3dx4/cia5rherx00rthkjx5wkktp8p/cia5rherx00ruhkjx6palbn57/cia5rherx00rvhkjxugk01wvb', + 'http://example.com/cia5rherx00rwhkjxzgza4olt/cia5rherx00rxhkjxwdssdcbj/cia5rherx00ryhkjxxnyxv3t6/cia5rherx00rzhkjxhr6w38i4', + 'http://example.com/cia5rherx00s0hkjxerf3k9ib/cia5rherx00s1hkjxnqqw079u/cia5rherx00s2hkjxifw1h8n3/cia5rherx00s3hkjxd05rx85o', + 'http://example.com/cia5rherx00s4hkjx2x89mh5w/cia5rherx00s5hkjx87d0h7li/cia5rherx00s6hkjxqjueoeqw/cia5rherx00s7hkjxyq9w3n82', + 'http://example.com/cia5rherx00s8hkjxzigd1zp5/cia5rherx00s9hkjxdtx2amst/cia5rherx00sahkjx2onc2f21/cia5rherx00sbhkjx4hgu22zb', + 'http://example.com/cia5rherx00schkjxrz7trjeu/cia5rherx00sdhkjxwmrp365i/cia5rherx00sehkjx7eq317yf/cia5rherx00sfhkjxo93dnhcw', + 'http://example.com/cia5rherx00sghkjx2zsmv5zb/cia5rherx00shhkjxuu1u80qs/cia5rherx00sihkjx8avektr4/cia5rherx00sjhkjxpg3tjre5', + 'http://example.com/cia5rherx00skhkjxm9hrr8dp/cia5rherx00slhkjxmklu8fxx/cia5rherx00smhkjxpz58b4co/cia5rherx00snhkjx6kflkbwz', + 'http://example.com/cia5rherx00sohkjx6zveco0b/cia5rherx00sphkjx9tzv10q9/cia5rherx00sqhkjx5kw9y9vt/cia5rherx00srhkjx5q2dpc7o', + 'http://example.com/cia5rherx00sshkjxfmf3zzhg/cia5rherx00sthkjx085rnzf5/cia5rherx00suhkjxo7eaxytl/cia5rherx00svhkjx3bfbsur9', + 'http://example.com/cia5rherx00swhkjxzihyd64f/cia5rherx00sxhkjxor1bcxl3/cia5rhery00syhkjxlensj1wa/cia5rhery00szhkjxwk1jdpzj', + 'http://example.com/cia5rhery00t0hkjxz5hf0kfl/cia5rhery00t1hkjx36ar1r16/cia5rhery00t2hkjxcv7t3hqq/cia5rhery00t3hkjxkzgqe0a6', + 'http://example.com/cia5rhery00t4hkjxpmbibq16/cia5rhery00t5hkjxrqr5n4lt/cia5rhery00t6hkjx4npmmnvj/cia5rhery00t7hkjxewqanavg', + 'http://example.com/cia5rhery00t8hkjxci9wud4s/cia5rhery00t9hkjxui809qxy/cia5rhery00tahkjx870oqect/cia5rhery00tbhkjx8g24crc0', + 'http://example.com/cia5rhery00tchkjxzjllkr5i/cia5rhery00tdhkjxqnnjio0r/cia5rhery00tehkjxnice4c5a/cia5rhery00tfhkjx0b0tfkbd', + 'http://example.com/cia5rhery00tghkjx1hfn5jnm/cia5rhery00thhkjx0m68lrh0/cia5rhery00tihkjxe24uktvm/cia5rhery00tjhkjxwudbxvgf', + 'http://example.com/cia5rhery00tkhkjxyupjrqmt/cia5rhery00tlhkjxns9kt84a/cia5rhery00tmhkjxnjnkvsza/cia5rhery00tnhkjx2u1kf42m', + 'http://example.com/cia5rhery00tohkjxv8euxxvv/cia5rhery00tphkjxcewtixg8/cia5rhery00tqhkjxm0fvhod1/cia5rhery00trhkjxzfels6hy', + 'http://example.com/cia5rhery00tshkjxcnmhpytv/cia5rhery00tthkjxgmwy284j/cia5rhery00tuhkjxvl9f0bet/cia5rhery00tvhkjxvd9h00tu', + 'http://example.com/cia5rhery00twhkjxworddumj/cia5rhery00txhkjx5hn3bob3/cia5rhery00tyhkjxwxhy5o31/cia5rhery00tzhkjxy7swe892', + 'http://example.com/cia5rhery00u0hkjx9v2rskyu/cia5rhery00u1hkjxt65535lp/cia5rhery00u2hkjxiephk9x0/cia5rhery00u3hkjxylul9icr', + 'http://example.com/cia5rhery00u4hkjxo1tucbyj/cia5rhery00u5hkjxchfewpdu/cia5rhery00u6hkjxzh1f7dsv/cia5rhery00u7hkjxow4myzvc', + 'http://example.com/cia5rhery00u8hkjxjkljwzmx/cia5rhery00u9hkjxb2hq0zff/cia5rhery00uahkjx3zil5iye/cia5rhery00ubhkjxpx6l4i62', + 'http://example.com/cia5rhery00uchkjxzybwg1aj/cia5rhery00udhkjxbc85v998/cia5rhery00uehkjx9x8k1ebt/cia5rhery00ufhkjxziinfjvs', + 'http://example.com/cia5rhery00ughkjxqjq7rbqe/cia5rhery00uhhkjxn422gi7s/cia5rhery00uihkjx6wdh0kru/cia5rhery00ujhkjxgx2z6i30', + 'http://example.com/cia5rhery00ukhkjxxwekiqd8/cia5rhery00ulhkjxti0u03ji/cia5rhery00umhkjxneh94911/cia5rhery00unhkjx5uothdt2', + 'http://example.com/cia5rhery00uohkjxh8wvz750/cia5rhery00uphkjxya408v8j/cia5rhery00uqhkjxowcw4c0j/cia5rhery00urhkjxp0hxhyjr', + 'http://example.com/cia5rhery00ushkjxj5ezkt47/cia5rhery00uthkjxfcbhp09u/cia5rhery00uuhkjxw14otqjw/cia5rhery00uvhkjxhkdyfxrs', + 'http://example.com/cia5rhery00uwhkjxehm078n0/cia5rhery00uxhkjxbadu0bio/cia5rhery00uyhkjxak1ocp8h/cia5rhery00uzhkjxing0qkah', + 'http://example.com/cia5rhery00v0hkjxl8s5to3x/cia5rhery00v1hkjx9t9obxjk/cia5rhery00v2hkjx37ndtfyo/cia5rhery00v3hkjxgsrqx8wo', + 'http://example.com/cia5rhery00v4hkjxcv3mqk3k/cia5rhery00v5hkjxm8gbw43x/cia5rhery00v6hkjxu55dmspc/cia5rhery00v7hkjx34me677j', + 'http://example.com/cia5rhery00v8hkjxwecik8go/cia5rhery00v9hkjxap89471j/cia5rhery00vahkjxllo77l7s/cia5rhery00vbhkjxrqbtypmt', + 'http://example.com/cia5rhery00vchkjx4uu12kzr/cia5rhery00vdhkjx5sxrg1cw/cia5rhery00vehkjxpimi78w5/cia5rhery00vfhkjxvu1h0bnc', + 'http://example.com/cia5rhery00vghkjxryq1kku2/cia5rhery00vhhkjx8yq7g6dg/cia5rhery00vihkjxrhoe95rs/cia5rhery00vjhkjxg036tj71', + 'http://example.com/cia5rhery00vkhkjxfk603ubw/cia5rhery00vlhkjxv6cvncpa/cia5rhery00vmhkjxtlptcbfj/cia5rhery00vnhkjx019qtozs', + 'http://example.com/cia5rhery00vohkjx55roqfyq/cia5rhery00vphkjxq51jjd0w/cia5rhery00vqhkjxzl5r049u/cia5rhery00vrhkjx8w085tma', + 'http://example.com/cia5rhery00vshkjx43gw6sxr/cia5rhery00vthkjx20l8em51/cia5rhery00vuhkjxdwoh2vk9/cia5rhery00vvhkjxw1g6vrut', + 'http://example.com/cia5rhery00vwhkjxvn2ebxzm/cia5rhery00vxhkjxz2hlqhzd/cia5rhery00vyhkjxqvadz9wb/cia5rhery00vzhkjx9rh2a3uh', + 'http://example.com/cia5rhery00w0hkjxa0rqkpov/cia5rhery00w1hkjxp833u09z/cia5rhery00w2hkjxn2awahj4/cia5rhery00w3hkjxwqcb9cgd', + 'http://example.com/cia5rhery00w4hkjx2qn7xtr0/cia5rhery00w5hkjx4mw9p5o5/cia5rhery00w6hkjx1h6f1nne/cia5rhery00w7hkjxqgof32en', + 'http://example.com/cia5rhery00w8hkjx40u225qd/cia5rhery00w9hkjx6jc1e5lj/cia5rhery00wahkjxqqn44l7k/cia5rhery00wbhkjxdwnm2lan', + 'http://example.com/cia5rhery00wchkjxmf3n36f9/cia5rhery00wdhkjxjitsfpkb/cia5rhery00wehkjxrwzzunbx/cia5rhery00wfhkjxlu5ar2zw', + 'http://example.com/cia5rhery00wghkjxkoktvbqj/cia5rhery00whhkjxjs6l9c1l/cia5rhery00wihkjx4c6l8wcz/cia5rhery00wjhkjxwrrx1kta', + 'http://example.com/cia5rhery00wkhkjx4o64pmsg/cia5rhery00wlhkjx5jwjko3n/cia5rhery00wmhkjxuj79fu0e/cia5rhery00wnhkjxdxjnmwu4', + 'http://example.com/cia5rhery00wohkjxnj9yz95o/cia5rhery00wphkjxqs39n7we/cia5rhery00wqhkjxw7enbbt8/cia5rhery00wrhkjxhirvzsj8', + 'http://example.com/cia5rhery00wshkjxklfnj99r/cia5rhery00wthkjxyvxvxzxm/cia5rhery00wuhkjxselr08ti/cia5rhery00wvhkjx0n4m1gwb', + 'http://example.com/cia5rhery00wwhkjxidbq1641/cia5rhery00wxhkjxer0uj0bx/cia5rhery00wyhkjx3rj98a5m/cia5rhery00wzhkjxar7ucjyp', + 'http://example.com/cia5rhery00x0hkjxikq8tega/cia5rhery00x1hkjxgq9t00p7/cia5rhery00x2hkjxzczhf4ta/cia5rhery00x3hkjxs8rl0xle', + 'http://example.com/cia5rhery00x4hkjxtpgdpam7/cia5rhery00x5hkjxnudpwm02/cia5rhery00x6hkjx96jfugp6/cia5rhery00x7hkjxugyl5bsm', + 'http://example.com/cia5rhery00x8hkjx26k3912r/cia5rhery00x9hkjx8j1o37fy/cia5rhery00xahkjxcnx1kjtl/cia5rhery00xbhkjxws0y4q9u', + 'http://example.com/cia5rhery00xchkjxnceot2tu/cia5rhery00xdhkjxmfcanvsn/cia5rhery00xehkjxti3dt4zk/cia5rhery00xfhkjx4r9pxmk8', + 'http://example.com/cia5rhery00xghkjxcl0s61iu/cia5rhery00xhhkjxy85ou2fq/cia5rhery00xihkjx8p53n3u3/cia5rhery00xjhkjx6dzo2asw', + 'http://example.com/cia5rhery00xkhkjxcfvflel3/cia5rhery00xlhkjxjs82vnte/cia5rhery00xmhkjxrisis221/cia5rhery00xnhkjx92ojt9kd', + 'http://example.com/cia5rhery00xohkjxxu18v57w/cia5rhery00xphkjx5gfp65ut/cia5rhery00xqhkjx0zug4xqu/cia5rhery00xrhkjxxqj8k3ce', + 'http://example.com/cia5rhery00xshkjxbpshqaoi/cia5rhery00xthkjxthb498xj/cia5rhery00xuhkjxu6o0heam/cia5rhery00xvhkjxcxp7f3yh', + 'http://example.com/cia5rhery00xwhkjxre2n0fww/cia5rhery00xxhkjxtu8s69t6/cia5rhery00xyhkjx7q1fm4xo/cia5rhery00xzhkjx6o1nu6ga', + 'http://example.com/cia5rhery00y0hkjxlsfyk6o1/cia5rhery00y1hkjxpaqyucwy/cia5rhery00y2hkjxapp8yfj0/cia5rhery00y3hkjx0fnzbtnb', + 'http://example.com/cia5rhery00y4hkjxlm2p2v6d/cia5rhery00y5hkjx0vzxj59x/cia5rhery00y6hkjxkamlg3ck/cia5rhery00y7hkjxkayvfx3a', + 'http://example.com/cia5rhery00y8hkjxdwdcp5cs/cia5rherz00y9hkjxjb0teg8f/cia5rherz00yahkjxs1uzi2l6/cia5rherz00ybhkjxp449moik', + 'http://example.com/cia5rherz00ychkjxwvi52xlt/cia5rherz00ydhkjxr7g9xhla/cia5rherz00yehkjxmrqqq92m/cia5rherz00yfhkjxljrtx13a', + 'http://example.com/cia5rherz00yghkjx6sxe01ps/cia5rherz00yhhkjxh27l6kvi/cia5rherz00yihkjxbrazjcpk/cia5rherz00yjhkjxmi6ft3qb', + 'http://example.com/cia5rherz00ykhkjxifkk8p2x/cia5rherz00ylhkjxpqz9t3cb/cia5rherz00ymhkjxc00r45v0/cia5rherz00ynhkjxgnbgvycv', + 'http://example.com/cia5rherz00yohkjxmr9yq0jk/cia5rherz00yphkjxaamwbi9t/cia5rherz00yqhkjxv53l03jj/cia5rherz00yrhkjxsvkylos9', + 'http://example.com/cia5rherz00yshkjx3vhhy1ut/cia5rherz00ythkjxnx4cssnw/cia5rherz00yuhkjxyojj0lzk/cia5rherz00yvhkjxoyozoftr', + 'http://example.com/cia5rherz00ywhkjxl9hqei9p/cia5rherz00yxhkjxyefcfdtf/cia5rherz00yyhkjxpmcgonjp/cia5rherz00yzhkjxtp7r9f3r', + 'http://example.com/cia5rherz00z0hkjxrhistyei/cia5rherz00z1hkjxinya1udt/cia5rherz00z2hkjxt2uibesw/cia5rherz00z3hkjxrv504cf7', + 'http://example.com/cia5rherz00z4hkjxo0a9j311/cia5rherz00z5hkjxazvef1je/cia5rherz00z6hkjxb75qrdko/cia5rherz00z7hkjxnv1dgzal', + 'http://example.com/cia5rherz00z8hkjx8ny20q55/cia5rherz00z9hkjxwlj70f0m/cia5rherz00zahkjx42hyz0m2/cia5rherz00zbhkjxwotnxn7y', + 'http://example.com/cia5rherz00zchkjxa1tke93x/cia5rherz00zdhkjxzd8wghy0/cia5rherz00zehkjxba383v6a/cia5rherz00zfhkjxjkcs4bwl', + 'http://example.com/cia5rherz00zghkjxk8sklzkb/cia5rherz00zhhkjxfwy3q53n/cia5rherz00zihkjxvheuc2pr/cia5rherz00zjhkjxbzljm6zl', + 'http://example.com/cia5rherz00zkhkjxv84kqu85/cia5rherz00zlhkjxso9bvlw7/cia5rherz00zmhkjxib6w80kz/cia5rherz00znhkjx41d5nf7s', + 'http://example.com/cia5rherz00zohkjxdd3iy5vu/cia5rherz00zphkjxwad53vl4/cia5rherz00zqhkjx4ad3e109/cia5rherz00zrhkjx4r6bwsia', + 'http://example.com/cia5rherz00zshkjx1rzmmdvs/cia5rherz00zthkjx6165yn1j/cia5rherz00zuhkjxnefkdvqx/cia5rherz00zvhkjxgm3q4960', + 'http://example.com/cia5rherz00zwhkjx78quz63t/cia5rherz00zxhkjxdj7uhzb1/cia5rherz00zyhkjxgfrciagd/cia5rherz00zzhkjx2fhew3jm', + 'http://example.com/cia5rherz0100hkjxpoguaspc/cia5rherz0101hkjxj8wf0hvv/cia5rherz0102hkjxft1bu9bg/cia5rherz0103hkjxh7rc7icq', + 'http://example.com/cia5rherz0104hkjxl336njqd/cia5rherz0105hkjxq5od0pbq/cia5rherz0106hkjxtgk2vqld/cia5rherz0107hkjxnyquy58x', + 'http://example.com/cia5rherz0108hkjx0e8vlkuv/cia5rherz0109hkjx1w4ao3zl/cia5rherz010ahkjx0uvsczyx/cia5rherz010bhkjxctueaktb', + 'http://example.com/cia5rherz010chkjxgocyaoln/cia5rherz010dhkjxvym9xjpu/cia5rherz010ehkjxkmv9qd1h/cia5rherz010fhkjxux4kwy9m', + 'http://example.com/cia5rherz010ghkjxgjg51jll/cia5rherz010hhkjxheyloqok/cia5rherz010ihkjxocdihpf0/cia5rherz010jhkjxkjob7g4l', + 'http://example.com/cia5rherz010khkjxyqnrc520/cia5rherz010lhkjxyfmva03e/cia5rherz010mhkjx7pf5a1q1/cia5rherz010nhkjxvcbajwq5', + 'http://example.com/cia5rherz010ohkjxlj1peujy/cia5rherz010phkjxbe0edddw/cia5rherz010qhkjxese0v0d0/cia5rherz010rhkjx3z5sqmvd', + 'http://example.com/cia5rherz010shkjx031cq64v/cia5rherz010thkjxo31twbuq/cia5rherz010uhkjxx0w89wkt/cia5rherz010vhkjxunc21rd5', + 'http://example.com/cia5rherz010whkjxokxwgntu/cia5rherz010xhkjx8zhcummr/cia5rherz010yhkjxrr19wgdd/cia5rherz010zhkjx3krrrmfy', + 'http://example.com/cia5rherz0110hkjxsnag5gy8/cia5rherz0111hkjxti3yt0uo/cia5rherz0112hkjxuvxezwly/cia5rherz0113hkjx42tjs0w1', + 'http://example.com/cia5rherz0114hkjxhap5ggkp/cia5rherz0115hkjxhpvs28ez/cia5rherz0116hkjxchtzr6ub/cia5rherz0117hkjxldrskwe8', + 'http://example.com/cia5rherz0118hkjx3moumoc6/cia5rherz0119hkjx8jmsv1po/cia5rherz011ahkjx5tbk1781/cia5rherz011bhkjxda8axg94', + 'http://example.com/cia5rherz011chkjxpch14tnq/cia5rherz011dhkjx779uxhge/cia5rherz011ehkjxd1gossfl/cia5rherz011fhkjxe6fsxfle', + 'http://example.com/cia5rherz011ghkjxkv0e5x6o/cia5rherz011hhkjxrmm01lop/cia5rherz011ihkjxadtpfth3/cia5rherz011jhkjxmsbqnjtx', + 'http://example.com/cia5rherz011khkjxs9v8q989/cia5rherz011lhkjxbb47ojmz/cia5rherz011mhkjxfqrbtm2s/cia5rherz011nhkjxf0ukz3z7', + 'http://example.com/cia5rherz011ohkjx1ljxj85v/cia5rherz011phkjxzyc52kr1/cia5rherz011qhkjxx03aq7rt/cia5rherz011rhkjxib8yi6wz', + 'http://example.com/cia5rherz011shkjxelvnazea/cia5rherz011thkjx6ge3ekjc/cia5rherz011uhkjxj6spkxml/cia5rherz011vhkjx2g4n6l67', + 'http://example.com/cia5rherz011whkjxoa9tcq7o/cia5rherz011xhkjx0aa7y41v/cia5rherz011yhkjx56c9pqwk/cia5rherz011zhkjx5iy36w89', + 'http://example.com/cia5rherz0120hkjx04d4k3fi/cia5rherz0121hkjxvl6nw4m5/cia5rherz0122hkjxxc1jbk55/cia5rherz0123hkjx0pjnf99r', + 'http://example.com/cia5rherz0124hkjx91z4pze5/cia5rherz0125hkjxg7qc4u38/cia5rherz0126hkjx1yegfvw4/cia5rherz0127hkjxl15ygp9h', + 'http://example.com/cia5rherz0128hkjx4tkyj2om/cia5rherz0129hkjx7h7oqbrp/cia5rherz012ahkjxkajenzcs/cia5rherz012bhkjxi4bowdxs', + 'http://example.com/cia5rherz012chkjx2szlut25/cia5rherz012dhkjxupa098p0/cia5rherz012ehkjxlg93n5ca/cia5rherz012fhkjx3e3lqc5s', + 'http://example.com/cia5rherz012ghkjxiypurty6/cia5rherz012hhkjxnpuba7yn/cia5rherz012ihkjxqajb87r0/cia5rherz012jhkjx3w8tfq58', + 'http://example.com/cia5rherz012khkjx6kpvfmqc/cia5rherz012lhkjxle4ozx4b/cia5rherz012mhkjxcx3zh6l8/cia5rherz012nhkjxwhol1p7z', + 'http://example.com/cia5rherz012ohkjx8hcbuea8/cia5rherz012phkjxjn78pjpu/cia5rherz012qhkjxcso0w7ob/cia5rherz012rhkjxuwsrzjnb', + 'http://example.com/cia5rherz012shkjxmfrs9kl4/cia5rherz012thkjxqd79q3jk/cia5rherz012uhkjx4y92c2l9/cia5rherz012vhkjxah5zn3ql', + 'http://example.com/cia5rherz012whkjxnoco7250/cia5rherz012xhkjx50xbnyn6/cia5rherz012yhkjxhiz0qo7f/cia5rherz012zhkjxpvm1udb0', + 'http://example.com/cia5rherz0130hkjxo3o4fndr/cia5rherz0131hkjxkz4fvrzq/cia5rherz0132hkjxz7tax104/cia5rherz0133hkjx1g0g06dn', + 'http://example.com/cia5rherz0134hkjx5y2031n3/cia5rherz0135hkjx7uqpqo6r/cia5rherz0136hkjxy33y4m0i/cia5rherz0137hkjxi2zwuxom', + 'http://example.com/cia5rherz0138hkjx2r63kcp2/cia5rherz0139hkjxqhukego2/cia5rherz013ahkjxrct5kwo5/cia5rherz013bhkjxlwdsrkdb', + 'http://example.com/cia5rherz013chkjxqb7drtld/cia5rherz013dhkjxhk7nzkp6/cia5rherz013ehkjxt59enx4r/cia5rherz013fhkjxmequgudl', + 'http://example.com/cia5rherz013ghkjxsvtndprv/cia5rherz013hhkjx5qzj9yky/cia5rherz013ihkjx7wi51091/cia5rherz013jhkjx07qd1vho', + 'http://example.com/cia5rherz013khkjxxhuteqrg/cia5rherz013lhkjxnytihmq0/cia5rherz013mhkjxxcuyopb7/cia5rherz013nhkjxz8wjm6kv', + 'http://example.com/cia5rherz013ohkjxti624ceh/cia5rherz013phkjx9es0m8z1/cia5rherz013qhkjx2thq0yiq/cia5rherz013rhkjxcz1h935h', + 'http://example.com/cia5rherz013shkjxy1fcs2p9/cia5rherz013thkjxqj1f3hzd/cia5rherz013uhkjx9n3img9m/cia5rherz013vhkjxbbsd9s7u', + 'http://example.com/cia5rherz013whkjxzrks74yw/cia5rherz013xhkjx19u9gnum/cia5rherz013yhkjxf189dqov/cia5rherz013zhkjxn840ifqp', + 'http://example.com/cia5rherz0140hkjxzflkd8o3/cia5rherz0141hkjxth1k5pcv/cia5rherz0142hkjxk4tx2d6t/cia5rherz0143hkjxua6in4hi', + 'http://example.com/cia5rherz0144hkjxh5223dp4/cia5rherz0145hkjxggdx0inf/cia5rherz0146hkjxukma20rn/cia5rherz0147hkjxbz6yr3vj', + 'http://example.com/cia5rherz0148hkjxj6yz49cp/cia5rherz0149hkjxnshaboc7/cia5rherz014ahkjx9k7w03oz/cia5rherz014bhkjxim3qdl32', + 'http://example.com/cia5rherz014chkjxkwm4bedt/cia5rherz014dhkjxd87owzz9/cia5rherz014ehkjx7gvsq5h8/cia5rherz014fhkjxvg5i2lzo', + 'http://example.com/cia5rhes0014ghkjx57nwx3bd/cia5rhes0014hhkjx7yg5lnmm/cia5rhes0014ihkjxpcj5y5pc/cia5rhes0014jhkjxx7pqjwyi', + 'http://example.com/cia5rhes0014khkjx8je94eu9/cia5rhes0014lhkjxj7s0ayqj/cia5rhes0014mhkjxaj8frq8f/cia5rhes0014nhkjxsvfwikzc', + 'http://example.com/cia5rhes0014ohkjx7w5zhsfa/cia5rhes0014phkjxb8znpn93/cia5rhes0014qhkjx26dojt2q/cia5rhes0014rhkjx4z51j3v1', + 'http://example.com/cia5rhes0014shkjxsrfqfh66/cia5rhes0014thkjxchkutc4l/cia5rhes0014uhkjx61hu197u/cia5rhes0014vhkjx88tbe055', + 'http://example.com/cia5rhes0014whkjxnsy6o8oh/cia5rhes0014xhkjxhf7qa11c/cia5rhes0014yhkjx8dg54bhs/cia5rhes0014zhkjxwddjjbfx', + 'http://example.com/cia5rhes00150hkjxvbx9bs0t/cia5rhes00151hkjx1ndja821/cia5rhes00152hkjxgre5jaft/cia5rhes00153hkjx403j16ab', + 'http://example.com/cia5rhes00154hkjx850qetqc/cia5rhes00155hkjx25fuxyq1/cia5rhes00156hkjxt0otyqf9/cia5rhes00157hkjxkrckit2g', + 'http://example.com/cia5rhes00158hkjxeka610dd/cia5rhes00159hkjxirohiw4g/cia5rhes0015ahkjx4a2e7hwj/cia5rhes0015bhkjx54ew959x', + 'http://example.com/cia5rhes0015chkjxdbymvixv/cia5rhes0015dhkjxehyc0l2p/cia5rhes0015ehkjxkkzsw7sr/cia5rhes0015fhkjx694x9jdr', + 'http://example.com/cia5rhes0015ghkjx8524c513/cia5rhes0015hhkjx9gbh0axg/cia5rhes0015ihkjxhux4m9va/cia5rhes0015jhkjxizfpn19a', + 'http://example.com/cia5rhes0015khkjxmy1viucg/cia5rhes0015lhkjx4k9cpi1x/cia5rhes0015mhkjxlwccit5i/cia5rhes0015nhkjxyyerl1x5', + 'http://example.com/cia5rhes0015ohkjxse6u4cq1/cia5rhes0015phkjxmbosv5k1/cia5rhes0015qhkjxagd18q9e/cia5rhes0015rhkjx18k37mza', + 'http://example.com/cia5rhes0015shkjxkf88qpr2/cia5rhes0015thkjxxjngloiv/cia5rhes0015uhkjxw3p51ph3/cia5rhes0015vhkjxuv117q6n', + 'http://example.com/cia5rhes0015whkjxepiuli5w/cia5rhes0015xhkjx6912ozju/cia5rhes0015yhkjxs50s4iw9/cia5rhes0015zhkjx4fqv3fj5', + 'http://example.com/cia5rhes00160hkjxaezek04y/cia5rhes00161hkjxo7jexmqa/cia5rhes00162hkjx8qt4t84r/cia5rhes00163hkjx0x35v1ea', + 'http://example.com/cia5rhes00164hkjxum5cztru/cia5rhes00165hkjxykw801f6/cia5rhes00166hkjx87cgbtl9/cia5rhes00167hkjxr5laneo4', + 'http://example.com/cia5rhes00168hkjx4675xx8q/cia5rhes00169hkjxa69bs98w/cia5rhes0016ahkjxgxbg1ktg/cia5rhes0016bhkjxcssrfeb6', + 'http://example.com/cia5rhes0016chkjxskrmxbeu/cia5rhes0016dhkjxchuf9w7d/cia5rhes0016ehkjx96tmup0w/cia5rhes0016fhkjx3b2ir9k8', + 'http://example.com/cia5rhes0016ghkjxshn9r5cd/cia5rhes0016hhkjxt0okuboo/cia5rhes0016ihkjx3xc5n1z7/cia5rhes0016jhkjxmm1rhinv', + 'http://example.com/cia5rhes0016khkjxs4md552m/cia5rhes0016lhkjxdqs7jlks/cia5rhes0016mhkjxsbqpbr27/cia5rhes0016nhkjxsw0eoqxh', + 'http://example.com/cia5rhes0016ohkjxj1uyawl4/cia5rhes0016phkjx4s3keqvp/cia5rhes0016qhkjxlr2ujty3/cia5rhes0016rhkjxshluxgzs', + 'http://example.com/cia5rhes0016shkjxmjm47a2n/cia5rhes0016thkjxvawl3vod/cia5rhes0016uhkjxlkwcmxrn/cia5rhes0016vhkjx0yrlq0k0', + 'http://example.com/cia5rhes0016whkjx47wacy0d/cia5rhes0016xhkjxx20x2adr/cia5rhes0016yhkjxpyafhbax/cia5rhes0016zhkjxkm7homqb', + 'http://example.com/cia5rhes00170hkjx58dppj32/cia5rhes00171hkjxgl9ekfiu/cia5rhes00172hkjx35hn4ajh/cia5rhes00173hkjx4k5x795i', + 'http://example.com/cia5rhes00174hkjx431k7e7c/cia5rhes00175hkjxdfvrfwqy/cia5rhes00176hkjx2wqmhu6x/cia5rhes00177hkjxd8ykmqj9', + 'http://example.com/cia5rhes00178hkjxpbwz0dv1/cia5rhes00179hkjxfxnx8xde/cia5rhes0017ahkjxnumozh8d/cia5rhes0017bhkjxjvgv7bu3', + 'http://example.com/cia5rhes0017chkjx6cbtaca4/cia5rhes0017dhkjxl6oa2n77/cia5rhes0017ehkjxv4e7c73p/cia5rhes0017fhkjxarapkb0w', + 'http://example.com/cia5rhes0017ghkjxvzviznq4/cia5rhes0017hhkjxkxkq2w0r/cia5rhes0017ihkjxfdhga9qz/cia5rhes0017jhkjxzr0zbpli', + 'http://example.com/cia5rhes0017khkjxd6x7dl0m/cia5rhes0017lhkjxpa472u8x/cia5rhes0017mhkjxi7scj2zd/cia5rhes0017nhkjxcrar3doc', + 'http://example.com/cia5rhes1017ohkjxim1b2tgs/cia5rhes1017phkjxido7zpq3/cia5rhes1017qhkjxzhgszmuh/cia5rhes1017rhkjxh9n4vlu9', + 'http://example.com/cia5rhes1017shkjxazazffwt/cia5rhes1017thkjxt1mu7dkg/cia5rhes1017uhkjx79p8vex3/cia5rhes1017vhkjxzk3rwfaj', + 'http://example.com/cia5rhes1017whkjxg9ldz44h/cia5rhes1017xhkjxubjc7d35/cia5rhes1017yhkjxyn9t58r7/cia5rhes1017zhkjx822o28jf', + 'http://example.com/cia5rhes10180hkjxy0zeitqi/cia5rhes10181hkjxuiumypud/cia5rhes10182hkjxqhf3xprn/cia5rhes10183hkjx9orcdf2t', + 'http://example.com/cia5rhes10184hkjx60vpjosn/cia5rhes10185hkjxiuxdbrjp/cia5rhes10186hkjxjazso4v3/cia5rhes10187hkjx1yda3p8i', + 'http://example.com/cia5rhes10188hkjx67qn30yn/cia5rhes10189hkjxd8e62yud/cia5rhes1018ahkjxr1ogihzw/cia5rhes1018bhkjxqa83yxs2', + 'http://example.com/cia5rhes1018chkjxcijm6ol2/cia5rhes1018dhkjxn27lkryl/cia5rhes1018ehkjxin74swtd/cia5rhes1018fhkjxc3n9hjub', + 'http://example.com/cia5rhes1018ghkjxs06i4n1v/cia5rhes1018hhkjxtbrrprdd/cia5rhes1018ihkjxh375u2d5/cia5rhes1018jhkjxwe4m1w3k', + 'http://example.com/cia5rhes1018khkjxc2fo3tyn/cia5rhes1018lhkjx11wqgr3o/cia5rhes1018mhkjx55cz73km/cia5rhes1018nhkjx027g05rh', + 'http://example.com/cia5rhes1018ohkjxuxt9w0qg/cia5rhes1018phkjxuppi0zpt/cia5rhes1018qhkjx3hedzfgq/cia5rhes1018rhkjxdbef85sg', + 'http://example.com/cia5rhes1018shkjxh23bn4hl/cia5rhes1018thkjx2jo33xt5/cia5rhes1018uhkjxgrf6z3q5/cia5rhes1018vhkjxs51u2bsq', + 'http://example.com/cia5rhes1018whkjxh98q42o4/cia5rhes1018xhkjxtktqtwob/cia5rhes1018yhkjxxf9qq7me/cia5rhes1018zhkjx540am2xr', + 'http://example.com/cia5rhes10190hkjxcom1s1af/cia5rhes10191hkjxr15i8zfz/cia5rhes10192hkjxbsyx6pqa/cia5rhes10193hkjx5lfk3tnz', + 'http://example.com/cia5rhes10194hkjx63khbmh5/cia5rhes10195hkjx23tzm25c/cia5rhes10196hkjx3tu55kps/cia5rhes10197hkjxt9kgwuye', + 'http://example.com/cia5rhes10198hkjxvsic8zyi/cia5rhes10199hkjxiqcxj6ha/cia5rhes1019ahkjxul53ymxv/cia5rhes1019bhkjx8j5i4gjo', + 'http://example.com/cia5rhes1019chkjxhkzab45h/cia5rhes1019dhkjxp9m537kv/cia5rhes1019ehkjxflgayfwl/cia5rhes1019fhkjxpxcfuwm7', + 'http://example.com/cia5rhes1019ghkjx0ec3mbfs/cia5rhes1019hhkjxpzum6b24/cia5rhes1019ihkjx8l7ygjw5/cia5rhes1019jhkjxc4kywxia', + 'http://example.com/cia5rhes1019khkjxcgdm2x8i/cia5rhes1019lhkjxsd4z7axk/cia5rhes1019mhkjxrivl4h0v/cia5rhes1019nhkjxwq5r7rjw', + 'http://example.com/cia5rhes1019ohkjxlbgrt2qs/cia5rhes1019phkjxrqx7xr97/cia5rhes1019qhkjxqxuaxnbc/cia5rhes1019rhkjx479nva7e', + 'http://example.com/cia5rhes1019shkjxo903skww/cia5rhes1019thkjx835fib01/cia5rhes1019uhkjxqnb5hb1c/cia5rhes1019vhkjx985hdr8a', + 'http://example.com/cia5rhes1019whkjxul29xzs7/cia5rhes1019xhkjx8v769rft/cia5rhes1019yhkjx8moz4avh/cia5rhes1019zhkjxltk1bmj1', + 'http://example.com/cia5rhes101a0hkjxgj1bgqcu/cia5rhes101a1hkjxam87hyv6/cia5rhes101a2hkjx6n7xfzcf/cia5rhes101a3hkjxwhuzx4bu', + 'http://example.com/cia5rhes101a4hkjxkz4gt4bb/cia5rhes101a5hkjx8jto6sbw/cia5rhes101a6hkjxdkz6q053/cia5rhes101a7hkjxafsa477k', + 'http://example.com/cia5rhes101a8hkjxmawq81f9/cia5rhes101a9hkjx7a1m25vl/cia5rhes101aahkjxn9vib2k1/cia5rhes101abhkjxnd9lr35m', + 'http://example.com/cia5rhes101achkjxaz60ife8/cia5rhes101adhkjxb9jyduwc/cia5rhes101aehkjximmqxxdc/cia5rhes101afhkjxrivbhs77', + 'http://example.com/cia5rhes101aghkjxrq2da1pd/cia5rhes101ahhkjxjcuopb44/cia5rhes101aihkjxmeqnm90k/cia5rhes101ajhkjxli2mp598', + 'http://example.com/cia5rhes101akhkjx3uzisjok/cia5rhes101alhkjx2z2rnozw/cia5rhes101amhkjxektigddg/cia5rhes101anhkjxvsxrqsn5', + 'http://example.com/cia5rhes101aohkjxfb78h44w/cia5rhes101aphkjx2g9hr8n2/cia5rhes101aqhkjx61plt1q1/cia5rhes101arhkjxajstaro6', + 'http://example.com/cia5rhes101ashkjxm5zox05g/cia5rhes101athkjxqnofipd6/cia5rhes101auhkjxr39j3scv/cia5rhes101avhkjxcnv43592', + 'http://example.com/cia5rhes101awhkjxi5amucip/cia5rhes101axhkjxy05rb4by/cia5rhes101ayhkjxoqbug0w2/cia5rhes101azhkjxobqv30io', + 'http://example.com/cia5rhes101b0hkjxgtxrq6a1/cia5rhes101b1hkjx3kckskk9/cia5rhes101b2hkjxgp3x3n2k/cia5rhes101b3hkjxjoa91opd', + 'http://example.com/cia5rhes101b4hkjxl0uryndo/cia5rhes101b5hkjxn8o6oumu/cia5rhes101b6hkjxrjbze70s/cia5rhes101b7hkjxv5mrjv5y', + 'http://example.com/cia5rhes101b8hkjx7yeg3s3m/cia5rhes101b9hkjxnchy3mil/cia5rhes101bahkjxawomopeo/cia5rhes101bbhkjx9oml99jy', + 'http://example.com/cia5rhes101bchkjxzaccplvr/cia5rhes101bdhkjxmv4u1l8n/cia5rhes101behkjxja90rgy0/cia5rhes101bfhkjxolfzxocw', + 'http://example.com/cia5rhes101bghkjxy1vfbaaw/cia5rhes101bhhkjxg6xznpan/cia5rhes101bihkjxlg9fzku8/cia5rhes101bjhkjxnh2hjnui', + 'http://example.com/cia5rhes101bkhkjxsclo2zp3/cia5rhes101blhkjxuvrcudv5/cia5rhes101bmhkjx605j2zjj/cia5rhes101bnhkjx2xml0fvu', + 'http://example.com/cia5rhes101bohkjx5gf3ijos/cia5rhes101bphkjxpe0su46e/cia5rhes101bqhkjxs22f7ad2/cia5rhes101brhkjx9agg7eo1', + 'http://example.com/cia5rhes101bshkjx3sn7g8yy/cia5rhes101bthkjxe04n3g8b/cia5rhes101buhkjxgy5w6ts0/cia5rhes101bvhkjx7q91193i', + 'http://example.com/cia5rhes101bwhkjxfgzjxdtg/cia5rhes101bxhkjx9fof34tp/cia5rhes101byhkjxoyqeyb8o/cia5rhes101bzhkjxs4h8rhgv', + 'http://example.com/cia5rhes101c0hkjxxj1zh5us/cia5rhes101c1hkjxoc2z40bk/cia5rhes101c2hkjxp2mh4dck/cia5rhes101c3hkjx11dpdt65', + 'http://example.com/cia5rhes101c4hkjxrfzebrql/cia5rhes101c5hkjxhgaz06ty/cia5rhes101c6hkjxrs79e85g/cia5rhes101c7hkjxggrhbqyx', + 'http://example.com/cia5rhes101c8hkjxvgxdgnbw/cia5rhes101c9hkjxyjttv60a/cia5rhes101cahkjxsq6fq2jl/cia5rhes101cbhkjx7q41av4q', + 'http://example.com/cia5rhes101cchkjxovipv8ev/cia5rhes101cdhkjxqmj2adv7/cia5rhes101cehkjxkac54kr0/cia5rhes101cfhkjxuqcmlixm', + 'http://example.com/cia5rhes101cghkjx8p3hy50r/cia5rhes101chhkjxitheakp9/cia5rhes101cihkjxrm1z2bcp/cia5rhes101cjhkjx6e60mdcr', + 'http://example.com/cia5rhes101ckhkjxavxp0z0w/cia5rhes101clhkjxo78s8ce3/cia5rhes101cmhkjx3q5plsy4/cia5rhes101cnhkjxbr2dyljs', + 'http://example.com/cia5rhes101cohkjxx6uzzh6z/cia5rhes101cphkjx39t0gmdt/cia5rhes101cqhkjx5cpi5gv2/cia5rhes101crhkjx8tiw2khg', + 'http://example.com/cia5rhes101cshkjxx26p2oew/cia5rhes101cthkjxh5x6cctw/cia5rhes101cuhkjxmqbok5qb/cia5rhes101cvhkjx98q4u6vg', + 'http://example.com/cia5rhes101cwhkjxca46qqdc/cia5rhes101cxhkjxkya9jblw/cia5rhes101cyhkjxsx55uj72/cia5rhes101czhkjx4px01ypv', + 'http://example.com/cia5rhes201d0hkjxrfq1bxuy/cia5rhes201d1hkjxum4pm3s6/cia5rhes201d2hkjx9djj6tvc/cia5rhes201d3hkjxkobt5p5a', + 'http://example.com/cia5rhes201d4hkjx6vbuy1h3/cia5rhes201d5hkjxtyrtq6sn/cia5rhes201d6hkjxyn0dbgeq/cia5rhes201d7hkjx9g1d2pu0', + 'http://example.com/cia5rhes201d8hkjxsci6f24w/cia5rhes201d9hkjxd8q6ugbk/cia5rhes201dahkjx8c8yunrs/cia5rhes201dbhkjxb657b9hh', + 'http://example.com/cia5rhes201dchkjxhpytp0es/cia5rhes201ddhkjxzz6or9dl/cia5rhes201dehkjxvt1iaj4e/cia5rhes201dfhkjxcovukh36', + 'http://example.com/cia5rhes201dghkjxs4wcuyr5/cia5rhes201dhhkjxa3ltvy94/cia5rhes201dihkjx1q3i72ys/cia5rhes201djhkjxjgthq1xi', + 'http://example.com/cia5rhes201dkhkjxsvsw8r7g/cia5rhes201dlhkjxho9dzz7z/cia5rhes201dmhkjxtd6y9lt9/cia5rhes201dnhkjx2mryfja5', + 'http://example.com/cia5rhes201dohkjx1qpsam6z/cia5rhes201dphkjxyqckmdus/cia5rhes201dqhkjx05x0cua4/cia5rhes201drhkjxlv48ezca', + 'http://example.com/cia5rhes201dshkjxv3tvrlnv/cia5rhes201dthkjxsb1vp68a/cia5rhes201duhkjxr3dpwbsl/cia5rhes201dvhkjxooy13asr', + 'http://example.com/cia5rhes201dwhkjxy2yxmf1a/cia5rhes201dxhkjxg7ddbk62/cia5rhes201dyhkjxyfw66d9i/cia5rhes201dzhkjxhiriqvpp', + 'http://example.com/cia5rhes201e0hkjxgnojdvfu/cia5rhes201e1hkjx35d46pkf/cia5rhes201e2hkjx8al6xyxc/cia5rhes201e3hkjxpm8o33n5', + 'http://example.com/cia5rhes201e4hkjxgmt6q22i/cia5rhes201e5hkjxcxujptph/cia5rhes201e6hkjxvjqvqv5y/cia5rhes201e7hkjx7frk9v00', + 'http://example.com/cia5rhes201e8hkjxwksc2h6k/cia5rhes201e9hkjxmrv2nebe/cia5rhes201eahkjxju4ycxem/cia5rhes201ebhkjxu63x5ai0', + 'http://example.com/cia5rhes201echkjxq4et8qb3/cia5rhes201edhkjxmawlqvb6/cia5rhes201eehkjx5mvbc5jf/cia5rhes201efhkjxf81g9ft0', + 'http://example.com/cia5rhes201eghkjxwc2n8rrz/cia5rhes201ehhkjx96jrb9qp/cia5rhes201eihkjxolmvqk0b/cia5rhes201ejhkjx8t4yxqdy', + 'http://example.com/cia5rhes201ekhkjxjj375p8m/cia5rhes201elhkjxd7n988u0/cia5rhes201emhkjx4sgv75jt/cia5rhes201enhkjx89v2rpwd', + 'http://example.com/cia5rhes201eohkjx441c02sl/cia5rhes201ephkjxicff9k4p/cia5rhes201eqhkjx5c7sjm4x/cia5rhes201erhkjxvl0a13y1', + 'http://example.com/cia5rhes201eshkjxf8inxrty/cia5rhes201ethkjxqjixrhe3/cia5rhes201euhkjx6cq543as/cia5rhes201evhkjxiq4rvbm6', + 'http://example.com/cia5rhes201ewhkjxpzr481o0/cia5rhes201exhkjxfqo3ya1u/cia5rhes201eyhkjxzaieceuz/cia5rhes201ezhkjxrp9aiyto', + 'http://example.com/cia5rhes201f0hkjxdxg04ktt/cia5rhes201f1hkjxc5xqh8w6/cia5rhes201f2hkjxxqt7mk69/cia5rhes201f3hkjxhz4mt35k', + 'http://example.com/cia5rhes201f4hkjxwif8ix73/cia5rhes201f5hkjxcnk41o2f/cia5rhes201f6hkjxqvmwkmte/cia5rhes201f7hkjxjhf0rwkd', + 'http://example.com/cia5rhes201f8hkjxgu0mbayd/cia5rhes201f9hkjxoirm2pi6/cia5rhes201fahkjx3eggxv1v/cia5rhes201fbhkjx3dr0v5lr', + 'http://example.com/cia5rhes201fchkjx9bl00653/cia5rhes201fdhkjxd986f7dy/cia5rhes201fehkjxcjhtezko/cia5rhes201ffhkjx2g6pp08r', + 'http://example.com/cia5rhes201fghkjxieabfnjf/cia5rhes201fhhkjxhncmkptc/cia5rhes201fihkjxd2idn405/cia5rhes201fjhkjxxh12k6dz', + 'http://example.com/cia5rhes201fkhkjxjhb8dl6c/cia5rhes201flhkjxjesmmxj5/cia5rhes201fmhkjxgdq4watu/cia5rhes201fnhkjxcx2v7046', + 'http://example.com/cia5rhes201fohkjxooyrgbd6/cia5rhes201fphkjxnswgkqhg/cia5rhes201fqhkjxr1olqtyi/cia5rhes201frhkjxpylkppc7', + 'http://example.com/cia5rhes201fshkjxss3f3m7a/cia5rhes201fthkjxtb752b31/cia5rhes201fuhkjxl8v5tked/cia5rhes201fvhkjxs83n3lna', + 'http://example.com/cia5rhes201fwhkjx7mn2ufyp/cia5rhes201fxhkjxykgvds9s/cia5rhes201fyhkjxzj880aau/cia5rhes201fzhkjx9hmzn5w1', + 'http://example.com/cia5rhes201g0hkjx40m23pfq/cia5rhes201g1hkjxu5axzq44/cia5rhes201g2hkjxtenwlezp/cia5rhes201g3hkjxeaanxtc3', + 'http://example.com/cia5rhes201g4hkjxfko2j17a/cia5rhes201g5hkjxdngk92iq/cia5rhes201g6hkjxyiixm3h3/cia5rhes201g7hkjx1e0o9rbr', + 'http://example.com/cia5rhes201g8hkjxzvuuf9mr/cia5rhes201g9hkjx9i4067eb/cia5rhes201gahkjxe0877b7o/cia5rhes201gbhkjxhjeqydx3', + 'http://example.com/cia5rhes201gchkjx28buxxph/cia5rhes201gdhkjx11mlvzu6/cia5rhes201gehkjx56z31f3p/cia5rhes201gfhkjxon8mxyaq', + 'http://example.com/cia5rhes201gghkjxzhavbsbu/cia5rhes201ghhkjxpalfbbgq/cia5rhes201gihkjxmg14pb0i/cia5rhes201gjhkjxz1k6lfox', + 'http://example.com/cia5rhes201gkhkjxr91y8n1x/cia5rhes201glhkjxcd8gf56b/cia5rhes201gmhkjxgmgi5aag/cia5rhes201gnhkjxzuskm0u3', + 'http://example.com/cia5rhes201gohkjxh6stmdzj/cia5rhes201gphkjxjhxmrc1z/cia5rhes201gqhkjxbsb6x26m/cia5rhes201grhkjx2qjq5azu', + 'http://example.com/cia5rhes201gshkjxwgykuiuh/cia5rhes201gthkjxshezzoh7/cia5rhes201guhkjxhxk5wn0c/cia5rhes201gvhkjxfgd3dy5o', + 'http://example.com/cia5rhes201gwhkjxrjdm59mt/cia5rhes201gxhkjx5p1au9tm/cia5rhes201gyhkjx2fhr9h8l/cia5rhes201gzhkjx1d6ey84l', + 'http://example.com/cia5rhes201h0hkjxw539qclb/cia5rhes201h1hkjxtasbgd4k/cia5rhes201h2hkjxnggs4jvi/cia5rhes201h3hkjx10i4oa01', + 'http://example.com/cia5rhes201h4hkjxc4yc7ah2/cia5rhes201h5hkjxpp8h6vjy/cia5rhes201h6hkjx3is219tv/cia5rhes201h7hkjxi5vczfdr', + 'http://example.com/cia5rhes201h8hkjx0pnfnjv8/cia5rhes201h9hkjxvab7mw12/cia5rhes201hahkjxpdkx31mo/cia5rhes201hbhkjx5qrrpii9', + 'http://example.com/cia5rhes201hchkjxxpstoh0r/cia5rhes201hdhkjxd3fqr26w/cia5rhes201hehkjxa89a8p00/cia5rhes201hfhkjxjb7dx816', + 'http://example.com/cia5rhes201hghkjxoundwtvv/cia5rhes201hhhkjxq0l8n544/cia5rhes201hihkjxfnxig9tq/cia5rhes201hjhkjxmthbhba3', + 'http://example.com/cia5rhes201hkhkjxh8del6ix/cia5rhes201hlhkjxbbkqryiz/cia5rhes201hmhkjxsrbt8rwc/cia5rhes201hnhkjxvjinr83g', + 'http://example.com/cia5rhes201hohkjxnm39jamh/cia5rhes201hphkjxbpdbg85s/cia5rhes201hqhkjxt4xsrvvw/cia5rhes201hrhkjx28uncmqm', + 'http://example.com/cia5rhes201hshkjx9y3havo3/cia5rhes201hthkjxsl3xf65k/cia5rhes201huhkjxevlc6mpu/cia5rhes201hvhkjxios9hjnc', + 'http://example.com/cia5rhes201hwhkjx1gqoupdx/cia5rhes201hxhkjxoezqmdn4/cia5rhes201hyhkjxsd34q556/cia5rhes201hzhkjxvkqinyu3', + 'http://example.com/cia5rhes201i0hkjxtuqp1qgj/cia5rhes201i1hkjxjq0bui86/cia5rhes201i2hkjxlu4behua/cia5rhes201i3hkjxbarxd26f', + 'http://example.com/cia5rhes201i4hkjx1dgyo81l/cia5rhes201i5hkjx3xb9oqcc/cia5rhes201i6hkjxgiyz2tkh/cia5rhes201i7hkjx6w3cspdt', + 'http://example.com/cia5rhes201i8hkjxgazao8qk/cia5rhes201i9hkjx9g0kulps/cia5rhes201iahkjxo3xkc8pd/cia5rhes201ibhkjx1dqefi47', + 'http://example.com/cia5rhes201ichkjxg5gkkixu/cia5rhes201idhkjxy4ocvh6v/cia5rhes201iehkjx4cyin399/cia5rhes201ifhkjxx2gjdbml', + 'http://example.com/cia5rhes201ighkjxhg0kk0p1/cia5rhes201ihhkjxt9erqxzy/cia5rhes201iihkjxorqialmn/cia5rhes201ijhkjxuj6s809s', + 'http://example.com/cia5rhes201ikhkjx0vne1gub/cia5rhes201ilhkjxvumlvx2e/cia5rhes201imhkjxkwp8knsu/cia5rhes201inhkjxi7n4t5yd', + 'http://example.com/cia5rhes201iohkjxzho5l61h/cia5rhes201iphkjxxe8fo8zr/cia5rhes201iqhkjxqnsimx8u/cia5rhes201irhkjxecgdhcvp', + 'http://example.com/cia5rhes201ishkjx2fi6dek5/cia5rhes201ithkjxf3i7k6mm/cia5rhes201iuhkjx0uioh430/cia5rhes201ivhkjxqw1gumyl', + 'http://example.com/cia5rhes201iwhkjxlqpv0zzb/cia5rhes201ixhkjxfxx80lsv/cia5rhes201iyhkjx6f1rt1ik/cia5rhes201izhkjx0pmgbqf9', + 'http://example.com/cia5rhes201j0hkjxi3jo8eqm/cia5rhes201j1hkjx8iahnhoa/cia5rhes201j2hkjxcp1bjuci/cia5rhes201j3hkjxushcyv9h', + 'http://example.com/cia5rhes201j4hkjxmfy3u8bq/cia5rhes201j5hkjxl6j0ozf5/cia5rhes201j6hkjx0jrm1hr3/cia5rhes201j7hkjxkgzgfuc2', + 'http://example.com/cia5rhes201j8hkjx6sbhqirt/cia5rhes201j9hkjx3jm8ttkr/cia5rhes201jahkjx4ww2jkjb/cia5rhes201jbhkjx2vwmj8mw', + 'http://example.com/cia5rhes201jchkjx6s6c38ry/cia5rhes201jdhkjxo5iduoju/cia5rhes201jehkjxl337z10k/cia5rhes201jfhkjxennzj2ed', + 'http://example.com/cia5rhes201jghkjx65xshc5s/cia5rhes201jhhkjxtrvnzhf5/cia5rhes201jihkjxers53yxq/cia5rhes201jjhkjxw8nisucr', + 'http://example.com/cia5rhes301jkhkjx7rpx2kp1/cia5rhes301jlhkjxa3840rux/cia5rhes301jmhkjx1943602l/cia5rhes301jnhkjxft42idno', + 'http://example.com/cia5rhes301johkjxoa2e62n8/cia5rhes301jphkjx8jfoflvc/cia5rhes301jqhkjxjt9rh0u6/cia5rhes301jrhkjxofoa9vq2', + 'http://example.com/cia5rhes301jshkjxnm4wl4ab/cia5rhes301jthkjx89ehf3ty/cia5rhes301juhkjxwubr4oap/cia5rhes301jvhkjxk2e8yz43', + 'http://example.com/cia5rhes301jwhkjx4bo3r583/cia5rhes301jxhkjxfmbottug/cia5rhes301jyhkjx86nc7v6s/cia5rhes301jzhkjx9h9x7167', + 'http://example.com/cia5rhes301k0hkjx0oc98odu/cia5rhes301k1hkjxdjynl4c1/cia5rhes301k2hkjxc471ye9i/cia5rhes301k3hkjxcawvse26', + 'http://example.com/cia5rhes301k4hkjxsqss9ydm/cia5rhes301k5hkjx9e2cz05j/cia5rhes301k6hkjxjb18jpjj/cia5rhes301k7hkjxai3k1edl', + 'http://example.com/cia5rhes301k8hkjxeqdbbtwd/cia5rhes301k9hkjxeliovzfz/cia5rhes301kahkjxt8kvwaw8/cia5rhes301kbhkjx334ytlc2', + 'http://example.com/cia5rhes301kchkjxl1lize37/cia5rhes301kdhkjxczqjsftr/cia5rhes301kehkjxercwjrhh/cia5rhes301kfhkjxdeb3fvgv', + 'http://example.com/cia5rhes301kghkjx0yk4gm2e/cia5rhes301khhkjxt4gdd4ly/cia5rhes301kihkjxegsqzb2u/cia5rhes301kjhkjxp1cug6e2', + 'http://example.com/cia5rhes301kkhkjxyhe6rxl6/cia5rhes301klhkjxqfebfzea/cia5rhes301kmhkjxrz0qs4pq/cia5rhes301knhkjxyskiwz5y', + 'http://example.com/cia5rhes301kohkjx3tpgvarg/cia5rhes301kphkjx19vihidz/cia5rhes301kqhkjxos60mu4k/cia5rhes301krhkjxbvenxr93', + 'http://example.com/cia5rhes301kshkjx9ysyvjir/cia5rhes301kthkjxrk2z4v9t/cia5rhes301kuhkjxitxi78qg/cia5rhes301kvhkjx6m0cf7dl', + 'http://example.com/cia5rhes301kwhkjxt4f6hr4z/cia5rhes301kxhkjxilxbilms/cia5rhes301kyhkjxotkf8aaj/cia5rhes301kzhkjx7czn8fdy', + 'http://example.com/cia5rhes301l0hkjxf9jhzc7i/cia5rhes301l1hkjx1by7b0y3/cia5rhes301l2hkjxoo8obxiq/cia5rhes301l3hkjxvoc40tkj', + 'http://example.com/cia5rhes301l4hkjxgjpxmlpv/cia5rhes301l5hkjx94yuj664/cia5rhes301l6hkjxr8e8y97y/cia5rhes301l7hkjxwznfxlhr', + 'http://example.com/cia5rhes301l8hkjxif8hgss9/cia5rhes301l9hkjxls026lu2/cia5rhes301lahkjx8g221cqp/cia5rhes301lbhkjx5nnfkl1o', + 'http://example.com/cia5rhes301lchkjx55outsg7/cia5rhes301ldhkjxa32ta3im/cia5rhes301lehkjxqzx1v4ag/cia5rhes301lfhkjxzs4h9iq8', + 'http://example.com/cia5rhes301lghkjx9zymf1is/cia5rhes301lhhkjxxi8tt4p0/cia5rhes301lihkjxwsqjsjxe/cia5rhes301ljhkjxa2tfzt6w', + 'http://example.com/cia5rhes301lkhkjxs9t7x1x9/cia5rhes301llhkjxo81fsok6/cia5rhes301lmhkjxgi0i3j3b/cia5rhes301lnhkjx5i1k3l6t', + 'http://example.com/cia5rhes301lohkjxj1t98ds7/cia5rhes301lphkjxdee7ecco/cia5rhes301lqhkjxgfslix18/cia5rhes301lrhkjx4w0teefo', + 'http://example.com/cia5rhes301lshkjxghppkl49/cia5rhes301lthkjx7b8lqwg7/cia5rhes301luhkjx9a10fkrm/cia5rhes301lvhkjxbbotm5de', + 'http://example.com/cia5rhes301lwhkjxmjdqwoun/cia5rhes301lxhkjxqscqyygp/cia5rhes301lyhkjx9v1twpxt/cia5rhes301lzhkjxedovrwz9', + 'http://example.com/cia5rhes301m0hkjxa79l057t/cia5rhes301m1hkjxi4lf4pam/cia5rhes301m2hkjxbc54aj2i/cia5rhes301m3hkjxh0uiocv9', + 'http://example.com/cia5rhes301m4hkjxehur1yoh/cia5rhes301m5hkjxa360j1dg/cia5rhes301m6hkjxxu566hbq/cia5rhes301m7hkjxx3ynzmno', + 'http://example.com/cia5rhes301m8hkjxboi8g565/cia5rhes301m9hkjxcw8d20dp/cia5rhes301mahkjxgep6vvnb/cia5rhes301mbhkjxaig0kixq', + 'http://example.com/cia5rhes301mchkjxrjfsemox/cia5rhes301mdhkjxwifr2cdy/cia5rhes301mehkjx0mfczty9/cia5rhes301mfhkjxw0d7du38', + 'http://example.com/cia5rhes301mghkjxma95b0fw/cia5rhes301mhhkjxe08g59uf/cia5rhes301mihkjx6uflwnsd/cia5rhes301mjhkjxrwodr62q', + 'http://example.com/cia5rhes301mkhkjx6rb2vspf/cia5rhes301mlhkjx95l13vr9/cia5rhes301mmhkjx8nf0whp6/cia5rhes301mnhkjxpa5k4qfz', + 'http://example.com/cia5rhes301mohkjx3lbw4jvc/cia5rhes301mphkjx80vx4999/cia5rhes301mqhkjxyyl6bvqt/cia5rhes301mrhkjxq4xrjjqk', + 'http://example.com/cia5rhes301mshkjx8bf0phng/cia5rhes301mthkjxorxsclwf/cia5rhes301muhkjxi92z3o3d/cia5rhes301mvhkjxc3pvc5j6', + 'http://example.com/cia5rhes301mwhkjxm1nnxnhb/cia5rhes301mxhkjxtkkdy3i1/cia5rhes301myhkjxcrbfcl0b/cia5rhes301mzhkjxq6p026u3', + 'http://example.com/cia5rhes301n0hkjx1c2gv11c/cia5rhes301n1hkjxxfi36cpe/cia5rhes301n2hkjx38o7jvti/cia5rhes301n3hkjx9x9p0qh6', + 'http://example.com/cia5rhes301n4hkjx04xmiymb/cia5rhes301n5hkjx1bimz4eh/cia5rhes301n6hkjxnjqswkw6/cia5rhes301n7hkjxx11qd98z', + 'http://example.com/cia5rhes301n8hkjxdgkuvg3u/cia5rhes301n9hkjx9408l8sy/cia5rhes301nahkjxbdy48hsf/cia5rhes301nbhkjx9gbl5y30', + 'http://example.com/cia5rhes301nchkjx4rj2l6gk/cia5rhes301ndhkjxw603iycn/cia5rhes301nehkjxq9p4xm5r/cia5rhes301nfhkjx17hqhk81', + 'http://example.com/cia5rhes301nghkjxm9macc2i/cia5rhes301nhhkjxubpvluxn/cia5rhes301nihkjxtznh1gve/cia5rhes301njhkjxwft7i8zr', + 'http://example.com/cia5rhes301nkhkjxf33xqqss/cia5rhes301nlhkjxt2nsxi7y/cia5rhes301nmhkjxx1k3jzgs/cia5rhes301nnhkjx377meb7x', + 'http://example.com/cia5rhes301nohkjx1iur2w22/cia5rhes301nphkjxrj1q40j2/cia5rhes301nqhkjxpv78uwi7/cia5rhes301nrhkjx74y43ako', + 'http://example.com/cia5rhes301nshkjxgqi4066n/cia5rhes301nthkjxzeax16t1/cia5rhes301nuhkjxkus1cy9e/cia5rhes301nvhkjxrk8s23la', + 'http://example.com/cia5rhes301nwhkjxtz5i4jno/cia5rhes301nxhkjxnve6r7to/cia5rhes301nyhkjxoy3cq981/cia5rhes301nzhkjxsteraq5a', + 'http://example.com/cia5rhes301o0hkjx0bjvkfri/cia5rhes301o1hkjxl3tpcl1b/cia5rhes301o2hkjxih6vk4ck/cia5rhes301o3hkjxh0vrl561', + 'http://example.com/cia5rhes301o4hkjxmqogfrad/cia5rhes301o5hkjx9m8hdpcc/cia5rhes301o6hkjxaluh3wcr/cia5rhes301o7hkjx4m8wommz', + 'http://example.com/cia5rhes301o8hkjxml1xa1az/cia5rhes301o9hkjxx678ystu/cia5rhes301oahkjxndq6nh65/cia5rhes301obhkjxadfjm3wa', + 'http://example.com/cia5rhes301ochkjxwz6f2spm/cia5rhes301odhkjxgvmfaaq8/cia5rhes301oehkjxt17j08ud/cia5rhes301ofhkjxneg64ahh', + 'http://example.com/cia5rhes301oghkjx2odplosg/cia5rhes301ohhkjx6lsxvvhc/cia5rhes301oihkjx1zjlr3lf/cia5rhes301ojhkjx4too7ovk', + 'http://example.com/cia5rhes301okhkjx5ie6svqi/cia5rhes301olhkjx1dvra0d8/cia5rhes301omhkjx01eottp8/cia5rhes301onhkjx0k4cthfm', + 'http://example.com/cia5rhes301oohkjx2uggrotk/cia5rhes301ophkjxo0nc672k/cia5rhes301oqhkjxyxv3yip2/cia5rhes301orhkjx1lzdi04w', + 'http://example.com/cia5rhes301oshkjx239gzsvl/cia5rhes301othkjxegmfaqs4/cia5rhes301ouhkjx3k7u7klw/cia5rhes301ovhkjxx0w3i22n', + 'http://example.com/cia5rhes301owhkjx43szuyvt/cia5rhes301oxhkjxwn8rt15b/cia5rhes301oyhkjxn9plrtrh/cia5rhes301ozhkjx939j8ua7', + 'http://example.com/cia5rhes301p0hkjx933v5a7c/cia5rhes301p1hkjxnptb4syc/cia5rhes301p2hkjxlbdlt4c7/cia5rhes301p3hkjxdnx9ndcb', + 'http://example.com/cia5rhes301p4hkjxgwkvdwyk/cia5rhes301p5hkjxu5l7j6a8/cia5rhes301p6hkjx69gflmy6/cia5rhes301p7hkjxl0ebaafj', + 'http://example.com/cia5rhes301p8hkjxf8355jja/cia5rhes301p9hkjxynm9lc74/cia5rhes301pahkjx9gj8htwg/cia5rhes301pbhkjx9dnwyvr7', + 'http://example.com/cia5rhes401pchkjxvc8103ko/cia5rhes401pdhkjxdvj8k8ys/cia5rhes401pehkjx1yvwz1t3/cia5rhes401pfhkjx9tdmf2pk', + 'http://example.com/cia5rhes401pghkjxm2moyuwg/cia5rhes401phhkjx6sd8pxql/cia5rhes401pihkjx0f56qfr0/cia5rhes401pjhkjxe09zm4ee', + 'http://example.com/cia5rhes401pkhkjxmw6xikqs/cia5rhes401plhkjxtrw32c5n/cia5rhes401pmhkjx2gs5r2uw/cia5rhes401pnhkjx3lnqjuvy', + 'http://example.com/cia5rhes401pohkjx18h593mm/cia5rhes401pphkjx74f4atkm/cia5rhes401pqhkjxl804wbka/cia5rhes401prhkjxvjq32png', + 'http://example.com/cia5rhes401pshkjxq7ig2fmw/cia5rhes401pthkjx94834nui/cia5rhes401puhkjxg5h7u1tk/cia5rhes401pvhkjx83fsa82j', + 'http://example.com/cia5rhes401pwhkjxslfaan9d/cia5rhes401pxhkjx5qqbf367/cia5rhes401pyhkjx9uafkt0z/cia5rhes401pzhkjxyk4qxvdq', + 'http://example.com/cia5rhes401q0hkjxxovjahis/cia5rhes401q1hkjx7811zjvy/cia5rhes401q2hkjx87k6qna2/cia5rhes401q3hkjxoj0w4dpu', + 'http://example.com/cia5rhes401q4hkjxln1jw5x1/cia5rhes401q5hkjxrh7gm7b7/cia5rhes401q6hkjx7r2y10bk/cia5rhes401q7hkjxhqkthpq6', + 'http://example.com/cia5rhes401q8hkjx6u394gyd/cia5rhes401q9hkjxrtrhrat9/cia5rhes401qahkjxdt7xqdcp/cia5rhes401qbhkjxm5ymdwfi', + 'http://example.com/cia5rhes401qchkjx025wiukn/cia5rhes401qdhkjxpovs1w4l/cia5rhes401qehkjxjdc5rv3v/cia5rhes401qfhkjxe3c0v82a', + 'http://example.com/cia5rhes401qghkjxzhl0kyt9/cia5rhes401qhhkjxx91x3w69/cia5rhes401qihkjxsldvc9au/cia5rhes401qjhkjxnag09g7f', + 'http://example.com/cia5rhes401qkhkjxp81l6si9/cia5rhes401qlhkjxdzg4648q/cia5rhes401qmhkjx5rysqo3m/cia5rhes401qnhkjxquhuyu1t', + 'http://example.com/cia5rhes401qohkjxsko3ojrg/cia5rhes401qphkjxvda749pk/cia5rhes401qqhkjxudg42xak/cia5rhes401qrhkjx7edixyt2', + 'http://example.com/cia5rhes401qshkjx9jc7o9ik/cia5rhes401qthkjxvhwfd027/cia5rhes401quhkjx22ja7ygg/cia5rhes401qvhkjxx2yc25pu', + 'http://example.com/cia5rhes401qwhkjxs1yqlawy/cia5rhes401qxhkjxap6eqaza/cia5rhes401qyhkjxhbq9zkww/cia5rhes401qzhkjx5j6rm35k', + 'http://example.com/cia5rhes401r0hkjxk8f23od8/cia5rhes401r1hkjxf2hw9jtn/cia5rhes401r2hkjx0dcvkzlo/cia5rhes401r3hkjxqgiol3kt', + 'http://example.com/cia5rhes401r4hkjxk8rzt66b/cia5rhes401r5hkjx4zc092sq/cia5rhes401r6hkjxdkgh2lu3/cia5rhes401r7hkjxrxdp47yk', + 'http://example.com/cia5rhes401r8hkjxl08yep62/cia5rhes401r9hkjx1xzzdt21/cia5rhes401rahkjxl1d6b0c6/cia5rhes401rbhkjx6zyydco5', + 'http://example.com/cia5rhes401rchkjx76v07kx8/cia5rhes401rdhkjxr6p5yan5/cia5rhes401rehkjxx6g8s1x3/cia5rhes401rfhkjxjwzjn0xv', + 'http://example.com/cia5rhes401rghkjxhiae442a/cia5rhes401rhhkjx28xtp7j6/cia5rhes401rihkjxqsail6xh/cia5rhes401rjhkjxr7kiu6ki', + 'http://example.com/cia5rhes401rkhkjxqne03tot/cia5rhes401rlhkjxjhxcypey/cia5rhes401rmhkjxsma2ekxx/cia5rhes401rnhkjx02z0sp28', + 'http://example.com/cia5rhes401rohkjxnrlforbh/cia5rhes401rphkjxuuk7smlp/cia5rhes401rqhkjxp387ih60/cia5rhes401rrhkjxxn9o7q8q', + 'http://example.com/cia5rhes401rshkjxh4lkmqca/cia5rhes401rthkjx0jm2hnqp/cia5rhes401ruhkjxeo73uqck/cia5rhes401rvhkjxjbn6t4yy', + 'http://example.com/cia5rhes401rwhkjxzepyozzy/cia5rhes401rxhkjx1pykuc1m/cia5rhes401ryhkjxpgqmomw4/cia5rhes401rzhkjx3zbmunev', + 'http://example.com/cia5rhes401s0hkjxjbbqwl70/cia5rhes401s1hkjx9xuj3zqs/cia5rhes401s2hkjxafsii503/cia5rhes401s3hkjxuu216w98', + 'http://example.com/cia5rhes401s4hkjxbbt02xp1/cia5rhes401s5hkjx1wbhsus2/cia5rhes401s6hkjx1ml5tjx2/cia5rhes401s7hkjxmxzwknq3', + 'http://example.com/cia5rhes401s8hkjxjna2smh1/cia5rhes401s9hkjxxqoxe1xs/cia5rhes401sahkjxta8cres1/cia5rhes401sbhkjxlobgkg5k', + 'http://example.com/cia5rhes401schkjx4a93mw54/cia5rhes401sdhkjx11zbb5rf/cia5rhes401sehkjxztk9dbrf/cia5rhes401sfhkjxgh3yzmo1', + 'http://example.com/cia5rhes401sghkjxc28fo8pm/cia5rhes401shhkjx94mcy08n/cia5rhes401sihkjxk9c7sc1a/cia5rhes401sjhkjx2kpauvo7', + 'http://example.com/cia5rhes401skhkjxil6rkwln/cia5rhes401slhkjx7rw9fbmh/cia5rhes401smhkjxp1azo0ra/cia5rhes401snhkjxjn6ske3g', + 'http://example.com/cia5rhes401sohkjxld97hjxq/cia5rhes401sphkjxw88rub86/cia5rhes401sqhkjxdepedlux/cia5rhes401srhkjxtz9wqykr', + 'http://example.com/cia5rhes401sshkjx23nj0gw3/cia5rhes401sthkjxamuty3aa/cia5rhes401suhkjxkzkkksxw/cia5rhes401svhkjxfy7t55xc', + 'http://example.com/cia5rhes401swhkjx2fv1t47w/cia5rhes401sxhkjx493ijzth/cia5rhes401syhkjxlt98ctc5/cia5rhes401szhkjxcgaqy6ks', + 'http://example.com/cia5rhes401t0hkjxibyaz6lz/cia5rhes401t1hkjxajqz3b7v/cia5rhes401t2hkjxutdwnzqk/cia5rhes401t3hkjxqr2zpknp', + 'http://example.com/cia5rhes401t4hkjx2me8lthv/cia5rhes401t5hkjxej9j1ggl/cia5rhes401t6hkjxoxxbptsl/cia5rhes401t7hkjx31lkyc9v', + 'http://example.com/cia5rhes401t8hkjxljaq5d24/cia5rhes401t9hkjxcj9ozsjc/cia5rhes401tahkjx45acwqjh/cia5rhes401tbhkjxsfepsuqn', + 'http://example.com/cia5rhes401tchkjx2d54v2mc/cia5rhes401tdhkjxg995kn83/cia5rhes401tehkjx3sa4rnpk/cia5rhes401tfhkjx9p5zj2fw', + 'http://example.com/cia5rhes401tghkjxetiwdot9/cia5rhes401thhkjxdzft6ee5/cia5rhes401tihkjxc44k574p/cia5rhes401tjhkjxhlaamwjt', + 'http://example.com/cia5rhes401tkhkjxfhcebmkr/cia5rhes401tlhkjx6d2ahwy8/cia5rhes401tmhkjxnvqrt43n/cia5rhes401tnhkjx6y3x0tl6', + 'http://example.com/cia5rhes401tohkjxm76vc3bd/cia5rhes401tphkjxe4toa8ix/cia5rhes401tqhkjx44k31o69/cia5rhes401trhkjx06h29ag1', + 'http://example.com/cia5rhes401tshkjx0en3ww5b/cia5rhes401tthkjxj0sbg5rs/cia5rhes401tuhkjx4mbcemrx/cia5rhes401tvhkjxzah5kckz', + 'http://example.com/cia5rhes401twhkjxbc8vo2b5/cia5rhes401txhkjx1quodrlw/cia5rhes401tyhkjx5q10omzn/cia5rhes401tzhkjxhoknv1pd', + 'http://example.com/cia5rhes401u0hkjxyboul8es/cia5rhes401u1hkjxb2vn5wu5/cia5rhes401u2hkjxat1dog9k/cia5rhes401u3hkjxg9cpxurx', + 'http://example.com/cia5rhes401u4hkjxcy69r1cg/cia5rhes401u5hkjxakh0jykj/cia5rhes401u6hkjxpsaz87je/cia5rhes401u7hkjx3ujs32jl', + 'http://example.com/cia5rhes401u8hkjxogdqi93b/cia5rhes401u9hkjxh0e8e5it/cia5rhes401uahkjxcypc72ho/cia5rhes401ubhkjx07fholph', + 'http://example.com/cia5rhes401uchkjx96q3s4y9/cia5rhes401udhkjxbw6z849k/cia5rhes401uehkjxhbtqh3g4/cia5rhes401ufhkjxbp1hjydk', + 'http://example.com/cia5rhes401ughkjxq6z30rsc/cia5rhes401uhhkjxgnc7011n/cia5rhes401uihkjx00l0t29g/cia5rhes401ujhkjxpdkefo86', + 'http://example.com/cia5rhes401ukhkjx5w5u4uez/cia5rhes401ulhkjxkp60rcm2/cia5rhes401umhkjx2o152chr/cia5rhes401unhkjxj1c837fv', + 'http://example.com/cia5rhes401uohkjxkm3hwgxw/cia5rhes401uphkjxr9fpwgxo/cia5rhes401uqhkjxbju1cc6a/cia5rhes401urhkjxnyjsugye', + 'http://example.com/cia5rhes401ushkjxnl9fzmwd/cia5rhes401uthkjx829ud4hl/cia5rhes401uuhkjxgzo6bd97/cia5rhes401uvhkjxninvqfmi', + 'http://example.com/cia5rhes401uwhkjx23xkeeyb/cia5rhes401uxhkjxr7f81k32/cia5rhes401uyhkjxu8gwxp2s/cia5rhes401uzhkjx0zbojk5h', + 'http://example.com/cia5rhes401v0hkjxnp3m2er4/cia5rhes401v1hkjxh6zxquzd/cia5rhes401v2hkjxcp9r8512/cia5rhes401v3hkjxfj2ziffr', + 'http://example.com/cia5rhes401v4hkjx450sdsy6/cia5rhes401v5hkjxid7nsxhs/cia5rhes401v6hkjx5umhcl29/cia5rhes401v7hkjx8c4ntx9f', + 'http://example.com/cia5rhes401v8hkjxm7493idl/cia5rhes401v9hkjxvp3boxa7/cia5rhes401vahkjxpdhxc0bd/cia5rhes401vbhkjxjq8g7bbv', + 'http://example.com/cia5rhes401vchkjxutbrig4f/cia5rhes401vdhkjxs6v3l5bs/cia5rhes401vehkjxz0s7ot2j/cia5rhes401vfhkjx6e86cpuy', + 'http://example.com/cia5rhes401vghkjxn3ce11di/cia5rhes401vhhkjx0lgmp1co/cia5rhes401vihkjxgjby3l0n/cia5rhes401vjhkjxh7bj2rti', + 'http://example.com/cia5rhes401vkhkjxq9xd82bm/cia5rhes401vlhkjxs2x6daye/cia5rhes401vmhkjxnv72qdm3/cia5rhes401vnhkjxjuu4sj2i', + 'http://example.com/cia5rhes401vohkjxf8wvg4tv/cia5rhes401vphkjxus1ibfvl/cia5rhes401vqhkjxaapjjznh/cia5rhes401vrhkjxhpfk9ana', + 'http://example.com/cia5rhes401vshkjxb314pxv2/cia5rhes401vthkjxcfenzpqi/cia5rhes401vuhkjxvbef4uzt/cia5rhes401vvhkjxcg2mtju1', + 'http://example.com/cia5rhes401vwhkjxobjolxrt/cia5rhes401vxhkjx8n6q1mbj/cia5rhes401vyhkjx1ffiobsm/cia5rhes401vzhkjx845f4yrb', + 'http://example.com/cia5rhes501w0hkjxnkd5nvx4/cia5rhes501w1hkjxd10zyp5f/cia5rhes501w2hkjxed1isr4c/cia5rhes501w3hkjx52w25p6h', + 'http://example.com/cia5rhes501w4hkjxz590qwcl/cia5rhes501w5hkjxirypp5am/cia5rhes501w6hkjx0la9fxfb/cia5rhes501w7hkjxqn8mmj3v', + 'http://example.com/cia5rhes501w8hkjxtvynj745/cia5rhes501w9hkjxicsfn3ft/cia5rhes501wahkjxawuf4y2u/cia5rhes501wbhkjxuioyhj09', + 'http://example.com/cia5rhes501wchkjxpzp5z6gl/cia5rhes501wdhkjxqyu6yfrv/cia5rhes501wehkjxksohpokd/cia5rhes501wfhkjxocde2wt3', + 'http://example.com/cia5rhes501wghkjxocbfchks/cia5rhes501whhkjx0tpiw1nt/cia5rhes501wihkjxslhtkvr0/cia5rhes501wjhkjx3f2wxmki', + 'http://example.com/cia5rhes501wkhkjxpmltynvl/cia5rhes501wlhkjxig3aj85x/cia5rhes501wmhkjxplefjg23/cia5rhes501wnhkjxanfao1fs', + 'http://example.com/cia5rhes501wohkjx9gnuza2e/cia5rhes501wphkjxi89ym1sn/cia5rhes501wqhkjxmpb91ix0/cia5rhes501wrhkjx6vdiefye', + 'http://example.com/cia5rhes501wshkjxxy1dl1w5/cia5rhes501wthkjxs40731ag/cia5rhes501wuhkjx5tu8ptk3/cia5rhes501wvhkjxb83m364e', + 'http://example.com/cia5rhes501wwhkjxyxi7zia4/cia5rhes501wxhkjxfttjkfl1/cia5rhes501wyhkjx73a609nu/cia5rhes501wzhkjxhrqkcsc9', + 'http://example.com/cia5rhes501x0hkjxpzoc18gx/cia5rhes501x1hkjx2evbj8dh/cia5rhes501x2hkjxcmt0dte5/cia5rhes501x3hkjxs2o08cdn', + 'http://example.com/cia5rhes501x4hkjxn8ppwkl6/cia5rhes501x5hkjxve994e14/cia5rhes501x6hkjxp3nroxzg/cia5rhes501x7hkjxdzh6iphg', + 'http://example.com/cia5rhes501x8hkjx3dxj6rdf/cia5rhes501x9hkjx0uek477t/cia5rhes501xahkjxyomgqdjw/cia5rhes501xbhkjx0adcgz3e', + 'http://example.com/cia5rhes501xchkjxjr38fuho/cia5rhes501xdhkjxi9h8gxgv/cia5rhes501xehkjx9lnq5x48/cia5rhes501xfhkjx6x7q34qn', + 'http://example.com/cia5rhes501xghkjx7kdv4j16/cia5rhes501xhhkjxzh3h1621/cia5rhes501xihkjx2tll48zr/cia5rhes501xjhkjx2mgqrjx7', + 'http://example.com/cia5rhes501xkhkjxb38ktam2/cia5rhes501xlhkjxqe4kly98/cia5rhes501xmhkjxs8kc7y4g/cia5rhes501xnhkjxon8hd6t9', + 'http://example.com/cia5rhes501xohkjxvh07sqak/cia5rhes501xphkjxa8cjku7k/cia5rhes501xqhkjx7czbmtzz/cia5rhes501xrhkjx2v5gm68q', + 'http://example.com/cia5rhes501xshkjxhafeuujz/cia5rhes501xthkjx83z1ik2e/cia5rhes501xuhkjxfwfbdp20/cia5rhes501xvhkjxat92izys', + 'http://example.com/cia5rhes501xwhkjxmxkavbl1/cia5rhes501xxhkjxjmwfaudp/cia5rhes501xyhkjxjb6y5ckv/cia5rhes501xzhkjxx1qzw43n', + 'http://example.com/cia5rhes501y0hkjxjqq91tnx/cia5rhes501y1hkjx5fqvt95y/cia5rhes501y2hkjx213i79od/cia5rhes501y3hkjx2bhrwh3c', + 'http://example.com/cia5rhes501y4hkjxpg0w8tm1/cia5rhes501y5hkjx4rfqmukn/cia5rhes501y6hkjxdmm6zlwo/cia5rhes501y7hkjxuuszpo6e', + 'http://example.com/cia5rhes501y8hkjx1dijvw0o/cia5rhes501y9hkjxh1co3ai2/cia5rhes501yahkjxfcerdd6h/cia5rhes501ybhkjx52yrnztr', + 'http://example.com/cia5rhes501ychkjxmuxdm6gn/cia5rhes501ydhkjxk9cu7gzp/cia5rhes501yehkjxt9czxhe8/cia5rhes501yfhkjxf1hpxe7k', + 'http://example.com/cia5rhes501yghkjxzyfsx9ee/cia5rhes501yhhkjxoobntt4j/cia5rhes501yihkjxbv4l41i4/cia5rhes501yjhkjx9cg8i6yq', + 'http://example.com/cia5rhes501ykhkjxb8micj52/cia5rhes501ylhkjxi810y8kg/cia5rhes501ymhkjx35pqd2dp/cia5rhes501ynhkjx411ay6w2', + 'http://example.com/cia5rhes501yohkjxp230m8o4/cia5rhes501yphkjx85aei3f0/cia5rhes501yqhkjx39awmvdg/cia5rhes501yrhkjxabhea8z7', + 'http://example.com/cia5rhes501yshkjxmy4w0zr0/cia5rhes501ythkjxxxtlmezs/cia5rhes501yuhkjx8mwm07hi/cia5rhes501yvhkjx1l5p3sr0', + 'http://example.com/cia5rhes501ywhkjxdrcc28nn/cia5rhes501yxhkjxqcqd6ogs/cia5rhes501yyhkjxim858nwj/cia5rhes501yzhkjxpi5u68xr', + 'http://example.com/cia5rhes501z0hkjxgxk8ryu8/cia5rhes501z1hkjx7jqdu67h/cia5rhes501z2hkjx21zj3rmt/cia5rhes501z3hkjxcq1lwavz', + 'http://example.com/cia5rhes501z4hkjxgx1266ef/cia5rhes501z5hkjxi6uyr5et/cia5rhes501z6hkjxhr0eot9n/cia5rhes501z7hkjxyz2oyzjs', + 'http://example.com/cia5rhes501z8hkjx5s5s200w/cia5rhes501z9hkjx67v1yb2z/cia5rhes501zahkjxw5u36sb5/cia5rhes501zbhkjxl17xibdr', + 'http://example.com/cia5rhes501zchkjxpx05d6o1/cia5rhes501zdhkjxiiadtum2/cia5rhes501zehkjxoj9i56gl/cia5rhes501zfhkjxqcxmjy73', + 'http://example.com/cia5rhes501zghkjxegc7tvdy/cia5rhes501zhhkjxqeeoq63e/cia5rhes501zihkjxysrggeqs/cia5rhes501zjhkjxf24x4w8j', + 'http://example.com/cia5rhes501zkhkjx36w5g359/cia5rhes501zlhkjxuornb7pf/cia5rhes501zmhkjx4pvpci2q/cia5rhes501znhkjxbv1oa4fp', + 'http://example.com/cia5rhes501zohkjxb6t1a9pz/cia5rhes501zphkjxg5ezhfdv/cia5rhes501zqhkjxl3efud9l/cia5rhes501zrhkjxcqb7r2sc', + 'http://example.com/cia5rhes501zshkjxd7wcvoav/cia5rhes501zthkjxelhdxd7w/cia5rhes501zuhkjxh07pf32p/cia5rhes501zvhkjxgcxn3nvl', + 'http://example.com/cia5rhes501zwhkjx95ri5zb5/cia5rhes501zxhkjxci9sujxb/cia5rhes501zyhkjx1hzc65ou/cia5rhes501zzhkjxf1kbgic9', + 'http://example.com/cia5rhes50200hkjxphxlxmld/cia5rhes50201hkjx0sveusk8/cia5rhes50202hkjxg5822asq/cia5rhes50203hkjxxle2qnr4', + 'http://example.com/cia5rhes50204hkjxswna3iww/cia5rhes50205hkjxo41y7z2t/cia5rhes50206hkjx1auwgf30/cia5rhes50207hkjx3vyiy15y', + 'http://example.com/cia5rhes50208hkjx6n640dxz/cia5rhes50209hkjxxb3tliuh/cia5rhes5020ahkjxht8vaioj/cia5rhes5020bhkjxqjo5gr27', + 'http://example.com/cia5rhes5020chkjxh9wu9gbv/cia5rhes5020dhkjxbrv63660/cia5rhes5020ehkjxbmozonad/cia5rhes5020fhkjxsek9b1wa', + 'http://example.com/cia5rhes5020ghkjxrlfea9iv/cia5rhes5020hhkjxt7qh369y/cia5rhes5020ihkjxkn7yslxt/cia5rhes5020jhkjx2ge4xq51', + 'http://example.com/cia5rhes5020khkjx2sp9c2gt/cia5rhes5020lhkjx1ks9juca/cia5rhes5020mhkjxrova7tax/cia5rhes5020nhkjxnaxah6tg', + 'http://example.com/cia5rhes5020ohkjx9btins8g/cia5rhes5020phkjxy4or4s6u/cia5rhes5020qhkjxxrqcpd3n/cia5rhes5020rhkjxm6xw3z2x', + 'http://example.com/cia5rhes5020shkjxz31fkpjb/cia5rhes5020thkjxsxivj1tx/cia5rhes5020uhkjx218dg3oe/cia5rhes5020vhkjxpxflwg9k', + 'http://example.com/cia5rhes5020whkjx3xpogsrh/cia5rhes5020xhkjxv5k6yvhb/cia5rhes5020yhkjxmg5wu4xg/cia5rhes5020zhkjx49u1376r', + 'http://example.com/cia5rhes50210hkjxu07iog9j/cia5rhes50211hkjxe2zq097b/cia5rhes50212hkjx7d2n5bis/cia5rhes50213hkjx98z0f1wd', + 'http://example.com/cia5rhes50214hkjxxz2fxal3/cia5rhes50215hkjx4cdss157/cia5rhes50216hkjxgemb403b/cia5rhes50217hkjxcx1to7hv', + 'http://example.com/cia5rhes50218hkjxlm8ctocp/cia5rhes50219hkjx1fcacxy3/cia5rhes5021ahkjxx59gdemf/cia5rhes5021bhkjxa8w89mbs', + 'http://example.com/cia5rhes5021chkjxgbgtxsby/cia5rhes5021dhkjxpsb7jlci/cia5rhes5021ehkjxo8ytwukr/cia5rhes5021fhkjxtpoy84xh', + 'http://example.com/cia5rhes5021ghkjxyk2hucae/cia5rhes5021hhkjxyiywhstb/cia5rhes5021ihkjx1sdmxxsc/cia5rhes5021jhkjxp5btccgt', + 'http://example.com/cia5rhes5021khkjxav298li6/cia5rhes5021lhkjx4ba0mhnf/cia5rhes5021mhkjxngkomyhl/cia5rhes5021nhkjxxtqqmtir', + 'http://example.com/cia5rhes5021ohkjxbavqb4tz/cia5rhes5021phkjx1f18irux/cia5rhes5021qhkjxgef61ilr/cia5rhes5021rhkjxeh1y04kj', + 'http://example.com/cia5rhes5021shkjxr4s9i0ob/cia5rhes5021thkjxfocdh5vi/cia5rhes5021uhkjxjcwajris/cia5rhes5021vhkjxitwdjshb', + 'http://example.com/cia5rhes5021whkjxwhlm3an5/cia5rhes5021xhkjx5dcoj15s/cia5rhes5021yhkjxy9biyupr/cia5rhes5021zhkjx6wit7c1p', + 'http://example.com/cia5rhes50220hkjxco3srhrz/cia5rhes50221hkjxn8kb150i/cia5rhes50222hkjxcfl48mla/cia5rhes50223hkjx5wzddel7', + 'http://example.com/cia5rhes50224hkjxv4kbq0bu/cia5rhes50225hkjxdlcujhtv/cia5rhes50226hkjx0nm0ncdj/cia5rhes50227hkjx4hnvg7w9', + 'http://example.com/cia5rhes50228hkjxn2hoexz0/cia5rhes50229hkjx5a0zae0n/cia5rhes5022ahkjx7kw3lf0v/cia5rhes5022bhkjx9uaqp2w5', + 'http://example.com/cia5rhes5022chkjxmllq37r4/cia5rhes5022dhkjxuogvq5kp/cia5rhes5022ehkjxegsxagw5/cia5rhes5022fhkjx25d5a5z8', + 'http://example.com/cia5rhes5022ghkjxwwoecae0/cia5rhes5022hhkjxli8zm9vs/cia5rhes5022ihkjxzxcky0jv/cia5rhes5022jhkjxvsb9g2qa', + 'http://example.com/cia5rhes5022khkjxhwpswkll/cia5rhes5022lhkjxow1y1vc4/cia5rhes5022mhkjxh0o8b4r5/cia5rhes5022nhkjxjsyoo9le', + 'http://example.com/cia5rhes5022ohkjx50pmnu22/cia5rhes5022phkjxfdh1jhl2/cia5rhes5022qhkjxh67gv4up/cia5rhes5022rhkjxmpux301t', + 'http://example.com/cia5rhes5022shkjxmgm2q2tv/cia5rhes5022thkjx7ivn1k01/cia5rhes5022uhkjxs4j1z1st/cia5rhes5022vhkjxh3y1ak61', + 'http://example.com/cia5rhes5022whkjxy2vkf9qu/cia5rhes5022xhkjxotujbeup/cia5rhes5022yhkjx5qiu2ujp/cia5rhes5022zhkjxluajf32y', + 'http://example.com/cia5rhes50230hkjxk7stw4db/cia5rhes60231hkjxf7aj9i0m/cia5rhes60232hkjxziydwog0/cia5rhes60233hkjxx3x1fbuc', + 'http://example.com/cia5rhes60234hkjxg2uqu0ml/cia5rhes60235hkjxq7n4gpgv/cia5rhes60236hkjxolpslbdw/cia5rhes60237hkjxyn1lp5ir', + 'http://example.com/cia5rhes60238hkjxwis4nirx/cia5rhes60239hkjxaiqtx5n6/cia5rhes6023ahkjxsgrablt0/cia5rhes6023bhkjxc06147lu', + 'http://example.com/cia5rhes6023chkjxxge8xmjn/cia5rhes6023dhkjx5j31jwgd/cia5rhes6023ehkjxwuz388j6/cia5rhes6023fhkjx3pdltokg', + 'http://example.com/cia5rhes6023ghkjx6dffsn9x/cia5rhes6023hhkjxzjoqqtor/cia5rhes6023ihkjx3bz79voa/cia5rhes6023jhkjxa7bb04th', + 'http://example.com/cia5rhes6023khkjxhg5ub876/cia5rhes6023lhkjxrklzuro9/cia5rhes6023mhkjx8xmhpdqm/cia5rhes6023nhkjxch1jn490', + 'http://example.com/cia5rhes6023ohkjxhad7g229/cia5rhes6023phkjx4zaksvdn/cia5rhes6023qhkjxx6ko1cpf/cia5rhes6023rhkjx0vireriy', + 'http://example.com/cia5rhes6023shkjxhvae8jtn/cia5rhes6023thkjxw4de6xi4/cia5rhes6023uhkjxzfqht8ml/cia5rhes6023vhkjxs8ul3zvc', + 'http://example.com/cia5rhes6023whkjxdsyyu08r/cia5rhes6023xhkjxhddko66j/cia5rhes6023yhkjxnfhgsx6b/cia5rhes6023zhkjxt63bqpbs', + 'http://example.com/cia5rhes60240hkjxa7oafjex/cia5rhes60241hkjx74x1e2f3/cia5rhes60242hkjxiaptta0r/cia5rhes60243hkjxingpv6qf', + 'http://example.com/cia5rhes60244hkjx832w9v0m/cia5rhes60245hkjxbtb4g19e/cia5rhes60246hkjxahthge6j/cia5rhes60247hkjxhqj3m07o', + 'http://example.com/cia5rhes60248hkjxcf7nc4li/cia5rhes60249hkjxyaeee0po/cia5rhes6024ahkjxz0zbl31v/cia5rhes6024bhkjxyli25oi7', + 'http://example.com/cia5rhes6024chkjxqymyzh67/cia5rhes6024dhkjx41mtrlwg/cia5rhes6024ehkjxupbohin3/cia5rhes6024fhkjx1wtwax3q', + 'http://example.com/cia5rhes6024ghkjxbhnnx8qm/cia5rhes6024hhkjx330f907k/cia5rhes6024ihkjxt8kevs6h/cia5rhes6024jhkjx6fz60hhj', + 'http://example.com/cia5rhes6024khkjx6jh6byd0/cia5rhes6024lhkjxnqak5lqd/cia5rhes6024mhkjx6qi3ka0d/cia5rhes6024nhkjxmydiqa1w', + 'http://example.com/cia5rhes6024ohkjx1wzyvp8g/cia5rhes6024phkjxcpe4crtr/cia5rhes6024qhkjx5k672peu/cia5rhes6024rhkjxrgc14c0o', + 'http://example.com/cia5rhes6024shkjxt3phdd6y/cia5rhes6024thkjxrcolx8rw/cia5rhes6024uhkjx1m8lrl96/cia5rhes6024vhkjx1ub0usjq', + 'http://example.com/cia5rhes6024whkjx30q3vye6/cia5rhes6024xhkjxqhicyl5l/cia5rhes6024yhkjxewkiuvcd/cia5rhes6024zhkjxpi0s95q6', + 'http://example.com/cia5rhes60250hkjx7x45wchz/cia5rhes60251hkjx29nj5yrn/cia5rhes60252hkjxmjtv4j8t/cia5rhes60253hkjx62flt3ct', + 'http://example.com/cia5rhes60254hkjxj24tyltz/cia5rhes60255hkjxu43vfkjt/cia5rhes60256hkjxorb3l17v/cia5rhes60257hkjxuusa9260', + 'http://example.com/cia5rhes60258hkjx2mtr4h7o/cia5rhes60259hkjxfni1laoe/cia5rhes6025ahkjxi8p6cxws/cia5rhes6025bhkjxms0v3mvk', + 'http://example.com/cia5rhes6025chkjxak2ehrye/cia5rhes6025dhkjxkkwv08j7/cia5rhes6025ehkjxmviua90r/cia5rhes6025fhkjxxz5403tq', + 'http://example.com/cia5rhes6025ghkjxw2zi9e42/cia5rhes6025hhkjxcpaquver/cia5rhes6025ihkjxdza15efa/cia5rhes6025jhkjxj10ftcde', + 'http://example.com/cia5rhes6025khkjxzdgyklzu/cia5rhes6025lhkjxepec48wo/cia5rhes6025mhkjxrr0rxhsw/cia5rhes6025nhkjxbx5apxib', + 'http://example.com/cia5rhes6025ohkjxmw1aiv3f/cia5rhes6025phkjxf2m420e9/cia5rhes6025qhkjxjiwth0yz/cia5rhes6025rhkjxrmxufevy', + 'http://example.com/cia5rhes6025shkjxusdiwv01/cia5rhes6025thkjxds425t8m/cia5rhes6025uhkjxuqrtt7if/cia5rhes6025vhkjxowk5zvf3', + 'http://example.com/cia5rhes6025whkjxh652j091/cia5rhes6025xhkjxg7n9opan/cia5rhes6025yhkjxhx4aysaj/cia5rhes6025zhkjxu82h4n54', + 'http://example.com/cia5rhes60260hkjxi674w0z0/cia5rhes60261hkjxojs9dwc5/cia5rhes60262hkjx9zme8232/cia5rhes60263hkjxg3tduw2q', + 'http://example.com/cia5rhes60264hkjxen5f1emm/cia5rhes60265hkjx9wlrydmg/cia5rhes60266hkjxyk0z00l1/cia5rhes60267hkjxim57nlkk', + 'http://example.com/cia5rhes60268hkjx0dxjfg9r/cia5rhes60269hkjxvsd7fx55/cia5rhes6026ahkjxr4wv79py/cia5rhes6026bhkjxbtuynf74', + 'http://example.com/cia5rhes6026chkjx0hbrlens/cia5rhes6026dhkjx4oarjdzi/cia5rhes6026ehkjxcfh9kh1i/cia5rhes6026fhkjxdvhhj9ps', + 'http://example.com/cia5rhes6026ghkjxzbxwxiwi/cia5rhes6026hhkjx10dmy3ck/cia5rhes6026ihkjxrh57qzib/cia5rhes6026jhkjxa6wqf4ro', + 'http://example.com/cia5rhes6026khkjxw4rqjhaq/cia5rhes6026lhkjxuc55dmgp/cia5rhes6026mhkjxlv6a6sz0/cia5rhes6026nhkjxwxm1u6cu', + 'http://example.com/cia5rhes6026ohkjxcezmtk1t/cia5rhes6026phkjxt8hncf2i/cia5rhes6026qhkjxuxprl91o/cia5rhes6026rhkjx9ujzo2je', + 'http://example.com/cia5rhes6026shkjxxutau6ka/cia5rhes6026thkjxa2hy9mje/cia5rhes6026uhkjxr2vho147/cia5rhes6026vhkjx7h70z8i9', + 'http://example.com/cia5rhes6026whkjx1nagxk22/cia5rhes6026xhkjxke02jgeq/cia5rhes6026yhkjxhemx0l0x/cia5rhes6026zhkjx8uhw94o4', + 'http://example.com/cia5rhes60270hkjxtpo8z0gx/cia5rhes60271hkjxaldlng02/cia5rhes60272hkjxi6u6vyos/cia5rhes60273hkjx8t4gz8q3', + 'http://example.com/cia5rhes60274hkjxzetzmgfp/cia5rhes60275hkjxqtd9rh66/cia5rhes60276hkjxo38ak1v6/cia5rhes60277hkjx3t2grzdi', + 'http://example.com/cia5rhes60278hkjxssjf92tp/cia5rhes60279hkjxtdiimuwo/cia5rhes6027ahkjxv7i327um/cia5rhes6027bhkjx34iyiwau', + 'http://example.com/cia5rhes6027chkjxsalv7vq1/cia5rhes6027dhkjxj1qa0eqe/cia5rhes6027ehkjxdstykpct/cia5rhes6027fhkjxep1lg57f', + 'http://example.com/cia5rhes6027ghkjxir6tvp5r/cia5rhes6027hhkjx37mwtxmp/cia5rhes6027ihkjxajh8kdk0/cia5rhes6027jhkjxprxxf6bf', + 'http://example.com/cia5rhes6027khkjxtx8rt4eg/cia5rhes6027lhkjx6stckrq2/cia5rhes6027mhkjxbp2scl06/cia5rhes6027nhkjx5tcodm70', + 'http://example.com/cia5rhes6027ohkjx02hq4e4i/cia5rhes6027phkjxpj98682x/cia5rhes6027qhkjxi6t9w6j8/cia5rhes6027rhkjxdoo5aitq', + 'http://example.com/cia5rhes6027shkjxq61ipcpf/cia5rhes6027thkjx4c95chxk/cia5rhes6027uhkjx5yp65br8/cia5rhes6027vhkjxgaj3cw9t', + 'http://example.com/cia5rhes6027whkjxx18if78t/cia5rhes6027xhkjxeruuk14w/cia5rhes6027yhkjxzur0jh40/cia5rhes6027zhkjx2zxmcdyy', + 'http://example.com/cia5rhes60280hkjxrh298dzu/cia5rhes60281hkjx5m40ppz3/cia5rhes60282hkjxfak6x0vp/cia5rhes60283hkjxcokmxlit', + 'http://example.com/cia5rhes60284hkjx58dts12q/cia5rhes60285hkjx7hgaud95/cia5rhes60286hkjxdycu90lv/cia5rhes60287hkjxjj4cgdk8', + 'http://example.com/cia5rhes60288hkjxai7gc5c8/cia5rhes60289hkjxbnomezv6/cia5rhes6028ahkjxw7wxahj2/cia5rhes6028bhkjx1smzie0j', + 'http://example.com/cia5rhes6028chkjxa57aiiju/cia5rhes6028dhkjxs1etgvw7/cia5rhes6028ehkjxtsbz6p0z/cia5rhes6028fhkjxmo1vsspv', + 'http://example.com/cia5rhes6028ghkjxieobtxp5/cia5rhes6028hhkjx9ragsscj/cia5rhes6028ihkjx385kpk1h/cia5rhes6028jhkjxotj68l1k', + 'http://example.com/cia5rhes6028khkjxea5reemm/cia5rhes6028lhkjx0kwzwbyo/cia5rhes6028mhkjx4nqjjcde/cia5rhes6028nhkjxzrrex5ue', + 'http://example.com/cia5rhes6028ohkjx7t2lhe7z/cia5rhes6028phkjx46qyubif/cia5rhes6028qhkjxjolbuqus/cia5rhes6028rhkjx8r7ii6z7', + 'http://example.com/cia5rhes6028shkjxilpnvd7j/cia5rhes6028thkjxof8m415p/cia5rhes6028uhkjxjp4mywli/cia5rhes6028vhkjxcw58yxw0', + 'http://example.com/cia5rhes6028whkjxhya97tqs/cia5rhes6028xhkjxpezwz1pe/cia5rhes6028yhkjxx59c4igt/cia5rhes6028zhkjxdjv35rpr', + 'http://example.com/cia5rhes60290hkjxnthanean/cia5rhes60291hkjxni7pjxv4/cia5rhes60292hkjx0flrl74n/cia5rhes60293hkjxm9x63zo7', + 'http://example.com/cia5rhes60294hkjxpnfmclsw/cia5rhes60295hkjx56ccc80r/cia5rhes60296hkjx4s91lrwv/cia5rhes60297hkjxf132ofl7', + 'http://example.com/cia5rhes60298hkjxl3mctpt0/cia5rhes60299hkjxvlg5nt62/cia5rhes6029ahkjx336mdt5q/cia5rhes6029bhkjxx1be21if', + 'http://example.com/cia5rhes6029chkjxo22y49m7/cia5rhes6029dhkjx1llimb0p/cia5rhes6029ehkjxt13ucuxv/cia5rhes6029fhkjxh2xoljln', + 'http://example.com/cia5rhes6029ghkjx68wd962d/cia5rhes6029hhkjx387d5swn/cia5rhes6029ihkjxh34aue0p/cia5rhes6029jhkjxfh61fg9l', + 'http://example.com/cia5rhes6029khkjxuz53ttqc/cia5rhes6029lhkjxvrp7a6bu/cia5rhes6029mhkjx5ug57g8j/cia5rhes6029nhkjxiv7fjxr3', + 'http://example.com/cia5rhes6029ohkjx2im4dkbc/cia5rhes6029phkjxk2vkitw7/cia5rhes6029qhkjx1g18697q/cia5rhes6029rhkjxu7cv0cp5', + 'http://example.com/cia5rhes6029shkjxzfgxcfx5/cia5rhes6029thkjx6bi4op1u/cia5rhes6029uhkjx57v7j2tp/cia5rhes6029vhkjxqsn3ros1', + 'http://example.com/cia5rhes7029whkjx33b3346i/cia5rhes7029xhkjxnhbvzlyl/cia5rhes7029yhkjxhofpksax/cia5rhes7029zhkjxpckp9le4', + 'http://example.com/cia5rhes702a0hkjx6pzs7e5d/cia5rhes702a1hkjxp2x65zqo/cia5rhes702a2hkjxu66pcizj/cia5rhes702a3hkjx7o8r0f06', + 'http://example.com/cia5rhes702a4hkjxs3nk500n/cia5rhes702a5hkjxg0rbzm6k/cia5rhes702a6hkjx234c6g7e/cia5rhes702a7hkjx9ocd54xq', + 'http://example.com/cia5rhes702a8hkjxhv3xsjpp/cia5rhes702a9hkjxofxw9mdy/cia5rhes702aahkjxwtmyec4h/cia5rhes702abhkjxly8sn8hi', + 'http://example.com/cia5rhes702achkjx7zlau40c/cia5rhes702adhkjx6i9t1hdm/cia5rhes702aehkjx3w115jp6/cia5rhes702afhkjx3spdsa1v', + 'http://example.com/cia5rhes702aghkjxd4i1f3k7/cia5rhes702ahhkjx1o7338m9/cia5rhes702aihkjx3issv8lp/cia5rhes702ajhkjxkkpxy74s', + 'http://example.com/cia5rhes702akhkjxdng2ft24/cia5rhes702alhkjxvf0nimyo/cia5rhes702amhkjxubx3l0hc/cia5rhes702anhkjxjdg78083', + 'http://example.com/cia5rhes702aohkjxb6np3w0m/cia5rhes702aphkjxbmp49sgd/cia5rhes702aqhkjx3wm23ff0/cia5rhes702arhkjx9ht9wc86', + 'http://example.com/cia5rhes702ashkjxw56jbjfz/cia5rhes702athkjx6js735z5/cia5rhes702auhkjxucfu5lpt/cia5rhes702avhkjxbyglt9ex', + 'http://example.com/cia5rhes702awhkjx18s0uu13/cia5rhes702axhkjxi3zrv40h/cia5rhes702ayhkjx3a8cp916/cia5rhes702azhkjxczqrzngo', + 'http://example.com/cia5rhes702b0hkjxglj4n5o7/cia5rhes702b1hkjx63bg4kb1/cia5rhes702b2hkjx60relgsi/cia5rhes702b3hkjxiol0e8ym', + 'http://example.com/cia5rhes702b4hkjxjfpk1sg5/cia5rhes702b5hkjxk428e7bk/cia5rhes702b6hkjxr97qxcy0/cia5rhes702b7hkjxdyz4rzzn', + 'http://example.com/cia5rhes702b8hkjx7vylah33/cia5rhes702b9hkjxinhs95fl/cia5rhes702bahkjxpengba9m/cia5rhes702bbhkjxh5smj013', + 'http://example.com/cia5rhes702bchkjxqce1aoab/cia5rhes702bdhkjxaiyf10a3/cia5rhes702behkjx5yqopkqf/cia5rhes702bfhkjx3hiu4jp5', + 'http://example.com/cia5rhes702bghkjx27997nof/cia5rhes702bhhkjxh131a1mu/cia5rhes702bihkjxdv7jmcf7/cia5rhes702bjhkjxu56c6np2', + 'http://example.com/cia5rhes702bkhkjxqpt1iswl/cia5rhes702blhkjxxvuevm79/cia5rhes702bmhkjxlb6egm5v/cia5rhes702bnhkjx0frya4zv', + 'http://example.com/cia5rhes702bohkjx62rqvbxx/cia5rhes702bphkjxn5543qcw/cia5rhes702bqhkjxo6xrcl3m/cia5rhes702brhkjxxiyxytk6', + 'http://example.com/cia5rhes702bshkjxtupz79qv/cia5rhes702bthkjx46tmi8da/cia5rhes702buhkjxa076ev9b/cia5rhes702bvhkjxwzgfevcu', + 'http://example.com/cia5rhes702bwhkjxwmx0x18a/cia5rhes702bxhkjxpq4el7be/cia5rhes702byhkjxwlypdgqk/cia5rhes702bzhkjxf16uiqj9', + 'http://example.com/cia5rhes702c0hkjx0oylz3z7/cia5rhes702c1hkjxnka3undy/cia5rhes702c2hkjx9pvadq7q/cia5rhes702c3hkjxubumi03d', + 'http://example.com/cia5rhes702c4hkjxv1je61d0/cia5rhes702c5hkjx3gud1w7h/cia5rhes702c6hkjxhbputn4m/cia5rhes702c7hkjx2fwamiyv', + 'http://example.com/cia5rhes702c8hkjxbgkmje13/cia5rhes702c9hkjxlumxva5q/cia5rhes702cahkjxmiet3v1x/cia5rhes702cbhkjx8ibo8t0v', + 'http://example.com/cia5rhes702cchkjxyl6aj596/cia5rhes702cdhkjxuk4jdais/cia5rhes702cehkjxznkrhgcf/cia5rhes702cfhkjxedld1xxc', + 'http://example.com/cia5rhes702cghkjxc2ry2vt4/cia5rhes702chhkjxahplgyzs/cia5rhes702cihkjxdfgeirre/cia5rhes702cjhkjx5k6zbwnv', + 'http://example.com/cia5rhes702ckhkjxt8jo94yh/cia5rhes702clhkjxsjs9l544/cia5rhes702cmhkjxob8bd0zc/cia5rhes702cnhkjx6cfcl3n9', + 'http://example.com/cia5rhes702cohkjxb9cd9ogj/cia5rhes702cphkjxpoorw1yg/cia5rhes702cqhkjxykcpxjap/cia5rhes702crhkjx3469lxlp', + 'http://example.com/cia5rhes702cshkjxmwi9wm5t/cia5rhes702cthkjx8tmzifvh/cia5rhes702cuhkjx4l68blak/cia5rhes702cvhkjxdxodcgpw', + 'http://example.com/cia5rhes702cwhkjx0tbp18xa/cia5rhes702cxhkjxa9e95679/cia5rhes702cyhkjxpunm4oge/cia5rhes702czhkjxsxewphj9', + 'http://example.com/cia5rhes702d0hkjx1a2yy8af/cia5rhes702d1hkjx4f2cssht/cia5rhes702d2hkjxa1d631y5/cia5rhes702d3hkjx5isc7bl5', + 'http://example.com/cia5rhes702d4hkjxxf0dzxl4/cia5rhes702d5hkjxxnd097v7/cia5rhes702d6hkjx98mpvdya/cia5rhes702d7hkjx284luop7', + 'http://example.com/cia5rhes702d8hkjxy6hghmfk/cia5rhes702d9hkjxr4ozxswm/cia5rhes702dahkjx4aemrdzl/cia5rhes702dbhkjx3b9om3gn', + 'http://example.com/cia5rhes702dchkjx2q559yuu/cia5rhes702ddhkjxr1frvgb5/cia5rhes702dehkjx59to46ip/cia5rhes702dfhkjxtjmix0kn', + 'http://example.com/cia5rhes702dghkjxk4m6a2s0/cia5rhes702dhhkjxfwaeszqy/cia5rhes702dihkjx4zf8y4ca/cia5rhes702djhkjxvhfrquil', + 'http://example.com/cia5rhes702dkhkjx2orxsnm3/cia5rhes702dlhkjx47rdcwpv/cia5rhes702dmhkjx8j62q07m/cia5rhes702dnhkjxt3qftg4a', + 'http://example.com/cia5rhes702dohkjxer57v1ky/cia5rhes702dphkjxjbishjq1/cia5rhes702dqhkjxt8r2fmuw/cia5rhes702drhkjx8etd1xkq', + 'http://example.com/cia5rhes702dshkjxwbjmsogs/cia5rhes702dthkjxzjt0f26i/cia5rhes702duhkjxrspfet0e/cia5rhes702dvhkjx24ih1puf', + 'http://example.com/cia5rhes702dwhkjx4qx5ofni/cia5rhes702dxhkjxyxhxsw0c/cia5rhes702dyhkjx8mi9wbce/cia5rhes702dzhkjxr9gk1g19', + 'http://example.com/cia5rhes702e0hkjxin8zq13k/cia5rhes702e1hkjxn5bq0ikw/cia5rhes702e2hkjxxb2qoxsk/cia5rhes702e3hkjxbco0q0qj', + 'http://example.com/cia5rhes702e4hkjxhxbl6l43/cia5rhes702e5hkjx0zz697fh/cia5rhes702e6hkjxfdsk112c/cia5rhes702e7hkjxabbxyd7j', + 'http://example.com/cia5rhes702e8hkjx3vnctynz/cia5rhes702e9hkjxg4zopm86/cia5rhes702eahkjxo3bg8ml3/cia5rhes702ebhkjxp3aeugu4', + 'http://example.com/cia5rhes702echkjxal3j832h/cia5rhes702edhkjx1lyibi15/cia5rhes702eehkjxstdtwkp6/cia5rhes702efhkjxdnbnyno0', + 'http://example.com/cia5rhes702eghkjx55wp2mw0/cia5rhes702ehhkjxwmxwjl29/cia5rhes702eihkjxg7t126ld/cia5rhes702ejhkjx15qdziu1', + 'http://example.com/cia5rhes702ekhkjxc0im9wy4/cia5rhes702elhkjxh2jd7hzr/cia5rhes702emhkjxcu8r9pzm/cia5rhes702enhkjx9jbgidf1', + 'http://example.com/cia5rhes702eohkjxhlu6h4ep/cia5rhes702ephkjx4mwoc3ql/cia5rhes702eqhkjxe2bwkjv6/cia5rhes702erhkjxh8shrs32', + 'http://example.com/cia5rhes702eshkjxs8w53l9b/cia5rhes702ethkjx1xsjdbbm/cia5rhes702euhkjxjrkym5vf/cia5rhes702evhkjxsuode17c', + 'http://example.com/cia5rhes702ewhkjxj1bzme2d/cia5rhes702exhkjx88mzjzre/cia5rhes702eyhkjxst5flmg9/cia5rhes702ezhkjxdar3h55h', + 'http://example.com/cia5rhes702f0hkjxrdjoki1j/cia5rhes702f1hkjx7iz1lpso/cia5rhes702f2hkjxvsyy2boh/cia5rhes702f3hkjxe4lwxkjq', + 'http://example.com/cia5rhes702f4hkjxhsgvfwf9/cia5rhes702f5hkjxtemdddm6/cia5rhes702f6hkjx8t7z5qmo/cia5rhes702f7hkjxgb9mzb5t', + 'http://example.com/cia5rhes702f8hkjxen7vbt3a/cia5rhes702f9hkjxozpijk1f/cia5rhes702fahkjxh2l1f7h6/cia5rhes702fbhkjxxojzw7gn', + 'http://example.com/cia5rhes702fchkjx0tvnzt2w/cia5rhes702fdhkjxbi6zt33e/cia5rhes702fehkjxd54fxgzx/cia5rhes702ffhkjxsayc02os', + 'http://example.com/cia5rhes702fghkjxpygjjz89/cia5rhes702fhhkjxbct2ojjb/cia5rhes702fihkjxe46ngi4m/cia5rhes702fjhkjxq7azlfig', + 'http://example.com/cia5rhes702fkhkjx1ff4tumn/cia5rhes702flhkjxosiemsy8/cia5rhes702fmhkjx0o6ktv9m/cia5rhes702fnhkjxj9yp67gs', + 'http://example.com/cia5rhes702fohkjxqro5xqt8/cia5rhes702fphkjx6s3gi6y0/cia5rhes702fqhkjxkkab85zz/cia5rhes702frhkjxo03b56tw', + 'http://example.com/cia5rhes702fshkjxlvaiv6rz/cia5rhes702fthkjxvkg1r7dy/cia5rhes702fuhkjx3txhokr4/cia5rhes702fvhkjxtvqvs9ei', + 'http://example.com/cia5rhes702fwhkjx5mknq1w5/cia5rhes702fxhkjxrj6a3pub/cia5rhes702fyhkjxvhu05ms3/cia5rhes702fzhkjxjby42qra', + 'http://example.com/cia5rhes702g0hkjxrcf4pcw1/cia5rhes702g1hkjxn081wq4r/cia5rhes702g2hkjxaf91n239/cia5rhes702g3hkjxxlcnut0h', + 'http://example.com/cia5rhes702g4hkjxboifrcf9/cia5rhes702g5hkjxzdowoz5o/cia5rhes702g6hkjxukarx97t/cia5rhes702g7hkjxccz4m3ra', + 'http://example.com/cia5rhes702g8hkjxcojon0ux/cia5rhes702g9hkjxldlady20/cia5rhes702gahkjxzy3fh4eg/cia5rhes702gbhkjxrbfe6e4i', + 'http://example.com/cia5rhes702gchkjxk66e8nbf/cia5rhes702gdhkjxudeemkvv/cia5rhes702gehkjx3c3hpe66/cia5rhes702gfhkjxn9olbr7q', + 'http://example.com/cia5rhes702gghkjxjutmvz9r/cia5rhes702ghhkjxevjnumc0/cia5rhes702gihkjxcsgdpbt7/cia5rhes702gjhkjxkajsb5n7', + 'http://example.com/cia5rhes702gkhkjxpctjecch/cia5rhes702glhkjx4psglrrf/cia5rhes702gmhkjxqsa29brc/cia5rhes702gnhkjxtu5lc4me', + 'http://example.com/cia5rhes702gohkjxy4ljuvei/cia5rhes702gphkjxscllm1ij/cia5rhes702gqhkjx1e8d9ndd/cia5rhes702grhkjxvt3mx80t', + 'http://example.com/cia5rhes702gshkjxex3gg0nz/cia5rhes702gthkjxlonhrzjs/cia5rhes702guhkjxl4vdp4al/cia5rhes702gvhkjxvu5xtj65', + 'http://example.com/cia5rhes702gwhkjx2eqa8s8p/cia5rhes702gxhkjxzs0d96f8/cia5rhes702gyhkjxh5qhyc6d/cia5rhes702gzhkjxit6h6kq6', + 'http://example.com/cia5rhes702h0hkjxovyyxzzh/cia5rhes702h1hkjxumx2doq9/cia5rhes702h2hkjxe8rwx6ye/cia5rhes702h3hkjxd0biux3c', + 'http://example.com/cia5rhes702h4hkjx0r9rhds4/cia5rhes802h5hkjxxe3pbik6/cia5rhes802h6hkjxkoyqybob/cia5rhes802h7hkjx0s2gcxkk', + 'http://example.com/cia5rhes802h8hkjx1b212net/cia5rhes802h9hkjxhye14m2j/cia5rhes802hahkjxf87hamb3/cia5rhes802hbhkjxvqh1ek5s', + 'http://example.com/cia5rhes802hchkjx5hdnwwul/cia5rhes802hdhkjxyc9ojtpr/cia5rhes802hehkjxdnrgdch1/cia5rhes802hfhkjxj6gwgjbt', + 'http://example.com/cia5rhes802hghkjxehwpywy9/cia5rhes802hhhkjx4lbi6x6l/cia5rhes802hihkjxnpf2cz93/cia5rhes802hjhkjxv9bgej4e', + 'http://example.com/cia5rhes802hkhkjxta1aj8pd/cia5rhes802hlhkjxqot5lx49/cia5rhes802hmhkjxs0uj77o1/cia5rhes802hnhkjx69uqlhl9', + 'http://example.com/cia5rhes802hohkjxsetak465/cia5rhes802hphkjx7cc4cvnw/cia5rhes802hqhkjxyz2rd85f/cia5rhes802hrhkjxwwwj80zy', + 'http://example.com/cia5rhes802hshkjxcxpfz2zy/cia5rhes802hthkjx0mg13xvr/cia5rhes802huhkjxl8tf2f1j/cia5rhes802hvhkjxkkdxui48', + 'http://example.com/cia5rhes802hwhkjxnt5u3nhm/cia5rhes802hxhkjxbffb2x8l/cia5rhes802hyhkjxd0tm0h6e/cia5rhes802hzhkjxua697jh2', + 'http://example.com/cia5rhes802i0hkjx5thy2y3q/cia5rhes802i1hkjx3jr1y269/cia5rhes802i2hkjxwwksi6eg/cia5rhes802i3hkjxor5nv0z2', + 'http://example.com/cia5rhes802i4hkjx4ttg4je9/cia5rhes802i5hkjxqzq7w677/cia5rhes802i6hkjxeldnbsf2/cia5rhes802i7hkjxk8rmgjfv', + 'http://example.com/cia5rhes802i8hkjx6eb7w4np/cia5rhes802i9hkjxstgvt28t/cia5rhes802iahkjx8b9vwdzr/cia5rhes802ibhkjx1pnrsc7b', + 'http://example.com/cia5rhes802ichkjxvo1nrawf/cia5rhes802idhkjxgivthtjh/cia5rhes802iehkjxx967w9dk/cia5rhes802ifhkjxuu3hsee9', + 'http://example.com/cia5rhes802ighkjxeijczff2/cia5rhes802ihhkjxer0knjjl/cia5rhes802iihkjx116p0tfc/cia5rhes802ijhkjxuomqb7a0', + 'http://example.com/cia5rhes802ikhkjxzw0s6ejs/cia5rhes802ilhkjx1fgypntw/cia5rhes802imhkjx7jreimgw/cia5rhes802inhkjx3shm6234', + 'http://example.com/cia5rhes802iohkjx28sv1ivu/cia5rhes802iphkjxr4p098ji/cia5rhes802iqhkjxdsotusgp/cia5rhes802irhkjx5kudhhd2', + 'http://example.com/cia5rhes802ishkjxixiz6mp1/cia5rhes802ithkjxcagn4wzv/cia5rhes802iuhkjxnulj8edc/cia5rhes802ivhkjxkc73vmwx', + 'http://example.com/cia5rhes802iwhkjx0gdh9w2o/cia5rhes802ixhkjxcl50g4e1/cia5rhes802iyhkjxrptys42g/cia5rhes802izhkjx4w62hqht', + 'http://example.com/cia5rhes802j0hkjx692slfdu/cia5rhes802j1hkjxv3bwwytl/cia5rhes802j2hkjxx3gky18b/cia5rhes802j3hkjx3vsa5jra', + 'http://example.com/cia5rhes802j4hkjx2xe9zlx3/cia5rhes802j5hkjx4f3wi9rl/cia5rhes802j6hkjx2qr5bzrp/cia5rhes802j7hkjxrw3fcfe5', + 'http://example.com/cia5rhes802j8hkjx14k6emm1/cia5rhes802j9hkjxcll7rahj/cia5rhes802jahkjx6dmkabft/cia5rhes802jbhkjxj2d4kvm5', + 'http://example.com/cia5rhes802jchkjxu3olmu84/cia5rhes802jdhkjx1kyxhqd9/cia5rhes802jehkjxmzlxuvus/cia5rhes802jfhkjxt7cvj5h1', + 'http://example.com/cia5rhes802jghkjx80q77jzc/cia5rhes802jhhkjxtj8xxa1e/cia5rhes802jihkjxu4lnwkqf/cia5rhes802jjhkjx33w6a2yi', + 'http://example.com/cia5rhes802jkhkjxefk60o55/cia5rhes802jlhkjx9yuz30ib/cia5rhes802jmhkjxhuhfcbcy/cia5rhes802jnhkjxf6wkt9ht', + 'http://example.com/cia5rhes802johkjxbbd9f8zb/cia5rhes802jphkjxzk5mtk4f/cia5rhes802jqhkjxwb6eerfn/cia5rhes802jrhkjxpuyyqgqw', + 'http://example.com/cia5rhes802jshkjxbgeqep0t/cia5rhes802jthkjxdkbxnh69/cia5rhes802juhkjx03vlhdpu/cia5rhes802jvhkjx8zhdnauu', + 'http://example.com/cia5rhes802jwhkjxkw5bbsew/cia5rhes802jxhkjxgzdwzev2/cia5rhes802jyhkjxeutiz8ot/cia5rhes802jzhkjxeigt9qdf', + 'http://example.com/cia5rhes802k0hkjxvmfunndw/cia5rhes802k1hkjx6j968gws/cia5rhes802k2hkjxjdz6yfk6/cia5rhes802k3hkjxnsoiuwfm', + 'http://example.com/cia5rhes802k4hkjx7fezq8em/cia5rhes802k5hkjxsgmtvhig/cia5rhes802k6hkjx8h5r0ac5/cia5rhes802k7hkjxm0tczqr3', + 'http://example.com/cia5rhes802k8hkjx3ej667en/cia5rhes802k9hkjxeta3mqrs/cia5rhes802kahkjxttm3mtbc/cia5rhes802kbhkjx08tnchxt', + 'http://example.com/cia5rhes802kchkjxs9ys1d1h/cia5rhes802kdhkjxa7zfmkfh/cia5rhes802kehkjx0f1f3s5x/cia5rhes802kfhkjx5bkfptdv', + 'http://example.com/cia5rhes802kghkjxfbx0j2be/cia5rhes802khhkjx2796rmnr/cia5rhes802kihkjxc8qjmfqv/cia5rhes802kjhkjx5l18ngbo', + 'http://example.com/cia5rhes802kkhkjxuvxeycqp/cia5rhes802klhkjxt3xak01c/cia5rhes802kmhkjxsnbinf75/cia5rhes802knhkjxdke5f04u', + 'http://example.com/cia5rhes802kohkjxlr2esas9/cia5rhes802kphkjxoi8bubek/cia5rhes802kqhkjx652tsdtk/cia5rhes802krhkjxx0d9sapx', + 'http://example.com/cia5rhes802kshkjx01h9i4q2/cia5rhes802kthkjxnvp9j4x1/cia5rhes802kuhkjxyfb118if/cia5rhes802kvhkjxcajg7k7x', + 'http://example.com/cia5rhes802kwhkjx1ahqqj6a/cia5rhes802kxhkjx576izsui/cia5rhes802kyhkjxdopj85lq/cia5rhes802kzhkjxrak3td4w', + 'http://example.com/cia5rhes802l0hkjx4oalj3hp/cia5rhes802l1hkjxuiaufryz/cia5rhes802l2hkjx3yx8z13v/cia5rhes802l3hkjxql0nh4mw', + 'http://example.com/cia5rhes802l4hkjxh2yk4att/cia5rhes802l5hkjx1ld7evsc/cia5rhes802l6hkjx64mt6pcs/cia5rhes802l7hkjxoa0hr513', + 'http://example.com/cia5rhes802l8hkjxb8puz3pu/cia5rhes802l9hkjx40l0wzy4/cia5rhes802lahkjxvqaauxku/cia5rhes802lbhkjxxe2r13sb', + 'http://example.com/cia5rhes802lchkjxp07dxy3z/cia5rhes802ldhkjx4p23lqcu/cia5rhes802lehkjxj1swfy96/cia5rhes802lfhkjxeppnm27y', + 'http://example.com/cia5rhes802lghkjxowt32sxr/cia5rhes802lhhkjxr2wyl9ej/cia5rhes802lihkjx62orwsjq/cia5rhes802ljhkjxw99oj10o', + 'http://example.com/cia5rhes802lkhkjxieavj07d/cia5rhes802llhkjxrglllbwb/cia5rhes802lmhkjxjbmalhyj/cia5rhes802lnhkjx54ff2569', + 'http://example.com/cia5rhes802lohkjxgar3wut3/cia5rhes802lphkjx3y6byab9/cia5rhes802lqhkjx2ki1hks2/cia5rhes802lrhkjx867oulq5', + 'http://example.com/cia5rhes802lshkjxb7hkzrqs/cia5rhes802lthkjxfu4yyljq/cia5rhes802luhkjxqswmaz83/cia5rhes802lvhkjxcgjxwpin', + 'http://example.com/cia5rhes802lwhkjxnow6sr6f/cia5rhes802lxhkjxbtxn02ok/cia5rhes802lyhkjxtreu397w/cia5rhes802lzhkjx9fbk1l2s', + 'http://example.com/cia5rhes802m0hkjxuov8bbjf/cia5rhes802m1hkjxfjxcjswu/cia5rhes802m2hkjxnuumriep/cia5rhes802m3hkjxv3abuieh', + 'http://example.com/cia5rhes802m4hkjx7chzbj7m/cia5rhes802m5hkjxamwwacgg/cia5rhes802m6hkjxalw3n1b1/cia5rhes802m7hkjx4oobkiqi', + 'http://example.com/cia5rhes802m8hkjx01qpkvwg/cia5rhes802m9hkjx8s23cjy5/cia5rhes802mahkjx7zklmzeg/cia5rhes802mbhkjxs9htzggq', + 'http://example.com/cia5rhes802mchkjxn4kg4arq/cia5rhes802mdhkjxrlms5rxt/cia5rhes802mehkjx51y6d37q/cia5rhes802mfhkjxgq01e010', + 'http://example.com/cia5rhes802mghkjxcuk2pmky/cia5rhes802mhhkjxcy28ajz9/cia5rhes802mihkjxm3xz72d2/cia5rhes802mjhkjxfecrsmb1', + 'http://example.com/cia5rhes802mkhkjx976oo1q6/cia5rhes802mlhkjxt1d0ks1h/cia5rhes802mmhkjxlya2lnkr/cia5rhes802mnhkjxcjv6cg22', + 'http://example.com/cia5rhes802mohkjxbmd2ljcc/cia5rhes802mphkjxqmpsf43e/cia5rhes802mqhkjx1018sa5h/cia5rhes802mrhkjx83lcx364', + 'http://example.com/cia5rhes802mshkjxx317boaq/cia5rhes802mthkjxvgax0zmu/cia5rhes802muhkjxfgnulq5x/cia5rhes802mvhkjxg9czop5a', + 'http://example.com/cia5rhes802mwhkjx3qru605e/cia5rhes802mxhkjxo14r6mbk/cia5rhes802myhkjxnxvtblhe/cia5rhes802mzhkjxp4fuyyvq', + 'http://example.com/cia5rhes802n0hkjxta6r34nn/cia5rhes802n1hkjxn2des330/cia5rhes802n2hkjxut3wbscg/cia5rhes802n3hkjxi3sjsek8', + 'http://example.com/cia5rhes802n4hkjxlrze879b/cia5rhes802n5hkjxl9d2bptv/cia5rhes802n6hkjxe2pyq523/cia5rhes802n7hkjx3d7uk0va', + 'http://example.com/cia5rhes802n8hkjxsm87le7w/cia5rhes802n9hkjxilk0wcph/cia5rhes802nahkjx2n4ghjd4/cia5rhes802nbhkjx2z0n5kej', + 'http://example.com/cia5rhes802nchkjxjyazwvt2/cia5rhes802ndhkjxg9kfrprk/cia5rhes802nehkjxbgdpif6f/cia5rhes802nfhkjxe0456keb', + 'http://example.com/cia5rhes802nghkjxjhpr22o6/cia5rhes802nhhkjxplhnqcb8/cia5rhes802nihkjxzys0lxo2/cia5rhes802njhkjx2q01z427', + 'http://example.com/cia5rhes802nkhkjx2bh3a4jg/cia5rhes802nlhkjxwr4hs5z6/cia5rhes802nmhkjxuj8y14q2/cia5rhes802nnhkjxuhl3zdhl', + 'http://example.com/cia5rhes802nohkjxsghqd6qb/cia5rhes802nphkjxzwvmt5ut/cia5rhes802nqhkjxr3vatvee/cia5rhes802nrhkjx2bozv5k1', + 'http://example.com/cia5rhes802nshkjx2d7r9wfy/cia5rhes802nthkjxcxj3kn6a/cia5rhes802nuhkjxdria7pkp/cia5rhes802nvhkjx6uliansr', + 'http://example.com/cia5rhes802nwhkjx8nqjhhq1/cia5rhes802nxhkjxc2k2euc9/cia5rhes802nyhkjxdv6dq6vu/cia5rhes802nzhkjxidl9ujw8', + 'http://example.com/cia5rhes802o0hkjxt3hs5pt1/cia5rhes802o1hkjxouvuo74k/cia5rhes802o2hkjx46xz3nds/cia5rhes802o3hkjxrrrkqadg', + 'http://example.com/cia5rhes902o4hkjx2apdepej/cia5rhes902o5hkjxqqttkkkz/cia5rhes902o6hkjxh46l0jeu/cia5rhes902o7hkjxl7h17xdc', + 'http://example.com/cia5rhes902o8hkjxbafzc6v5/cia5rhes902o9hkjxcuowkvn1/cia5rhes902oahkjxasvphtbh/cia5rhes902obhkjxgp6ckpu5', + 'http://example.com/cia5rhes902ochkjxfb99zhss/cia5rhes902odhkjx0idz3cqv/cia5rhes902oehkjxy0f9nkn1/cia5rhes902ofhkjxnhrq2m1r', + 'http://example.com/cia5rhes902oghkjx24kuk19k/cia5rhes902ohhkjx5hx5puqb/cia5rhes902oihkjxcaqprqtz/cia5rhes902ojhkjx3zh6ivhp', + 'http://example.com/cia5rhes902okhkjxuk062elz/cia5rhes902olhkjxpv0ezkgb/cia5rhes902omhkjx6gkm3rj1/cia5rhes902onhkjxmckdzmmf', + 'http://example.com/cia5rhes902oohkjx6667yepw/cia5rhes902ophkjxkhrilcux/cia5rhes902oqhkjxubgywv84/cia5rhes902orhkjxl2z5gfhv', + 'http://example.com/cia5rhes902oshkjxwnznffds/cia5rhes902othkjx2nrd505l/cia5rhes902ouhkjxor8wvi62/cia5rhes902ovhkjxkknnf2c5', + 'http://example.com/cia5rhes902owhkjx0xvzj6j4/cia5rhes902oxhkjxm8wjviav/cia5rhes902oyhkjxd48tw0nv/cia5rhes902ozhkjxy55fth1m', + 'http://example.com/cia5rhes902p0hkjxhf2ln9fg/cia5rhes902p1hkjxn1kh849s/cia5rhes902p2hkjx7w18z1ij/cia5rhes902p3hkjx1iukw4f9', + 'http://example.com/cia5rhes902p4hkjx94e9yno8/cia5rhes902p5hkjxgg7krrow/cia5rhes902p6hkjxs7qbcgio/cia5rhes902p7hkjxjy5ubg21', + 'http://example.com/cia5rhes902p8hkjxc76syimq/cia5rhes902p9hkjxr4crms15/cia5rhes902pahkjxnijggak5/cia5rhes902pbhkjxzj7ajf4p', + 'http://example.com/cia5rhes902pchkjxtq8dybc1/cia5rhes902pdhkjxwqg0v1ob/cia5rhes902pehkjxig150nfx/cia5rhes902pfhkjx4pn0r7va', + 'http://example.com/cia5rhes902pghkjxg86s4zod/cia5rhes902phhkjxc16il6yq/cia5rhes902pihkjx25j53w11/cia5rhes902pjhkjxar484o36', + 'http://example.com/cia5rhes902pkhkjxqjtgnf6o/cia5rhes902plhkjxx22y2p6c/cia5rhes902pmhkjxu72lfdom/cia5rhes902pnhkjxv7bb9e9q', + 'http://example.com/cia5rhes902pohkjxxb029uj1/cia5rhes902pphkjx4ujdzzo5/cia5rhes902pqhkjxx4lnnhw7/cia5rhes902prhkjx6x2u79ck', + 'http://example.com/cia5rhes902pshkjxd1hhakk6/cia5rhes902pthkjxmpu8mcyi/cia5rhes902puhkjxzpcbicof/cia5rhes902pvhkjxij383b25', + 'http://example.com/cia5rhes902pwhkjxiur6rdbh/cia5rhes902pxhkjxyhkhpxrq/cia5rhes902pyhkjx29a11uyj/cia5rhes902pzhkjxf1p8g30r', + 'http://example.com/cia5rhes902q0hkjxotowbqgb/cia5rhes902q1hkjxmb7p5sr6/cia5rhes902q2hkjx378apexd/cia5rhes902q3hkjxjkglr1c4', + 'http://example.com/cia5rhes902q4hkjxxcw4jsq6/cia5rhes902q5hkjxqenj7c97/cia5rhes902q6hkjx2ye8s3q1/cia5rhes902q7hkjxtxm7sdya', + 'http://example.com/cia5rhes902q8hkjxkc9vstb2/cia5rhes902q9hkjxzwok7ng9/cia5rhes902qahkjx8ygp04d1/cia5rhes902qbhkjx4qux7aki', + 'http://example.com/cia5rhes902qchkjx57dfmt8h/cia5rhes902qdhkjx9b0035cy/cia5rhes902qehkjxvebgxts8/cia5rhes902qfhkjxu6yi37mb', + 'http://example.com/cia5rhes902qghkjx6xi3dyjx/cia5rhes902qhhkjx9x0aclfr/cia5rhes902qihkjx7mxvg28t/cia5rhes902qjhkjx9q2wphpa', + 'http://example.com/cia5rhes902qkhkjxu6s6q2q0/cia5rhes902qlhkjxlgcpgxpx/cia5rhes902qmhkjxc1dxbvr1/cia5rhes902qnhkjx6bvhf6hr', + 'http://example.com/cia5rhes902qohkjx1tm0hkvs/cia5rhes902qphkjx3pu22rbr/cia5rhes902qqhkjxdjcth8ug/cia5rhes902qrhkjxwhg6mr88', + 'http://example.com/cia5rhes902qshkjxji4arcck/cia5rhes902qthkjx5t1kjk9o/cia5rhes902quhkjx88zgme2o/cia5rhes902qvhkjxhs22agoc', + 'http://example.com/cia5rhes902qwhkjxkqdfy8em/cia5rhes902qxhkjxo11waca0/cia5rhes902qyhkjxthqzds0b/cia5rhes902qzhkjx890jrftn', + 'http://example.com/cia5rhes902r0hkjx2scv74kv/cia5rhes902r1hkjxhczgr5iw/cia5rhes902r2hkjxd9v3ewx1/cia5rhes902r3hkjxx4tpj5xh', + 'http://example.com/cia5rhes902r4hkjx8217649m/cia5rhes902r5hkjx954lwwvc/cia5rhes902r6hkjxzczxe9o4/cia5rhes902r7hkjxadtvbtm3', + 'http://example.com/cia5rhes902r8hkjx1su72qpn/cia5rhes902r9hkjxexh45oq0/cia5rhes902rahkjxah76ntxr/cia5rhes902rbhkjxnnfojf19', + 'http://example.com/cia5rhes902rchkjxc85n1zzu/cia5rhes902rdhkjxix5w6nkz/cia5rhes902rehkjxogcyyb50/cia5rhes902rfhkjx3r7glwov', + 'http://example.com/cia5rhes902rghkjxtck9dhwc/cia5rhes902rhhkjxru36hy4a/cia5rhes902rihkjxmmyc9tpx/cia5rhes902rjhkjxmbsypxaq', + 'http://example.com/cia5rhes902rkhkjx2fo040pu/cia5rhes902rlhkjxr65jltb9/cia5rhes902rmhkjxnzf86rqg/cia5rhes902rnhkjxca9gnhfv', + 'http://example.com/cia5rhes902rohkjx3t12qfew/cia5rhes902rphkjx0uy0q6x0/cia5rhes902rqhkjxytr1mozv/cia5rhes902rrhkjxti5cpfhq', + 'http://example.com/cia5rhes902rshkjxtzuesbvw/cia5rhes902rthkjx4imx7yq2/cia5rhes902ruhkjxv5rwbdfw/cia5rhes902rvhkjxx9dyruvh', + 'http://example.com/cia5rhes902rwhkjx80skj5fy/cia5rhes902rxhkjxs2roo0or/cia5rhes902ryhkjx0f0egqew/cia5rhes902rzhkjx2qyobgwd', + 'http://example.com/cia5rhes902s0hkjxwzjb0ibj/cia5rhes902s1hkjxthhdzgdb/cia5rhes902s2hkjxmp0am5hc/cia5rhes902s3hkjxou8fe0bw', + 'http://example.com/cia5rhes902s4hkjxy807y0wz/cia5rhes902s5hkjxyi0ucjpj/cia5rhes902s6hkjx57r4913i/cia5rhes902s7hkjx5zyg25co', + 'http://example.com/cia5rhes902s8hkjxtv0y9qsr/cia5rhes902s9hkjxmara3sln/cia5rhes902sahkjx16zbww31/cia5rhes902sbhkjxk3yfnqrf', + 'http://example.com/cia5rhes902schkjxmqs7wb8e/cia5rhes902sdhkjxbzqsikjf/cia5rhes902sehkjxifnkxd42/cia5rhes902sfhkjxeslnix9t', + 'http://example.com/cia5rhes902sghkjx9csqi025/cia5rhes902shhkjx03m41rdk/cia5rhes902sihkjx7o16p436/cia5rhes902sjhkjxuopqyoaf', + 'http://example.com/cia5rhes902skhkjxkj9lox0l/cia5rhes902slhkjx4siwdfz6/cia5rhes902smhkjxkz6smrk5/cia5rhes902snhkjxbydhx9sr', + 'http://example.com/cia5rhes902sohkjxmt7rn0m7/cia5rhes902sphkjxbr1rrero/cia5rhes902sqhkjxgsa5faxo/cia5rhes902srhkjxkaypi7hq', + 'http://example.com/cia5rhes902sshkjxcdabjgaq/cia5rhes902sthkjxdj1l2sdw/cia5rhes902suhkjx4w18whjz/cia5rhes902svhkjx00bsy24i', + 'http://example.com/cia5rhes902swhkjx1sxzd3bs/cia5rhes902sxhkjxwihbb32s/cia5rhes902syhkjxjv82ql1y/cia5rhes902szhkjxhx2p1tjw', + 'http://example.com/cia5rhes902t0hkjxlq1v45l8/cia5rhes902t1hkjxpcb65x6c/cia5rhes902t2hkjxqd79lp9t/cia5rhes902t3hkjxzlu0vgsq', + 'http://example.com/cia5rhes902t4hkjxchoh1xz9/cia5rhes902t5hkjxi7ja8w34/cia5rhes902t6hkjxcibihy5j/cia5rhes902t7hkjxzxhj2llf', + 'http://example.com/cia5rhes902t8hkjxe6kjteus/cia5rhes902t9hkjxct0osy9c/cia5rhes902tahkjxkpn37x26/cia5rhes902tbhkjxs1i3y06r', + 'http://example.com/cia5rhes902tchkjxkijsvrry/cia5rhes902tdhkjx478e7b15/cia5rhes902tehkjxe15r2zp0/cia5rhes902tfhkjx0xdr6u4g', + 'http://example.com/cia5rhes902tghkjxre727axs/cia5rhes902thhkjx8tjhkncn/cia5rhes902tihkjxfwe9moa8/cia5rhes902tjhkjxrw37is68', + 'http://example.com/cia5rhes902tkhkjx1vha7oxy/cia5rhes902tlhkjxorgrss4a/cia5rhes902tmhkjx5v2vjvpc/cia5rhes902tnhkjxe13xjwvn', + 'http://example.com/cia5rhes902tohkjxhh415ghg/cia5rhes902tphkjxewddudgl/cia5rhes902tqhkjxlt904su4/cia5rhes902trhkjxvox3ueb9', + 'http://example.com/cia5rhes902tshkjx565cdwgu/cia5rhes902tthkjx7v8dxnp1/cia5rhes902tuhkjx9lkhhc8x/cia5rhes902tvhkjxet30fwnm', + 'http://example.com/cia5rhes902twhkjx50zbd0gj/cia5rhes902txhkjxcxmjzp6i/cia5rhes902tyhkjx4wwog6sc/cia5rhes902tzhkjxl5k35m8y', + 'http://example.com/cia5rhes902u0hkjxmixf873e/cia5rhes902u1hkjxqkzx249g/cia5rhes902u2hkjxq1h6e73c/cia5rhes902u3hkjxy0raorlv', + 'http://example.com/cia5rhes902u4hkjxp7qu708r/cia5rhes902u5hkjxjq511roe/cia5rhes902u6hkjx24zsjlw7/cia5rhes902u7hkjxxao19ibw', + 'http://example.com/cia5rhes902u8hkjxjj8e4qjy/cia5rhes902u9hkjxnpfmyyee/cia5rhesa02uahkjx2cbqjqlj/cia5rhesa02ubhkjx2sb57ho9', + 'http://example.com/cia5rhesa02uchkjxi7stcunb/cia5rhesa02udhkjxf13m1va9/cia5rhesa02uehkjxbrbvzlts/cia5rhesa02ufhkjxv7c5sg8p', + 'http://example.com/cia5rhesa02ughkjxc4bg17mm/cia5rhesa02uhhkjx1fodi8bu/cia5rhesa02uihkjx1pgynm8w/cia5rhesa02ujhkjx21oibarf', + 'http://example.com/cia5rhesa02ukhkjx9l592wdv/cia5rhesa02ulhkjxvbp05nkt/cia5rhesa02umhkjxosf9qynb/cia5rhesa02unhkjx7bb91ukh', + 'http://example.com/cia5rhesa02uohkjxkss6ccme/cia5rhesa02uphkjxvd93brv7/cia5rhesa02uqhkjxoc61qqcx/cia5rhesa02urhkjxawfbw41u', + 'http://example.com/cia5rhesa02ushkjxgxz51hyw/cia5rhesa02uthkjxewu4vl7k/cia5rhesa02uuhkjxknapklva/cia5rhesa02uvhkjxmdf6weyv', + 'http://example.com/cia5rhesa02uwhkjx9egqxjsi/cia5rhesa02uxhkjxfzhkd5yr/cia5rhesa02uyhkjx8hv3p08k/cia5rhesa02uzhkjx45psji1y', + 'http://example.com/cia5rhesa02v0hkjx4l8lsl3u/cia5rhesa02v1hkjxngdy1ar6/cia5rhesa02v2hkjx6jo5h3qu/cia5rhesa02v3hkjxzbv3dpni', + 'http://example.com/cia5rhesa02v4hkjxtcyq3hrj/cia5rhesa02v5hkjxi1yvcnxy/cia5rhesa02v6hkjxw7v5871t/cia5rhesa02v7hkjxe8a1rpaz', + 'http://example.com/cia5rhesa02v8hkjx11e94e28/cia5rhesa02v9hkjxy79y9wsa/cia5rhesa02vahkjx4x1k6e7p/cia5rhesa02vbhkjx4nugadg0', + 'http://example.com/cia5rhesa02vchkjxyff973f9/cia5rhesa02vdhkjxxeylqp99/cia5rhesa02vehkjxup7kbh2i/cia5rhesa02vfhkjxvik2bnru', + 'http://example.com/cia5rhesa02vghkjxojpmez35/cia5rhesa02vhhkjxrsr1rbtw/cia5rhesa02vihkjxz51r21kh/cia5rhesa02vjhkjxnry87ysd', + 'http://example.com/cia5rhesa02vkhkjxc7kcgnod/cia5rhesa02vlhkjxou8csehx/cia5rhesa02vmhkjx4g46j5vv/cia5rhesa02vnhkjx0xdxordo', + 'http://example.com/cia5rhesa02vohkjxcpd7futv/cia5rhesa02vphkjxjhhvuq13/cia5rhesa02vqhkjx1jx0mwyq/cia5rhesa02vrhkjxatheqhre', + 'http://example.com/cia5rhesa02vshkjxhoxrm7du/cia5rhesa02vthkjxp3j2d7pl/cia5rhesa02vuhkjxfajs3kp2/cia5rhesa02vvhkjx094w7t5z', + 'http://example.com/cia5rhesa02vwhkjx8zsoc546/cia5rhesa02vxhkjxbbwesmgs/cia5rhesa02vyhkjxah7vbsl2/cia5rhesa02vzhkjxccc1osvb', + 'http://example.com/cia5rhesa02w0hkjxilmp1gcj/cia5rhesa02w1hkjxpax3mj4u/cia5rhesa02w2hkjxl4830fix/cia5rhesa02w3hkjxushwofrd', + 'http://example.com/cia5rhesa02w4hkjx0ayq3lna/cia5rhesa02w5hkjxtyfjinxi/cia5rhesa02w6hkjx6v3jk6np/cia5rhesa02w7hkjxb3kzmwfz', + 'http://example.com/cia5rhesa02w8hkjxpfztdog3/cia5rhesa02w9hkjxxu1jj9ro/cia5rhesa02wahkjx9x02t4s6/cia5rhesa02wbhkjxmudm4let', + 'http://example.com/cia5rhesa02wchkjxf8gwzm46/cia5rhesa02wdhkjxnogroqj5/cia5rhesa02wehkjxzcswjm19/cia5rhesa02wfhkjxd7sq70cn', + 'http://example.com/cia5rhesa02wghkjxzl71wo9i/cia5rhesa02whhkjx4qzdc4en/cia5rhesa02wihkjxqcwczavg/cia5rhesa02wjhkjx5v1mo7io', + 'http://example.com/cia5rhesa02wkhkjx17rwy1u4/cia5rhesa02wlhkjxupgzmlhz/cia5rhesa02wmhkjxy4guynpo/cia5rhesa02wnhkjx7hwclzmy', + 'http://example.com/cia5rhesa02wohkjxbhrfwyae/cia5rhesa02wphkjxg54k6a1v/cia5rhesa02wqhkjxxx6ovpcu/cia5rhesa02wrhkjx7chazbg3', + 'http://example.com/cia5rhesa02wshkjxsqxk0429/cia5rhesa02wthkjxjmhikrl3/cia5rhesa02wuhkjxojb6ebr1/cia5rhesa02wvhkjxrcv5ezdg', + 'http://example.com/cia5rhesa02wwhkjx6oc3w8ov/cia5rhesa02wxhkjxm2i72vec/cia5rhesa02wyhkjx3fh9ne9a/cia5rhesa02wzhkjx0971hhm1', + 'http://example.com/cia5rhesa02x0hkjxwenvo26l/cia5rhesa02x1hkjxtfilhs8a/cia5rhesa02x2hkjxpqvnoyqk/cia5rhesa02x3hkjx23vjztdc', + 'http://example.com/cia5rhesa02x4hkjxsgmx5os7/cia5rhesa02x5hkjxgrehs28q/cia5rhesa02x6hkjxvxtnze9l/cia5rhesa02x7hkjx0vlv9z1s', + 'http://example.com/cia5rhesa02x8hkjxge35kywm/cia5rhesa02x9hkjxw2tbefo5/cia5rhesa02xahkjxv137f9qt/cia5rhesa02xbhkjxnz9ep47k', + 'http://example.com/cia5rhesa02xchkjx00anlyr6/cia5rhesa02xdhkjx79zjud7w/cia5rhesa02xehkjxrb6rk7rw/cia5rhesa02xfhkjxphslyr6m', + 'http://example.com/cia5rhesa02xghkjxv656h0en/cia5rhesa02xhhkjxwt9sllti/cia5rhesa02xihkjxbblv9n51/cia5rhesa02xjhkjxoms9ldox', + 'http://example.com/cia5rhesa02xkhkjxb0ljdnru/cia5rhesa02xlhkjxuysb8km6/cia5rhesa02xmhkjx7sdb3ap1/cia5rhesa02xnhkjx556d8gld', + 'http://example.com/cia5rhesa02xohkjxmh07tx4r/cia5rhesa02xphkjxzekp4dcp/cia5rhesa02xqhkjxywdkdqe0/cia5rhesa02xrhkjxn2exhmk3', + 'http://example.com/cia5rhesa02xshkjxot0nxe5o/cia5rhesa02xthkjxxns5dc6l/cia5rhesa02xuhkjxppx37eq1/cia5rhesa02xvhkjxa0n1ft75', + 'http://example.com/cia5rhesa02xwhkjxdlz42y7u/cia5rhesa02xxhkjxyyrh6ehu/cia5rhesa02xyhkjxc2snloyu/cia5rhesa02xzhkjx7hniv0l2', + 'http://example.com/cia5rhesa02y0hkjx1ufllm5d/cia5rhesa02y1hkjxyciovmxi/cia5rhesa02y2hkjxeecnhafz/cia5rhesa02y3hkjxka899vl8', + 'http://example.com/cia5rhesa02y4hkjxas1scma5/cia5rhesa02y5hkjx7hx7cgry/cia5rhesa02y6hkjxt4r854cj/cia5rhesa02y7hkjxl2d58z7z', + 'http://example.com/cia5rhesa02y8hkjx6uvep3v0/cia5rhesa02y9hkjxzkox5acn/cia5rhesa02yahkjxbsttsd42/cia5rhesa02ybhkjx9s0eooqr', + 'http://example.com/cia5rhesa02ychkjxs38kr0ju/cia5rhesa02ydhkjxp7i0dm4v/cia5rhesa02yehkjxytdwg00m/cia5rhesa02yfhkjx6kojbk3h', + 'http://example.com/cia5rhesa02yghkjxu47g5rne/cia5rhesa02yhhkjxp5jiqzck/cia5rhesa02yihkjx9n0k7a86/cia5rhesa02yjhkjxawsmggmg', + 'http://example.com/cia5rhesa02ykhkjxpz5t3big/cia5rhesa02ylhkjxd7mv52ko/cia5rhesa02ymhkjxaq7e3qjp/cia5rhesa02ynhkjxx6n59eel', + 'http://example.com/cia5rhesa02yohkjx151ccq4m/cia5rhesa02yphkjxs6vfdnyb/cia5rhesa02yqhkjxakcfphvj/cia5rhesa02yrhkjxrtgqqlkl', + 'http://example.com/cia5rhesa02yshkjx5nthrj0p/cia5rhesa02ythkjxaoa6xfw1/cia5rhesa02yuhkjxsyk94k5s/cia5rhesa02yvhkjxp1fsbr6q', + 'http://example.com/cia5rhesa02ywhkjxxrjgdzm9/cia5rhesa02yxhkjxmt4liicf/cia5rhesa02yyhkjxqnavwf3w/cia5rhesa02yzhkjxwb6efk2q', + 'http://example.com/cia5rhesa02z0hkjxai4gwv4i/cia5rhesb02z1hkjxotmfrbh1/cia5rhesb02z2hkjxcorstega/cia5rhesb02z3hkjxmm6qrl72', + 'http://example.com/cia5rhesb02z4hkjx119h63fe/cia5rhesb02z5hkjx51zhx95d/cia5rhesb02z6hkjx13iaxgj7/cia5rhesb02z7hkjxnhttadyh', + 'http://example.com/cia5rhesb02z8hkjx7n279k7d/cia5rhesb02z9hkjxtyhtwvh4/cia5rhesb02zahkjxuxc8tnjw/cia5rhesb02zbhkjx9f5w1igd', + 'http://example.com/cia5rhesb02zchkjxu6u03gpq/cia5rhesb02zdhkjxad5fmie8/cia5rhesb02zehkjx82hi1ubw/cia5rhesb02zfhkjxlz014bc9', + 'http://example.com/cia5rhesb02zghkjxpcp41mh4/cia5rhesb02zhhkjxwtmgx1un/cia5rhesb02zihkjxvdlzs9gj/cia5rhesb02zjhkjxxnbuzjtx', + 'http://example.com/cia5rhesb02zkhkjxszpr5g1b/cia5rhesb02zlhkjx5r4u5x2d/cia5rhesb02zmhkjxj9k1c9lb/cia5rhesb02znhkjx76dnsetw', + 'http://example.com/cia5rhesb02zohkjxk9w5hbj0/cia5rhesb02zphkjxsz3yi7na/cia5rhesb02zqhkjxn35x1ss7/cia5rhesb02zrhkjxcmyedvkx', + 'http://example.com/cia5rhesb02zshkjxhuw9xl6g/cia5rhesb02zthkjxnkalu85l/cia5rhesb02zuhkjx0nb3kn0f/cia5rhesb02zvhkjxekzyryxq', + 'http://example.com/cia5rhesb02zwhkjxl2y2pyxt/cia5rhesb02zxhkjxeqoaa6v8/cia5rhesb02zyhkjxu5g8zso5/cia5rhesb02zzhkjx4t9www0y', + 'http://example.com/cia5rhesb0300hkjxdl9hwbgu/cia5rhesb0301hkjxed3e759g/cia5rhesb0302hkjxhsksjcb6/cia5rhesb0303hkjx3fn4nqpg', + 'http://example.com/cia5rhesb0304hkjx0j1w4t7r/cia5rhesb0305hkjx8v918aao/cia5rhesb0306hkjxvdvahvsq/cia5rhesb0307hkjxooa62xp7', + 'http://example.com/cia5rhesb0308hkjx33xkvxg7/cia5rhesb0309hkjxws0twzy4/cia5rhesb030ahkjxtpn8o3r0/cia5rhesb030bhkjxh3wusg8f', + 'http://example.com/cia5rhesb030chkjxt4rz9ksr/cia5rhesb030dhkjxklvupy26/cia5rhesb030ehkjxxf7wxi4e/cia5rhesb030fhkjxa11i0b7u', + 'http://example.com/cia5rhesb030ghkjxc1beup56/cia5rhesb030hhkjxxcgs2kpz/cia5rhesb030ihkjxc7a9g482/cia5rhesb030jhkjxjb51smqy', + 'http://example.com/cia5rhesb030khkjxu6a7pi1g/cia5rhesb030lhkjxo0qkosjn/cia5rhesb030mhkjxucjtxcjj/cia5rhesb030nhkjxbn92q469', + 'http://example.com/cia5rhesb030ohkjxalklya9k/cia5rhesb030phkjx2s6kj633/cia5rhesb030qhkjx5dus8zhl/cia5rhesb030rhkjxexszt1qv', + 'http://example.com/cia5rhesb030shkjxs325mfs5/cia5rhesb030thkjxhj36rw16/cia5rhesb030uhkjxzs6na9ek/cia5rhesb030vhkjxcahu5xdq', + 'http://example.com/cia5rhesb030whkjxbq6vx29f/cia5rhesb030xhkjxzcefm4gv/cia5rhesb030yhkjxcx0o6obs/cia5rhesb030zhkjxy6a654xp', + 'http://example.com/cia5rhesb0310hkjx74o7x2ol/cia5rhesb0311hkjxbaa60v2j/cia5rhesb0312hkjxws59xi68/cia5rhesb0313hkjxe024wl25', + 'http://example.com/cia5rhesb0314hkjxsmo6r7qy/cia5rhesb0315hkjxyo4vsu02/cia5rhesb0316hkjxq7bv4sl0/cia5rhesb0317hkjxnu8xcpds', + 'http://example.com/cia5rhesb0318hkjxfllzbwdp/cia5rhesb0319hkjx9jpu3qeb/cia5rhesb031ahkjx1yw2rxmr/cia5rhesb031bhkjx0kh6iq8e', + 'http://example.com/cia5rhesb031chkjxdjrdwexa/cia5rhesb031dhkjxpdkelmj0/cia5rhesb031ehkjxo0q4659i/cia5rhesb031fhkjxqj6po4aw', + 'http://example.com/cia5rhesb031ghkjxydbw4cnp/cia5rhesb031hhkjx7ak9k5ib/cia5rhesb031ihkjxshjg8guf/cia5rhesb031jhkjxgc7isxom', + 'http://example.com/cia5rhesb031khkjxilu3xhrq/cia5rhesb031lhkjx7qa2tmqv/cia5rhesb031mhkjx2gqx175d/cia5rhesb031nhkjxcclzxfj5', + 'http://example.com/cia5rhesb031ohkjxnqa5u0yu/cia5rhesb031phkjxvj65rgc0/cia5rhesb031qhkjxps94ct2m/cia5rhesb031rhkjx13vf7hqf', + 'http://example.com/cia5rhesb031shkjx4c0rxkqe/cia5rhesb031thkjx2f1rlhtc/cia5rhesb031uhkjxc6mmo6r9/cia5rhesb031vhkjxmkdbf7tz', + 'http://example.com/cia5rhesb031whkjx4wrcwah4/cia5rhesb031xhkjxjy564fgv/cia5rhesb031yhkjxel00ycv5/cia5rhesb031zhkjxwv42qge9', + 'http://example.com/cia5rhesb0320hkjx579rdytw/cia5rhesb0321hkjxqwktfaa3/cia5rhesb0322hkjx5uguvgm4/cia5rhesb0323hkjxod265vm7', + 'http://example.com/cia5rhesb0324hkjxo91kfm12/cia5rhesb0325hkjx7eeoo34p/cia5rhesb0326hkjxkcbju4fy/cia5rhesb0327hkjx9rgv9jej', + 'http://example.com/cia5rhesb0328hkjxu29htifz/cia5rhesb0329hkjx833v3icl/cia5rhesb032ahkjxp93q4nqo/cia5rhesb032bhkjx4tktxa61', + 'http://example.com/cia5rhesb032chkjxli18annx/cia5rhesb032dhkjxoin4rpsb/cia5rhesb032ehkjxkezkkq9n/cia5rhesb032fhkjxxq4syq15', + 'http://example.com/cia5rhesb032ghkjxjr48ia8g/cia5rhesb032hhkjxaz6zhm4c/cia5rhesb032ihkjxriefifyj/cia5rhesb032jhkjxt06hn2ix', + 'http://example.com/cia5rhesb032khkjxl0o2c8hq/cia5rhesb032lhkjxvlfg1dcu/cia5rhesb032mhkjxa6neghbc/cia5rhesb032nhkjxomcdlu3w', + 'http://example.com/cia5rhesb032ohkjxnrefhx6j/cia5rhesb032phkjx05xbd8mi/cia5rhesb032qhkjx22ncbb1j/cia5rhesb032rhkjx8mqw8vvb', + 'http://example.com/cia5rhesb032shkjxzw7wur7z/cia5rhesb032thkjxdybqu2ix/cia5rhesb032uhkjxjqrudsu0/cia5rhesb032vhkjx60p88zgu', + 'http://example.com/cia5rhesb032whkjxsj2cgd7r/cia5rhesb032xhkjxjv4oyt79/cia5rhesb032yhkjxlzlkj3x2/cia5rhesb032zhkjxhkvllyb6', + 'http://example.com/cia5rhesb0330hkjxhduykvat/cia5rhesb0331hkjxqg1x6769/cia5rhesb0332hkjx8scwhj5n/cia5rhesb0333hkjxous8ibmx', +]; diff --git a/dom/cache/test/mochitest/message_receiver.html b/dom/cache/test/mochitest/message_receiver.html new file mode 100644 index 0000000000..82cb587c72 --- /dev/null +++ b/dom/cache/test/mochitest/message_receiver.html @@ -0,0 +1,6 @@ +<!DOCTYPE html> +<script> + navigator.serviceWorker.onmessage = function(e) { + window.parent.postMessage(e.data, "*"); + }; +</script> diff --git a/dom/cache/test/mochitest/mirror.sjs b/dom/cache/test/mochitest/mirror.sjs new file mode 100644 index 0000000000..0006aba8f0 --- /dev/null +++ b/dom/cache/test/mochitest/mirror.sjs @@ -0,0 +1,5 @@ +function handleRequest(request, response) { + response.setStatusLine(request.httpVersion, 200, "OK"); + response.setHeader("Mirrored", request.getHeader("Mirror")); + response.write(request.getHeader("Mirror")); +} diff --git a/dom/cache/test/mochitest/mochitest.ini b/dom/cache/test/mochitest/mochitest.ini new file mode 100644 index 0000000000..92ebb7ad3f --- /dev/null +++ b/dom/cache/test/mochitest/mochitest.ini @@ -0,0 +1,47 @@ +[DEFAULT] +support-files = + test_cache.js + test_cache_add.js + worker_driver.js + worker_wrapper.js + frame.html + message_receiver.html + driver.js + serviceworker_driver.js + test_cache_match_request.js + test_cache_matchAll_request.js + test_cache_overwrite.js + mirror.sjs + test_cache_match_vary.js + vary.sjs + test_caches.js + test_cache_keys.js + test_cache_put.js + test_cache_requestCache.js + test_cache_delete.js + test_cache_put_reorder.js + test_cache_redirect.js + test_cache_https.js + large_url_list.js + empty.html + +[test_cache.html] +[test_cache_add.html] +[test_cache_match_request.html] +[test_cache_matchAll_request.html] +[test_cache_overwrite.html] +[test_cache_match_vary.html] +[test_caches.html] +[test_cache_keys.html] +[test_cache_put.html] +[test_cache_requestCache.html] +[test_cache_delete.html] +[test_cache_put_reorder.html] +[test_cache_https.html] +[test_cache_redirect.html] +[test_cache_restart.html] +[test_cache_shrink.html] +[test_cache_orphaned_cache.html] +[test_cache_orphaned_body.html] +[test_cache_untrusted.html] +[test_chrome_constructor.html] diff --git a/dom/cache/test/mochitest/serviceworker_driver.js b/dom/cache/test/mochitest/serviceworker_driver.js new file mode 100644 index 0000000000..e5e8e47caa --- /dev/null +++ b/dom/cache/test/mochitest/serviceworker_driver.js @@ -0,0 +1,44 @@ +// Any copyright is dedicated to the Public Domain. +// http://creativecommons.org/publicdomain/zero/1.0/ + +function serviceWorkerTestExec(testFile) { + var isB2G = !navigator.userAgent.includes("Android") && + /Mobile|Tablet/.test(navigator.userAgent); + if (isB2G) { + // TODO B2G doesn't support running service workers for now due to bug 1137683. + dump("Skipping running the test in SW until bug 1137683 gets fixed.\n"); + return Promise.resolve(); + } + return new Promise(function(resolve, reject) { + function setupSW(registration) { + var worker = registration.waiting || + registration.active; + + window.addEventListener("message",function onMessage(event) { + if (event.data.context != "ServiceWorker") { + return; + } + if (event.data.type == 'finish') { + window.removeEventListener("message", onMessage); + registration.unregister() + .then(resolve) + .catch(reject); + } else if (event.data.type == 'status') { + ok(event.data.status, event.data.context + ": " + event.data.msg); + } + }, false); + + worker.onerror = reject; + + var iframe = document.createElement("iframe"); + iframe.src = "message_receiver.html"; + iframe.onload = function() { + worker.postMessage({ script: testFile }); + }; + document.body.appendChild(iframe); + } + + navigator.serviceWorker.ready.then(setupSW); + navigator.serviceWorker.register("worker_wrapper.js", {scope: "."}); + }); +} diff --git a/dom/cache/test/mochitest/test_cache.html b/dom/cache/test/mochitest/test_cache.html new file mode 100644 index 0000000000..ef945b75e8 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Interfaces Exposed to Workers</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache.js b/dom/cache/test/mochitest/test_cache.js new file mode 100644 index 0000000000..c8927956c9 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache.js @@ -0,0 +1,131 @@ +var c = null +var request = "http://example.com/hmm?q=foobar" + context; +var response = new Response("This is some Response!"); +var name = 'snafu' + context; +var foobar = 'foobar' + context; + +ok(!!caches, 'caches object should be available on global'); +caches.open(name).then(function(openCache) { + ok(openCache instanceof Cache, 'cache object should be resolved from caches.open'); + return caches.has(name); +}).then(function(hasResult) { + ok(hasResult, 'caches.has() should resolve true'); + return caches.keys(); +}).then(function(keys) { + ok(!!keys, 'caches.keys() should resolve to a truthy value'); + ok(keys.length >= 1, 'caches.keys() should resolve to an array of length at least 1'); + ok(keys.indexOf(name) >= 0, 'caches.keys() should resolve to an array containing key'); + return caches.delete(name); +}).then(function(deleteResult) { + ok(deleteResult, 'caches.delete() should resolve true'); + return caches.has(name); +}).then(function(hasMissingCache) { + ok(!hasMissingCache, 'missing key should return false from has'); +}).then(function() { + return caches.open(name); +}).then(function(snafu) { + return snafu.keys(); +}).then(function(empty) { + is(0, empty.length, 'cache.keys() should resolve to an array of length 0'); +}).then(function() { + return caches.open(name); +}).then(function(snafu) { + var req = './cachekey'; + var res = new Response("Hello world"); + return snafu.put('ftp://invalid', res).then(function() { + ok(false, 'This should fail'); + }).catch(function (err) { + is(err.name, 'TypeError', 'put() should throw TypeError for invalid scheme'); + return snafu.put(req, res); + }).then(function(v) { + return snafu; + }); +}).then(function(snafu) { + return Promise.all([snafu, snafu.keys()]); +}).then(function(args) { + var snafu = args[0]; + var keys = args[1]; + is(1, keys.length, 'cache.keys() should resolve to an array of length 1'); + ok(keys[0] instanceof Request, 'key should be a Request'); + ok(keys[0].url.match(/cachekey$/), 'Request URL should match original'); + return Promise.all([snafu, snafu.match(keys[0]), snafu.match('ftp://invalid')]); +}).then(function(args) { + var snafu = args[0]; + var response = args[1]; + ok(response instanceof Response, 'value should be a Response'); + is(response.status, 200, 'Response status should be 200'); + is(undefined, args[2], 'Match with invalid scheme should resolve undefined'); + return Promise.all([snafu, snafu.put('./cachekey2', response)]); +}).then(function(args) { + var snafu = args[0] + return snafu.match('./cachekey2'); +}).then(function(response) { + return response.text().then(function(v) { + is(v, "Hello world", "Response body should match original"); + }); +}).then(function() { + // FIXME(nsm): Can't use a Request object for now since the operations + // consume it's 'body'. See + // https://github.com/slightlyoff/ServiceWorker/issues/510. + return caches.open(foobar); +}).then(function(openCache) { + c = openCache; + return c.put(request, response); +}).then(function(putResponse) { + is(putResponse, undefined, 'The promise should resolve to undefined'); + return c.keys(request); +}).then(function(keys) { + ok(keys, 'Valid keys object expected'); + is(keys.length, 1, 'Only one key is expected'); + return c.keys(); +}).then(function(keys) { + ok(keys, 'Valid keys object expected'); + is(keys.length, 1, 'Only one key is expected'); + return c.matchAll(request); +}).then(function(matchAllResponses) { + ok(matchAllResponses, 'matchAll should succeed'); + is(matchAllResponses.length, 1, 'Only one match is expected'); + return c.match(request); +}).then(function(matchResponse) { + ok(matchResponse, 'match should succeed'); + return caches.match(request); +}).then(function(storageMatchResponse) { + ok(storageMatchResponse, 'storage match should succeed'); + return caches.match(request, {cacheName:foobar}); +}).then(function(storageMatchResponse) { + ok(storageMatchResponse, 'storage match with cacheName should succeed'); + var request2 = new Request("http://example.com/hmm?q=snafu" + context); + return c.match(request2, {ignoreSearch:true}); +}).then(function(match2Response) { + ok(match2Response, 'match should succeed'); + return c.delete(request); +}).then(function(deleteResult) { + ok(deleteResult, 'delete should succeed'); + return c.keys(); +}).then(function(keys) { + ok(keys, 'Valid keys object expected'); + is(keys.length, 0, 'Zero keys is expected'); + return c.matchAll(request); +}).then(function(matchAll2Responses) { + ok(matchAll2Responses, 'matchAll should succeed'); + is(matchAll2Responses.length, 0, 'Zero matches is expected'); + return caches.has(foobar); +}).then(function(hasResult) { + ok(hasResult, 'has should succeed'); + return caches.keys(); +}).then(function(keys) { + ok(keys, 'Valid keys object expected'); + ok(keys.length >= 2, 'At least two keys are expected'); + ok(keys.indexOf(name) >= 0, 'snafu should exist'); + ok(keys.indexOf(foobar) >= keys.indexOf(name), 'foobar should come after it'); + return caches.delete(foobar); +}).then(function(deleteResult) { + ok(deleteResult, 'delete should succeed'); + return caches.has(foobar); +}).then(function(hasMissingCache) { + ok(!hasMissingCache, 'has should have a result'); + return caches.delete(name); +}).then(function(deleteResult) { + ok(deleteResult, 'delete should succeed'); + testDone(); +}) diff --git a/dom/cache/test/mochitest/test_cache_add.html b/dom/cache/test/mochitest/test_cache_add.html new file mode 100644 index 0000000000..81c84d5fb7 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_add.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Interfaces Exposed to Workers</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_add.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_add.js b/dom/cache/test/mochitest/test_cache_add.js new file mode 100644 index 0000000000..0719f1a10a --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_add.js @@ -0,0 +1,55 @@ +var singleUrl = './test_cache_add.js'; +var urlList = [ + './empty.html', + './frame.html', + './test_cache.js' +]; +var cache; +var name = "adder" + context; +caches.open(name).then(function(openCache) { + cache = openCache; + return cache.add('ftp://example.com/invalid' + context); +}).catch(function (err) { + is(err.name, 'TypeError', 'add() should throw TypeError for invalid scheme'); + return cache.addAll(['http://example.com/valid' + context, 'ftp://example.com/invalid' + context]); +}).catch(function (err) { + is(err.name, 'TypeError', 'addAll() should throw TypeError for invalid scheme'); + var promiseList = urlList.map(function(url) { + return cache.match(url); + }); + promiseList.push(cache.match(singleUrl)); + return Promise.all(promiseList); +}).then(function(resultList) { + is(urlList.length + 1, resultList.length, 'Expected number of results'); + resultList.every(function(result) { + is(undefined, result, 'URLs should not already be in the cache'); + }); + return cache.add(singleUrl); +}).then(function(result) { + is(undefined, result, 'Successful add() should resolve undefined'); + return cache.addAll(urlList); +}).then(function(result) { + is(undefined, result, 'Successful addAll() should resolve undefined'); + var promiseList = urlList.map(function(url) { + return cache.match(url); + }); + promiseList.push(cache.match(singleUrl)); + return Promise.all(promiseList); +}).then(function(resultList) { + is(urlList.length + 1, resultList.length, 'Expected number of results'); + resultList.every(function(result) { + ok(!!result, 'Responses should now be in cache for each URL.'); + }); + return cache.matchAll(); +}).then(function(resultList) { + is(urlList.length + 1, resultList.length, 'Expected number of results'); + resultList.every(function(result) { + ok(!!result, 'Responses should now be in cache for each URL.'); + }); + return caches.delete(name); +}).then(function() { + testDone(); +}).catch(function(err) { + ok(false, 'Caught error: ' + err); + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_cache_delete.html b/dom/cache/test/mochitest/test_cache_delete.html new file mode 100644 index 0000000000..a9bcf4328f --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_delete.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate the Cache.delete() method</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_delete.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_delete.js b/dom/cache/test/mochitest/test_cache_delete.js new file mode 100644 index 0000000000..aa7bc94eb8 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_delete.js @@ -0,0 +1,111 @@ +var name = "delete" + context; +var c; + +function setupTest(reqs) { + return new Promise(function(resolve, reject) { + var cache; + caches.open(name).then(function(c) { + cache = c; + return c.addAll(reqs); + }).then(function() { + resolve(cache); + }).catch(function(err) { + reject(err); + }); + }); +} + +function testBasics() { + var tests = [ + "//mochi.test:8888/?foo" + context, + "//mochi.test:8888/?bar" + context, + ]; + var cache; + return setupTest(tests) + .then(function(c) { + cache = c; + return cache.delete("//mochi.test:8888/?baz"); + }).then(function(deleted) { + ok(!deleted, "Deleting a non-existing entry should fail"); + return cache.keys(); + }).then(function(keys) { + is(keys.length, 2, "No entries from the cache should be deleted"); + return cache.delete(tests[0]); + }).then(function(deleted) { + ok(deleted, "Deleting an existing entry should succeed"); + return cache.keys(); + }).then(function(keys) { + is(keys.length, 1, "Only one entry should exist now"); + ok(keys[0].url.indexOf(tests[1]) >= 0, "The correct entry must be deleted"); + }); +} + +function testFragment() { + var tests = [ + "//mochi.test:8888/?foo" + context, + "//mochi.test:8888/?bar" + context, + "//mochi.test:8888/?baz" + context + "#fragment", + ]; + var cache; + return setupTest(tests) + .then(function(c) { + cache = c; + return cache.delete(tests[0] + "#fragment"); + }).then(function(deleted) { + ok(deleted, "Deleting an existing entry should succeed"); + return cache.keys(); + }).then(function(keys) { + is(keys.length, 2, "Only one entry should exist now"); + ok(keys[0].url.indexOf(tests[1]) >= 0, "The correct entry must be deleted"); + ok(keys[1].url.indexOf(tests[2].replace("#fragment", "")) >= 0, "The correct entry must be deleted"); + // Now, delete a request that was added with a fragment + return cache.delete("//mochi.test:8888/?baz" + context); + }).then(function(deleted) { + ok(deleted, "Deleting an existing entry should succeed"); + return cache.keys(); + }).then(function(keys) { + is(keys.length, 1, "Only one entry should exist now"); + ok(keys[0].url.indexOf(tests[1]) >= 0, "3The correct entry must be deleted"); + }); +} + +function testInterleaved() { + var tests = [ + "//mochi.test:8888/?foo" + context, + "//mochi.test:8888/?bar" + context, + ]; + var newURL = "//mochi.test:8888/?baz" + context; + var cache; + return setupTest(tests) + .then(function(c) { + cache = c; + // Simultaneously add and delete a request + return Promise.all([ + cache.delete(newURL), + cache.add(newURL), + ]); + }).then(function(result) { + ok(!result[1], "deletion should fail"); + return cache.keys(); + }).then(function(keys) { + is(keys.length, 3, "Tree entries should still exist"); + ok(keys[0].url.indexOf(tests[0]) >= 0, "The correct entry must be deleted"); + ok(keys[1].url.indexOf(tests[1]) >= 0, "The correct entry must be deleted"); + ok(keys[2].url.indexOf(newURL) >= 0, "The new entry should be correctly inserted"); + }); +} + +// Make sure to clean up after each test step. +function step(testPromise) { + return testPromise.then(function() { + caches.delete(name); + }); +} + +step(testBasics()).then(function() { + return step(testFragment()); +}).then(function() { + return step(testInterleaved()); +}).then(function() { + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_cache_https.html b/dom/cache/test/mochitest/test_cache_https.html new file mode 100644 index 0000000000..8ec509f0e8 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_https.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Interfaces Exposed to Workers</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_https.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_https.js b/dom/cache/test/mochitest/test_cache_https.js new file mode 100644 index 0000000000..b12b6d4f3a --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_https.js @@ -0,0 +1,31 @@ +var cache = null; +var name = 'https_' + context; +var urlBase = 'https://example.com/tests/dom/cache/test/mochitest'; +var url1 = urlBase + '/test_cache.js'; +var url2 = urlBase + '/test_cache_add.js'; + +function addOpaque(cache, url) { + return fetch(new Request(url, { mode: 'no-cors' })).then(function(response) { + return cache.put(url, response); + }); +} + +caches.open(name).then(function(c) { + cache = c; + return Promise.all([ + addOpaque(cache, url1), + addOpaque(cache, url2) + ]); +}).then(function() { + return cache.delete(url1); +}).then(function(result) { + ok(result, 'Cache entry should be deleted'); + return cache.delete(url2); +}).then(function(result) { + ok(result, 'Cache entry should be deleted'); + cache = null; + return caches.delete(name); +}).then(function(result) { + ok(result, 'Cache should be deleted'); + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_cache_keys.html b/dom/cache/test/mochitest/test_cache_keys.html new file mode 100644 index 0000000000..684f36f4ec --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_keys.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate the Cache.keys() method</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_keys.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_keys.js b/dom/cache/test/mochitest/test_cache_keys.js new file mode 100644 index 0000000000..6c1fa5ee4d --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_keys.js @@ -0,0 +1,74 @@ +var name = "keys" + context; +var c; + +var tests = [ + "//mochi.test:8888/?page" + context, + "//mochi.test:8888/?another" + context, +]; + +caches.open(name).then(function(cache) { + c = cache; + return c.addAll(tests); +}).then(function() { + // Add another cache entry using Cache.add + var another = "//mochi.test:8888/?yetanother" + context; + tests.push(another); + return c.add(another); +}).then(function() { + // Add another cache entry with URL fragment using Cache.add + var anotherWithFragment = "//mochi.test:8888/?fragment" + context + "#fragment"; + tests.push(anotherWithFragment); + return c.add(anotherWithFragment); +}).then(function() { + return c.keys(); +}).then(function(keys) { + is(keys.length, tests.length, "Same number of elements"); + // Verify both the insertion order of the requests and their validity. + keys.forEach(function(r, i) { + ok(r instanceof Request, "Valid request object"); + ok(r.url.indexOf(tests[i]) >= 0, "Valid URL"); + }); + // Try searching for just one request + return c.keys(tests[1]); +}).then(function(keys) { + is(keys.length, 1, "One match should be found"); + ok(keys[0].url.indexOf(tests[1]) >= 0, "Valid URL"); + // Try to see if ignoreSearch works as expected. + return c.keys(new Request("//mochi.test:8888/?foo"), {ignoreSearch: true}); +}).then(function(keys) { + is(keys.length, tests.length, "Same number of elements"); + keys.forEach(function(r, i) { + ok(r instanceof Request, "Valid request object"); + ok(r.url.indexOf(tests[i]) >= 0, "Valid URL"); + }); + // Try to see if ignoreMethod works as expected + return Promise.all( + ["POST", "PUT", "DELETE", "OPTIONS"] + .map(function(method) { + var req = new Request(tests[2], {method: method}); + return c.keys(req) + .then(function(keys) { + is(keys.length, 0, "No request should be matched without ignoreMethod"); + return c.keys(req, {ignoreMethod: true}); + }).then(function(keys) { + is(keys.length, 1, "One match should be found"); + ok(keys[0].url.indexOf(tests[2]) >= 0, "Valid URL"); + }); + }) + ); +}).then(function() { + // But HEAD should be allowed even without ignoreMethod + return c.keys(new Request(tests[0], {method: "HEAD"})); +}).then(function(keys) { + is(keys.length, 1, "One match should be found"); + ok(keys[0].url.indexOf(tests[0]) >= 0, "Valid URL"); + // Make sure cacheName is ignored. + return c.keys(tests[0], {cacheName: "non-existing-cache"}); +}).then(function(keys) { + is(keys.length, 1, "One match should be found"); + ok(keys[0].url.indexOf(tests[0]) >= 0, "Valid URL"); + return caches.delete(name); +}).then(function(deleted) { + ok(deleted, "The cache should be successfully deleted"); + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_cache_matchAll_request.html b/dom/cache/test/mochitest/test_cache_matchAll_request.html new file mode 100644 index 0000000000..e507b56cf8 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_matchAll_request.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate calling matchAll with a Request object</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_matchAll_request.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_matchAll_request.js b/dom/cache/test/mochitest/test_cache_matchAll_request.js new file mode 100644 index 0000000000..9e6715e804 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_matchAll_request.js @@ -0,0 +1,183 @@ +var request1 = new Request("//mochi.test:8888/?1&" + context + "#fragment"); +var request2 = new Request("//mochi.test:8888/?2&" + context); +var request3 = new Request("//mochi.test:8888/?3&" + context); +var requestWithAltQS = new Request("//mochi.test:8888/?queryString"); +var unknownRequest = new Request("//mochi.test:8888/non/existing/path?" + context); +var response1, response3; +var c; +var response1Text, response3Text; +var name = "matchAll-request" + context; + +function checkResponse(r, response, responseText) { + ok(r !== response, "The objects should not be the same"); + is(r.url, response.url.replace("#fragment", ""), + "The URLs should be the same"); + is(r.status, response.status, "The status codes should be the same"); + is(r.type, response.type, "The response types should be the same"); + is(r.ok, response.ok, "Both responses should have succeeded"); + is(r.statusText, response.statusText, + "Both responses should have the same status text"); + return r.text().then(function(text) { + // Avoid dumping out the large response text to the log if they're equal. + if (text !== responseText) { + is(text, responseText, "The response body should be correct"); + } + }); +} + +fetch(new Request(request1)).then(function(r) { + response1 = r; + return response1.text(); +}).then(function(text) { + response1Text = text; + return fetch(new Request(request3)); +}).then(function(r) { + response3 = r; + return response3.text(); +}).then(function(text) { + response3Text = text; + return testRequest(request1, request2, request3, unknownRequest, + requestWithAltQS, + request1.url.replace("#fragment", "#other")); +}).then(function() { + return testRequest(request1.url, request2.url, request3.url, + unknownRequest.url, requestWithAltQS.url, + request1.url.replace("#fragment", "#other")); +}).then(function() { + testDone(); +}); + +// The request arguments can either be a URL string, or a Request object. +function testRequest(request1, request2, request3, unknownRequest, + requestWithAlternateQueryString, + requestWithDifferentFragment) { + return caches.open(name).then(function(cache) { + c = cache; + return c.add(request1); + }).then(function() { + return c.add(request3); + }).then(function() { + return Promise.all( + ["HEAD", "POST", "PUT", "DELETE", "OPTIONS"] + .map(function(method) { + var r = new Request(request1, {method: method}); + return c.add(r) + .then(function() { + ok(false, "Promise should be rejected"); + }, function(err) { + is(err.name, "TypeError", "Adding a request with type '" + method + "' should fail"); + }); + }) + ); + }).then(function() { + return c.matchAll(request1); + }).then(function(r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }).then(function() { + return c.matchAll(new Request(request1, {method: "HEAD"})); + }).then(function(r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, ""); + }).then(function() { + return c.matchAll(new Request(request1, {method: "HEAD"}), {ignoreMethod: true}); + }).then(function(r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }).then(function() { + return Promise.all( + ["POST", "PUT", "DELETE", "OPTIONS"] + .map(function(method) { + var req = new Request(request1, {method: method}); + return c.matchAll(req) + .then(function(r) { + is(r.length, 0, "Searching for a request with a non-GET/HEAD method should not succeed"); + return c.matchAll(req, {ignoreMethod: true}); + }).then(function(r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }); + }) + ); + }).then(function() { + return c.matchAll(requestWithDifferentFragment); + }).then(function(r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }).then(function() { + return c.matchAll(requestWithAlternateQueryString, + {ignoreSearch: true}); + }).then(function(r) { + is(r.length, 2, "Should find 2 items"); + return Promise.all([ + checkResponse(r[0], response1, response1Text), + checkResponse(r[1], response3, response3Text) + ]); + }).then(function() { + return c.matchAll(request3); + }).then(function(r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response3, response3Text); + }).then(function() { + return c.matchAll(); + }).then(function(r) { + is(r.length, 2, "Should find 2 items"); + return Promise.all([ + checkResponse(r[0], response1, response1Text), + checkResponse(r[1], response3, response3Text) + ]); + }).then(function() { + return caches.match(request1, {cacheName: name + "mambojambo"}) + .then(function() { + is(typeof r, "undefined", 'Searching in the wrong cache should resolve to undefined'); + return caches.has(name + "mambojambo"); + }).then(function(hasCache) { + ok(!hasCache, 'The wrong cache should still not exist'); + }); + }).then(function() { + return c.matchAll(unknownRequest); + }).then(function(r) { + is(r.length, 0, "Searching for an unknown request should not succeed"); + return caches.match(unknownRequest, {cacheName: name}); + }).then(function(r) { + is(typeof r, "undefined", "Searching for an unknown request should not succeed"); + // Make sure that cacheName is ignored on Cache + return c.matchAll(request1, {cacheName: name + "mambojambo"}); + }).then(function(r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }).then(function() { + return caches.delete(name); + }).then(function(success) { + ok(success, "We should be able to delete the cache successfully"); + // Make sure that the cache is still usable after deletion. + return c.matchAll(request1); + }).then(function(r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response1, response1Text); + }).then(function() { + return c.matchAll(request3); + }).then(function(r) { + is(r.length, 1, "Should only find 1 item"); + return checkResponse(r[0], response3, response3Text); + }).then(function() { + return c.matchAll(); + }).then(function(r) { + is(r.length, 2, "Should find 2 items"); + return Promise.all([ + checkResponse(r[0], response1, response1Text), + checkResponse(r[1], response3, response3Text) + ]); + }).then(function() { + // Now, drop the cache, reopen and verify that we can't find the request any more. + c = null; + return caches.open(name); + }).then(function(cache) { + return cache.matchAll(); + }).then(function(r) { + is(r.length, 0, "Searching in the cache after deletion should not succeed"); + return caches.delete(name); + }).then(function(deleted) { + ok(deleted, "The cache should be deleted successfully"); + }); +} diff --git a/dom/cache/test/mochitest/test_cache_match_request.html b/dom/cache/test/mochitest/test_cache_match_request.html new file mode 100644 index 0000000000..9696c164de --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_match_request.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate calling match with a Request object</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_match_request.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_match_request.js b/dom/cache/test/mochitest/test_cache_match_request.js new file mode 100644 index 0000000000..455f814c31 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_match_request.js @@ -0,0 +1,145 @@ +var request = new Request("//mochi.test:8888/?" + context + "#fragment"); +var requestWithAltQS = new Request("//mochi.test:8888/?queryString"); +var unknownRequest = new Request("//mochi.test:8888/non/existing/path?" + context); +var response; +var c; +var responseText; +var name = "match-request" + context; + +function checkResponse(r, expectedBody) { + if (expectedBody === undefined) { + expectedBody = responseText; + } + ok(r !== response, "The objects should not be the same"); + is(r.url, response.url.replace("#fragment", ""), + "The URLs should be the same"); + is(r.status, response.status, "The status codes should be the same"); + is(r.type, response.type, "The response types should be the same"); + is(r.ok, response.ok, "Both responses should have succeeded"); + is(r.statusText, response.statusText, + "Both responses should have the same status text"); + return r.text().then(function(text) { + // Avoid dumping out the large response text to the log if they're equal. + if (text !== expectedBody) { + is(text, responseText, "The response body should be correct"); + } + }); +} +fetch(new Request(request)).then(function(r) { + response = r; + return response.text(); +}).then(function(text) { + responseText = text; + return testRequest(request, unknownRequest, requestWithAltQS, + request.url.replace("#fragment", "#other")); +}).then(function() { + return testRequest(request.url, unknownRequest.url, requestWithAltQS.url, + request.url.replace("#fragment", "#other")); +}).then(function() { + testDone(); +}); +// The request argument can either be a URL string, or a Request object. +function testRequest(request, unknownRequest, requestWithAlternateQueryString, + requestWithDifferentFragment) { + return caches.open(name).then(function(cache) { + c = cache; + return c.add(request); + }).then(function() { + return Promise.all( + ["HEAD", "POST", "PUT", "DELETE", "OPTIONS"] + .map(function(method) { + var r = new Request(request, {method: method}); + return c.add(r) + .then(function() { + ok(false, "Promise should be rejected"); + }, function(err) { + is(err.name, "TypeError", "Adding a request with type '" + method + "' should fail"); + }); + }) + ); + }).then(function() { + return c.match(request); + }).then(function(r) { + return checkResponse(r); + }).then(function() { + return c.match(new Request(request, {method: "HEAD"})); + }).then(function(r) { + return checkResponse(r, ''); + }).then(function() { + return c.match(new Request(request, {method: "HEAD"}), {ignoreMethod: true}); + }).then(function(r) { + return checkResponse(r); + }).then(function() { + return Promise.all( + ["POST", "PUT", "DELETE", "OPTIONS"] + .map(function(method) { + var req = new Request(request, {method: method}); + return c.match(req) + .then(function(r) { + is(typeof r, "undefined", "Searching for a request with a non-GET/HEAD method should not succeed"); + return c.match(req, {ignoreMethod: true}); + }).then(function(r) { + return checkResponse(r); + }); + }) + ); + }).then(function() { + return caches.match(request); + }).then(function(r) { + return checkResponse(r); + }).then(function() { + return caches.match(requestWithDifferentFragment); + }).then(function(r) { + return checkResponse(r); + }).then(function() { + return caches.match(requestWithAlternateQueryString, + {ignoreSearch: true, cacheName: name}); + }).then(function(r) { + return checkResponse(r); + }).then(function() { + return caches.match(request, {cacheName: name}); + }).then(function(r) { + return checkResponse(r); + }).then(function() { + return caches.match(request, {cacheName: name + "mambojambo"}) + .then(function(result) { + is(typeof r, "undefined", 'Searching in the wrong cache should resolve to undefined'); + return caches.has(name + "mambojambo"); + }).then(function(hasCache) { + ok(!hasCache, 'The wrong cache should still not exist'); + }); + }).then(function() { + // Make sure that cacheName is ignored on Cache + return c.match(request, {cacheName: name + "mambojambo"}); + }).then(function(r) { + return checkResponse(r); + }).then(function() { + return c.match(unknownRequest); + }).then(function(r) { + is(typeof r, "undefined", "Searching for an unknown request should not succeed"); + return caches.match(unknownRequest); + }).then(function(r) { + is(typeof r, "undefined", "Searching for an unknown request should not succeed"); + return caches.match(unknownRequest, {cacheName: name}); + }).then(function(r) { + is(typeof r, "undefined", "Searching for an unknown request should not succeed"); + return caches.delete(name); + }).then(function(success) { + ok(success, "We should be able to delete the cache successfully"); + // Make sure that the cache is still usable after deletion. + return c.match(request); + }).then(function(r) { + return checkResponse(r); + }).then(function() { + // Now, drop the cache, reopen and verify that we can't find the request any more. + c = null; + return caches.open(name); + }).then(function(cache) { + return cache.match(request); + }).then(function(r) { + is(typeof r, "undefined", "Searching in the cache after deletion should not succeed"); + return caches.delete(name); + }).then(function(deleted) { + ok(deleted, "The cache should be deleted successfully"); + }); +} diff --git a/dom/cache/test/mochitest/test_cache_match_vary.html b/dom/cache/test/mochitest/test_cache_match_vary.html new file mode 100644 index 0000000000..927184ac99 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_match_vary.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate calling match with requests involving the Vary header</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_match_vary.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_match_vary.js b/dom/cache/test/mochitest/test_cache_match_vary.js new file mode 100644 index 0000000000..9b4ad538a2 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_match_vary.js @@ -0,0 +1,334 @@ +var requestURL = "//mochi.test:8888/tests/dom/cache/test/mochitest/vary.sjs?" + context; +var name = "match-vary" + context; + +function checkResponse(r, response, responseText) { + ok(r !== response, "The objects should not be the same"); + is(r.url, response.url.replace("#fragment", ""), + "The URLs should be the same"); + is(r.status, response.status, "The status codes should be the same"); + is(r.type, response.type, "The response types should be the same"); + is(r.ok, response.ok, "Both responses should have succeeded"); + is(r.statusText, response.statusText, + "Both responses should have the same status text"); + is(r.headers.get("Vary"), response.headers.get("Vary"), + "Both responses should have the same Vary header"); + return r.text().then(function(text) { + is(text, responseText, "The response body should be correct"); + }); +} + +// Returns a Promise that will be resolved to an object with the following +// properties: +// * cache: A Cache object that contains one entry fetched with headers. +// * response: A Response object which is the result of fetching a request +// with the specified headers. +// * responseText: The body of the above response object. +function setupTest(headers) { + return setupTestMultipleEntries([headers]).then(function(test) { + return {response: test.response[0], + responseText: test.responseText[0], + cache: test.cache}; + }); +} +function setupTestMultipleEntries(headers) { + ok(Array.isArray(headers), "headers should be an array"); + return new Promise(function(resolve, reject) { + var response, responseText, cache; + Promise.all(headers.map(function(h) { + return fetch(requestURL, {headers: h}); + })).then(function(r) { + response = r; + return Promise.all(response.map(function(r) { + return r.text(); + })); + }).then(function(text) { + responseText = text; + return caches.open(name); + }).then(function(c) { + cache = c; + return Promise.all(headers.map(function(h) { + return c.add(new Request(requestURL, {headers: h})); + })); + }).then(function() { + resolve({response: response, responseText: responseText, cache: cache}); + }).catch(function(err) { + reject(err); + }); + }); +} + +function testBasics() { + var test; + return setupTest({"WhatToVary": "Custom"}) + .then(function(t) { + test = t; + // Ensure that searching without specifying a Custom header succeeds. + return test.cache.match(requestURL); + }).then(function(r) { + return checkResponse(r, test.response, test.responseText); + }).then(function() { + // Ensure that searching with a non-matching value for the Custom header fails. + return test.cache.match(new Request(requestURL, {headers: {"Custom": "foo=bar"}})); + }).then(function(r) { + is(typeof r, "undefined", "Searching for a request with an unknown Vary header should not succeed"); + // Ensure that searching with a non-matching value for the Custom header but with ignoreVary set succeeds. + return test.cache.match(new Request(requestURL, {headers: {"Custom": "foo=bar"}}), + {ignoreVary: true}); + }).then(function(r) { + return checkResponse(r, test.response, test.responseText); + }); +} + +function testBasicKeys() { + function checkRequest(reqs) { + is(reqs.length, 1, "One request expected"); + ok(reqs[0].url.indexOf(requestURL) >= 0, "The correct request expected"); + ok(reqs[0].headers.get("WhatToVary"), "Custom", "The correct request headers expected"); + } + var test; + return setupTest({"WhatToVary": "Custom"}) + .then(function(t) { + test = t; + // Ensure that searching without specifying a Custom header succeeds. + return test.cache.keys(requestURL); + }).then(function(r) { + return checkRequest(r); + }).then(function() { + // Ensure that searching with a non-matching value for the Custom header fails. + return test.cache.keys(new Request(requestURL, {headers: {"Custom": "foo=bar"}})); + }).then(function(r) { + is(r.length, 0, "Searching for a request with an unknown Vary header should not succeed"); + // Ensure that searching with a non-matching value for the Custom header but with ignoreVary set succeeds. + return test.cache.keys(new Request(requestURL, {headers: {"Custom": "foo=bar"}}), + {ignoreVary: true}); + }).then(function(r) { + return checkRequest(r); + }); +} + +function testStar() { + function ensurePromiseRejected(promise) { + return promise + .then(function() { + ok(false, "Promise should be rejected"); + }, function(err) { + is(err.name, "TypeError", "Attempting to store a Response with a Vary:* header must fail"); + }); + } + var test; + return new Promise(function(resolve, reject) { + var cache; + caches.open(name).then(function(c) { + cache = c; + Promise.all([ + ensurePromiseRejected( + cache.add(new Request(requestURL + "1", {headers: {"WhatToVary": "*"}}))), + ensurePromiseRejected( + cache.addAll([ + new Request(requestURL + "2", {headers: {"WhatToVary": "*"}}), + requestURL + "3", + ])), + ensurePromiseRejected( + fetch(new Request(requestURL + "4", {headers: {"WhatToVary": "*"}})) + .then(function(response) { + return cache.put(requestURL + "4", response); + })), + ensurePromiseRejected( + cache.add(new Request(requestURL + "5", {headers: {"WhatToVary": "*,User-Agent"}}))), + ensurePromiseRejected( + cache.addAll([ + new Request(requestURL + "6", {headers: {"WhatToVary": "*,User-Agent"}}), + requestURL + "7", + ])), + ensurePromiseRejected( + fetch(new Request(requestURL + "8", {headers: {"WhatToVary": "*,User-Agent"}})) + .then(function(response) { + return cache.put(requestURL + "8", response); + })), + ensurePromiseRejected( + cache.add(new Request(requestURL + "9", {headers: {"WhatToVary": "User-Agent,*"}}))), + ensurePromiseRejected( + cache.addAll([ + new Request(requestURL + "10", {headers: {"WhatToVary": "User-Agent,*"}}), + requestURL + "10", + ])), + ensurePromiseRejected( + fetch(new Request(requestURL + "11", {headers: {"WhatToVary": "User-Agent,*"}})) + .then(function(response) { + return cache.put(requestURL + "11", response); + })), + ]).then(reject, resolve); + }); + }); +} + +function testMatch() { + var test; + return setupTest({"WhatToVary": "Custom", "Custom": "foo=bar"}) + .then(function(t) { + test = t; + // Ensure that searching with a different Custom header fails. + return test.cache.match(new Request(requestURL, {headers: {"Custom": "bar=baz"}})); + }).then(function(r) { + is(typeof r, "undefined", "Searching for a request with a non-matching Custom header should not succeed"); + // Ensure that searching with the same Custom header succeeds. + return test.cache.match(new Request(requestURL, {headers: {"Custom": "foo=bar"}})); + }).then(function(r) { + return checkResponse(r, test.response, test.responseText); + }); +} + +function testInvalidHeaderName() { + var test; + return setupTest({"WhatToVary": "Foo/Bar, Custom-User-Agent"}) + .then(function(t) { + test = t; + // Ensure that searching with a different User-Agent header fails. + return test.cache.match(new Request(requestURL, {headers: {"Custom-User-Agent": "MyUA"}})); + }).then(function(r) { + is(typeof r, "undefined", "Searching for a request with a non-matching Custom-User-Agent header should not succeed"); + // Ensure that searching with a different Custom-User-Agent header but with ignoreVary succeeds. + return test.cache.match(new Request(requestURL, {headers: {"Custom-User-Agent": "MyUA"}}), + {ignoreVary: true}); + }).then(function(r) { + return checkResponse(r, test.response, test.responseText); + }).then(function() { + // Ensure that we do not mistakenly recognize the tokens in the invalid header name. + return test.cache.match(new Request(requestURL, {headers: {"Foo": "foobar"}})); + }).then(function(r) { + return checkResponse(r, test.response, test.responseText); + }); +} + +function testMultipleHeaders() { + var test; + return setupTest({"WhatToVary": "Custom-Referer,\tCustom-Accept-Encoding"}) + .then(function(t) { + test = t; + // Ensure that searching with a different Referer header fails. + return test.cache.match(new Request(requestURL, {headers: {"Custom-Referer": "https://somesite.com/"}})); + }).then(function(r) { + is(typeof r, "undefined", "Searching for a request with a non-matching Custom-Referer header should not succeed"); + // Ensure that searching with a different Custom-Referer header but with ignoreVary succeeds. + return test.cache.match(new Request(requestURL, {headers: {"Custom-Referer": "https://somesite.com/"}}), + {ignoreVary: true}); + }).then(function(r) { + return checkResponse(r, test.response, test.responseText); + }).then(function() { + // Ensure that searching with a different Custom-Accept-Encoding header fails. + return test.cache.match(new Request(requestURL, {headers: {"Custom-Accept-Encoding": "myencoding"}})); + }).then(function(r) { + is(typeof r, "undefined", "Searching for a request with a non-matching Custom-Accept-Encoding header should not succeed"); + // Ensure that searching with a different Custom-Accept-Encoding header but with ignoreVary succeeds. + return test.cache.match(new Request(requestURL, {headers: {"Custom-Accept-Encoding": "myencoding"}}), + {ignoreVary: true}); + }).then(function(r) { + return checkResponse(r, test.response, test.responseText); + }).then(function() { + // Ensure that searching with an empty Custom-Referer header succeeds. + return test.cache.match(new Request(requestURL, {headers: {"Custom-Referer": ""}})); + }).then(function(r) { + return checkResponse(r, test.response, test.responseText); + }).then(function() { + // Ensure that searching with an empty Custom-Accept-Encoding header succeeds. + return test.cache.match(new Request(requestURL, {headers: {"Custom-Accept-Encoding": ""}})); + }).then(function(r) { + return checkResponse(r, test.response, test.responseText); + }).then(function() { + // Ensure that searching with an empty Custom-Referer header but with a different Custom-Accept-Encoding header fails. + return test.cache.match(new Request(requestURL, {headers: {"Custom-Referer": "", + "Custom-Accept-Encoding": "myencoding"}})); + }).then(function(r) { + is(typeof r, "undefined", "Searching for a request with a non-matching Custom-Accept-Encoding header should not succeed"); + // Ensure that searching with an empty Custom-Referer header but with a different Custom-Accept-Encoding header and ignoreVary succeeds. + return test.cache.match(new Request(requestURL, {headers: {"Custom-Referer": "", + "Custom-Accept-Encoding": "myencoding"}}), + {ignoreVary: true}); + }).then(function(r) { + return checkResponse(r, test.response, test.responseText); + }); +} + +function testMultipleCacheEntries() { + var test; + return setupTestMultipleEntries([ + {"WhatToVary": "Accept-Language", "Accept-Language": "en-US"}, + {"WhatToVary": "Accept-Language", "Accept-Language": "en-US, fa-IR"}, + ]).then(function(t) { + test = t; + return test.cache.matchAll(); + }).then(function (r) { + is(r.length, 2, "Two cache entries should be stored in the DB"); + // Ensure that searching without specifying an Accept-Language header fails. + return test.cache.matchAll(requestURL); + }).then(function(r) { + is(r.length, 0, "Searching for a request without specifying an Accept-Language header should not succeed"); + // Ensure that searching without specifying an Accept-Language header but with ignoreVary succeeds. + return test.cache.matchAll(requestURL, {ignoreVary: true}); + }).then(function(r) { + return Promise.all([ + checkResponse(r[0], test.response[0], test.responseText[0]), + checkResponse(r[1], test.response[1], test.responseText[1]), + ]); + }).then(function() { + // Ensure that searching with Accept-Language: en-US succeeds. + return test.cache.matchAll(new Request(requestURL, {headers: {"Accept-Language": "en-US"}})); + }).then(function(r) { + is(r.length, 1, "One cache entry should be found"); + return checkResponse(r[0], test.response[0], test.responseText[0]); + }).then(function() { + // Ensure that searching with Accept-Language: en-US,fa-IR succeeds. + return test.cache.matchAll(new Request(requestURL, {headers: {"Accept-Language": "en-US, fa-IR"}})); + }).then(function(r) { + is(r.length, 1, "One cache entry should be found"); + return checkResponse(r[0], test.response[1], test.responseText[1]); + }).then(function() { + // Ensure that searching with a valid Accept-Language header but with ignoreVary returns both entries. + return test.cache.matchAll(new Request(requestURL, {headers: {"Accept-Language": "en-US"}}), + {ignoreVary: true}); + }).then(function(r) { + return Promise.all([ + checkResponse(r[0], test.response[0], test.responseText[0]), + checkResponse(r[1], test.response[1], test.responseText[1]), + ]); + }).then(function() { + // Ensure that searching with Accept-Language: fa-IR fails. + return test.cache.matchAll(new Request(requestURL, {headers: {"Accept-Language": "fa-IR"}})); + }).then(function(r) { + is(r.length, 0, "Searching for a request with a different Accept-Language header should not succeed"); + // Ensure that searching with Accept-Language: fa-IR but with ignoreVary should succeed. + return test.cache.matchAll(new Request(requestURL, {headers: {"Accept-Language": "fa-IR"}}), + {ignoreVary: true}); + }).then(function(r) { + is(r.length, 2, "Two cache entries should be found"); + return Promise.all([ + checkResponse(r[0], test.response[0], test.responseText[0]), + checkResponse(r[1], test.response[1], test.responseText[1]), + ]); + }); +} + +// Make sure to clean up after each test step. +function step(testPromise) { + return testPromise.then(function() { + caches.delete(name); + }, function() { + caches.delete(name); + }); +} + +step(testBasics()).then(function() { + return step(testBasicKeys()); +}).then(function() { + return step(testStar()); +}).then(function() { + return step(testMatch()); +}).then(function() { + return step(testInvalidHeaderName()); +}).then(function() { + return step(testMultipleHeaders()); +}).then(function() { + return step(testMultipleCacheEntries()); +}).then(function() { + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_cache_orphaned_body.html b/dom/cache/test/mochitest/test_cache_orphaned_body.html new file mode 100644 index 0000000000..049a97a794 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_orphaned_body.html @@ -0,0 +1,235 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache with QuotaManager Restart</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="large_url_list.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function clearStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function storageUsage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var cb = SpecialPowers.wrapCallback(function(request) { + var result = request.result; + resolve(result.usage, result.fileUsage); + }); + qms.getUsageForPrincipal(principal, cb); + }); +} + +function groupUsage() { + return new Promise(function(resolve, reject) { + navigator.storage.estimate().then(storageEstimation => { + resolve(storageEstimation.usage, 0); + }); + }); +} + +function workerGroupUsage() { + return new Promise(function(resolve, reject) { + function workerScript() { + navigator.storage.estimate().then(storageEstimation => { + postMessage(storageEstimation.usage); + }); + } + + let url = + URL.createObjectURL(new Blob(["(", workerScript.toSource(), ")()"])); + + let worker = new Worker(url); + worker.onmessage = function (e) { + resolve(e.data, 0); + }; + }); +} + +function resetStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var request = qms.reset(); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function gc() { + return new Promise(function(resolve, reject) { + SpecialPowers.exactGC(resolve); + }); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true], + ["dom.storageManager.enabled", true]], +}, function() { + var name = 'orphanedBodyOwner'; + var cache = null; + var response = null; + var initialUsage = 0; + var fullUsage = 0; + var resetUsage = 0; + var endUsage = 0; + var url = 'test_cache_add.js'; + + // start from a fresh origin directory so other tests do not influence our + // results + setupTestIframe().then(function() { + return clearStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + is(0, usage, 'disk usage should be zero to start'); + }) + + // Initialize and populate an initial cache to get the base sqlite pages + // and directory structure allocated. + .then(function() { + return caches.open(name); + }).then(function(c) { + return c.add(url); + }).then(function() { + return gc(); + }).then(function() { + return caches.delete(name); + }).then(function(deleted) { + ok(deleted, 'cache should be deleted'); + + // This is a bit superfluous, but its necessary to make sure the Cache is + // fully deleted before we proceed. The deletion actually takes place in + // two async steps. We don't want to resetStorage() until the second step + // has taken place. This extra Cache operation ensure that all the + // runnables have been flushed through the threads, etc. + return caches.has(name); + }) + + // Now measure initial disk usage + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + initialUsage = usage; + }) + + // Now re-populate the Cache object + .then(function() { + return caches.open(name); + }).then(function(c) { + cache = c; + return cache.add(url); + }) + + // Get a reference to the body we've stored in the Cache. + .then(function() { + return cache.match(url); + }).then(function(r) { + response = r; + return cache.delete(url); + }).then(function(result) { + ok(result, "Cache entry should be deleted"); + }) + + // Reset the quota dir while the cache entry is deleted, but still referenced + // from the DOM. This forces the body to be orphaned. + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + fullUsage = usage; + ok(fullUsage > initialUsage, 'disk usage should have grown'); + }) + + // Test groupUsage() + .then(function() { + return resetStorage(); + }).then(function() { + return groupUsage(); + }).then(function(usage) { + fullUsage = usage; + ok(fullUsage > initialUsage, 'disk group usage should have grown'); + }) + + // Test workerGroupUsage() + .then(function() { + return resetStorage(); + }).then(function() { + return workerGroupUsage(); + }).then(function(usage) { + fullUsage = usage; + ok(fullUsage > initialUsage, 'disk group usage on worker should have grown'); + }) + + // Now perform a new Cache operation that will reopen the origin. This + // should clean up the orphaned body. + .then(function() { + return caches.match(url); + }).then(function(r) { + ok(!r, 'response should not exist in storage'); + }) + + // Finally, verify orphaned data was cleaned up by re-checking the disk + // usage. Reset the storage first to ensure any WAL transaction files + // are flushed before measuring the usage. + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + endUsage = usage; + dump("### ### initial:" + initialUsage + ", full:" + fullUsage + + ", end:" + endUsage + "\n"); + ok(endUsage < fullUsage, 'disk usage should have shrank'); + is(endUsage, initialUsage, 'disk usage should return to original'); + }) + + // Verify that the stale, orphaned response cannot be put back into + // the cache. + .then(function() { + ok(!response.bodyUsed, 'response body should not be considered used'); + return cache.put(url, response).then(function() { + ok(false, 'Should not be able to store stale orphaned body.'); + }).catch(function(e) { + is(e.name, 'TypeError', 'storing a stale orphaned body should throw TypeError'); + }); + }).then(function() { + ok(response.bodyUsed, 'attempting to store response should mark body used'); + }) + + .then(function() { + SimpleTest.finish(); + }); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_orphaned_cache.html b/dom/cache/test/mochitest/test_cache_orphaned_cache.html new file mode 100644 index 0000000000..c6086e4877 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_orphaned_cache.html @@ -0,0 +1,165 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache with QuotaManager Restart</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="large_url_list.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function clearStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function storageUsage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var cb = SpecialPowers.wrapCallback(function(request) { + var result = request.result; + resolve(result.usage, result.fileUsage); + }); + qms.getUsageForPrincipal(principal, cb); + }); +} + +function resetStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var request = qms.reset(); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function gc() { + return new Promise(function(resolve, reject) { + SpecialPowers.exactGC(resolve); + }); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true]], +}, function() { + var name = 'toBeOrphaned'; + var cache = null; + var initialUsage = 0; + var fullUsage = 0; + var resetUsage = 0; + var endUsage = 0; + var url = 'test_cache_add.js'; + + // start from a fresh origin directory so other tests do not influence our + // results + setupTestIframe().then(function() { + return clearStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + is(0, usage, 'disk usage should be zero to start'); + }) + + // Initialize and populate an initial cache to get the base sqlite pages + // and directory structure allocated. + .then(function() { + return caches.open(name); + }).then(function(c) { + return c.add(url); + }).then(function() { + return gc(); + }).then(function() { + return caches.delete(name); + }).then(function(deleted) { + ok(deleted, 'cache should be deleted'); + + // This is a bit superfluous, but its necessary to make sure the Cache is + // fully deleted before we proceed. The deletion actually takes place in + // two async steps. We don't want to resetStorage() until the second step + // has taken place. This extra Cache operation ensure that all the + // runnables have been flushed through the threads, etc. + return caches.has(name); + }) + + // Now measure initial disk usage + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + initialUsage = usage; + }) + + // Now re-populate the Cache object + .then(function() { + return caches.open(name); + }).then(function(c) { + cache = c; + return cache.add(url); + }).then(function() { + return caches.delete(name); + }).then(function(deleted) { + ok(deleted, 'cache should be deleted'); + }) + + // Reset the quota dir while the cache is deleted, but still referenced + // from the DOM. This forces it to be orphaned. + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + fullUsage = usage; + ok(fullUsage > initialUsage, 'disk usage should have grown'); + }) + + // Now perform a new Cache operation that will reopen the origin. This + // should clean up the orphaned Cache data. + .then(function() { + return caches.has(name); + }).then(function(result) { + ok(!result, 'cache should not exist in storage'); + }) + + // Finally, verify orphaned data was cleaned up by re-checking the disk + // usage. Reset the storage first to ensure any WAL transaction files + // are flushed before measuring the usage. + .then(function() { + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + endUsage = usage; + dump("### ### initial:" + initialUsage + ", full:" + fullUsage + + ", end:" + endUsage + "\n"); + ok(endUsage < fullUsage, 'disk usage should have shrank'); + is(endUsage, initialUsage, 'disk usage should return to original'); + SimpleTest.finish(); + }); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_overwrite.html b/dom/cache/test/mochitest/test_cache_overwrite.html new file mode 100644 index 0000000000..2e1976a4cb --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_overwrite.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test what happens when you overwrite a cache entry</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_overwrite.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_overwrite.js b/dom/cache/test/mochitest/test_cache_overwrite.js new file mode 100644 index 0000000000..5b1c0142bf --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_overwrite.js @@ -0,0 +1,47 @@ +var requestURL = "//mochi.test:8888/tests/dom/cache/test/mochitest/mirror.sjs?" + context; +var response; +var c; +var responseText; +var name = "match-mirror" + context; + +function checkResponse(r) { + ok(r !== response, "The objects should not be the same"); + is(r.url, response.url.replace("#fragment", ""), + "The URLs should be the same"); + is(r.status, response.status, "The status codes should be the same"); + is(r.type, response.type, "The response types should be the same"); + is(r.ok, response.ok, "Both responses should have succeeded"); + is(r.statusText, response.statusText, + "Both responses should have the same status text"); + is(r.headers.get("Mirrored"), response.headers.get("Mirrored"), + "Both responses should have the same Mirrored header"); + return r.text().then(function(text) { + is(text, responseText, "The response body should be correct"); + }); +} + +fetch(new Request(requestURL, {headers: {"Mirror": "bar"}})).then(function(r) { + is(r.headers.get("Mirrored"), "bar", "The server should give back the correct header"); + response = r; + return response.text(); +}).then(function(text) { + responseText = text; + return caches.open(name); +}).then(function(cache) { + c = cache; + return c.add(new Request(requestURL, {headers: {"Mirror": "foo"}})); +}).then(function() { + // Overwrite the request, to replace the entry stored in response_headers + // with a different value. + return c.add(new Request(requestURL, {headers: {"Mirror": "bar"}})); +}).then(function() { + return c.matchAll(); +}).then(function(r) { + is(r.length, 1, "Only one request should be in the cache"); + return checkResponse(r[0]); +}).then(function() { + return caches.delete(name); +}).then(function(deleted) { + ok(deleted, "The cache should be deleted successfully"); + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_cache_put.html b/dom/cache/test/mochitest/test_cache_put.html new file mode 100644 index 0000000000..1f3ccec8a5 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_put.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Interfaces Exposed to Workers</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_put.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_put.js b/dom/cache/test/mochitest/test_cache_put.js new file mode 100644 index 0000000000..29e1b4bd5c --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_put.js @@ -0,0 +1,50 @@ +var url = 'test_cache.js'; +var cache; +var fetchResponse; +Promise.all([fetch(url), + caches.open('putter' + context)]).then(function(results) { + fetchResponse = results[0]; + cache = results[1]; + return cache.put(url, fetchResponse.clone()); +}).then(function(result) { + is(undefined, result, 'Successful put() should resolve undefined'); + return cache.match(url); +}).then(function(response) { + ok(response, 'match() should find resppnse that was previously put()'); + ok(response.url.endsWith(url), 'matched response should match original url'); + return Promise.all([fetchResponse.text(), + response.text()]); +}).then(function(results) { + // suppress large assert spam unless it's relevent + if (results[0] !== results[1]) { + is(results[0], results[1], 'stored response body should match original'); + } + + // Now, try to overwrite the request with a different response object. + return cache.put(url, new Response("overwritten")); +}).then(function() { + return cache.matchAll(url); +}).then(function(result) { + is(result.length, 1, "Only one entry should exist"); + return result[0].text(); +}).then(function(body) { + is(body, "overwritten", "The cache entry should be successfully overwritten"); + + // Now, try to write a URL with a fragment + return cache.put(url + "#fragment", new Response("more overwritten")); +}).then(function() { + return cache.matchAll(url + "#differentFragment"); +}).then(function(result) { + is(result.length, 1, "Only one entry should exist"); + return result[0].text(); +}).then(function(body) { + is(body, "more overwritten", "The cache entry should be successfully overwritten"); + + // TODO: Verify that trying to store a response with an error raises a TypeError + // when bug 1147178 is fixed. + + return caches.delete('putter' + context); +}).then(function(deleted) { + ok(deleted, "The cache should be deleted successfully"); + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_cache_put_reorder.html b/dom/cache/test/mochitest/test_cache_put_reorder.html new file mode 100644 index 0000000000..33934176cd --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_put_reorder.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Ensure that using Cache.put to overwrite an entry will change its insertion order</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_put_reorder.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_put_reorder.js b/dom/cache/test/mochitest/test_cache_put_reorder.js new file mode 100644 index 0000000000..df8beecf93 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_put_reorder.js @@ -0,0 +1,31 @@ +var name = "putreorder" + context; +var c; + +var reqs = [ + "//mochi.test:8888/?foo" + context, + "//mochi.test:8888/?bar" + context, + "//mochi.test:8888/?baz" + context, +]; + +caches.open(name).then(function(cache) { + c = cache; + return c.addAll(reqs); +}).then(function() { + return c.put(reqs[1], new Response("overwritten")); +}).then(function() { + return c.keys(); +}).then(function(keys) { + is(keys.length, 3, "Correct number of entries expected"); + ok(keys[0].url.indexOf(reqs[0]) >= 0, "The first entry should be untouched"); + ok(keys[2].url.indexOf(reqs[1]) >= 0, "The second entry should be moved to the end"); + ok(keys[1].url.indexOf(reqs[2]) >= 0, "The third entry should now be the second one"); + return c.match(reqs[1]); +}).then(function(r) { + return r.text(); +}).then(function(body) { + is(body, "overwritten", "The body should be overwritten"); + return caches.delete(name); +}).then(function(deleted) { + ok(deleted, "The cache should be deleted successfully"); + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_cache_redirect.html b/dom/cache/test/mochitest/test_cache_redirect.html new file mode 100644 index 0000000000..e2372d5ddb --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_redirect.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Cache storage of redirect responses</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_redirect.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_redirect.js b/dom/cache/test/mochitest/test_cache_redirect.js new file mode 100644 index 0000000000..7547d6528b --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_redirect.js @@ -0,0 +1,14 @@ +let cache; +let url = 'foo.html'; +let redirectURL = 'http://example.com/foo-bar.html'; +caches.open('redirect-' + context).then(c => { + cache = c; + var response = Response.redirect(redirectURL); + is(response.headers.get('Location'), redirectURL); + return cache.put(url, response); +}).then(_ => { + return cache.match(url); +}).then(response => { + is(response.headers.get('Location'), redirectURL); + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_cache_requestCache.html b/dom/cache/test/mochitest/test_cache_requestCache.html new file mode 100644 index 0000000000..910e96f54d --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_requestCache.html @@ -0,0 +1,20 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate the Cache.keys() method</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + runTests("test_cache_requestCache.js") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_requestCache.js b/dom/cache/test/mochitest/test_cache_requestCache.js new file mode 100644 index 0000000000..5e11b0f3d1 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_requestCache.js @@ -0,0 +1,27 @@ +var name = "requestCache" + context; +var c; + +var reqWithoutCache = new Request("//mochi.test:8888/?noCache" + context); +var reqWithCache = new Request("//mochi.test:8888/?withCache" + context, + {cache: "force-cache"}); + +// Sanity check +is(reqWithoutCache.cache, "default", "Correct default value"); +is(reqWithCache.cache, "force-cache", "Correct value set by the ctor"); + +caches.open(name).then(function(cache) { + c = cache; + return c.addAll([reqWithoutCache, reqWithCache]); +}).then(function() { + return c.keys(); +}).then(function(keys) { + is(keys.length, 2, "Correct number of requests"); + is(keys[0].url, reqWithoutCache.url, "Correct URL"); + is(keys[0].cache, reqWithoutCache.cache, "Correct cache attribute"); + is(keys[1].url, reqWithCache.url, "Correct URL"); + is(keys[1].cache, reqWithCache.cache, "Correct cache attribute"); + return caches.delete(name); +}).then(function(deleted) { + ok(deleted, "The cache should be successfully deleted"); + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_cache_restart.html b/dom/cache/test/mochitest/test_cache_restart.html new file mode 100644 index 0000000000..68649bf618 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_restart.html @@ -0,0 +1,70 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache with QuotaManager Restart</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function resetStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var request = qms.reset(); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true]], +}, function() { + var name = 'foo'; + var url = './test_cache_add.js'; + var cache; + setupTestIframe().then(function() { + return caches.open(name); + }).then(function(c) { + cache = c; + return cache.add(url); + }).then(function() { + return resetStorage(); + }).then(function() { + return cache.match(url).then(function(resp) { + ok(false, 'old cache reference should not work after reset'); + }).catch(function(err) { + ok(true, 'old cache reference should not work after reset'); + }); + }).then(function() { + return caches.open(name); + }).then(function(c) { + cache = c; + return cache.match(url); + }).then(function(resp) { + ok(!!resp, 'cache should work after QM reset'); + return caches.delete(name); + }).then(function(success) { + ok(success, 'cache should be deleted'); + SimpleTest.finish(); + }); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_shrink.html b/dom/cache/test/mochitest/test_cache_shrink.html new file mode 100644 index 0000000000..b7136cb750 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_shrink.html @@ -0,0 +1,132 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache with QuotaManager Restart</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="large_url_list.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +function clearStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var request = qms.clearStoragesForPrincipal(principal); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function storageUsage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var principal = SpecialPowers.wrap(document).nodePrincipal; + var cb = SpecialPowers.wrapCallback(function(request) { + var result = request.result; + resolve(result.usage, result.fileUsage); + }); + qms.getUsageForPrincipal(principal, cb); + }); +} + +function resetStorage() { + return new Promise(function(resolve, reject) { + var qms = SpecialPowers.Services.qms; + var request = qms.reset(); + var cb = SpecialPowers.wrapCallback(resolve); + request.callback = cb; + }); +} + +function gc() { + return new Promise(function(resolve, reject) { + SpecialPowers.exactGC(resolve); + }); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true], + ["dom.quotaManager.testing", true]], +}, function() { + var name = 'foo'; + var cache = null; + var initialUsage = 0; + var fullUsage = 0; + var endUsage = 0; + // start from a fresh origin directory so other tests do not influence our + // results + setupTestIframe().then(function() { + return clearStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + is(0, usage, 'disk usage should be zero to start'); + return caches.open(name); + }).then(function(c) { + cache = c; + return storageUsage(); + }).then(function(usage) { + initialUsage = usage; + return Promise.all(largeUrlList.map(function(url) { + return cache.put(new Request(url), new Response()); + })); + }).then(function() { + return cache.keys(); + }).then(function(keyList) { + is(keyList.length, largeUrlList.length, 'Large URL list is stored in cache'); + cache = null; + // Ensure the Cache DOM object is gone before proceeding. If its alive + // it will keep the related entries on-disk as well. + return gc(); + }).then(function() { + // reset the quota manager storage to ensure the DB connection is flushed + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + fullUsage = usage; + ok(fullUsage > initialUsage, 'disk usage should have grown'); + return caches.delete(name); + }).then(function(result) { + ok(result, 'cache should be deleted'); + // This is a bit superfluous, but its necessary to make sure the Cache is + // fully deleted before we proceed. The deletion actually takes place in + // two async steps. We don't want to resetStorage() until the second step + // has taken place. This extra Cache operation ensure that all the + // runnables have been flushed through the threads, etc. + return caches.has(name); + }).then(function(result) { + ok(!result, 'cache should not exist in storage'); + // reset the quota manager storage to ensure the DB connection is flushed + return resetStorage(); + }).then(function() { + return storageUsage(); + }).then(function(usage) { + endUsage = usage; + dump("### ### initial:" + initialUsage + ", full:" + fullUsage + + ", end:" + endUsage + "\n"); + ok(endUsage < (fullUsage / 2), 'disk usage should have shrank significantly'); + ok(endUsage > initialUsage, 'disk usage should not shrink back to orig size'); + SimpleTest.finish(); + }); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_cache_untrusted.html b/dom/cache/test/mochitest/test_cache_untrusted.html new file mode 100644 index 0000000000..1dff8c0380 --- /dev/null +++ b/dom/cache/test/mochitest/test_cache_untrusted.html @@ -0,0 +1,40 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test Cache with QuotaManager Restart</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <script type="text/javascript" src="large_url_list.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> +function setupTestIframe() { + return new Promise(function(resolve) { + var iframe = document.createElement("iframe"); + iframe.src = "empty.html"; + iframe.onload = function() { + window.caches = iframe.contentWindow.caches; + resolve(); + }; + document.body.appendChild(iframe); + }); +} + +SimpleTest.waitForExplicitFinish(); +SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true]], +}, function() { + setupTestIframe().then(function() { + return caches.open('foo'); + }).then(function(usage) { + ok(false, 'caches should not be usable in untrusted http origin'); + }).catch(function(err) { + is(err.name, 'SecurityError', 'caches should reject with SecurityError'); + SimpleTest.finish(); + }); +}); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_caches.html b/dom/cache/test/mochitest/test_caches.html new file mode 100644 index 0000000000..e06deabf1e --- /dev/null +++ b/dom/cache/test/mochitest/test_caches.html @@ -0,0 +1,22 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Test the CacheStorage API</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> + <script type="text/javascript" src="driver.js"></script> +</head> +<body> +<iframe id="frame"></iframe> +<script class="testbody" type="text/javascript"> + // These tests can only be run in sequential mode because we need to be able + // to rely on the global state of the CacheStorage at all times. + runTests("test_caches.js", "sequential") + .then(function() { + SimpleTest.finish(); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/test_caches.js b/dom/cache/test/mochitest/test_caches.js new file mode 100644 index 0000000000..e57ceb1127 --- /dev/null +++ b/dom/cache/test/mochitest/test_caches.js @@ -0,0 +1,122 @@ +function arraysHaveSameContent(arr1, arr2) { + if (arr1.length != arr2.length) { + return false; + } + return arr1.every(function(value, index) { + return arr2[index] === value; + }); +} + +function testHas() { + var name = "caches-has" + context; + return caches.has(name).then(function(has) { + ok(!has, name + " should not exist yet"); + return caches.open(name); + }).then(function(c) { + return caches.has(name); + }).then(function(has) { + ok(has, name + " should now exist"); + return caches.delete(name); + }).then(function(deleted) { + ok(deleted, "The deletion should finish successfully"); + return caches.has(name); + }).then(function(has) { + ok(!has, name + " should not exist any more"); + }); +} + +function testKeys() { + var names = [ + // The names here are intentionally unsorted, to ensure the insertion order + // and make sure we don't confuse it with an alphabetically sorted list. + "caches-keys4" + context, + "caches-keys0" + context, + "caches-keys1" + context, + "caches-keys3" + context, + ]; + return caches.keys().then(function(keys) { + is(keys.length, 0, "No keys should exist yet"); + return Promise.all(names.map(function(name) { + return caches.open(name); + })); + }).then(function() { + return caches.keys(); + }).then(function(keys) { + ok(arraysHaveSameContent(keys, names), "Keys must match in insertion order"); + return Promise.all(names.map(function(name) { + return caches.delete(name); + })); + }).then(function(deleted) { + ok(arraysHaveSameContent(deleted, [true, true, true, true]), "All deletions must succeed"); + return caches.keys(); + }).then(function(keys) { + is(keys.length, 0, "No keys should exist any more"); + }); +} + +function testMatchAcrossCaches() { + var tests = [ + // The names here are intentionally unsorted, to ensure the insertion order + // and make sure we don't confuse it with an alphabetically sorted list. + { + name: "caches-xmatch5" + context, + request: "//mochi.test:8888/?5" + context, + }, + { + name: "caches-xmatch2" + context, + request: "//mochi.test:8888/tests/dom/cache/test/mochitest/test_caches.js?2" + context, + }, + { + name: "caches-xmatch4" + context, + request: "//mochi.test:8888/?4" + context, + }, + ]; + return Promise.all(tests.map(function(test) { + return caches.open(test.name).then(function(c) { + return c.add(test.request); + }); + })).then(function() { + return caches.match("//mochi.test:8888/?5" + context, {ignoreSearch: true}); + }).then(function(match) { + ok(match.url.indexOf("?5") > 0, "Match should come from the first cache"); + return caches.delete("caches-xmatch2" + context); // This should not change anything! + }).then(function(deleted) { + ok(deleted, "Deletion should finish successfully"); + return caches.match("//mochi.test:8888/?" + context, {ignoreSearch: true}); + }).then(function(match) { + ok(match.url.indexOf("?5") > 0, "Match should still come from the first cache"); + return caches.delete("caches-xmatch5" + context); // This should eliminate the first match! + }).then(function(deleted) { + ok(deleted, "Deletion should finish successfully"); + return caches.match("//mochi.test:8888/?" + context, {ignoreSearch: true}); + }).then(function(match) { + ok(match.url.indexOf("?4") > 0, "Match should come from the third cache"); + return caches.delete("caches-xmatch4" + context); // Game over! + }).then(function(deleted) { + ok(deleted, "Deletion should finish successfully"); + return caches.match("//mochi.test:8888/?" + context, {ignoreSearch: true}); + }).then(function(match) { + is(typeof match, "undefined", "No matches should be found"); + }); +} + +function testDelete() { + return caches.delete("delete" + context).then(function(deleted) { + ok(!deleted, "Attempting to delete a non-existing cache should fail"); + return caches.open("delete" + context); + }).then(function() { + return caches.delete("delete" + context); + }).then(function(deleted) { + ok(deleted, "Delete should now succeed"); + }); +} + +testHas().then(function() { + return testKeys(); +}).then(function() { + return testMatchAcrossCaches(); +}).then(function() { + return testDelete(); +}).then(function() { + testDone(); +}); diff --git a/dom/cache/test/mochitest/test_chrome_constructor.html b/dom/cache/test/mochitest/test_chrome_constructor.html new file mode 100644 index 0000000000..1951c655fc --- /dev/null +++ b/dom/cache/test/mochitest/test_chrome_constructor.html @@ -0,0 +1,44 @@ +<!-- Any copyright is dedicated to the Public Domain. + - http://creativecommons.org/publicdomain/zero/1.0/ --> +<!DOCTYPE HTML> +<html> +<head> + <title>Validate Interfaces Exposed to Workers</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script class="testbody" type="text/javascript"> + SimpleTest.waitForExplicitFinish(); + + SpecialPowers.pushPrefEnv({ + "set": [["dom.caches.enabled", true], + ["dom.caches.testing.enabled", true]], + }, function() { + // attach to a different origin's CacheStorage + var url = 'http://example.com/'; + var storage = SpecialPowers.createChromeCache('content', url); + + // verify we can use the other origin's CacheStorage as normal + var req = new Request('http://example.com/index.html'); + var res = new Response('hello world'); + var cache; + storage.open('foo').then(function(c) { + cache = c; + ok(cache, 'storage should create cache'); + return cache.put(req, res.clone()); + }).then(function() { + return cache.match(req); + }).then(function(foundResponse) { + return Promise.all([res.text(), foundResponse.text()]); + }).then(function(results) { + is(results[0], results[1], 'cache should contain response'); + return storage.delete('foo'); + }).then(function(deleted) { + ok(deleted, 'storage should delete cache'); + SimpleTest.finish(); + }); + }); +</script> +</body> +</html> diff --git a/dom/cache/test/mochitest/vary.sjs b/dom/cache/test/mochitest/vary.sjs new file mode 100644 index 0000000000..6659ac852b --- /dev/null +++ b/dom/cache/test/mochitest/vary.sjs @@ -0,0 +1,9 @@ +function handleRequest(request, response) { + response.setStatusLine(request.httpVersion, 200, "OK"); + var header = "no WhatToVary header"; + if (request.hasHeader("WhatToVary")) { + header = request.getHeader("WhatToVary"); + response.setHeader("Vary", header); + } + response.write(header); +} diff --git a/dom/cache/test/mochitest/worker_driver.js b/dom/cache/test/mochitest/worker_driver.js new file mode 100644 index 0000000000..71b541f30a --- /dev/null +++ b/dom/cache/test/mochitest/worker_driver.js @@ -0,0 +1,86 @@ +// Any copyright is dedicated to the Public Domain. +// http://creativecommons.org/publicdomain/zero/1.0/ +// +// Utility script for writing worker tests. In your main document do: +// +// <script type="text/javascript" src="worker_driver.js"></script> +// <script type="text/javascript"> +// workerTestExec('myWorkerTestCase.js') +// </script> +// +// This will then spawn a worker, define some utility functions, and then +// execute the code in myWorkerTestCase.js. You can then use these +// functions in your worker-side test: +// +// ok() - like the SimpleTest assert +// is() - like the SimpleTest assert +// workerTestDone() - like SimpleTest.finish() indicating the test is complete +// +// There are also some functions for requesting information that requires +// SpecialPowers or other main-thread-only resources: +// +// workerTestGetPrefs() - request an array of prefs value from the main thread +// workerTestGetPermissions() - request an array permissions from the MT +// workerTestGetVersion() - request the current version string from the MT +// workerTestGetUserAgent() - request the user agent string from the MT +// +// For an example see test_worker_interfaces.html and test_worker_interfaces.js. + +function workerTestExec(script) { + return new Promise(function(resolve, reject) { + var worker = new Worker('worker_wrapper.js'); + worker.onmessage = function(event) { + is(event.data.context, "Worker", + "Correct context for messages received on the worker"); + if (event.data.type == 'finish') { + worker.terminate(); + SpecialPowers.forceGC(); + resolve(); + + } else if (event.data.type == 'status') { + ok(event.data.status, event.data.context + ": " + event.data.msg); + + } else if (event.data.type == 'getPrefs') { + var result = {}; + event.data.prefs.forEach(function(pref) { + result[pref] = SpecialPowers.Services.prefs.getBoolPref(pref); + }); + worker.postMessage({ + type: 'returnPrefs', + prefs: event.data.prefs, + result: result + }); + + } else if (event.data.type == 'getPermissions') { + var result = {}; + event.data.permissions.forEach(function(permission) { + result[permission] = SpecialPowers.hasPermission(permission, window.document); + }); + worker.postMessage({ + type: 'returnPermissions', + permissions: event.data.permissions, + result: result + }); + + } else if (event.data.type == 'getVersion') { + var result = SpecialPowers.Cc['@mozilla.org/xre/app-info;1'].getService(SpecialPowers.Ci.nsIXULAppInfo).version; + worker.postMessage({ + type: 'returnVersion', + result: result + }); + + } else if (event.data.type == 'getUserAgent') { + worker.postMessage({ + type: 'returnUserAgent', + result: navigator.userAgent + }); + } + } + + worker.onerror = function(event) { + reject('Worker had an error: ' + event.data); + }; + + worker.postMessage({ script: script }); + }); +} diff --git a/dom/cache/test/mochitest/worker_wrapper.js b/dom/cache/test/mochitest/worker_wrapper.js new file mode 100644 index 0000000000..864201498e --- /dev/null +++ b/dom/cache/test/mochitest/worker_wrapper.js @@ -0,0 +1,129 @@ +// Any copyright is dedicated to the Public Domain. +// http://creativecommons.org/publicdomain/zero/1.0/ +// +// ServiceWorker equivalent of worker_wrapper.js. + +var client; +var context; + +function ok(a, msg) { + client.postMessage({type: 'status', status: !!a, + msg: a + ": " + msg, context: context}); +} + +function is(a, b, msg) { + client.postMessage({type: 'status', status: a === b, + msg: a + " === " + b + ": " + msg, context: context }); +} + +function workerTestArrayEquals(a, b) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length != b.length) { + return false; + } + for (var i = 0, n = a.length; i < n; ++i) { + if (a[i] !== b[i]) { + return false; + } + } + return true; +} + +function testDone() { + client.postMessage({ type: 'finish', context: context }); +} + +function workerTestGetPrefs(prefs, cb) { + addEventListener('message', function workerTestGetPrefsCB(e) { + if (e.data.type != 'returnPrefs' || + !workerTestArrayEquals(prefs, e.data.prefs)) { + return; + } + removeEventListener('message', workerTestGetPrefsCB); + cb(e.data.result); + }); + client.postMessage({ + type: 'getPrefs', + context: context, + prefs: prefs + }); +} + +function workerTestGetPermissions(permissions, cb) { + addEventListener('message', function workerTestGetPermissionsCB(e) { + if (e.data.type != 'returnPermissions' || + !workerTestArrayEquals(permissions, e.data.permissions)) { + return; + } + removeEventListener('message', workerTestGetPermissionsCB); + cb(e.data.result); + }); + client.postMessage({ + type: 'getPermissions', + context: context, + permissions: permissions + }); +} + +function workerTestGetVersion(cb) { + addEventListener('message', function workerTestGetVersionCB(e) { + if (e.data.type !== 'returnVersion') { + return; + } + removeEventListener('message', workerTestGetVersionCB); + cb(e.data.result); + }); + client.postMessage({ + context: context, + type: 'getVersion' + }); +} + +function workerTestGetUserAgent(cb) { + addEventListener('message', function workerTestGetUserAgentCB(e) { + if (e.data.type !== 'returnUserAgent') { + return; + } + removeEventListener('message', workerTestGetUserAgentCB); + cb(e.data.result); + }); + client.postMessage({ + context: context, + type: 'getUserAgent' + }); +} + +addEventListener('message', function workerWrapperOnMessage(e) { + removeEventListener('message', workerWrapperOnMessage); + var data = e.data; + function runScript() { + try { + importScripts(data.script); + } catch(e) { + client.postMessage({ + type: 'status', + status: false, + context: context, + msg: 'worker failed to import ' + data.script + "; error: " + e.message + }); + } + } + if ("ServiceWorker" in self) { + self.clients.matchAll().then(function(clients) { + for (var i = 0; i < clients.length; ++i) { + if (clients[i].url.indexOf("message_receiver.html") > -1) { + client = clients[i]; + break; + } + } + if (!client) { + dump("We couldn't find the message_receiver window, the test will fail\n"); + } + context = "ServiceWorker"; + runScript(); + }); + } else { + client = self; + context = "Worker"; + runScript(); + } +}); diff --git a/dom/cache/test/xpcshell/head.js b/dom/cache/test/xpcshell/head.js new file mode 100644 index 0000000000..3d51929b35 --- /dev/null +++ b/dom/cache/test/xpcshell/head.js @@ -0,0 +1,77 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + * + * All images in schema_15_profile.zip are from https://github.com/mdn/sw-test/ + * and are CC licensed by https://www.flickr.com/photos/legofenris/. + */ + +var Cc = Components.classes; +var Ci = Components.interfaces; +var Cu = Components.utils; + +// services required be initialized in order to run CacheStorage +var ss = Cc['@mozilla.org/storage/service;1'] + .createInstance(Ci.mozIStorageService); +var sts = Cc['@mozilla.org/network/stream-transport-service;1'] + .getService(Ci.nsIStreamTransportService); +var hash = Cc['@mozilla.org/security/hash;1'] + .createInstance(Ci.nsICryptoHash); + +// Expose Cache and Fetch symbols on the global +Cu.importGlobalProperties(['caches', 'fetch']); + +// Extract a zip file into the profile +function create_test_profile(zipFileName) { + do_get_profile(); + + var directoryService = Cc['@mozilla.org/file/directory_service;1'] + .getService(Ci.nsIProperties); + var profileDir = directoryService.get('ProfD', Ci.nsIFile); + var currentDir = directoryService.get('CurWorkD', Ci.nsIFile); + + var packageFile = currentDir.clone(); + packageFile.append(zipFileName); + + var zipReader = Cc['@mozilla.org/libjar/zip-reader;1'] + .createInstance(Ci.nsIZipReader); + zipReader.open(packageFile); + + var entryNames = []; + var entries = zipReader.findEntries(null); + while (entries.hasMore()) { + var entry = entries.getNext(); + entryNames.push(entry); + } + entryNames.sort(); + + for (var entryName of entryNames) { + var zipentry = zipReader.getEntry(entryName); + + var file = profileDir.clone(); + entryName.split('/').forEach(function(part) { + file.append(part); + }); + + if (zipentry.isDirectory) { + file.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt('0755', 8)); + } else { + var istream = zipReader.getInputStream(entryName); + + var ostream = Cc['@mozilla.org/network/file-output-stream;1'] + .createInstance(Ci.nsIFileOutputStream); + ostream.init(file, -1, parseInt('0644', 8), 0); + + var bostream = Cc['@mozilla.org/network/buffered-output-stream;1'] + .createInstance(Ci.nsIBufferedOutputStream); + bostream.init(ostream, 32 * 1024); + + bostream.writeFrom(istream, istream.available()); + + istream.close(); + bostream.close(); + } + } + + zipReader.close(); +} diff --git a/dom/cache/test/xpcshell/make_profile.js b/dom/cache/test/xpcshell/make_profile.js new file mode 100644 index 0000000000..5928000e69 --- /dev/null +++ b/dom/cache/test/xpcshell/make_profile.js @@ -0,0 +1,131 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + * + * All images in schema_15_profile.zip are from https://github.com/mdn/sw-test/ + * and are CC licensed by https://www.flickr.com/photos/legofenris/. + */ + +var Cc = Components.classes; +var Ci = Components.interfaces; +var Cu = Components.utils; + +// Enumerate the directory tree and store results in entryList as +// +// { path: 'a/b/c', file: <nsIFile> } +// +// The algorithm starts with the first entry already in entryList. +function enumerate_tree(entryList) { + for (var index = 0; index < entryList.length; ++index) { + var path = entryList[index].path; + var file = entryList[index].file; + + if (file.isDirectory()) { + var dirList = file.directoryEntries; + while (dirList.hasMoreElements()) { + var dirFile = dirList.getNext().QueryInterface(Ci.nsIFile); + entryList.push({ path: path + '/' + dirFile.leafName, file: dirFile }); + } + } + } +} + +function zip_profile(zipFile, profileDir) { + var zipWriter = Cc['@mozilla.org/zipwriter;1'] + .createInstance(Ci.nsIZipWriter); + zipWriter.open(zipFile, 0x04 | 0x08 | 0x20); + + var root = profileDir.clone(); + root.append('storage'); + root.append('default'); + root.append('chrome'); + + var entryList = [{path: 'storage/default/chrome', file: root}]; + enumerate_tree(entryList); + + entryList.forEach(function(entry) { + if (entry.file.isDirectory()) { + zipWriter.addEntryDirectory(entry.path, entry.file.lastModifiedTime, + false); + } else { + var istream = Cc['@mozilla.org/network/file-input-stream;1'] + .createInstance(Ci.nsIFileInputStream); + istream.init(entry.file, -1, -1, 0); + zipWriter.addEntryStream(entry.path, entry.file.lastModifiedTime, + Ci.nsIZipWriter.COMPRESSION_DEFAULT, istream, + false); + istream.close(); + } + }); + + zipWriter.close(); +} + +function exactGC() { + return new Promise(function(resolve) { + var count = 0; + function doPreciseGCandCC() { + function scheduleGCCallback() { + Cu.forceCC(); + + if (++count < 2) { + doPreciseGCandCC(); + } else { + resolve(); + } + } + Cu.schedulePreciseGC(scheduleGCCallback); + } + doPreciseGCandCC(); + }); +} + +function resetQuotaManager() { + return new Promise(function(resolve) { + var qm = Cc['@mozilla.org/dom/quota/manager;1'] + .getService(Ci.nsIQuotaManager); + + var prefService = Cc['@mozilla.org/preferences-service;1'] + .getService(Ci.nsIPrefService); + + // enable quota manager testing mode + var pref = 'dom.quotaManager.testing'; + prefService.getBranch(null).setBoolPref(pref, true); + + var request = qm.reset(); + request.callback = resolve; + + // disable quota manager testing mode + //prefService.getBranch(null).setBoolPref(pref, false); + }); +} + +function run_test() { + do_test_pending(); + do_get_profile(); + + var directoryService = Cc['@mozilla.org/file/directory_service;1'] + .getService(Ci.nsIProperties); + var profileDir = directoryService.get('ProfD', Ci.nsIFile); + var currentDir = directoryService.get('CurWorkD', Ci.nsIFile); + + var zipFile = currentDir.clone(); + zipFile.append('new_profile.zip'); + if (zipFile.exists()) { + zipFile.remove(false); + } + ok(!zipFile.exists()); + + caches.open('xpcshell-test').then(function(c) { + var request = new Request('http://example.com/index.html'); + var response = new Response('hello world'); + return c.put(request, response); + }).then(exactGC).then(resetQuotaManager).then(function() { + zip_profile(zipFile, profileDir); + dump('### ### created zip at: ' + zipFile.path + '\n'); + do_test_finished(); + }).catch(function(e) { + do_test_finished(); + ok(false, e); + }); +} diff --git a/dom/cache/test/xpcshell/schema_15_profile.zip b/dom/cache/test/xpcshell/schema_15_profile.zip Binary files differnew file mode 100644 index 0000000000..32cc8f2eeb --- /dev/null +++ b/dom/cache/test/xpcshell/schema_15_profile.zip diff --git a/dom/cache/test/xpcshell/test_migration.js b/dom/cache/test/xpcshell/test_migration.js new file mode 100644 index 0000000000..8ccccda9d8 --- /dev/null +++ b/dom/cache/test/xpcshell/test_migration.js @@ -0,0 +1,49 @@ +/** + * Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + * + * All images in schema_15_profile.zip are from https://github.com/mdn/sw-test/ + * and are CC licensed by https://www.flickr.com/photos/legofenris/. + */ + +function run_test() { + do_test_pending(); + create_test_profile('schema_15_profile.zip'); + + var cache; + caches.open('xpcshell-test').then(function(c) { + cache = c; + ok(cache, 'cache exists'); + return cache.keys(); + }).then(function(requestList) { + ok(requestList.length > 0, 'should have at least one request in cache'); + requestList.forEach(function(request) { + ok(request, 'each request in list should be non-null'); + ok(request.redirect === 'follow', 'request.redirect should default to "follow"'); + ok(request.cache === 'default', 'request.cache should have been updated to "default"' + request.cache); + ok(request.mode === 'navigate', 'request.mode should have been updated to "navigate"'); + ok(request.referrerPolicy === 'no-referrer-when-downgrade', 'request.referrerPolicy should have been updated to "no-referrer-when-downgrade"'); + }); + return Promise.all(requestList.map(function(request) { + return cache.match(request); + })); + }).then(function(responseList) { + ok(responseList.length > 0, 'should have at least one response in cache'); + responseList.forEach(function(response) { + ok(response, 'each response in list should be non-null'); + // reponse.url is a empty string in current test file. It should test for + // not being a empty string once thet test file is updated. + ok(typeof response.url === 'string', 'each response.url in list should be a string'); + // reponse.redirected may be changed once test file is updated. It should + // be false since current reponse.url is a empty string. + ok(response.redirected === false, 'each response.redirected in list should be false'); + do_check_eq(response.headers.get('Content-Type'), 'text/plain;charset=UTF-8', + 'the response should have the correct header'); + }); + }).then(function() { + do_test_finished(); + }).catch(function(e) { + ok(false, 'caught exception ' + e); + do_test_finished(); + }); +} diff --git a/dom/cache/test/xpcshell/xpcshell.ini b/dom/cache/test/xpcshell/xpcshell.ini new file mode 100644 index 0000000000..6cbba6ab3e --- /dev/null +++ b/dom/cache/test/xpcshell/xpcshell.ini @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +[DEFAULT] +head = head.js +tail = +support-files = + schema_15_profile.zip + +# dummy test entry to generate profile zip files +[make_profile.js] + skip-if = true + +[test_migration.js] |