Attachment 8832157 Details for Bug 1217544

[patch] Part 6: mozStorageConnectionUtils

bug1217544.background.sync.part6.mozstorageincrementalvacuum.patch (text/plain), 28.79 KB, created by Fernando Jiménez Moreno [:ferjm]

(hide)

Description:

Filename:

MIME Type:

Creator: Fernando Jiménez Moreno [:ferjm]

Size: 28.79 KB

patch

obsolete

># HG changeset patch
># User Fernando Jimenez <ferjmoreno@gmail.com>
># Parent  e72944c7835a7c4875c39d02637d05b63b69cde1
>Bug 1217544 - Implement one-off BackgroundSync API. Part 6: mozStorageConnectionUtils. r=bkelly
>
>diff --git a/storage/IncrementalVacuumConnection.cpp b/storage/IncrementalVacuumConnection.cpp
>new file mode 100644
>--- /dev/null
>+++ b/storage/IncrementalVacuumConnection.cpp
>@@ -0,0 +1,298 @@
>+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
>+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
>+/* This Source Code Form is subject to the terms of the Mozilla Public
>+ * License, v. 2.0. If a copy of the MPL was not distributed with this
>+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
>+
>+#include "IncrementalVacuumConnection.h"
>+
>+#include "mozStorageConnectionUtils.h"
>+#include "mozStorageHelper.h"
>+#include "nsClassHashtable.h"
>+#include "nsPrintfCString.h"
>+#include "nsTArray.h"
>+#include "nsThreadUtils.h"
>+
>+namespace mozilla {
>+namespace storage {
>+
>+using dom::quota::QuotaObject;
>+
>+NS_IMPL_ISUPPORTS(IncrementalVacuumConnection,
>+                  mozIStorageAsyncConnection,
>+                  mozIStorageConnection);
>+
>+IncrementalVacuumConnection::IncrementalVacuumConnection(
>+    mozIStorageConnection* aBase, const int32_t aMaxFreePages)
>+  : mBase(aBase)
>+  , mMaxFreePages(aMaxFreePages)
>+  , mClosed(false)
>+{
>+  MOZ_ASSERT(mBase);
>+}
>+
>+IncrementalVacuumConnection::~IncrementalVacuumConnection()
>+{
>+  NS_ASSERT_OWNINGTHREAD(IncrementalVacuumConnection);
>+  MOZ_ALWAYS_SUCCEEDS(Close());
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::Close()
>+{
>+  NS_ASSERT_OWNINGTHREAD(IncrementalVacuumConnection);
>+
>+  if (mClosed) {
>+    return NS_OK;
>+  }
>+  mClosed = true;
>+
>+  // If we are closing here, then we must not have a transaction
>+  // open anywhere else. This should be guaranteed to succeed.
>+  MOZ_ALWAYS_SUCCEEDS(utils::IncrementalVacuum(mBase, mMaxFreePages));
>+
>+  return mBase->Close();
>+}
>+
>+// The following methods are all boilerplate that either forward to the
>+// base connection or block the method. All the async execution methods
>+// are blocked because we do not use them and they would require more
>+// work to wrap properly.
>+
>+// mozIStorageAsyncConnection methods
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::AsyncClose(mozIStorageCompletionCallback*)
>+{
>+  // async methods are not supported.
>+  return NS_ERROR_NOT_IMPLEMENTED;
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::AsyncClone(bool, mozIStorageCompletionCallback*)
>+{
>+  // async methods are not supported.
>+  return NS_ERROR_NOT_IMPLEMENTED;
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::GetDatabaseFile(nsIFile** aFileOut)
>+{
>+  return mBase->GetDatabaseFile(aFileOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::CreateAsyncStatement(const nsACString&,
>+                                                  mozIStorageAsyncStatement**)
>+{
>+  // async methods are not supported.
>+  return NS_ERROR_NOT_IMPLEMENTED;
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::ExecuteAsync(mozIStorageBaseStatement**, uint32_t,
>+                                          mozIStorageStatementCallback*,
>+                                          mozIStoragePendingStatement**)
>+{
>+  // async methods are not supported.
>+  return NS_ERROR_NOT_IMPLEMENTED;
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::ExecuteSimpleSQLAsync(const nsACString&,
>+    mozIStorageStatementCallback*, mozIStoragePendingStatement**)
>+{
>+  // async methods are not supported.
>+  return NS_ERROR_NOT_IMPLEMENTED;
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::CreateFunction(const nsACString& aFunctionName,
>+                                            int32_t aNumArguments,
>+                                            mozIStorageFunction* aFunction)
>+{
>+  // async methods are not supported.
>+  return NS_ERROR_NOT_IMPLEMENTED;
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::CreateAggregateFunction(
>+    const nsACString& aFunctionName, int32_t aNumArguments,
>+    mozIStorageAggregateFunction* aFunction)
>+{
>+  return mBase->CreateAggregateFunction(aFunctionName, aNumArguments,
>+                                        aFunction);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::RemoveFunction(const nsACString& aFunctionName)
>+{
>+  return mBase->RemoveFunction(aFunctionName);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::SetProgressHandler(
>+    int32_t aGranularity, mozIStorageProgressHandler* aHandler,
>+    mozIStorageProgressHandler** aHandlerOut)
>+{
>+  return mBase->SetProgressHandler(aGranularity, aHandler, aHandlerOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::RemoveProgressHandler(
>+    mozIStorageProgressHandler** aHandlerOut)
>+{
>+  return mBase->RemoveProgressHandler(aHandlerOut);
>+}
>+
>+// mozIStorageConnection methods
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::Clone(bool aReadOnly,
>+                                   mozIStorageConnection** aConnectionOut)
>+{
>+  nsCOMPtr<mozIStorageConnection> conn;
>+  nsresult rv = mBase->Clone(aReadOnly, getter_AddRefs(conn));
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  nsCOMPtr<mozIStorageConnection> wrapped =
>+    new IncrementalVacuumConnection(conn, mMaxFreePages);
>+  wrapped.forget(aConnectionOut);
>+
>+  return rv;
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::GetDefaultPageSize(int32_t* aSizeOut)
>+{
>+  return mBase->GetDefaultPageSize(aSizeOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::GetConnectionReady(bool* aReadyOut)
>+{
>+  return mBase->GetConnectionReady(aReadyOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::GetLastInsertRowID(int64_t* aRowIdOut)
>+{
>+  return mBase->GetLastInsertRowID(aRowIdOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::GetAffectedRows(int32_t* aCountOut)
>+{
>+  return mBase->GetAffectedRows(aCountOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::GetLastError(int32_t* aErrorOut)
>+{
>+  return mBase->GetLastError(aErrorOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::GetLastErrorString(nsACString& aErrorOut)
>+{
>+  return mBase->GetLastErrorString(aErrorOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::GetSchemaVersion(int32_t* aVersionOut)
>+{
>+  return mBase->GetSchemaVersion(aVersionOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::SetSchemaVersion(int32_t aVersion)
>+{
>+  return mBase->SetSchemaVersion(aVersion);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::CreateStatement(const nsACString& aQuery,
>+                                             mozIStorageStatement** aStatementOut)
>+{
>+  return mBase->CreateStatement(aQuery, aStatementOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::ExecuteSimpleSQL(const nsACString& aQuery)
>+{
>+  return mBase->ExecuteSimpleSQL(aQuery);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::TableExists(const nsACString& aTableName,
>+                                         bool* aExistsOut)
>+{
>+  return mBase->TableExists(aTableName, aExistsOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::IndexExists(const nsACString& aIndexName,
>+                                         bool* aExistsOut)
>+{
>+  return mBase->IndexExists(aIndexName, aExistsOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::GetTransactionInProgress(bool* aResultOut)
>+{
>+  return mBase->GetTransactionInProgress(aResultOut);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::BeginTransaction()
>+{
>+  return mBase->BeginTransaction();
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::BeginTransactionAs(int32_t aType)
>+{
>+  return mBase->BeginTransactionAs(aType);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::CommitTransaction()
>+{
>+  return mBase->CommitTransaction();
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::RollbackTransaction()
>+{
>+  return mBase->RollbackTransaction();
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::CreateTable(const char* aTable,
>+                                         const char* aSchema)
>+{
>+  return mBase->CreateTable(aTable, aSchema);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::SetGrowthIncrement(int32_t aIncrement,
>+                                                const nsACString& aDatabase)
>+{
>+  return mBase->SetGrowthIncrement(aIncrement, aDatabase);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::EnableModule(const nsACString& aModule)
>+{
>+  return mBase->EnableModule(aModule);
>+}
>+
>+NS_IMETHODIMP
>+IncrementalVacuumConnection::GetQuotaObjects(QuotaObject** aDatabaseQuotaObject,
>+                                             QuotaObject** aJournalQuotaObject)
>+{
>+  return mBase->GetQuotaObjects(aDatabaseQuotaObject, aJournalQuotaObject);
>+}
>+
>+} // namespace storage
>+} // namespace mozilla
>diff --git a/storage/IncrementalVacuumConnection.h b/storage/IncrementalVacuumConnection.h
>new file mode 100644
>--- /dev/null
>+++ b/storage/IncrementalVacuumConnection.h
>@@ -0,0 +1,37 @@
>+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
>+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
>+/* This Source Code Form is subject to the terms of the Mozilla Public
>+ * License, v. 2.0. If a copy of the MPL was not distributed with this
>+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
>+
>+#ifndef mozilla_storage_IncrementalVacuumConnection_h
>+#define mozilla_storage_IncrementalVacuumConnection_h
>+
>+#include "mozIStorageConnection.h"
>+#include "nsCOMPtr.h"
>+
>+namespace mozilla {
>+namespace storage {
>+
>+class IncrementalVacuumConnection final : public mozIStorageConnection
>+{
>+public:
>+  IncrementalVacuumConnection(mozIStorageConnection* aBase,
>+                              const int32_t aMaxFreePages);
>+
>+private:
>+  ~IncrementalVacuumConnection();
>+
>+  nsCOMPtr<mozIStorageConnection> mBase;
>+  int32_t mMaxFreePages;
>+  bool mClosed;
>+
>+  NS_DECL_ISUPPORTS
>+  NS_DECL_MOZISTORAGEASYNCCONNECTION
>+  NS_DECL_MOZISTORAGECONNECTION
>+};
>+
>+} // namespace storage
>+} // namespace mozilla
>+
>+#endif // mozilla_storage_IncrementalVacuumConnection_h
>diff --git a/storage/moz.build b/storage/moz.build
>--- a/storage/moz.build
>+++ b/storage/moz.build
>@@ -40,30 +40,34 @@ EXPORTS += [
> 
> EXPORTS.mozilla += [
>     'storage.h',
> ]
> 
> # NOTE When adding something to this list, you probably need to add it to the
> #      storage.h file too.
> EXPORTS.mozilla.storage += [
>+    'IncrementalVacuumConnection.h',
>+    'mozStorageConnectionUtils.h',
>     'StatementCache.h',
>     'Variant.h',
>     'Variant_inl.h',
> ]
> # SEE ABOVE NOTE!
> 
> UNIFIED_SOURCES += [
>     'FileSystemModule.cpp',
>+    'IncrementalVacuumConnection.cpp',
>     'mozStorageArgValueArray.cpp',
>     'mozStorageAsyncStatement.cpp',
>     'mozStorageAsyncStatementExecution.cpp',
>     'mozStorageAsyncStatementJSHelper.cpp',
>     'mozStorageAsyncStatementParams.cpp',
>     'mozStorageBindingParamsArray.cpp',
>+    'mozStorageConnectionUtils.cpp',
>     'mozStorageError.cpp',
>     'mozStoragePrivateHelpers.cpp',
>     'mozStorageResultSet.cpp',
>     'mozStorageRow.cpp',
>     'mozStorageService.cpp',
>     'mozStorageSQLFunctions.cpp',
>     'mozStorageStatement.cpp',
>     'mozStorageStatementJSHelper.cpp',
>diff --git a/storage/mozStorageConnectionUtils.cpp b/storage/mozStorageConnectionUtils.cpp
>new file mode 100644
>--- /dev/null
>+++ b/storage/mozStorageConnectionUtils.cpp
>@@ -0,0 +1,429 @@
>+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
>+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
>+/* This Source Code Form is subject to the terms of the Mozilla Public
>+ * License, v. 2.0. If a copy of the MPL was not distributed with this
>+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
>+
>+#include "mozStorageConnectionUtils.h"
>+
>+namespace mozilla {
>+namespace storage {
>+namespace utils {
>+
>+namespace {
>+
>+class MOZ_RAII AutoDisableForeignKeyChecking
>+{
>+public:
>+  explicit AutoDisableForeignKeyChecking(mozIStorageConnection* aConn)
>+    : mConn(aConn)
>+    , mForeignKeyCheckingDisabled(false)
>+  {
>+    nsCOMPtr<mozIStorageStatement> state;
>+    nsresult rv = mConn->CreateStatement(NS_LITERAL_CSTRING(
>+      "PRAGMA foreign_keys;"
>+    ), getter_AddRefs(state));
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return; }
>+
>+    bool hasMoreData = false;
>+    rv = state->ExecuteStep(&hasMoreData);
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return; }
>+
>+    int32_t mode;
>+    rv = state->GetInt32(0, &mode);
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return; }
>+
>+    if (mode) {
>+      nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
>+        "PRAGMA foreign_keys = OFF;"
>+      ));
>+      if (NS_WARN_IF(NS_FAILED(rv))) { return; }
>+      mForeignKeyCheckingDisabled = true;
>+    }
>+  }
>+
>+  ~AutoDisableForeignKeyChecking()
>+  {
>+    if (mForeignKeyCheckingDisabled) {
>+      nsresult rv = mConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
>+        "PRAGMA foreign_keys = ON;"
>+      ));
>+      if (NS_WARN_IF(NS_FAILED(rv))) { return; }
>+    }
>+  }
>+
>+private:
>+  nsCOMPtr<mozIStorageConnection> mConn;
>+  bool mForeignKeyCheckingDisabled;
>+};
>+
>+} // anonymous namespace
>+
>+nsresult
>+Validate(mozIStorageConnection* aConn,
>+         const int32_t aLatestSchemaVersion,
>+         const nsTArray<Expect>& aExpectedSchema)
>+{
>+  int32_t schemaVersion;
>+  nsresult rv = aConn->GetSchemaVersion(&schemaVersion);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  if (NS_WARN_IF(schemaVersion != aLatestSchemaVersion)) {
>+    return NS_ERROR_FAILURE;
>+  }
>+
>+#ifdef DEBUG
>+  // Read the schema from the sqlite_master table and compare.
>+  nsCOMPtr<mozIStorageStatement> state;
>+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
>+    "SELECT name, type, sql FROM sqlite_master;"
>+  ), getter_AddRefs(state));
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  bool hasMoreData = false;
>+  while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) {
>+    nsAutoCString name;
>+    rv = state->GetUTF8String(0, name);
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+    nsAutoCString type;
>+    rv = state->GetUTF8String(1, type);
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+    nsAutoCString sql;
>+    rv = state->GetUTF8String(2, sql);
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+    bool foundMatch = false;
>+    for (uint32_t i = 0; i < aExpectedSchema.Length(); ++i) {
>+      if (name == aExpectedSchema[i].mName) {
>+        if (type != aExpectedSchema[i].mType) {
>+          NS_WARNING(nsPrintfCString("Unexpected type for schema entry %s",
>+                     name.get()).get());
>+          return NS_ERROR_FAILURE;
>+        }
>+
>+        if (!aExpectedSchema[i].mIgnoreSql && sql != aExpectedSchema[i].mSql) {
>+          NS_WARNING(nsPrintfCString("Unexpected SQL for schema entry %s",
>+                     name.get()).get());
>+          return NS_ERROR_FAILURE;
>+        }
>+
>+        foundMatch = true;
>+        break;
>+      }
>+    }
>+
>+    if (NS_WARN_IF(!foundMatch)) {
>+      NS_WARNING(nsPrintfCString("Unexpected schema entry %s in database",
>+                 name.get()).get());
>+      return NS_ERROR_FAILURE;
>+    }
>+  }
>+#endif
>+
>+  return rv;
>+}
>+
>+nsresult
>+RewriteSchema(mozIStorageConnection* aConn,
>+              const nsACString& aTable,
>+              const nsACString& aSql)
>+{
>+  nsresult rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
>+    "PRAGMA writable_schema = ON"
>+  ));
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  nsCOMPtr<mozIStorageStatement> state;
>+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
>+    "UPDATE sqlite_master SET sql=:sql WHERE name=:name"
>+  ), getter_AddRefs(state));
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("sql"), aSql);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  rv = state->BindUTF8StringByName(NS_LITERAL_CSTRING("name"), aTable);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  rv = state->Execute();
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  rv = aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
>+    "PRAGMA writable_schema = OFF"
>+  ));
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  return rv;
>+}
>+
>+nsresult
>+Migrate(mozIStorageConnection* aConn,
>+        const int32_t aFirstShippedSchemaVersion,
>+        const int32_t aLatestSchemaVersion,
>+        const nsTArray<Migration>& aMigrationList)
>+{
>+  MOZ_ASSERT(!NS_IsMainThread());
>+  MOZ_ASSERT(aConn);
>+
>+  int32_t currentVersion = 0;
>+  nsresult rv = aConn->GetSchemaVersion(&currentVersion);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  nsClassHashtable<nsCStringHashKey, nsCString> schemaRewrites;
>+
>+  while (currentVersion < aLatestSchemaVersion) {
>+    // Wiping old database should be handled elsewhere because it requires
>+    // making a whole new mozIStorageConnection.  Make sure we don't
>+    // accidentally get here for one of those old databases.
>+    MOZ_ASSERT(currentVersion >= aFirstShippedSchemaVersion);
>+
>+    for (uint32_t i = 0; i < aMigrationList.Length(); ++i) {
>+      if (aMigrationList[i].mFromVersion == currentVersion) {
>+        nsCString table, sql;
>+        rv = aMigrationList[i].mFunc(aConn, table, sql);
>+        if (!table.IsEmpty() && !schemaRewrites.Contains(table)) {
>+          schemaRewrites.Put(table, new nsCString(sql));
>+        }
>+        break;
>+      }
>+    }
>+
>+    DebugOnly<int32_t> lastVersion = currentVersion;
>+    rv = aConn->GetSchemaVersion(&currentVersion);
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+    MOZ_ASSERT(currentVersion > lastVersion);
>+  }
>+
>+  MOZ_ASSERT(currentVersion == aLatestSchemaVersion);
>+
>+  // Now overwrite the master SQL for each table that requires so.
>+  for (auto iter = schemaRewrites.Iter(); !iter.Done(); iter.Next()) {
>+    rv = RewriteSchema(aConn, iter.Key(), *schemaRewrites.Get(iter.Key()));
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+  }
>+
>+  schemaRewrites.Clear();
>+
>+  return rv;
>+}
>+
>+nsresult
>+CreateOrMigrateSchema(mozIStorageConnection* aConn,
>+                      const int32_t aFirstShippedSchemaVersion,
>+                      const int32_t aLatestSchemaVersion,
>+                      const nsTArray<nsCString>& aTablesSql,
>+                      const nsTArray<Expect>& aExpectedSchema,
>+                      const nsTArray<Migration>& aMigrationList)
>+{
>+  MOZ_ASSERT(!NS_IsMainThread());
>+  MOZ_ASSERT(aConn);
>+
>+  int32_t schemaVersion;
>+  nsresult rv = aConn->GetSchemaVersion(&schemaVersion);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  if (schemaVersion == aLatestSchemaVersion) {
>+    // We already have the correct schema version.  Validate it matches
>+    // our expected schema and then proceed.
>+    rv = Validate(aConn, aLatestSchemaVersion, aExpectedSchema);
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+    return rv;
>+  }
>+
>+  // Turn off checking foreign keys before starting a transaction, and restore
>+  // it once we're done.
>+  AutoDisableForeignKeyChecking restoreForeignKeyChecking(aConn);
>+  mozStorageTransaction trans(aConn, false,
>+                              mozIStorageConnection::TRANSACTION_IMMEDIATE);
>+  bool needVacuum = false;
>+
>+  if (schemaVersion) {
>+    // A schema exists, but its not the current version.  Attempt to
>+    // migrate it to our new schema.
>+    rv = Migrate(aConn, aFirstShippedSchemaVersion, aLatestSchemaVersion,
>+                 aMigrationList);
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+    // Migrations happen infrequently and reflect a chance in DB structure.
>+    // This is a good time to rebuild the database.  It also helps catch
>+    // if a new migration is incorrect by fast failing on the corruption.
>+    needVacuum = true;
>+  } else {
>+    // There is no schema installed.  Create the database from scratch.
>+    for (uint32_t i = 0; i < aTablesSql.Length(); i++) {
>+      rv = aConn->ExecuteSimpleSQL(aTablesSql[i]);
>+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+    }
>+
>+    rv = aConn->SetSchemaVersion(aLatestSchemaVersion);
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+    rv = aConn->GetSchemaVersion(&schemaVersion);
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+  }
>+
>+  rv = Validate(aConn, aLatestSchemaVersion, aExpectedSchema);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  rv = trans.Commit();
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  if (needVacuum) {
>+    // Unfortunately, this must be performed outside of the transaction.
>+    aConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("VACUUM"));
>+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+  }
>+
>+  return rv;
>+}
>+
>+nsresult
>+InitializeConnection(mozIStorageConnection* aConn,
>+                     const uint32_t aPageSize,
>+                     const uint32_t aGrowthSize,
>+                     const uint32_t aWalAutoCheckpointPages,
>+                     const uint32_t aWalAutoCheckpointSize)
>+{
>+  MOZ_ASSERT(!NS_IsMainThread());
>+  MOZ_ASSERT(aConn);
>+
>+  // This function needs to perform per-connection initialization tasks that
>+  // need to happen regardless of the schema.
>+
>+  nsPrintfCString pragmas(
>+    // Use a smaller page size to improve perf/footprint; default is too large
>+    "PRAGMA page_size = %u; "
>+    // Enable auto_vacuum; this must happen after page_size and before WAL
>+    "PRAGMA auto_vacuum = INCREMENTAL; "
>+    "PRAGMA foreign_keys = ON; ",
>+    aPageSize
>+  );
>+
>+  // Note, the default encoding of UTF-8 is preferred.  mozStorage does all
>+  // the work necessary to convert UTF-16 nsString values for us.  We don't
>+  // need ordering and the binary equality operations are correct.  So, do
>+  // NOT set PRAGMA encoding to UTF-16.
>+
>+  nsresult rv = aConn->ExecuteSimpleSQL(pragmas);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  // Limit fragmentation by growing the database by many pages at once.
>+  rv = aConn->SetGrowthIncrement(aGrowthSize, EmptyCString());
>+  if (rv == NS_ERROR_FILE_TOO_BIG) {
>+    NS_WARNING("Not enough disk space to set sqlite growth increment.");
>+    rv = NS_OK;
>+  }
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  // Enable WAL journaling.  This must be performed in a separate transaction
>+  // after changing the page_size and enabling auto_vacuum.
>+  nsPrintfCString wal(
>+    // WAL journal can grow to given number of *pages*
>+    "PRAGMA wal_autocheckpoint = %u; "
>+    // Always truncate the journal back to given number of *bytes*
>+    "PRAGMA journal_size_limit = %u; "
>+    // WAL must be enabled at the end to allow page size to be changed, etc.
>+    "PRAGMA journal_mode = WAL; ",
>+    aWalAutoCheckpointPages,
>+    aWalAutoCheckpointSize
>+  );
>+
>+  rv = aConn->ExecuteSimpleSQL(wal);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  // Verify that we successfully set the vacuum mode to incremental.  It
>+  // is very easy to put the database in a state where the auto_vacuum
>+  // pragma above fails silently.
>+#ifdef DEBUG
>+  nsCOMPtr<mozIStorageStatement> state;
>+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
>+    "PRAGMA auto_vacuum;"
>+  ), getter_AddRefs(state));
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  bool hasMoreData = false;
>+  rv = state->ExecuteStep(&hasMoreData);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  int32_t mode;
>+  rv = state->GetInt32(0, &mode);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  // integer value 2 is incremental mode
>+  if (NS_WARN_IF(mode != 2)) { return NS_ERROR_UNEXPECTED; }
>+#endif
>+
>+  return NS_OK;
>+}
>+
>+nsresult
>+IncrementalVacuum(mozIStorageConnection* aConn,
>+                  const int32_t aMaxFreePages)
>+{
>+  // Determine how much free space is in the database.
>+  nsCOMPtr<mozIStorageStatement> state;
>+  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
>+    "PRAGMA freelist_count;"
>+  ), getter_AddRefs(state));
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  bool hasMoreData = false;
>+  rv = state->ExecuteStep(&hasMoreData);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  int32_t freePages = 0;
>+  rv = state->GetInt32(0, &freePages);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  // We have a relatively small page size, so we want to be careful to avoid
>+  // fragmentation.  We already use a growth incremental which will cause
>+  // sqlite to allocate and release multiple pages at the same time.  We can
>+  // further reduce fragmentation by making our allocated chunks a bit
>+  // "sticky".  This is done by creating some hysteresis where we allocate
>+  // pages/chunks as soon as we need them, but we only release pages/chunks
>+  // when we have a large amount of free space.  This helps with the case
>+  // where a page is adding and remove resources causing it to dip back and
>+  // forth across a chunk boundary.
>+  //
>+  // So only proceed with releasing pages if we have more than our constant
>+  // threshold.
>+  if (freePages <= aMaxFreePages) {
>+    return NS_OK;
>+  }
>+
>+  // Release the excess pages back to the sqlite VFS.  This may also release
>+  // chunks of multiple pages back to the OS.
>+  int32_t pagesToRelease = freePages - aMaxFreePages;
>+
>+  rv = aConn->ExecuteSimpleSQL(nsPrintfCString(
>+    "PRAGMA incremental_vacuum(%d);", pagesToRelease
>+  ));
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  // Verify that our incremental vacuum actually did something
>+#ifdef DEBUG
>+  rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
>+    "PRAGMA freelist_count;"
>+  ), getter_AddRefs(state));
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  hasMoreData = false;
>+  rv = state->ExecuteStep(&hasMoreData);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  freePages = 0;
>+  rv = state->GetInt32(0, &freePages);
>+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
>+
>+  MOZ_ASSERT(freePages <= aMaxFreePages);
>+#endif
>+
>+  return NS_OK;
>+}
>+
>+} // namespace utils
>+} // namespace storage
>+} // namespace mozilla
>diff --git a/storage/mozStorageConnectionUtils.h b/storage/mozStorageConnectionUtils.h
>new file mode 100644
>--- /dev/null
>+++ b/storage/mozStorageConnectionUtils.h
>@@ -0,0 +1,78 @@
>+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
>+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
>+/* This Source Code Form is subject to the terms of the Mozilla Public
>+ * License, v. 2.0. If a copy of the MPL was not distributed with this
>+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
>+
>+#ifndef mozilla_storage_mozStorageConnectionUtils_h
>+#define mozilla_storage_mozStorageConnectionUtils_h
>+
>+#include "mozIStorageConnection.h"
>+#include "nsCOMPtr.h"
>+
>+namespace mozilla {
>+namespace storage {
>+namespace utils {
>+
>+struct Migration;
>+
>+struct Expect
>+{
>+  // Expect exact SQL
>+  Expect(const char* aName, const char* aType, const char* aSql)
>+    : mName(aName)
>+    , mType(aType)
>+    , mSql(aSql)
>+    , mIgnoreSql(false)
>+  { }
>+
>+  // Ignore SQL
>+  Expect(const char* aName, const char* aType)
>+    : mName(aName)
>+    , mType(aType)
>+    , mIgnoreSql(true)
>+  { }
>+
>+  const nsCString mName;
>+  const nsCString mType;
>+  const nsCString mSql;
>+  const bool mIgnoreSql;
>+};
>+
>+typedef nsresult (*MigrationFunc)(mozIStorageConnection*, nsACString&,
>+                                  nsACString&);
>+
>+struct Migration
>+{
>+  constexpr Migration(int32_t aFromVersion, MigrationFunc aFunc)
>+    : mFromVersion(aFromVersion)
>+    , mFunc(aFunc)
>+  { }
>+  int32_t mFromVersion;
>+  MigrationFunc mFunc;
>+};
>+
>+nsresult
>+CreateOrMigrateSchema(mozIStorageConnection* aConn,
>+                      const int32_t aFirstShippedSchemaVersion,
>+                      const int32_t aLatestSchemaVersion,
>+                      const nsTArray<nsCString>& aTablesSql,
>+                      const nsTArray<Expect>& aExpectedSchema,
>+                      const nsTArray<Migration>& aMigrationList);
>+
>+nsresult
>+InitializeConnection(mozIStorageConnection* aConn,
>+                     const uint32_t aPageSize,
>+                     const uint32_t aGrowthSize,
>+                     const uint32_t aWalAutoCheckpointPages,
>+                     const uint32_t aWalAutoCheckpointSize);
>+
>+nsresult
>+IncrementalVacuum(mozIStorageConnection* aConn,
>+                  const int32_t aMaxFreePages);
>+
>+} // namespace utils
>+} // namespace storage
>+} // namespace mozilla
>+
>+#endif // mozilla_storage_IncrementalVacuumConnection_h
>diff --git a/storage/mozStorageSQLFunctions.cpp b/storage/mozStorageSQLFunctions.cpp
>--- a/storage/mozStorageSQLFunctions.cpp
>+++ b/storage/mozStorageSQLFunctions.cpp
>@@ -2,16 +2,17 @@
>  * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
>  * This Source Code Form is subject to the terms of the Mozilla Public
>  * License, v. 2.0. If a copy of the MPL was not distributed with this
>  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
> 
> #include "mozilla/ArrayUtils.h"
> 
> #include "mozStorageSQLFunctions.h"
>+#include "nsTArray.h"
> #include "nsUnicharUtils.h"
> #include <algorithm>
> 
> namespace mozilla {
> namespace storage {
> 
> ////////////////////////////////////////////////////////////////////////////////
> //// Local Helper Functions
>diff --git a/storage/storage.h b/storage/storage.h
>--- a/storage/storage.h
>+++ b/storage/storage.h
>@@ -28,12 +28,14 @@
> #include "mozIStorageCompletionCallback.h"
> #include "mozIStorageAsyncStatement.h"
> #include "mozIStorageAsyncConnection.h"
> 
> ////////////////////////////////////////////////////////////////////////////////
> //// Native Language Helpers
> 
> #include "mozStorageHelper.h"
>+#include "mozilla/storage/IncrementalVacuumConnection.h"
>+#include "mozilla/storage/mozStorageConnectionUtils.h"
> #include "mozilla/storage/StatementCache.h"
> #include "mozilla/storage/Variant.h"
> 
> #endif // mozilla_storage_h_

Flags:

asuth: review+

Actions: View | Diff | Review

Attachments on bug 1217544: 8705632 | 8705633 | 8705634 | 8707035 | 8712697 | 8712699 | 8712744 | 8712745 | 8712746 | 8721385 | 8721387 | 8721388 | 8721389 | 8721408 | 8727182 | 8731368 | 8731371 | 8731374 | 8731377 | 8733495 | 8733497 | 8736435 | 8736438 | 8743357 | 8743359 | 8743360 | 8743362 | 8743363 | 8743364 | 8743365 | 8743366 | 8759773 | 8763785 | 8763788 | 8763790 | 8763791 | 8763794 | 8763795 | 8763796 | 8763797 | 8763799 | 8777369 | 8777370 | 8777373 | 8777386 | 8777388 | 8777390 | 8777844 | 8788555 | 8788558 | 8788560 | 8788562 | 8788567 | 8788569 | 8788571 | 8788574 | 8788575 | 8788576 | 8832151 | 8832152 | 8832153 | 8832154 | 8832155 | 8832156 | 8832157 | 8832158 | 8832159 | 8832167 | 8832450 | 8866279 | 8866280 | 9053609