<commit_before>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/sync/syncable/directory_backing_store.h"

#include "build/build_config.h"

#if defined(OS_MACOSX)
#include <CoreFoundation/CoreFoundation.h>
#endif

#include <limits>

#include "base/hash_tables.h"
#include "base/logging.h"
#include "chrome/browser/sync/protocol/bookmark_specifics.pb.h"
#include "chrome/browser/sync/protocol/service_constants.h"
#include "chrome/browser/sync/protocol/sync.pb.h"
#include "chrome/browser/sync/syncable/syncable-inl.h"
#include "chrome/browser/sync/syncable/syncable_columns.h"
#include "chrome/browser/sync/util/crypto_helpers.h"
#include "chrome/common/sqlite_utils.h"
#include "third_party/sqlite/preprocessed/sqlite3.h"

// Sometimes threads contend on the DB lock itself, especially when one thread
// is calling SaveChanges.  In the worst case scenario, the user can put his
// laptop to sleep during db contention, and wake up the laptop days later, so
// infinity seems like the best choice here.
const int kDirectoryBackingStoreBusyTimeoutMs = std::numeric_limits<int>::max();

using std::string;

namespace syncable {

// This just has to be big enough to hold an UPDATE or INSERT statement that
// modifies all the columns in the entry table.
static const string::size_type kUpdateStatementBufferSize = 2048;

// Increment this version whenever updating DB tables.
extern const int32 kCurrentDBVersion = 69;  // Extern only for our unittest.

namespace {

int ExecQuery(sqlite3* dbhandle, const char* query) {
  SQLStatement statement;
  int result = statement.prepare(dbhandle, query);
  if (SQLITE_OK != result)
    return result;
  do {
    result = statement.step();
  } while (SQLITE_ROW == result);

  return result;
}

string GenerateCacheGUID() {
  return Generate128BitRandomHexString();
}

}  // namespace


// Iterate over the fields of |entry| and bind each to |statement| for
// updating.  Returns the number of args bound.
int BindFields(const EntryKernel& entry, SQLStatement* statement) {
  int index = 0;
  int i = 0;
  for (i = BEGIN_FIELDS; i < INT64_FIELDS_END; ++i) {
    statement->bind_int64(index++, entry.ref(static_cast<Int64Field>(i)));
  }
  for ( ; i < ID_FIELDS_END; ++i) {
    statement->bind_string(index++, entry.ref(static_cast<IdField>(i)).s_);
  }
  for ( ; i < BIT_FIELDS_END; ++i) {
    statement->bind_int(index++, entry.ref(static_cast<BitField>(i)));
  }
  for ( ; i < STRING_FIELDS_END; ++i) {
    statement->bind_string(index++, entry.ref(static_cast<StringField>(i)));
  }
  std::string temp;
  for ( ; i < PROTO_FIELDS_END; ++i) {
    entry.ref(static_cast<ProtoField>(i)).SerializeToString(&temp);
    statement->bind_blob(index++, temp.data(), temp.length());
  }
  return index;
}

// The caller owns the returned EntryKernel*.
EntryKernel* UnpackEntry(SQLStatement* statement) {
  EntryKernel* result = NULL;
  int query_result = statement->step();
  if (SQLITE_ROW == query_result) {
    result = new EntryKernel;
    result->clear_dirty();
    CHECK(statement->column_count() == static_cast<int>(FIELD_COUNT));
    int i = 0;
    for (i = BEGIN_FIELDS; i < INT64_FIELDS_END; ++i) {
      result->put(static_cast<Int64Field>(i), statement->column_int64(i));
    }
    for ( ; i < ID_FIELDS_END; ++i) {
      result->mutable_ref(static_cast<IdField>(i)).s_ =
          statement->column_string(i);
    }
    for ( ; i < BIT_FIELDS_END; ++i) {
      result->put(static_cast<BitField>(i), (0 != statement->column_int(i)));
    }
    for ( ; i < STRING_FIELDS_END; ++i) {
      result->put(static_cast<StringField>(i),
          statement->column_string(i));
    }
    for ( ; i < PROTO_FIELDS_END; ++i) {
      result->mutable_ref(static_cast<ProtoField>(i)).ParseFromArray(
          statement->column_blob(i), statement->column_bytes(i));
    }
    ZeroFields(result, i);
  } else {
    CHECK(SQLITE_DONE == query_result);
    result = NULL;
  }
  return result;
}

namespace {

string ComposeCreateTableColumnSpecs() {
  const ColumnSpec* begin = g_metas_columns;
  const ColumnSpec* end = g_metas_columns + arraysize(g_metas_columns);
  string query;
  query.reserve(kUpdateStatementBufferSize);
  char separator = '(';
  for (const ColumnSpec* column = begin; column != end; ++column) {
    query.push_back(separator);
    separator = ',';
    query.append(column->name);
    query.push_back(' ');
    query.append(column->spec);
  }
  query.push_back(')');
  return query;
}

void AppendColumnList(std::string* output) {
  const char* joiner = " ";
  // Be explicit in SELECT order to match up with UnpackEntry.
  for (int i = BEGIN_FIELDS; i < BEGIN_FIELDS + FIELD_COUNT; ++i) {
    output->append(joiner);
    output->append(ColumnName(i));
    joiner = ", ";
  }
}

}  // namespace

///////////////////////////////////////////////////////////////////////////////
// DirectoryBackingStore implementation.

DirectoryBackingStore::DirectoryBackingStore(const string& dir_name,
                                             const FilePath& backing_filepath)
    : load_dbhandle_(NULL),
      save_dbhandle_(NULL),
      dir_name_(dir_name),
      backing_filepath_(backing_filepath),
      needs_column_refresh_(false) {
}

DirectoryBackingStore::~DirectoryBackingStore() {
  if (NULL != load_dbhandle_) {
    sqlite3_close(load_dbhandle_);
    load_dbhandle_ = NULL;
  }
  if (NULL != save_dbhandle_) {
    sqlite3_close(save_dbhandle_);
    save_dbhandle_ = NULL;
  }
}

bool DirectoryBackingStore::OpenAndConfigureHandleHelper(
    sqlite3** handle) const {
  if (SQLITE_OK == OpenSqliteDb(backing_filepath_, handle)) {
    sqlite3_busy_timeout(*handle, std::numeric_limits<int>::max());
    {
      SQLStatement statement;
      statement.prepare(*handle, "PRAGMA fullfsync = 1");
      if (SQLITE_DONE != statement.step()) {
        LOG(FATAL) << sqlite3_errmsg(*handle);
      }
    }
    {
      SQLStatement statement;
      statement.prepare(*handle, "PRAGMA synchronous = 2");
      if (SQLITE_DONE != statement.step()) {
        LOG(FATAL) << sqlite3_errmsg(*handle);
      }
    }
    sqlite3_busy_timeout(*handle, kDirectoryBackingStoreBusyTimeoutMs);
#if defined(OS_WIN)
    // Do not index this file. Scanning can occur every time we close the file,
    // which causes long delays in SQLite's file locking.
    const DWORD attrs = GetFileAttributes(backing_filepath_.value().c_str());
    const BOOL attrs_set =
      SetFileAttributes(backing_filepath_.value().c_str(),
                        attrs | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED);
#endif

    return true;
  }
  return false;
}

DirOpenResult DirectoryBackingStore::Load(MetahandlesIndex* entry_bucket,
    ExtendedAttributes* xattrs_bucket,
    Directory::KernelLoadInfo* kernel_load_info) {
  if (!BeginLoad())
    return FAILED_OPEN_DATABASE;

  DirOpenResult result = InitializeTables();
  if (OPENED != result)
    return result;

  DropDeletedEntries();
  LoadEntries(entry_bucket);
  LoadExtendedAttributes(xattrs_bucket);
  LoadInfo(kernel_load_info);

  EndLoad();
  return OPENED;
}

bool DirectoryBackingStore::BeginLoad() {
  DCHECK(load_dbhandle_ == NULL);
  return OpenAndConfigureHandleHelper(&load_dbhandle_);
}

void DirectoryBackingStore::EndLoad() {
  sqlite3_close(load_dbhandle_);
  load_dbhandle_ = NULL;  // No longer used.
}

bool DirectoryBackingStore::SaveChanges(
    const Directory::SaveChangesSnapshot& snapshot) {
  sqlite3* dbhandle = LazyGetSaveHandle();

  SQLTransaction transaction(dbhandle);
  if (SQLITE_OK != transaction.BeginExclusive())
    return false;

  for (OriginalEntries::const_iterator i = snapshot.dirty_metas.begin();
       i != snapshot.dirty_metas.end(); ++i) {
    DCHECK(i->is_dirty());
    if (!SaveEntryToDB(*i))
      return false;
  }

  for (ExtendedAttributes::const_iterator i = snapshot.dirty_xattrs.begin();
       i != snapshot.dirty_xattrs.end(); ++i) {
    DCHECK(i->second.dirty);
    if (i->second.is_deleted) {
      if (!DeleteExtendedAttributeFromDB(i))
        return false;
    } else {
      if (!SaveExtendedAttributeToDB(i))
        return false;
    }
  }

  if (Directory::KERNEL_SHARE_INFO_DIRTY == snapshot.kernel_info_status) {
    const Directory::PersistedKernelInfo& info = snapshot.kernel_info;
    SQLStatement update;
    update.prepare(dbhandle, "UPDATE share_info "
                   "SET last_sync_timestamp = ?, initial_sync_ended = ?, "
                   "store_birthday = ?, "
                   "next_id = ?");
    update.bind_int64(0, info.last_sync_timestamp);
    update.bind_bool(1, info.initial_sync_ended);
    update.bind_string(2, info.store_birthday);
    update.bind_int64(3, info.next_id);
<commit_msg>Don't acquire a database lock in syncable::DirectoryBackingStore::SaveChanges if there is nothing to save.<commit_after>// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/sync/syncable/directory_backing_store.h"

#include "build/build_config.h"

#if defined(OS_MACOSX)
#include <CoreFoundation/CoreFoundation.h>
#endif

#include <limits>

#include "base/hash_tables.h"
#include "base/logging.h"
#include "chrome/browser/sync/protocol/bookmark_specifics.pb.h"
#include "chrome/browser/sync/protocol/service_constants.h"
#include "chrome/browser/sync/protocol/sync.pb.h"
#include "chrome/browser/sync/syncable/syncable-inl.h"
#include "chrome/browser/sync/syncable/syncable_columns.h"
#include "chrome/browser/sync/util/crypto_helpers.h"
#include "chrome/common/sqlite_utils.h"
#include "third_party/sqlite/preprocessed/sqlite3.h"

// Sometimes threads contend on the DB lock itself, especially when one thread
// is calling SaveChanges.  In the worst case scenario, the user can put his
// laptop to sleep during db contention, and wake up the laptop days later, so
// infinity seems like the best choice here.
const int kDirectoryBackingStoreBusyTimeoutMs = std::numeric_limits<int>::max();

using std::string;

namespace syncable {

// This just has to be big enough to hold an UPDATE or INSERT statement that
// modifies all the columns in the entry table.
static const string::size_type kUpdateStatementBufferSize = 2048;

// Increment this version whenever updating DB tables.
extern const int32 kCurrentDBVersion = 69;  // Extern only for our unittest.

namespace {

int ExecQuery(sqlite3* dbhandle, const char* query) {
  SQLStatement statement;
  int result = statement.prepare(dbhandle, query);
  if (SQLITE_OK != result)
    return result;
  do {
    result = statement.step();
  } while (SQLITE_ROW == result);

  return result;
}

string GenerateCacheGUID() {
  return Generate128BitRandomHexString();
}

}  // namespace


// Iterate over the fields of |entry| and bind each to |statement| for
// updating.  Returns the number of args bound.
int BindFields(const EntryKernel& entry, SQLStatement* statement) {
  int index = 0;
  int i = 0;
  for (i = BEGIN_FIELDS; i < INT64_FIELDS_END; ++i) {
    statement->bind_int64(index++, entry.ref(static_cast<Int64Field>(i)));
  }
  for ( ; i < ID_FIELDS_END; ++i) {
    statement->bind_string(index++, entry.ref(static_cast<IdField>(i)).s_);
  }
  for ( ; i < BIT_FIELDS_END; ++i) {
    statement->bind_int(index++, entry.ref(static_cast<BitField>(i)));
  }
  for ( ; i < STRING_FIELDS_END; ++i) {
    statement->bind_string(index++, entry.ref(static_cast<StringField>(i)));
  }
  std::string temp;
  for ( ; i < PROTO_FIELDS_END; ++i) {
    entry.ref(static_cast<ProtoField>(i)).SerializeToString(&temp);
    statement->bind_blob(index++, temp.data(), temp.length());
  }
  return index;
}

// The caller owns the returned EntryKernel*.
EntryKernel* UnpackEntry(SQLStatement* statement) {
  EntryKernel* result = NULL;
  int query_result = statement->step();
  if (SQLITE_ROW == query_result) {
    result = new EntryKernel;
    result->clear_dirty();
    CHECK(statement->column_count() == static_cast<int>(FIELD_COUNT));
    int i = 0;
    for (i = BEGIN_FIELDS; i < INT64_FIELDS_END; ++i) {
      result->put(static_cast<Int64Field>(i), statement->column_int64(i));
    }
    for ( ; i < ID_FIELDS_END; ++i) {
      result->mutable_ref(static_cast<IdField>(i)).s_ =
          statement->column_string(i);
    }
    for ( ; i < BIT_FIELDS_END; ++i) {
      result->put(static_cast<BitField>(i), (0 != statement->column_int(i)));
    }
    for ( ; i < STRING_FIELDS_END; ++i) {
      result->put(static_cast<StringField>(i),
          statement->column_string(i));
    }
    for ( ; i < PROTO_FIELDS_END; ++i) {
      result->mutable_ref(static_cast<ProtoField>(i)).ParseFromArray(
          statement->column_blob(i), statement->column_bytes(i));
    }
    ZeroFields(result, i);
  } else {
    CHECK(SQLITE_DONE == query_result);
    result = NULL;
  }
  return result;
}

namespace {

string ComposeCreateTableColumnSpecs() {
  const ColumnSpec* begin = g_metas_columns;
  const ColumnSpec* end = g_metas_columns + arraysize(g_metas_columns);
  string query;
  query.reserve(kUpdateStatementBufferSize);
  char separator = '(';
  for (const ColumnSpec* column = begin; column != end; ++column) {
    query.push_back(separator);
    separator = ',';
    query.append(column->name);
    query.push_back(' ');
    query.append(column->spec);
  }
  query.push_back(')');
  return query;
}

void AppendColumnList(std::string* output) {
  const char* joiner = " ";
  // Be explicit in SELECT order to match up with UnpackEntry.
  for (int i = BEGIN_FIELDS; i < BEGIN_FIELDS + FIELD_COUNT; ++i) {
    output->append(joiner);
    output->append(ColumnName(i));
    joiner = ", ";
  }
}

}  // namespace

///////////////////////////////////////////////////////////////////////////////
// DirectoryBackingStore implementation.

DirectoryBackingStore::DirectoryBackingStore(const string& dir_name,
                                             const FilePath& backing_filepath)
    : load_dbhandle_(NULL),
      save_dbhandle_(NULL),
      dir_name_(dir_name),
      backing_filepath_(backing_filepath),
      needs_column_refresh_(false) {
}

DirectoryBackingStore::~DirectoryBackingStore() {
  if (NULL != load_dbhandle_) {
    sqlite3_close(load_dbhandle_);
    load_dbhandle_ = NULL;
  }
  if (NULL != save_dbhandle_) {
    sqlite3_close(save_dbhandle_);
    save_dbhandle_ = NULL;
  }
}

bool DirectoryBackingStore::OpenAndConfigureHandleHelper(
    sqlite3** handle) const {
  if (SQLITE_OK == OpenSqliteDb(backing_filepath_, handle)) {
    sqlite3_busy_timeout(*handle, std::numeric_limits<int>::max());
    {
      SQLStatement statement;
      statement.prepare(*handle, "PRAGMA fullfsync = 1");
      if (SQLITE_DONE != statement.step()) {
        LOG(FATAL) << sqlite3_errmsg(*handle);
      }
    }
    {
      SQLStatement statement;
      statement.prepare(*handle, "PRAGMA synchronous = 2");
      if (SQLITE_DONE != statement.step()) {
        LOG(FATAL) << sqlite3_errmsg(*handle);
      }
    }
    sqlite3_busy_timeout(*handle, kDirectoryBackingStoreBusyTimeoutMs);
#if defined(OS_WIN)
    // Do not index this file. Scanning can occur every time we close the file,
    // which causes long delays in SQLite's file locking.
    const DWORD attrs = GetFileAttributes(backing_filepath_.value().c_str());
    const BOOL attrs_set =
      SetFileAttributes(backing_filepath_.value().c_str(),
                        attrs | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED);
#endif

    return true;
  }
  return false;
}

DirOpenResult DirectoryBackingStore::Load(MetahandlesIndex* entry_bucket,
    ExtendedAttributes* xattrs_bucket,
    Directory::KernelLoadInfo* kernel_load_info) {
  if (!BeginLoad())
    return FAILED_OPEN_DATABASE;

  DirOpenResult result = InitializeTables();
  if (OPENED != result)
    return result;

  DropDeletedEntries();
  LoadEntries(entry_bucket);
  LoadExtendedAttributes(xattrs_bucket);
  LoadInfo(kernel_load_info);

  EndLoad();
  return OPENED;
}

bool DirectoryBackingStore::BeginLoad() {
  DCHECK(load_dbhandle_ == NULL);
  return OpenAndConfigureHandleHelper(&load_dbhandle_);
}

void DirectoryBackingStore::EndLoad() {
  sqlite3_close(load_dbhandle_);
  load_dbhandle_ = NULL;  // No longer used.
}

bool DirectoryBackingStore::SaveChanges(
    const Directory::SaveChangesSnapshot& snapshot) {
  sqlite3* dbhandle = LazyGetSaveHandle();

  // SQLTransaction::BeginExclusive causes a disk write to occur. This is not
  // something that should happen every 10 seconds when this function runs, so
  // just stop here if there's nothing to save.
  bool save_info =
    (Directory::KERNEL_SHARE_INFO_DIRTY == snapshot.kernel_info_status);
  if (snapshot.dirty_metas.size() < 1 && snapshot.dirty_xattrs.size() < 1 &&
      !save_info)
    return true;

  SQLTransaction transaction(dbhandle);
  if (SQLITE_OK != transaction.BeginExclusive())
    return false;

  for (OriginalEntries::const_iterator i = snapshot.dirty_metas.begin();
       i != snapshot.dirty_metas.end(); ++i) {
    DCHECK(i->is_dirty());
    if (!SaveEntryToDB(*i))
      return false;
  }

  for (ExtendedAttributes::const_iterator i = snapshot.dirty_xattrs.begin();
       i != snapshot.dirty_xattrs.end(); ++i) {
    DCHECK(i->second.dirty);
    if (i->second.is_deleted) {
      if (!DeleteExtendedAttributeFromDB(i))
        return false;
    } else {
      if (!SaveExtendedAttributeToDB(i))
        return false;
    }
  }

  if (save_info) {
    const Directory::PersistedKernelInfo& info = snapshot.kernel_info;
    SQLStatement update;
    update.prepare(dbhandle, "UPDATE share_info "
                   "SET last_sync_timestamp = ?, initial_sync_ended = ?, "
                   "store_birthday = ?, "
                   "next_id = ?");
    update.bind_int64(0, info.last_sync_timestamp);
    update.bind_bool(1, info.initial_sync_ended);
    update.bind_string(2, info.store_birthday);
    update.bind_int64(3, info.next_id);

{'fraction_non_alphanumeric': 0.13242388953585094, 'fraction_numerical': 0.005989020129762102, 'mean_word_length': 3.7234154007333684, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6610365', 'n_tokens_mistral': 5878, 'n_tokens_neox': 5739, 'n_words': 1255}