<commit_before>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/safe_browsing/safe_browsing_database.h"

#include "base/command_line.h"
#include "base/file_util.h"
#include "base/histogram.h"
#include "base/time.h"
#include "base/message_loop.h"
#include "base/process_util.h"
#include "base/sha2.h"
#include "base/stats_counters.h"
#include "chrome/browser/safe_browsing/bloom_filter.h"
#include "chrome/browser/safe_browsing/safe_browsing_database_bloom.h"
#include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
#include "chrome/browser/safe_browsing/safe_browsing_store_sqlite.h"
#include "chrome/common/chrome_switches.h"
#include "googleurl/src/gurl.h"

namespace {

// Filename suffix for the bloom filter.
const FilePath::CharType kBloomFilterFile[] = FILE_PATH_LITERAL(" Filter 2");

// The maximum staleness for a cached entry.
const int kMaxStalenessMinutes = 45;

// To save space, the incoming |chunk_id| and |list_id| are combined
// into an |encoded_chunk_id| for storage by shifting the |list_id|
// into the low-order bits.  These functions decode that information.
int DecodeListId(const int encoded_chunk_id) {
  return encoded_chunk_id & 1;
}
int DecodeChunkId(int encoded_chunk_id) {
  return encoded_chunk_id >> 1;
}
int EncodeChunkId(int chunk, int list_id) {
  DCHECK(list_id == 0 || list_id == 1);
  return chunk << 1 | list_id;
}

// Generate the set of prefixes to check for |url|.
// TODO(shess): This function is almost the same as
// |CompareFullHashes()| in safe_browsing_util.cc, except that code
// does an early exit on match.  Since match should be the infrequent
// case (phishing or malware found), consider combining this function
// with that one.
void PrefixesToCheck(const GURL& url, std::vector<SBPrefix>* prefixes) {
  std::vector<std::string> hosts;
  if (url.HostIsIPAddress()) {
    hosts.push_back(url.host());
  } else {
    safe_browsing_util::GenerateHostsToCheck(url, &hosts);
  }

  std::vector<std::string> paths;
  safe_browsing_util::GeneratePathsToCheck(url, &paths);

  for (size_t i = 0; i < hosts.size(); ++i) {
    for (size_t j = 0; j < paths.size(); ++j) {
      SBFullHash full_hash;
      base::SHA256HashString(hosts[i] + paths[j], &full_hash,
                             sizeof(SBFullHash));
      prefixes->push_back(full_hash.prefix);
    }
  }
}

// Find the entries in |full_hashes| with prefix in |prefix_hits|, and
// add them to |full_hits| if not expired.  "Not expired" is when
// either |last_update| was recent enough, or the item has been
// received recently enough.  Expired items are not deleted because a
// future update may make them acceptable again.
//
// For efficiency reasons the code walks |prefix_hits| and
// |full_hashes| in parallel, so they must be sorted by prefix.
void GetCachedFullHashes(const std::vector<SBPrefix>& prefix_hits,
                         const std::vector<SBAddFullHash>& full_hashes,
                         std::vector<SBFullHashResult>* full_hits,
                         base::Time last_update) {
  const base::Time expire_time =
      base::Time::Now() - base::TimeDelta::FromMinutes(kMaxStalenessMinutes);

  std::vector<SBPrefix>::const_iterator piter = prefix_hits.begin();
  std::vector<SBAddFullHash>::const_iterator hiter = full_hashes.begin();

  while (piter != prefix_hits.end() && hiter != full_hashes.end()) {
    if (*piter < hiter->full_hash.prefix) {
      ++piter;
    } else if (hiter->full_hash.prefix < *piter) {
      ++hiter;
    } else {
      if (expire_time < last_update ||
          expire_time.ToTimeT() < hiter->received) {
        SBFullHashResult result;
        const int list_id = DecodeListId(hiter->chunk_id);
        result.list_name = safe_browsing_util::GetListName(list_id);
        result.add_chunk_id = DecodeChunkId(hiter->chunk_id);
        result.hash = hiter->full_hash;
        full_hits->push_back(result);
      }

      // Only increment |hiter|, |piter| might have multiple hits.
      ++hiter;
    }
  }
}

// Helper for |UpdateStarted()|.  Separates |chunks| into malware and
// phishing vectors, and converts the results into range strings.
void GetChunkIds(const std::vector<int>& chunks,
                 std::string* malware_list, std::string* phishing_list) {
  std::vector<int> malware_chunks;
  std::vector<int> phishing_chunks;

  for (std::vector<int>::const_iterator iter = chunks.begin();
       iter != chunks.end(); ++iter) {
    if (safe_browsing_util::MALWARE == DecodeListId(*iter)) {
      malware_chunks.push_back(DecodeChunkId(*iter));
    } else if (safe_browsing_util::PHISH == DecodeListId(*iter)) {
      phishing_chunks.push_back(DecodeChunkId(*iter));
    } else {
      NOTREACHED();
    }
  }

  std::sort(malware_chunks.begin(), malware_chunks.end());
  std::vector<ChunkRange> malware_ranges;
  ChunksToRanges(malware_chunks, &malware_ranges);
  RangesToString(malware_ranges, malware_list);

  std::sort(phishing_chunks.begin(), phishing_chunks.end());
  std::vector<ChunkRange> phishing_ranges;
  ChunksToRanges(phishing_chunks, &phishing_ranges);
  RangesToString(phishing_ranges, phishing_list);
}

// Order |SBAddFullHash| on the prefix part.  |SBAddPrefixLess()| from
// safe_browsing_store.h orders on both chunk-id and prefix.
bool SBAddFullHashPrefixLess(const SBAddFullHash& a, const SBAddFullHash& b) {
  return a.full_hash.prefix < b.full_hash.prefix;
}

}  // namespace

// Factory method.
// TODO(shess): Proposed staging of the rolling:
// - Ship "old" to dev channel to provide a safe fallback.
// - If that proves stable, change to "newsqlite".  This changes the
//   code which manipulates the data, without changing the data
//   format.  At this point all changes could be reverted without
//   having to resync everyone's database from scratch.
// - If SafeBrowsingDatabaseNew proves stable, change the default to
//   "newfile", which will change the file format.  Changing back
//   would require resync from scratch.
// - Once enough users are converted to "newfile", remove all of the
//   redundent indirection classes and functions, perhaps leaving
//   SafeBrowsingStoreSqlite for on-the-fly conversions.
// - Once there are few remaining SQLite-format users, remove
//   SafeBrowsingStoreSqlite.  Remaining users will resync their
//   safe-browsing database from scratch.  If users haven't sync'ed
//   their database in months, this probably won't be more expensive
//   than an incremental sync.
SafeBrowsingDatabase* SafeBrowsingDatabase::Create() {
  const CommandLine& command_line = *CommandLine::ForCurrentProcess();
  std::string value =
      command_line.GetSwitchValueASCII(switches::kSafeBrowsingDatabaseStore);
  if (!value.compare("newfile")) {
    return new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile);
  } else if (!value.compare("newsqlite")) {
    return new SafeBrowsingDatabaseNew(new SafeBrowsingStoreSqlite);
  } else if (!value.compare("old")) {
    return new SafeBrowsingDatabaseBloom;
  } else {
    // Default.
    DCHECK(value.empty());
    return new SafeBrowsingDatabaseNew(new SafeBrowsingStoreSqlite);
  }
}

SafeBrowsingDatabase::~SafeBrowsingDatabase() {
}

// static
FilePath SafeBrowsingDatabase::BloomFilterForFilename(
    const FilePath& db_filename) {
  return FilePath(db_filename.value() + kBloomFilterFile);
}

SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew(SafeBrowsingStore* store)
    : creation_loop_(MessageLoop::current()),
      store_(store),
      ALLOW_THIS_IN_INITIALIZER_LIST(reset_factory_(this)) {
  DCHECK(store_.get());
}

SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew()
    : creation_loop_(MessageLoop::current()),
      store_(new SafeBrowsingStoreSqlite),
      ALLOW_THIS_IN_INITIALIZER_LIST(reset_factory_(this)) {
  DCHECK(store_.get());
}

SafeBrowsingDatabaseNew::~SafeBrowsingDatabaseNew() {
  DCHECK_EQ(creation_loop_, MessageLoop::current());
}

void SafeBrowsingDatabaseNew::Init(const FilePath& filename) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  // NOTE: There is no need to grab the lock in this function, since
  // until it returns, there are no pointers to this class on other
  // threads.  Then again, that means there is no possibility of
  // contention on the lock...
  AutoLock locked(lookup_lock_);

  DCHECK(filename_.empty());  // Ensure we haven't been run before.

  filename_ = filename;
  store_->Init(
      filename_,
      NewCallback(this, &SafeBrowsingDatabaseNew::HandleCorruptDatabase));

  full_hashes_.clear();
  pending_hashes_.clear();

  bloom_filter_filename_ = BloomFilterForFilename(filename_);
  LoadBloomFilter();
}

bool SafeBrowsingDatabaseNew::ResetDatabase() {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  // Delete files on disk.
  // TODO(shess): Hard to see where one might want to delete without a
  // reset.  Perhaps inline |Delete()|?
  if (!Delete())
    return false;

  // Reset objects in memory.
  {
    AutoLock locked(lookup_lock_);
    full_hashes_.clear();
    pending_hashes_.clear();
    prefix_miss_cache_.clear();
    // TODO(shess): This could probably be |bloom_filter_.reset()|.
    bloom_filter_ = new BloomFilter(BloomFilter::kBloomFilterMinSize *
                                    BloomFilter::kBloomFilterSizeRatio);
  }

  return true;
}

bool SafeBrowsingDatabaseNew::ContainsUrl(
    const GURL& url,
    std::string* matching_list,
    std::vector<SBPrefix>* prefix_hits,
    std::vector<SBFullHashResult>* full_hits,
    base::Time last_update) {
  // Clear the results first.
  matching_list->clear();
  prefix_hits->clear();
  full_hits->clear();

  std::vector<SBPrefix> prefixes;
  PrefixesToCheck(url, &prefixes);
  if (prefixes.empty())
    return false;

  // Prevent changes to bloom filter and caches.
  AutoLock locked(lookup_lock_);

  if (!bloom_filter_.get())
    return false;

  // TODO(erikkay): Not filling in matching_list - is that OK?
  size_t miss_count = 0;
  for (size_t i = 0; i < prefixes.size(); ++i) {
    if (bloom_filter_->Exists(prefixes[i])) {
      prefix_hits->push_back(prefixes[i]);
      if (prefix_miss_cache_.count(prefixes[i]) > 0)
        ++miss_count;
    }
  }

  // If all the prefixes are cached as 'misses', don't issue a GetHash.
  if (miss_count == prefix_hits->size())
    return false;

  // Find the matching full-hash results.  |full_hashes_| are from the
  // database, |pending_hashes_| are from GetHash requests between
  // updates.
  std::sort(prefix_hits->begin(), prefix_hits->end());
  GetCachedFullHashes(*prefix_hits, full_hashes_, full_hits, last_update);
  GetCachedFullHashes(*prefix_hits, pending_hashes_, full_hits, last_update);
  return true;
}

// Helper to insert entries for all of the prefixes or full hashes in
// |entry| into the store.
void SafeBrowsingDatabaseNew::InsertAdd(int chunk_id, SBPrefix host,
                                        const SBEntry* entry, int list_id) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  STATS_COUNTER("SB.HostInsert", 1);
  const int encoded_chunk_id = EncodeChunkId(chunk_id, list_id);
  const int count = entry->prefix_count();

  DCHECK(!entry->IsSub());
  if (!count) {
    // No prefixes, use host instead.
    STATS_COUNTER("SB.PrefixAdd", 1);
    store_->WriteAddPrefix(encoded_chunk_id, host);
  } else if (entry->IsPrefix()) {
    // Prefixes only.
    for (int i = 0; i < count; i++) {
      const SBPrefix prefix = entry->PrefixAt(i);
      STATS_COUNTER("SB.PrefixAdd", 1);
      store_->WriteAddPrefix(encoded_chunk_id, prefix);
    }
  } else {
    // Prefixes and hashes.
    const base::Time receive_time = base::Time::Now();
    for (int i = 0; i < count; ++i) {
      const SBFullHash full_hash = entry->FullHashAt(i);
      const SBPrefix prefix = full_hash.prefix;

      STATS_COUNTER("SB.PrefixAdd", 1);
      store_->WriteAddPrefix(encoded_chunk_id, prefix);

      STATS_COUNTER("SB.PrefixAddFull", 1);
      store_->WriteAddHash(encoded_chunk_id, receive_time, full_hash);
    }
  }
}

// Helper to iterate over all the entries in the hosts in |chunks| and
// add them to the store.
void SafeBrowsingDatabaseNew::InsertAddChunks(int list_id,
                                              const SBChunkList& chunks) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());
  for (SBChunkList::const_iterator citer = chunks.begin();
       citer != chunks.end(); ++citer) {
    const int chunk_id = citer->chunk_number;

    // The server can give us a chunk that we already have because
    // it's part of a range.  Don't add it again.
    const int encoded_chunk_id = EncodeChunkId(chunk_id, list_id);
    if (store_->CheckAddChunk(encoded_chunk_id))
      continue;

    store_->SetAddChunk(encoded_chunk_id);
    for (std::deque<SBChunkHost>::const_iterator hiter = citer->hosts.begin();
         hiter != citer->hosts.end(); ++hiter) {
      // NOTE: Could pass |encoded_chunk_id|, but then inserting add
      // chunks would look different from inserting sub chunks.
      InsertAdd(chunk_id, hiter->host, hiter->entry, list_id);
    }
  }
}

// Helper to insert entries for all of the prefixes or full hashes in
// |entry| into the store.
void SafeBrowsingDatabaseNew::InsertSub(int chunk_id, SBPrefix host,
                                        const SBEntry* entry, int list_id) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  STATS_COUNTER("SB.HostDelete", 1);
  const int encoded_chunk_id = EncodeChunkId(chunk_id, list_id);
  const int count = entry->prefix_count();

  DCHECK(entry->IsSub());
  if (!count) {
    // No prefixes, use host instead.
    STATS_COUNTER("SB.PrefixSub", 1);
    const int add_chunk_id = EncodeChunkId(entry->chunk_id(), list_id);
    store_->WriteSubPrefix(encoded_chunk_id, add_chunk_id, host);
  } else if (entry->IsPrefix()) {
    // Prefixes only.
    for (int i = 0; i < count; i++) {
      const SBPrefix prefix = entry->PrefixAt(i);
      const int add_chunk_id =
          EncodeChunkId(entry->ChunkIdAtPrefix(i), list_id);

      STATS_COUNTER("SB.PrefixSub", 1);
      store_->WriteSubPrefix(encoded_chunk_id, add_chunk_id, prefix);
    }
  } else {
    // Prefixes and hashes.
    for (int i = 0; i < count; ++i) {
      const SBFullHash full_hash = entry->FullHashAt(i);
      const int add_chunk_id =
          EncodeChunkId(entry->ChunkIdAtPrefix(i), list_id);

      STATS_COUNTER("SB.PrefixSub", 1);
      store_->WriteSubPrefix(encoded_chunk_id, add_chunk_id, full_hash.prefix);

      STATS_COUNTER("SB.PrefixSubFull", 1);
      store_->WriteSubHash(encoded_chunk_id, add_chunk_id, full_hash);
    }
  }
}

// Helper to iterate over all the entries in the hosts in |chunks| and
// add them to the store.
void SafeBrowsingDatabaseNew::InsertSubChunks(int list_id,
                                              const SBChunkList& chunks) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());
  for (SBChunkList::const_iterator citer = chunks.begin();
       citer != chunks.end(); ++citer) {
    const int chunk_id = citer->chunk_number;

    // The server can give us a chunk that we already have because
    // it's part of a range.  Don't add it again.
    const int encoded_chunk_id = EncodeChunkId(chunk_id, list_id);
    if (store_->CheckSubChunk(encoded_chunk_id))
      continue;

    store_->SetSubChunk(encoded_chunk_id);
    for (std::deque<SBChunkHost>::const_iterator hiter = citer->hosts.begin();
         hiter != citer->hosts.end(); ++hiter) {
      InsertSub(chunk_id, hiter->host, hiter->entry, list_id);
    }
  }
}

void SafeBrowsingDatabaseNew::InsertChunks(const std::string& list_name,
                                           const SBChunkList& chunks) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  if (chunks.empty())
    return;

  const base::Time insert_start = base::Time::Now();

  const int list_id = safe_browsing_util::GetListId(list_name);
  store_->BeginChunk();
  if (chunks.front().is_add) {
    InsertAddChunks(list_id, chunks);
  } else {
    InsertSubChunks(list_id, chunks);
  }
  store_->FinishChunk();

  UMA_HISTOGRAM_TIMES("SB2.ChunkInsert", base::Time::Now() - insert_start);
}

void SafeBrowsingDatabaseNew::DeleteChunks(
    const std::vector<SBChunkDelete>& chunk_deletes) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  if (chunk_deletes.empty())
    return;

  const std::string& list_name = chunk_deletes.front().list_name;
  const int list_id = safe_browsing_util::GetListId(list_name);

  for (size_t i = 0; i < chunk_deletes.size(); ++i) {
    std::vector<int> chunk_numbers;
    RangesToChunks(chunk_deletes[i].chunk_del, &chunk_numbers);
    for (size_t j = 0; j < chunk_numbers.size(); ++j) {
      const int encoded_chunk_id = EncodeChunkId(chunk_numbers[j], list_id);
      if (chunk_deletes[i].is_sub_del)
        store_->DeleteSubChunk(encoded_chunk_id);
      else
        store_->DeleteAddChunk(encoded_chunk_id);
    }
  }
}

void SafeBrowsingDatabaseNew::CacheHashResults(
    const std::vector<SBPrefix>& prefixes,
    const std::vector<SBFullHashResult>& full_hits) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  // This is on the same thread as other updates, lock against
  // |ContainsUrl()|.
  AutoLock locked(lookup_lock_);

  if (full_hits.empty()) {
    prefix_miss_cache_.insert(prefixes.begin(), prefixes.end());
    return;
  }

  // TODO(shess): SBFullHashResult and SBAddFullHash are very similar.
  // Refactor to make them identical.
  const base::Time now = base::Time::Now();
  const size_t orig_size = pending_hashes_.size();
  for (std::vector<SBFullHashResult>::const_iterator iter = full_hits.begin();
       iter != full_hits.end(); ++iter) {
    const int list_id = safe_browsing_util::GetListId(iter->list_name);
    const int encoded_chunk_id = EncodeChunkId(iter->add_chunk_id, list_id);
    pending_hashes_.push_back(SBAddFullHash(encoded_chunk_id, now, iter->hash));
  }

  // Sort new entries then merge with the previously-sorted entries.
  std::vector<SBAddFullHash>::iterator
      orig_end = pending_hashes_.begin() + orig_size;
  std::sort(orig_end, pending_hashes_.end(), SBAddFullHashPrefixLess);
  std::inplace_merge(pending_hashes_.begin(), orig_end, pending_hashes_.end(),
                     SBAddFullHashPrefixLess);
}

bool SafeBrowsingDatabaseNew::UpdateStarted(
    std::vector<SBListChunkRanges>* lists) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());
  DCHECK(lists);

  // If |BeginUpdate()| fails, reset the database.
  if (!store_->BeginUpdate()) {
    HandleCorruptDatabase();
    return false;
  }

  SBListChunkRanges malware(safe_browsing_util::kMalwareList);
  SBListChunkRanges phishing(safe_browsing_util::kPhishingList);

  std::vector<int> add_chunks;
  store_->GetAddChunks(&add_chunks);
  GetChunkIds(add_chunks, &malware.adds, &phishing.adds);

  std::vector<int> sub_chunks;
  store_->GetSubChunks(&sub_chunks);
  GetChunkIds(sub_chunks, &malware.subs, &phishing.subs);

  lists->push_back(malware);
  lists->push_back(phishing);

  return true;
}

void SafeBrowsingDatabaseNew::UpdateFinished(bool update_succeeded) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  // Unroll any partially-received transaction.
  if (!update_succeeded) {
    store_->CancelUpdate();
    return;
  }

  // Copy out the pending add hashes.  Copy rather than swapping in
  // case |ContainsURL()| is called before the new filter is complete.
  std::vector<SBAddFullHash> pending_add_hashes;
  {
    AutoLock locked(lookup_lock_);
    pending_add_hashes.insert(pending_add_hashes.end(),
                              pending_hashes_.begin(), pending_hashes_.end());
  }

  std::vector<SBAddPrefix> add_prefixes;
  std::vector<SBAddFullHash> add_full_hashes;
  if (!store_->FinishUpdate(pending_add_hashes,
                            &add_prefixes, &add_full_hashes))
    return;

  // Measure the amount of IO during the bloom filter build.
  base::IoCounters io_before, io_after;
  base::ProcessHandle handle = base::Process::Current().handle();
  scoped_ptr<base::ProcessMetrics> metric(
#if !defined(OS_MACOSX)
      base::ProcessMetrics::CreateProcessMetrics(handle)
#else
      // Getting stats only for the current process is enough, so NULL is fine.
      base::ProcessMetrics::CreateProcessMetrics(handle, NULL)
#endif
  );

  // IoCounters are currently not supported on Mac, and may not be
  // available for Linux, so we check the result and only show IO
  // stats if they are available.
  const bool got_counters = metric->GetIOCounters(&io_before);

  const base::Time before = base::Time::Now();

  // Create and populate |filter| from |add_prefixes|.
  // TODO(shess): The bloom filter doesn't need to be a
  // scoped_refptr<> for this code.  Refactor that away.
  const int filter_size =
      BloomFilter::FilterSizeForKeyCount(add_prefixes.size());
  scoped_refptr<BloomFilter> filter(new BloomFilter(filter_size));
  for (size_t i = 0; i < add_prefixes.size(); ++i) {
    filter->Insert(add_prefixes[i].prefix);
  }

  // This needs to be in sorted order by prefix for efficient access.
  std::sort(add_full_hashes.begin(), add_full_hashes.end(),
            SBAddFullHashPrefixLess);

  // Swap in the newly built filter and cache.
  {
    AutoLock locked(lookup_lock_);
    full_hashes_.swap(add_full_hashes);
    pending_hashes_.clear();
    prefix_miss_cache_.clear();
    bloom_filter_.swap(filter);
  }

  const base::TimeDelta bloom_gen = base::Time::Now() - before;

  // Persist the bloom filter to disk.  Since only this thread changes
  // |bloom_filter_|, there is no need to lock.
  WriteBloomFilter();

  // Gather statistics.
  if (got_counters && metric->GetIOCounters(&io_after)) {
    UMA_HISTOGRAM_COUNTS("SB2.BuildReadBytes",
                         static_cast<int>(io_after.ReadTransferCount -
                                          io_before.ReadTransferCount));
    UMA_HISTOGRAM_COUNTS("SB2.BuildWriteBytes",
                         static_cast<int>(io_after.WriteTransferCount -
                                          io_before.WriteTransferCount));
    UMA_HISTOGRAM_COUNTS("SB2.BuildReadOperations",
                         static_cast<int>(io_after.ReadOperationCount -
                                          io_before.ReadOperationCount));
    UMA_HISTOGRAM_COUNTS("SB2.BuildWriteOperations",<commit_msg>Relax DCHECK in SafeBrowsingDatabaseNew::CacheHashResults().<commit_after>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/safe_browsing/safe_browsing_database.h"

#include "base/command_line.h"
#include "base/file_util.h"
#include "base/histogram.h"
#include "base/time.h"
#include "base/message_loop.h"
#include "base/process_util.h"
#include "base/sha2.h"
#include "base/stats_counters.h"
#include "chrome/browser/safe_browsing/bloom_filter.h"
#include "chrome/browser/safe_browsing/safe_browsing_database_bloom.h"
#include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
#include "chrome/browser/safe_browsing/safe_browsing_store_sqlite.h"
#include "chrome/common/chrome_switches.h"
#include "googleurl/src/gurl.h"

namespace {

// Filename suffix for the bloom filter.
const FilePath::CharType kBloomFilterFile[] = FILE_PATH_LITERAL(" Filter 2");

// The maximum staleness for a cached entry.
const int kMaxStalenessMinutes = 45;

// To save space, the incoming |chunk_id| and |list_id| are combined
// into an |encoded_chunk_id| for storage by shifting the |list_id|
// into the low-order bits.  These functions decode that information.
int DecodeListId(const int encoded_chunk_id) {
  return encoded_chunk_id & 1;
}
int DecodeChunkId(int encoded_chunk_id) {
  return encoded_chunk_id >> 1;
}
int EncodeChunkId(int chunk, int list_id) {
  DCHECK(list_id == 0 || list_id == 1);
  return chunk << 1 | list_id;
}

// Generate the set of prefixes to check for |url|.
// TODO(shess): This function is almost the same as
// |CompareFullHashes()| in safe_browsing_util.cc, except that code
// does an early exit on match.  Since match should be the infrequent
// case (phishing or malware found), consider combining this function
// with that one.
void PrefixesToCheck(const GURL& url, std::vector<SBPrefix>* prefixes) {
  std::vector<std::string> hosts;
  if (url.HostIsIPAddress()) {
    hosts.push_back(url.host());
  } else {
    safe_browsing_util::GenerateHostsToCheck(url, &hosts);
  }

  std::vector<std::string> paths;
  safe_browsing_util::GeneratePathsToCheck(url, &paths);

  for (size_t i = 0; i < hosts.size(); ++i) {
    for (size_t j = 0; j < paths.size(); ++j) {
      SBFullHash full_hash;
      base::SHA256HashString(hosts[i] + paths[j], &full_hash,
                             sizeof(SBFullHash));
      prefixes->push_back(full_hash.prefix);
    }
  }
}

// Find the entries in |full_hashes| with prefix in |prefix_hits|, and
// add them to |full_hits| if not expired.  "Not expired" is when
// either |last_update| was recent enough, or the item has been
// received recently enough.  Expired items are not deleted because a
// future update may make them acceptable again.
//
// For efficiency reasons the code walks |prefix_hits| and
// |full_hashes| in parallel, so they must be sorted by prefix.
void GetCachedFullHashes(const std::vector<SBPrefix>& prefix_hits,
                         const std::vector<SBAddFullHash>& full_hashes,
                         std::vector<SBFullHashResult>* full_hits,
                         base::Time last_update) {
  const base::Time expire_time =
      base::Time::Now() - base::TimeDelta::FromMinutes(kMaxStalenessMinutes);

  std::vector<SBPrefix>::const_iterator piter = prefix_hits.begin();
  std::vector<SBAddFullHash>::const_iterator hiter = full_hashes.begin();

  while (piter != prefix_hits.end() && hiter != full_hashes.end()) {
    if (*piter < hiter->full_hash.prefix) {
      ++piter;
    } else if (hiter->full_hash.prefix < *piter) {
      ++hiter;
    } else {
      if (expire_time < last_update ||
          expire_time.ToTimeT() < hiter->received) {
        SBFullHashResult result;
        const int list_id = DecodeListId(hiter->chunk_id);
        result.list_name = safe_browsing_util::GetListName(list_id);
        result.add_chunk_id = DecodeChunkId(hiter->chunk_id);
        result.hash = hiter->full_hash;
        full_hits->push_back(result);
      }

      // Only increment |hiter|, |piter| might have multiple hits.
      ++hiter;
    }
  }
}

// Helper for |UpdateStarted()|.  Separates |chunks| into malware and
// phishing vectors, and converts the results into range strings.
void GetChunkIds(const std::vector<int>& chunks,
                 std::string* malware_list, std::string* phishing_list) {
  std::vector<int> malware_chunks;
  std::vector<int> phishing_chunks;

  for (std::vector<int>::const_iterator iter = chunks.begin();
       iter != chunks.end(); ++iter) {
    if (safe_browsing_util::MALWARE == DecodeListId(*iter)) {
      malware_chunks.push_back(DecodeChunkId(*iter));
    } else if (safe_browsing_util::PHISH == DecodeListId(*iter)) {
      phishing_chunks.push_back(DecodeChunkId(*iter));
    } else {
      NOTREACHED();
    }
  }

  std::sort(malware_chunks.begin(), malware_chunks.end());
  std::vector<ChunkRange> malware_ranges;
  ChunksToRanges(malware_chunks, &malware_ranges);
  RangesToString(malware_ranges, malware_list);

  std::sort(phishing_chunks.begin(), phishing_chunks.end());
  std::vector<ChunkRange> phishing_ranges;
  ChunksToRanges(phishing_chunks, &phishing_ranges);
  RangesToString(phishing_ranges, phishing_list);
}

// Order |SBAddFullHash| on the prefix part.  |SBAddPrefixLess()| from
// safe_browsing_store.h orders on both chunk-id and prefix.
bool SBAddFullHashPrefixLess(const SBAddFullHash& a, const SBAddFullHash& b) {
  return a.full_hash.prefix < b.full_hash.prefix;
}

}  // namespace

// Factory method.
// TODO(shess): Proposed staging of the rolling:
// - Ship "old" to dev channel to provide a safe fallback.
// - If that proves stable, change to "newsqlite".  This changes the
//   code which manipulates the data, without changing the data
//   format.  At this point all changes could be reverted without
//   having to resync everyone's database from scratch.
// - If SafeBrowsingDatabaseNew proves stable, change the default to
//   "newfile", which will change the file format.  Changing back
//   would require resync from scratch.
// - Once enough users are converted to "newfile", remove all of the
//   redundent indirection classes and functions, perhaps leaving
//   SafeBrowsingStoreSqlite for on-the-fly conversions.
// - Once there are few remaining SQLite-format users, remove
//   SafeBrowsingStoreSqlite.  Remaining users will resync their
//   safe-browsing database from scratch.  If users haven't sync'ed
//   their database in months, this probably won't be more expensive
//   than an incremental sync.
SafeBrowsingDatabase* SafeBrowsingDatabase::Create() {
  const CommandLine& command_line = *CommandLine::ForCurrentProcess();
  std::string value =
      command_line.GetSwitchValueASCII(switches::kSafeBrowsingDatabaseStore);
  if (!value.compare("newfile")) {
    return new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile);
  } else if (!value.compare("newsqlite")) {
    return new SafeBrowsingDatabaseNew(new SafeBrowsingStoreSqlite);
  } else if (!value.compare("old")) {
    return new SafeBrowsingDatabaseBloom;
  } else {
    // Default.
    DCHECK(value.empty());
    return new SafeBrowsingDatabaseNew(new SafeBrowsingStoreSqlite);
  }
}

SafeBrowsingDatabase::~SafeBrowsingDatabase() {
}

// static
FilePath SafeBrowsingDatabase::BloomFilterForFilename(
    const FilePath& db_filename) {
  return FilePath(db_filename.value() + kBloomFilterFile);
}

SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew(SafeBrowsingStore* store)
    : creation_loop_(MessageLoop::current()),
      store_(store),
      ALLOW_THIS_IN_INITIALIZER_LIST(reset_factory_(this)) {
  DCHECK(store_.get());
}

SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew()
    : creation_loop_(MessageLoop::current()),
      store_(new SafeBrowsingStoreSqlite),
      ALLOW_THIS_IN_INITIALIZER_LIST(reset_factory_(this)) {
  DCHECK(store_.get());
}

SafeBrowsingDatabaseNew::~SafeBrowsingDatabaseNew() {
  DCHECK_EQ(creation_loop_, MessageLoop::current());
}

void SafeBrowsingDatabaseNew::Init(const FilePath& filename) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  // NOTE: There is no need to grab the lock in this function, since
  // until it returns, there are no pointers to this class on other
  // threads.  Then again, that means there is no possibility of
  // contention on the lock...
  AutoLock locked(lookup_lock_);

  DCHECK(filename_.empty());  // Ensure we haven't been run before.

  filename_ = filename;
  store_->Init(
      filename_,
      NewCallback(this, &SafeBrowsingDatabaseNew::HandleCorruptDatabase));

  full_hashes_.clear();
  pending_hashes_.clear();

  bloom_filter_filename_ = BloomFilterForFilename(filename_);
  LoadBloomFilter();
}

bool SafeBrowsingDatabaseNew::ResetDatabase() {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  // Delete files on disk.
  // TODO(shess): Hard to see where one might want to delete without a
  // reset.  Perhaps inline |Delete()|?
  if (!Delete())
    return false;

  // Reset objects in memory.
  {
    AutoLock locked(lookup_lock_);
    full_hashes_.clear();
    pending_hashes_.clear();
    prefix_miss_cache_.clear();
    // TODO(shess): This could probably be |bloom_filter_.reset()|.
    bloom_filter_ = new BloomFilter(BloomFilter::kBloomFilterMinSize *
                                    BloomFilter::kBloomFilterSizeRatio);
  }

  return true;
}

bool SafeBrowsingDatabaseNew::ContainsUrl(
    const GURL& url,
    std::string* matching_list,
    std::vector<SBPrefix>* prefix_hits,
    std::vector<SBFullHashResult>* full_hits,
    base::Time last_update) {
  // Clear the results first.
  matching_list->clear();
  prefix_hits->clear();
  full_hits->clear();

  std::vector<SBPrefix> prefixes;
  PrefixesToCheck(url, &prefixes);
  if (prefixes.empty())
    return false;

  // This function is called on the I/O thread, prevent changes to
  // bloom filter and caches.
  AutoLock locked(lookup_lock_);

  if (!bloom_filter_.get())
    return false;

  // TODO(erikkay): Not filling in matching_list - is that OK?
  size_t miss_count = 0;
  for (size_t i = 0; i < prefixes.size(); ++i) {
    if (bloom_filter_->Exists(prefixes[i])) {
      prefix_hits->push_back(prefixes[i]);
      if (prefix_miss_cache_.count(prefixes[i]) > 0)
        ++miss_count;
    }
  }

  // If all the prefixes are cached as 'misses', don't issue a GetHash.
  if (miss_count == prefix_hits->size())
    return false;

  // Find the matching full-hash results.  |full_hashes_| are from the
  // database, |pending_hashes_| are from GetHash requests between
  // updates.
  std::sort(prefix_hits->begin(), prefix_hits->end());
  GetCachedFullHashes(*prefix_hits, full_hashes_, full_hits, last_update);
  GetCachedFullHashes(*prefix_hits, pending_hashes_, full_hits, last_update);
  return true;
}

// Helper to insert entries for all of the prefixes or full hashes in
// |entry| into the store.
void SafeBrowsingDatabaseNew::InsertAdd(int chunk_id, SBPrefix host,
                                        const SBEntry* entry, int list_id) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  STATS_COUNTER("SB.HostInsert", 1);
  const int encoded_chunk_id = EncodeChunkId(chunk_id, list_id);
  const int count = entry->prefix_count();

  DCHECK(!entry->IsSub());
  if (!count) {
    // No prefixes, use host instead.
    STATS_COUNTER("SB.PrefixAdd", 1);
    store_->WriteAddPrefix(encoded_chunk_id, host);
  } else if (entry->IsPrefix()) {
    // Prefixes only.
    for (int i = 0; i < count; i++) {
      const SBPrefix prefix = entry->PrefixAt(i);
      STATS_COUNTER("SB.PrefixAdd", 1);
      store_->WriteAddPrefix(encoded_chunk_id, prefix);
    }
  } else {
    // Prefixes and hashes.
    const base::Time receive_time = base::Time::Now();
    for (int i = 0; i < count; ++i) {
      const SBFullHash full_hash = entry->FullHashAt(i);
      const SBPrefix prefix = full_hash.prefix;

      STATS_COUNTER("SB.PrefixAdd", 1);
      store_->WriteAddPrefix(encoded_chunk_id, prefix);

      STATS_COUNTER("SB.PrefixAddFull", 1);
      store_->WriteAddHash(encoded_chunk_id, receive_time, full_hash);
    }
  }
}

// Helper to iterate over all the entries in the hosts in |chunks| and
// add them to the store.
void SafeBrowsingDatabaseNew::InsertAddChunks(int list_id,
                                              const SBChunkList& chunks) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());
  for (SBChunkList::const_iterator citer = chunks.begin();
       citer != chunks.end(); ++citer) {
    const int chunk_id = citer->chunk_number;

    // The server can give us a chunk that we already have because
    // it's part of a range.  Don't add it again.
    const int encoded_chunk_id = EncodeChunkId(chunk_id, list_id);
    if (store_->CheckAddChunk(encoded_chunk_id))
      continue;

    store_->SetAddChunk(encoded_chunk_id);
    for (std::deque<SBChunkHost>::const_iterator hiter = citer->hosts.begin();
         hiter != citer->hosts.end(); ++hiter) {
      // NOTE: Could pass |encoded_chunk_id|, but then inserting add
      // chunks would look different from inserting sub chunks.
      InsertAdd(chunk_id, hiter->host, hiter->entry, list_id);
    }
  }
}

// Helper to insert entries for all of the prefixes or full hashes in
// |entry| into the store.
void SafeBrowsingDatabaseNew::InsertSub(int chunk_id, SBPrefix host,
                                        const SBEntry* entry, int list_id) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  STATS_COUNTER("SB.HostDelete", 1);
  const int encoded_chunk_id = EncodeChunkId(chunk_id, list_id);
  const int count = entry->prefix_count();

  DCHECK(entry->IsSub());
  if (!count) {
    // No prefixes, use host instead.
    STATS_COUNTER("SB.PrefixSub", 1);
    const int add_chunk_id = EncodeChunkId(entry->chunk_id(), list_id);
    store_->WriteSubPrefix(encoded_chunk_id, add_chunk_id, host);
  } else if (entry->IsPrefix()) {
    // Prefixes only.
    for (int i = 0; i < count; i++) {
      const SBPrefix prefix = entry->PrefixAt(i);
      const int add_chunk_id =
          EncodeChunkId(entry->ChunkIdAtPrefix(i), list_id);

      STATS_COUNTER("SB.PrefixSub", 1);
      store_->WriteSubPrefix(encoded_chunk_id, add_chunk_id, prefix);
    }
  } else {
    // Prefixes and hashes.
    for (int i = 0; i < count; ++i) {
      const SBFullHash full_hash = entry->FullHashAt(i);
      const int add_chunk_id =
          EncodeChunkId(entry->ChunkIdAtPrefix(i), list_id);

      STATS_COUNTER("SB.PrefixSub", 1);
      store_->WriteSubPrefix(encoded_chunk_id, add_chunk_id, full_hash.prefix);

      STATS_COUNTER("SB.PrefixSubFull", 1);
      store_->WriteSubHash(encoded_chunk_id, add_chunk_id, full_hash);
    }
  }
}

// Helper to iterate over all the entries in the hosts in |chunks| and
// add them to the store.
void SafeBrowsingDatabaseNew::InsertSubChunks(int list_id,
                                              const SBChunkList& chunks) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());
  for (SBChunkList::const_iterator citer = chunks.begin();
       citer != chunks.end(); ++citer) {
    const int chunk_id = citer->chunk_number;

    // The server can give us a chunk that we already have because
    // it's part of a range.  Don't add it again.
    const int encoded_chunk_id = EncodeChunkId(chunk_id, list_id);
    if (store_->CheckSubChunk(encoded_chunk_id))
      continue;

    store_->SetSubChunk(encoded_chunk_id);
    for (std::deque<SBChunkHost>::const_iterator hiter = citer->hosts.begin();
         hiter != citer->hosts.end(); ++hiter) {
      InsertSub(chunk_id, hiter->host, hiter->entry, list_id);
    }
  }
}

void SafeBrowsingDatabaseNew::InsertChunks(const std::string& list_name,
                                           const SBChunkList& chunks) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  if (chunks.empty())
    return;

  const base::Time insert_start = base::Time::Now();

  const int list_id = safe_browsing_util::GetListId(list_name);
  store_->BeginChunk();
  if (chunks.front().is_add) {
    InsertAddChunks(list_id, chunks);
  } else {
    InsertSubChunks(list_id, chunks);
  }
  store_->FinishChunk();

  UMA_HISTOGRAM_TIMES("SB2.ChunkInsert", base::Time::Now() - insert_start);
}

void SafeBrowsingDatabaseNew::DeleteChunks(
    const std::vector<SBChunkDelete>& chunk_deletes) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  if (chunk_deletes.empty())
    return;

  const std::string& list_name = chunk_deletes.front().list_name;
  const int list_id = safe_browsing_util::GetListId(list_name);

  for (size_t i = 0; i < chunk_deletes.size(); ++i) {
    std::vector<int> chunk_numbers;
    RangesToChunks(chunk_deletes[i].chunk_del, &chunk_numbers);
    for (size_t j = 0; j < chunk_numbers.size(); ++j) {
      const int encoded_chunk_id = EncodeChunkId(chunk_numbers[j], list_id);
      if (chunk_deletes[i].is_sub_del)
        store_->DeleteSubChunk(encoded_chunk_id);
      else
        store_->DeleteAddChunk(encoded_chunk_id);
    }
  }
}

void SafeBrowsingDatabaseNew::CacheHashResults(
    const std::vector<SBPrefix>& prefixes,
    const std::vector<SBFullHashResult>& full_hits) {
  // This is called on the I/O thread, lock against updates.
  AutoLock locked(lookup_lock_);

  if (full_hits.empty()) {
    prefix_miss_cache_.insert(prefixes.begin(), prefixes.end());
    return;
  }

  // TODO(shess): SBFullHashResult and SBAddFullHash are very similar.
  // Refactor to make them identical.
  const base::Time now = base::Time::Now();
  const size_t orig_size = pending_hashes_.size();
  for (std::vector<SBFullHashResult>::const_iterator iter = full_hits.begin();
       iter != full_hits.end(); ++iter) {
    const int list_id = safe_browsing_util::GetListId(iter->list_name);
    const int encoded_chunk_id = EncodeChunkId(iter->add_chunk_id, list_id);
    pending_hashes_.push_back(SBAddFullHash(encoded_chunk_id, now, iter->hash));
  }

  // Sort new entries then merge with the previously-sorted entries.
  std::vector<SBAddFullHash>::iterator
      orig_end = pending_hashes_.begin() + orig_size;
  std::sort(orig_end, pending_hashes_.end(), SBAddFullHashPrefixLess);
  std::inplace_merge(pending_hashes_.begin(), orig_end, pending_hashes_.end(),
                     SBAddFullHashPrefixLess);
}

bool SafeBrowsingDatabaseNew::UpdateStarted(
    std::vector<SBListChunkRanges>* lists) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());
  DCHECK(lists);

  // If |BeginUpdate()| fails, reset the database.
  if (!store_->BeginUpdate()) {
    HandleCorruptDatabase();
    return false;
  }

  SBListChunkRanges malware(safe_browsing_util::kMalwareList);
  SBListChunkRanges phishing(safe_browsing_util::kPhishingList);

  std::vector<int> add_chunks;
  store_->GetAddChunks(&add_chunks);
  GetChunkIds(add_chunks, &malware.adds, &phishing.adds);

  std::vector<int> sub_chunks;
  store_->GetSubChunks(&sub_chunks);
  GetChunkIds(sub_chunks, &malware.subs, &phishing.subs);

  lists->push_back(malware);
  lists->push_back(phishing);

  return true;
}

void SafeBrowsingDatabaseNew::UpdateFinished(bool update_succeeded) {
  DCHECK_EQ(creation_loop_, MessageLoop::current());

  // Unroll any partially-received transaction.
  if (!update_succeeded) {
    store_->CancelUpdate();
    return;
  }

  // Copy out the pending add hashes.  Copy rather than swapping in
  // case |ContainsURL()| is called before the new filter is complete.
  std::vector<SBAddFullHash> pending_add_hashes;
  {
    AutoLock locked(lookup_lock_);
    pending_add_hashes.insert(pending_add_hashes.end(),
                              pending_hashes_.begin(), pending_hashes_.end());
  }

  std::vector<SBAddPrefix> add_prefixes;
  std::vector<SBAddFullHash> add_full_hashes;
  if (!store_->FinishUpdate(pending_add_hashes,
                            &add_prefixes, &add_full_hashes))
    return;

  // Measure the amount of IO during the bloom filter build.
  base::IoCounters io_before, io_after;
  base::ProcessHandle handle = base::Process::Current().handle();
  scoped_ptr<base::ProcessMetrics> metric(
#if !defined(OS_MACOSX)
      base::ProcessMetrics::CreateProcessMetrics(handle)
#else
      // Getting stats only for the current process is enough, so NULL is fine.
      base::ProcessMetrics::CreateProcessMetrics(handle, NULL)
#endif
  );

  // IoCounters are currently not supported on Mac, and may not be
  // available for Linux, so we check the result and only show IO
  // stats if they are available.
  const bool got_counters = metric->GetIOCounters(&io_before);

  const base::Time before = base::Time::Now();

  // Create and populate |filter| from |add_prefixes|.
  // TODO(shess): The bloom filter doesn't need to be a
  // scoped_refptr<> for this code.  Refactor that away.
  const int filter_size =
      BloomFilter::FilterSizeForKeyCount(add_prefixes.size());
  scoped_refptr<BloomFilter> filter(new BloomFilter(filter_size));
  for (size_t i = 0; i < add_prefixes.size(); ++i) {
    filter->Insert(add_prefixes[i].prefix);
  }

  // This needs to be in sorted order by prefix for efficient access.
  std::sort(add_full_hashes.begin(), add_full_hashes.end(),
            SBAddFullHashPrefixLess);

  // Swap in the newly built filter and cache.
  {
    AutoLock locked(lookup_lock_);
    full_hashes_.swap(add_full_hashes);

    // TODO(shess): If |CacheHashResults()| is posted between the
    // earlier lock and this clear, those pending hashes will be lost.
    // It could be fixed by only removing hashes which were collected
    // at the earlier point.  I believe that is fail-safe as-is (the
    // hash will be fetched again).
    pending_hashes_.clear();

    prefix_miss_cache_.clear();
    bloom_filter_.swap(filter);
  }

  const base::TimeDelta bloom_gen = base::Time::Now() - before;

  // Persist the bloom filter to disk.  Since only this thread changes
  // |bloom_filter_|, there is no need to lock.
  WriteBloomFilter();

  // Gather statistics.
  if (got_counters && metric->GetIOCounters(&io_after)) {
    UMA_HISTOGRAM_COUNTS("SB2.BuildReadBytes",
                         static_cast<int>(io_after.ReadTransferCount -
                                          io_before.ReadTransferCount));
    UMA_HISTOGRAM_COUNTS("SB2.BuildWriteBytes",
                         static_cast<int>(io_after.WriteTransferCount -
                                          io_before.WriteTransferCount));
    UMA_HISTOGRAM_COUNTS("SB2.BuildReadOperations",
                         static_cast<int>(io_after.ReadOperationCount -
                                          io_before.ReadOperationCount));
    UMA_HISTOGRAM_COUNTS("SB2.BuildWriteOperations",
{'fraction_non_alphanumeric': 0.11786971246430705, 'fraction_numerical': 0.0019036235252451468, 'mean_word_length': 3.754577983582404, 'pattern_counts': {'":': 0, '<': 111, '<?xml version=': 0, '>': 221, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '368700', 'n_tokens_mistral': 14686, 'n_tokens_neox': 14304, 'n_words': 3421}