<commit_before>#pragma once

namespace goobly {
inline ::rocksdb::Options dboptsFactory() {
  rocksdb::Options o{};
  // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
  // does starts a thread per core
  o.IncreaseParallelism();
  // For big databases, this is the best style compaction. Universal
  // Compaction has issues if your DB is > 150 GB.  See docs.
  // This option also sets compression after the first 2 levels
  o.OptimizeLevelStyleCompaction();
  // rocksdb mailing list optimization for queues
  // https://www.facebook.com/groups/rocksdb.dev
  o.setMaxOpenFiles(-1);
  o.create_if_missing = true;
  o.create_missing_column_families = true;
  o.allow_mmap_reads = true;
  o.allow_mmap_writes = true;
  o.access_hint_on_compaction_start = rocksdb::DBOptions::SEQUENTIAL;
  // TODO(add block table config & use bloomfilter)
  // final BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
  // tableOptions.setFilter(new BloomFilter(10, false));
  // options.setTableFormatConfig(tableOptions);
  // Please make sure you set up BloomFilter with parameter useBlockBasedMode=false
  return o;
}

inline ::rocksdb::WriteOptions woptsFactory() {
  rocksdb::WriteOptions o{};
  o.sync = true;
  return o;
}

inline ::rocksdb::ReadOptions roptsFactory(bool tailing=true) {
  rocksdb::ReadOptions o{};
  /*if true use read-only tailing iterators*/
  o.tailing = tailing;
  o.verify_checksums = false;
  return o;
}

}<commit_msg>fixed compilation<commit_after>#pragma once
#include <rocksdb/db.h>
namespace goobly {
inline ::rocksdb::Options dboptsFactory() {
  rocksdb::Options o{};
  // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
  // does starts a thread per core
  o.IncreaseParallelism();
  // For big databases, this is the best style compaction. Universal
  // Compaction has issues if your DB is > 150 GB.  See docs.
  // This option also sets compression after the first 2 levels
  o.OptimizeLevelStyleCompaction();
  // rocksdb mailing list optimization for queues
  // https://www.facebook.com/groups/rocksdb.dev
  o.max_open_files = -1;
  o.create_if_missing = true;
  o.create_missing_column_families = true;
  o.allow_mmap_reads = true;
  o.allow_mmap_writes = true;
  o.access_hint_on_compaction_start = rocksdb::DBOptions::SEQUENTIAL;
  rocksdb::BlockBasedTableOptions table_options;
  table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
  o.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
  return o;
}

inline ::rocksdb::WriteOptions woptsFactory() {
  rocksdb::WriteOptions o{};
  // Note: If you don't end up using rocksdb for the index, then definitely
  // set this to false as you already have the write-ahead-log entry synced
  // on disk
  o.sync = true;
  return o;
}

inline ::rocksdb::ReadOptions roptsFactory(bool tailing=true) {
  rocksdb::ReadOptions o{};
  /*if true use read-only tailing iterators*/
  o.tailing = tailing;
  o.verify_checksums = false;
  return o;
}

}
{'fraction_non_alphanumeric': 0.10307234886025768, 'fraction_numerical': 0.004625041295011563, 'mean_word_length': 4.293706293706293, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6942213', 'n_tokens_mistral': 992, 'n_tokens_neox': 960, 'n_words': 277}