<commit_before>#include "index.hpp"

namespace vg {

using namespace std;

Index::Index(void) {
    reset_options();
}

Index::Index(string& dir) : name(dir) {
    reset_options();
}

void Index::reset_options(void) {
    start_sep = '\x00';
    end_sep = '\xff';
    write_options = rocksdb::WriteOptions();
    options.create_if_missing = true;
    //options.env->SetBackgroundThreads(omp_get_num_procs());
    //options.compression = rocksdb::kBZip2Compression;
    //options.compression = rocksdb::kSnappyCompression;
    options.compression = rocksdb::kZlibCompression;
    //options.compaction_style = rocksdb::kCompactionStyleUniversal;
    options.compaction_style = rocksdb::kCompactionStyleLevel;
    //options.allow_os_buffer = true;
    /*
    options.max_background_compactions = threads;
    options.max_background_flushes = threads;<commit_msg>well the problem wasn't universal compaction<commit_after>#include "index.hpp"

namespace vg {

using namespace std;

Index::Index(void) {
    reset_options();
}

Index::Index(string& dir) : name(dir) {
    reset_options();
}

void Index::reset_options(void) {
    start_sep = '\x00';
    end_sep = '\xff';
    write_options = rocksdb::WriteOptions();
    options.create_if_missing = true;
    //options.env->SetBackgroundThreads(omp_get_num_procs());
    //options.compression = rocksdb::kBZip2Compression;
    //options.compression = rocksdb::kSnappyCompression;
    options.compression = rocksdb::kZlibCompression;
    options.compaction_style = rocksdb::kCompactionStyleUniversal;
    //options.compaction_style = rocksdb::kCompactionStyleLevel;
    //options.allow_os_buffer = true;
    /*
    options.max_background_compactions = threads;
    options.max_background_flushes = threads;
{'fraction_non_alphanumeric': 0.12521440823327615, 'fraction_numerical': 0.003430531732418525, 'mean_word_length': 4.608974358974359, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7403482', 'n_tokens_mistral': 563, 'n_tokens_neox': 552, 'n_words': 84}