<commit_before>#include "lidx.h"

#include <stdlib.h>

#include <leveldb/db.h>
#include <leveldb/status.h>
#include <leveldb/write_batch.h>

#include "lidx-utils.h"
#include "lidx-icu-utils.h"
#include "lidx-encode.h"

#include <set>
#include <map>

static int db_put(lidx * index, std::string & key, std::string & value);
static int db_get(lidx * index, std::string & key, std::string * p_value);
static int db_delete(lidx * index, std::string & key);
static int db_flush(lidx * index);

// . -> next word id
// ,[docid] -> [words ids]
// /[word id] -> word
// word -> [word id], [docs ids]

struct lidx {
  leveldb::DB * lidx_db;
  std::map<std::string, std::string> * lidx_buffer;
  std::set<std::string> * lidx_buffer_dirty;
  std::set<std::string> * lidx_deleted;
};

lidx * lidx_new(void)
{
  lidx_init_icu_utils();
  lidx * result = (lidx *) calloc(1, sizeof(* result));
  result->lidx_buffer = new std::map<std::string, std::string>();
  result->lidx_buffer_dirty = new std::set<std::string>();
  result->lidx_deleted = new std::set<std::string>();
  return result;
}

void lidx_free(lidx * index)
{
  delete index->lidx_buffer;
  delete index->lidx_buffer_dirty;
  delete index->lidx_deleted;
  free(index);
}

int lidx_open(lidx * index, const char * filename)
{
  leveldb::Options options;
  leveldb::Status status;

  options.max_open_files = 0;
  options.create_if_missing = true;
  status = leveldb::DB::Open(options, filename, &index->lidx_db);
  if (!status.ok()) {
    return -1;
  }
  
  return 0;
}

void lidx_close(lidx * index)
{
  if (index->lidx_db == NULL) {
    return;
  }
  db_flush(index);
  delete index->lidx_db;
  index->lidx_db = NULL;<commit_msg>Removed setting value unsupported in standard LevelDB<commit_after>#include "lidx.h"

#include <stdlib.h>

#include <leveldb/db.h>
#include <leveldb/status.h>
#include <leveldb/write_batch.h>

#include "lidx-utils.h"
#include "lidx-icu-utils.h"
#include "lidx-encode.h"

#include <set>
#include <map>

static int db_put(lidx * index, std::string & key, std::string & value);
static int db_get(lidx * index, std::string & key, std::string * p_value);
static int db_delete(lidx * index, std::string & key);
static int db_flush(lidx * index);

// . -> next word id
// ,[docid] -> [words ids]
// /[word id] -> word
// word -> [word id], [docs ids]

struct lidx {
  leveldb::DB * lidx_db;
  std::map<std::string, std::string> * lidx_buffer;
  std::set<std::string> * lidx_buffer_dirty;
  std::set<std::string> * lidx_deleted;
};

lidx * lidx_new(void)
{
  lidx_init_icu_utils();
  lidx * result = (lidx *) calloc(1, sizeof(* result));
  result->lidx_buffer = new std::map<std::string, std::string>();
  result->lidx_buffer_dirty = new std::set<std::string>();
  result->lidx_deleted = new std::set<std::string>();
  return result;
}

void lidx_free(lidx * index)
{
  delete index->lidx_buffer;
  delete index->lidx_buffer_dirty;
  delete index->lidx_deleted;
  free(index);
}

int lidx_open(lidx * index, const char * filename)
{
  leveldb::Options options;
  leveldb::Status status;

  options.create_if_missing = true;
  status = leveldb::DB::Open(options, filename, &index->lidx_db);
  if (!status.ok()) {
    return -1;
  }
  
  return 0;
}

void lidx_close(lidx * index)
{
  if (index->lidx_db == NULL) {
    return;
  }
  db_flush(index);
  delete index->lidx_db;
  index->lidx_db = NULL;
{'fraction_non_alphanumeric': 0.17191211401425177, 'fraction_numerical': 0.0020783847980997625, 'mean_word_length': 4.460291734197731, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3544347', 'n_tokens_mistral': 1305, 'n_tokens_neox': 1301, 'n_words': 279}