<commit_before>/**
 * @file forward_index.cpp
 * @author <NAME>
 */

#include "index/forward_index.h"
#include "index/postings_data.h"
#include "index/string_list.h"
#include "index/string_list_writer.h"
#include "index/vocabulary_map.h"
#include "io/libsvm_parser.h"
#include "tokenizers/tokenizer.h"
#include "util/disk_vector.h"
#include "util/shim.h"

namespace meta {
namespace index {

forward_index::forward_index(const cpptoml::toml_group & config):
    disk_index{config},
    _index_name{*cpptoml::get_as<std::string>(config, "forward-index")}
{ /* nothing */ }

std::string forward_index::index_name() const
{
    return _index_name;
}

std::string forward_index::liblinear_data(doc_id d_id) const
{
    return "";
}

void forward_index::load_index()
{
    LOG(info) << "Loading index from disk: " << _index_name << ENDLG;
    init_metadata();
    _doc_id_mapping = make_unique<string_list>(_index_name
                                                    + "/docids.mapping");
}

void forward_index::create_index(const std::string & config_file)
{
    LOG(info) << "Creating index: " << _index_name << ENDLG;

    filesystem::copy_file(config_file, _index_name + "/config.toml");
    auto config = cpptoml::parse_file(_index_name + "/config.toml");

    // if the corpus is a single libsvm formatted file, then we are done;
    // otherwise, we will create an inverted index and the uninvert it
    if(is_libsvm_format(config))
    {
        create_libsvm_postings(config);
        create_libsvm_metadata(config);
    }
    else
    {
        uninvert(config);
        create_uninverted_metadata(config);
    }

    // now that the files are tokenized, we can create the string_list
    _doc_id_mapping = make_unique<string_list>(_index_name
                                                    + "/docids.mapping");

    LOG(info) << "Done creating index: " << _index_name << ENDLG;
}

void forward_index::create_libsvm_postings(const cpptoml::toml_group& config)
{
    auto prefix = config.get_as<std::string>("prefix");
    if (!prefix)
        throw forward_index_exception{"prefix missing from configuration file"};

    auto dataset = config.get_as<std::string>("dataset");
    if (!dataset)
        throw forward_index_exception{"dataset missing from configuration file"};

    std::string existing_file = *prefix + "/"
        + *dataset + "/" + *dataset + ".dat";

    filesystem::copy_file(existing_file, _index_name + "/postings.index");

    init_metadata();

    // now, assign byte locations for libsvm doc starting points
    _postings = make_unique<io::mmap_file>(_index_name + "/postings.index");
    doc_id d_id{0};
    uint8_t last_byte = '\n';
    for(uint64_t idx = 0; idx < _postings->size(); ++idx)
    {<commit_msg>forward_index can create and load libsvm formatted corpora<commit_after>/**
 * @file forward_index.cpp
 * @author <NAME>
 */

#include "index/forward_index.h"
#include "index/postings_data.h"
#include "index/string_list.h"
#include "index/string_list_writer.h"
#include "index/vocabulary_map.h"
#include "io/libsvm_parser.h"
#include "tokenizers/tokenizer.h"
#include "util/disk_vector.h"
#include "util/mapping.h"
#include "util/shim.h"

namespace meta {
namespace index {

forward_index::forward_index(const cpptoml::toml_group & config):
    disk_index{config},
    _index_name{*cpptoml::get_as<std::string>(config, "forward-index")}
{ /* nothing */ }

std::string forward_index::index_name() const
{
    return _index_name;
}

std::string forward_index::liblinear_data(doc_id d_id) const
{
    return "";
}

void forward_index::load_index()
{
    LOG(info) << "Loading index from disk: " << _index_name << ENDLG;

    init_metadata();

    _doc_id_mapping = make_unique<string_list>(_index_name + "/docids.mapping");
    _postings = make_unique<io::mmap_file>(
        _index_name + "/postings.index"
    );

    auto config = cpptoml::parse_file(_index_name + "/config.toml");
    if(is_libsvm_format(config))
        _term_id_mapping = nullptr;
    else
    {
        _term_id_mapping =
            make_unique<vocabulary_map>(_index_name + "/termids.mapping");
    }

    map::load_mapping(_label_ids, _index_name + "/labelids.mapping");
    _tokenizer = tokenizers::tokenizer::load(config);
}

void forward_index::create_index(const std::string & config_file)
{
    LOG(info) << "Creating index: " << _index_name << ENDLG;

    filesystem::copy_file(config_file, _index_name + "/config.toml");
    auto config = cpptoml::parse_file(_index_name + "/config.toml");
    _tokenizer = tokenizers::tokenizer::load(config);

    // if the corpus is a single libsvm formatted file, then we are done;
    // otherwise, we will create an inverted index and the uninvert it
    if(is_libsvm_format(config))
    {
        create_libsvm_postings(config);
        create_libsvm_metadata(config);
        _term_id_mapping = nullptr; // we don't know what the terms are!
    }
    else
    {
        uninvert(config);
        create_uninverted_metadata(config);
    }

    // now that the files are tokenized, we can create the string_list
    _doc_id_mapping = make_unique<string_list>(_index_name
                                                    + "/docids.mapping");
    map::save_mapping(_label_ids, _index_name + "/labelids.mapping");

    LOG(info) << "Done creating index: " << _index_name << ENDLG;
}

void forward_index::create_libsvm_postings(const cpptoml::toml_group& config)
{
    auto prefix = config.get_as<std::string>("prefix");
    if (!prefix)
        throw forward_index_exception{"prefix missing from configuration file"};

    auto dataset = config.get_as<std::string>("dataset");
    if (!dataset)
        throw forward_index_exception{"dataset missing from configuration file"};

    std::string existing_file = *prefix + "/"
        + *dataset + "/" + *dataset + ".dat";

    filesystem::copy_file(existing_file, _index_name + "/postings.index");

    init_metadata();

    // now, assign byte locations for libsvm doc starting points
    _postings = make_unique<io::mmap_file>(_index_name + "/postings.index");
    doc_id d_id{0};
    uint8_t last_byte = '\n';
    for(uint64_t idx = 0; idx < _postings->size(); ++idx)
    {
{'fraction_non_alphanumeric': 0.1310946076848563, 'fraction_numerical': 0.0016144656118824668, 'mean_word_length': 3.623134328358209, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3683652', 'n_tokens_mistral': 2046, 'n_tokens_neox': 2001, 'n_words': 407}