<commit_before>/*
 * Transform a TSV format factor graph file and output corresponding binary
 * format used in DeepDive
 */

#include "text2bin.h"
#include "binary_format.h"
#include <assert.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdint.h>
#include <stdlib.h>
#include <vector>

namespace dd {

constexpr char field_delim = '\t';  // tsv file delimiter

// read variables and convert to binary format
void load_var(std::string input_filename, std::string output_filename) {
  std::ifstream fin(input_filename);
  std::ofstream fout(output_filename, std::ios::binary | std::ios::out);
  num_variables_t count = 0;
  int is_evidence;
  uint16_t var_type;
  variable_id_t vid;
  variable_value_t initial_value;
  num_variable_values_t cardinality;
  while (fin >> vid >> is_evidence >> initial_value >> var_type >>
         cardinality) {
    write_be(fout, vid);
    write_be<uint8_t>(fout, is_evidence);
    write_be(fout, initial_value);
    write_be(fout, var_type);
    write_be(fout, cardinality);
    ++count;
  }
  std::cout << count << std::endl;
}

// convert weights
void load_weight(std::string input_filename, std::string output_filename) {
  std::ifstream fin(input_filename);
  std::ofstream fout(output_filename, std::ios::binary | std::ios::out);
  long count = 0;
  int isfixed;
  weight_id_t wid;
  weight_value_t initial_value;
  while (fin >> wid >> isfixed >> initial_value) {
    write_be(fout, wid);
    write_be<uint8_t>(fout, isfixed);
    write_be(fout, initial_value);
    ++count;
  }
  std::cout << count << std::endl;
}

static inline long parse_pgarray(
    std::istream &input, std::function<void(const std::string &)> parse_element,
    long expected_count = -1) {
  if (input.peek() == '{') {
    input.get();
    std::string element;
    bool ended = false;
    long count = 0;
    while (getline(input, element, ',')) {
      if (element.at(element.length() - 1) == '}') {
        ended = true;
        element = element.substr(0, element.length() - 1);
      }
      parse_element(element);
      count++;
      if (ended) break;
    }
    assert(expected_count < 0 || count == expected_count);
    return count;
  } else {
    return -1;
  }
}
static inline long parse_pgarray_or_die(
    std::istream &input, std::function<void(const std::string &)> parse_element,
    long expected_count = -1) {
  long count = parse_pgarray(input, parse_element, expected_count);
  if (count >= 0) {
    return count;
  } else {
    std::cerr << "Expected an array '{' but found: " << input.get()
              << std::endl;
    std::abort();
  }
}

// load factors
// wid, vids
void load_factor(std::string input_filename, std::string output_filename,
                 factor_function_type_t funcid, factor_arity_t arity_expected,
                 const std::vector<bool> &positives_vec) {
  std::ifstream fin(input_filename);
  std::ofstream fout(output_filename, std::ios::binary | std::ios::out);
  num_edges_t total_edges = 0;
  std::vector<variable_id_t> vids;
  std::vector<weight_id_t> vals_and_weights;  // FIXME separate the two
  std::string line;
  std::string array_piece;
  while (getline(fin, line)) {
    std::string field;
    istringstream ss(line);
    vids.clear();
    // factor type
    write_be<uint16_t>(fout, funcid);
    // variable ids
    factor_arity_t arity = 0;
    auto parse_variableid = [&vids, &arity,
                             &total_edges](const std::string &element) {
      vids.push_back(atol(element.c_str()));
      ++total_edges;
      ++arity;
    };
    for (factor_arity_t i = 0; i < arity_expected; ++i) {
      getline(ss, field, field_delim);
      // try parsing as an array first
      // FIXME remove this?  parsing vid arrays is probably broken since this
      // doesn't create a cross product of factors but simply widens the arity
      istringstream fieldinput(field);
      if (parse_pgarray(fieldinput, parse_variableid) < 0) {
        // otherwise, parse it as a single variable
        parse_variableid(field);
      }
    }
    write_be(fout, arity);
    for (factor_arity_t i = 0; i < vids.size(); ++i) {
      write_be(fout, vids[i]);
      variable_value_t should_equal_to = positives_vec.at(i) ? 1 : 0;
      write_be(fout, should_equal_to);
    }
    // weight ids
    switch (funcid) {
      case FUNC_AND_CATEGORICAL: {
        vals_and_weights.clear();
        // IN  Format: NUM_WEIGHTS [VAR1 VAL ID] [VAR2 VAL ID] ... [WEIGHT ID]
        // OUT Format: NUM_WEIGHTS [[VAR1_VALi, VAR2_VALi, ..., WEIGHTi]]<commit_msg>Scrubs some types in text2bin<commit_after>/*
 * Transform a TSV format factor graph file and output corresponding binary
 * format used in DeepDive
 */

#include "text2bin.h"
#include "binary_format.h"
#include <assert.h>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdint.h>
#include <stdlib.h>
#include <vector>

namespace dd {

constexpr char field_delim = '\t';  // tsv file delimiter

// read variables and convert to binary format
void load_var(std::string input_filename, std::string output_filename) {
  std::ifstream fin(input_filename);
  std::ofstream fout(output_filename, std::ios::binary | std::ios::out);
  num_variables_t count = 0;
  int is_evidence;
  uint16_t var_type;
  variable_id_t vid;
  variable_value_t initial_value;
  num_variable_values_t cardinality;
  while (fin >> vid >> is_evidence >> initial_value >> var_type >>
         cardinality) {
    write_be(fout, vid);
    write_be<uint8_t>(fout, is_evidence);
    write_be(fout, initial_value);
    write_be(fout, var_type);
    write_be(fout, cardinality);
    ++count;
  }
  std::cout << count << std::endl;
}

// convert weights
void load_weight(std::string input_filename, std::string output_filename) {
  std::ifstream fin(input_filename);
  std::ofstream fout(output_filename, std::ios::binary | std::ios::out);
  num_weights_t count = 0;
  int isfixed;
  weight_id_t wid;
  weight_value_t initial_value;
  while (fin >> wid >> isfixed >> initial_value) {
    write_be(fout, wid);
    write_be<uint8_t>(fout, isfixed);
    write_be(fout, initial_value);
    ++count;
  }
  std::cout << count << std::endl;
}

static inline size_t parse_pgarray(
    std::istream &input, std::function<void(const std::string &)> parse_element,
    size_t expected_count = -1) {
  if (input.peek() == '{') {
    input.get();
    std::string element;
    bool ended = false;
    size_t count = 0;
    while (getline(input, element, ',')) {
      if (element.at(element.length() - 1) == '}') {
        ended = true;
        element = element.substr(0, element.length() - 1);
      }
      parse_element(element);
      count++;
      if (ended) break;
    }
    assert(expected_count == -1 || count == expected_count);
    return count;
  } else {
    return -1;
  }
}
static inline size_t parse_pgarray_or_die(
    std::istream &input, std::function<void(const std::string &)> parse_element,
    size_t expected_count = -1) {
  size_t count = parse_pgarray(input, parse_element, expected_count);
  if (count != -1) {
    return count;
  } else {
    std::cerr << "Expected an array '{' but found: " << input.get()
              << std::endl;
    std::abort();
  }
}

// load factors
// wid, vids
void load_factor(std::string input_filename, std::string output_filename,
                 factor_function_type_t funcid, factor_arity_t arity_expected,
                 const std::vector<bool> &positives_vec) {
  std::ifstream fin(input_filename);
  std::ofstream fout(output_filename, std::ios::binary | std::ios::out);
  num_edges_t total_edges = 0;
  std::vector<variable_id_t> vids;
  std::vector<weight_id_t> vals_and_weights;  // FIXME separate the two
  std::string line;
  std::string array_piece;
  while (getline(fin, line)) {
    std::string field;
    istringstream ss(line);
    vids.clear();
    // factor type
    write_be<uint16_t>(fout, funcid);
    // variable ids
    factor_arity_t arity = 0;
    auto parse_variableid = [&vids, &arity,
                             &total_edges](const std::string &element) {
      vids.push_back(atol(element.c_str()));
      ++total_edges;
      ++arity;
    };
    for (factor_arity_t i = 0; i < arity_expected; ++i) {
      getline(ss, field, field_delim);
      // try parsing as an array first
      // FIXME remove this?  parsing vid arrays is probably broken since this
      // doesn't create a cross product of factors but simply widens the arity
      istringstream fieldinput(field);
      if (parse_pgarray(fieldinput, parse_variableid) == -1) {
        // otherwise, parse it as a single variable
        parse_variableid(field);
      }
    }
    write_be(fout, arity);
    for (factor_arity_t i = 0; i < vids.size(); ++i) {
      write_be(fout, vids[i]);
      variable_value_t should_equal_to = positives_vec.at(i) ? 1 : 0;
      write_be(fout, should_equal_to);
    }
    // weight ids
    switch (funcid) {
      case FUNC_AND_CATEGORICAL: {
        vals_and_weights.clear();
        // IN  Format: NUM_WEIGHTS [VAR1 VAL ID] [VAR2 VAL ID] ... [WEIGHT ID]
        // OUT Format: NUM_WEIGHTS [[VAR1_VALi, VAR2_VALi, ..., WEIGHTi]]
{'fraction_non_alphanumeric': 0.13214363915042698, 'fraction_numerical': 0.006459382526822859, 'mean_word_length': 3.325284090909091, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1017345', 'n_tokens_mistral': 3147, 'n_tokens_neox': 3083, 'n_words': 747}