<commit_before>#include "precomp.h"

#include <iterator>
#include "csv_parser.h"

namespace lib_parsing {
template <typename T>
CsvParser<T>::CsvParser() {}

template <typename T>
CsvParser<T>::~CsvParser() {}

template <typename T>
inline bool CsvParser<T>::Parse(sp<lib_data::MlDataFrame<T>> data,
                                string filepath) {
  std::ifstream open(filepath);
  if (open.fail()) {
    open.close();
    return false;
  }

  auto ret = Parse(data, open);
  open.close();

  return ret;
}

template <typename T>
bool CsvParser<T>::Parse(sp<lib_data::MlDataFrame<T>> data,
                         col_array<char>& raw_data) {
  if (raw_data.empty()) return false;
  std::stringstream stream(raw_data.data());
  return Parse(data, stream);
}

template <typename T>
bool CsvParser<T>::Parse(sp<lib_data::MlDataFrame<T>> data, const char* raw_data) {
  std::stringstream stream(raw_data);
  return Parse(data, stream);
}

template <typename T>
template <typename Ts>
inline bool CsvParser<T>::Parse(sp<lib_data::MlDataFrame<T>> data, Ts& stream) {
  std::locale x(std::locale::classic(), new csv_ctype);

  col_array<T> targets;
  col_array<col_array<T>> samples;
  col_array<T> att_ids;
  col_array<string> attributes;
  col_array<col_map<string, T>> attribute_col_map;

  string line;
  std::getline(stream, line);
  if (stream.eof()) return false;

  std::stringstream linestream(line);
  linestream.imbue(x);

  std::copy(std::istream_iterator<string>(linestream),
            std::istream_iterator<string>(), std::back_inserter(attributes));
  attribute_col_map.assign(attributes.size(), col_map<string, T>());
  att_ids.assign(attributes.size(), 0);
  samples.assign(attributes.size() - 1, col_array<T>());

  auto target_id = attributes.size() - 1;
  for (int i = 0; i < attributes.size(); ++i)
    if (attributes[i].compare("class") == 0 ||
        attributes[i].compare("target") == 0 ||
        attributes[i].compare("output") == 0 ||
		attributes[i].compare("regression") == 0)
      target_id = i;

  T value;
  string part;
  do {
    std::getline(stream, line);
	if (line.empty()) break;
    std::stringstream lstream(line);
    lstream.imbue(x);

    int att_id = 0;
    while (!lstream.eof()) {
      auto& att_col_map = attribute_col_map[att_id];
      lstream >> part;
	  if (lstream.fail()) break;
      std::stringstream pstream(part);
      pstream >> value;

      if (pstream.fail()) {
        if (att_col_map.find(part) == att_col_map.end()) {
          att_col_map[part] = att_ids[att_id];
          ++att_ids[att_id];
        }

        value = att_col_map[part];
      } else
        att_col_map[part] = value;

      if (att_id == target_id)
        targets.emplace_back(value);
      else
        samples[att_id].emplace_back(value);

      ++att_id;
    }
    if (att_id != attributes.size()) return false;
  } while (!stream.eof());

  col_array<T> sample_packed;
  sample_packed.assign(samples.size() * samples[0].size(), 0);
  for (int i = 0; i < samples.size(); ++i)
    std::copy(samples[i].begin(), samples[i].end(),
              sample_packed.begin() + i * samples[0].size());

  data->AddTargetData(targets);
  data->AddSampleData(sample_packed);
  data->AddTargetMap(attribute_col_map.back());
  attribute_col_map.pop_back();
  data->AddFeatureMap(attribute_col_map);
  return true;
}

template CsvParser<float>::CsvParser();
template CsvParser<double>::CsvParser();
}<commit_msg>- fixed parser error when target variable is not the last column<commit_after>#include "precomp.h"

#include <iterator>
#include "csv_parser.h"

namespace lib_parsing {
template <typename T>
CsvParser<T>::CsvParser() {}

template <typename T>
CsvParser<T>::~CsvParser() {}

template <typename T>
inline bool CsvParser<T>::Parse(sp<lib_data::MlDataFrame<T>> data,
                                string filepath) {
  std::ifstream open(filepath);
  if (open.fail()) {
    open.close();
    return false;
  }

  auto ret = Parse(data, open);
  open.close();

  return ret;
}

template <typename T>
bool CsvParser<T>::Parse(sp<lib_data::MlDataFrame<T>> data,
                         col_array<char>& raw_data) {
  if (raw_data.empty()) return false;
  std::stringstream stream(raw_data.data());
  return Parse(data, stream);
}

template <typename T>
bool CsvParser<T>::Parse(sp<lib_data::MlDataFrame<T>> data, const char* raw_data) {
  std::stringstream stream(raw_data);
  return Parse(data, stream);
}

template <typename T>
template <typename Ts>
inline bool CsvParser<T>::Parse(sp<lib_data::MlDataFrame<T>> data, Ts& stream) {
  std::locale x(std::locale::classic(), new csv_ctype);

  col_array<T> targets;
  col_array<col_array<T>> samples;
  col_array<T> att_ids;
  col_array<string> attributes;
  col_array<col_map<string, T>> attribute_col_map;

  string line;
  std::getline(stream, line);
  if (stream.eof()) return false;

  std::stringstream linestream(line);
  linestream.imbue(x);

  std::copy(std::istream_iterator<string>(linestream),
            std::istream_iterator<string>(), std::back_inserter(attributes));
  attribute_col_map.assign(attributes.size(), col_map<string, T>());
  att_ids.assign(attributes.size(), 0);
  samples.assign(attributes.size() - 1, col_array<T>());

  auto target_id = attributes.size() - 1;
  for (int i = 0; i < attributes.size(); ++i)
    if (attributes[i].compare("class") == 0 ||
        attributes[i].compare("target") == 0 ||
        attributes[i].compare("output") == 0 ||
		attributes[i].compare("regression") == 0)
      target_id = i;

  T value;
  string part;
  do {
    std::getline(stream, line);
	if (line.empty()) break;
    std::stringstream lstream(line);
    lstream.imbue(x);

    int att_id = 0;
    while (!lstream.eof()) {
      auto& att_col_map =
          attribute_col_map[att_id];
      lstream >> part;
      if (lstream.fail()) break;
      std::stringstream pstream(part);
      pstream >> value;

      if (pstream.fail()) {
        if (att_col_map.find(part) == att_col_map.end()) {
          att_col_map[part] = att_ids[att_id];
          ++att_ids[att_id];
        }

        value = att_col_map[part];
      } else
        att_col_map[part] = value;

      if (att_id == target_id)
        targets.emplace_back(value);
      else {
        samples[att_id - (att_id > target_id ? 1 : 0)].emplace_back(value);
      }

      ++att_id;
    }
    if (att_id != attributes.size()) return false;
  } while (!stream.eof());

  col_array<T> sample_packed;
  sample_packed.assign(samples.size() * samples[0].size(), 0);
  for (int i = 0; i < samples.size(); ++i)
    std::copy(samples[i].begin(), samples[i].end(),
              sample_packed.begin() + i * samples[0].size());

  data->AddTargetData(targets);
  data->AddSampleData(sample_packed);
  data->AddTargetMap(attribute_col_map[target_id]);
  attribute_col_map.erase(attribute_col_map.begin() + target_id);
  data->AddFeatureMap(attribute_col_map);
  return true;
}

template CsvParser<float>::CsvParser();
template CsvParser<double>::CsvParser();
}
{'fraction_non_alphanumeric': 0.16241629861803675, 'fraction_numerical': 0.0039891722467587975, 'mean_word_length': 3.8181194234728895, 'pattern_counts': {'":': 0, '<': 81, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '500150', 'n_tokens_mistral': 2482, 'n_tokens_neox': 2412, 'n_words': 468}