<commit_before>// Copyright 2018 Google LLC. All Rights Reserved.
/*
  Copyright (C) 2005 <NAME>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
*/

#include "stats/DataTable.hpp"
#include "stats/moments.hpp"

#include <cctype>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>

#include "Models/CategoricalData.hpp"
#include "cpputil/DefaultVnames.hpp"
#include "cpputil/Ptr.hpp"
#include "cpputil/math_utils.hpp"
#include "cpputil/report_error.hpp"
#include "cpputil/string_utils.hpp"

namespace BOOM {
  using std::endl;


  //---------------------------------------------------------------------
  // Determine the type of variable stored in vs.

  MixedDataOrganizer::MixedDataOrganizer()
      : numeric_count_(0),
        categorical_count_(0),
        unknown_count_(0)
  {}

  void MixedDataOrganizer::add_variable(VariableType type) {
    int index = variable_types_.size();
    variable_types_.push_back(type);
    if (type == VariableType::numeric) {
      type_map_[index] = std::make_pair(type, numeric_count_++);
    } else if (type == VariableType::categorical) {
      type_map_[index] = std::make_pair(type, categorical_count_++);
    } else {
      ++unknown_count_;
      report_error("Numeric and categorical the the only currently supported"
                   " types.");
    }
  }

  std::pair<VariableType, int> MixedDataOrganizer::type_map(int i) const {
    auto it = type_map_.find(i);
    if (it == type_map_.end()) {
      return std::make_pair(VariableType::unknown, -1);
    } else {
      return it->second;
    }
  }

  void MixedDataOrganizer::diagnose_types(const std::vector<std::string> &vs) {
    uint nfields = vs.size();
    variable_types_ = std::vector<VariableType>(nfields, VariableType::unknown);
    for (uint i = 0; i < vs.size(); ++i) {
      variable_types_[i] =
          is_numeric(vs[i]) ? VariableType::numeric : VariableType::categorical;
    }
  }

  bool MixedDataOrganizer::check_type(int i, const std::string &s) const {
    VariableType type = variable_types_[i];
    if (is_numeric(s)) {
      if (type == VariableType::numeric) return true;
    } else {  // s is not numeric
      if (type == VariableType::categorical) return true;
    }
    return false;
  }

  bool MixedDataOrganizer::operator==(const MixedDataOrganizer &rhs) const {
    return numeric_count_ == rhs.numeric_count_
        && categorical_count_ == rhs.categorical_count_
        && unknown_count_ == rhs.unknown_count_
        && variable_types_ == rhs.variable_types_;
  }

  //===========================================================================
  CategoricalVariable::CategoricalVariable(
      const std::vector<std::string> &raw_data)
      : key_(make_catkey(raw_data)) {
    for (int i = 0; i < raw_data.size(); ++i) {
      Ptr<CategoricalData> dp = new CategoricalData(raw_data[i], key_);
      data_.push_back(dp);
    }
  }

  const Vector &get(const std::map<uint, Vector> &m, uint i) {
    return m.find(i)->second;
  }

  const CategoricalVariable &get(const std::map<uint, CategoricalVariable> &m,
                                 uint i) {
    return m.find(i)->second;
  }

  inline void field_length_error(const std::string &fname, uint line, uint nfields,
                                 uint prev_nfields) {
    std::ostringstream msg;
    msg << "file: " << fname << endl
        << " line number " << line << " has " << nfields
        << " fields.  Previous lines had " << prev_nfields << "fields." << endl;
    report_error(msg.str());
  }

  //-----------------------------------------------------------------
  inline void wrong_type_error(uint line_num, uint field_num) {
    std::ostringstream msg;
    msg << "line number " << line_num << " field number " << field_num << endl;
    report_error(msg.str());
  }

  inline void unknown_type() { report_error("unknown type"); }
  //-----------------------------------------------------------------

  DataTable::DataTable() {}

  DataTable::DataTable(const std::string &fname,
                       bool header,
                       const std::string &sep) {
    ifstream in(fname.c_str());
    if (!in) {
      std::string msg = "bad file name ";
      report_error(msg + fname);
    }

    StringSplitter split(sep);
    std::string line;
    uint nfields = 0;
    uint line_number = 0;

    std::vector<std::vector<std::string>> categorical_data;
    std::vector<Vector> numeric_data;
<commit_msg>Add data organizer to constructor.<commit_after>// Copyright 2018 Google LLC. All Rights Reserved.
/*
  Copyright (C) 2005 <NAME>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
*/

#include "stats/DataTable.hpp"
#include "stats/moments.hpp"

#include <cctype>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>

#include "Models/CategoricalData.hpp"
#include "cpputil/DefaultVnames.hpp"
#include "cpputil/Ptr.hpp"
#include "cpputil/math_utils.hpp"
#include "cpputil/report_error.hpp"
#include "cpputil/string_utils.hpp"

namespace BOOM {
  using std::endl;


  //---------------------------------------------------------------------
  // Determine the type of variable stored in vs.

  MixedDataOrganizer::MixedDataOrganizer()
      : numeric_count_(0),
        categorical_count_(0),
        unknown_count_(0)
  {}

  void MixedDataOrganizer::add_variable(VariableType type) {
    int index = variable_types_.size();
    variable_types_.push_back(type);
    if (type == VariableType::numeric) {
      type_map_[index] = std::make_pair(type, numeric_count_++);
    } else if (type == VariableType::categorical) {
      type_map_[index] = std::make_pair(type, categorical_count_++);
    } else {
      ++unknown_count_;
      report_error("Numeric and categorical the the only currently supported"
                   " types.");
    }
  }

  std::pair<VariableType, int> MixedDataOrganizer::type_map(int i) const {
    auto it = type_map_.find(i);
    if (it == type_map_.end()) {
      return std::make_pair(VariableType::unknown, -1);
    } else {
      return it->second;
    }
  }

  void MixedDataOrganizer::diagnose_types(const std::vector<std::string> &vs) {
    uint nfields = vs.size();
    variable_types_ = std::vector<VariableType>(nfields, VariableType::unknown);
    for (uint i = 0; i < vs.size(); ++i) {
      variable_types_[i] =
          is_numeric(vs[i]) ? VariableType::numeric : VariableType::categorical;
    }
  }

  bool MixedDataOrganizer::check_type(int i, const std::string &s) const {
    VariableType type = variable_types_[i];
    if (is_numeric(s)) {
      if (type == VariableType::numeric) return true;
    } else {  // s is not numeric
      if (type == VariableType::categorical) return true;
    }
    return false;
  }

  bool MixedDataOrganizer::operator==(const MixedDataOrganizer &rhs) const {
    return numeric_count_ == rhs.numeric_count_
        && categorical_count_ == rhs.categorical_count_
        && unknown_count_ == rhs.unknown_count_
        && variable_types_ == rhs.variable_types_;
  }

  //===========================================================================
  CategoricalVariable::CategoricalVariable(
      const std::vector<std::string> &raw_data)
      : key_(make_catkey(raw_data)) {
    for (int i = 0; i < raw_data.size(); ++i) {
      Ptr<CategoricalData> dp = new CategoricalData(raw_data[i], key_);
      data_.push_back(dp);
    }
  }

  const Vector &get(const std::map<uint, Vector> &m, uint i) {
    return m.find(i)->second;
  }

  const CategoricalVariable &get(const std::map<uint, CategoricalVariable> &m,
                                 uint i) {
    return m.find(i)->second;
  }

  inline void field_length_error(const std::string &fname, uint line, uint nfields,
                                 uint prev_nfields) {
    std::ostringstream msg;
    msg << "file: " << fname << endl
        << " line number " << line << " has " << nfields
        << " fields.  Previous lines had " << prev_nfields << "fields." << endl;
    report_error(msg.str());
  }

  //-----------------------------------------------------------------
  inline void wrong_type_error(uint line_num, uint field_num) {
    std::ostringstream msg;
    msg << "line number " << line_num << " field number " << field_num << endl;
    report_error(msg.str());
  }

  inline void unknown_type() { report_error("unknown type"); }
  //-----------------------------------------------------------------

  DataTable::DataTable()
      : data_organizer_(new MixedDataOrganizer)
  {}

  DataTable::DataTable(const std::string &fname,
                       bool header,
                       const std::string &sep)
      : data_organizer_(new MixedDataOrganizer)
  {
    ifstream in(fname.c_str());
    if (!in) {
      std::string msg = "bad file name ";
      report_error(msg + fname);
    }

    StringSplitter split(sep);
    std::string line;
    uint nfields = 0;
    uint line_number = 0;

    std::vector<std::vector<std::string>> categorical_data;
    std::vector<Vector> numeric_data;

{'fraction_non_alphanumeric': 0.16703359816215183, 'fraction_numerical': 0.005551833062123097, 'mean_word_length': 2.818713450292398, 'pattern_counts': {'":': 0, '<': 107, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5174920', 'n_tokens_mistral': 3190, 'n_tokens_neox': 3040, 'n_words': 828}