<commit_before>#ifndef GUARD_cluster_h
#define GUARD_cluster_h

// #include <iostream>
#include <string>
#include <sstream>
#include <vector>
#include <map>
#include <set>
#include <stdio.h>
#include "assert.h"
//
#include "utils.h"
#include "Suffstats.h"


template <class T> class Suffstats;
template <class T> class Cluster;
template <typename T> std::ostream& operator<<(std::ostream& os,
					       const Cluster<T>& cT);

template <class T>
class Cluster {
 public:
  Cluster<T>(): num_cols(5) { init_suffstats(); };
  Cluster<T>(int NUM_COLS): num_cols(NUM_COLS) { init_suffstats(); };
  double insert_row(std::vector<T> vT, int row_idx);
  double remove_row(std::vector<T> vT, int row_idx);
  double calc_data_logp(std::vector<T> vT) const;
  double get_score() const;
  int get_count() const;
  std::set<int> get_global_row_indices();
  std::set<int> get_global_col_indices();
  friend std::ostream& operator<< <>(std::ostream& os, const Cluster<T>& cT);
  Suffstats<T> get_suffstats_i(int idx) const;
  //
  std::map<int, double> calc_logps();
  double calc_sum_logp();
  double get_vector_logp(std::vector<T> vT); // to be removed when test ensures calc_data_logps correctness
 private:
  double score;
  int num_cols;
  int count;
  void init_suffstats();
  std::map<int, Suffstats<T> > suffstats_m;
  std::set<int> global_row_indices;
  std::set<int> global_col_indices;
};

template <class T>
std::map<int, double> Cluster<T>::calc_logps() {
  std::map<int, double> ret_map;
  typename std::map<int, Suffstats<T> >::iterator it = suffstats_m.begin();
  for(; it!=suffstats_m.end(); it++) {
    double logp = it->second.calc_logp();
    ret_map[it->first] = logp;
  }
  return ret_map;
}

template <class T>
double Cluster<T>::calc_sum_logp() {
  double sum_logp = 0;
  std::map<int, double> logp_map = calc_logps();
  typename std::map<int, double>::iterator it = logp_map.begin();
  for(; it!=logp_map.end(); it++) {
    sum_logp += it->second;
  }
  return sum_logp;
}

template <class T>
Suffstats<T> Cluster<T>::get_suffstats_i(int idx) const {
  typename std::map<int, Suffstats<T> >::const_iterator it = \
    suffstats_m.find(idx);
  if(it == suffstats_m.end()) {
    // FIXME : how to fail properly?
    assert(1==0);
    Suffstats<T> st;
    return st;
  }
  return it->second;
}

template <class T>
std::set<int> Cluster<T>::get_global_row_indices() {
  return global_row_indices;
}

template <class T>
std::set<int> Cluster<T>::get_global_col_indices() {
  return global_col_indices;
}

template <class T>
double Cluster<T>::get_score() const {
  return score;
}

template <class T>
int Cluster<T>::get_count() const {
  return count;
}

template <typename T>
std::ostream& operator<<(std::ostream& os, const Cluster<T>& cT) {
  typename std::map<int, Suffstats<T> >::const_iterator it = cT.suffstats_m.begin();
  for(; it!= cT.suffstats_m.end(); it++) {
    os << "suffstats idx: " << it->first << " :: ";
    os <<  cT.global_row_indices << " :: " << it->second;
  }
  os << "========" << std::endl;
  os <<"cluster score: " << cT.get_score() << std::endl;
  return os;
}

#endif // GUARD_cluster_h<commit_msg>add endl after each cluster print<commit_after>#ifndef GUARD_cluster_h
#define GUARD_cluster_h

// #include <iostream>
#include <string>
#include <sstream>
#include <vector>
#include <map>
#include <set>
#include <stdio.h>
#include "assert.h"
//
#include "utils.h"
#include "Suffstats.h"


template <class T> class Suffstats;
template <class T> class Cluster;
template <typename T> std::ostream& operator<<(std::ostream& os,
					       const Cluster<T>& cT);

template <class T>
class Cluster {
 public:
  Cluster<T>(): num_cols(5) { init_suffstats(); };
  Cluster<T>(int NUM_COLS): num_cols(NUM_COLS) { init_suffstats(); };
  double insert_row(std::vector<T> vT, int row_idx);
  double remove_row(std::vector<T> vT, int row_idx);
  double calc_data_logp(std::vector<T> vT) const;
  double get_score() const;
  int get_count() const;
  std::set<int> get_global_row_indices();
  std::set<int> get_global_col_indices();
  friend std::ostream& operator<< <>(std::ostream& os, const Cluster<T>& cT);
  Suffstats<T> get_suffstats_i(int idx) const;
  //
  std::map<int, double> calc_logps();
  double calc_sum_logp();
  double get_vector_logp(std::vector<T> vT); // to be removed when test ensures calc_data_logps correctness
 private:
  double score;
  int num_cols;
  int count;
  void init_suffstats();
  std::map<int, Suffstats<T> > suffstats_m;
  std::set<int> global_row_indices;
  std::set<int> global_col_indices;
};

template <class T>
std::map<int, double> Cluster<T>::calc_logps() {
  std::map<int, double> ret_map;
  typename std::map<int, Suffstats<T> >::iterator it = suffstats_m.begin();
  for(; it!=suffstats_m.end(); it++) {
    double logp = it->second.calc_logp();
    ret_map[it->first] = logp;
  }
  return ret_map;
}

template <class T>
double Cluster<T>::calc_sum_logp() {
  double sum_logp = 0;
  std::map<int, double> logp_map = calc_logps();
  typename std::map<int, double>::iterator it = logp_map.begin();
  for(; it!=logp_map.end(); it++) {
    sum_logp += it->second;
  }
  return sum_logp;
}

template <class T>
Suffstats<T> Cluster<T>::get_suffstats_i(int idx) const {
  typename std::map<int, Suffstats<T> >::const_iterator it = \
    suffstats_m.find(idx);
  if(it == suffstats_m.end()) {
    // FIXME : how to fail properly?
    assert(1==0);
    Suffstats<T> st;
    return st;
  }
  return it->second;
}

template <class T>
std::set<int> Cluster<T>::get_global_row_indices() {
  return global_row_indices;
}

template <class T>
std::set<int> Cluster<T>::get_global_col_indices() {
  return global_col_indices;
}

template <class T>
double Cluster<T>::get_score() const {
  return score;
}

template <class T>
int Cluster<T>::get_count() const {
  return count;
}

template <typename T>
std::ostream& operator<<(std::ostream& os, const Cluster<T>& cT) {
  typename std::map<int, Suffstats<T> >::const_iterator it = cT.suffstats_m.begin();
  for(; it!= cT.suffstats_m.end(); it++) {
    os << "suffstats idx: " << it->first << " :: ";
    os <<  cT.global_row_indices << " :: " << it->second << std::endl;
  }
  os << "========" << std::endl;
  os <<"cluster score: " << cT.get_score() << std::endl;
  return os;
}

#endif // GUARD_cluster_h
{'fraction_non_alphanumeric': 0.17213504517356157, 'fraction_numerical': 0.0012680297987002695, 'mean_word_length': 4.8861940298507465, 'pattern_counts': {'":': 0, '<': 177, '<?xml version=': 0, '>': 131, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2329764', 'n_tokens_mistral': 2442, 'n_tokens_neox': 2367, 'n_words': 539}