<commit_before>/*
 *  Data.cpp
 *  met - Minimum Error Training
 *
 *  Created by <NAME> on 13/05/08.
 *
 */

#include <algorithm>
#include "util/check.hh"
#include <cmath>
#include <fstream>

#include "Data.h"
#include "FileStream.h"
#include "Scorer.h"
#include "ScorerFactory.h"
#include "Util.h"

Data::Data()
  : theScorer(NULL),
    number_of_scores(0),
    _sparse_flag(false),
    scoredata(NULL),
    featdata(NULL) {}

Data::Data(Scorer& ptr)
    : theScorer(&ptr),
      score_type(theScorer->getName()),
      number_of_scores(0),
      _sparse_flag(false),
      scoredata(new ScoreData(*theScorer)),
      featdata(new FeatureData)
{
  TRACE_ERR("Data::score_type " << score_type << std::endl);
  TRACE_ERR("Data::Scorer type from Scorer: " << theScorer->getName() << endl);
}

Data::~Data() {
  if (featdata) {
    delete featdata;
    featdata = NULL;
  }
  if (scoredata) {
    delete scoredata;
    scoredata = NULL;
  }
}

//ADDED BY TS
void Data::remove_duplicates() {

  uint nSentences = featdata->size();
  assert(scoredata->size() == nSentences);

  for (uint s=0; s < nSentences; s++) {

    FeatureArray& feat_array =  featdata->get(s);
    ScoreArray& score_array =  scoredata->get(s);

    assert(feat_array.size() == score_array.size());

    //serves as a hash-map:
    std::map<double, std::vector<uint> > lookup;

    uint end_pos = feat_array.size() - 1;

    uint nRemoved = 0;
    for (uint k=0; k <= end_pos; k++) {

      const FeatureStats& cur_feats = feat_array.get(k);

      double sum = 0.0;
      for (uint l=0; l < cur_feats.size(); l++)
	sum += cur_feats.get(l);

      if (lookup.find(sum) != lookup.end()) {

	//std::cerr << "hit" << std::endl;

	std::vector<uint>& cur_list = lookup[sum];

	uint l=0;
	for (l=0; l < cur_list.size(); l++) {
	  
	  uint j=cur_list[l];

	  if (cur_feats == feat_array.get(j)
	      && score_array.get(k) == score_array.get(j)) {

	    if (k < end_pos) {
	      
	      feat_array.swap(k,end_pos);
	      score_array.swap(k,end_pos);
	      
	      k--;
	    }
	    
	    end_pos--;
	    nRemoved++;
	    break;
	  }
	}

	if (l == lookup[sum].size())
	  cur_list.push_back(k);
      }
      else
	lookup[sum].push_back(k);

      // for (uint j=0; j < k; j++) {

      // 	if (feat_array.get(k) == feat_array.get(j)
      // 	    && score_array.get(k) == score_array.get(j)) {

      // 	  if (k < end_pos) {

      // 	    feat_array.swap(k,end_pos);
      // 	    score_array.swap(k,end_pos);

      // 	    k--;
      // 	  }<commit_msg>uint -> size_t<commit_after>/*
 *  Data.cpp
 *  met - Minimum Error Training
 *
 *  Created by <NAME> on 13/05/08.
 *
 */

#include <algorithm>
#include "util/check.hh"
#include <cmath>
#include <fstream>

#include "Data.h"
#include "FileStream.h"
#include "Scorer.h"
#include "ScorerFactory.h"
#include "Util.h"

Data::Data()
  : theScorer(NULL),
    number_of_scores(0),
    _sparse_flag(false),
    scoredata(NULL),
    featdata(NULL) {}

Data::Data(Scorer& ptr)
    : theScorer(&ptr),
      score_type(theScorer->getName()),
      number_of_scores(0),
      _sparse_flag(false),
      scoredata(new ScoreData(*theScorer)),
      featdata(new FeatureData)
{
  TRACE_ERR("Data::score_type " << score_type << std::endl);
  TRACE_ERR("Data::Scorer type from Scorer: " << theScorer->getName() << endl);
}

Data::~Data() {
  if (featdata) {
    delete featdata;
    featdata = NULL;
  }
  if (scoredata) {
    delete scoredata;
    scoredata = NULL;
  }
}

//ADDED BY TS
void Data::remove_duplicates() {

  size_t nSentences = featdata->size();
  assert(scoredata->size() == nSentences);

  for (size_t s=0; s < nSentences; s++) {

    FeatureArray& feat_array =  featdata->get(s);
    ScoreArray& score_array =  scoredata->get(s);

    assert(feat_array.size() == score_array.size());

    //serves as a hash-map:
    std::map<double, std::vector<size_t> > lookup;

    size_t end_pos = feat_array.size() - 1;

    size_t nRemoved = 0;
    for (size_t k=0; k <= end_pos; k++) {

      const FeatureStats& cur_feats = feat_array.get(k);

      double sum = 0.0;
      for (size_t l=0; l < cur_feats.size(); l++)
	sum += cur_feats.get(l);

      if (lookup.find(sum) != lookup.end()) {

	//std::cerr << "hit" << std::endl;

	std::vector<size_t>& cur_list = lookup[sum];

	size_t l=0;
	for (l=0; l < cur_list.size(); l++) {
	  
	  size_t j=cur_list[l];

	  if (cur_feats == feat_array.get(j)
	      && score_array.get(k) == score_array.get(j)) {

	    if (k < end_pos) {
	      
	      feat_array.swap(k,end_pos);
	      score_array.swap(k,end_pos);
	      
	      k--;
	    }
	    
	    end_pos--;
	    nRemoved++;
	    break;
	  }
	}

	if (l == lookup[sum].size())
	  cur_list.push_back(k);
      }
      else
	lookup[sum].push_back(k);

      // for (size_t j=0; j < k; j++) {

      // 	if (feat_array.get(k) == feat_array.get(j)
      // 	    && score_array.get(k) == score_array.get(j)) {

      // 	  if (k < end_pos) {

      // 	    feat_array.swap(k,end_pos);
      // 	    score_array.swap(k,end_pos);

      // 	    k--;
      // 	  }
{'fraction_non_alphanumeric': 0.1786208259237305, 'fraction_numerical': 0.007113218731475993, 'mean_word_length': 2.6105563480741796, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2916435', 'n_tokens_mistral': 2069, 'n_tokens_neox': 1967, 'n_words': 363}