<commit_before>#include <iostream>
#include <string>
#include <cstring>
#include <cmath>
#include <ctime>
#include <fstream>
#include <sstream>
#include <exception>
#include <cstring>
//#include <boost/tokenizer.hpp>
#include <boost/circular_buffer.hpp>
#include <boost/filesystem.hpp>
#include <map>
//#include <set>
//#include <unordered_map>
#include <cstdarg>
#include "options.hpp"
#include "basic_utils/stream_reader.hpp"

typedef int64_t Index;
std::string provenance;

#include "basic_utils/utils.hpp"
//#include "basic_utils/stream_reader.hpp"
#include "vocabulary.hpp"

Index cnt_words_processed;
//Index cnt_words_processed_old;

Index cnt_bigrams;
typedef std::map<Index,Index> Accumulator;
std::vector<Accumulator> counters;
Vocabulary vocab;
#include "basic_utils/file_io.hpp"

void accumulate(Accumulator & ac,Index w)
{
    if (ac.find( w ) != ac.end())
        ac[w]++;
    else
    {
        ac.insert(std::make_pair(w,1));
        cnt_bigrams++;
    }
}

void accumulate(Index first, Index second)
{
    if ((first<0)||(second<0)) return;
    //if (second.length()<2) return;
    //if (counters.find( first ) == counters.end())
      //  counters.insert(std::make_pair(first,Accumulator()));
    accumulate(counters[first],second);
}

void load_bigrams(std::string str_path_in,const Options & options)
{
    DirReader dr(str_path_in);
    auto size_window = options.size_window;
    boost::circular_buffer<int64_t> cb(size_window); //- size of n-grams
    for (uint64_t i =0 ; i<size_window; i++) cb.push_back(-1);
    wchar_t * word;
    while ((word=dr.get_word())!=NULL )
    {
        if (vocab.is_word_valid(std::wstring(word)))
        {
            //std::cerr<<wstring_to_utf8(std::wstring(word))<<"\n";
            cnt_words_processed++;
            Index id_current = vocab.get_id(word);
            if (word[0]==L'.')      
            {
                if (options.obey_sentence_bounds)
                {
                    id_current=-1;
                    for (size_t j=1;j<cb.size();j++)
                        cb.push_back(id_current);
                }
                continue;
            }
             //freq_per_id[id_current]++;
            if ((cnt_words_processed % 500000) == 0)
                std::cerr<<"bgrams "<<cnt_words_processed/1000<<"k of "<<vocab.cnt_words_processed/1000<<"k (" <<100.0*cnt_words_processed/vocab.cnt_words_processed<<"%)\n";
            cb.push_back(id_current);
            //auto i = cb.begin();
            auto last = cb[cb.size()-1];
            for (size_t j=0;j<cb.size()-1;j++)
            {
                //std::cerr<<"accing "<<last<<" - "<<cb[j]<<"\n";
                //std::cerr<<"accing "<<wstring_to_utf8(lst_id2word[last])<<" - "<<wstring_to_utf8(lst_id2word[cb[j]])<<"\n";
                accumulate(last,cb[j]);
                accumulate(cb[j],last);
             }
        }
    }
}

struct window_params<commit_msg>use eos and eop markrs<commit_after>#include <iostream>
#include <string>
#include <cstring>
#include <cmath>
#include <ctime>
#include <fstream>
#include <sstream>
#include <exception>
#include <cstring>
//#include <boost/tokenizer.hpp>
#include <boost/circular_buffer.hpp>
#include <boost/filesystem.hpp>
#include <map>
//#include <set>
//#include <unordered_map>
#include <cstdarg>
#include "options.hpp"
#include "basic_utils/stream_reader.hpp"

typedef int64_t Index;
std::string provenance;

#include "basic_utils/utils.hpp"
//#include "basic_utils/stream_reader.hpp"
#include "vocabulary.hpp"

Index cnt_words_processed;
//Index cnt_words_processed_old;

Index cnt_bigrams;
typedef std::map<Index,Index> Accumulator;
std::vector<Accumulator> counters;
Vocabulary vocab;
#include "basic_utils/file_io.hpp"

void accumulate(Accumulator & ac,Index w)
{
    if (ac.find( w ) != ac.end())
        ac[w]++;
    else
    {
        ac.insert(std::make_pair(w,1));
        cnt_bigrams++;
    }
}

void accumulate(Index first, Index second)
{
    if ((first<0)||(second<0)) return;
    //if (second.length()<2) return;
    //if (counters.find( first ) == counters.end())
      //  counters.insert(std::make_pair(first,Accumulator()));
    accumulate(counters[first],second);
}

void load_bigrams(std::string str_path_in,const Options & options)
{
    DirReader dr(str_path_in);
    auto size_window = options.size_window;
    boost::circular_buffer<int64_t> cb(size_window); //- size of n-grams
    for (uint64_t i =0 ; i<size_window; i++) cb.push_back(-1);
    wchar_t * word;
    while ((word=dr.get_word())!=NULL )
    {
        if (vocab.is_word_valid(std::wstring(word)))
        {
            //std::cerr<<wstring_to_utf8(std::wstring(word))<<"\n";
            cnt_words_processed++;
            Index id_current = vocab.get_id(word);
            if ( (word[0]==L'.') ||  (word[0]==L'eos') ||  (word[0]==L'eop') )
            {
                if (options.obey_sentence_bounds)
                {
                    id_current=-1;
                    for (size_t j=1;j<cb.size();j++)
                        cb.push_back(id_current);
                }
                continue;
            }
             //freq_per_id[id_current]++;
            if ((cnt_words_processed % 500000) == 0)
                std::cerr<<"bgrams "<<cnt_words_processed/1000<<"k of "<<vocab.cnt_words_processed/1000<<"k (" <<100.0*cnt_words_processed/vocab.cnt_words_processed<<"%)\n";
            cb.push_back(id_current);
            //auto i = cb.begin();
            auto last = cb[cb.size()-1];
            for (size_t j=0;j<cb.size()-1;j++)
            {
                //std::cerr<<"accing "<<last<<" - "<<cb[j]<<"\n";
                //std::cerr<<"accing "<<wstring_to_utf8(lst_id2word[last])<<" - "<<wstring_to_utf8(lst_id2word[cb[j]])<<"\n";
                accumulate(last,cb[j]);
                accumulate(cb[j],last);
             }
        }
    }
}

struct window_params
{'fraction_non_alphanumeric': 0.16771559789079776, 'fraction_numerical': 0.014628338152747066, 'mean_word_length': 2.6865203761755487, 'pattern_counts': {'":': 0, '<': 129, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4604565', 'n_tokens_mistral': 2171, 'n_tokens_neox': 2080, 'n_words': 333}