<commit_before>#include "Tokenizer.h"
using namespace sf1r::laser::clustering::type;
using namespace ilplib::knlp;

namespace sf1r { namespace laser {

Tokenizer::Tokenizer(const std::string& pcaDict, const std::string& termDict)
    : termDict_(NULL)
    , tok(NULL)
{
    termDict_ = new TermDictionary(termDict);
    tok = new TitlePCA(pcaDict);
}

Tokenizer::~Tokenizer()
{
    if (NULL != termDict_)
    {
        delete termDict_;
        termDict_ = NULL;
    }

    if(tok != NULL)
    {
        delete tok;
        tok = NULL;
    }
}

void Tokenizer::tokenize(const std::string& title, boost::unordered_map<std::string, float>& vec) const
{
    typedef std::pair<std::string, float> TermPair;
    std::vector<TermPair> tks;
    std::vector<TermPair> subtks;
    std::string brand, mdt;
    tok->pca(title, tks, brand, mdt, subtks, false);
    if (!brand.empty())
    {
        tks.push_back(std::make_pair(brand, 0.5));
    }
    if (!mdt.empty())
    {
        tks.push_back(std::make_pair(mdt, 0.5));
    }
    float total = 0;
    for (size_t i = 0; i < tks.size(); ++i)
    {
        std::cout<<tks[i].first<<"\n";
        boost::unordered_map<std::string, std::pair<int, int> >::const_iterator it = termDict_->getTerms().find(tks[i].first);
        if(it != termDict_->getTerms().end())
        {
            std::cout<<"YES\n";
            vec[tks[i].first] = tks[i].second;
            total+=tks[i].second;
        }
    }

    if(total > 0)
    {
        for (boost::unordered_map<std::string, float>::iterator it = vec.begin(); it != vec.end(); it++)
        {
            std::cout<<it->first<<"\n";
            it->second /= total;
        }
    }
    else
    {
        vec.clear();
    }
}
    
void Tokenizer::tokenize(const std::string& title, boost::unordered_map<int, float>& vec) const
{
    typedef std::pair<std::string, float> TermPair;
    std::vector<TermPair> tks;
    std::vector<TermPair> subtks;
    std::string brand, mdt;
    tok->pca(title, tks, brand, mdt, subtks, false);
    
    float total = 0;
    for (size_t i = 0; i < tks.size(); ++i)
    {
        boost::unordered_map<std::string, std::pair<int, int> >::const_iterator it = termDict_->getTerms().find(tks[i].first);
        if(it != termDict_->getTerms().end())
        {
            vec[it->second.second] = tks[i].second;
            total+=tks[i].second;
        }
    }

    if(total > 0)
    {
        for (boost::unordered_map<int, float>::iterator it = vec.begin(); it != vec.end(); it++)<commit_msg>Revert "tiny"<commit_after>#include "Tokenizer.h"
using namespace sf1r::laser::clustering::type;
using namespace ilplib::knlp;

namespace sf1r { namespace laser {

Tokenizer::Tokenizer(const std::string& pcaDict, const std::string& termDict)
    : termDict_(NULL)
    , tok(NULL)
{
    termDict_ = new TermDictionary(termDict);
    tok = new TitlePCA(pcaDict);
}

Tokenizer::~Tokenizer()
{
    if (NULL != termDict_)
    {
        delete termDict_;
        termDict_ = NULL;
    }

    if(tok != NULL)
    {
        delete tok;
        tok = NULL;
    }
}

void Tokenizer::tokenize(const std::string& title, boost::unordered_map<std::string, float>& vec) const
{
    typedef std::pair<std::string, float> TermPair;
    std::vector<TermPair> tks;
    std::vector<TermPair> subtks;
    std::string brand, mdt;
    tok->pca(title, tks, brand, mdt, subtks, false);
    
    float total = 0;
    for (size_t i = 0; i < tks.size(); ++i)
    {
        boost::unordered_map<std::string, std::pair<int, int> >::const_iterator it = termDict_->getTerms().find(tks[i].first);
        if(it != termDict_->getTerms().end())
        {
            vec[tks[i].first] = tks[i].second;
            total+=tks[i].second;
        }
    }

    if(total > 0)
    {
        for (boost::unordered_map<std::string, float>::iterator it = vec.begin(); it != vec.end(); it++)
        {
            it->second /= total;
        }
    }
    else
    {
        vec.clear();
    }
}
    
void Tokenizer::tokenize(const std::string& title, boost::unordered_map<int, float>& vec) const
{
    typedef std::pair<std::string, float> TermPair;
    std::vector<TermPair> tks;
    std::vector<TermPair> subtks;
    std::string brand, mdt;
    tok->pca(title, tks, brand, mdt, subtks, false);
    
    float total = 0;
    for (size_t i = 0; i < tks.size(); ++i)
    {
        boost::unordered_map<std::string, std::pair<int, int> >::const_iterator it = termDict_->getTerms().find(tks[i].first);
        if(it != termDict_->getTerms().end())
        {
            vec[it->second.second] = tks[i].second;
            total+=tks[i].second;
        }
    }

    if(total > 0)
    {
        for (boost::unordered_map<int, float>::iterator it = vec.begin(); it != vec.end(); it++)
{'fraction_non_alphanumeric': 0.17355546183045129, 'fraction_numerical': 0.004217629692113032, 'mean_word_length': 2.6569005397070162, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2352039', 'n_tokens_mistral': 1703, 'n_tokens_neox': 1669, 'n_words': 323}