<commit_before>#include "PcaProductTokenizer.h"
#include <la-manager/TitlePCAWrapper.h>
#include <string>
#include <vector>
#include <map>
#include <set>

using namespace sf1r;
using izenelib::util::UString;

void PcaProductTokenizer::tokenize(ProductTokenParam& param)
{
    typedef std::vector<std::pair<std::string, float> > TokenScoreVec;
    TokenScoreVec tokens, subTokens;
    std::string brand;
    std::string model;

    TitlePCAWrapper* titlePCA = TitlePCAWrapper::get();
    titlePCA->pca(param.query, tokens, brand, model, subTokens, false);

    double scoreSum = 0;
    double maxScore = 0;
    std::string maxToken;

    typedef std::map<std::string, float> TokenScoreMap;
    TokenScoreMap tokenScoreMap;

    for (TokenScoreVec::const_iterator it = tokens.begin();
         it != tokens.end(); ++it)
    {
        if (tokenScoreMap.find(it->first) != tokenScoreMap.end())
            continue;

        tokenScoreMap[it->first] = it->second;
        scoreSum += it->second;

        if (it->second > maxScore)
        {
            maxScore = it->second;
            maxToken = it->first;
        }
    }

    std::vector<std::string> majorTokens;
    if (!maxToken.empty()) majorTokens.push_back(maxToken);
    if (!brand.empty()) majorTokens.push_back(brand);
    if (!model.empty()) majorTokens.push_back(model);

    for (std::vector<std::string>::const_iterator it = majorTokens.begin();
         it != majorTokens.end(); ++it)
    {
        TokenScoreMap::iterator mapIter = tokenScoreMap.find(*it);
        UString ustr(mapIter->first, UString::UTF_8);
        double score = mapIter->second / scoreSum;
        std::pair<UString, double> tokenScore(ustr, score);

        tokenScoreMap.erase(mapIter);
        param.majorTokens.push_back(tokenScore);

        if (param.isRefineResult)
        {
            param.refinedResult.append(ustr);
            param.refinedResult.push_back(SPACE_UCHAR);
        }
    }

    for (TokenScoreMap::const_iterator it = tokenScoreMap.begin();
         it != tokenScoreMap.end(); ++it)
    {
        UString ustr(it->first, UString::UTF_8);
        double score = it->second / scoreSum;
        std::pair<UString, double> tokenScore(ustr, score);<commit_msg>Fix a bug in dcc969e98bae3db7c91c70ea065e0a09783a9cb8.<commit_after>#include "PcaProductTokenizer.h"
#include <la-manager/TitlePCAWrapper.h>
#include <string>
#include <vector>
#include <map>
#include <set>

using namespace sf1r;
using izenelib::util::UString;

void PcaProductTokenizer::tokenize(ProductTokenParam& param)
{
    typedef std::vector<std::pair<std::string, float> > TokenScoreVec;
    TokenScoreVec tokens, subTokens;
    std::string brand;
    std::string model;

    TitlePCAWrapper* titlePCA = TitlePCAWrapper::get();
    titlePCA->pca(param.query, tokens, brand, model, subTokens, false);

    double scoreSum = 0;
    double maxScore = 0;
    std::string maxToken;

    typedef std::map<std::string, float> TokenScoreMap;
    TokenScoreMap tokenScoreMap;

    for (TokenScoreVec::const_iterator it = tokens.begin();
         it != tokens.end(); ++it)
    {
        if (tokenScoreMap.find(it->first) != tokenScoreMap.end())
            continue;

        tokenScoreMap[it->first] = it->second;
        scoreSum += it->second;

        if (it->second > maxScore)
        {
            maxScore = it->second;
            maxToken = it->first;
        }
    }

    std::vector<std::string> majorTokens;
    if (!maxToken.empty() &&
        maxToken != brand && maxToken != model)
    {
        majorTokens.push_back(maxToken);
    }
    if (!brand.empty())
    {
        majorTokens.push_back(brand);
    }
    if (!model.empty())
    {
        majorTokens.push_back(model);
    }

    for (std::vector<std::string>::const_iterator it = majorTokens.begin();
         it != majorTokens.end(); ++it)
    {
        TokenScoreMap::iterator mapIter = tokenScoreMap.find(*it);

        if (mapIter == tokenScoreMap.end())
            continue;

        UString ustr(mapIter->first, UString::UTF_8);
        double score = mapIter->second / scoreSum;
        std::pair<UString, double> tokenScore(ustr, score);

        tokenScoreMap.erase(mapIter);
        param.majorTokens.push_back(tokenScore);

        if (param.isRefineResult)
        {
            param.refinedResult.append(ustr);
            param.refinedResult.push_back(SPACE_UCHAR);
        }
    }

    for (TokenScoreMap::const_iterator it = tokenScoreMap.begin();
         it != tokenScoreMap.end(); ++it)
    {
        UString ustr(it->first, UString::UTF_8);
        double score = it->second / scoreSum;
        std::pair<UString, double> tokenScore(ustr, score);
{'fraction_non_alphanumeric': 0.13660733433411967, 'fraction_numerical': 0.006862534848809779, 'mean_word_length': 2.939189189189189, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4940778', 'n_tokens_mistral': 1485, 'n_tokens_neox': 1468, 'n_words': 275}