<commit_before>#include "property_diversity_reranker.h"

#include <mining-manager/group-label-logger/GroupLabelLogger.h>

#include <list>
#include <algorithm>
#include <glog/logging.h>

using sf1r::GroupLabelLogger;

NS_FACETED_BEGIN

PropertyDiversityReranker::PropertyDiversityReranker(
    const std::string& property,
    const GroupManager::PropValueMap& propValueMap,
    const std::string& boostingProperty
)
    :property_(property)
    ,propValueMap_(propValueMap)
    ,groupLabelLogger_(NULL)
    ,boostingProperty_(boostingProperty)
{
}

PropertyDiversityReranker::~PropertyDiversityReranker()
{
}

void PropertyDiversityReranker::simplererank(
    std::vector<unsigned int>& docIdList,
    std::vector<float>& rankScoreList
)
{
    typedef std::list<std::pair<unsigned int,float> > DocIdList;
    typedef std::map<PropValueTable::pvid_t, DocIdList> DocIdMap;

    GroupManager::PropValueMap::const_iterator pvIt = propValueMap_.find(property_);
    if (pvIt == propValueMap_.end())
    {
        LOG(ERROR) << "in GroupManager: group index file is not loaded for group property " << property_;
        return;
    }

    const PropValueTable& pvTable = pvIt->second;
    const std::vector<PropValueTable::pvid_t>& idTable = pvTable.propIdTable();

    std::size_t numDoc = docIdList.size();
    std::vector<unsigned int> newDocIdList;
    std::vector<float> newScoreList;	
    newDocIdList.reserve(numDoc);
    newScoreList.reserve(numDoc);

    DocIdMap docIdMap;
    DocIdList missDocs;
    for (std::size_t i = 0; i < numDoc; ++i)
    {
        docid_t docId = docIdList[i];
	
        // this doc has not built group index data
        if (docId >= idTable.size())
            continue;
	
        PropValueTable::pvid_t pvId = idTable[docId];
        if (pvId != 0)
        {
            docIdMap[pvId].push_back(std::make_pair(docId, rankScoreList[i]));
        }
        else
        {
            missDocs.push_back(std::make_pair(docId, rankScoreList[i]));
        }
    }
    if(docIdMap.size() <= 1) return; // single property or empty
    do{
        DocIdMap::iterator mapIt = docIdMap.begin();
        while(mapIt != docIdMap.end())
        {
            if(mapIt->second.empty()) 
            {
                docIdMap.erase(mapIt++);
            }
            else
            {
                std::pair<unsigned int, float> element = mapIt->second.front();
                mapIt->second.pop_front();
                newDocIdList.push_back(element.first);
                newScoreList.push_back(element.second);
                ++mapIt;
            }
        }
    }while(!docIdMap.empty());

    for(DocIdList::iterator missIt = missDocs.begin(); missIt != missDocs.end(); ++missIt)
    {
        newDocIdList.push_back(missIt->first);
        newScoreList.push_back(missIt->second);
    }

    using std::swap;
    swap(docIdList, newDocIdList);
    swap(rankScoreList, newScoreList);
}

void PropertyDiversityReranker::rerank(
    std::vector<unsigned int>& docIdList,
    std::vector<float>& rankScoreList,
    const std::string& query
)
{
    typedef std::list<std::pair<unsigned int,float> > DocIdList;
    typedef std::map<PropValueTable::pvid_t, DocIdList> DocIdMap;

    std::vector<std::string> propValueVec;
    std::vector<int> freqVec;
    if(groupLabelLogger_)
    {
        groupLabelLogger_->getFreqLabel(query, 1, propValueVec, freqVec);
    }

    if(!propValueVec.empty())
    {
        std::string& boostingCategoryLabel = propValueVec[0];
        //std::cout<<"boosting category "<<boostingCategoryLabel<<std::endl;		
        GroupManager::PropValueMap::const_iterator pvIt = propValueMap_.find(boostingProperty_);
        if (pvIt == propValueMap_.end())
        {
            LOG(ERROR) << "in GroupManager: group index file is not loaded for group property " << boostingProperty_;
            simplererank(docIdList, rankScoreList);<commit_msg>print boosting category for ease of debug.<commit_after>#include "property_diversity_reranker.h"

#include <mining-manager/group-label-logger/GroupLabelLogger.h>

#include <list>
#include <algorithm>
#include <glog/logging.h>

using sf1r::GroupLabelLogger;

NS_FACETED_BEGIN

PropertyDiversityReranker::PropertyDiversityReranker(
    const std::string& property,
    const GroupManager::PropValueMap& propValueMap,
    const std::string& boostingProperty
)
    :property_(property)
    ,propValueMap_(propValueMap)
    ,groupLabelLogger_(NULL)
    ,boostingProperty_(boostingProperty)
{
}

PropertyDiversityReranker::~PropertyDiversityReranker()
{
}

void PropertyDiversityReranker::simplererank(
    std::vector<unsigned int>& docIdList,
    std::vector<float>& rankScoreList
)
{
    typedef std::list<std::pair<unsigned int,float> > DocIdList;
    typedef std::map<PropValueTable::pvid_t, DocIdList> DocIdMap;

    GroupManager::PropValueMap::const_iterator pvIt = propValueMap_.find(property_);
    if (pvIt == propValueMap_.end())
    {
        LOG(ERROR) << "in GroupManager: group index file is not loaded for group property " << property_;
        return;
    }

    const PropValueTable& pvTable = pvIt->second;
    const std::vector<PropValueTable::pvid_t>& idTable = pvTable.propIdTable();

    std::size_t numDoc = docIdList.size();
    std::vector<unsigned int> newDocIdList;
    std::vector<float> newScoreList;	
    newDocIdList.reserve(numDoc);
    newScoreList.reserve(numDoc);

    DocIdMap docIdMap;
    DocIdList missDocs;
    for (std::size_t i = 0; i < numDoc; ++i)
    {
        docid_t docId = docIdList[i];
	
        // this doc has not built group index data
        if (docId >= idTable.size())
            continue;
	
        PropValueTable::pvid_t pvId = idTable[docId];
        if (pvId != 0)
        {
            docIdMap[pvId].push_back(std::make_pair(docId, rankScoreList[i]));
        }
        else
        {
            missDocs.push_back(std::make_pair(docId, rankScoreList[i]));
        }
    }
    if(docIdMap.size() <= 1) return; // single property or empty
    do{
        DocIdMap::iterator mapIt = docIdMap.begin();
        while(mapIt != docIdMap.end())
        {
            if(mapIt->second.empty()) 
            {
                docIdMap.erase(mapIt++);
            }
            else
            {
                std::pair<unsigned int, float> element = mapIt->second.front();
                mapIt->second.pop_front();
                newDocIdList.push_back(element.first);
                newScoreList.push_back(element.second);
                ++mapIt;
            }
        }
    }while(!docIdMap.empty());

    for(DocIdList::iterator missIt = missDocs.begin(); missIt != missDocs.end(); ++missIt)
    {
        newDocIdList.push_back(missIt->first);
        newScoreList.push_back(missIt->second);
    }

    using std::swap;
    swap(docIdList, newDocIdList);
    swap(rankScoreList, newScoreList);
}

void PropertyDiversityReranker::rerank(
    std::vector<unsigned int>& docIdList,
    std::vector<float>& rankScoreList,
    const std::string& query
)
{
    typedef std::list<std::pair<unsigned int,float> > DocIdList;
    typedef std::map<PropValueTable::pvid_t, DocIdList> DocIdMap;

    std::vector<std::string> propValueVec;
    std::vector<int> freqVec;
    if(groupLabelLogger_)
    {
        groupLabelLogger_->getFreqLabel(query, 1, propValueVec, freqVec);
    }

    if(!propValueVec.empty())
    {
        std::string& boostingCategoryLabel = propValueVec[0];
        std::cout<<"boosting category "<<boostingCategoryLabel<<std::endl;		
        GroupManager::PropValueMap::const_iterator pvIt = propValueMap_.find(boostingProperty_);
        if (pvIt == propValueMap_.end())
        {
            LOG(ERROR) << "in GroupManager: group index file is not loaded for group property " << boostingProperty_;
            simplererank(docIdList, rankScoreList);
{'fraction_non_alphanumeric': 0.12392889116255276, 'fraction_numerical': 0.0015347231103721704, 'mean_word_length': 3.068678459937565, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7153212', 'n_tokens_mistral': 2539, 'n_tokens_neox': 2467, 'n_words': 431}