<commit_before>#include "ZambeziSearch.h"
#include "ZambeziFilter.h"
#include "ZambeziScoreNormalizer.h"
#include "SearchManagerPreProcessor.h"
#include "Sorter.h"
#include "QueryBuilder.h"
#include "HitQueue.h"
#include <la-manager/AttrTokenizeWrapper.h>
#include <query-manager/SearchKeywordOperation.h>
#include <query-manager/QueryTypeDef.h> // FilteringType
#include <common/ResultType.h>
#include <common/ResourceManager.h>
#include <common/PropSharedLockSet.h>
#include <document-manager/DocumentManager.h>
#include <mining-manager/MiningManager.h>
#include <mining-manager/zambezi-manager/ZambeziManager.h>
#include <mining-manager/group-manager/GroupFilterBuilder.h>
#include <mining-manager/group-manager/GroupFilter.h>
#include <mining-manager/product-scorer/ProductScorer.h>
#include <mining-manager/util/convert_ustr.h>
#include <b5m-manager/product_matcher.h>
#include <ir/index_manager/utility/BitVector.h>
#include <util/ClockTimer.h>
#include <glog/logging.h>
#include <iostream>
#include <set>
#include <boost/scoped_ptr.hpp>

namespace sf1r
{

namespace
{
const std::size_t kAttrTopDocNum = 200;
const std::size_t kZambeziTopKNum = 5e5;

const std::string kTopLabelPropName = "Category";
const size_t kRootCateNum = 10;

const std::string kMerchantPropName = "Source";
const izenelib::util::UString::EncodingType kEncodeType =
    izenelib::util::UString::UTF_8;
const izenelib::util::UString kAttrExcludeMerchant =
    izenelib::util::UString("淘宝网", kEncodeType);

const izenelib::util::UString::CharT kUCharSpace = ' ';

const MonomorphicFilter<true> kAllPassFilter;
}

ZambeziSearch::ZambeziSearch(
    DocumentManager& documentManager,
    SearchManagerPreProcessor& preprocessor,
    QueryBuilder& queryBuilder)
    : documentManager_(documentManager)
    , preprocessor_(preprocessor)
    , queryBuilder_(queryBuilder)
    , groupFilterBuilder_(NULL)
    , zambeziManager_(NULL)
    , categoryValueTable_(NULL)
    , merchantValueTable_(NULL)
{
}

void ZambeziSearch::setMiningManager(
    const boost::shared_ptr<MiningManager>& miningManager)
{
    groupFilterBuilder_ = miningManager->GetGroupFilterBuilder();
    zambeziManager_ = miningManager->getZambeziManager();
    categoryValueTable_ = miningManager->GetPropValueTable(kTopLabelPropName);
    merchantValueTable_ = miningManager->GetPropValueTable(kMerchantPropName);
    zambeziScoreNormalizer_.reset(new ZambeziScoreNormalizer(*miningManager));
}

bool ZambeziSearch::search(
    const SearchKeywordOperation& actionOperation,
    KeywordSearchResult& searchResult,
    std::size_t limit,
    std::size_t offset)
{
    const std::string& query = actionOperation.actionItem_.env_.queryString_;
    LOG(INFO) << "zambezi search for query: " << query;

    if (query.empty())
        return false;

    std::vector<docid_t> candidates;
    std::vector<uint32_t> scores;

    if (!zambeziManager_)
    {
        LOG(WARNING) << "the instance of ZambeziManager is empty";
        return false;
    }

    faceted::GroupParam& groupParam = actionOperation.actionItem_.groupParam_;
    const bool originIsAttrGroup = groupParam.isAttrGroup_;
    groupParam.isAttrGroup_ = false;

    PropSharedLockSet propSharedLockSet;
    boost::scoped_ptr<ProductScorer> productScorer(
        preprocessor_.createProductScorer(actionOperation.actionItem_, propSharedLockSet, NULL));

    boost::shared_ptr<faceted::GroupFilter> groupFilter;
    if (groupFilterBuilder_)
    {
        groupFilter.reset(
            groupFilterBuilder_->createFilter(groupParam, propSharedLockSet));
    }

    ConditionsNode& filterTree =
        actionOperation.actionItem_.filterTree_;

    boost::shared_ptr<InvertedIndexManager::FilterBitmapT> filterBitmap;
    boost::shared_ptr<izenelib::ir::indexmanager::BitVector> filterBitVector;

    if (!filterTree.empty())
    {
        queryBuilder_.prepare_filter(filterTree, filterBitmap);
        filterBitVector.reset(new izenelib::ir::indexmanager::BitVector);
        filterBitVector->importFromEWAH(*filterBitmap);
    }

    AttrTokenizeWrapper* attrTokenize = AttrTokenizeWrapper::get();
    std::vector<std::pair<std::string, int> > tokenList;
    attrTokenize->attr_tokenize(query, tokenList);
    getAnalyzedQuery_(query, searchResult.analyzedQuery_);

    zambeziManager_->search(tokenList, kAllPassFilter, kZambeziTopKNum,
                            candidates, scores);

    if (candidates.empty())
    {
        std::vector<std::pair<std::string, int> > subTokenList;
        if (attrTokenize->attr_subtokenize(tokenList, subTokenList))
        {
            zambeziManager_->search(subTokenList, kAllPassFilter, kZambeziTopKNum,
                                    candidates, scores);
        }
    }

    if (candidates.empty())
    {
        LOG(INFO) << "empty search result for query: " << query;
        return false;
    }

    if (candidates.size() != scores.size())
    {
        LOG(WARNING) << "mismatch size of candidate docid and score";
        return false;
    }

    //normalize relevance scores
    if (tokenList.size() > 1)
    {
        uint32_t normalizerScore = 0;
        for (std::vector<std::pair<std::string, int> >::iterator i = tokenList.begin();
            i != tokenList.end(); ++i)
        {
            normalizerScore += i->second;
        }
        for (std::vector<uint32_t>::iterator i = scores.begin(); i != scores.end(); ++i)
        {
            (*i) = (*i)/normalizerScore;
        }
    }

    izenelib::util::ClockTimer timer;

    boost::shared_ptr<Sorter> sorter;
    CustomRankerPtr customRanker;
    preprocessor_.prepareSorterCustomRanker(actionOperation,
                                            sorter,
                                            customRanker);

    boost::scoped_ptr<HitQueue> scoreItemQueue;
    const std::size_t heapSize = limit + offset;

    if (sorter)
    {
        scoreItemQueue.reset(new PropertySortedHitQueue(sorter,
                                                        heapSize,
                                                        propSharedLockSet));
    }<commit_msg>fix Arithmetic exception in ZambeziSearch temporarily<commit_after>#include "ZambeziSearch.h"
#include "ZambeziFilter.h"
#include "ZambeziScoreNormalizer.h"
#include "SearchManagerPreProcessor.h"
#include "Sorter.h"
#include "QueryBuilder.h"
#include "HitQueue.h"
#include <la-manager/AttrTokenizeWrapper.h>
#include <query-manager/SearchKeywordOperation.h>
#include <query-manager/QueryTypeDef.h> // FilteringType
#include <common/ResultType.h>
#include <common/ResourceManager.h>
#include <common/PropSharedLockSet.h>
#include <document-manager/DocumentManager.h>
#include <mining-manager/MiningManager.h>
#include <mining-manager/zambezi-manager/ZambeziManager.h>
#include <mining-manager/group-manager/GroupFilterBuilder.h>
#include <mining-manager/group-manager/GroupFilter.h>
#include <mining-manager/product-scorer/ProductScorer.h>
#include <mining-manager/util/convert_ustr.h>
#include <b5m-manager/product_matcher.h>
#include <ir/index_manager/utility/BitVector.h>
#include <util/ClockTimer.h>
#include <glog/logging.h>
#include <iostream>
#include <set>
#include <boost/scoped_ptr.hpp>

namespace sf1r
{

namespace
{
const std::size_t kAttrTopDocNum = 200;
const std::size_t kZambeziTopKNum = 5e5;

const std::string kTopLabelPropName = "Category";
const size_t kRootCateNum = 10;

const std::string kMerchantPropName = "Source";
const izenelib::util::UString::EncodingType kEncodeType =
    izenelib::util::UString::UTF_8;
const izenelib::util::UString kAttrExcludeMerchant =
    izenelib::util::UString("淘宝网", kEncodeType);

const izenelib::util::UString::CharT kUCharSpace = ' ';

const MonomorphicFilter<true> kAllPassFilter;
}

ZambeziSearch::ZambeziSearch(
    DocumentManager& documentManager,
    SearchManagerPreProcessor& preprocessor,
    QueryBuilder& queryBuilder)
    : documentManager_(documentManager)
    , preprocessor_(preprocessor)
    , queryBuilder_(queryBuilder)
    , groupFilterBuilder_(NULL)
    , zambeziManager_(NULL)
    , categoryValueTable_(NULL)
    , merchantValueTable_(NULL)
{
}

void ZambeziSearch::setMiningManager(
    const boost::shared_ptr<MiningManager>& miningManager)
{
    groupFilterBuilder_ = miningManager->GetGroupFilterBuilder();
    zambeziManager_ = miningManager->getZambeziManager();
    categoryValueTable_ = miningManager->GetPropValueTable(kTopLabelPropName);
    merchantValueTable_ = miningManager->GetPropValueTable(kMerchantPropName);
    zambeziScoreNormalizer_.reset(new ZambeziScoreNormalizer(*miningManager));
}

bool ZambeziSearch::search(
    const SearchKeywordOperation& actionOperation,
    KeywordSearchResult& searchResult,
    std::size_t limit,
    std::size_t offset)
{
    const std::string& query = actionOperation.actionItem_.env_.queryString_;
    LOG(INFO) << "zambezi search for query: " << query;

    if (query.empty())
        return false;

    std::vector<docid_t> candidates;
    std::vector<uint32_t> scores;

    if (!zambeziManager_)
    {
        LOG(WARNING) << "the instance of ZambeziManager is empty";
        return false;
    }

    faceted::GroupParam& groupParam = actionOperation.actionItem_.groupParam_;
    const bool originIsAttrGroup = groupParam.isAttrGroup_;
    groupParam.isAttrGroup_ = false;

    PropSharedLockSet propSharedLockSet;
    boost::scoped_ptr<ProductScorer> productScorer(
        preprocessor_.createProductScorer(actionOperation.actionItem_, propSharedLockSet, NULL));

    boost::shared_ptr<faceted::GroupFilter> groupFilter;
    if (groupFilterBuilder_)
    {
        groupFilter.reset(
            groupFilterBuilder_->createFilter(groupParam, propSharedLockSet));
    }

    ConditionsNode& filterTree =
        actionOperation.actionItem_.filterTree_;

    boost::shared_ptr<InvertedIndexManager::FilterBitmapT> filterBitmap;
    boost::shared_ptr<izenelib::ir::indexmanager::BitVector> filterBitVector;

    if (!filterTree.empty())
    {
        queryBuilder_.prepare_filter(filterTree, filterBitmap);
        filterBitVector.reset(new izenelib::ir::indexmanager::BitVector);
        filterBitVector->importFromEWAH(*filterBitmap);
    }

    AttrTokenizeWrapper* attrTokenize = AttrTokenizeWrapper::get();
    std::vector<std::pair<std::string, int> > tokenList;
    attrTokenize->attr_tokenize(query, tokenList);
    getAnalyzedQuery_(query, searchResult.analyzedQuery_);

    zambeziManager_->search(tokenList, kAllPassFilter, kZambeziTopKNum,
                            candidates, scores);

    if (candidates.empty())
    {
        std::vector<std::pair<std::string, int> > subTokenList;
        if (attrTokenize->attr_subtokenize(tokenList, subTokenList))
        {
            zambeziManager_->search(subTokenList, kAllPassFilter, kZambeziTopKNum,
                                    candidates, scores);
        }
    }

    if (candidates.empty())
    {
        LOG(INFO) << "empty search result for query: " << query;
        return false;
    }

    if (candidates.size() != scores.size())
    {
        LOG(WARNING) << "mismatch size of candidate docid and score";
        return false;
    }

    //normalize relevance scores
    if (tokenList.size() > 1)
    {
        uint32_t normalizerScore = 0;
        for (std::vector<std::pair<std::string, int> >::iterator i = tokenList.begin();
            i != tokenList.end(); ++i)
        {
            normalizerScore += i->second;
        }
        normalizerScore = normalizerScore == 0 ? 1 : normalizerScore;
        for (std::vector<uint32_t>::iterator i = scores.begin(); i != scores.end(); ++i)
        {
            (*i) = (*i)/normalizerScore;
        }
    }

    izenelib::util::ClockTimer timer;

    boost::shared_ptr<Sorter> sorter;
    CustomRankerPtr customRanker;
    preprocessor_.prepareSorterCustomRanker(actionOperation,
                                            sorter,
                                            customRanker);

    boost::scoped_ptr<HitQueue> scoreItemQueue;
    const std::size_t heapSize = limit + offset;

    if (sorter)
    {
        scoreItemQueue.reset(new PropertySortedHitQueue(sorter,
                                                        heapSize,
                                                        propSharedLockSet));
    }
{'fraction_non_alphanumeric': 0.11199869579393544, 'fraction_numerical': 0.0030974894033257254, 'mean_word_length': 3.57286619455833, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7239656', 'n_tokens_mistral': 3766, 'n_tokens_neox': 3682, 'n_words': 614}