<commit_before>#include "SuffixMatchMiningTask.hpp"
#include <document-manager/DocumentManager.h>
#include <boost/filesystem.hpp>
#include <glog/logging.h>
#include <icma/icma.h>
#include <la-manager/LAPool.h>
#include <mining-manager/util/split_ustr.h>
#include "FilterManager.h"
#include "FMIndexManager.h"
namespace sf1r
{

SuffixMatchMiningTask::SuffixMatchMiningTask(
        boost::shared_ptr<DocumentManager>& document_manager,
        boost::shared_ptr<FMIndexManager>& fmi_manager,
        boost::shared_ptr<FilterManager>& filter_manager,
        std::string data_root_path,
        boost::shared_mutex& mutex)
    : document_manager_(document_manager)
    , fmi_manager_(fmi_manager)
    , filter_manager_(filter_manager)
    , data_root_path_(data_root_path)
    , mutex_(mutex)
{
}

SuffixMatchMiningTask::~SuffixMatchMiningTask()
{

}

bool SuffixMatchMiningTask::preProcess()
{
    new_filter_manager.reset(new FilterManager(filter_manager_->getGroupManager(), data_root_path_,
                filter_manager_->getAttrManager(), filter_manager_->getNumericTableBuilder()));
    new_filter_manager->copyPropertyInfo(filter_manager_);
    new_filter_manager->generatePropertyId();

    new_fmi_manager.reset(new FMIndexManager(data_root_path_, document_manager_, new_filter_manager));

    std::vector<std::string> properties;
    for (int i = 0; i < FMIndexManager::FM_TYPE_COUNT; ++i)
    {
        fmi_manager_->getProperties(properties, (FMIndexManager::PropertyFMType)i);
        new_fmi_manager->addProperties(properties, (FMIndexManager::PropertyFMType)i);
        std::vector<std::string>().swap(properties);
    }

    size_t last_docid = fmi_manager_ ? fmi_manager_->docCount() : 0;
    need_rebuild = false;

    std::vector<uint32_t> del_docid_list;
    document_manager_->getDeletedDocIdList(del_docid_list);
    if (last_docid == document_manager_->getMaxDocId())
    {
        // check if there is any new deleted doc.
        std::vector<size_t> doclen_list(del_docid_list.size(), 0);
        fmi_manager_->getDocLenList(del_docid_list, doclen_list);
        for (size_t i = 0; i < doclen_list.size(); ++i)
        {
            if (doclen_list[i] > 0)
            {
                need_rebuild = true;
                break;
            }
        }

        if (!need_rebuild)
        {
            /*
            @brief : in here document_manager_->isThereRtypeDoc() just means for the -R SCD, 
            because the Rtype doc in -U SCD is not in this flow.
            */
            if (document_manager_->isThereRtypeDoc()) 
            {
                LOG (INFO) << "Update for R-type SCD" << endl;
                const std::vector<std::pair<int32_t, std::string> >& prop_list = filter_manager_->getProp_list();
                for (std::vector<std::pair<int32_t, std::string> >::const_iterator i = prop_list.begin(); i != prop_list.end(); ++i)
                {
                    std::set<string>::iterator iter = document_manager_->RtypeDocidPros_.find((*i).second);
                    if (iter == document_manager_->RtypeDocidPros_.end())
                    {
                        new_filter_manager->addUnchangedProperty((*i).second);
                    }
                }
            }
        }
    }
    else
    {
        LOG(INFO) << "old fmi docCount is : " << last_docid << ", document_manager count:" << document_manager_->getMaxDocId();
        need_rebuild = true;
    }

    if (need_rebuild)
    {
        LOG(INFO) << "rebuilding in fm-index is needed.";
    }
    else
    {<commit_msg>fix the bug of when there is no filter property in Rytpe SCD<commit_after>#include "SuffixMatchMiningTask.hpp"
#include <document-manager/DocumentManager.h>
#include <boost/filesystem.hpp>
#include <glog/logging.h>
#include <icma/icma.h>
#include <la-manager/LAPool.h>
#include <mining-manager/util/split_ustr.h>
#include "FilterManager.h"
#include "FMIndexManager.h"
namespace sf1r
{

SuffixMatchMiningTask::SuffixMatchMiningTask(
        boost::shared_ptr<DocumentManager>& document_manager,
        boost::shared_ptr<FMIndexManager>& fmi_manager,
        boost::shared_ptr<FilterManager>& filter_manager,
        std::string data_root_path,
        boost::shared_mutex& mutex)
    : document_manager_(document_manager)
    , fmi_manager_(fmi_manager)
    , filter_manager_(filter_manager)
    , data_root_path_(data_root_path)
    , mutex_(mutex)
{
}

SuffixMatchMiningTask::~SuffixMatchMiningTask()
{

}

bool SuffixMatchMiningTask::preProcess()
{
    new_filter_manager.reset(new FilterManager(filter_manager_->getGroupManager(), data_root_path_,
                filter_manager_->getAttrManager(), filter_manager_->getNumericTableBuilder()));
    new_filter_manager->copyPropertyInfo(filter_manager_);
    new_filter_manager->generatePropertyId();

    new_fmi_manager.reset(new FMIndexManager(data_root_path_, document_manager_, new_filter_manager));

    std::vector<std::string> properties;
    for (int i = 0; i < FMIndexManager::FM_TYPE_COUNT; ++i)
    {
        fmi_manager_->getProperties(properties, (FMIndexManager::PropertyFMType)i);
        new_fmi_manager->addProperties(properties, (FMIndexManager::PropertyFMType)i);
        std::vector<std::string>().swap(properties);
    }

    size_t last_docid = fmi_manager_ ? fmi_manager_->docCount() : 0;
    need_rebuild = false;

    std::vector<uint32_t> del_docid_list;
    document_manager_->getDeletedDocIdList(del_docid_list);
    if (last_docid == document_manager_->getMaxDocId())
    {
        // check if there is any new deleted doc.
        std::vector<size_t> doclen_list(del_docid_list.size(), 0);
        fmi_manager_->getDocLenList(del_docid_list, doclen_list);
        for (size_t i = 0; i < doclen_list.size(); ++i)
        {
            if (doclen_list[i] > 0)
            {
                need_rebuild = true;
                break;
            }
        }

        if (!need_rebuild)
        {
            /*
            @brief : in here document_manager_->isThereRtypeDoc() just means for the -R SCD, 
            because the Rtype doc in -U SCD is not in this flow.
            */
            if (document_manager_->isThereRtypeDoc()) 
            {
                LOG (INFO) << "Update for R-type SCD" << endl;
                const std::vector<std::pair<int32_t, std::string> >& prop_list = filter_manager_->getProp_list();
                uint32_t count = 0;
                for (std::vector<std::pair<int32_t, std::string> >::const_iterator i = prop_list.begin(); i != prop_list.end(); ++i)
                {
                    std::set<string>::iterator iter = document_manager_->RtypeDocidPros_.find((*i).second);
                    if (iter == document_manager_->RtypeDocidPros_.end())
                    {
                        count++;
                        if (!filter_manager_->isNumericProp((*i).second))
                        {
                            new_filter_manager->addUnchangedProperty((*i).second);
                        }
                    }
                }
                if (count == prop_list.size())// means no filter property need rebuild, so return;
                {
                    LOG (INFO) << "All the filter properties do not need rebuild ..." << endl;
                    return false;
                }

            }
        }
    }
    else
    {
        LOG(INFO) << "old fmi docCount is : " << last_docid << ", document_manager count:" << document_manager_->getMaxDocId();
        need_rebuild = true;
    }

    if (need_rebuild)
    {
        LOG(INFO) << "rebuilding in fm-index is needed.";
    }
    else
    {
{'fraction_non_alphanumeric': 0.1191757448484053, 'fraction_numerical': 0.0035437721485759286, 'mean_word_length': 2.565746373420683, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 78, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6441990', 'n_tokens_mistral': 2378, 'n_tokens_neox': 2298, 'n_words': 398}