<commit_before>#include "ProductScdReceiver.h"
#include <bundles/index/IndexTaskService.h>
#include <boost/lexical_cast.hpp>
#include <boost/format.hpp>

using namespace sf1r;
namespace bfs = boost::filesystem;
ProductScdReceiver::ProductScdReceiver(const std::string& syncID, const std::string& collectionName)
:index_service_(NULL)
,syncID_(syncID)
,collectionName_(collectionName)
{
    syncConsumer_ = SynchroFactory::getConsumer(syncID);
    syncConsumer_->watchProducer(
            boost::bind(&ProductScdReceiver::Run, this, _1),
            collectionName);
}

bool ProductScdReceiver::onReceived(const std::string& scd_source_dir)
{
    if(!index_service_->index(0))
    {
        return false;
    }

    return true;
}

bool ProductScdReceiver::Run(const std::string& scd_source_dir)
{
    LOG(INFO)<<"ProductScdReceiver::Run "<<scd_source_dir<<std::endl;
    if(index_service_==NULL)
    {
        return false;
    }
    if (!scd_source_dir.empty())
    {
        std::string index_scd_dir = index_service_->getScdDir();
        bfs::create_directories(index_scd_dir);
        //copy scds in scd_source_dir/ to index_scd_dir/
        ScdParser parser(izenelib::util::UString::UTF_8);
        static const bfs::directory_iterator kItrEnd;
        std::vector<bfs::path> scd_list;
        for (bfs::directory_iterator itr(scd_source_dir); itr != kItrEnd; ++itr)
        {
            if (bfs::is_regular_file(itr->status()))
            {
                std::string fileName = itr->path().filename().string();
                if (parser.checkSCDFormat(fileName) )
                {
                    LOG(INFO)<<"[ProductScdReceiver] find SCD "<<fileName<<std::endl;
                    scd_list.push_back(itr->path());
                }
            }
        }
        if(scd_list.empty())
        {
            LOG(INFO)<<"[ProductScdReceiver] No SCD file found."<<std::endl;
            return true;
        }
        bfs::path to_dir(index_scd_dir);
        if(!CopyFileListToDir_(scd_list, to_dir))
        {
            return false;
        }
    }

    //call buildCollection
    if (index_service_->getDocNum() < 1)
    {
        return true;
    }

    if(!index_service_->index(0))
    {
        return false;
    }

    return true;
}

bool ProductScdReceiver::CopyFileListToDir_(const std::vector<boost::filesystem::path>& file_list, const boost::filesystem::path& to_dir)
{
    std::vector<bfs::path> copied_file;
    for(uint32_t i=0;i<file_list.size();i++)
    {
        std::string to_filename = file_list[i].filename().string();
        bool failed = true;
        while(true)
        {
            bfs::path to_file = to_dir/to_filename;
            try
            {
                if(!bfs::exists(to_file))
                {
                    bfs::copy_file(file_list[i], to_file);
                    failed = false;
                }
            }<commit_msg>index after get SCD<commit_after>#include "ProductScdReceiver.h"
#include <bundles/index/IndexTaskService.h>
#include <boost/lexical_cast.hpp>
#include <boost/format.hpp>

using namespace sf1r;
namespace bfs = boost::filesystem;
ProductScdReceiver::ProductScdReceiver(const std::string& syncID, const std::string& collectionName)
:index_service_(NULL)
,syncID_(syncID)
,collectionName_(collectionName)
{
    syncConsumer_ = SynchroFactory::getConsumer(syncID);
    syncConsumer_->watchProducer(
            boost::bind(&ProductScdReceiver::Run, this, _1),
            collectionName);
}

bool ProductScdReceiver::onReceived(const std::string& scd_source_dir)
{
    if(!index_service_->index(0))
    {
        return false;
    }

    return true;
}

bool ProductScdReceiver::Run(const std::string& scd_source_dir)
{
    LOG(INFO)<<"ProductScdReceiver::Run "<<scd_source_dir<<std::endl;
    if(index_service_==NULL)
    {
        return false;
    }
    if (!scd_source_dir.empty())
    {
        std::string index_scd_dir = index_service_->getScdDir();
        bfs::create_directories(index_scd_dir);
        //copy scds in scd_source_dir/ to index_scd_dir/
        ScdParser parser(izenelib::util::UString::UTF_8);
        static const bfs::directory_iterator kItrEnd;
        std::vector<bfs::path> scd_list;
        for (bfs::directory_iterator itr(scd_source_dir); itr != kItrEnd; ++itr)
        {
            if (bfs::is_regular_file(itr->status()))
            {
                std::string fileName = itr->path().filename().string();
                if (parser.checkSCDFormat(fileName) )
                {
                    LOG(INFO)<<"[ProductScdReceiver] find SCD "<<fileName<<std::endl;
                    scd_list.push_back(itr->path());
                }
            }
        }
        if(scd_list.empty())
        {
            LOG(INFO)<<"[ProductScdReceiver] No SCD file found."<<std::endl;
            return true;
        }
        bfs::path to_dir(index_scd_dir);
        if(!CopyFileListToDir_(scd_list, to_dir))
        {
            return false;
        }
    }

    if(!index_service_->index(0))
    {
        return false;
    }

    return true;
}

bool ProductScdReceiver::CopyFileListToDir_(const std::vector<boost::filesystem::path>& file_list, const boost::filesystem::path& to_dir)
{
    std::vector<bfs::path> copied_file;
    for(uint32_t i=0;i<file_list.size();i++)
    {
        std::string to_filename = file_list[i].filename().string();
        bool failed = true;
        while(true)
        {
            bfs::path to_file = to_dir/to_filename;
            try
            {
                if(!bfs::exists(to_file))
                {
                    bfs::copy_file(file_list[i], to_file);
                    failed = false;
                }
            }
{'fraction_non_alphanumeric': 0.13012977902490355, 'fraction_numerical': 0.002981410031567871, 'mean_word_length': 2.435542168674699, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6531684', 'n_tokens_mistral': 1841, 'n_tokens_neox': 1813, 'n_words': 274}