<commit_before>#include "RecommendBundleActivator.h"
#include "RecommendBundleConfiguration.h"
#include <recommend-manager/UserManager.h>
#include <recommend-manager/ItemManager.h>
#include <recommend-manager/VisitManager.h>
#include <recommend-manager/PurchaseManager.h>
#include <recommend-manager/CartManager.h>
#include <recommend-manager/OrderManager.h>
#include <recommend-manager/EventManager.h>
#include <recommend-manager/RateManager.h>
#include <recommend-manager/RecommenderFactory.h>
#include <recommend-manager/ItemIdGenerator.h>
#include <bundles/index/IndexSearchService.h>

#include <aggregator-manager/SearchWorker.h>

#include <ir/id_manager/IDManager.h>

#include <common/SFLogger.h>

#include <memory> // auto_ptr

#include <glog/logging.h>

namespace bfs = boost::filesystem;

using izenelib::ir::idmanager::IDManager;

namespace sf1r
{

RecommendBundleActivator::RecommendBundleActivator()
    :context_(NULL)
{
}

RecommendBundleActivator::~RecommendBundleActivator()
{
}

void RecommendBundleActivator::start(IBundleContext::ConstPtr context)
{
    context_ = context;
    config_ = static_pointer_cast<RecommendBundleConfiguration>(context_->getBundleConfig());

    indexSearchTracker_.reset(new ServiceTracker(context, "IndexSearchService", this));
    indexSearchTracker_->startTracking();
}

void RecommendBundleActivator::stop(IBundleContext::ConstPtr context)
{
    if(indexSearchTracker_)
    {
        indexSearchTracker_->stopTracking();
        indexSearchTracker_.reset();
    }

    if (taskServiceReg_)
    {
        taskServiceReg_->unregister();
        taskServiceReg_.reset();
        taskService_.reset();
    }

    if (searchServiceReg_)
    {
        searchServiceReg_->unregister();
        searchServiceReg_.reset();
        searchService_.reset();
    }

    userManager_.reset();
    itemManager_.reset();
    visitManager_.reset();
    purchaseManager_.reset();
    cartManager_.reset();
    orderManager_.reset();
    eventManager_.reset();
    rateManager_.reset();
    recommenderFactory_.reset();
    userIdGenerator_.reset();
    itemIdGenerator_.reset();
    coVisitManager_.reset();
    itemCFManager_.reset();
}

bool RecommendBundleActivator::addingService(const ServiceReference& ref)
{
    if (ref.getServiceName() == "IndexSearchService")
    {
        Properties props = ref.getServiceProperties();
        if (props.get("collection") == config_->collectionName_)
        {
            IndexSearchService* service = reinterpret_cast<IndexSearchService*>(const_cast<IService*>(ref.getService()));
            if (! init_(service))
            {
                LOG(ERROR) << "failed in RecommendBundleActivator::init_(), collection:  " << config_->collectionName_;
                return false;
            }

            return true;
        }
    }

    return false;
}

void RecommendBundleActivator::removedService(const ServiceReference& ref)
{
}

bool RecommendBundleActivator::init_(IndexSearchService* indexSearchService)
{
    createSCDDir_();

    if (! createDataDir_())
        return false;

    createUser_();
    createItem_(indexSearchService);
    createMining_();
    createEvent_();
    createOrder_();
    createRecommender_();
    createService_();

    return true;
}

bool RecommendBundleActivator::createDataDir_()
{
    std::string dir;
    if (! openDataDirectory_(dir))
    {
        LOG(ERROR) << "failed in RecommendBundleActivator::openDataDirectory_(), collection: "
                   << config_->collectionName_;
        return false;
    }

    dataDir_ = dir;
    bfs::create_directories(dataDir_);
    return true;
}

bool RecommendBundleActivator::openDataDirectory_(std::string& dataDir)
{
    std::vector<std::string>& directories = config_->collectionDataDirectories_;
    if (directories.size() == 0)
    {
        std::cout<<"no data dir config"<<std::endl;
        return false;
    }

    directoryRotator_.setCapacity(directories.size());
    std::vector<bfs::path> dirtyDirectories;
    typedef std::vector<std::string>::const_iterator iterator;
    bfs::path dataPath = config_->collPath_.getCollectionDataPath();
    for (iterator it = directories.begin(); it != directories.end(); ++it)
    {
        bfs::path dir = dataPath / *it;
        if (!directoryRotator_.appendDirectory(dir))
        {
            std::string msg = dir.file_string() + " corrupted, delete it!";
            LOG(ERROR) <<msg <<endl;
            //clean the corrupt dir
            bfs::remove_all(dir);
        }
    }

    directoryRotator_.rotateToNewest();
    boost::shared_ptr<Directory> newest = directoryRotator_.currentDirectory();
    if (newest)
    {
        dataDir = newest->path().string();
        std::vector<bfs::path>::iterator it = dirtyDirectories.begin();
        for( ; it != dirtyDirectories.end(); ++it)
            directoryRotator_.appendDirectory(*it);
        return true;
    }
    else
    {
        std::vector<bfs::path>::iterator it = dirtyDirectories.begin();
        for( ; it != dirtyDirectories.end(); ++it)
            directoryRotator_.appendDirectory(*it);
		
        directoryRotator_.rotateToNewest();
        boost::shared_ptr<Directory> dir = directoryRotator_.currentDirectory();
        if(dir)
        {
            dataDir = newest->path().string();
            return true;
        }
    }

    return false;
}

void RecommendBundleActivator::createSCDDir_()
{
    bfs::create_directories(config_->userSCDPath());
    bfs::create_directories(config_->orderSCDPath());
}

void RecommendBundleActivator::createUser_()
{
    bfs::path userDir = dataDir_ / "user";
    bfs::create_directory(userDir);
    userManager_.reset(new UserManager((userDir / "user.db").string()));<commit_msg>fix a bug in RecommendBundleActivator::openDataDirectory_(), when cookie is false, although it could remove dirty directory, it fails to generate a new directory.<commit_after>#include "RecommendBundleActivator.h"
#include "RecommendBundleConfiguration.h"
#include <recommend-manager/UserManager.h>
#include <recommend-manager/ItemManager.h>
#include <recommend-manager/VisitManager.h>
#include <recommend-manager/PurchaseManager.h>
#include <recommend-manager/CartManager.h>
#include <recommend-manager/OrderManager.h>
#include <recommend-manager/EventManager.h>
#include <recommend-manager/RateManager.h>
#include <recommend-manager/RecommenderFactory.h>
#include <recommend-manager/ItemIdGenerator.h>
#include <bundles/index/IndexSearchService.h>

#include <aggregator-manager/SearchWorker.h>

#include <ir/id_manager/IDManager.h>

#include <common/SFLogger.h>

#include <memory> // auto_ptr

#include <glog/logging.h>

namespace bfs = boost::filesystem;

using izenelib::ir::idmanager::IDManager;

namespace sf1r
{

RecommendBundleActivator::RecommendBundleActivator()
    :context_(NULL)
{
}

RecommendBundleActivator::~RecommendBundleActivator()
{
}

void RecommendBundleActivator::start(IBundleContext::ConstPtr context)
{
    context_ = context;
    config_ = static_pointer_cast<RecommendBundleConfiguration>(context_->getBundleConfig());

    indexSearchTracker_.reset(new ServiceTracker(context, "IndexSearchService", this));
    indexSearchTracker_->startTracking();
}

void RecommendBundleActivator::stop(IBundleContext::ConstPtr context)
{
    if(indexSearchTracker_)
    {
        indexSearchTracker_->stopTracking();
        indexSearchTracker_.reset();
    }

    if (taskServiceReg_)
    {
        taskServiceReg_->unregister();
        taskServiceReg_.reset();
        taskService_.reset();
    }

    if (searchServiceReg_)
    {
        searchServiceReg_->unregister();
        searchServiceReg_.reset();
        searchService_.reset();
    }

    userManager_.reset();
    itemManager_.reset();
    visitManager_.reset();
    purchaseManager_.reset();
    cartManager_.reset();
    orderManager_.reset();
    eventManager_.reset();
    rateManager_.reset();
    recommenderFactory_.reset();
    userIdGenerator_.reset();
    itemIdGenerator_.reset();
    coVisitManager_.reset();
    itemCFManager_.reset();
}

bool RecommendBundleActivator::addingService(const ServiceReference& ref)
{
    if (ref.getServiceName() == "IndexSearchService")
    {
        Properties props = ref.getServiceProperties();
        if (props.get("collection") == config_->collectionName_)
        {
            IndexSearchService* service = reinterpret_cast<IndexSearchService*>(const_cast<IService*>(ref.getService()));
            if (! init_(service))
            {
                LOG(ERROR) << "failed in RecommendBundleActivator::init_(), collection:  " << config_->collectionName_;
                return false;
            }

            return true;
        }
    }

    return false;
}

void RecommendBundleActivator::removedService(const ServiceReference& ref)
{
}

bool RecommendBundleActivator::init_(IndexSearchService* indexSearchService)
{
    createSCDDir_();

    if (! createDataDir_())
        return false;

    createUser_();
    createItem_(indexSearchService);
    createMining_();
    createEvent_();
    createOrder_();
    createRecommender_();
    createService_();

    return true;
}

bool RecommendBundleActivator::createDataDir_()
{
    std::string dir;
    if (! openDataDirectory_(dir))
    {
        LOG(ERROR) << "failed in RecommendBundleActivator::openDataDirectory_(), collection: "
                   << config_->collectionName_;
        return false;
    }

    dataDir_ = dir;
    bfs::create_directories(dataDir_);
    return true;
}

bool RecommendBundleActivator::openDataDirectory_(std::string& dataDir)
{
    std::vector<std::string>& directories = config_->collectionDataDirectories_;
    if (directories.size() == 0)
    {
        std::cout<<"no data dir config"<<std::endl;
        return false;
    }

    directoryRotator_.setCapacity(directories.size());
    std::vector<bfs::path> dirtyDirectories;
    typedef std::vector<std::string>::const_iterator iterator;
    bfs::path dataPath = config_->collPath_.getCollectionDataPath();
    for (iterator it = directories.begin(); it != directories.end(); ++it)
    {
        bfs::path dir = dataPath / *it;
        if (!directoryRotator_.appendDirectory(dir))
        {
            std::string msg = dir.file_string() + " corrupted, delete it!";
            LOG(ERROR) <<msg <<endl;
            //clean the corrupt dir
            bfs::remove_all(dir);
            dirtyDirectories.push_back(dir);
        }
    }

    directoryRotator_.rotateToNewest();
    boost::shared_ptr<Directory> newest = directoryRotator_.currentDirectory();
    if (newest)
    {
        dataDir = newest->path().string();
        std::vector<bfs::path>::iterator it = dirtyDirectories.begin();
        for( ; it != dirtyDirectories.end(); ++it)
            directoryRotator_.appendDirectory(*it);
        return true;
    }
    else
    {
        std::vector<bfs::path>::iterator it = dirtyDirectories.begin();
        for( ; it != dirtyDirectories.end(); ++it)
            directoryRotator_.appendDirectory(*it);
		
        directoryRotator_.rotateToNewest();
        boost::shared_ptr<Directory> dir = directoryRotator_.currentDirectory();
        if(dir)
        {
            dataDir = dir->path().string();
            return true;
        }
    }

    return false;
}

void RecommendBundleActivator::createSCDDir_()
{
    bfs::create_directories(config_->userSCDPath());
    bfs::create_directories(config_->orderSCDPath());
}

void RecommendBundleActivator::createUser_()
{
    bfs::path userDir = dataDir_ / "user";
    bfs::create_directory(userDir);
    userManager_.reset(new UserManager((userDir / "user.db").string()));
{'fraction_non_alphanumeric': 0.12849402253375763, 'fraction_numerical': 0.00034402683409305926, 'mean_word_length': 3.576151121605667, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6197997', 'n_tokens_mistral': 3409, 'n_tokens_neox': 3294, 'n_words': 543}