<commit_before>/*
 * QueryDataBase.cpp
 *
 *  Created on: 2009-11-25
 *      Author: jinglei
 */

#include "RecommendManager.h"
#include <mining-manager/concept-id-manager.h>
#include <mining-manager/auto-fill-submanager/AutoFillSubManager.h>
#include <mining-manager/query-correction-submanager/QueryCorrectionSubmanager.h>
#include <mining-manager/util/TermUtil.hpp>

#include <document-manager/DocumentManager.h>
#include <common/SFLogger.h>
#include <log-manager/LogManager.h>
#include <log-manager/UserQuery.h>
#include <log-manager/PropertyLabel.h>
#include <query-manager/QMCommonFunc.h>

#include <idmlib/util/idm_analyzer.h>
#include <glog/logging.h>

#include <cmath>

using namespace sf1r;

RecommendManager::RecommendManager(
        const std::string& collection_name,
        const CollectionPath& collectionPath,
        const MiningSchema& mining_schema,
        const MiningConfig& mining_config,
        const boost::shared_ptr<DocumentManager>& documentManager,
        boost::shared_ptr<QueryCorrectionSubmanager> query_correction,
        idmlib::util::IDMAnalyzer* analyzer,
        uint32_t logdays
)
    : isOpen_(false)
    , collection_name_(collection_name)
    , collectionPath_(collectionPath)
    , mining_schema_(mining_schema)
    , mining_config_(mining_config)
    , info_(0)
    , serInfo_(collectionPath.getQueryDataPath()+"/ser_info")
    , document_manager_(documentManager)
    , recommend_db_(NULL), concept_id_manager_(NULL)
    , autofill_(new AutoFillSubManager())
    , query_correction_(query_correction)
    , analyzer_(analyzer)
    , logdays_(logdays)
    , dir_switcher_(collectionPath.getQueryDataPath())
    , max_docid_(0), max_docid_file_(collectionPath.getQueryDataPath()+"/max_id")
{
    open();
}

RecommendManager::~RecommendManager()
{
    close();
}

bool RecommendManager::open()
{
    if (isOpen_) return true;
    if (max_docid_file_.Load())
    {
        max_docid_ = max_docid_file_.GetValue();
    }
    serInfo_.setCacheSize(1);
    serInfo_.open();
    bool b = serInfo_.get(0, info_);
    if (!b)
    {
        serInfo_.update(0, info_);
    }
    serInfo_.flush();
    if (!dir_switcher_.Open())
    {
        return false;
    }
    std::string current_recommend_path;
    if (!dir_switcher_.GetCurrent(current_recommend_path))
    {
        return false;
    }
    try {
        LOG(INFO) << "open ir manager on " << current_recommend_path << std::endl;
        recommend_db_ = new MIRDatabase(current_recommend_path);
        recommend_db_->setCacheSize<0>(100000000);
        recommend_db_->setCacheSize<1>(0);
        recommend_db_->open();
    }
    catch(std::exception& ex)
    {
        LOG(ERROR) << ex.what() << std::endl;
        return false;
    }
    LOG(INFO) << "open ir manager finished: " << current_recommend_path << std::endl;
    std::string path_tocreate = current_recommend_path+"/concept-id";
    boost::filesystem::create_directories(path_tocreate);
    concept_id_manager_ = new ConceptIDManager(path_tocreate);
    if (!concept_id_manager_->Open())
    {
        //TODO
    }
    if(!autofill_->Init(collectionPath_, collection_name_, mining_config_.autofill_param.cron)) return false;
    isOpen_ = true;
    return true;
}

void RecommendManager::flush()
{
    if (isOpen_)
    {
        serInfo_.flush();
        if(recommend_db_)
            recommend_db_->flush();
        if (concept_id_manager_)
            concept_id_manager_->Flush();
    }
    if (autofill_) autofill_->flush();
}

void RecommendManager::close()
{
    if (isOpen_)
    {
        serInfo_.close();
        if (recommend_db_ != NULL)
        {
            recommend_db_->close();
            delete recommend_db_;
            recommend_db_ = NULL;
        }
        if (concept_id_manager_ != NULL)
        {
            concept_id_manager_->Close();
            delete concept_id_manager_;
            concept_id_manager_ = NULL;
        }
        isOpen_ = false;<commit_msg>tiny fix<commit_after>/*
 * QueryDataBase.cpp
 *
 *  Created on: 2009-11-25
 *      Author: jinglei
 */

#include "RecommendManager.h"
#include <mining-manager/concept-id-manager.h>
#include <mining-manager/auto-fill-submanager/AutoFillSubManager.h>
#include <mining-manager/query-correction-submanager/QueryCorrectionSubmanager.h>
#include <mining-manager/util/TermUtil.hpp>

#include <document-manager/DocumentManager.h>
#include <common/SFLogger.h>
#include <log-manager/LogManager.h>
#include <log-manager/UserQuery.h>
#include <log-manager/PropertyLabel.h>
#include <query-manager/QMCommonFunc.h>

#include <idmlib/util/idm_analyzer.h>
#include <glog/logging.h>

#include <cmath>

using namespace sf1r;

RecommendManager::RecommendManager(
        const std::string& collection_name,
        const CollectionPath& collectionPath,
        const MiningSchema& mining_schema,
        const MiningConfig& mining_config,
        const boost::shared_ptr<DocumentManager>& documentManager,
        boost::shared_ptr<QueryCorrectionSubmanager> query_correction,
        idmlib::util::IDMAnalyzer* analyzer,
        uint32_t logdays
)
    : isOpen_(false)
    , collection_name_(collection_name)
    , collectionPath_(collectionPath)
    , mining_schema_(mining_schema)
    , mining_config_(mining_config)
    , info_(0)
    , serInfo_(collectionPath.getQueryDataPath()+"/ser_info")
    , document_manager_(documentManager)
    , recommend_db_(NULL), concept_id_manager_(NULL)
    , autofill_(new AutoFillSubManager())
    , query_correction_(query_correction)
    , analyzer_(analyzer)
    , logdays_(logdays)
    , dir_switcher_(collectionPath.getQueryDataPath())
    , max_docid_(0), max_docid_file_(collectionPath.getQueryDataPath()+"/max_id")
{
    open();
}

RecommendManager::~RecommendManager()
{
    close();
}

bool RecommendManager::open()
{
    if (isOpen_) return true;
    if (max_docid_file_.Load())
    {
        max_docid_ = max_docid_file_.GetValue();
    }
    serInfo_.setCacheSize(1);
    serInfo_.open();
    bool b = serInfo_.get(0, info_);
    if (!b)
    {
        serInfo_.update(0, info_);
    }
    serInfo_.flush();
    if (!dir_switcher_.Open())
    {
        return false;
    }
    std::string current_recommend_path;
    if (!dir_switcher_.GetCurrent(current_recommend_path))
    {
        return false;
    }
    try {
        LOG(INFO) << "open ir manager on " << current_recommend_path << std::endl;
        recommend_db_ = new MIRDatabase(current_recommend_path);
        recommend_db_->setCacheSize<0>(100000000);
        recommend_db_->setCacheSize<1>(0);
        recommend_db_->open();
    }
    catch(std::exception& ex)
    {
        LOG(ERROR) << ex.what() << std::endl;
        return false;
    }
    LOG(INFO) << "open ir manager finished: " << current_recommend_path << std::endl;
    std::string path_tocreate = current_recommend_path+"/concept-id";
    boost::filesystem::create_directories(path_tocreate);
    concept_id_manager_ = new ConceptIDManager(path_tocreate);
    if (!concept_id_manager_->Open())
    {
        //TODO
    }
    if(!autofill_->Init(collectionPath_, collection_name_, mining_config_.autofill_param.cron)) return false;
    isOpen_ = true;
    return true;
}

void RecommendManager::flush()
{
    if (isOpen_)
    {
        try{
        serInfo_.flush();
        if(recommend_db_)
            recommend_db_->flush();
        if (concept_id_manager_)
            concept_id_manager_->Flush();
        }catch(const std::exception& e)
        {
            LOG(ERROR) << "flush error in " << __FUNCTION__ << ", info :" << e.what();
        }
    }
    if (autofill_) autofill_->flush();
}

void RecommendManager::close()
{
    if (isOpen_)
    {
        serInfo_.close();
        if (recommend_db_ != NULL)
        {
            recommend_db_->close();
            delete recommend_db_;
            recommend_db_ = NULL;
        }
        if (concept_id_manager_ != NULL)
        {
            concept_id_manager_->Close();
            delete concept_id_manager_;
            concept_id_manager_ = NULL;
        }
        isOpen_ = false;
{'fraction_non_alphanumeric': 0.12377405338299192, 'fraction_numerical': 0.006952203600248293, 'mean_word_length': 3.1504379185986604, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5833833', 'n_tokens_mistral': 2664, 'n_tokens_neox': 2582, 'n_words': 401}