<commit_before>/*
 * QueryDataBase.cpp
 *
 *  Created on: 2009-11-25
 *      Author: jinglei
 */

#include "RecommendManager.h"
#include <mining-manager/util/TermUtil.hpp>
#include <common/SFLogger.h>
#include <cmath>
#include <log-manager/LogManager.h>
#include <log-manager/UserQuery.h>
#include <log-manager/PropertyLabel.h>
#include <query-manager/QMCommonFunc.h>
#include <mining-manager/query-correction-submanager/QueryCorrectionSubmanager.h>
using namespace sf1r;

RecommendManager::RecommendManager(
    const std::string& path,
    const std::string& collection_name,
    const MiningSchema& mining_schema,
    const boost::shared_ptr<DocumentManager>& documentManager,
    boost::shared_ptr<QueryCorrectionSubmanager> query_correction,
    idmlib::util::IDMAnalyzer* analyzer,
    uint32_t logdays
)
    : path_(path)
    , isOpen_(false)
    , collection_name_(collection_name)
    , mining_schema_(mining_schema)
    , info_(0)
    , serInfo_(path_+"/ser_info")
    , document_manager_(documentManager)
    , recommend_db_(NULL), concept_id_manager_(NULL)
    , autofill_(new AutoFillSubManager())
    , query_correction_(query_correction)
    , analyzer_(analyzer)
    , logdays_(logdays)
    , dir_switcher_(path)
    , max_docid_(0), max_docid_file_(path+"/max_id")
{
    open();
}

RecommendManager::~RecommendManager()
{
    close();
}

bool RecommendManager::open()
{
    if (isOpen_) return true;
    if (max_docid_file_.Load())
    {
        max_docid_ = max_docid_file_.GetValue();
    }
    serInfo_.setCacheSize(1);
    serInfo_.open();
    bool b = serInfo_.get(0, info_);
    if (!b)
    {
        serInfo_.update(0, info_);
    }
    serInfo_.flush();
    if (!dir_switcher_.Open())
    {
        return false;
    }
    std::string current_recommend_path;
    if (!dir_switcher_.GetCurrent(current_recommend_path))
    {
        return false;
    }
    std::cout<<"open recommend manager on "<<current_recommend_path<<std::endl;
    recommend_db_ = new MIRDatabase(current_recommend_path);
    recommend_db_->setCacheSize<0>(100000000);
    recommend_db_->setCacheSize<1>(0);
    recommend_db_->open();
    std::string path_tocreate = current_recommend_path+"/concept-id";
    boost::filesystem::create_directories(path_tocreate);
    concept_id_manager_ = new ConceptIDManager(path_tocreate);
    if (!concept_id_manager_->Open())
    {
        //TODO
    }
    if(!autofill_->Init(path_+"/autofill")) return false;
    isOpen_ = true;
    return true;
}

void RecommendManager::close()
{
    if (isOpen_)
    {
        serInfo_.close();
        if (recommend_db_ != NULL)
        {
            recommend_db_->close();
            delete recommend_db_;
            recommend_db_ = NULL;
        }
        if (concept_id_manager_ != NULL)
        {
            concept_id_manager_->Close();
            delete concept_id_manager_;
            concept_id_manager_ = NULL;
        }
        isOpen_ = false;
    }
}

void RecommendManager::RebuildForAll()
{
    boost::posix_time::ptime time_now = boost::posix_time::second_clock::local_time();
    uint32_t days = logdays_;
    boost::gregorian::days dd(days);
    boost::posix_time::ptime p = time_now-dd;
    std::string time_string = boost::posix_time::to_iso_string(p);
    typedef std::map<std::string, std::string> DbRecordType;

    std::string query_sql = "SELECT query, count(*) AS freq, max(hit_docs_num) AS df FROM user_queries WHERE collection='" + collection_name_ + "' AND hit_docs_num>0 AND TimeStamp>='" + time_string + "' GROUP BY query";
    std::list<DbRecordType> query_records;
    UserQuery::find_by_sql(query_sql, query_records);
    std::list<QueryLogType> queryList;
    for (std::list<DbRecordType>::iterator it = query_records.begin();
            it != query_records.end(); ++it)
    {
        izenelib::util::UString uquery((*it)["query"], izenelib::util::UString::UTF_8);
        if (QueryUtility::isRestrictWord(uquery))
        {
            continue;
        }
        uint32_t freq = boost::lexical_cast<uint32_t>((*it)["freq"]);
        uint32_t df = boost::lexical_cast<uint32_t>((*it)["df"]);
        queryList.push_back(QueryLogType(freq, df, uquery));
    }
<commit_msg>fix the sql clause for selecting from user_queries<commit_after>/*
 * QueryDataBase.cpp
 *
 *  Created on: 2009-11-25
 *      Author: jinglei
 */

#include "RecommendManager.h"
#include <mining-manager/util/TermUtil.hpp>
#include <common/SFLogger.h>
#include <cmath>
#include <log-manager/LogManager.h>
#include <log-manager/UserQuery.h>
#include <log-manager/PropertyLabel.h>
#include <query-manager/QMCommonFunc.h>
#include <mining-manager/query-correction-submanager/QueryCorrectionSubmanager.h>
using namespace sf1r;

RecommendManager::RecommendManager(
    const std::string& path,
    const std::string& collection_name,
    const MiningSchema& mining_schema,
    const boost::shared_ptr<DocumentManager>& documentManager,
    boost::shared_ptr<QueryCorrectionSubmanager> query_correction,
    idmlib::util::IDMAnalyzer* analyzer,
    uint32_t logdays
)
    : path_(path)
    , isOpen_(false)
    , collection_name_(collection_name)
    , mining_schema_(mining_schema)
    , info_(0)
    , serInfo_(path_+"/ser_info")
    , document_manager_(documentManager)
    , recommend_db_(NULL), concept_id_manager_(NULL)
    , autofill_(new AutoFillSubManager())
    , query_correction_(query_correction)
    , analyzer_(analyzer)
    , logdays_(logdays)
    , dir_switcher_(path)
    , max_docid_(0), max_docid_file_(path+"/max_id")
{
    open();
}

RecommendManager::~RecommendManager()
{
    close();
}

bool RecommendManager::open()
{
    if (isOpen_) return true;
    if (max_docid_file_.Load())
    {
        max_docid_ = max_docid_file_.GetValue();
    }
    serInfo_.setCacheSize(1);
    serInfo_.open();
    bool b = serInfo_.get(0, info_);
    if (!b)
    {
        serInfo_.update(0, info_);
    }
    serInfo_.flush();
    if (!dir_switcher_.Open())
    {
        return false;
    }
    std::string current_recommend_path;
    if (!dir_switcher_.GetCurrent(current_recommend_path))
    {
        return false;
    }
    std::cout<<"open recommend manager on "<<current_recommend_path<<std::endl;
    recommend_db_ = new MIRDatabase(current_recommend_path);
    recommend_db_->setCacheSize<0>(100000000);
    recommend_db_->setCacheSize<1>(0);
    recommend_db_->open();
    std::string path_tocreate = current_recommend_path+"/concept-id";
    boost::filesystem::create_directories(path_tocreate);
    concept_id_manager_ = new ConceptIDManager(path_tocreate);
    if (!concept_id_manager_->Open())
    {
        //TODO
    }
    if(!autofill_->Init(path_+"/autofill")) return false;
    isOpen_ = true;
    return true;
}

void RecommendManager::close()
{
    if (isOpen_)
    {
        serInfo_.close();
        if (recommend_db_ != NULL)
        {
            recommend_db_->close();
            delete recommend_db_;
            recommend_db_ = NULL;
        }
        if (concept_id_manager_ != NULL)
        {
            concept_id_manager_->Close();
            delete concept_id_manager_;
            concept_id_manager_ = NULL;
        }
        isOpen_ = false;
    }
}

void RecommendManager::RebuildForAll()
{
    boost::posix_time::ptime time_now = boost::posix_time::second_clock::local_time();
    uint32_t days = logdays_;
    boost::gregorian::days dd(days);
    boost::posix_time::ptime p = time_now-dd;
    std::string time_string = boost::posix_time::to_iso_string(p);
    typedef std::map<std::string, std::string> DbRecordType;

    std::string query_sql = "SELECT query, count(*) AS freq, hit_docs_num AS df FROM user_queries WHERE collection='" + collection_name_ + "' AND hit_docs_num>0 AND TimeStamp>='" + time_string + "' GROUP BY query";
    std::list<DbRecordType> query_records;
    UserQuery::find_by_sql(query_sql, query_records);
    std::list<QueryLogType> queryList;
    for (std::list<DbRecordType>::iterator it = query_records.begin();
            it != query_records.end(); ++it)
    {
        izenelib::util::UString uquery((*it)["query"], izenelib::util::UString::UTF_8);
        if (QueryUtility::isRestrictWord(uquery))
        {
            continue;
        }
        uint32_t freq = boost::lexical_cast<uint32_t>((*it)["freq"]);
        uint32_t df = boost::lexical_cast<uint32_t>((*it)["df"]);
        queryList.push_back(QueryLogType(freq, df, uquery));
    }

{'fraction_non_alphanumeric': 0.1330318895763922, 'fraction_numerical': 0.00951927653498334, 'mean_word_length': 3.592896174863388, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3874813', 'n_tokens_mistral': 2896, 'n_tokens_neox': 2803, 'n_words': 447}