<commit_before>#include "RpcImageServer.h"
#include "ImageServerCfg.h"
#include "ImageProcess.h"
#include "ImageProcessTask.h"
#include <image-manager/ImageServerStorage.h>
#include <image-manager/ImageServerRequest.h>
#include "errno.h"
#include <boost/bind.hpp>
#include <boost/filesystem.hpp>
#include <boost/thread.hpp>
#include <glog/logging.h>

#include <tfs_client_api.h>

using namespace tfs::client;
using namespace tfs::common;

namespace sf1r
{

RpcImageServer::RpcImageServer(const std::string& host, uint16_t port, uint32_t threadNum)
    : host_(host)
    , port_(port)
    , threadNum_(threadNum)
    , is_exporting_(false)
    , export_thread_(NULL)
{
}

RpcImageServer::~RpcImageServer()
{
    std::cout << "~RpcImageServer()" << std::endl;
    stop();
}

void RpcImageServer::start()
{
    instance.listen(host_, port_);
    instance.start(threadNum_);
}

void RpcImageServer::join()
{
    instance.join();
}

void RpcImageServer::run()
{
    start();
    join();
}

void RpcImageServer::stop()
{
    instance.end();
    instance.join();
    if(export_thread_)
    {
        export_thread_->interrupt();
        export_thread_->join();
        delete export_thread_;
    }
}

void RpcImageServer::dispatch(msgpack::rpc::request req)
{
    try
    {
        std::string method;
        req.method().convert(&method);

        if (method == ImageServerRequest::method_names[ImageServerRequest::METHOD_TEST])
        {
            msgpack::type::tuple<bool> params;
            req.params().convert(&params);

            req.result(true);
        }
        else if (method == ImageServerRequest::method_names[ImageServerRequest::METHOD_GET_IMAGE_COLOR])
        {
            msgpack::type::tuple<ImageColorData> params;
            req.params().convert(&params);
            ImageColorData& reqdata = params.get<0>();

            reqdata.success = ImageServerStorage::get()->GetImageColor(reqdata.img_file, reqdata.result_img_color);
            if(!reqdata.success)
            {
                // try to recalculate the color
                //std::string picFilePath = ImageServerCfg::get()->getImageFilePath() + "/" + reqdata.img_file;
                char* data = NULL;
                std::size_t datasize = 0;
                if(ImageServerStorage::get()->DownloadImageData(reqdata.img_file, data, datasize))
                {
                    int indexCollection[COLOR_RET_NUM];
                    bool ret = getImageColor(data, datasize, indexCollection, COLOR_RET_NUM);
                    if(ret){
                        std::ostringstream oss;
                        for(std::size_t i = 0; i < COLOR_RET_NUM - 1; ++i)
                        {
                            oss << indexCollection[i] << ",";
                        }
                        oss << indexCollection[COLOR_RET_NUM - 1]; 

                        reqdata.success = ImageServerStorage::get()->SetImageColor(reqdata.img_file, oss.str());
                        reqdata.result_img_color = oss.str();
                        //LOG(INFO) << "got request image color for fileName=" << reqdata.img_file << "result:" <<
                        //   oss.str() << std::endl;
                    }else{
                        LOG(WARNING) << "getImageColor failed imagePath=" << reqdata.img_file << std::endl;
                        reqdata.success = false;
                    }
                    delete[] data;
                }
                else
                {
                    LOG(INFO) << "getImage data failed while get image color imagePath=" << reqdata.img_file << std::endl;
                    reqdata.success = false;
                }
            }
            req.result(reqdata);
        }
        else if (method == ImageServerRequest::method_names[ImageServerRequest::METHOD_UPLOAD_IMAGE])
        {
            struct timeval tv_start, tv_end;
            gettimeofday(&tv_start, NULL);
            LOG(INFO) << "get upload request : " << tv_start.tv_sec << endl;
            msgpack::type::tuple<UploadImageData> params;
            req.params().convert(&params);
            UploadImageData& reqdata = params.get<0>();
            std::string ret_file;
            if(reqdata.param_type == 0)
            {
                reqdata.success = ImageServerStorage::get()->UploadImageFile(reqdata.img_file, ret_file);
            }
            else if(reqdata.param_type == 1)
            {
                reqdata.success = ImageServerStorage::get()->UploadImageData(reqdata.img_file.data(),
                    reqdata.img_file.size(), ret_file);
            }
            else
            {
                reqdata.success = false;
            }
            gettimeofday(&tv_end, NULL);
            int interval = tv_end.tv_sec - tv_start.tv_sec;
            if( interval > 1)
                LOG(WARNING) << "upload time larger than 1s : " << interval << endl;
            if(reqdata.success)
            {
                // write log file
                //
                ofstream ofs;
                try
                {
                    ofs.open(ImageServerCfg::get()->getUploadImageLog().c_str(), ofstream::app);
                    ofs << ret_file.c_str() << std::endl;
                    ofs.close();
                }
                catch(std::exception& e)
                {
                    LOG(ERROR) << "write upload log failed: " << ret_file << std::endl;
                    ofs.close();
                }
            }
            else
            {
                if(reqdata.param_type == 0)
                    LOG(WARNING) << "file upload failed: " << reqdata.img_file << std::endl;
                else
                    LOG(WARNING) << "image data upload failed: " << std::endl;
            }
            reqdata.img_file = ret_file;
            req.result(reqdata);
            gettimeofday(&tv_end, NULL);
            LOG(INFO) << "finish upload request : " << tv_end.tv_sec << endl;
            interval = tv_end.tv_sec - tv_start.tv_sec;
            if( interval > 1)
                LOG(WARNING) << "upload time total larger than 1s : " << interval << endl;
        }
        else if (method == ImageServerRequest::method_names[ImageServerRequest::METHOD_DELETE_IMAGE])
        {
            msgpack::type::tuple<DeleteImageData> params;
            req.params().convert(&params);
            DeleteImageData& reqdata = params.get<0>();
            reqdata.success = ImageServerStorage::get()->DeleteImage(reqdata.img_file);
            req.result(reqdata);
        }
        else if (method == ImageServerRequest::method_names[ImageServerRequest::METHOD_EXPORT_IMAGE])
        {
            msgpack::type::tuple<ExportImageData> params;
            req.params().convert(&params);
            ExportImageData& reqdata = params.get<0>();
            req.result(reqdata);
            if(is_exporting_)
            {
                LOG(WARNING) << "image is already exporting! Ignore any more request." << std::endl;
                return;
            }
            is_exporting_ = true;
            if(export_thread_)
            {<commit_msg>tiny log for image server<commit_after>#include "RpcImageServer.h"
#include "ImageServerCfg.h"
#include "ImageProcess.h"
#include "ImageProcessTask.h"
#include <image-manager/ImageServerStorage.h>
#include <image-manager/ImageServerRequest.h>
#include "errno.h"
#include <boost/bind.hpp>
#include <boost/filesystem.hpp>
#include <boost/thread.hpp>
#include <glog/logging.h>

#include <tfs_client_api.h>

using namespace tfs::client;
using namespace tfs::common;

namespace sf1r
{

RpcImageServer::RpcImageServer(const std::string& host, uint16_t port, uint32_t threadNum)
    : host_(host)
    , port_(port)
    , threadNum_(threadNum)
    , is_exporting_(false)
    , export_thread_(NULL)
{
}

RpcImageServer::~RpcImageServer()
{
    std::cout << "~RpcImageServer()" << std::endl;
    stop();
}

void RpcImageServer::start()
{
    instance.listen(host_, port_);
    instance.start(threadNum_);
}

void RpcImageServer::join()
{
    instance.join();
}

void RpcImageServer::run()
{
    start();
    join();
}

void RpcImageServer::stop()
{
    instance.end();
    instance.join();
    if(export_thread_)
    {
        export_thread_->interrupt();
        export_thread_->join();
        delete export_thread_;
    }
}

void RpcImageServer::dispatch(msgpack::rpc::request req)
{
    try
    {
        std::string method;
        req.method().convert(&method);

        if (method == ImageServerRequest::method_names[ImageServerRequest::METHOD_TEST])
        {
            msgpack::type::tuple<bool> params;
            req.params().convert(&params);

            req.result(true);
        }
        else if (method == ImageServerRequest::method_names[ImageServerRequest::METHOD_GET_IMAGE_COLOR])
        {
            msgpack::type::tuple<ImageColorData> params;
            req.params().convert(&params);
            ImageColorData& reqdata = params.get<0>();

            reqdata.success = ImageServerStorage::get()->GetImageColor(reqdata.img_file, reqdata.result_img_color);
            if(!reqdata.success)
            {
                // try to recalculate the color
                //std::string picFilePath = ImageServerCfg::get()->getImageFilePath() + "/" + reqdata.img_file;
                char* data = NULL;
                std::size_t datasize = 0;
                if(ImageServerStorage::get()->DownloadImageData(reqdata.img_file, data, datasize))
                {
                    int indexCollection[COLOR_RET_NUM];
                    bool ret = getImageColor(data, datasize, indexCollection, COLOR_RET_NUM);
                    if(ret){
                        std::ostringstream oss;
                        for(std::size_t i = 0; i < COLOR_RET_NUM - 1; ++i)
                        {
                            oss << indexCollection[i] << ",";
                        }
                        oss << indexCollection[COLOR_RET_NUM - 1]; 

                        reqdata.success = ImageServerStorage::get()->SetImageColor(reqdata.img_file, oss.str());
                        reqdata.result_img_color = oss.str();
                        //LOG(INFO) << "got request image color for fileName=" << reqdata.img_file << "result:" <<
                        //   oss.str() << std::endl;
                    }else{
                        LOG(WARNING) << "getImageColor failed imagePath=" << reqdata.img_file << std::endl;
                        reqdata.success = false;
                    }
                    delete[] data;
                }
                else
                {
                    LOG(INFO) << "getImage data failed while get image color imagePath=" << reqdata.img_file << std::endl;
                    reqdata.success = false;
                }
            }
            req.result(reqdata);
        }
        else if (method == ImageServerRequest::method_names[ImageServerRequest::METHOD_UPLOAD_IMAGE])
        {
            struct timeval tv_start, tv_end;
            gettimeofday(&tv_start, NULL);
            //LOG(INFO) << "get upload request : " << tv_start.tv_sec << endl;
            msgpack::type::tuple<UploadImageData> params;
            req.params().convert(&params);
            UploadImageData& reqdata = params.get<0>();
            std::string ret_file;
            if(reqdata.param_type == 0)
            {
                reqdata.success = ImageServerStorage::get()->UploadImageFile(reqdata.img_file, ret_file);
            }
            else if(reqdata.param_type == 1)
            {
                reqdata.success = ImageServerStorage::get()->UploadImageData(reqdata.img_file.data(),
                    reqdata.img_file.size(), ret_file);
            }
            else
            {
                reqdata.success = false;
            }
            gettimeofday(&tv_end, NULL);
            int interval = tv_end.tv_sec - tv_start.tv_sec;
            if( interval > 1)
                LOG(WARNING) << "upload time larger than 1s : " << interval << endl;
            if(reqdata.success)
            {
                // write log file
                //
                ofstream ofs;
                try
                {
                    ofs.open(ImageServerCfg::get()->getUploadImageLog().c_str(), ofstream::app);
                    ofs << ret_file.c_str() << ", time: " << tv_end.tv_sec << std::endl;
                    ofs.close();
                }
                catch(std::exception& e)
                {
                    LOG(ERROR) << "write upload log failed: " << ret_file << std::endl;
                    ofs.close();
                }
            }
            else
            {
                if(reqdata.param_type == 0)
                    LOG(WARNING) << "file upload failed: " << reqdata.img_file << std::endl;
                else
                    LOG(WARNING) << "image data upload failed: " << std::endl;
            }
            reqdata.img_file = ret_file;
            req.result(reqdata);
            gettimeofday(&tv_end, NULL);
            //LOG(INFO) << "finish upload request : " << tv_end.tv_sec << endl;
            interval = tv_end.tv_sec - tv_start.tv_sec;
            if( interval > 1)
                LOG(WARNING) << "upload time total larger than 1s : " << interval << endl;
        }
        else if (method == ImageServerRequest::method_names[ImageServerRequest::METHOD_DELETE_IMAGE])
        {
            msgpack::type::tuple<DeleteImageData> params;
            req.params().convert(&params);
            DeleteImageData& reqdata = params.get<0>();
            reqdata.success = ImageServerStorage::get()->DeleteImage(reqdata.img_file);
            LOG(INFO) << "delete request for : " << reqdata.img_file << std::endl;
            req.result(reqdata);
        }
        else if (method == ImageServerRequest::method_names[ImageServerRequest::METHOD_EXPORT_IMAGE])
        {
            msgpack::type::tuple<ExportImageData> params;
            req.params().convert(&params);
            ExportImageData& reqdata = params.get<0>();
            req.result(reqdata);
            if(is_exporting_)
            {
                LOG(WARNING) << "image is already exporting! Ignore any more request." << std::endl;
                return;
            }
            is_exporting_ = true;
            if(export_thread_)
            {
{'fraction_non_alphanumeric': 0.12346881959910913, 'fraction_numerical': 0.0027839643652561247, 'mean_word_length': 1.742175572519084, 'pattern_counts': {'":': 0, '<': 207, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7482381', 'n_tokens_mistral': 4053, 'n_tokens_neox': 3876, 'n_words': 694}