<commit_before>/**
 * Created by chriscai on 2015/1/12.
 */
var http = require('http');

var  log4js = require('log4js'),
     Apply = require('../model/Apply'),
     http = require('http'),
     _ = require('underscore'),
     logger = log4js.getLogger(),
     ORM = require("orm");


var dateFormat  = function (date , fmt){
    var o = {
        "M+": date.getMonth() + 1, //月份
        "d+": date.getDate(), //日
        "h+": date.getHours(), //小时
        "m+": date.getMinutes(), //分
        "s+": date.getSeconds(), //秒
        "q+": Math.floor((date.getMonth() + 3) / 3), //季度
        "S": date.getMilliseconds() //毫秒
    };
    if (/(y+)/.test(fmt)) fmt = fmt.replace(RegExp.$1, (date.getFullYear() + "").substr(4 - RegExp.$1.length));
    for (var k in o)
        if (new RegExp("(" + k + ")").test(fmt)) fmt = fmt.replace(RegExp.$1, (RegExp.$1.length == 1) ? (o[k]) : (("00" + o[k]).substr(("" + o[k]).length)));
    return fmt;
};

var StatisticsService = function (){

    this.statisticsDao = global.models.statisticsDao;
    this.applyDao = global.models.applyDao;


    this.url = GLOBAL.pjconfig.storage.errorMsgTopUrl;

    logger.debug('query url : ' + this.url);
};



StatisticsService.prototype = {
    queryById : function (param  , callback){
        this.statisticsDao.find({projectId :param.projectId , startDate : dateFormat(param.startDate , 'yyyy-MM-dd hh:mm:ss')} , function (err , items){
            if(err){
                callback(err);
                return;
            }



            if(items[0] ){
                items[0].content = JSON.parse(items[0].content);
                items[0].content = _.map(items[0].content, function(value, key){
                    return {title : key , total : value};
                });

                items[0].content = items[0].content.sort(function ( a, b){
                    if(a.total < b.total){
                        return 1;
                    }else {
                        return -1;
                    }
                });

                if(param.top){
                    items[0].content = items[0].content.slice(0 , param.top);
                }

            }
            callback(null,  items);
        });

    },
    queryByChart : function (param  , callback){
        //筛选参数
        var s_params = {};
        if(param.projectId !=-1){
            s_params.projectId = param.projectId;
        }

        //时间开始范围 7天内 或者1个月内
        var oneDay = 1000 * 60 * 60 * 24;
        var day = param.timeScope ==1?7:30;
        param.startTime = new Date() - oneDay * day;
        param.startTime = dateFormat(new Date(param.startTime), 'yyyy-MM-dd');

        if(GLOBAL.DEBUG){
            logger.info("query start time is "+ param.startTime);
        }
        this.statisticsDao.find(s_params).only("endDate" , "startDate" , "projectId" , "id" , "total").where("startDate >=?", [param.startTime]).all(function (err, items)  {
            if(err){
                callback(err);
                return;
            }
            callback(null,{ret:0, msg:"success", data: items });
        }).where(function(){

        });
    },
    fetchAndSave : function (id , startDate , cb){
        var self = this;
        http.get((this.url + '?id=' + id + '&startDate=' + (startDate -0 ))  , function(res){
            var buffer = '';
            res.on('data' , function (chunk){
                buffer += chunk.toString();
            }).on('end' , function (){
                var saveModel = {};
                try {
                    var result = JSON.parse(buffer);

                    saveModel = {
                        startDate : new Date(result.startDate),
                        endDate : new Date(result.endDate),
                        content : JSON.stringify(result.result),
                        projectId : id,
                        total : 0
                    };

                    _.each(result.result , function (value ,key ){
                            saveModel.total += result.result[key];
                    })

                }catch(err){
                    logger.error('error :' + err);
                    saveModel = {
                        startDate: startDate,
                        endDate: startDate + 86400000,
                        content: "{}",
                        total: 0
                    }
                }

                self.statisticsDao.create(saveModel , function (err , items){
                    if(err){
                        logger.error("Insert into b_statistics error(id=", id + ") :  " +  err);
                    }
                    logger.info("Insert into b_statistics success(id=", id + ") :  " + buffer.toString());
                    cb && cb(err);
                });
            })

        }).on('error' , function (err){
            logger.error('error :' + err);
        });
    },

    startMonitor : function (){
        var self = this;


        var getTomorrowDay = function (){
            var tomorrow = new Date(nowDate);
            tomorrow.setHours(1, 0 , 0 ,0);
            tomorrow.setDate(tomorrow.getDate() + 1);
            return tomorrow;
//            return new Date(nowDate.getFullYear() + "-" + (nowDate.getMonth() + 1)  + "-" + (nowDate.getDate()+1) + " 01:00:00");
        }

        var getStartDay = function (){
            var startDate = new Date(nowDate);
            startDate.setHours(0 , 0 , 0 ,0);
            return startDate;
        }<commit_msg>close log<commit_after>/**
 * Created by chriscai on 2015/1/12.
 */
var http = require('http');

var  log4js = require('log4js'),
     Apply = require('../model/Apply'),
     http = require('http'),
     _ = require('underscore'),
     logger = log4js.getLogger(),
     ORM = require("orm");


var dateFormat  = function (date , fmt){
    var o = {
        "M+": date.getMonth() + 1, //月份
        "d+": date.getDate(), //日
        "h+": date.getHours(), //小时
        "m+": date.getMinutes(), //分
        "s+": date.getSeconds(), //秒
        "q+": Math.floor((date.getMonth() + 3) / 3), //季度
        "S": date.getMilliseconds() //毫秒
    };
    if (/(y+)/.test(fmt)) fmt = fmt.replace(RegExp.$1, (date.getFullYear() + "").substr(4 - RegExp.$1.length));
    for (var k in o)
        if (new RegExp("(" + k + ")").test(fmt)) fmt = fmt.replace(RegExp.$1, (RegExp.$1.length == 1) ? (o[k]) : (("00" + o[k]).substr(("" + o[k]).length)));
    return fmt;
};

var StatisticsService = function (){

    this.statisticsDao = global.models.statisticsDao;
    this.applyDao = global.models.applyDao;


    this.url = GLOBAL.pjconfig.storage.errorMsgTopUrl;

    logger.debug('query url : ' + this.url);
};



StatisticsService.prototype = {
    queryById : function (param  , callback){
        this.statisticsDao.find({projectId :param.projectId , startDate : dateFormat(param.startDate , 'yyyy-MM-dd hh:mm:ss')} , function (err , items){
            if(err){
                callback(err);
                return;
            }



            if(items[0] ){
                items[0].content = JSON.parse(items[0].content);
                items[0].content = _.map(items[0].content, function(value, key){
                    return {title : key , total : value};
                });

                items[0].content = items[0].content.sort(function ( a, b){
                    if(a.total < b.total){
                        return 1;
                    }else {
                        return -1;
                    }
                });

                if(param.top){
                    items[0].content = items[0].content.slice(0 , param.top);
                }

            }
            callback(null,  items);
        });

    },
    queryByChart : function (param  , callback){
        //筛选参数
        var s_params = {};
        if(param.projectId !=-1){
            s_params.projectId = param.projectId;
        }

        //时间开始范围 7天内 或者1个月内
        var oneDay = 1000 * 60 * 60 * 24;
        var day = param.timeScope ==1?7:30;
        param.startTime = new Date() - oneDay * day;
        param.startTime = dateFormat(new Date(param.startTime), 'yyyy-MM-dd');

        if(GLOBAL.DEBUG){
            logger.info("query start time is "+ param.startTime);
        }
        this.statisticsDao.find(s_params).only("endDate" , "startDate" , "projectId" , "id" , "total").where("startDate >=?", [param.startTime]).all(function (err, items)  {
            if(err){
                callback(err);
                return;
            }
            callback(null,{ret:0, msg:"success", data: items });
        }).where(function(){

        });
    },
    fetchAndSave : function (id , startDate , cb){
        var self = this;
        http.get((this.url + '?id=' + id + '&startDate=' + (startDate -0 ))  , function(res){
            var buffer = '';
            res.on('data' , function (chunk){
                buffer += chunk.toString();
            }).on('end' , function (){
                var saveModel = {};
                try {
                    var result = JSON.parse(buffer);

                    saveModel = {
                        startDate : new Date(result.startDate),
                        endDate : new Date(result.endDate),
                        content : JSON.stringify(result.result),
                        projectId : id,
                        total : 0
                    };

                    _.each(result.result , function (value ,key ){
                            saveModel.total += result.result[key];
                    })

                }catch(err){
                    logger.error('error :' + err);
                    saveModel = {
                        startDate: startDate,
                        endDate: startDate + 86400000,
                        content: "{}",
                        total: 0
                    }
                }

                self.statisticsDao.create(saveModel , function (err , items){
                    if(err){
                        logger.error("Insert into b_statistics error(id=", id + ") :  " +  err);
                    }
                    logger.info("Insert into b_statistics success(id=", id + ") :  " );
                    cb && cb(err);
                });
            })

        }).on('error' , function (err){
            logger.error('error :' + err);
        });
    },

    startMonitor : function (){
        var self = this;


        var getTomorrowDay = function (){
            var tomorrow = new Date(nowDate);
            tomorrow.setHours(1, 0 , 0 ,0);
            tomorrow.setDate(tomorrow.getDate() + 1);
            return tomorrow;
//            return new Date(nowDate.getFullYear() + "-" + (nowDate.getMonth() + 1)  + "-" + (nowDate.getDate()+1) + " 01:00:00");
        }

        var getStartDay = function (){
            var startDate = new Date(nowDate);
            startDate.setHours(0 , 0 , 0 ,0);
            return startDate;
        }
{'fraction_non_alphanumeric': 0.1536213352817609, 'fraction_numerical': 0.014430541601972782, 'mean_word_length': 1.4257864421798847, 'pattern_counts': {'":': 14, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5422074', 'n_tokens_mistral': 3338, 'n_tokens_neox': 3130, 'n_words': 634}