<commit_before>/**
 * Created by james on 13-12-5.
 * spider middleware
 */

var crypto = require('crypto');
var url =  require("url");
var util = require('util');
var async = require('async');
var myredis = require('../lib/myredis.js');

var spider = function(spiderCore){
    this.spiderCore = spiderCore;
    this.queue_length = 0;
    this.driller_rules_updated = 0;
    this.driller_rules = {};
    logger = spiderCore.settings.logger;
}

////report to spidercore standby////////////////////////
spider.prototype.assembly = function(callback){
    var self = this;
    var dbtype = 'redis';
    if(this.spiderCore.settings['use_ssdb'])dbtype = 'ssdb';
    async.series([
        function(cb){
            myredis.createClient(
                self.spiderCore.settings['driller_info_redis_db'][0],
                self.spiderCore.settings['driller_info_redis_db'][1],
                self.spiderCore.settings['driller_info_redis_db'][2],
                dbtype,
                function(err,cli){
                    self.redis_cli0 = cli;
                    cb(err);
                });
        },
        function(cb){
            myredis.createClient(
                self.spiderCore.settings['url_info_redis_db'][0],
                self.spiderCore.settings['url_info_redis_db'][1],
                self.spiderCore.settings['url_info_redis_db'][2],
                dbtype,
                function(err,cli){
                    self.redis_cli1 = cli;
                    cb(err);
                });
        },
        function(cb){
            myredis.createClient(
                self.spiderCore.settings['url_report_redis_db'][0],
                self.spiderCore.settings['url_report_redis_db'][1],
                self.spiderCore.settings['url_report_redis_db'][2],
                dbtype,
                function(err,cli){
                    self.redis_cli2 = cli;
                    cb(err);
                });
        }
    ],function(err,result){
        if(callback)callback(null,'done');
    });
}
/**
 * smart parse string to json object deeply(level2)
 * @param source
 */
spider.prototype.jsonSmartDeepParse = function(obj){
    var dataobj = {};
    var numberPattern = new RegExp("^\-?[0-9]+$");
    for(i in obj){
        if(obj.hasOwnProperty(i)){
            if(typeof(obj[i])==='string'&&(obj[i].charAt(0)==='{'||obj[i].charAt(0)==='[')){
                dataobj[i] = JSON.parse(obj[i]);
            }else if(numberPattern.test(obj[i])){
                dataobj[i] = parseInt(obj[i]);
            }else if(obj[i]==='true'){
                dataobj[i] = true;
            }else if(obj[i]==='false'){
                dataobj[i] = false;
            }else dataobj[i] = obj[i];
        }
    }
    return dataobj;
}

//refresh the driller rules//////////////////////////////
spider.prototype.refreshDrillerRules = function(){
    var self = this;
    var redis_cli = this.redis_cli0;
        redis_cli.get('updated:driller:rule',function(err,value){
            if (err)throw(err);
            if(self.driller_rules_updated!==parseInt(value)){//driller is changed
                logger.debug('driller rules is changed');
                redis_cli.hlist('driller:*',function(err,values){
                    if (err)throw(err);
                    self.tmp_driller_rules = {};
                    self.tmp_driller_rules_length = values.length;
                    for(var i=0;i<values.length;i++){
                        self.wrapper_rules(values[i]);
                    }
                });
                self.driller_rules_updated=parseInt(value);
            }else{
                logger.debug('driller rules is not changed, queue length: '+self.queue_length);
                setTimeout(function(){self.refreshDrillerRules();},self.spiderCore.settings['check_driller_rules_interval']*1000);
            }
        })
}
//wrapper each rule
spider.prototype.wrapper_rules = function(key){
    var self = this;
    var redis_cli = this.redis_cli0;
    redis_cli.hgetall(key, function(err,value){//for synchronized using object variable
        if(self.tmp_driller_rules==undefined)self.tmp_driller_rules = {};
        var isActive = value['active']=='true'||value['active']==true||value['active']=='1'||value['active']==1?true:false;
        if(isActive||self.spiderCore.settings['test']) {
            logger.debug('Load rule: '+key);
            if (self.tmp_driller_rules[value['domain']] == undefined)self.tmp_driller_rules[value['domain']] = {};<commit_msg>refresh rules log level change to info<commit_after>/**
 * Created by james on 13-12-5.
 * spider middleware
 */

var crypto = require('crypto');
var url =  require("url");
var util = require('util');
var async = require('async');
var myredis = require('../lib/myredis.js');

var spider = function(spiderCore){
    this.spiderCore = spiderCore;
    this.queue_length = 0;
    this.driller_rules_updated = 0;
    this.driller_rules = {};
    logger = spiderCore.settings.logger;
}

////report to spidercore standby////////////////////////
spider.prototype.assembly = function(callback){
    var self = this;
    var dbtype = 'redis';
    if(this.spiderCore.settings['use_ssdb'])dbtype = 'ssdb';
    async.series([
        function(cb){
            myredis.createClient(
                self.spiderCore.settings['driller_info_redis_db'][0],
                self.spiderCore.settings['driller_info_redis_db'][1],
                self.spiderCore.settings['driller_info_redis_db'][2],
                dbtype,
                function(err,cli){
                    self.redis_cli0 = cli;
                    cb(err);
                });
        },
        function(cb){
            myredis.createClient(
                self.spiderCore.settings['url_info_redis_db'][0],
                self.spiderCore.settings['url_info_redis_db'][1],
                self.spiderCore.settings['url_info_redis_db'][2],
                dbtype,
                function(err,cli){
                    self.redis_cli1 = cli;
                    cb(err);
                });
        },
        function(cb){
            myredis.createClient(
                self.spiderCore.settings['url_report_redis_db'][0],
                self.spiderCore.settings['url_report_redis_db'][1],
                self.spiderCore.settings['url_report_redis_db'][2],
                dbtype,
                function(err,cli){
                    self.redis_cli2 = cli;
                    cb(err);
                });
        }
    ],function(err,result){
        if(callback)callback(null,'done');
    });
}
/**
 * smart parse string to json object deeply(level2)
 * @param source
 */
spider.prototype.jsonSmartDeepParse = function(obj){
    var dataobj = {};
    var numberPattern = new RegExp("^\-?[0-9]+$");
    for(i in obj){
        if(obj.hasOwnProperty(i)){
            if(typeof(obj[i])==='string'&&(obj[i].charAt(0)==='{'||obj[i].charAt(0)==='[')){
                dataobj[i] = JSON.parse(obj[i]);
            }else if(numberPattern.test(obj[i])){
                dataobj[i] = parseInt(obj[i]);
            }else if(obj[i]==='true'){
                dataobj[i] = true;
            }else if(obj[i]==='false'){
                dataobj[i] = false;
            }else dataobj[i] = obj[i];
        }
    }
    return dataobj;
}

//refresh the driller rules//////////////////////////////
spider.prototype.refreshDrillerRules = function(){
    var self = this;
    var redis_cli = this.redis_cli0;
        redis_cli.get('updated:driller:rule',function(err,value){
            if (err)throw(err);
            if(self.driller_rules_updated!==parseInt(value)){//driller is changed
                logger.info('driller rules is changed');
                redis_cli.hlist('driller:*',function(err,values){
                    if (err)throw(err);
                    self.tmp_driller_rules = {};
                    self.tmp_driller_rules_length = values.length;
                    for(var i=0;i<values.length;i++){
                        self.wrapper_rules(values[i]);
                    }
                });
                self.driller_rules_updated=parseInt(value);
            }else{
                logger.debug('driller rules is not changed, queue length: '+self.queue_length);
                setTimeout(function(){self.refreshDrillerRules();},self.spiderCore.settings['check_driller_rules_interval']*1000);
            }
        })
}
//wrapper each rule
spider.prototype.wrapper_rules = function(key){
    var self = this;
    var redis_cli = this.redis_cli0;
    redis_cli.hgetall(key, function(err,value){//for synchronized using object variable
        if(self.tmp_driller_rules==undefined)self.tmp_driller_rules = {};
        var isActive = value['active']=='true'||value['active']==true||value['active']=='1'||value['active']==1?true:false;
        if(isActive||self.spiderCore.settings['test']) {
            logger.info('Load rule: '+key);
            if (self.tmp_driller_rules[value['domain']] == undefined)self.tmp_driller_rules[value['domain']] = {};
{'fraction_non_alphanumeric': 0.15748031496062992, 'fraction_numerical': 0.0073195075967616725, 'mean_word_length': 2.041483979763912, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '573039', 'n_tokens_mistral': 2789, 'n_tokens_neox': 2702, 'n_words': 392}