<commit_before>var utility = require('./utility');
var config = require('./config');
var processors = require('./processors');
var getData = utility.getData;
var db = require('./db');
var r = require('./redis');
var redis = r.client;
var jobs = r.jobs;
var kue = r.kue;
var logger = utility.logger;
var generateJob = utility.generateJob;
var async = require('async');
var operations = require('./operations');
var insertMatch = operations.insertMatch;
var queueReq = operations.queueReq;
var fullhistory = require('./tasks/fullhistory');
var updatenames = require('./tasks/updatenames');
var selector = require('./selector');
var domain = require('domain');
var trackedPlayers = {};
var activePlayers = {};
var ratingPlayers = {};
/*
var seaport = require('seaport');
var server = seaport.createServer();
server.listen(config.REGISTRY_PORT);
*/
var retrievers = config.RETRIEVER_HOST;
//don't need these handlers when kue supports job ttl in 0.9?
process.on('SIGTERM', function() {
    clearActiveJobs(function(err) {
        process.exit(err || 1);
    });
});
process.on('SIGINT', function() {
    clearActiveJobs(function(err) {
        process.exit(err || 1);
    });
});
var d = domain.create();
d.on('error', function(err) {
    console.log(err.stack);
    clearActiveJobs(function(err2) {
        process.exit(err2 || err || 1);
    });
});
d.run(function() {
    console.log("[WORKER] starting worker");
    build(function() {
        startScan();
        jobs.promote();
        jobs.process('api', config.STEAM_API_KEY.split(",").length, processors.processApi);
        jobs.process('mmr', processors.processMmr);
        jobs.process('request', processors.processApi);
        setInterval(fullhistory, 17 * 60 * 1000, function() {});
        setInterval(updatenames, 3 * 60 * 1000, function() {});
        setInterval(build, 3 * 60 * 1000, function() {});
        //todo implement redis window check 
        //setInterval(apiStatus, 2 * 60 * 1000);
    });
});

function clearActiveJobs(cb) {
    jobs.active(function(err, ids) {
        if (err) {
            return cb(err);
        }
        async.mapSeries(ids, function(id, cb) {
            kue.Job.get(id, function(err, job) {<commit_msg>change full history to 5 minutes<commit_after>var utility = require('./utility');
var config = require('./config');
var processors = require('./processors');
var getData = utility.getData;
var db = require('./db');
var r = require('./redis');
var redis = r.client;
var jobs = r.jobs;
var kue = r.kue;
var logger = utility.logger;
var generateJob = utility.generateJob;
var async = require('async');
var operations = require('./operations');
var insertMatch = operations.insertMatch;
var queueReq = operations.queueReq;
var fullhistory = require('./tasks/fullhistory');
var updatenames = require('./tasks/updatenames');
var selector = require('./selector');
var domain = require('domain');
var trackedPlayers = {};
var activePlayers = {};
var ratingPlayers = {};
/*
var seaport = require('seaport');
var server = seaport.createServer();
server.listen(config.REGISTRY_PORT);
*/
var retrievers = config.RETRIEVER_HOST;
//don't need these handlers when kue supports job ttl in 0.9?
process.on('SIGTERM', function() {
    clearActiveJobs(function(err) {
        process.exit(err || 1);
    });
});
process.on('SIGINT', function() {
    clearActiveJobs(function(err) {
        process.exit(err || 1);
    });
});
var d = domain.create();
d.on('error', function(err) {
    console.log(err.stack);
    clearActiveJobs(function(err2) {
        process.exit(err2 || err || 1);
    });
});
d.run(function() {
    console.log("[WORKER] starting worker");
    build(function() {
        startScan();
        jobs.promote();
        jobs.process('api', config.STEAM_API_KEY.split(",").length, processors.processApi);
        jobs.process('mmr', processors.processMmr);
        jobs.process('request', processors.processApi);
        setInterval(fullhistory, 5 * 60 * 1000, function() {});
        setInterval(updatenames, 3 * 60 * 1000, function() {});
        setInterval(build, 3 * 60 * 1000, function() {});
        //todo implement redis window check 
        //setInterval(apiStatus, 2 * 60 * 1000);
    });
});

function clearActiveJobs(cb) {
    jobs.active(function(err, ids) {
        if (err) {
            return cb(err);
        }
        async.mapSeries(ids, function(id, cb) {
            kue.Job.get(id, function(err, job) {
{'fraction_non_alphanumeric': 0.16877256317689532, 'fraction_numerical': 0.016245487364620937, 'mean_word_length': 3.523469387755102, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '849500', 'n_tokens_mistral': 1436, 'n_tokens_neox': 1343, 'n_words': 317}