<commit_before>var utility = require('./utility');
var processors = require('./processors');
var getData = utility.getData;
var db = require('./db');
var r = require('./redis');
var redis = r.client;
var jobs = r.jobs;
var kue = r.kue;
var logger = utility.logger;
var generateJob = utility.generateJob;
var async = require('async');
var operations = require('./operations');
var insertMatch = operations.insertMatch;
var queueReq = operations.queueReq;
var fullhistory = require('./tasks/fullhistory');
var updatenames = require('./tasks/updatenames');
var selector = require('./selector');
var getmissing = require('./tasks/getmissing');

var trackedPlayers = {};
var ratingPlayers = {};

console.log("[WORKER] starting worker");
build(function() {
    startScan();
    jobs.promote();
    jobs.process('api', processors.processApi);
    jobs.process('mmr', processors.processMmr);
    setInterval(clearActiveJobs, 1 * 60 * 1000, function() {});
    setInterval(fullhistory, 60 * 60 * 1000, function() {});
    setInterval(updatenames, 5 * 60 * 1000, function() {});
    setInterval(getmissing, 5 * 60 * 1000, function() {});
    setInterval(build, 5 * 60 * 1000, function() {});
});

function build(cb) {
    db.players.find(selector("tracked"), function(err, docs) {
        if (err) {
            return build(cb);
        }
        var t = {};
        var r = {};
        var b = [];
        docs.forEach(function(player) {
            t[player.account_id] = true;
        });
        async.map(utility.getRetrieverUrls(), function(url, cb) {
            getData(url, function(err, body) {
                for (var key in body.accounts) {
                    b.push(body.accounts[key]);
                }
                for (var key in body.accountToIdx) {
                    r[key] = url + "?account_id=" + key;
                }
                cb(err);
            });
        }, function(err) {
            if (err) {
                build(cb);
            }
            trackedPlayers = t;
            ratingPlayers = r;
            redis.set("bots", JSON.stringify(b));
            redis.set("ratingPlayers", JSON.stringify(r));<commit_msg>added return to build error<commit_after>var utility = require('./utility');
var processors = require('./processors');
var getData = utility.getData;
var db = require('./db');
var r = require('./redis');
var redis = r.client;
var jobs = r.jobs;
var kue = r.kue;
var logger = utility.logger;
var generateJob = utility.generateJob;
var async = require('async');
var operations = require('./operations');
var insertMatch = operations.insertMatch;
var queueReq = operations.queueReq;
var fullhistory = require('./tasks/fullhistory');
var updatenames = require('./tasks/updatenames');
var selector = require('./selector');
var getmissing = require('./tasks/getmissing');

var trackedPlayers = {};
var ratingPlayers = {};

console.log("[WORKER] starting worker");
build(function() {
    startScan();
    jobs.promote();
    jobs.process('api', processors.processApi);
    jobs.process('mmr', processors.processMmr);
    setInterval(clearActiveJobs, 1 * 60 * 1000, function() {});
    setInterval(fullhistory, 60 * 60 * 1000, function() {});
    setInterval(updatenames, 5 * 60 * 1000, function() {});
    setInterval(getmissing, 5 * 60 * 1000, function() {});
    setInterval(build, 5 * 60 * 1000, function() {});
});

function build(cb) {
    db.players.find(selector("tracked"), function(err, docs) {
        if (err) {
            return build(cb);
        }
        var t = {};
        var r = {};
        var b = [];
        docs.forEach(function(player) {
            t[player.account_id] = true;
        });
        async.map(utility.getRetrieverUrls(), function(url, cb) {
            getData(url, function(err, body) {
                for (var key in body.accounts) {
                    b.push(body.accounts[key]);
                }
                for (var key in body.accountToIdx) {
                    r[key] = url + "?account_id=" + key;
                }
                cb(err);
            });
        }, function(err) {
            if (err) {
                return build(cb);
            }
            trackedPlayers = t;
            ratingPlayers = r;
            redis.set("bots", JSON.stringify(b));
            redis.set("ratingPlayers", JSON.stringify(r));
{'fraction_non_alphanumeric': 0.15754215754215753, 'fraction_numerical': 0.016632016632016633, 'mean_word_length': 2.409448818897638, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4682265', 'n_tokens_mistral': 1354, 'n_tokens_neox': 1241, 'n_words': 294}