<commit_before>var queries = require('./queries');
var insertPlayer = queries.insertPlayer;
var utility = require('./utility');
var getData = utility.getData;
var async = require('async');
var db = require('./db');
start();

function start()
{
    getSummaries(function(err)
    {
        if (err)
        {
            console.error(err);
        }
        return start();
    });
}

function getSummaries(cb)
{
    db.raw("select account_id from players offset random() * (select count(*) from players) limit 100").asCallback(function(err, results)
    {
        if (err)
        {
            return cb(err);
        }
        var container = utility.generateJob("api_summaries",
        {
            players: results.rows
        });
        getData(container.url, function(err, body)
        {
            if (err)
            {
                //couldn't get data from api, non-retryable
                return cb(JSON.stringify(err));
            }<commit_msg>add delay<commit_after>var queries = require('./queries');
var insertPlayer = queries.insertPlayer;
var utility = require('./utility');
var getData = utility.getData;
var async = require('async');
var db = require('./db');
start();

function start()
{
    getSummaries(function(err)
    {
        if (err)
        {
            console.error(err);
        }
        return setTimeout(start, 1000);
    });
}

function getSummaries(cb)
{
    db.raw("select account_id from players offset random() * (select count(*) from players) limit 100").asCallback(function(err, results)
    {
        if (err)
        {
            return cb(err);
        }
        var container = utility.generateJob("api_summaries",
        {
            players: results.rows
        });
        getData(container.url, function(err, body)
        {
            if (err)
            {
                //couldn't get data from api, non-retryable
                return cb(JSON.stringify(err));
            }
{'fraction_non_alphanumeric': 0.1309034907597536, 'fraction_numerical': 0.00513347022587269, 'mean_word_length': 1.9755725190839695, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '703698', 'n_tokens_mistral': 560, 'n_tokens_neox': 549, 'n_words': 130}