<commit_before>var utility = require('./utility');
var queue = require('./queue');
var pQueue = queue.getQueue('parse');
var rQueue = queue.getQueue('request');
var getData = utility.getData;
var queries = require('./queries');
var redis = require('./redis');
var insertMatch = queries.insertMatch;
var db = require('./db');
rQueue.process(100, processRequest);

function processRequest(job, cb)
{
    var payload = job.data.payload;
    if (payload.match_id)
    {
        //request match id, get data from API
        getData(job.data.url, function(err, body)
        {
            if (err)
            {
                //couldn't get data from api, non-retryable
                return cb(JSON.stringify(err));
            }
            //match details response
            var match = body.result;
            match.parse_status = 0;
            insertMatch(db, redis, match,
            {
                type: "api",
                attempts: 1<commit_msg>make requests lifo<commit_after>var utility = require('./utility');
var queue = require('./queue');
var pQueue = queue.getQueue('parse');
var rQueue = queue.getQueue('request');
var getData = utility.getData;
var queries = require('./queries');
var redis = require('./redis');
var insertMatch = queries.insertMatch;
var db = require('./db');
rQueue.process(100, processRequest);

function processRequest(job, cb)
{
    var payload = job.data.payload;
    if (payload.match_id)
    {
        //request match id, get data from API
        getData(job.data.url, function(err, body)
        {
            if (err)
            {
                //couldn't get data from api, non-retryable
                return cb(JSON.stringify(err));
            }
            //match details response
            var match = body.result;
            match.parse_status = 0;
            insertMatch(db, redis, match,
            {
                type: "api",
                attempts: 1,
                lifo: true
{'fraction_non_alphanumeric': 0.1276161306789178, 'fraction_numerical': 0.005104645227156712, 'mean_word_length': 2.001531393568147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5846221', 'n_tokens_mistral': 554, 'n_tokens_neox': 540, 'n_words': 147}