<commit_before>var uuid = require('node-uuid');
var urlLib = require('url');
var request = require('request');
var Joi = require('joi');

var log = require('../logger.js');
var storage = require('../storage.js');
var config = require('../config.js');
var unzip = require('../unzip.js');
var getReport = require('../report/index.js');
var runnerConfig = require('../report/runnerConfig.js');
var formatReport = require('../formatReport.js');


var internals = {};


var jobs = require('../jobs.js');

var processStamp = '_' + Date.now() + process.pid;

internals.phantomWorker = function(data, next){
    var id = data && data.id;
    log.info(id + ' processStamp', {processStamp: processStamp, id: id});
    getReport(data, function (err, harvest, report) {
        data.state.processed = new Date();
        if (err) {
            data.error = {
                error: true,
                err: {
                    message: err.message,
                    stack: err.stack
                },
                message: 'An error occured. Code 002.\n\r ' + err.message
            };
            log.error('An error occured. Code 002. (Runtime Error):', {error: err, id: id});
            storage.set(id, data);
            // set success on job to avoid retries
            return next();
        }

        var hasNoAppError = (data && !data.error);
        var hasOutput = harvest && report;

        log.info('getReport returned result from phamtom', {
            'id': id,
            'hasNoAppError': !!hasNoAppError,
            'hasOutput': !!hasOutput
        });

        if (hasNoAppError || ((!err || hasNoAppError) && data && hasOutput)) {
            data.harvest = harvest;
            data.report = formatReport(report);
        }

        log.info('Formating done, saving result.', {'id': id});
        storage.set(id, data, next);
     });
};

var _queue;
internals.queue = function(id){
    if (!_queue){
        _queue = jobs(internals.phantomWorker, 5, 60);
    }<commit_msg>timeout jobs, somewhat linked to #20<commit_after>var uuid = require('node-uuid');
var urlLib = require('url');
var request = require('request');
var Joi = require('joi');

var log = require('../logger.js');
var storage = require('../storage.js');
var config = require('../config.js');
var unzip = require('../unzip.js');
var getReport = require('../report/index.js');
var runnerConfig = require('../report/runnerConfig.js');
var formatReport = require('../formatReport.js');


var internals = {};


var jobs = require('../jobs.js');

var processStamp = '_' + Date.now() + process.pid;

var TIMEOUT_SECONDS = 120;

internals.phantomWorker = function(data, next){
    var id = data && data.id;
    log.info(id + ' processStamp', {processStamp: processStamp, id: id});

    var timeout = new Date(data.state.queued);
    timeout.setSeconds(timeout.getSeconds() + TIMEOUT_SECONDS);

    // dont allow same job to retry for too long, if level-jobs/queue-system doesnt abort job
    var now = new Date();
    if (now >= timeout){
        data.state.timedout = new Date();
        data.error = {
            error: true,
            message: 'Something went wrong, processing timed out. Code 002.'
        };
        log.error('Something went wrong, processing timed out. Code 002. (Runtime Error):', {id: id});
        return storage.set(id, data, next);
    }

    getReport(data, function (err, harvest, report) {

        data.state.processed = new Date();

        if (err) {
            data.error = {
                error: true,
                err: {
                    message: err.message,
                    stack: err.stack
                },
                message: 'An error occured. Code 002.\n\r ' + err.message
            };
            log.error('An error occured. Code 002. (Runtime Error):', {error: err, id: id});
            storage.set(id, data);
            // set success on job to avoid retries
            return next();
        }

        var hasNoAppError = (data && !data.error);
        var hasOutput = harvest && report;

        log.info('getReport returned result from phamtom', {
            'id': id,
            'hasNoAppError': !!hasNoAppError,
            'hasOutput': !!hasOutput
        });

        if (hasNoAppError || ((!err || hasNoAppError) && data && hasOutput)) {
            data.harvest = harvest;
            try{
                data.report = formatReport(report);
            } catch(e){
                data.error = {
                    message: 'Formatting of report failed for '+id,
                    error: true,
                    err: {
                        message: e.message,
                        stack: e.stack
                    }
                };
                log.error(data.error.message);
            }
        }

        log.info('Formating done, saving result.', {'id': id});
        storage.set(id, data, next);
     });
};

var _queue;
internals.queue = function(id){
    if (!_queue){
        _queue = jobs(internals.phantomWorker, 5, 60);
    }
{'fraction_non_alphanumeric': 0.1489065606361829, 'fraction_numerical': 0.005765407554671968, 'mean_word_length': 1.940385739333723, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4516347', 'n_tokens_mistral': 1490, 'n_tokens_neox': 1416, 'n_words': 384}