<commit_before>'use strict';

var Disqueue = require('disqueue-node');
var async = require('async');

var config = require('./config');
var constants = require('./constants');
var Run = require('../models/run');
var Log = require('../models/log');

var disq = new Disqueue(config.disque);
var debug = require('debug')('tilloo:runs');

var Runs = {};

Runs.killRun = function killRun(runId, force, callback) {
    Run.findByRunId(runId, function(err, run) {
        if(err) {
            return callback(err);
        }
        else {
            async.parallel([
                    function(done) {
                        disq.addJob({queue: constants.QUEUES.KILL_PREFIX + run.worker, job: JSON.stringify({pid: run.pid}), timeout: 0}, function(err) {
                            if (err) {
                                debug('Unable to queue kill for worker: %s, pid: %d', run.worker, run.pid);
                            }
                            else {
                                debug('kill sent to worker: %s, pid: %d', run.worker, run.pid);
                            }
                            done();
                        });
                    },
                    function(done) {
                        if(force) {
                            disq.addJob({queue: constants.QUEUES.STATUS, job: JSON.stringify({status: constants.JOBSTATUS.FAIL, runId: run._id, type:constants.KILLTYPE.MANUAL}), timeout: 0}, function(err) {
                                    if(err) {
                                        debug('Unable to queue fail status for jobId: %s, runId: %s, status: %s', run.jobId, run._id, run.status);
                                    }
                                    else {
                                        debug('fail status sent');
                                    }
                                    done();
                                });
                        }
                        else {
                            setImmediate(done);
                        }
                    }
                ],
            callback);
        }
    });
};

Runs.deleteRunsOlderThan = function deleteRunsOlderThan(days, callback) {
    Run.findRunsOlderThan(days, function (err, runs) {
        console.info('Deleting ' + runs.length + ' runs');
        async.eachLimit(runs, 10, function(run, done) {
            Log.deleteOutputForRun(run._id, function(err) {
                if(err) {
                    console.error('log delete err: ' + err);
                    done();
                }
                else {
                    run.remove(done);
                }
            });
        }, callback);
    });
};


module.exports = Runs;<commit_msg>Changed Behavior of expireruns.  If a mongodb timeout is encountered it will keep trying to expire other records<commit_after>'use strict';

var Disqueue = require('disqueue-node');
var async = require('async');

var config = require('./config');
var constants = require('./constants');
var Run = require('../models/run');
var Log = require('../models/log');

var disq = new Disqueue(config.disque);
var debug = require('debug')('tilloo:runs');

var Runs = {};

Runs.killRun = function killRun(runId, force, callback) {
    Run.findByRunId(runId, function(err, run) {
        if(err) {
            return callback(err);
        }
        else {
            async.parallel([
                    function(done) {
                        disq.addJob({queue: constants.QUEUES.KILL_PREFIX + run.worker, job: JSON.stringify({pid: run.pid}), timeout: 0}, function(err) {
                            if (err) {
                                debug('Unable to queue kill for worker: %s, pid: %d', run.worker, run.pid);
                            }
                            else {
                                debug('kill sent to worker: %s, pid: %d', run.worker, run.pid);
                            }
                            done();
                        });
                    },
                    function(done) {
                        if(force) {
                            disq.addJob({queue: constants.QUEUES.STATUS, job: JSON.stringify({status: constants.JOBSTATUS.FAIL, runId: run._id, type:constants.KILLTYPE.MANUAL}), timeout: 0}, function(err) {
                                    if(err) {
                                        debug('Unable to queue fail status for jobId: %s, runId: %s, status: %s', run.jobId, run._id, run.status);
                                    }
                                    else {
                                        debug('fail status sent');
                                    }
                                    done();
                                });
                        }
                        else {
                            setImmediate(done);
                        }
                    }
                ],
            callback);
        }
    });
};

Runs.deleteRunsOlderThan = function deleteRunsOlderThan(days, callback) {
    Run.findRunsOlderThan(days, function (err, runs) {
        console.info('Deleting ' + runs.length + ' runs');
        async.eachLimit(runs, 10, function(run, done) {
            Log.deleteOutputForRun(run._id, function(err) {
                if(err) {
                    console.error('log delete err: ' + err);
                    // Eat the error and keep deleting records
                    done();
                }
                else {
                    run.remove(function (err) {
                        if (err) {
                            console.error('run delete err: ' + err);
                        }
                        // Eat the error and keep deleting records
                        done();
                    });
                }
            });
        }, callback);
    });
};


module.exports = Runs;
{'fraction_non_alphanumeric': 0.11704834605597965, 'fraction_numerical': 0.0013570822731128074, 'mean_word_length': 1.0826563051925115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1139201', 'n_tokens_mistral': 1486, 'n_tokens_neox': 1412, 'n_words': 320}