<commit_before>/**
 * Created by kestas on 3/24/2017.
 */

var fs = require('fs');
const path = require('path');
var AWS_S3 = require('./AWS_S3');
var remove = require('rimraf');
var Job = require('../models/job');

var io_emitter = require('socket.io-emitter')({ host: process.env['NEPTUNE_REDIS_HOST'], port: process.env['NEPTUNE_REDIS_PORT'] });
 
const getAllFiles = function(dirPath, arrayOfFiles) {
  files = fs.readdirSync(dirPath)
 
  arrayOfFiles = arrayOfFiles || []
 
  files.forEach(function(file) {
    if (fs.statSync(dirPath + "/" + file).isDirectory()) {
      arrayOfFiles = getAllFiles(dirPath + "/" + file, arrayOfFiles)
    } else {
      arrayOfFiles.push(path.join(dirPath,file))
    }
  })
 
  return arrayOfFiles
}

const createAllFileObjects = function(files_array, data){
    return new Promise( async function (resolve, reject){
            for (var i = 0; i < files_array.length; i++) {
                console.log("filename: " + files_array[i]);
                var filename = path.basename(files_array[i])
                var extension = path.extname(files_array[i])
                const fileContent = fs.readFileSync(files_array[i]);
                await data.createFile(filename, extension, fileContent);
            }
            resolve()
        }
    )
}

module.exports.compile = function(req, res) {
    var jobid = req.body.jobid;
    console.log("JOB ID:", jobid);
    var jobdir = req.body.jobdir;
    console.log("JOB DIR:", jobdir);

    var mintname = req.body.sourcefilename;
    var ininame = req.body.configfilename;

    var FLUIGI_BINARY_PATH = path.join(global.Neptune_ROOT_DIR, "jobs", "Fluigi-jar-with-dependencies.jar");
    var mint_path = path.join(global.Neptune_ROOT_DIR, jobdir, mintname);
    var ini_path = path.join(global.Neptune_ROOT_DIR, jobdir, ininame);
    var out_path = path.join(global.Neptune_ROOT_DIR, jobdir, "output");
    var cwd = path.join(global.Neptune_ROOT_DIR, jobdir);
    var logpath = path.join(global.Neptune_ROOT_DIR, jobdir, "log.txt");

    console.log('COMPILING!');
    console.log('MINT PATH: %s', mint_path);
    console.log('INI PATH: %s', ini_path);
    console.log('OUT PATH: %s', out_path);

    console.log("java -jar ",FLUIGI_BINARY_PATH," ",mint_path," -i ",ini_path," -o sej");
    var par_terminal = require('child_process').spawn(
        'java', ['-jar', FLUIGI_BINARY_PATH, mint_path, '-i', ini_path, '-o', 'sej'], { cwd: cwd }
    );

    par_terminal.stdout.on('data', function(data) {
        fs.appendFile(logpath, data.toString(), function(err) {
            if (err) throw err;
        });
        io_emitter.to(jobid).emit('stdout', data.toString());
        console.log(data.toString());
    });

    par_terminal.stderr.on("data", function(data) {<commit_msg>Updated max ram limit for java fluigi<commit_after>/**
 * Created by kestas on 3/24/2017.
 */

var fs = require('fs');
const path = require('path');
var AWS_S3 = require('./AWS_S3');
var remove = require('rimraf');
var Job = require('../models/job');

var io_emitter = require('socket.io-emitter')({ host: process.env['NEPTUNE_REDIS_HOST'], port: process.env['NEPTUNE_REDIS_PORT'] });
 
const getAllFiles = function(dirPath, arrayOfFiles) {
  files = fs.readdirSync(dirPath)
 
  arrayOfFiles = arrayOfFiles || []
 
  files.forEach(function(file) {
    if (fs.statSync(dirPath + "/" + file).isDirectory()) {
      arrayOfFiles = getAllFiles(dirPath + "/" + file, arrayOfFiles)
    } else {
      arrayOfFiles.push(path.join(dirPath,file))
    }
  })
 
  return arrayOfFiles
}

const createAllFileObjects = function(files_array, data){
    return new Promise( async function (resolve, reject){
            for (var i = 0; i < files_array.length; i++) {
                console.log("filename: " + files_array[i]);
                var filename = path.basename(files_array[i])
                var extension = path.extname(files_array[i])
                const fileContent = fs.readFileSync(files_array[i]);
                await data.createFile(filename, extension, fileContent);
            }
            resolve()
        }
    )
}

module.exports.compile = function(req, res) {
    var jobid = req.body.jobid;
    console.log("JOB ID:", jobid);
    var jobdir = req.body.jobdir;
    console.log("JOB DIR:", jobdir);

    var mintname = req.body.sourcefilename;
    var ininame = req.body.configfilename;

    var FLUIGI_BINARY_PATH = path.join(global.Neptune_ROOT_DIR, "jobs", "Fluigi-jar-with-dependencies.jar");
    var mint_path = path.join(global.Neptune_ROOT_DIR, jobdir, mintname);
    var ini_path = path.join(global.Neptune_ROOT_DIR, jobdir, ininame);
    var out_path = path.join(global.Neptune_ROOT_DIR, jobdir, "output");
    var cwd = path.join(global.Neptune_ROOT_DIR, jobdir);
    var logpath = path.join(global.Neptune_ROOT_DIR, jobdir, "log.txt");

    console.log('COMPILING!');
    console.log('MINT PATH: %s', mint_path);
    console.log('INI PATH: %s', ini_path);
    console.log('OUT PATH: %s', out_path);

    console.log("java -jar ",FLUIGI_BINARY_PATH," ",mint_path," -i ",ini_path," -o sej");
    var par_terminal = require('child_process').spawn(
        'java', ['-Xmx', '5G', '-jar', FLUIGI_BINARY_PATH, mint_path, '-i', ini_path, '-o', 'sej'], { cwd: cwd }
    );

    par_terminal.stdout.on('data', function(data) {
        fs.appendFile(logpath, data.toString(), function(err) {
            if (err) throw err;
        });
        io_emitter.to(jobid).emit('stdout', data.toString());
        console.log(data.toString());
    });

    par_terminal.stderr.on("data", function(data) {
{'fraction_non_alphanumeric': 0.15699964196204796, 'fraction_numerical': 0.0037593984962406013, 'mean_word_length': 3.271406727828746, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7449228', 'n_tokens_mistral': 1956, 'n_tokens_neox': 1881, 'n_words': 368}