<commit_before>

define(['q'], function(Q) {
    'use strict';
    return {
	executeOnHost: function(cmds, ip, user, cb_message, cb_complete, cb_processing, cb_error) {
	    var self = this;
	    var ssh2shell = require('ssh2shell');
	    var output = {
		user: user,
		ip: ip,
		stdout: ''
	    };
	    
	    var host = {
		server: {
		    host: ip,
		    port: 22,
		    userName: user.name,
		    privateKey: require('fs').readFileSync(user.key)
		},
		commands: cmds,
		idleTimeOut: 10000,
		msg: {
		    send: function( message ) {
			if (cb_message)
			    cb_message(message);
		    }
		},
		onCommandProcessing: function( command, response, sshObj, stream ) {
		    if (cb_processing) {
			cb_processing(command, response);
		    }
		},
		onCommandComplete: function( command, response, sshObj ) {
		    if (cb_complete)
			cb_complete(command, response);
		},
		onCommandTimeout: function( command, response, sshObj, stream, connection) {
		    // pass through, we don't want to time out
		},
		onEnd: function( sessionText, sshObj ) {
		    var stdout = sessionText;
		    stdout = stdout.replace('Connected to '+ip,'');
		    stdout = stdout.replace(new RegExp(user.name + '@.+\$'), '');
		    output.stdout = stdout;
		}
	    };
	    var ssh = new ssh2shell(host);
	    /*
	      ssh.on ("error", function(err, type, close, cb) {
	      reject(err + ' on ' + user.name + '@' + ip);
	      if (cb_error)
	      cb_error(err, type, close, cb);
	      });
	    */
	    var deferred = Q.defer();
	    ssh.on("close", function(hadError) {
		if (hadError) {
		    deferred.reject();
		}
		else {
		    deferred.resolve(output);
		}
	    });
	    ssh.connect();
	    return deferred.promise;
	},
	parseMakePercentOutput: function(output) {
	    var regex = /[0-9]+%/gm;
	    var match = null;
	    var retVals = [];
	    while (match = regex.exec(output)) {
		var percent = parseInt(new String(match).replace('%',''), 10);
		retVals.push(percent);
	    }
	    return retVals;
	},
	parseMakeErrorOutput: function(output) {
	    var regex = /([^:^\n]+):(\d+):(\d+):\s(\w+\s*\w*):\s(.+)\n(\s+)(.*)\s+\^+/gm;
	    var match = null;
	    var retVals = [];
	    while (match = regex.exec(output)) {
		retVals.push({
		    filename:       match[1].replace(self.gen_dir + '/src/', ''),
		    packageName:    filename.split('/')[0],
		    line:           parseInt(match[2]),
		    column:         parseInt(match[3]),
		    type:           match[4],
		    text:           match[5],
		    codeWhitespace: match[6],
		    code:           match[7],
		    adjustedColumn: column - codeWhitespace.length
		});
	    }
	    return retVals;
	},
	parsePsAuxOutput: function(output) {
	    return output;
	},
	mkdirRemote: function(dir, ip, user) {
	    var client = require('scp2');
	    client.defaults({
		host: ip,
		username: user.name,
		privateKey: require('fs').readFileSync(user.key),
	    });
	    var deferred = Q.defer();
	    client.mkdir(dir, function(err) {
		if (err)
		    deferred.reject(err);
		else
		    deferred.resolve();
	    });
	    return deferred.promise;
	},
	copyToHost: function(from, to, ip, user) {
	    var client = require('scp2');
	    
	    var deferred = Q.defer();
	    client.scp(from, {
		host: ip,
		username: user.name,
		privateKey: require('fs').readFileSync(user.key),
		path: to
	    }, function(err) {
		if (err)
		    deferred.reject(err);
		else
		    deferred.resolve();<commit_msg>fixed bug somehow related to scp2 which prevented parallel multi-host compilation from working.  workaround just uses utility function I already had.<commit_after>

define(['q'], function(Q) {
    'use strict';
    return {
	executeOnHost: function(cmds, ip, user, cb_message, cb_complete, cb_processing, cb_error) {
	    var self = this;
	    var ssh2shell = require('ssh2shell');
	    var output = {
		user: user,
		ip: ip,
		stdout: ''
	    };
	    
	    var host = {
		server: {
		    host: ip,
		    port: 22,
		    userName: user.name,
		    privateKey: require('fs').readFileSync(user.key)
		},
		commands: cmds,
		idleTimeOut: 10000,
		msg: {
		    send: function( message ) {
			if (cb_message)
			    cb_message(message);
		    }
		},
		onCommandProcessing: function( command, response, sshObj, stream ) {
		    if (cb_processing) {
			cb_processing(command, response);
		    }
		},
		onCommandComplete: function( command, response, sshObj ) {
		    if (cb_complete)
			cb_complete(command, response);
		},
		onCommandTimeout: function( command, response, sshObj, stream, connection) {
		    // pass through, we don't want to time out
		},
		onEnd: function( sessionText, sshObj ) {
		    var stdout = sessionText;
		    stdout = stdout.replace('Connected to '+ip,'');
		    stdout = stdout.replace(new RegExp(user.name + '@.+\$'), '');
		    output.stdout = stdout;
		}
	    };
	    var ssh = new ssh2shell(host);
	    /*
	      ssh.on ("error", function(err, type, close, cb) {
	      reject(err + ' on ' + user.name + '@' + ip);
	      if (cb_error)
	      cb_error(err, type, close, cb);
	      });
	    */
	    var deferred = Q.defer();
	    ssh.on("close", function(hadError) {
		if (hadError) {
		    deferred.reject();
		}
		else {
		    deferred.resolve(output);
		}
	    });
	    ssh.connect();
	    return deferred.promise;
	},
	parseMakePercentOutput: function(output) {
	    var regex = /[0-9]+%/gm;
	    var match = null;
	    var retVals = [];
	    while (match = regex.exec(output)) {
		var percent = parseInt(new String(match).replace('%',''), 10);
		retVals.push(percent);
	    }
	    return retVals;
	},
	parseMakeErrorOutput: function(output) {
	    var regex = /([^:^\n]+):(\d+):(\d+):\s(\w+\s*\w*):\s(.+)\n(\s+)(.*)\s+\^+/gm;
	    var match = null;
	    var retVals = [];
	    while (match = regex.exec(output)) {
		retVals.push({
		    filename:       match[1].replace(self.gen_dir + '/src/', ''),
		    packageName:    filename.split('/')[0],
		    line:           parseInt(match[2]),
		    column:         parseInt(match[3]),
		    type:           match[4],
		    text:           match[5],
		    codeWhitespace: match[6],
		    code:           match[7],
		    adjustedColumn: column - codeWhitespace.length
		});
	    }
	    return retVals;
	},
	parsePsAuxOutput: function(output) {
	    return output;
	},
	mkdirRemote: function(dir, ip, user) {
	    var self = this;
	    return self.executeOnHost(['mkdir -p ' + dir],
				      ip,
				      user);
	    /*
	    var client = require('scp2');
	    client.defaults({
		host: ip,
		username: user.name,
		privateKey: require('fs').readFileSync(user.key),
	    });
	    var deferred = Q.defer();
	    client.mkdir(dir, function(err) {
		if (err)
		    deferred.reject(err);
		else
		    deferred.resolve();
	    });
	    return deferred.promise;
	    */
	},
	copyToHost: function(from, to, ip, user) {
	    var client = require('scp2');
	    
	    var deferred = Q.defer();
	    client.scp(from, {
		host: ip,
		username: user.name,
		privateKey: require('fs').readFileSync(user.key),
		path: to
	    }, function(err) {
		if (err)
		    deferred.reject(err);
		else
		    deferred.resolve();
{'fraction_non_alphanumeric': 0.16142675877625828, 'fraction_numerical': 0.006908219371211053, 'mean_word_length': 2.4487117160913954, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6058669', 'n_tokens_mistral': 2742, 'n_tokens_neox': 2372, 'n_words': 518}