<commit_before>/**
 * Duplicity CLI helper
 * Used in the backup model
 */
(function(require, m)
{

    'use strict';

    var exec = require('child_process').exec;
    var os = require('os');
    var moment = require('moment');

    var module = function()
    {

        var process = null;
        var cancelled = false;
        var outputCallback = null;

        /**
         * Checks if a process is running
         */
        this.isProcessing = function()
        {
            return process !== null;
        };

        /**
         * Registers a callback to be triggered when an output is received from the CLI tool
         * @param callback
         */
        this.onOutput = function(callback)
        {
            outputCallback = callback;
        };

        /**
         * Starts a backup task
         * @param data
         * @param type (full | "")
         * @param callback
         */
        this.doBackup = function(data, type, callback)
        {
            var options = {env: {PASSPHRASE: data.passphrase, TMPDIR: os.tmpdir()}};
            var command = 'duplicity ' + type + ' "' + data.path + '" "' + data.url + '" ' + data.cli_options + ' --verbosity info';
            process = exec(command, options, function(error, stdout, stderr)
            {
                process = null;
                callback(cancelled || error !== null);
            });
            process.stdout.on('data', _onStdout.bind(this));
            process.stderr.on('data', _onStderr.bind(this));
        };

        /**
         * Tries to get a file and save it on the given path
         * @param data
         * @param path
         * @param dest_path
         * @param callback
         */
        this.restoreFile = function(data, path, dest_path, callback)
        {
            var options = {env: {PASSPHRASE: data.passphrase, TMPDIR: os.tmpdir()}};
            var command = 'duplicity restore --file-to-restore "' + path + '" "' + data.url + '" "' + dest_path + '"' + ' ' + data.cli_options + ' --verbosity info';
            process = exec(command, options, function(error, stdout, stderr)
            {
                process = null;
                callback(cancelled || error !== null);
            });
            process.stdout.on('data', _onStdout.bind(this));
            process.stderr.on('data', _onStderr.bind(this));
        };

        /**
         * Tries to restore a backup
         * @param data
         * @param dest_path
         * @param callback
         */
        this.restoreTree = function(data, dest_path, callback)
        {
            var options = {env: {PASSPHRASE: data.passphrase, TMPDIR: os.tmpdir()}};
            var command = 'duplicity restore "' + data.url + '" "' + dest_path + '"' + ' ' + data.cli_options + ' --verbosity info';
            process = exec(command, options, function(error, stdout, stderr)
            {
                process = null;
                callback(cancelled || error !== null);
            });
            process.stdout.on('data', _onStdout.bind(this));
            process.stderr.on('data', _onStderr.bind(this));
        };

        /**
         * Lists the current files in a backup
         * @param data
         * @param callback
         */
        this.getFiles = function(data, callback)
        {
            var options = {env: {PASSPHRASE: data.passphrase, TMPDIR: os.tmpdir()}};
            var command = 'duplicity list-current-files ' + data.url + ' ' + data.cli_options + ' --verbosity info';
            process = exec(command, options, function(error, stdout, stderr)
            {
                var regex = /^[a-zA-Z]{3} [a-zA-Z]{3} [0-9 :]+(.*)$/gm;
                var tree = [];
                var match;
                while (match = regex.exec(stdout))
                {
                    if (match[1] !== 'undefined')
                    {
                        var path = match[1];
                        if (path !== '.' && path !== '..')
                        {
                            tree.push({
                                path: path,
                                dir: path.search('/') !== -1 ? path.substring(0, path.lastIndexOf('/')) : '.',
                                name: path.substring(path.lastIndexOf('/') + 1)
                            });
                        }
                    }
                }
                process = null;
                callback(error !== null, tree);
            });
            process.stdout.on('data', _onStdout.bind(this));
            process.stderr.on('data', _onStderr.bind(this));
        };

        /**
         * Gets the current status of a backup
         * @param data
         * @param callback
         */
        this.getStatus = function(data, callback)
        {
            var options = {env: {PASSPHRASE: data.passphrase, TMPDIR: os.tmpdir()}};
            var command1 = 'duplicity collection-status ' + data.url + ' ' + data.cli_options + ' --verbosity info';
            var command2 = 'duplicity incremental ' + data.path + ' ' + data.url + ' ' + data.cli_options + ' --verbosity info --dry-run';
            process = exec(command1 + ' && ' + command2, options, function(error, stdout, stderr)
            {
                var data = {};
                var chain_start_time = new RegExp('Chain start time: ([^\n]+)', 'gm').exec(stdout);
                var chain_end_time = new RegExp('Chain end time: ([^\n]+)', 'gm').exec(stdout);
                var backup_sets = new RegExp('Number of contained backup sets: ([0-9]+)', 'gm').exec(stdout);
                var source_files = new RegExp('SourceFiles ([0-9]+)', 'g').exec(stdout);
                var source_file_size = new RegExp('SourceFileSize [0-9]+ [(]([^)]+)[)]', 'g').exec(stdout);
                var backup_volumes = new RegExp('Total number of contained volumes: ([0-9]+)', 'gm').exec(stdout);
                data.chain_start_time = chain_start_time !== null && typeof chain_start_time[1] !== 'undefined' ? Date.parse(chain_start_time[1]) : '';
                data.chain_end_time = chain_end_time !== null && typeof chain_end_time[1] !== 'undefined' ? Date.parse(chain_end_time[1]) : '';
                data.backup_sets = backup_sets !== null && typeof backup_sets[1] !== 'undefined' ? backup_sets[1] : '';
                data.backup_volumes = backup_volumes !== null && typeof backup_volumes[1] !== 'undefined' ? backup_volumes[1] : '';
                data.chain_start_time = data.chain_start_time !== '' ? moment(data.chain_start_time).format('YYYY-MM-DD HH:mm') : '';
                data.chain_end_time = data.chain_end_time !== '' ? moment(data.chain_end_time).format('YYYY-MM-DD HH:mm') : '';
                data.source_files = source_files !== null && typeof source_files[1] !== 'undefined' ? source_files[1] : '';<commit_msg>Increases max buffer size when doing CLI tasks<commit_after>/**
 * Duplicity CLI helper
 * Used in the backup model
 */
(function(require, m)
{

    'use strict';

    var exec = require('child_process').exec;
    var os = require('os');
    var moment = require('moment');
    var maxBuffer = 1024 * 1000;

    var module = function()
    {

        var process = null;
        var cancelled = false;
        var outputCallback = null;

        /**
         * Checks if a process is running
         */
        this.isProcessing = function()
        {
            return process !== null;
        };

        /**
         * Registers a callback to be triggered when an output is received from the CLI tool
         * @param callback
         */
        this.onOutput = function(callback)
        {
            outputCallback = callback;
        };

        /**
         * Starts a backup task
         * @param data
         * @param type (full | "")
         * @param callback
         */
        this.doBackup = function(data, type, callback)
        {
            var options = {env: {PASSPHRASE: data.passphrase, TMPDIR: os.tmpdir()}, maxBuffer: maxBuffer};
            var command = 'duplicity ' + type + ' "' + data.path + '" "' + data.url + '" ' + data.cli_options + ' --verbosity info';
            process = exec(command, options, function(error, stdout, stderr)
            {
                process = null;
                callback(cancelled || error !== null);
            });
            process.stdout.on('data', _onStdout.bind(this));
            process.stderr.on('data', _onStderr.bind(this));
        };

        /**
         * Tries to get a file and save it on the given path
         * @param data
         * @param path
         * @param dest_path
         * @param callback
         */
        this.restoreFile = function(data, path, dest_path, callback)
        {
            var options = {env: {PASSPHRASE: data.passphrase, TMPDIR: os.tmpdir()}, maxBuffer: maxBuffer};
            var command = 'duplicity restore --file-to-restore "' + path + '" "' + data.url + '" "' + dest_path + '"' + ' ' + data.cli_options + ' --verbosity info';
            process = exec(command, options, function(error, stdout, stderr)
            {
                process = null;
                callback(cancelled || error !== null);
            });
            process.stdout.on('data', _onStdout.bind(this));
            process.stderr.on('data', _onStderr.bind(this));
        };

        /**
         * Tries to restore a backup
         * @param data
         * @param dest_path
         * @param callback
         */
        this.restoreTree = function(data, dest_path, callback)
        {
            var options = {env: {PASSPHRASE: data.passphrase, TMPDIR: os.tmpdir()}, maxBuffer: maxBuffer};
            var command = 'duplicity restore "' + data.url + '" "' + dest_path + '"' + ' ' + data.cli_options + ' --verbosity info';
            process = exec(command, options, function(error, stdout, stderr)
            {
                process = null;
                callback(cancelled || error !== null);
            });
            process.stdout.on('data', _onStdout.bind(this));
            process.stderr.on('data', _onStderr.bind(this));
        };

        /**
         * Lists the current files in a backup
         * @param data
         * @param callback
         */
        this.getFiles = function(data, callback)
        {
            var options = {env: {PASSPHRASE: data.passphrase, TMPDIR: os.tmpdir()}, maxBuffer: maxBuffer};
            var command = 'duplicity list-current-files ' + data.url + ' ' + data.cli_options + ' --verbosity info';
            process = exec(command, options, function(error, stdout, stderr)
            {
                var regex = /^[a-zA-Z]{3} [a-zA-Z]{3} [0-9 :]+(.*)$/gm;
                var tree = [];
                var match;
                while (match = regex.exec(stdout))
                {
                    if (match[1] !== 'undefined')
                    {
                        var path = match[1];
                        if (path !== '.' && path !== '..')
                        {
                            tree.push({
                                path: path,
                                dir: path.search('/') !== -1 ? path.substring(0, path.lastIndexOf('/')) : '.',
                                name: path.substring(path.lastIndexOf('/') + 1)
                            });
                        }
                    }
                }
                process = null;
                callback(error !== null, tree);
            });
            process.stdout.on('data', _onStdout.bind(this));
            process.stderr.on('data', _onStderr.bind(this));
        };

        /**
         * Gets the current status of a backup
         * @param data
         * @param callback
         */
        this.getStatus = function(data, callback)
        {
            var options = {env: {PASSPHRASE: data.passphrase, TMPDIR: os.tmpdir()}, maxBuffer: maxBuffer};
            var command1 = 'duplicity collection-status ' + data.url + ' ' + data.cli_options + ' --verbosity info';
            var command2 = 'duplicity incremental ' + data.path + ' ' + data.url + ' ' + data.cli_options + ' --verbosity info --dry-run';
            process = exec(command1 + ' && ' + command2, options, function(error, stdout, stderr)
            {
                var data = {};
                var chain_start_time = new RegExp('Chain start time: ([^\n]+)', 'gm').exec(stdout);
                var chain_end_time = new RegExp('Chain end time: ([^\n]+)', 'gm').exec(stdout);
                var backup_sets = new RegExp('Number of contained backup sets: ([0-9]+)', 'gm').exec(stdout);
                var source_files = new RegExp('SourceFiles ([0-9]+)', 'g').exec(stdout);
                var source_file_size = new RegExp('SourceFileSize [0-9]+ [(]([^)]+)[)]', 'g').exec(stdout);
                var backup_volumes = new RegExp('Total number of contained volumes: ([0-9]+)', 'gm').exec(stdout);
                data.chain_start_time = chain_start_time !== null && typeof chain_start_time[1] !== 'undefined' ? Date.parse(chain_start_time[1]) : '';
                data.chain_end_time = chain_end_time !== null && typeof chain_end_time[1] !== 'undefined' ? Date.parse(chain_end_time[1]) : '';
                data.backup_sets = backup_sets !== null && typeof backup_sets[1] !== 'undefined' ? backup_sets[1] : '';
                data.backup_volumes = backup_volumes !== null && typeof backup_volumes[1] !== 'undefined' ? backup_volumes[1] : '';
                data.chain_start_time = data.chain_start_time !== '' ? moment(data.chain_start_time).format('YYYY-MM-DD HH:mm') : '';
                data.chain_end_time = data.chain_end_time !== '' ? moment(data.chain_end_time).format('YYYY-MM-DD HH:mm') : '';
                data.source_files = source_files !== null && typeof source_files[1] !== 'undefined' ? source_files[1] : '';
{'fraction_non_alphanumeric': 0.14633617574794044, 'fraction_numerical': 0.005058534470299176, 'mean_word_length': 1.6861413043478262, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1341774', 'n_tokens_mistral': 3950, 'n_tokens_neox': 3819, 'n_words': 885}