<commit_before>/*globals define */
/*jshint node:true, browser:true, esversion: 6*/

define([
    'plugin/CreateExecution/CreateExecution/CreateExecution',
    'deepforge/plugin/PtrCodeGen',
    'common/storage/constants',
    'common/core/constants',
    'q',
    'text!./metadata.json',
    './templates/index',
    'deepforge/plugin/LocalExecutor',
    'executor/ExecutorClient',
    'underscore'
], function (
    CreateExecution,
    PtrCodeGen,
    STORAGE_CONSTANTS,
    CONSTANTS,
    Q,
    pluginMetadata,
    Templates,
    LocalExecutor,  // DeepForge operation primitives
    ExecutorClient,
    _
) {
    'use strict';

    pluginMetadata = JSON.parse(pluginMetadata);
    /**
     * Initializes a new instance of ExecutePipeline.
     * @class
     * @augments {CreateExecution}
     * @classdesc This class represents the plugin ExecutePipeline.
     * @constructor
     */
    var OUTPUT_INTERVAL = 1500,
        STDOUT_FILE = 'job_stdout.txt';
    var ExecutePipeline = function () {
        // Call base class' constructor.
        CreateExecution.call(this);
        this.pluginMetadata = pluginMetadata;

        this._currentSave = Q();
        this.initRun();
    };

    /**
     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
     * This is also available at the instance at this.pluginMetadata.
     * @type {object}
     */
    ExecutePipeline.metadata = pluginMetadata;
    ExecutePipeline.UPDATE_INTERVAL = 1500;

    // Prototypical inheritance from CreateExecution.
    ExecutePipeline.prototype = Object.create(CreateExecution.prototype);
    ExecutePipeline.prototype.constructor = ExecutePipeline;

    ExecutePipeline.prototype.initRun = function () {
        // Cache
        this.nodes = {};

        // Record keeping for running operations
        this.opFor = {};
        this.incomingCounts = {};
        this.outputsOf = {};
        this.inputPortsFor = {};
        this.inputs = {};

        this.finished = {};
        this.completedCount = 0;
        this.totalCount = 0;
        this.outputLineCount = {};

        // When a pipeline fails, it will let all running jobs finish and record
        // the results of each job
        //
        // The following variables are used to...
        //   - keep track of the number of jobs currently running
        //   - keep track if the pipeline has errored
        //     - if so, don't start any more jobs
        this.pipelineError = null;
        this.runningJobs = 0;
    };

    /**
     * Main function for the plugin to execute. This will perform the execution.
     * Notes:
     * - Always log with the provided logger.[error,warning,info,debug].
     * - Do NOT put any user interaction logic UI, etc. inside this method.
     * - callback always has to be called even if error happened.
     *
     * @param {function(string, plugin.PluginResult)} callback - the result callback
     */
    ExecutePipeline.prototype.main = function (callback) {
        // This will probably need to execute the operations, too, because the
        // inputs for the next operation cannot be created until the inputs have
        // been generated

        this.initRun();
        var startPromise;
        if (this.core.isTypeOf(this.activeNode, this.META.Pipeline)) {
            // If starting with a pipeline, we will create an Execution first
            startPromise = this.createExecution(this.activeNode)
                .then(execNode => {
                    this.activeNode = execNode;
                    return this.core.loadSubTree(this.activeNode);
                });
        } else if (this.core.isTypeOf(this.activeNode, this.META.Execution)) {
            startPromise = this.core.loadSubTree(this.activeNode);
        } else {
            return callback('Current node is not a Pipeline or Execution!', this.result);
        }

        // Set debug and the final callback
        this.debug = true;  // this.getCurrentConfig().debug;
        this._callback = callback;

        startPromise.then(subtree => {
            var children = subtree
                .filter(n => this.core.getParent(n) === this.activeNode);

            this.pipelineName = this.core.getAttribute(this.activeNode, 'name');
            this.buildCache(subtree);
            this.parsePipeline(children);  // record deps, etc

            return this.clearResults();
        })
        .then(() => this.executePipeline())
        .fail(e => this.logger.error(e));
    };

    // Override 'save' to prevent race conditions while saving
    ExecutePipeline.prototype.save = function (msg) {
        // When 'save'  is called, it should still finish any current save op
        // before continuing
        this._currentSave = this._currentSave
            .then(() => CreateExecution.prototype.save.call(this, msg))
            .then(result => {
                var msg;
                if (result.status === STORAGE_CONSTANTS.FORKED) {
                    msg = `"${this.pipelineName}" execution has forked to "${result.forkName}"`;
                    this.sendNotification(msg);
                }
            });

        return this._currentSave;
    };

    ExecutePipeline.prototype.isInputData = function (node) {
        var prnt = this.core.getParent(node);
        return this.core.isTypeOf(prnt, this.META.Inputs);
    };

    ExecutePipeline.prototype.clearResults = function () {
        var nodes = Object.keys(this.nodes).map(id => this.nodes[id]);
        // Clear the pipeline's results<commit_msg>Created better execution fork names. Fixes #494 (#496)<commit_after>/*globals define */
/*jshint node:true, browser:true, esversion: 6*/

define([
    'plugin/CreateExecution/CreateExecution/CreateExecution',
    'deepforge/plugin/PtrCodeGen',
    'common/storage/constants',
    'common/core/constants',
    'q',
    'text!./metadata.json',
    './templates/index',
    'deepforge/plugin/LocalExecutor',
    'executor/ExecutorClient',
    'underscore'
], function (
    CreateExecution,
    PtrCodeGen,
    STORAGE_CONSTANTS,
    CONSTANTS,
    Q,
    pluginMetadata,
    Templates,
    LocalExecutor,  // DeepForge operation primitives
    ExecutorClient,
    _
) {
    'use strict';

    pluginMetadata = JSON.parse(pluginMetadata);
    /**
     * Initializes a new instance of ExecutePipeline.
     * @class
     * @augments {CreateExecution}
     * @classdesc This class represents the plugin ExecutePipeline.
     * @constructor
     */
    var OUTPUT_INTERVAL = 1500,
        STDOUT_FILE = 'job_stdout.txt';
    var ExecutePipeline = function () {
        // Call base class' constructor.
        CreateExecution.call(this);
        this.pluginMetadata = pluginMetadata;

        this._currentSave = Q();
        this.initRun();
    };

    /**
     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
     * This is also available at the instance at this.pluginMetadata.
     * @type {object}
     */
    ExecutePipeline.metadata = pluginMetadata;
    ExecutePipeline.UPDATE_INTERVAL = 1500;

    // Prototypical inheritance from CreateExecution.
    ExecutePipeline.prototype = Object.create(CreateExecution.prototype);
    ExecutePipeline.prototype.constructor = ExecutePipeline;

    ExecutePipeline.prototype.initRun = function () {
        // Cache
        this.nodes = {};

        // Record keeping for running operations
        this.opFor = {};
        this.incomingCounts = {};
        this.outputsOf = {};
        this.inputPortsFor = {};
        this.inputs = {};

        this.finished = {};
        this.completedCount = 0;
        this.totalCount = 0;
        this.outputLineCount = {};

        // When a pipeline fails, it will let all running jobs finish and record
        // the results of each job
        //
        // The following variables are used to...
        //   - keep track of the number of jobs currently running
        //   - keep track if the pipeline has errored
        //     - if so, don't start any more jobs
        this.pipelineError = null;
        this.runningJobs = 0;
    };

    /**
     * Main function for the plugin to execute. This will perform the execution.
     * Notes:
     * - Always log with the provided logger.[error,warning,info,debug].
     * - Do NOT put any user interaction logic UI, etc. inside this method.
     * - callback always has to be called even if error happened.
     *
     * @param {function(string, plugin.PluginResult)} callback - the result callback
     */
    ExecutePipeline.prototype.main = function (callback) {
        // This will probably need to execute the operations, too, because the
        // inputs for the next operation cannot be created until the inputs have
        // been generated

        this.initRun();
        var startPromise;
        if (this.core.isTypeOf(this.activeNode, this.META.Pipeline)) {
            // If starting with a pipeline, we will create an Execution first
            startPromise = this.createExecution(this.activeNode)
                .then(execNode => {
                    this.activeNode = execNode;
                });
        } else if (this.core.isTypeOf(this.activeNode, this.META.Execution)) {
            startPromise = Q();
        } else {
            return callback('Current node is not a Pipeline or Execution!', this.result);
        }

        // Set debug and the final callback
        this.debug = true;  // this.getCurrentConfig().debug;
        this._callback = callback;

        startPromise
        .then(() => this.core.loadSubTree(this.activeNode))
        .then(subtree => {
            var children = subtree
                .filter(n => this.core.getParent(n) === this.activeNode);

            this.pipelineName = this.core.getAttribute(this.activeNode, 'name');
            this.buildCache(subtree);
            this.parsePipeline(children);  // record deps, etc

            return this.clearResults();
        })
        .then(() => this.executePipeline())
        .fail(e => this.logger.error(e));
    };

    ExecutePipeline.prototype.updateForkName = function () {
        var basename = this.pipelineName + '_fork';
        return this.project.getBranches().then(branches => {
            var names = Object.keys(branches),
                name = basename,
                i = 2;

            while (names.indexOf(name) !== -1) {
                name = basename + '_' + i;
                i++;
            }

            this.forkName = name;
        });
    };

    // Override 'save' to prevent race conditions while saving
    ExecutePipeline.prototype.save = function (msg) {
        // When 'save'  is called, it should still finish any current save op
        // before continuing
        this._currentSave = this._currentSave
            .then(() => this.updateForkName())
            .then(() => CreateExecution.prototype.save.call(this, msg))
            .then(result => {
                var msg;
                if (result.status === STORAGE_CONSTANTS.FORKED) {
                    msg = `"${this.pipelineName}" execution has forked to "${result.forkName}"`;
                    this.sendNotification(msg);
                }
            });

        return this._currentSave;
    };

    ExecutePipeline.prototype.isInputData = function (node) {
        var prnt = this.core.getParent(node);
        return this.core.isTypeOf(prnt, this.META.Inputs);
    };

    ExecutePipeline.prototype.clearResults = function () {
        var nodes = Object.keys(this.nodes).map(id => this.nodes[id]);
        // Clear the pipeline's results
{'fraction_non_alphanumeric': 0.1089032258064516, 'fraction_numerical': 0.0027526881720430106, 'mean_word_length': 2.1177259318852237, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2265808', 'n_tokens_mistral': 3165, 'n_tokens_neox': 3041, 'n_words': 817}