<commit_before>var getopt = require('node-getopt')
    , q = require('q')
    , _ = require('underscore')
    , util = require('util')
    , fs = require('fs')
    , novautils = require('../component-utils')
    , novaform = require('novaform')
    , novastl = require('novastl')
    , AWS = require('aws-sdk')
    , uuid = require('node-uuid')
    , moment = require('moment')
    , stackUtils = require('../stack-utils')
    , s3utils = require('../s3utils')
    , config = require('../configuration')
    , assert = require('assert');

var cmdopts = module.exports.opts = getopt.create([
    ['w', 'wait', 'Wait for completion'],
    ['n', 'noop', 'Do not actually deploy'],
    ['', 'template-output=ARG', 'Dump the generated CloudFormation template to a file'],
    ['h', 'help', 'Display help']
]);

cmdopts.setHelp('[[OPTIONS]]\n');

function Command(config, commonOptions, args, helpCallback) {
    if (!(this instanceof Command)) {
        return new Command(name, properties);
    }

    this.config = config;
    this.displayHelpAndExit = helpCallback;

    var opts = this.opts = cmdopts.parse(args);
    this.commonOptions = commonOptions;
    this.commandOptions = this.opts.options;

    if (opts.options.help) {
        helpCallback();
        return;
    }

    if (_(opts.argv).isEmpty()) {
        helpCallback('Missing project/component reference');
        return;
    } else if (opts.argv.length !== 1) {
        helpCallback('Too many project/component references specified');
        return;
    }

    var ref = opts.argv[0];
    ref = this.ref = Ref.parse(ref);
    if (!this.ref) {
        helpCallback('Invalid project ref');
        return;
    }
    if (!this.ref.component) {
        helpCallback('Component was not specified')
        return;
    }

    this.project = Project.load(this.ref.project, function(e) {
        helpCallback(util.format('Failed to load project "%s": %s', ref.project, e.message));
    });
    if (!this.project) {
        helpCallback(util.format('Could not find project "%s"', this.ref.project));
        return;
    }

    this.component = this.project.findComponent(this.ref.component);
    if (!this.component) {
        helpCallback(util.format('Component "%s" does not exist', this.ref.component));
        return;
    }
}

Command.prototype._waitForStack = function(options, shouldStopCallback) {
    var cfn = options.cfn;
    var stackName = options.stackName;
    var maxWaitSeconds = options.maxWaitSeconds || 15 * 60;
    var waitSeconds = options.waitSeconds || 1;

    var start = moment();
    var maxEnd = moment(start);
    maxEnd.add(maxWaitSeconds, 'seconds');

    var getStackStatus = q.nbind(stackUtils.getStackStatus, stackUtils);

    var f = function() {
        return getStackStatus(cfn, stackName).then(function(status) {
            var callbackResult = shouldStopCallback(status);
            if (callbackResult) {
                return callbackResult;
            }
            if (moment().isAfter(maxEnd)) {
                throw new Error('Timeout');
            }
            return q.delay(waitSeconds * 1000).then(f);
        });
    };

    return f();
};

Command.prototype.execute = function() {
    var that = this;

    return q().then(function() {
        // init deployment

        var stackName = that.ref.makeStackName();

        var deploymentDate = moment.utc();
        var deploymentId = uuid.v4();

        config.currentDeployment.id = deploymentId;
        config.currentDeployment.date = deploymentDate;
        config.currentDeployment.ref = that.ref;

        return {
            projectName: that.ref.project,
            componentName: that.ref.component,
            deploymentDate: deploymentDate,
            deploymentId: deploymentId,
            stackName: stackName,
        };
    }).then(function(deploymentConfig) {
        // TODO: validate project's components, make sure dependencies exist
        var deplist = [];

        function walkDeps(result, componentName, walked) {
            if (!walked) {
                walked = [];
            }
            if (walked.indexOf(componentName) !== -1) {
                throw new Error('recursive dependency');
            }

            walked.push(componentName);

            var component = that.project.findComponent(componentName);
            if (!component) {
                throw new Error(util.format('Could not find dependent component "%s"', componentName));
            }
            var deps = component.dependencies.map(function(depname) {
                var w = walked ? walked.slice() : [];
                return walkDeps([depname], depname, w);
            });

            deps.sort(function(a, b) { return a.length - b.length; });

            deps.forEach(function(deps) {
                deps.reverse();
                deps.forEach(function(d) {
                    var idx = result.indexOf(d);
                    if (idx !== -1) {
                        result.splice(idx, 1);
                    }
                    result.unshift(d);
                });
            });

            return result;
        }

        var deplist = walkDeps([], that.component.name);

        return _.extend(deploymentConfig, {
            dependentComponents: deplist,
        });
    }).then(function(deploymentConfig) {
        // fetch output for each dependent component

        var getStackOutput = q.nbind(stackUtils.getStackOutput, stackUtils);
        if (that.commonOptions.verbose) {
            console.log('Fetching outputs of dependent stacks...');
        }

        var region = that.component.region;
        var cfn = that.cfn = new AWS.CloudFormation({ region : region });

        var componentNames = deploymentConfig.dependentComponents;

        var outputsPromises = componentNames.map(function(depname) {
            return Ref(that.ref.project, depname).makeStackName();
        }).map(function(stackName) {
            return getStackOutput(cfn, stackName);
        });

        return q.all(outputsPromises).then(function(results) {
            var dependencyObject = _.object(_.zip(componentNames, results));
            return _.extend(deploymentConfig, {
                dependencies: dependencyObject
            });
        }).catch(function(e) {
            if (e === stackUtils.Status.DOES_NOT_EXIST) {
                throw new Error('One of the dependent stacks is not yet deployed!');
            }
            throw e;
        });
    }).then(function(deploymentConfig) {
        // build the component
        if (that.commonOptions.verbose) {
            console.log('Building component...');
        }

        function returnResult(result) {
            return _.extend(deploymentConfig, {
                buildResult: result,
            });
        }

        var doneDeferred = q.defer();

        var options = {}; // Currently unused but reserved for the future use.
        var result = that.component.build(deploymentConfig.dependencies, options, doneDeferred.makeNodeResolver());
        if (typeof result === 'undefined') {
            // looks like component wants to use async building, lets wait for done callback to be called.
            return doneDeferred.promise.timeout(30000).then(returnResult);
        } else {
            // async building with promises. Assume build() returned a promise
            return result.then(returnResult);
        }
    }).then(function(deploymentConfig) {
        // build cloudformation resources
        if (that.commonOptions.verbose) {
            console.log('Generating cloudformation template...');
        }

        var stack = novaform.Stack(deploymentConfig.stackName);
        stack.add(deploymentConfig.buildResult.resourceGroups || []);
        stack.add(deploymentConfig.buildResult.outputs || []);

        if (stack.isEmpty()) {
            throw new Error('Nothing to deploy. Lets call it a success!');
        }

        var templateBody = stack.toJson();

        var templateOutput = that.commandOptions['template-output'];
        if (templateOutput === '-') {
            console.log(templateBody);
        } else if (templateOutput) {
            var fd = fs.openSync(templateOutput, 'w');
            fs.writeSync(fd, templateBody);
            fs.closeSync(fd);
        }

        return _.extend(deploymentConfig, {
            templateBody: templateBody,
        });
    }).then(function(deploymentConfig) {
        if (that.commandOptions.noop) {
            return deploymentConfig;
        }

        if (that.commonOptions.verbose) {
            console.log('Uploading cloudformation template to S3...');
        }

        var bucketname = that.config.s3.bucket;
        var region = that.config.s3.region;
        var datestring = deploymentConfig.deploymentDate.format();
        var keypath = util.format('%s%s/%s/%s/templates/%s-%s',
            that.config.s3.keyPrefix,
            that.ref.project, that.ref.component,
            deploymentConfig.deploymentId,
            deploymentConfig.stackName, datestring);

        var params = {
            Bucket: bucketname,
            Key: keypath,
            Body: deploymentConfig.templateBody,
        };
<commit_msg>Append .json file extension to uploaded cfn templates<commit_after>var getopt = require('node-getopt')
    , q = require('q')
    , _ = require('underscore')
    , util = require('util')
    , fs = require('fs')
    , novautils = require('../component-utils')
    , novaform = require('novaform')
    , novastl = require('novastl')
    , AWS = require('aws-sdk')
    , uuid = require('node-uuid')
    , moment = require('moment')
    , stackUtils = require('../stack-utils')
    , s3utils = require('../s3utils')
    , config = require('../configuration')
    , assert = require('assert');

var cmdopts = module.exports.opts = getopt.create([
    ['w', 'wait', 'Wait for completion'],
    ['n', 'noop', 'Do not actually deploy'],
    ['', 'template-output=ARG', 'Dump the generated CloudFormation template to a file'],
    ['h', 'help', 'Display help']
]);

cmdopts.setHelp('[[OPTIONS]]\n');

function Command(config, commonOptions, args, helpCallback) {
    if (!(this instanceof Command)) {
        return new Command(name, properties);
    }

    this.config = config;
    this.displayHelpAndExit = helpCallback;

    var opts = this.opts = cmdopts.parse(args);
    this.commonOptions = commonOptions;
    this.commandOptions = this.opts.options;

    if (opts.options.help) {
        helpCallback();
        return;
    }

    if (_(opts.argv).isEmpty()) {
        helpCallback('Missing project/component reference');
        return;
    } else if (opts.argv.length !== 1) {
        helpCallback('Too many project/component references specified');
        return;
    }

    var ref = opts.argv[0];
    ref = this.ref = Ref.parse(ref);
    if (!this.ref) {
        helpCallback('Invalid project ref');
        return;
    }
    if (!this.ref.component) {
        helpCallback('Component was not specified')
        return;
    }

    this.project = Project.load(this.ref.project, function(e) {
        helpCallback(util.format('Failed to load project "%s": %s', ref.project, e.message));
    });
    if (!this.project) {
        helpCallback(util.format('Could not find project "%s"', this.ref.project));
        return;
    }

    this.component = this.project.findComponent(this.ref.component);
    if (!this.component) {
        helpCallback(util.format('Component "%s" does not exist', this.ref.component));
        return;
    }
}

Command.prototype._waitForStack = function(options, shouldStopCallback) {
    var cfn = options.cfn;
    var stackName = options.stackName;
    var maxWaitSeconds = options.maxWaitSeconds || 15 * 60;
    var waitSeconds = options.waitSeconds || 1;

    var start = moment();
    var maxEnd = moment(start);
    maxEnd.add(maxWaitSeconds, 'seconds');

    var getStackStatus = q.nbind(stackUtils.getStackStatus, stackUtils);

    var f = function() {
        return getStackStatus(cfn, stackName).then(function(status) {
            var callbackResult = shouldStopCallback(status);
            if (callbackResult) {
                return callbackResult;
            }
            if (moment().isAfter(maxEnd)) {
                throw new Error('Timeout');
            }
            return q.delay(waitSeconds * 1000).then(f);
        });
    };

    return f();
};

Command.prototype.execute = function() {
    var that = this;

    return q().then(function() {
        // init deployment

        var stackName = that.ref.makeStackName();

        var deploymentDate = moment.utc();
        var deploymentId = uuid.v4();

        config.currentDeployment.id = deploymentId;
        config.currentDeployment.date = deploymentDate;
        config.currentDeployment.ref = that.ref;

        return {
            projectName: that.ref.project,
            componentName: that.ref.component,
            deploymentDate: deploymentDate,
            deploymentId: deploymentId,
            stackName: stackName,
        };
    }).then(function(deploymentConfig) {
        // TODO: validate project's components, make sure dependencies exist
        var deplist = [];

        function walkDeps(result, componentName, walked) {
            if (!walked) {
                walked = [];
            }
            if (walked.indexOf(componentName) !== -1) {
                throw new Error('recursive dependency');
            }

            walked.push(componentName);

            var component = that.project.findComponent(componentName);
            if (!component) {
                throw new Error(util.format('Could not find dependent component "%s"', componentName));
            }
            var deps = component.dependencies.map(function(depname) {
                var w = walked ? walked.slice() : [];
                return walkDeps([depname], depname, w);
            });

            deps.sort(function(a, b) { return a.length - b.length; });

            deps.forEach(function(deps) {
                deps.reverse();
                deps.forEach(function(d) {
                    var idx = result.indexOf(d);
                    if (idx !== -1) {
                        result.splice(idx, 1);
                    }
                    result.unshift(d);
                });
            });

            return result;
        }

        var deplist = walkDeps([], that.component.name);

        return _.extend(deploymentConfig, {
            dependentComponents: deplist,
        });
    }).then(function(deploymentConfig) {
        // fetch output for each dependent component

        var getStackOutput = q.nbind(stackUtils.getStackOutput, stackUtils);
        if (that.commonOptions.verbose) {
            console.log('Fetching outputs of dependent stacks...');
        }

        var region = that.component.region;
        var cfn = that.cfn = new AWS.CloudFormation({ region : region });

        var componentNames = deploymentConfig.dependentComponents;

        var outputsPromises = componentNames.map(function(depname) {
            return Ref(that.ref.project, depname).makeStackName();
        }).map(function(stackName) {
            return getStackOutput(cfn, stackName);
        });

        return q.all(outputsPromises).then(function(results) {
            var dependencyObject = _.object(_.zip(componentNames, results));
            return _.extend(deploymentConfig, {
                dependencies: dependencyObject
            });
        }).catch(function(e) {
            if (e === stackUtils.Status.DOES_NOT_EXIST) {
                throw new Error('One of the dependent stacks is not yet deployed!');
            }
            throw e;
        });
    }).then(function(deploymentConfig) {
        // build the component
        if (that.commonOptions.verbose) {
            console.log('Building component...');
        }

        function returnResult(result) {
            return _.extend(deploymentConfig, {
                buildResult: result,
            });
        }

        var doneDeferred = q.defer();

        var options = {}; // Currently unused but reserved for the future use.
        var result = that.component.build(deploymentConfig.dependencies, options, doneDeferred.makeNodeResolver());
        if (typeof result === 'undefined') {
            // looks like component wants to use async building, lets wait for done callback to be called.
            return doneDeferred.promise.timeout(30000).then(returnResult);
        } else {
            // async building with promises. Assume build() returned a promise
            return result.then(returnResult);
        }
    }).then(function(deploymentConfig) {
        // build cloudformation resources
        if (that.commonOptions.verbose) {
            console.log('Generating cloudformation template...');
        }

        var stack = novaform.Stack(deploymentConfig.stackName);
        stack.add(deploymentConfig.buildResult.resourceGroups || []);
        stack.add(deploymentConfig.buildResult.outputs || []);

        if (stack.isEmpty()) {
            throw new Error('Nothing to deploy. Lets call it a success!');
        }

        var templateBody = stack.toJson();

        var templateOutput = that.commandOptions['template-output'];
        if (templateOutput === '-') {
            console.log(templateBody);
        } else if (templateOutput) {
            var fd = fs.openSync(templateOutput, 'w');
            fs.writeSync(fd, templateBody);
            fs.closeSync(fd);
        }

        return _.extend(deploymentConfig, {
            templateBody: templateBody,
        });
    }).then(function(deploymentConfig) {
        if (that.commandOptions.noop) {
            return deploymentConfig;
        }

        if (that.commonOptions.verbose) {
            console.log('Uploading cloudformation template to S3...');
        }

        var bucketname = that.config.s3.bucket;
        var region = that.config.s3.region;
        var datestring = deploymentConfig.deploymentDate.format();
        var keypath = util.format('%s%s/%s/%s/templates/%s-%s.json',
            that.config.s3.keyPrefix,
            that.ref.project, that.ref.component,
            deploymentConfig.deploymentId,
            deploymentConfig.stackName, datestring);

        var params = {
            Bucket: bucketname,
            Key: keypath,
            Body: deploymentConfig.templateBody,
        };

{'fraction_non_alphanumeric': 0.12389572380900764, 'fraction_numerical': 0.0028182754322258956, 'mean_word_length': 2.1095382541287497, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5366696', 'n_tokens_mistral': 4983, 'n_tokens_neox': 4823, 'n_words': 1089}