<commit_before>var _ = require('lodash')
    , Artifact = require('../artifact')
    , AWS = require('aws-sdk')
    , componentUtils = require('../component-utils')
    , fs = require('fs')
    , moment = require('moment')
    , program = require('../program')
    , Project = require('../project')
    , path = require('path')
    , util = require('util')
    , utils = require('../utils');

function Command(opts) {
    if (!(this instanceof Command)) {
        return new Command(opts);
    }

    this.commandOptions = opts.options;

    if (_(opts.argv).isEmpty()) {
        throw new Error('Missing project reference');
    } else if (opts.argv.length !== 1) {
        throw new Error('Too many project references specified');
    }

    this.project = new Project(opts.argv[0]);
    this.artifactName = this.project.name;
}

Command.options = [
    ['', 'artifact=ARG', 'Will upload and use specified artifact'],
    ['', 'name=ARG', 'The name to use for the artifact'],
    ['', 'region=ARG', 'The region the artifact belongs to'],
];
Command.usageText = '[options] project_name';
Command.descriptionText = 'Builds artifacts for the project';

function ensureBucketsExist(regions) {
    var regionsWithoutBuckets = _.filter(regions, region => {
        return program.getRootBucket().region !== region
            && program.getSecondaryBucket(region) === null;
    });

    if (regionsWithoutBuckets.length === 0) {
        return Promise.resolve();
    }

    var question = util.format(
'Artifacts cannot be placed in regions without nova buckets.\n\
You can ether run nova init --secondary and create the buckets\n\
or nova can create default buckets for you.\n\
Should nova create buckets in regions %s and carry on?\
', regionsWithoutBuckets.join(','));

    return utils.yesorno(question).then(yesorno => {
        if (!yesorno) {
            return Promise.reject(new Error('Please init buckets in region(s) ' + regionsWithoutBuckets.join(',') + '. See nova init.'));
        }
        var userId = program.getUserId();
        // Params for aws.createBucket
        var bucketsToCreate = _.map(regionsWithoutBuckets, region => {
            return {
                Bucket: util.format('nova-%s-%s', region, userId),
                ACL: 'private',
                CreateBucketConfiguration: {
                    LocationConstraint: region,
                },
            };
        });
        if (program.options.verbose) {
            console.log('Creating buckets: ' + _.map(bucketsToCreate, 'Bucket').join(','));
        }
        var createBucketPromises = _.map(bucketsToCreate, bucketParams => {
            var region = bucketParams.CreateBucketConfiguration.LocationConstraint;
            var s3 = new AWS.S3({ region: region, signatureVersion: 'v4' });
            var createBucket = utils.pbind(s3.createBucket, s3, bucketParams);
            return createBucket();
        });
        return Promise.all(createBucketPromises).then(values => {
            // Transform into objects accepted by Program.setSecondaryBucket
            return _.map(values, (bucketData, index) => {
                return {
                    region: regionsWithoutBuckets[index],
                    name: bucketsToCreate[index].Bucket,
                    location: bucketData.Location,
                };
            });
        })  ;
    }).then(buckets => {
        return _.map(buckets, bucket => {
            return program.setSecondaryBucket(bucket);
        });
    });
}

Command.prototype.execute = function() {
    var metaInfo = {
        timestamp: moment().utc().format('YYYYMMDDTHHmmss'),
    }

    if (program.options.verbose) {
        console.log('Loading project ' + this.project.name);
    }

    this.project.load(componentUtils.bind({
        projectName: this.project.name,
    }));

    var artifactDef = this.project.findArtifact(this.artifactName);
    if (!artifactDef) {
        throw new Error(`Failed to find artifact '${this.artifactName}' in project '${this.project.name}'`);
    }

    metaInfo.regions = typeof artifactDef.region === 'string'
        ? [artifactDef.region]
        : artifactDef.region;

    if (program.options.verbose) {
        console.log('Ensuring nova buckets in regions ' + metaInfo.regions.join(','));
    }

    return ensureBucketsExist(metaInfo.regions).then(() => {
        if (program.options.verbose) {
            console.log('Building artifact ' + this.artifactName);
        }

        var options = {}; // Currently unused but reserved for future use.
        return utils.pdone(artifactDef.build, artifactDef, options);
    }).then(artifact => {
        if (artifact.type !== Artifact.Type.EBApp) {
            throw new Error('Only EBApp artifact type supported currently');
        }

        if (program.options.verbose) {
            console.log('Uploading artifact to regions ' + metaInfo.regions.join(','));
        }

        var uploadInfo = _.map(metaInfo.regions, region => {
            var bucket = program.getBucket(region);
            var baseName = path.basename(artifact.path);
            var keyPath = path.join(
                'data',
                this.project.name,
                'artifacts',
                this.artifactName,
                metaInfo.timestamp,
                baseName);
            return {
                bucketName: bucket.name,
                bucketRegion: region,
                bucketKeyPath: keyPath,
                artifactReadStream: fs.createReadStream(artifact.path)
            };
        });

        var uploadPromises = _.map(uploadInfo, info => {
            var params = {
                Bucket: info.bucketName,
                Key: info.bucketKeyPath,
                Body: info.artifactReadStream,
            };
            var s3 = new AWS.S3({ region : info.bucketRegion, signatureVersion: 'v4' });

            var s3ManagedUploader = s3.upload(params);
            var lastProgress = 0;
            if (program.options.verbose) {
                s3ManagedUploader.on('httpUploadProgress', function(event) {
                    var thisProgress = event.loaded / event.total * 100;
                    if (thisProgress === 100 || thisProgress > lastProgress + 7) {
                        console.log('Progress (', info.bucketRegion, ')', thisProgress.toFixed(1), '% -', event.loaded, 'of', event.total);
                        lastProgress = thisProgress;
                    }
                });
            }

            var s3ManagedUploaderSend = utils.pbind(s3ManagedUploader.send, s3ManagedUploader);
            return s3ManagedUploaderSend().then(function(data) {
                return {
                    Bucket: info.bucketName,
                    Key: info.bucketKeyPath,
                    region: info.bucketRegion,
                    url: data.Location,
                };
            });
        });

        return Promise.all(uploadPromises);
    }).then(values => {
        if (program.options.verbose) {
            console.log('Updating meta info...');
        }<commit_msg>Factor out upload artifacts to S3<commit_after>var _ = require('lodash')
    , Artifact = require('../artifact')
    , AWS = require('aws-sdk')
    , componentUtils = require('../component-utils')
    , fs = require('fs')
    , moment = require('moment')
    , program = require('../program')
    , Project = require('../project')
    , path = require('path')
    , util = require('util')
    , utils = require('../utils');

function Command(opts) {
    if (!(this instanceof Command)) {
        return new Command(opts);
    }

    this.commandOptions = opts.options;

    if (_(opts.argv).isEmpty()) {
        throw new Error('Missing project reference');
    } else if (opts.argv.length !== 1) {
        throw new Error('Too many project references specified');
    }

    this.project = new Project(opts.argv[0]);
    this.artifactName = this.project.name;
}

Command.options = [
    ['', 'artifact=ARG', 'Will upload and use specified artifact'],
    ['', 'name=ARG', 'The name to use for the artifact'],
    ['', 'region=ARG', 'The region the artifact belongs to'],
];
Command.usageText = '[options] project_name';
Command.descriptionText = 'Builds artifacts for the project';

function ensureBucketsExist(regions) {
    var regionsWithoutBuckets = _.filter(regions, region => {
        return program.getRootBucket().region !== region
            && program.getSecondaryBucket(region) === null;
    });

    if (regionsWithoutBuckets.length === 0) {
        return Promise.resolve();
    }

    var question = util.format(
'Artifacts cannot be placed in regions without nova buckets.\n\
You can ether run nova init --secondary and create the buckets\n\
or nova can create default buckets for you.\n\
Should nova create buckets in regions %s and carry on?\
', regionsWithoutBuckets.join(','));

    return utils.yesorno(question).then(yesorno => {
        if (!yesorno) {
            return Promise.reject(new Error('Please init buckets in region(s) ' + regionsWithoutBuckets.join(',') + '. See nova init.'));
        }
        var userId = program.getUserId();
        // Params for aws.createBucket
        var bucketsToCreate = _.map(regionsWithoutBuckets, region => {
            return {
                Bucket: util.format('nova-%s-%s', region, userId),
                ACL: 'private',
                CreateBucketConfiguration: {
                    LocationConstraint: region,
                },
            };
        });
        if (program.options.verbose) {
            console.log('Creating buckets: ' + _.map(bucketsToCreate, 'Bucket').join(','));
        }
        var createBucketPromises = _.map(bucketsToCreate, bucketParams => {
            var region = bucketParams.CreateBucketConfiguration.LocationConstraint;
            var s3 = new AWS.S3({ region: region, signatureVersion: 'v4' });
            var createBucket = utils.pbind(s3.createBucket, s3, bucketParams);
            return createBucket();
        });
        return Promise.all(createBucketPromises).then(values => {
            // Transform into objects accepted by Program.setSecondaryBucket
            return _.map(values, (bucketData, index) => {
                return {
                    region: regionsWithoutBuckets[index],
                    name: bucketsToCreate[index].Bucket,
                    location: bucketData.Location,
                };
            });
        })  ;
    }).then(buckets => {
        return _.map(buckets, bucket => {
            return program.setSecondaryBucket(bucket);
        });
    });
}

function uploadArtifactToS3(keyPath, artifact, regions) {
    var uploadInfo = _.map(regions, region => {
        var bucket = program.getBucket(region);
        return {
            bucketName: bucket.name,
            bucketRegion: region,
            bucketKeyPath: keyPath,
            artifactReadStream: fs.createReadStream(artifact.path)
        };
    });

    var uploadPromises = _.map(uploadInfo, info => {
        var params = {
            Bucket: info.bucketName,
            Key: info.bucketKeyPath,
            Body: info.artifactReadStream,
        };
        var s3 = new AWS.S3({ region : info.bucketRegion, signatureVersion: 'v4' });

        var s3ManagedUploader = s3.upload(params);
        var lastProgress = 0;
        if (program.options.verbose) {
            s3ManagedUploader.on('httpUploadProgress', function(event) {
                var thisProgress = event.loaded / event.total * 100;
                if (thisProgress === 100 || thisProgress > lastProgress + 7) {
                    console.log('Progress (', info.bucketRegion, ')', thisProgress.toFixed(1), '% -', event.loaded, 'of', event.total);
                    lastProgress = thisProgress;
                }
            });
        }

        var s3ManagedUploaderSend = utils.pbind(s3ManagedUploader.send, s3ManagedUploader);
        return s3ManagedUploaderSend().then(function(data) {
            return {
                Bucket: info.bucketName,
                Key: info.bucketKeyPath,
                region: info.bucketRegion,
                url: data.Location,
            };
        });
    });

    return Promise.all(uploadPromises);
}

Command.prototype.execute = function() {
    var metaInfo = {
        timestamp: moment().utc().format('YYYYMMDDTHHmmss'),
    }

    if (program.options.verbose) {
        console.log('Loading project ' + this.project.name);
    }

    this.project.load(componentUtils.bind({
        projectName: this.project.name,
    }));

    var artifactDef = this.project.findArtifact(this.artifactName);
    if (!artifactDef) {
        throw new Error(`Failed to find artifact '${this.artifactName}' in project '${this.project.name}'`);
    }

    metaInfo.regions = typeof artifactDef.region === 'string'
        ? [artifactDef.region]
        : artifactDef.region;

    if (program.options.verbose) {
        console.log('Ensuring nova buckets in regions ' + metaInfo.regions.join(','));
    }

    return ensureBucketsExist(metaInfo.regions).then(() => {
        if (program.options.verbose) {
            console.log('Building artifact ' + this.artifactName);
        }

        var options = {}; // Currently unused but reserved for future use.
        return utils.pdone(artifactDef.build, artifactDef, options);
    }).then(artifact => {
        if (artifact.type !== Artifact.Type.EBApp) {
            throw new Error('Only EBApp artifact type supported currently');
        }

        if (program.options.verbose) {
            console.log('Uploading artifact to regions ' + metaInfo.regions.join(','));
        }

        var baseName = path.basename(artifact.path);
        var keyPath = path.join(
            'data',
            this.project.name,
            'artifacts',
            this.artifactName,
            metaInfo.timestamp,
            baseName);

        return uploadArtifactToS3(keyPath, artifact, metaInfo.regions);
    }).then(values => {
        if (program.options.verbose) {
            console.log('Updating meta info...');
        }
{'fraction_non_alphanumeric': 0.12712044857690397, 'fraction_numerical': 0.004045709418695436, 'mean_word_length': 2.1241685144124167, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1164554', 'n_tokens_mistral': 3906, 'n_tokens_neox': 3666, 'n_words': 847}