<commit_before>#!/usr/bin/env node

// config is coffee....
require('coffee-script/register');

var fs     = require('fs');
var s3     = require('s3');
var path   = require('path');
var argv   = require('yargs').argv;
var config = require('../config/server');

var gzippableRe = /\.(css|js|svg|gz)(?:$|\?)/;
var versionedRe = /\.(css|js)(?:$|\?)/;

function getS3Params(file, stat, cb) {
    var s3Params = {};

    if (gzippableRe.test(path.extname(file))) {
        s3Params.ContentEncoding = 'gzip';
    }

    if (versionedRe.test(path.extname(file))) {
        s3Params.Expires = new Date((new Date).setYear((new Date).getFullYear() + 1))
    }

    cb(null, s3Params);
}

function getUploadParams(isProduction) {
    var params = {
        localDir: "app/public/",
        deleteRemoved: false, // default false, whether to remove s3 objects 
                              // that have no corresponding local file.
        s3Params: {
            Bucket: config.buckets.ASSETS
            // other options supported by putObject, except Body and ContentLength. 
            // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property 
        },

        getS3Params: getS3Params
    };

    return params;
}

function getClient(creds) {
    var client = s3.createClient({
        maxAsyncS3: 20,     // this is the default
        s3RetryCount: 3,    // this is the default
        s3RetryDelay: 1000, // this is the default
        multipartUploadThreshold: 20971520, // this is the default (20 MB)
        multipartUploadSize: 15728640, // this is the default (15 MB)
        s3Options: {
            accessKeyId: creds.aws.id,
            secretAccessKey: creds.aws.key,
            region : creds.aws.region
            // any other options are passed to new AWS.S3()
            // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property
        },
    });

    return client;
}

function startUploader(client, params, cb) {
    var uploader = client.uploadDir(params);
    uploader.on('error', function(err) {
        console.error("unable to sync:", err.stack);
    });
    uploader.on('progress', function() {
        console.log("progress", uploader.progressAmount, uploader.progressTotal);
    });
    uploader.on('end', function() {
        console.log("done uploading");
    });
}

module.exports = function deploy() {
    var isProduction = !!argv.production;
    var creds = {}, client, uploadParams;

    try {
        creds = require('../credentials.coffee');
    } catch (e) {
        throw new Error('Wat... No credentials.coffee file... No access')
    }

    client = getClient(creds);
    uploadParams = getUploadParams(isProduction);

    startUploader(client, uploadParams);
};
<commit_msg>Add doodle bucket deployment<commit_after>#!/usr/bin/env node

// config is coffee....
require('coffee-script/register');

var fs     = require('fs');
var s3     = require('s3');
var path   = require('path');
var config = require('../config/server');

var gzippableRe = /\.(css|js|svg|gz)(?:$|\?)/;
var versionedRe = /\.(css|js)(?:$|\?)/;

function getCredentials() {
    var creds = {};

    try {
        creds = require('../credentials.coffee');
    } catch (e) {
        throw new Error('Wat... No credentials.coffee file... No access')
    }

    return creds;
}

function getDoodlePath(arg) {
    var path;

    if (!arg) {
        throw new Error('Need to provide a path as argument in format `username/doodleName`');
    }

    try {
        if (fs.lstatSync('doodles/'+arg).isDirectory()) {
            path = 'doodles/'+arg;
        }
    }
    catch (e) {
        throw new Error('Path provided for doodle is wrong / empty, remember just pass `username/doodleName`');
    }

    return path;
}

function getS3ParamsAssets(file, stat, cb) {
    var s3Params = {};

    if (gzippableRe.test(path.extname(file))) {
        s3Params.ContentEncoding = 'gzip';
    }

    if (versionedRe.test(path.extname(file))) {
        s3Params.Expires = new Date((new Date).setYear((new Date).getFullYear() + 1))
    }

    cb(null, s3Params);
}

function getUploadParams(deployingAssets, localDir, bucket) {
    console.log('getUploadParams\n');
    console.log(localDir+'\n');
    var params = {
        localDir: localDir,
        deleteRemoved: false, // default false, whether to remove s3 objects 
                              // that have no corresponding local file.
        s3Params: {
            Bucket: bucket
            // other options supported by putObject, except Body and ContentLength. 
            // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property 
        }
    };

    if (deployingAssets) {
        params.getS3Params = getS3ParamsAssets;
    } else {
        params.s3Params.Prefix = localDir.split('doodles/')[1];
    }

    return params;
}

function getClient(creds) {
    var client = s3.createClient({
        maxAsyncS3: 20,     // this is the default
        s3RetryCount: 3,    // this is the default
        s3RetryDelay: 1000, // this is the default
        multipartUploadThreshold: 20971520, // this is the default (20 MB)
        multipartUploadSize: 15728640, // this is the default (15 MB)
        s3Options: {
            accessKeyId: creds.aws.id,
            secretAccessKey: creds.aws.key,
            region : creds.aws.region
            // any other options are passed to new AWS.S3()
            // See: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property
        },
    });

    return client;
}

function startUploader(client, params, cb) {
    console.log('\n\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n');
    console.log('UPLOADING TO BUCKET '+params.s3Params.Bucket);
    console.log('\n\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\n');

    var uploader = client.uploadDir(params);
    uploader.on('error', function(err) {
        console.error("unable to sync:", err.stack);
    });
    uploader.on('progress', function() {
        console.log("progress", uploader.progressAmount, uploader.progressTotal);
    });
    uploader.on('end', function() {
        console.log("done uploading");
    });
}

function deployAssets() {
    var creds, client, uploadParams;

    creds        = getCredentials();
    client       = getClient(creds);
    uploadParams = getUploadParams(true, "app/public/", config.buckets.ASSETS);

    startUploader(client, uploadParams);
};

function deployDoodle(toLive, path) {
    var doodlePath, creds, client, uploadParams;
    var bucket = toLive ? config.buckets.SOURCE : config.buckets.PENDING;

    doodlePath   = getDoodlePath(path)
    creds        = getCredentials();
    client       = getClient(creds);
    uploadParams = getUploadParams(false, doodlePath, bucket);

    startUploader(client, uploadParams);
}

module.exports = {
    deployAssets        : deployAssets,
    deployDoodlePending : deployDoodle.bind(null, false),
    deployDoodleLive    : deployDoodle.bind(null, true)
};

{'fraction_non_alphanumeric': 0.1570282819755171, 'fraction_numerical': 0.013367102856338822, 'mean_word_length': 2.3417959567465916, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7111641', 'n_tokens_mistral': 2252, 'n_tokens_neox': 2152, 'n_words': 485}