<commit_before>/*
 * Grunt Task File
 * ---------------
 *
 * Task: S3
 * Description: Move files to and from s3
 * Dependencies: knox, async, underscore.deferred
 *
 */

module.exports = function (grunt) {

  /**
   * Module dependencies.
   */

  // Core.
  var util = require('util');
  var crypto = require('crypto');
  var fs = require('fs');
  var path = require('path');
  var url = require('url');

  // Npm.
  var knox = require('knox');
  var async = require('async');

  /**
   * Grunt aliases.
   */
  var log = grunt.log;
  var _ = grunt.utils._;

  _.mixin(require('underscore.deferred'));

  /**
   * Success/error messages.
   */

  const MSG_UPLOAD_SUCCESS = '✓ Uploaded: %s (%s)';
  const MSG_DOWNLOAD_SUCCESS = '✓ Downloaded: %s (%s)';

  const MSG_ERR_NOT_FOUND = 'File not found: %s';
  const MSG_ERR_UPLOAD = 'Upload error: %s';
  const MSG_ERR_DOWNLOAD = 'Download error: %s';
  const MSG_ERR_CHECKSUM = 'Expected remote hash: %s but found %s';
  const MSG_ERR_VERIFY = 'Unable to verify upload: %s => %s';

  /**
   * Create an Error object based off of a formatted message. Arguments
   * are identical to those of util.format.
   */
  function makeError () {
    var msg = util.format.apply(util, _.toArray(arguments));
    return new Error(msg);
  }

  /**
   * Transfer files to/from s3.
   */
  grunt.registerTask('s3', 'Publishes files to s3.', function () {
    var done = this.async();
    var config = _.defaults(grunt.config('s3') || {}, {
      upload: [],
      download: []
    });

    var transfers = [];

    config.upload.forEach(function(upload) {
      // Expand list of files to download.
      var files = grunt.file.expandFiles(upload.src);

      files.forEach(function(file) {
        // If there is only 1 file and it matches the original file wildcard,
        // we know this is a single file transfer. Otherwise, we need to build
        // the destination.
        var dest = (files.length === 1 && file === upload.src)
          ? upload.dest
          : path.join(upload.dest, path.basename(file));

        transfers.push(grunt.helper('s3.put', file, dest, upload));
      });
    });

    config.download.forEach(function(download) {
      transfers.push(grunt.helper('s3.pull', download.src, download.dest, download));
    });

    var total = transfers.length;
    var errors = 0;

    transfers.forEach(function(transfer) {
      transfer.done(function(msg) {
        log.ok(msg);
      });

      transfer.fail(function(msg) {
        log.error(msg);
        ++errors;
      });

      transfer.always(function() {
        if (--total === 0) {
          done(!errors);
        }
      });
    });
  });

  /**
   * Publishes the local file at src to the s3 dest.
   *
   * Verifies that the upload was successful by downloading the file and comparing
   * an md5 checksum of the local and remote versions.
   */
  grunt.registerHelper('s3.put', function (src, dest, options) {
    var dfd = new _.Deferred();
    var config = _.defaults(options, grunt.config('s3') || {});

    var headers = options.headers || {};

    var localHash = crypto.createHash('md5');
    var remoteHash;

    if (options.access) {
      headers['x-amz-acl'] = options.access;
    }

    // Make sure the local file exists.
    if (!path.existsSync(src)) {
      return dfd.reject(makeError(MSG_ERR_NOT_FOUND, src));
    }

    // Get an md5 of the file so we can verify the uploads.
    localHash = localHash.update(grunt.file.read(src)).digest('hex');

    // Create a new s3 client using the current endpoint.
    var client = knox.createClient(config);

    // Upload the file to this endpoint.
    client.putFile(src, dest, headers, function (err, res) {
      // If there was an upload error any status other than a 200, we
      // can assume something went wrong.
      if (err || res.statusCode !== 200) {
        return dfd.reject(makeError(MSG_ERR_UPLOAD, res.statusCode));
      }

      // The etag has double quotes around it. Strip them out.
      remoteHash = res.headers.etag.replace(/"/g, '');

      if (remoteHash === localHash) {
        var msg = util.format(MSG_UPLOAD_SUCCESS, src, localHash);
        dfd.resolve(msg);
      }
      else {
        dfd.reject(makeError(MSG_ERR_CHECKSUM, localHash, remoteHash));
      }
    });

    return dfd;
  });

  /**
   * Similar to s3.put but, you know, the other way.
   */
  grunt.registerHelper('s3.pull', function (src, dest, options) {
    var dfd = new _.Deferred();

    var config = _.defaults(options, grunt.config('s3') || {});

    // Create a local stream we can write the downloaded file to.
    var file = fs.createWriteStream(dest);

    // We'll want to verify the download once it's finished.
    var localHash = crypto.createHash('md5');

    // Create a new s3 client using the current endpoint.
    var client = knox.createClient(config);
<commit_msg>Cleanup.<commit_after>/*
 * Grunt Task File
 * ---------------
 *
 * Task: S3
 * Description: Move files to and from s3
 * Dependencies: knox, async, underscore.deferred
 *
 */

module.exports = function (grunt) {

  /**
   * Module dependencies.
   */

  // Core.
  var util = require('util');
  var crypto = require('crypto');
  var fs = require('fs');
  var path = require('path');
  var url = require('url');

  // Npm.
  var knox = require('knox');
  var async = require('async');
  var deferred = require('underscore.deferred');

  /**
   * Grunt aliases.
   */
  var log = grunt.log;
  var _ = grunt.utils._;

  _.mixin(deferred);

  /**
   * Success/error messages.
   */

  const MSG_UPLOAD_SUCCESS = '✓ Uploaded: %s (%s)';
  const MSG_DOWNLOAD_SUCCESS = '✓ Downloaded: %s (%s)';

  const MSG_ERR_NOT_FOUND = 'File not found: %s';
  const MSG_ERR_UPLOAD = 'Upload error: %s';
  const MSG_ERR_DOWNLOAD = 'Download error: %s';
  const MSG_ERR_CHECKSUM = 'Expected hash: %s but found %s';
  const MSG_ERR_VERIFY = 'Unable to verify upload: %s => %s';

  /**
   * Create an Error object based off of a formatted message. Arguments
   * are identical to those of util.format.
   *
   * @param {String} Format.
   * @param {...string|number} Values to insert into Format.
   */
  function makeError () {
    var msg = util.format.apply(util, _.toArray(arguments));
    return new Error(msg);
  }

  /**
   * Transfer files to/from s3.
   *
   * Uses global s3 grunt config.
   */
  grunt.registerTask('s3', 'Publishes files to s3.', function () {
    var done = this.async();
    var config = _.defaults(grunt.config('s3') || {}, {
      upload: [],
      download: []
    });

    var transfers = [];

    config.upload.forEach(function(upload) {
      // Expand list of files to download.
      var files = grunt.file.expandFiles(upload.src);

      files.forEach(function(file) {
        // If there is only 1 file and it matches the original file wildcard,
        // we know this is a single file transfer. Otherwise, we need to build
        // the destination.
        var dest = (files.length === 1 && file === upload.src)
          ? upload.dest
          : path.join(upload.dest, path.basename(file));

        transfers.push(grunt.helper('s3.put', file, dest, upload));
      });
    });

    config.download.forEach(function(download) {
      transfers.push(grunt.helper('s3.pull', download.src, download.dest, download));
    });

    var total = transfers.length;
    var errors = 0;

    transfers.forEach(function(transfer) {
      transfer.done(function(msg) {
        log.ok(msg);
      });

      transfer.fail(function(msg) {
        log.error(msg);
        ++errors;
      });

      transfer.always(function() {
        if (--total === 0) {
          done(!errors);
        }
      });
    });
  });

  /**
   * Publishes the local file at src to the s3 dest.
   *
   * Verifies that the upload was successful by comparing an md5 checksum of
   * the local and remote versions.
   *
   * @param {String} src The local path to the file to upload.
   * @param {String} dest The s3 path, relative to the bucket, to which the src is
   *     uploaded.
   * @param {Object} [options] An object containing options which override any option
   *     declared in the global s3 config.
   */
  grunt.registerHelper('s3.put', function (src, dest, options) {
    var dfd = new _.Deferred();
    var config = _.defaults(options, grunt.config('s3') || {});

    var headers = options.headers || {};

    var localHash = crypto.createHash('md5');
    var remoteHash;

    if (options.access) {
      headers['x-amz-acl'] = options.access;
    }

    // Make sure the local file exists.
    if (!path.existsSync(src)) {
      return dfd.reject(makeError(MSG_ERR_NOT_FOUND, src));
    }

    // Get an md5 of the file so we can verify the uploads.
    localHash = localHash.update(grunt.file.read(src)).digest('hex');

    // Create a new s3 client using the current endpoint.
    var client = knox.createClient(config);

    // Upload the file to this endpoint.
    client.putFile(src, dest, headers, function (err, res) {
      // If there was an upload error any status other than a 200, we
      // can assume something went wrong.
      if (err || res.statusCode !== 200) {
        return dfd.reject(makeError(MSG_ERR_UPLOAD, res.statusCode));
      }

      // The etag has double quotes around it. Strip them out.
      remoteHash = res.headers.etag.replace(/"/g, '');

      if (remoteHash === localHash) {
        var msg = util.format(MSG_UPLOAD_SUCCESS, src, localHash);
        dfd.resolve(msg);
      }
      else {
        dfd.reject(makeError(MSG_ERR_CHECKSUM, localHash, remoteHash));
      }
    });

    return dfd;
  });

  /**
   * Download a file from s3..
   *
   * Verifies that the download was successful by downloading the file and
   * comparing an md5 checksum of the local and remote versions.
   *
   * @param {String} src The s3 path, relative to the bucket, of the file being
   *     downloaded.
   * @param {String} dest The local path where the download will be saved.
   * @param {Object} [options] An object containing options which override any option
   *     declared in the global s3 config.
   */
  grunt.registerHelper('s3.pull', function (src, dest, options) {
    var dfd = new _.Deferred();

    var config = _.defaults(options, grunt.config('s3') || {});

    // Create a local stream we can write the downloaded file to.
    var file = fs.createWriteStream(dest);

    // We'll want to verify the download once it's finished.
    var localHash = crypto.createHash('md5');

    // Create a new s3 client using the current endpoint.
    var client = knox.createClient(config);

{'fraction_non_alphanumeric': 0.1371606334841629, 'fraction_numerical': 0.006221719457013574, 'mean_word_length': 2.545788770053476, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5224911', 'n_tokens_mistral': 3386, 'n_tokens_neox': 3285, 'n_words': 992}