<commit_before>/*jshint esnext:true*/
/*globals module:true, require:true, process:true*/

/**
 * Module dependencies.
 */

// Core.
const util = require('util');
const crypto = require('crypto');
const fs = require('fs');
const path = require('path');
const url = require('url');
const zlib = require('zlib');

// Npm.
const knox = require('knox');
const mime = require('mime');
const deferred = require('underscore.deferred');

// Local
const common = require('./common');

// Avoid warnings.
const existsSync = ('existsSync' in fs) ? fs.existsSync : path.existsSync;

/**
 * Success/error messages.
 */
const MSG_UPLOAD_SUCCESS = '↗'.blue + ' Uploaded: %s (%s)';
const MSG_DOWNLOAD_SUCCESS = '↙'.yellow + ' Downloaded: %s (%s)';
const MSG_DELETE_SUCCESS = '✗'.red + ' Deleted: %s';
const MSG_COPY_SUCCESS = '→'.cyan + ' Copied: %s to %s';

const MSG_UPLOAD_DEBUG = '↗'.blue + ' Upload: ' + '%s'.grey + ' to ' + '%s:%s'.cyan;
const MSG_DOWNLOAD_DEBUG = '↙'.yellow + ' Download: ' + '%s:%s'.cyan + ' to ' + '%s'.grey;
const MSG_DELETE_DEBUG = '✗'.red + ' Delete: ' + '%s:%s'.cyan;
const MSG_COPY_DEBUG = '→'.cyan + ' Copy: ' + '%s'.cyan + ' to ' + '%s:%s'.cyan;

const MSG_ERR_NOT_FOUND = '¯\\_(ツ)_/¯ File not found: %s';
const MSG_ERR_UPLOAD = 'Upload error: %s (%s)';
const MSG_ERR_DOWNLOAD = 'Download error: %s (%s)';
const MSG_ERR_DELETE = 'Delete error: %s (%s)';
const MSG_ERR_COPY = 'Copy error: %s to %s';
const MSG_ERR_CHECKSUM = '%s error: expected hash: %s but found %s for %s';

exports.init = function (grunt) {
  var async = grunt.util.async,
      _ = grunt.util._;

  _.mixin(deferred);

  var exports = {};

  /**
   * Create an Error object based off of a formatted message. Arguments
   * are identical to those of util.format.
   *
   * @param {String} Format.
   * @param {...string|number} Values to insert into Format.
   * @returns {Error}
   */
  var makeError = exports.makeError = function () {
    var msg = util.format.apply(util, _.toArray(arguments));
    return new Error(msg);
  };

  /**
   * Publishes the local file at src to the s3 dest.
   *
   * Verifies that the upload was successful by comparing an md5 checksum of
   * the local and remote versions.
   *
   * @param {String} src The local path to the file to upload.
   * @param {String} dest The s3 path, relative to the bucket, to which the src
   *     is uploaded.
   * @param {Object} [options] An object containing options which override any
   *     option declared in the global s3 config.
   */
  exports.put = exports.upload = function (src, dest, opts) {
    var dfd = new _.Deferred();
    var options = _.clone(opts, true);

    // Make sure the local file exists.
    if (!existsSync(src)) {
      return dfd.reject(makeError(MSG_ERR_NOT_FOUND, src));
    }

    var config = options;
    var headers = options.headers || {};

    if (options.access) {
      headers['x-amz-acl'] = options.access;
    }

    // Pick out the configuration options we need for the client.
    var client = knox.createClient(_(config).pick([
      'endpoint', 'port', 'key', 'secret', 'access', 'bucket', 'secure'
    ]));

    if (config.debug) {
      return dfd.resolve(util.format(MSG_UPLOAD_DEBUG, path.relative(process.cwd(), src), client.bucket, dest)).promise();
    }

    // Encapsulate this logic to make it easier to gzip the file first if
    // necesssary.
    var upload = function (cb) {
      cb = cb || function () {};

      // Upload the file to s3.
      client.putFile(src, dest, headers, function (err, res) {
        // If there was an upload error or any status other than a 200, we
        // can assume something went wrong.
        if (err || res.statusCode !== 200) {
          cb(makeError(MSG_ERR_UPLOAD, src, err || res.statusCode));
        }
        else {
          // Read the local file so we can get its md5 hash.
          fs.readFile(src, function (err, data) {
            if (err) {
              cb(makeError(MSG_ERR_UPLOAD, src, err));
            }
            else {
              // The etag head in the response from s3 has double quotes around
              // it. Strip them out.
              var remoteHash = res.headers.etag.replace(/"/g, '');

              // Get an md5 of the local file so we can verify the upload.
              var localHash = crypto.createHash('md5').update(data).digest('hex');

              if (remoteHash === localHash) {
                var msg = util.format(MSG_UPLOAD_SUCCESS, src, localHash);
                cb(null, msg);
              }
              else {
                cb(makeError(MSG_ERR_CHECKSUM, 'Upload', localHash, remoteHash, src));
              }
            }
          });
        }
      });
    }

    // If gzip is enabled, gzip the file into a temp file and then perform the
    // upload.
    if (options.gzip) {
      headers['Content-Encoding'] = 'gzip';
      headers['Content-Type'] = mime.lookup(src);

      var charset = mime.charsets.lookup(headers['Content-Type'], null);
      if (charset) {
        headers['Content-Type'] += '; charset=' + charset;
      }<commit_msg>added semicolon after function decl<commit_after>/*jshint esnext:true*/
/*globals module:true, require:true, process:true*/

/**
 * Module dependencies.
 */

// Core.
const util = require('util');
const crypto = require('crypto');
const fs = require('fs');
const path = require('path');
const url = require('url');
const zlib = require('zlib');

// Npm.
const knox = require('knox');
const mime = require('mime');
const deferred = require('underscore.deferred');

// Local
const common = require('./common');

// Avoid warnings.
const existsSync = ('existsSync' in fs) ? fs.existsSync : path.existsSync;

/**
 * Success/error messages.
 */
const MSG_UPLOAD_SUCCESS = '↗'.blue + ' Uploaded: %s (%s)';
const MSG_DOWNLOAD_SUCCESS = '↙'.yellow + ' Downloaded: %s (%s)';
const MSG_DELETE_SUCCESS = '✗'.red + ' Deleted: %s';
const MSG_COPY_SUCCESS = '→'.cyan + ' Copied: %s to %s';

const MSG_UPLOAD_DEBUG = '↗'.blue + ' Upload: ' + '%s'.grey + ' to ' + '%s:%s'.cyan;
const MSG_DOWNLOAD_DEBUG = '↙'.yellow + ' Download: ' + '%s:%s'.cyan + ' to ' + '%s'.grey;
const MSG_DELETE_DEBUG = '✗'.red + ' Delete: ' + '%s:%s'.cyan;
const MSG_COPY_DEBUG = '→'.cyan + ' Copy: ' + '%s'.cyan + ' to ' + '%s:%s'.cyan;

const MSG_ERR_NOT_FOUND = '¯\\_(ツ)_/¯ File not found: %s';
const MSG_ERR_UPLOAD = 'Upload error: %s (%s)';
const MSG_ERR_DOWNLOAD = 'Download error: %s (%s)';
const MSG_ERR_DELETE = 'Delete error: %s (%s)';
const MSG_ERR_COPY = 'Copy error: %s to %s';
const MSG_ERR_CHECKSUM = '%s error: expected hash: %s but found %s for %s';

exports.init = function (grunt) {
  var async = grunt.util.async,
      _ = grunt.util._;

  _.mixin(deferred);

  var exports = {};

  /**
   * Create an Error object based off of a formatted message. Arguments
   * are identical to those of util.format.
   *
   * @param {String} Format.
   * @param {...string|number} Values to insert into Format.
   * @returns {Error}
   */
  var makeError = exports.makeError = function () {
    var msg = util.format.apply(util, _.toArray(arguments));
    return new Error(msg);
  };

  /**
   * Publishes the local file at src to the s3 dest.
   *
   * Verifies that the upload was successful by comparing an md5 checksum of
   * the local and remote versions.
   *
   * @param {String} src The local path to the file to upload.
   * @param {String} dest The s3 path, relative to the bucket, to which the src
   *     is uploaded.
   * @param {Object} [options] An object containing options which override any
   *     option declared in the global s3 config.
   */
  exports.put = exports.upload = function (src, dest, opts) {
    var dfd = new _.Deferred();
    var options = _.clone(opts, true);

    // Make sure the local file exists.
    if (!existsSync(src)) {
      return dfd.reject(makeError(MSG_ERR_NOT_FOUND, src));
    }

    var config = options;
    var headers = options.headers || {};

    if (options.access) {
      headers['x-amz-acl'] = options.access;
    }

    // Pick out the configuration options we need for the client.
    var client = knox.createClient(_(config).pick([
      'endpoint', 'port', 'key', 'secret', 'access', 'bucket', 'secure'
    ]));

    if (config.debug) {
      return dfd.resolve(util.format(MSG_UPLOAD_DEBUG, path.relative(process.cwd(), src), client.bucket, dest)).promise();
    }

    // Encapsulate this logic to make it easier to gzip the file first if
    // necesssary.
    var upload = function (cb) {
      cb = cb || function () {};

      // Upload the file to s3.
      client.putFile(src, dest, headers, function (err, res) {
        // If there was an upload error or any status other than a 200, we
        // can assume something went wrong.
        if (err || res.statusCode !== 200) {
          cb(makeError(MSG_ERR_UPLOAD, src, err || res.statusCode));
        }
        else {
          // Read the local file so we can get its md5 hash.
          fs.readFile(src, function (err, data) {
            if (err) {
              cb(makeError(MSG_ERR_UPLOAD, src, err));
            }
            else {
              // The etag head in the response from s3 has double quotes around
              // it. Strip them out.
              var remoteHash = res.headers.etag.replace(/"/g, '');

              // Get an md5 of the local file so we can verify the upload.
              var localHash = crypto.createHash('md5').update(data).digest('hex');

              if (remoteHash === localHash) {
                var msg = util.format(MSG_UPLOAD_SUCCESS, src, localHash);
                cb(null, msg);
              }
              else {
                cb(makeError(MSG_ERR_CHECKSUM, 'Upload', localHash, remoteHash, src));
              }
            }
          });
        }
      });
    };

    // If gzip is enabled, gzip the file into a temp file and then perform the
    // upload.
    if (options.gzip) {
      headers['Content-Encoding'] = 'gzip';
      headers['Content-Type'] = mime.lookup(src);

      var charset = mime.charsets.lookup(headers['Content-Type'], null);
      if (charset) {
        headers['Content-Type'] += '; charset=' + charset;
      }
{'fraction_non_alphanumeric': 0.1481881567318079, 'fraction_numerical': 0.0029460866149464793, 'mean_word_length': 2.5521450994070456, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2797673', 'n_tokens_mistral': 3311, 'n_tokens_neox': 3247, 'n_words': 990}