<commit_before>/**
 * Module dependencies
 */

var path = require('path');
var Writable = require('stream').Writable;
var Transform = require('stream').Transform;
var concat = require('concat-stream');
var _ = require('lodash');
_.defaultsDeep = require('merge-defaults');
var knox = require('knox');
var S3MultipartUpload = require('knox-mpu');
var S3Lister = require('s3-lister');


/**
 * skipper-s3
 *
 * @param  {Object} globalOpts
 * @return {Object}
 */

module.exports = function SkipperS3 (globalOpts) {
  globalOpts = globalOpts || {};


  _.defaults(globalOpts, {

      // By default, create new files on disk
      // using their uploaded filenames.
      // (no overwrite-checking is performed!!)
      saveAs: function (__newFile) {
        return __newFile.filename;
      },

      // Max bytes (defaults to ~15MB)
      maxBytes: 15000000,

      // The bucket we're going to upload stuff into
      // bucket: '',

      // Our S3 API key
      // key: '',

      // Our S3 API secret
      // secret: '',

      // By default, upload files to `/` (within the bucket)
      dirname: '/'
    });

  var adapter = {

    read: function (filename, cb) {

      // Determine location where file should be written:
      var dirPath = globalOpts.dirname;
      var filePath = path.join(dirPath, filename);

      var client = knox.createClient({
        key: globalOpts.key,
        secret: globalOpts.secret,
        bucket: globalOpts.bucket
      });

      // Build a noop transform stream that will pump the S3 output through
      var __transform__ = new Transform();
      __transform__._transform = function (chunk, encoding, callback) {
        return callback(null, chunk);
      };

      client.get(filePath).on('response', function(s3res){
        // Handle explicit s3res errors
        s3res.once('error', function (err) {
          __transform__.emit('error', err);
        });

        // check whether we got an actual file stream:
        if (s3res.statusCode < 300) {
          s3res.pipe(__transform__);
        }
        // or an error:
        else {
          // Wait for the body of the error message to stream in:
          var body = '';
          s3res.setEncoding('utf8');
          s3res.on('readable', function (){
            var chunk = s3res.read();
            if (typeof chunk === 'string') body += chunk;
          });
          // Then build the error and emit it
          s3res.once('end', function () {
            var err = new Error();
            err.status = s3res.statusCode;
            err.headers = s3res.headers;
            err.message = 'Non-200 status code returned from S3 for requested file.';
            if (body) err.message += ('\n'+body);
            __transform__.emit('error', err);
          });
        }
      })
      .end();

      if (cb) {
        var firedCb = false;
        __transform.once('error', function (err) {
          if (firedCb) return;
          firedCb = true;
          cb(err);
        });
        __transform__.pipe(concat(function (data) {
          if (firedCb) return;
          firedCb = true;
          cb(null, data);
        }));
      }

      return __transform__;
    },

    rm: function (filepath, cb){
      return cb(new Error('TODO'));
    },
    ls: function (dirpath, cb) {
      var client = knox.createClient({
        key: globalOpts.key,
        secret: globalOpts.secret,
        bucket: globalOpts.bucket
      });

      // TODO: take a look at maxKeys
      // https://www.npmjs.org/package/s3-lister

      // Allow empty dirpath
      if (!dirpath) {
        dirpath='/';
      }
      // Strip leading slash from dirpath to form prefix
      var prefix = dirpath.replace(/^\//, '');

      var lister = new S3Lister(client, {
        prefix : prefix
      });

      if (!cb) {
        return lister;
      }
      else {
        var firedCb;
        lister.once('error', function (err) {
          if(firedCb)return;
          firedCb=true;
          cb(err);
        });
        lister.pipe(concat(function (data) {
          console.log('______ DATA _______\n', data);
          if(firedCb)return;
          firedCb=true;
          cb(null, data);
        }));
        return lister;
      }
    },

    receiver: S3Receiver,
    receive: S3Receiver
  };

  return adapter;

  /**
   * A simple receiver for Skipper that writes Upstreams to
   * S3 to the configured bucket at the configured path.
   *
   * Includes a garbage-collection mechanism for failed
   * uploads.
   *
   * @param  {Object} options
   * @return {Stream.Writable}
   */<commit_msg>Makes most of the adapter tests pass.<commit_after>/**
 * Module dependencies
 */

var path = require('path');
var Writable = require('stream').Writable;
var Transform = require('stream').Transform;
var concat = require('concat-stream');
var _ = require('lodash');
_.defaultsDeep = require('merge-defaults');
var knox = require('knox');
var S3MultipartUpload = require('knox-mpu');
var S3Lister = require('s3-lister');


/**
 * skipper-s3
 *
 * @param  {Object} globalOpts
 * @return {Object}
 */

module.exports = function SkipperS3 (globalOpts) {
  globalOpts = globalOpts || {};


  _.defaults(globalOpts, {

      // By default, create new files on disk
      // using their uploaded filenames.
      // (no overwrite-checking is performed!!)
      saveAs: function (__newFile) {
        return __newFile.filename;
      },

      // Max bytes (defaults to ~15MB)
      maxBytes: 15000000,

      // The bucket we're going to upload stuff into
      // bucket: '',

      // Our S3 API key
      // key: '',

      // Our S3 API secret
      // secret: '',

      // By default, upload files to `/` (within the bucket)
      dirname: '/'
    });

  var adapter = {

    read: function (filepath, cb) {

      // DONT trim leading slash to form prefix!!
      var prefix = filepath;
      // var prefix = dirpath.replace(/^\//, '');
      // console.log('Trying to look up:', prefix);


      var client = knox.createClient({
        key: globalOpts.key,
        secret: globalOpts.secret,
        bucket: globalOpts.bucket
      });

      // Build a noop transform stream that will pump the S3 output through
      var __transform__ = new Transform();
      __transform__._transform = function (chunk, encoding, callback) {
        return callback(null, chunk);
      };

      client.get(prefix).on('response', function(s3res){
        // Handle explicit s3res errors
        s3res.once('error', function (err) {
          __transform__.emit('error', err);
        });

        // check whether we got an actual file stream:
        if (s3res.statusCode < 300) {
          s3res.pipe(__transform__);
        }
        // or an error:
        else {
          // Wait for the body of the error message to stream in:
          var body = '';
          s3res.setEncoding('utf8');
          s3res.on('readable', function (){
            var chunk = s3res.read();
            if (typeof chunk === 'string') body += chunk;
          });
          // Then build the error and emit it
          s3res.once('end', function () {
            var err = new Error();
            err.status = s3res.statusCode;
            err.headers = s3res.headers;
            err.message = 'Non-200 status code returned from S3 for requested file.';
            if (body) err.message += ('\n'+body);
            __transform__.emit('error', err);
          });
        }
      })
      .end();

      if (cb) {
        var firedCb;
        __transform__.once('error', function (err) {
          if (firedCb) return;
          firedCb = true;
          cb(err);
        });
        __transform__.pipe(concat(function (data) {
          if (firedCb) return;
          firedCb = true;
          cb(null, data);
        }));
      }

      return __transform__;
    },

    rm: function (filepath, cb){
      return cb(new Error('TODO'));
    },
    ls: function (dirpath, cb) {
      var client = knox.createClient({
        key: globalOpts.key,
        secret: globalOpts.secret,
        bucket: globalOpts.bucket
      });

      // TODO: take a look at maxKeys
      // https://www.npmjs.org/package/s3-lister

      // Allow empty dirpath (defaults to `/`)
      if (!dirpath) {
        dirpath='/';
      }
      // Strip leading slash from dirpath to form prefix
      var prefix = dirpath.replace(/^\//, '');

      var lister = new S3Lister(client, {
        prefix : prefix
      });

      if (!cb) {
        return lister;
      }
      else {
        var firedCb;
        lister.once('error', function (err) {
          if(firedCb)return;
          firedCb=true;
          cb(err);
        });
        lister.pipe(concat(function (data) {
          if(firedCb)return;
          firedCb=true;

          // Pluck just the "Key" (i.e. file path)
          // and return only the filename (i.e. snip
          // off the path prefix)
          data = _.pluck(data, 'Key');
          data = _.map(data, function snipPathPrefixes (thisPath) {
            return thisPath.replace(/^.*[\/]([^\/]*)$/, '$1');
          });

          console.log('______ files _______\n', data);
          cb(null, data);
        }));

        // TODO: marshal each matched file in the stream
        // (using a Transform- take a look at all the
        //  "plucking" and stuff I have going on above ^)
        return lister;
      }
    },

    receiver: S3Receiver,
    receive: S3Receiver
  };

  return adapter;

  /**
   * A simple receiver for Skipper that writes Upstreams to
   * S3 to the configured bucket at the configured path.
   *
   * Includes a garbage-collection mechanism for failed
   * uploads.
   *
   * @param  {Object} options
   * @return {Stream.Writable}
   */
{'fraction_non_alphanumeric': 0.12967145790554416, 'fraction_numerical': 0.008726899383983572, 'mean_word_length': 1.8734513274336284, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6968883', 'n_tokens_mistral': 2928, 'n_tokens_neox': 2830, 'n_words': 807}