<commit_before>
'use strict';

//
// Module Dependencies
//

var url   = require('url');
var async = require('async');
var _     = require('underscore');

//
// IOStreams provides a grouping and generic linking functions
// for input and output stream connectors/generators
//
var IOStreams = function () {

  this.providers = [];

  return this;
};

//
// A method to add providers to IOStreams
// @param  {Object} provider Should have to callable functions getInputStream(config) and getOutputStream(config)
// @return {Number}          Number of providers now available
//
IOStreams.prototype.use = function(provider) {
  return this.providers.push(provider);
};

//
// Geds a provider that supports a certain protocol
// @param  {String} protocol e.g 'file:', 'http:' or 's3:'
// @return {Object}          A provider object
//
IOStreams.prototype.getProviderByProtocol = function(protocol) {

  var provider = _.find(this.providers, function(provider) {
    return provider.protocol === protocol;
  });

  if(!provider) {
    throw new Error('No provider found supporting the protocol `' + protocol + '`');
  }

  return provider;

};

//
// Parse config into a consistent structure
// @param  {String | Object} config
// @return {Object}
//
IOStreams.prototype.parseConfig = function(config) {
  if(typeof config === 'string') {
    config = url.parse(config);
  }
  return config;
};

//
// Opens a stream from wherever the destined input is
// @param  {String | Object}   config
// @param  {Function} callback (err, inputStream)
// @return {Void}
//
IOStreams.prototype.getInputStream = function(config, callback) {

  config = this.parseConfig(config);

  var provider;

  try {
    provider = this.getProviderByProtocol(config.protocol);
  } catch (err) {
    return callback(err);
  }

  return provider.getInputStream(config, callback);

};

//
// Opens a stream to wherever the destined output is
// @param  {String | Object}   config
// @param  {Function} callback (err, outputStream)
// @return {Void}
//
IOStreams.prototype.getOutputStream = function(config, callback) {

  config = this.parseConfig(config);

  var provider;

  try {
    provider = this.getProviderByProtocol(config.protocol);
  } catch (err) {
    return callback(err);
  }

  return provider.getOutputStream(config, callback);

};

//
// Lazy function to get Input and Output Streams
// @param  {String | Object}   inputConfig
// @param  {String | Object}   outputConfig [description]
// @param  {Function} callback     (err, inputStream, outputStream)
// @return {Void}
//
IOStreams.prototype.getInputOutputStreams = function(inputConfig, outputConfig, callback) {

  var self = this;

  async.parallel([
    function(callback) {
      self.getInputStream(inputConfig, callback);
    },
    function(callback) {
      self.getOutputStream(outputConfig, callback);
    }
  ], function(err, results) {
    callback(err, results[0], results[1]);
  });

};

//
// Create an instance of IOStreams
// @return {IOStreams}
//
var createInstance = function() {
  return new IOStreams();
};

//
// Exports
//

module.exports           = createInstance;
module.exports.IOStreams = IOStreams;<commit_msg>Remove url parsing config<commit_after>
'use strict';

//
// Module Dependencies
//

var url   = require('url');
var async = require('async');
var _     = require('underscore');

//
// IOStreams provides a grouping and generic linking functions
// for input and output stream connectors/generators
//
var IOStreams = function () {

  this.providers = [];

  return this;
};

//
// A method to add providers to IOStreams
// @param  {Object} provider Should have to callable functions getInputStream(config) and getOutputStream(config)
// @return {Number}          Number of providers now available
//
IOStreams.prototype.use = function(provider) {
  return this.providers.push(provider);
};

//
// Geds a provider that supports a certain protocol
// @param  {String} protocol e.g 'file:', 'http:' or 's3:'
// @return {Object}          A provider object
//
IOStreams.prototype.getProviderByProtocol = function(protocol) {

  var provider = _.find(this.providers, function(provider) {
    return provider.protocol === protocol;
  });

  if(!provider) {
    throw new Error('No provider found supporting the protocol `' + protocol + '`');
  }

  return provider;

};


//
// Opens a stream from wherever the destined input is
// @param  {String | Object}   config
// @param  {Function} callback (err, inputStream)
// @return {Void}
//
IOStreams.prototype.getInputStream = function(config, callback) {

  var provider;

  try {
    provider = this.getProviderByProtocol(config.protocol);
  } catch (err) {
    return callback(err);
  }

  return provider.getInputStream(config, callback);

};

//
// Opens a stream to wherever the destined output is
// @param  {String | Object}   config
// @param  {Function} callback (err, outputStream)
// @return {Void}
//
IOStreams.prototype.getOutputStream = function(config, callback) {

  var provider;

  try {
    provider = this.getProviderByProtocol(config.protocol);
  } catch (err) {
    return callback(err);
  }

  return provider.getOutputStream(config, callback);

};

//
// Lazy function to get Input and Output Streams
// @param  {String | Object}   inputConfig
// @param  {String | Object}   outputConfig [description]
// @param  {Function} callback     (err, inputStream, outputStream)
// @return {Void}
//
IOStreams.prototype.getInputOutputStreams = function(inputConfig, outputConfig, callback) {

  var self = this;

  async.parallel([
    function(callback) {
      self.getInputStream(inputConfig, callback);
    },
    function(callback) {
      self.getOutputStream(outputConfig, callback);
    }
  ], function(err, results) {
    callback(err, results[0], results[1]);
  });

};

//
// Create an instance of IOStreams
// @return {IOStreams}
//
var createInstance = function() {
  return new IOStreams();
};

//
// Exports
//

module.exports           = createInstance;
module.exports.IOStreams = IOStreams;
{'fraction_non_alphanumeric': 0.1343925852366766, 'fraction_numerical': 0.0009930486593843098, 'mean_word_length': 3.4433823529411764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '606653', 'n_tokens_mistral': 1801, 'n_tokens_neox': 1763, 'n_words': 499}