<commit_before>var util  = require("util");
var http  = require("http");
var https = require("https");
var EventEmitter = require('events').EventEmitter;
var isUrl = require('./lib/is-url');

var elasticdump = function(input, output, options){
  var self  = this;

  self.input   = input;
  self.output  = output;
  self.options = options;

  if (!self.options.searchBody)  {
      self.options.searchBody = {"query": { "match_all": {} }, "fields": ["*"], "_source": true };
  }

  if (self.options.toLog === null || self.options.toLog === undefined) {
    self.options.toLog = true;
  }

  self.validationErrors = self.validateOptions();

  if(options.maxSockets){
    self.log('globally setting maxSockets=' + options.maxSockets);
    http.globalAgent.maxSockets  = options.maxSockets;
    https.globalAgent.maxSockets = options.maxSockets;
  }

  var InputProto;
  if(self.options.input && !self.options.inputTransport){
    if(self.options.input === "$"){
      self.inputType = 'stdio';
    }else if(isUrl(self.options.input)){
      self.inputType = 'elasticsearch';
    }else{
      self.inputType  = 'file';
    }

    InputProto = require(__dirname + "/lib/transports/" + self.inputType)[self.inputType];
    self.input  = (new InputProto(self, self.options.input, self.options['input-index']));
  }else if(self.options.inputTransport){
    InputProto = require(self.options.inputTransport);
    var inputProtoKeys = Object.keys(InputProto);
    self.input = (new InputProto[inputProtoKeys[0]](self, self.options.input, self.options['input-index']));
  }

  var OutputProto;
  if(self.options.output && !self.options.outputTransport){
    if(self.options.output === "$"){
      self.outputType = 'stdio';
      self.options.toLog = false;
    }else if(isUrl(self.options.output)){
      self.outputType = 'elasticsearch';
    }else{
      self.outputType = 'file';
    }

    OutputProto = require(__dirname + "/lib/transports/" + self.outputType)[self.outputType];
    self.output = (new OutputProto(self, self.options.output, self.options['output-index']));
  }else if(self.options.outputTransport){
    OutputProto = require(self.options.outputTransport);
    var outputProtoKeys = Object.keys(OutputProto);
    self.output = (new OutputProto[outputProtoKeys[0]](self, self.options.output, self.options['output-index']));
  }
};

util.inherits(elasticdump, EventEmitter);

elasticdump.prototype.log = function(message){
  var self = this;

  if(typeof self.options.logger === 'function'){
    self.options.logger(message);
  }else if(self.options.toLog === true){
    self.emit("log", message);
  }
};

elasticdump.prototype.validateOptions = function(){
  var self = this;<commit_msg>Set inputType/outputType if using custom transport<commit_after>var util  = require("util");
var http  = require("http");
var https = require("https");
var EventEmitter = require('events').EventEmitter;
var isUrl = require('./lib/is-url');

var elasticdump = function(input, output, options){
  var self  = this;

  self.input   = input;
  self.output  = output;
  self.options = options;

  if (!self.options.searchBody)  {
      self.options.searchBody = {"query": { "match_all": {} }, "fields": ["*"], "_source": true };
  }

  if (self.options.toLog === null || self.options.toLog === undefined) {
    self.options.toLog = true;
  }

  self.validationErrors = self.validateOptions();

  if(options.maxSockets){
    self.log('globally setting maxSockets=' + options.maxSockets);
    http.globalAgent.maxSockets  = options.maxSockets;
    https.globalAgent.maxSockets = options.maxSockets;
  }

  var InputProto;
  if(self.options.input && !self.options.inputTransport){
    if(self.options.input === "$"){
      self.inputType = 'stdio';
    }else if(isUrl(self.options.input)){
      self.inputType = 'elasticsearch';
    }else{
      self.inputType  = 'file';
    }

    InputProto = require(__dirname + "/lib/transports/" + self.inputType)[self.inputType];
    self.input  = (new InputProto(self, self.options.input, self.options['input-index']));
  }else if(self.options.inputTransport){
    self.inputType = 'inputTransport';
    InputProto = require(self.options.inputTransport);
    var inputProtoKeys = Object.keys(InputProto);
    self.input = (new InputProto[inputProtoKeys[0]](self, self.options.input, self.options['input-index']));
  }

  var OutputProto;
  if(self.options.output && !self.options.outputTransport){
    if(self.options.output === "$"){
      self.outputType = 'stdio';
      self.options.toLog = false;
    }else if(isUrl(self.options.output)){
      self.outputType = 'elasticsearch';
    }else{
      self.outputType = 'file';
    }

    OutputProto = require(__dirname + "/lib/transports/" + self.outputType)[self.outputType];
    self.output = (new OutputProto(self, self.options.output, self.options['output-index']));
  }else if(self.options.outputTransport){
    self.outputType = 'outputTransport';
    OutputProto = require(self.options.outputTransport);
    var outputProtoKeys = Object.keys(OutputProto);
    self.output = (new OutputProto[outputProtoKeys[0]](self, self.options.output, self.options['output-index']));
  }
};

util.inherits(elasticdump, EventEmitter);

elasticdump.prototype.log = function(message){
  var self = this;

  if(typeof self.options.logger === 'function'){
    self.options.logger(message);
  }else if(self.options.toLog === true){
    self.emit("log", message);
  }
};

elasticdump.prototype.validateOptions = function(){
  var self = this;
{'fraction_non_alphanumeric': 0.16167664670658682, 'fraction_numerical': 0.0007258210851025222, 'mean_word_length': 4.02461257976299, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3984852', 'n_tokens_mistral': 1740, 'n_tokens_neox': 1712, 'n_words': 304}