<commit_before>var _ = require('lodash');
var util = require('../util');
var config = util.getConfig();
var dirs = util.dirs();
var log = require(dirs.core + 'log');
var moment = require('moment');

var adapter = config.adapters[config.importer.adapter];
var daterange = config.importer.daterange;

var from = moment.utc(daterange.from);

if(daterange.to) {
  var to = moment.utc(daterange.to);
} else {
  var to = moment().utc();
  log.debug(
    'No end date specified for importing, setting to',
    to.format('YYYY-MM-DD HH:mm:ss')
  );
}

var TradeBatcher = require(dirs.budfox + 'tradeBatcher');
var CandleManager = require(dirs.budfox + 'candleManager');
var exchangeChecker = require(dirs.core + 'exchangeChecker');

var error = exchangeChecker.cantFetchFullHistory(config.watch);
if(error)
  util.die(error, true);

var fetcher = require(dirs.importers + config.watch.exchange);

if(to <= from)
  util.die('This daterange does not make sense.')

var Market = function() {
  _.bindAll(this);
  this.exchangeSettings = exchangeChecker.settings(config.watch);

  this.tradeBatcher = new TradeBatcher(this.exchangeSettings.tid);
  this.candleManager = new CandleManager;
  this.fetcher = fetcher({
    to: to,
    from: from
  });

  this.done = false;

  this.fetcher.bus.on(
    'trades',
    this.processTrades
  );

  this.fetcher.bus.on(
    'done',
    function() {
      this.done = true;
    }.bind(this)
  )

  this.tradeBatcher.on(
    'new batch',
    this.candleManager.processTrades
  );

  this.candleManager.on(
    'candles',
    this.pushCandles
  );  

  Readable.call(this, {objectMode: true});

  this.get();
}

var Readable = require('stream').Readable;
Market.prototype = Object.create(Readable.prototype, {
  constructor: { value: Market }
});

Market.prototype._read = _.noop;

Market.prototype.pushCandles = function(candles) {
  _.each(candles, this.push);
}

Market.prototype.get = function() {
  this.fetcher.fetch();
}

Market.prototype.processTrades = function(trades) {
  this.tradeBatcher.write(trades);

  if(this.done)
    return log.info('Done importing!');

  setTimeout(this.get, 1000);
}

module.exports = Market;<commit_msg>validate import dates<commit_after>var _ = require('lodash');
var util = require('../util');
var config = util.getConfig();
var dirs = util.dirs();
var log = require(dirs.core + 'log');
var moment = require('moment');

var adapter = config.adapters[config.importer.adapter];
var daterange = config.importer.daterange;

var from = moment.utc(daterange.from);

if(daterange.to) {
  var to = moment.utc(daterange.to);
} else {
  var to = moment().utc();
  log.debug(
    'No end date specified for importing, setting to',
    to.format('YYYY-MM-DD HH:mm:ss')
  );
}

if(!from.isValid())
  util.die('invalid `from`');

if(!to.isValid())
  util.die('invalid `to`');

var TradeBatcher = require(dirs.budfox + 'tradeBatcher');
var CandleManager = require(dirs.budfox + 'candleManager');
var exchangeChecker = require(dirs.core + 'exchangeChecker');

var error = exchangeChecker.cantFetchFullHistory(config.watch);
if(error)
  util.die(error, true);

var fetcher = require(dirs.importers + config.watch.exchange);

if(to <= from)
  util.die('This daterange does not make sense.')

var Market = function() {
  _.bindAll(this);
  this.exchangeSettings = exchangeChecker.settings(config.watch);

  this.tradeBatcher = new TradeBatcher(this.exchangeSettings.tid);
  this.candleManager = new CandleManager;
  this.fetcher = fetcher({
    to: to,
    from: from
  });

  this.done = false;

  this.fetcher.bus.on(
    'trades',
    this.processTrades
  );

  this.fetcher.bus.on(
    'done',
    function() {
      this.done = true;
    }.bind(this)
  )

  this.tradeBatcher.on(
    'new batch',
    this.candleManager.processTrades
  );

  this.candleManager.on(
    'candles',
    this.pushCandles
  );  

  Readable.call(this, {objectMode: true});

  this.get();
}

var Readable = require('stream').Readable;
Market.prototype = Object.create(Readable.prototype, {
  constructor: { value: Market }
});

Market.prototype._read = _.noop;

Market.prototype.pushCandles = function(candles) {
  _.each(candles, this.push);
}

Market.prototype.get = function() {
  this.fetcher.fetch();
}

Market.prototype.processTrades = function(trades) {
  this.tradeBatcher.write(trades);

  if(this.done)
    return log.info('Done importing!');

  setTimeout(this.get, 300);
}

module.exports = Market;
{'fraction_non_alphanumeric': 0.15492006304886285, 'fraction_numerical': 0.0015762215717180816, 'mean_word_length': 4.0305775764439415, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2609199', 'n_tokens_mistral': 1604, 'n_tokens_neox': 1547, 'n_words': 295}