<commit_before>var _ = require('lodash'),
    Promise = require('bluebird'),
    Readable = require('stream').Readable,
    util = require('util');

function MarketoStream(marketoResultPromise, options) {
  options = _.defaults(options || {}, {objectMode: true});
  Readable.call(this, options);
  this._setData(marketoResultPromise || Promise.resolve({}));
}
util.inherits(MarketoStream, Readable);

MarketoStream.prototype._setData = function(dataPromise) {
  this._ready = false;

  var self = this;
  this._dataPromise = dataPromise;
  return dataPromise.then(
    function(data) {
      if (data.result.length > 0) {
        self._data = data;
        self._resultIndex = 0;
        self._ready = true;
        self._pushNext();
      } else {
        self.emit('error', 'Leads list is empty');
        // end the stream
        self.push(null);
      }
    },
    function(err) {
      self.emit('error', err);
      // end the stream
      self.push(null);
    });
};

MarketoStream.prototype._read = function() {

  if (!this._ready) {
    return;
  }

  if (this._data.result) {
    if (this._pushNext()) {
      return;
    } else if (this._data.moreResult) {
      this._setData(this._data.nextPage())
    } else {
      // No data left and no more data from marketo<commit_msg>validating data.result is not null<commit_after>var _ = require('lodash'),
    Promise = require('bluebird'),
    Readable = require('stream').Readable,
    util = require('util');

function MarketoStream(marketoResultPromise, options) {
  options = _.defaults(options || {}, {objectMode: true});
  Readable.call(this, options);
  this._setData(marketoResultPromise || Promise.resolve({}));
}
util.inherits(MarketoStream, Readable);

MarketoStream.prototype._setData = function(dataPromise) {
  this._ready = false;

  var self = this;
  this._dataPromise = dataPromise;
  return dataPromise.then(
    function(data) {
      if (data.result && data.result.length > 0) {
        self._data = data;
        self._resultIndex = 0;
        self._ready = true;
        self._pushNext();
      } else {
        self.emit('error', 'Leads list is empty');
        // end the stream
        self.push(null);
      }
    },
    function(err) {
      self.emit('error', err);
      // end the stream
      self.push(null);
    });
};

MarketoStream.prototype._read = function() {

  if (!this._ready) {
    return;
  }

  if (this._data.result) {
    if (this._pushNext()) {
      return;
    } else if (this._data.moreResult) {
      this._setData(this._data.nextPage())
    } else {
      // No data left and no more data from marketo
{'fraction_non_alphanumeric': 0.14537107880642694, 'fraction_numerical': 0.001530221882172915, 'mean_word_length': 2.762589928057554, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '860581', 'n_tokens_mistral': 820, 'n_tokens_neox': 807, 'n_words': 172}