<commit_before>var request = require('request')
var split = require('split2')

module.exports = function(url, opts) {
  if (!opts) opts = {}
  if (typeof opts.retry !== 'number' && opts.retry !== false) opts.retry = 3000

  var dataBuf = ''
  var event = ''
  var id = ''
  var req
  var timeout

  var parse = split(function(line) {
    if (!line) {
      if (!dataBuf) return
      var data = dataBuf
      dataBuf = ''
      console.log('data', data)
      return {
        id : id,
        event : event,
        data : data
      }
    }
    if (line.indexOf('data: ') === 0) dataBuf += (dataBuf ? '\n' : '') + line.slice(6)
    if (line.indexOf('id: ') === 0) id = line.slice(6)
    if (line.indexOf('event: ') === 0) event = line.slice(6)

  })

  var connect = function() {
    dataBuf = ''
    req = request(url)

    req.on('error', function(err) {
      if (!opts.retry) parse.emit('error', err)
    })

    req.on('complete', function() {
      if (destroyed) return
<commit_msg>removed console statement<commit_after>var request = require('request')
var split = require('split2')

module.exports = function(url, opts) {
  if (!opts) opts = {}
  if (typeof opts.retry !== 'number' && opts.retry !== false) opts.retry = 3000

  var dataBuf = ''
  var event = ''
  var id = ''
  var req
  var timeout

  var parse = split(function(line) {
    if (!line) {
      if (!dataBuf) return
      var data = dataBuf
      dataBuf = ''
      return {
        id : id,
        event : event,
        data : data
      }
    }
    if (line.indexOf('data: ') === 0) dataBuf += (dataBuf ? '\n' : '') + line.slice(6)
    if (line.indexOf('id: ') === 0) id = line.slice(6)
    if (line.indexOf('event: ') === 0) event = line.slice(6)

  })

  var connect = function() {
    dataBuf = ''
    req = request(url)

    req.on('error', function(err) {
      if (!opts.retry) parse.emit('error', err)
    })

    req.on('complete', function() {
      if (destroyed) return

{'fraction_non_alphanumeric': 0.1783893985728848, 'fraction_numerical': 0.011213047910295617, 'mean_word_length': 2.4198606271777003, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3149543', 'n_tokens_mistral': 675, 'n_tokens_neox': 657, 'n_words': 170}