<commit_before>require('colors-browserify')
var Hub = require('../../lib')
var http = require('http')
var JSONStream = require('JSONStream')
var hub = new Hub({ //eslint-disable-line
  adapter: {
    inject: require('../../lib/protocol/websocket'),
    id: 'mtv',
    websocket: {
      server: 3031,
      val: 'ws://localhost:3032'
    }
  },
  levelup: 'mtv'
  // datafromjson: false
})

hub.levelready.is(true, function () {
  if (!hub.datafromjson || hub.datafromjson.val !== true) {
    console.log('start loading!'.magenta)
    http.request({
      host:  'scraper.dev.vigour.io', //'scraper-de-staging.dev.vigour.io',
      path: '/new.json',
      port: 80,
      method: 'get',
      headers: {
        accepts: '*/*'
      }
    }, function (res) {
      res.pipe(JSONStream.parse('mtvData.*.*'))
      .on('data', function (data) {
        console.log('CHUNK!'.magenta)
        // make this atomic wat to heavy
        hub.set(data)
      })
      .on('end', function () {
        hub.setKey('datafromjson', true)
      })
    }).end()
  }
})
<commit_msg>show chunks<commit_after>require('colors-browserify')
var Hub = require('../../lib')
var http = require('http')
var JSONStream = require('JSONStream')
var hub = new Hub({ //eslint-disable-line
  adapter: {
    inject: require('../../lib/protocol/websocket'),
    id: 'mtv',
    websocket: {
      server: 3031,
      val: 'ws://localhost:3032'
    }
  },
  shows: {},
  levelup: 'mtv'
  // datafromjson: false
})

hub.levelready.is(true, function () {
  if (!hub.datafromjson || hub.datafromjson.val !== true) {
    console.log('start loading!'.magenta)
    http.request({
      host: 'scraper.dev.vigour.io',
      path: '/new.json',
      port: 80,
      method: 'get',
      headers: {
        accepts: '*/*'
      }
    }, function (res) {
      res.pipe(JSONStream.parse('mtvData.*.*.shows.*'))
      .on('data', function (data) {
        if (data.id) {
          hub.shows.setKey(data.id , data)
        }
      })
      .on('end', function () {
        hub.setKey('datafromjson', true)
        console.log('LOADED!'.magenta)
      })
    }).end()
  }
})

{'fraction_non_alphanumeric': 0.19689849624060152, 'fraction_numerical': 0.009398496240601503, 'mean_word_length': 2.3110419906687403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3191419', 'n_tokens_mistral': 758, 'n_tokens_neox': 744, 'n_words': 134}