<commit_before>'use strict'

var util = require('util')
var progress = require('progress-string')
var diff = require('ansi-diff-stream')()
var speedometer = require('speedometer')
var prettierBytes = require('prettier-bytes')
var Updater = require('../updater')
var debug = require('debug')(require('../package').name)

module.exports = function (db, opts) {
  var downloadSpeed = speedometer()
  var processingSpeed = speedometer()
  var updater = new Updater(db, opts)

  updater.downloaded = 0
  updater.bps = 0
  updater.pps = 0
  updater
    .on('init', function () {
      trackSyncing()
    })
    .on('downloaded', function (bytes) {
      updater.bps = downloadSpeed(bytes)
    })
    .on('processed', function () {
      updater.pps = processingSpeed(1)
    })
    .on('end', function () {
      process.exit(updater.live ? 1 : 0)
    })
    .on('error', function (err) {
      throw err
    })

  if (!debug.enabled) diff.pipe(process.stdout)

  function trackSyncing () {
    updater.feed.on('download', function (block) {
      updater.downloaded++
      if (updater.feed.blocks !== updater.feedLength) {
        var newBlocks = updater.feed.blocks - updater.feedLength
        updater.newBlocks = newBlocks
        updater.remaining += newBlocks
        updater.feedLength = updater.feed.blocks<commit_msg>update progress bar before exit<commit_after>'use strict'

var util = require('util')
var progress = require('progress-string')
var diff = require('ansi-diff-stream')()
var speedometer = require('speedometer')
var prettierBytes = require('prettier-bytes')
var Updater = require('../updater')
var debug = require('debug')(require('../package').name)

module.exports = function (db, opts) {
  var downloadSpeed = speedometer()
  var processingSpeed = speedometer()
  var updater = new Updater(db, opts)

  updater.downloaded = 0
  updater.bps = 0
  updater.pps = 0
  updater
    .on('init', function () {
      trackSyncing()
    })
    .on('downloaded', function (bytes) {
      updater.bps = downloadSpeed(bytes)
    })
    .on('processed', function () {
      updater.pps = processingSpeed(1)
    })
    .on('end', function () {
      printProgress()
      process.exit(updater.live ? 1 : 0)
    })
    .on('error', function (err) {
      throw err
    })

  if (!debug.enabled) diff.pipe(process.stdout)

  function trackSyncing () {
    updater.feed.on('download', function (block) {
      updater.downloaded++
      if (updater.feed.blocks !== updater.feedLength) {
        var newBlocks = updater.feed.blocks - updater.feedLength
        updater.newBlocks = newBlocks
        updater.remaining += newBlocks
        updater.feedLength = updater.feed.blocks
{'fraction_non_alphanumeric': 0.14477611940298507, 'fraction_numerical': 0.004477611940298508, 'mean_word_length': 3.2021943573667713, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '709579', 'n_tokens_mistral': 850, 'n_tokens_neox': 855, 'n_words': 182}