<commit_before>var levelDb = require('level-test')()('benchmark')
  , db = require('./level-revisions')(levelDb)
  , join = Array.prototype.join

  , input1 = join.call(Array(100), 'abc')
  , input2 = join.call(Array(200), 'add')
  , input3 = join.call(Array(2000), 'a') + 'foo' + join.call(Array(2000), 'b')
  , input4 = join.call(Array(2000), 'a') + 'bar' + join.call(Array(2000), 'b')

console.time('benchmark completely different input')
db.add('key', { body: input1, date: new Date(100) }, function () {
  db.add('key', { body: input2, date: new Date(200) }, function () {
    console.timeEnd('benchmark completely different input')

    console.time('common begining & ending')
    db.add('key2', { body: input3, date: new Date(0) }, function () {
      db.add('key2', { body: input4, date: new Date(100) }, function () {
        console.timeEnd('common begining & ending')
        console.log()
        console.log('saving loads of revisions')
        var start = Date.now()

        function write(index) {
          if (index % 100 === 0 && index > 0) {
            console.log('%s: %s ms', index, (Date.now() - start))
          }
          if (index === 500) {
            levelDb.get('key3', function (err, data) {
              console.log('uncompressed size:' + data.length / 1024 + 'kb')
            })
            return
          }

          var input = index % 2 === 0 ? input3 : input4

          db.add('key3', { body: input, date: new Date(index) }, function () {
            write(index + 1)
          })
        }

        write(0)

      })
    })
  })
})<commit_msg>a little bit better benchmark<commit_after>var levelDb = require('level-test')()('benchmark')
  , db = require('./level-revisions')(levelDb)
  , join = Array.prototype.join

  , input1 = join.call(Array(100), 'abc')
  , input2 = join.call(Array(200), 'add')
  , input3 = join.call(Array(2000), 'a') + 'foo' + join.call(Array(2000), 'b')
  , input4 = join.call(Array(2000), 'a') + 'bar' + join.call(Array(2000), 'b')

console.time('benchmark completely different input')
db.add('key', { body: input1, date: new Date(100) }, function () {
  db.add('key', { body: input2, date: new Date(200) }, function () {
    console.timeEnd('benchmark completely different input')

    console.time('common begining & ending')
    db.add('key2', { body: input3, date: new Date(0) }, function () {
      db.add('key2', { body: input4, date: new Date(100) }, function () {
        console.timeEnd('common begining & ending')
        console.log()
        console.log('saving loads of revisions')
        var start = Date.now()

        function _write(index) {
          var input = index % 2 === 0 ? input3 : input4
          if (index === 500) return running = false

          db.add('key3', { body: input, date: new Date(index) }, function () {
            write(index + 1)
          })
        }

        function write(index) {
          if (index % 100 === 0 && index > 0) {
            console.log('%s: %s ms', index, (Date.now() - start))
            console.time('get')
            db.get('key3', function () {
              console.timeEnd('get')
              start = Date.now()
              _write(index)
            })
          } else {
            _write(index)
          }
        }

        write(0)

      })
    })
  })
})
{'fraction_non_alphanumeric': 0.1784207353827607, 'fraction_numerical': 0.03616636528028933, 'mean_word_length': 2.3024875621890546, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1008152', 'n_tokens_mistral': 1131, 'n_tokens_neox': 1037, 'n_words': 246}