<commit_before>var level     = require('level-test')()
var sublevel  = require('level-sublevel/bytewise')
var pull      = require('pull-stream')
var ecc       = require('eccjs')
var tape      = require('tape')

var SSB       = require('../')
var u         = require('../util')
var w         = require('./util')
var replicate = require('../replicate')

var codec     = require('../codec')
var JSONB     = require('json-buffer')


function rand (n) {
  var a = []
  while(n--)
    a.push(Math.random())
  return a
}


module.exports = function (opts) {

  var create = require('../message')(opts)

  function createDB(name) {
    return SSB(sublevel(level(name, {
      valueEncoding: require('../codec')
    })), opts)
  }

  var MESSAGE = new Buffer('msg')

  function init (ssb, n, cb) {
    var keys = opts.keys.generate()
    var prev

    ssb.add(prev = create(keys, 'init', keys.public), function () {
      pull(
        pull.values(rand(n)),
        pull.asyncMap(function (r, cb) {
          ssb.add(prev =
            create(keys, 'msg', ''+r, prev), cb)
        }),
        pull.drain(null, cb)
      )
    })
      return keys
  }

  function compareDbs (a, b, cb) {

    var cbs = u.groups(next)

    pull(a.createFeedStream(), pull.collect(cbs()))
    pull(b.createFeedStream(), pull.collect(cbs()))

    function next(err, ary) {
      cb(err, ary && ary[0], ary && ary[1])
    }
  }

  return {
    createDB: createDB, init: init, compareDbs: compareDbs
  }

}<commit_msg>more test utils<commit_after>var level     = require('level-test')()
var sublevel  = require('level-sublevel/bytewise')
var pull      = require('pull-stream')
var ecc       = require('eccjs')
var tape      = require('tape')

var SSB       = require('../')
var u         = require('../util')
var w         = require('./util')
var replicate = require('../replicate')

var codec     = require('../codec')
var JSONB     = require('json-buffer')


function rand (n) {
  var a = []
  while(n--)
    a.push(Math.random())
  return a
}


module.exports = function (opts) {

  var create = require('../message')(opts)

  function createDB(name) {
    return SSB(sublevel(level(name, {
      valueEncoding: require('../codec')
    })), opts)
  }

  var MESSAGE = new Buffer('msg')

  function load (ssb, keys, n, cb) {
    var prev
    ssb.getLatest(opts.hash(keys.public), function (err, prev) {
      if(err) return cb(err)
      pull(
        pull.values(rand(n)),
        pull.asyncMap(function (r, cb) {
          ssb.add(prev =
            create(keys, 'msg', ''+r, prev), cb)
        }),
        pull.drain(null, cb)
      )
    })
  }

  function init (ssb, n, cb) {
    var keys = opts.keys.generate()
    var prev

    ssb.add(prev = create(keys, 'init', keys.public), function () {
      pull(
        pull.values(rand(n)),
        pull.asyncMap(function (r, cb) {
          ssb.add(prev =
            create(keys, 'msg', ''+r, prev), cb)
        }),
        pull.drain(null, cb)
      )
    })
      return keys
  }

  function compareDbs (a, b, cb) {

    var cbs = u.groups(next)

    pull(a.createFeedStream(), pull.collect(cbs()))
    pull(b.createFeedStream(), pull.collect(cbs()))

    function next(err, ary) {
      cb(err, ary && ary[0], ary && ary[1])
    }
  }

  return {
    createDB: createDB, init: init, compareDbs: compareDbs, load: load
  }

}
{'fraction_non_alphanumeric': 0.17253731343283582, 'fraction_numerical': 0.0011940298507462687, 'mean_word_length': 2.347652347652348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1143593', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1147, 'n_words': 264}