<commit_before>
var tape = require('tape')

var util = require('../util')
var tree = require('../slow')
var Merkle = require('../')
tree = Merkle.tree

function count (tree) {
  var s = 0
  if('string' === typeof tree)
    return 1
  else
    for (var k in tree.tree) {
      s += count(tree.tree[k])
    }
  return s + 1
}
var crypto = require('crypto')
function hash (n) {
  n = (n | 0).toString(16)
  if(n.length % 2)
    n = '0' + n
  return crypto.createHash('sha1').update(n, 'hex').digest('hex')
}

console.log('N, t, N/t')

var _t =  0.0001
var sqrt2 = Math.pow(2, 0.25)
var N = 1, j = 0
var a = []
//for(var i = 1; i <= 20; i ++) {
while(N < (1 << 21)) {
  while(a.length < N)
    a.push(hash(j++))
  var start = Date.now()
  var g = tree(a)
  
  var t = Date.now() - start || 0.0001
  var d = g.digest()
  console.log([~~N, t, N/t].join(', '))
  _t = t
  N = N * sqrt2
}
<commit_msg>iterate on tests that are small N<commit_after>
var tape = require('tape')

var util = require('../util')
var tree = require('../slow')
var Merkle = require('../')
tree = Merkle.tree

function count (tree) {
  var s = 0
  if('string' === typeof tree)
    return 1
  else
    for (var k in tree.tree) {
      s += count(tree.tree[k])
    }
  return s + 1
}
var crypto = require('crypto')
function hash (n) {
  n = (n | 0).toString(16)
  if(n.length % 2)
    n = '0' + n
  return crypto.createHash('sha1').update(n, 'hex').digest('hex')
}

console.log('N, t, N/t, hash')

var _t =  1
var sqrt2 = Math.pow(2, 0.25)
var N = 1, j = 0
var a = []

while(N < (1 << 21)) {
  while(a.length <= N)
    a.push(hash(j++))
  var start = Date.now()
  var m = 0, g
  while(Date.now() === start) {
    m ++
    g = tree(a)
  }
  
  var t = Date.now() - start || 1
  var d = g.digest()
  console.log([a.length, t/m, a.length/(t/m), d].join(', '))
  _t = t
  var _N = N
  while(N < _N + 1)
    N = N * sqrt2
}

{'fraction_non_alphanumeric': 0.2091295116772824, 'fraction_numerical': 0.030254777070063694, 'mean_word_length': 2.51024208566108, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '861289', 'n_tokens_mistral': 834, 'n_tokens_neox': 789, 'n_words': 224}