<commit_before>var Leveldown = require('../');
var assert = require('assert');

Leveldown = new Leveldown(__dirname+"/dbfoo");

  Leveldown.open({
        errorIfExists   : false
      , createIfMissing : true
      , cacheSize       : 8 << 20
      , writeBufferSize : 4 << 20
  })


// bench -----------------------------------

var count = 12000;
var val = '1337,1337,1337,1337,1337';

function putSync () {
  var start = Date.now();

  for (var i = 0; i < count; i++) {
    Leveldown.put(i, val);
  }

  var duration = Date.now()-start;
  log(true, count, 'put', duration);
}

function getSync () {
  start = Date.now();

  for (var i = 0; i < count; i++) {
    assert(Leveldown.get(i) == val);
  }

  duration = Date.now()-start;
  log(true, count, 'get', duration);
}

function putAsync(cb) {
  start = Date.now();

  var written = 0;
  for (var i = 0; i < count; i++) {
    Leveldown.put(i, val, function (err) {
      if (err) throw err;
      if (++written == count) {
        duration = Date.now()-start;
        log(false, count, 'put', duration);
        if (cb) cb();
      }
    })
  }
}

function getAsync (cb) {
  start = Date.now();

  var received = 0;
  for (var i = 0; i < count; i++) {
    Leveldown.get(i, function (err, value) {
      if (err) throw err;
      assert(value == val);
      if (++received == count) {
        duration = Date.now()-start;
        log(false, count, 'get', duration);
        if (cb) cb()
      }
    })
  }
}

function batchSync () {
  var start = Date.now();

  var arr = [];
  for (var i = 0; i < count; i++) {
    arr.push({type:"put",key:i, value:val});
  }

  var batch = Leveldown.batch(arr);

  var duration = Date.now()-start;
  log(true, count, 'batch', duration);
}

function batchAsync(cb) {
  var start = Date.now();

  var arr = [];
  for (var i = 0; i < count; i++) {
    arr.push({type:"put",key:i, value:val});
  }
  
  Leveldown.batch(arr, function (err) {
    if (err) throw err;
    var duration = Date.now() - start;
    log(false, count, 'batch', duration);
    if (cb) cb();
  });
}


/*
 * Get it started
 */

console.log('\n  benchmarking with ' + humanize(count) + ' records, ' + val.length + ' chars each\n');

putAsync(function () {
  putSync()
  batchAsync(function () {
    batchSync()
    getAsync(function () {
      getSync()
      console.log()
    })
  })
})

/*
 * Utility functions
 */

var last;
function log(sync, num, op, dur) {
  if (last && last != op) console.log();
  last = op;
  console.log([
    pad(op + (sync? 'Sync' : ''), 13),
    ':',
    pad(humanize(Math.floor(1000/dur * num)), 9),
    {'batch' : 'w', 'put' : 'w', 'get' : 'r', 'iterator' : 'r'}[op] + '/s',
    'in ' + pad(humanize(dur), 6) + 'ms'
   ].join(' '));
}

function pad(str, len) {
  str = str.toString();
  return Array(len - str.length + 1).join(' ') + str;
}<commit_msg>+ {asBuffer:false} benchmark test<commit_after>var Leveldown = require('../');
var assert = require('assert');

Leveldown = new Leveldown(__dirname+"/dbfoo");

  Leveldown.open({
        errorIfExists   : false
      , createIfMissing : true
      , cacheSize       : 8 << 20
      , writeBufferSize : 4 << 20
  })


// bench -----------------------------------

var count = 12000;
var val = '1337,1337,1337,1337,1337';

function putSync () {
  var start = Date.now();

  for (var i = 0; i < count; i++) {
    Leveldown.put(i, val);
  }

  var duration = Date.now()-start;
  log(true, count, 'put', duration);
}

function getSync () {
  start = Date.now();

  for (var i = 0; i < count; i++) {
    assert(Leveldown.get(i) == val);
  }

  duration = Date.now()-start;
  log(true, count, 'get', duration);
}

function getStrSync () {
  start = Date.now();

  for (var i = 0; i < count; i++) {
    assert(Leveldown.get(i, {asBuffer: false}) == val);
  }

  duration = Date.now()-start;
  log(true, count, 'get', duration, "asBuffer:false");
}

function putAsync(cb) {
  start = Date.now();

  var written = 0;
  for (var i = 0; i < count; i++) {
    Leveldown.put(i, val, function (err) {
      if (err) throw err;
      if (++written == count) {
        duration = Date.now()-start;
        log(false, count, 'put', duration);
        if (cb) cb();
      }
    })
  }
}

function getAsync (cb) {
  start = Date.now();

  var received = 0;
  for (var i = 0; i < count; i++) {
    Leveldown.get(i, function (err, value) {
      if (err) throw err;
      assert(value == val);
      if (++received == count) {
        duration = Date.now()-start;
        log(false, count, 'get', duration);
        if (cb) cb()
      }
    })
  }
}

function getStrAsync (cb) {
  start = Date.now();

  var received = 0;
  for (var i = 0; i < count; i++) {
    Leveldown.get(i, {asBuffer:false}, function (err, value) {
      if (err) throw err;
      assert(value == val);
      if (++received == count) {
        duration = Date.now()-start;
        log(false, count, 'get', duration, "asBuffer:false");
        if (cb) cb()
      }
    })
  }
}

function batchSync () {
  var start = Date.now();

  var arr = [];
  for (var i = 0; i < count; i++) {
    arr.push({type:"put",key:i, value:val});
  }

  var batch = Leveldown.batch(arr);

  var duration = Date.now()-start;
  log(true, count, 'batch', duration);
}

function batchAsync(cb) {
  var start = Date.now();

  var arr = [];
  for (var i = 0; i < count; i++) {
    arr.push({type:"put",key:i, value:val});
  }
  
  Leveldown.batch(arr, function (err) {
    if (err) throw err;
    var duration = Date.now() - start;
    log(false, count, 'batch', duration);
    if (cb) cb();
  });
}


/*
 * Get it started
 */

console.log('\n  benchmarking with ' + humanize(count) + ' records, ' + val.length + ' chars each\n');

putAsync(function () {
  putSync()
  batchAsync(function () {
    batchSync()
    getAsync(function () {
      getStrAsync(function () {
        getSync()
        getStrSync()
        console.log()
      })
    })
  })
})

/*
 * Utility functions
 */

var last;
function log(sync, num, op, dur, desc) {
  if (!desc) desc = ""
  else desc = "("+desc+")";
  if (last && last != op) console.log();
  last = op;
  console.log([
    pad(op + (sync? 'Sync' : ''), 13),
    ':',
    pad(humanize(Math.floor(1000/dur * num)), 9),
    {'batch' : 'w', 'put' : 'w', 'get' : 'r', 'iterator' : 'r'}[op] + '/s',
    'in ' + pad(humanize(dur), 6) + 'ms' + pad(desc, 18)
   ].join(' '));
}

function pad(str, len) {
  str = str.toString();
  return Array(len - str.length + 1).join(' ') + str;
}
{'fraction_non_alphanumeric': 0.20753551575046325, 'fraction_numerical': 0.015596046942557134, 'mean_word_length': 2.4470463012240553, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1248745', 'n_tokens_mistral': 2455, 'n_tokens_neox': 2341, 'n_words': 603}