<commit_before>var flat = require('./flat');
var hashFile = require('./hash-file');
var Merkle = require('level-merkle');
var Writable = require('stream').Writable;
var Transform = require('stream').Transform;
var through = require('through');

module.exports = build;

// Build a merkle tree from everything in `dir`
// and store it in `db`, then call `fn`.
function build(dir, db, fn) {
  var merkle = Merkle(db, 'merkle');

  // This one is ~4x faster but consumes
  // more memory.

  // Store all the file hashes in memory,
  // then write to db in one batch.
  if (false) {
    var ops = [];

    var wr = Writable({ objectMode: true });
    wr._write = function(file, _, done) {
      ops.push({
        key: hashFile(file.path, file.stat),
        value: file.path
      });
      done();
    };

    flat(dir)
    .pipe(wr)
    .on('finish', function() {
      db.batch(ops, fn);
    });
  }

  // This one is slower but consumes less memory,
  // as nothing is buffered

  // Stream files to the db.
  if (false) {
    var tr = Transform({ objectMode: true });
    tr._transform = function(file, _, done) {
      done(null, {
        key: hashFile(file.path, file.stat),
        value: file.path
      });
    };

    flat(dir)
      .pipe(tr)
      .pipe(db.createWriteStream())
      .on('close', fn);
  }
}

if (!module.parent) {
  var level = require('level');
  var sub = require('level-sublevel');

  level(__dirname + '/../db', function(err, db) {
    db = sub(db);

    var start = Date.now();
    console.log('GO');

    build(__dirname + '/../..', db, function(err) {
      if (err) throw err;
      console.log('built merkle tree in %sms', Date.now() - start);
    });
  });
}
<commit_msg>build-merkle: use faster but more mem consuming approach for now<commit_after>var flat = require('./flat');
var hashFile = require('./hash-file');
var Merkle = require('level-merkle');
var Writable = require('stream').Writable;
var Transform = require('stream').Transform;
var through = require('through');

module.exports = build;

// Build a merkle tree from everything in `dir`
// and store it in `db`, then call `fn`.
function build(dir, db, fn) {
  var merkle = Merkle(db, 'merkle');

  // This one is ~4x faster but consumes
  // more memory.

  // Store all the file hashes in memory,
  // then write to db in one batch.
  if (true) {
    var ops = [];

    var wr = Writable({ objectMode: true });
    wr._write = function(file, _, done) {
      ops.push({
        key: hashFile(file.path, file.stat),
        value: file.path
      });
      done();
    };

    flat(dir)
    .pipe(wr)
    .on('finish', function() {
      db.batch(ops, fn);
    });
  }

  // This one is slower but consumes less memory,
  // as nothing is buffered

  // Stream files to the db.
  if (false) {
    var tr = Transform({ objectMode: true });
    tr._transform = function(file, _, done) {
      done(null, {
        key: hashFile(file.path, file.stat),
        value: file.path
      });
    };

    flat(dir)
      .pipe(tr)
      .pipe(db.createWriteStream())
      .on('close', fn);
  }
}

if (!module.parent) {
  var level = require('level');
  var sub = require('level-sublevel');

  level(__dirname + '/../db', function(err, db) {
    db = sub(db);

    var start = Date.now();
    console.log('GO');

    build(__dirname + '/..', db, function(err) {
      if (err) throw err;
      console.log('built merkle tree in %sms', Date.now() - start);
    });
  });
}

{'fraction_non_alphanumeric': 0.1693106432073839, 'fraction_numerical': 0.0005768676088837611, 'mean_word_length': 2.488933601609658, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3666671', 'n_tokens_mistral': 1146, 'n_tokens_neox': 1133, 'n_words': 333}