<commit_before>#include <sys/types.h>
#include <sys/wait.h>

#include <db.h>

#include <chord_types.h>

#include <qhash.h>
#include "merkle.h"
#include "merkle_tree_disk.h"
#include "merkle_tree_bdb.h"
#include <misc_utils.h>
#include <id_utils.h>

typedef bhash<chordID, hashID> keys_t;

static char *indexpath = "/tmp/index.mrk";
static char *internalpath = "/tmp/internal.mrk";
static char *leafpath  = "/tmp/leaf.mrk";
static char *indexpathro = "/tmp/index.mrk.ro";
static char *internalpathro = "/tmp/internal.mrk.ro";
static char *leafpathro  = "/tmp/leaf.mrk.ro";

// Better be careful here!  We call rm -rf on this later.
static char *bdbpath = "/tmp/mtree.bdb";

typedef callback<merkle_tree *, bool>::ref merkle_allocator_t;

merkle_tree *
allocate_disk (bool master)
{
  // Master should be read-write
  return
    New merkle_tree_disk (indexpath, internalpath, leafpath, master);
}

merkle_tree *
allocate_bdb (bool master)
{
  return
    New merkle_tree_bdb (bdbpath, !master, !master);
  // slaves should join and be read-only;
  // master should not join and be read-write.
}

void
cleanup ()
{
  unlink (indexpath);
  unlink (internalpath);
  unlink (leafpath);
  unlink (indexpathro);
  unlink (internalpathro);
  unlink (leafpathro);
  char buf[80];
  sprintf (buf, "rm -rf %s", bdbpath);
  system (buf);
}

bool
reap (int pid, bool wait = true)
{
  int wpid, status, options = 0;

  if (!wait)
    options = WNOHANG;
  wpid = waitpid (pid, &status, options);
  if (wpid < 0) {
    perror("waitpid");
    exit (1);
  }
  if (!wait && (wpid == 0))
    return false;
  if (wpid != pid)
    fatal ("unexpected pid reaped: %d\n", wpid);
  if (!WIFEXITED(status) || WEXITSTATUS(status) != 0)
    fatal << "child exited unhappily\n";
  return true;
}

void
insert_blocks (merkle_tree *mtree, int upto, bool random,
    keys_t *keys = NULL)
{
  for (int i = 1; i <= upto; i++) {
    merkle_hash key;
    if (random) 
      key.randomize ();
    else
      key = i;

    if (keys)
      keys->insert (static_cast<bigint> (key));
    mtree->insert (key);
  }
}

void
test_numkeys (merkle_tree *tree, unsigned int cnt)
{
  merkle_node *root = tree->get_root ();
  assert (root->count == cnt);
  tree->lookup_release (root);
}

void
test_insertions (str msg,
    merkle_tree *mtree, uint nkeys, bool rand, keys_t &keys)
{
  uint okeyssz = keys.size ();
  warn << msg << " bulk insertion... ";
  u_int64_t start = getusec (true);
  mtree->set_rehash_on_modification (false);
  insert_blocks (mtree, nkeys, rand, &keys);
  warn << "completed in: " << (getusec (true)-start)/1000 << "ms... ";
  assert ((keys.size () - okeyssz) == nkeys);
  warn << "OK\n";

  warn << msg << " hash full tree... ";
  start = getusec (true);
  mtree->hash_tree ();
  warn << "completed in: " << (getusec (true)-start)/1000 << "ms... ";
  test_numkeys (mtree, keys.size ());
  warn << "OK\n";

  warn << msg << " check invariants... ";
  mtree->check_invariants ();
  warn << "OK.\n";

  mtree->set_rehash_on_modification (true);
}

const vec<chordID> &
test_reads (merkle_tree *mtree, const keys_t &keys)
{
  static vec<chordID> intree;
  intree.clear ();
  unsigned int nkeys = keys.size ();

  warn << "All key read (get_keyrange)... ";
  intree = mtree->get_keyrange (
      0, (chordID (1) << 160) - 1, nkeys + 1);
  assert (intree.size () == nkeys);
  for (uint i = 0; i < intree.size (); i++) {
    assert (keys[intree[i]]);
  }
  warn << "OK\n";

  warn << "Randomized key lookups... ";
  uint limit = 128 + int(nkeys * 0.2);
  if (limit > nkeys)
    limit = nkeys;
  uint order[nkeys];
  for (uint i=0; i<nkeys; i++) order[i] = i;
  for (uint i = 0; i < limit; i++) {
    uint j = i + random() % (nkeys-i);
    uint key = order[j]; order[j] = order[i]; order[i] = key;
    assert (mtree->key_exists (intree[key]));
    merkle_node *n = mtree->lookup (intree[key]);
    assert (n);
    assert (n->isleaf ());
  }
  warn << "OK\n";

  warn << "Missing key lookups... ";
  int todo = limit;
  while (todo) {
    chordID c = make_randomID ();
    if (keys[c])
      continue;
    todo--;
    assert (!mtree->key_exists (c));
    merkle_node *n = mtree->lookup (c);
    assert (n);
  }
  warn << "OK\n";

  warn << "Random range reads... ";
  chordID highest = (((chordID) 1) << 160) - 1;
  for (int i = 0; i < 32; i++) {
    chordID min = make_randomID ();
    chordID max = min + make_randomID () % (highest - min);
    //warn << "  get_keyrange (" << min << ", " << max << ", " << nkeys << ")\n";
    vec<chordID> rangekeys = mtree->get_keyrange (min, max, nkeys);

    keys_t subset;
    for (uint j = 0; j < nkeys; j++)
      if (betweenbothincl (min, max, intree[j]))
	subset.insert (intree[j]);
    assert (subset.size () == rangekeys.size ());
    for (uint j = 0; j < rangekeys.size (); j++)
      assert (subset[rangekeys[j]]);
  }
  warn << "OK\n";
  return intree;
}

void
test (str msg, merkle_tree *mtree, uint nkeys, bool rand = true)<commit_msg>Fix simple memory leak in test_merkle_tree.<commit_after>#include <sys/types.h>
#include <sys/wait.h>

#include <db.h>

#include <chord_types.h>

#include <qhash.h>
#include "merkle.h"
#include "merkle_tree_disk.h"
#include "merkle_tree_bdb.h"
#include <misc_utils.h>
#include <id_utils.h>

typedef bhash<chordID, hashID> keys_t;

static char *indexpath = "/tmp/index.mrk";
static char *internalpath = "/tmp/internal.mrk";
static char *leafpath  = "/tmp/leaf.mrk";
static char *indexpathro = "/tmp/index.mrk.ro";
static char *internalpathro = "/tmp/internal.mrk.ro";
static char *leafpathro  = "/tmp/leaf.mrk.ro";

// Better be careful here!  We call rm -rf on this later.
static char *bdbpath = "/tmp/mtree.bdb";

typedef callback<merkle_tree *, bool>::ref merkle_allocator_t;

merkle_tree *
allocate_disk (bool master)
{
  // Master should be read-write
  return
    New merkle_tree_disk (indexpath, internalpath, leafpath, master);
}

merkle_tree *
allocate_bdb (bool master)
{
  return
    New merkle_tree_bdb (bdbpath, !master, !master);
  // slaves should join and be read-only;
  // master should not join and be read-write.
}

void
cleanup ()
{
  unlink (indexpath);
  unlink (internalpath);
  unlink (leafpath);
  unlink (indexpathro);
  unlink (internalpathro);
  unlink (leafpathro);
  char buf[80];
  sprintf (buf, "rm -rf %s", bdbpath);
  system (buf);
}

bool
reap (int pid, bool wait = true)
{
  int wpid, status, options = 0;

  if (!wait)
    options = WNOHANG;
  wpid = waitpid (pid, &status, options);
  if (wpid < 0) {
    perror("waitpid");
    exit (1);
  }
  if (!wait && (wpid == 0))
    return false;
  if (wpid != pid)
    fatal ("unexpected pid reaped: %d\n", wpid);
  if (!WIFEXITED(status) || WEXITSTATUS(status) != 0)
    fatal << "child exited unhappily\n";
  return true;
}

void
insert_blocks (merkle_tree *mtree, int upto, bool random,
    keys_t *keys = NULL)
{
  for (int i = 1; i <= upto; i++) {
    merkle_hash key;
    if (random) 
      key.randomize ();
    else
      key = i;

    if (keys)
      keys->insert (static_cast<bigint> (key));
    mtree->insert (key);
  }
}

void
test_numkeys (merkle_tree *tree, unsigned int cnt)
{
  merkle_node *root = tree->get_root ();
  assert (root->count == cnt);
  tree->lookup_release (root);
}

void
test_insertions (str msg,
    merkle_tree *mtree, uint nkeys, bool rand, keys_t &keys)
{
  uint okeyssz = keys.size ();
  warn << msg << " bulk insertion... ";
  u_int64_t start = getusec (true);
  mtree->set_rehash_on_modification (false);
  insert_blocks (mtree, nkeys, rand, &keys);
  warn << "completed in: " << (getusec (true)-start)/1000 << "ms... ";
  assert ((keys.size () - okeyssz) == nkeys);
  warn << "OK\n";

  warn << msg << " hash full tree... ";
  start = getusec (true);
  mtree->hash_tree ();
  warn << "completed in: " << (getusec (true)-start)/1000 << "ms... ";
  test_numkeys (mtree, keys.size ());
  warn << "OK\n";

  warn << msg << " check invariants... ";
  mtree->check_invariants ();
  warn << "OK.\n";

  mtree->set_rehash_on_modification (true);
}

const vec<chordID> &
test_reads (merkle_tree *mtree, const keys_t &keys)
{
  static vec<chordID> intree;
  intree.clear ();
  unsigned int nkeys = keys.size ();

  warn << "All key read (get_keyrange)... ";
  intree = mtree->get_keyrange (
      0, (chordID (1) << 160) - 1, nkeys + 1);
  assert (intree.size () == nkeys);
  for (uint i = 0; i < intree.size (); i++) {
    assert (keys[intree[i]]);
  }
  warn << "OK\n";

  warn << "Randomized key lookups... ";
  uint limit = 128 + int(nkeys * 0.2);
  if (limit > nkeys)
    limit = nkeys;
  uint order[nkeys];
  for (uint i=0; i<nkeys; i++) order[i] = i;
  for (uint i = 0; i < limit; i++) {
    uint j = i + random() % (nkeys-i);
    uint key = order[j]; order[j] = order[i]; order[i] = key;
    assert (mtree->key_exists (intree[key]));
    merkle_node *n = mtree->lookup (intree[key]);
    assert (n);
    assert (n->isleaf ());
    mtree->lookup_release (n);
  }
  warn << "OK\n";

  warn << "Missing key lookups... ";
  int todo = limit;
  while (todo) {
    chordID c = make_randomID ();
    if (keys[c])
      continue;
    todo--;
    assert (!mtree->key_exists (c));
    merkle_node *n = mtree->lookup (c);
    assert (n);
    mtree->lookup_release (n);
  }
  warn << "OK\n";

  warn << "Random range reads... ";
  chordID highest = (((chordID) 1) << 160) - 1;
  for (int i = 0; i < 32; i++) {
    chordID min = make_randomID ();
    chordID max = min + make_randomID () % (highest - min);
    //warn << "  get_keyrange (" << min << ", " << max << ", " << nkeys << ")\n";
    vec<chordID> rangekeys = mtree->get_keyrange (min, max, nkeys);

    keys_t subset;
    for (uint j = 0; j < nkeys; j++)
      if (betweenbothincl (min, max, intree[j]))
	subset.insert (intree[j]);
    assert (subset.size () == rangekeys.size ());
    for (uint j = 0; j < rangekeys.size (); j++)
      assert (subset[rangekeys[j]]);
  }
  warn << "OK\n";
  return intree;
}

void
test (str msg, merkle_tree *mtree, uint nkeys, bool rand = true)
{'fraction_non_alphanumeric': 0.17700620372223333, 'fraction_numerical': 0.008605163097858715, 'mean_word_length': 3.18025930572982, 'pattern_counts': {'":': 0, '<': 177, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2727979', 'n_tokens_mistral': 3919, 'n_tokens_neox': 3838, 'n_words': 1022}