<commit_before>#include <chord_types.h>
#include <id_utils.h>
#include <rxx.h>
#include "merkle_tree_disk.h"
#include "dhash_common.h"

static FILE *
open_file (str name) {
  // make all the parent directories if applicable
  vec<str> dirs;
  static const rxx dirsplit("\\/");
  (void) split (&dirs, dirsplit, name);
  strbuf filestrbuf;
  for (uint i = 0; i < dirs.size()-1; i++) {
    filestrbuf << dirs[i] << "/";
    
    str dirpath (filestrbuf);
    // check for existence, make if necessary
    struct stat st;
    if (stat (dirpath, &st) < 0) {
      if (mkdir(dirpath, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH) != 0) {
	fatal << "open_file: mkdir (" << dirpath << "): " << strerror (errno) << ".\n";
      }
    }
  }

  FILE *f = fopen (name, "r+");
  if (f == NULL) {
    f = fopen (name, "w+");
  }
  if (f == NULL) {
    fatal << "open_file: fopen (" << name << "): " << strerror (errno) << ".\n"; 
  }
  return f;
}

static int
copy_file (str src, str dst)
{
  int sfd = open (src, O_RDONLY);
  if (sfd < 0) {
    perror ("copy_file: src");
    return sfd;
  }

  str tmpfile = dst << "~";
  unlink (tmpfile);
  int dfd = open (tmpfile, O_CREAT|O_WRONLY, 0644);
  if (dfd < 0) {
    perror ("copy_file: dst");
    close (sfd);
    return dfd;
  }
#define COPY_ERROR(msg) \
  do {						  \
    int oerrno = errno;				  \
    warn ("copy_file: " msg ": %m");		  \
    close (sfd); close (dfd); unlink (tmpfile);	  \
    errno = oerrno;				  \
    return -1;					  \
  } while (0)

  char buf[8192];
  ssize_t nread;
  while ((nread = read (sfd, buf, sizeof (buf))) > 0) {
    ssize_t nwrote = write (dfd, buf, nread);
    if (nwrote < 0) {
      COPY_ERROR ("bad write");
    }
    if (nwrote != nread) {
      COPY_ERROR ("short write");
    }
  }
  if (nread < 0) {
    COPY_ERROR ("read error");
  }
  close (sfd); close (dfd);
  rename (tmpfile, dst);
#undef COPY_ERROR
  return 0;
}

//////////////// merkle_node_disk /////////////////

merkle_node_disk::merkle_node_disk (FILE *internal, FILE *leaf, 
				    MERKLE_DISK_TYPE type, u_int32_t block_no) :
  merkle_node (),
  hashes (NULL),
  children (NULL),
  _internal (internal),
  _leaf (leaf), 
  _type (type),
  _block_no (block_no)
{
  if (isleaf()) {
    merkle_leaf_node leaf;
    int seekval = fseek (_leaf, _block_no*sizeof (merkle_leaf_node), SEEK_SET);
    assert (seekval == 0);
    fread (&leaf, sizeof (merkle_leaf_node), 1, _leaf);

    count = ntohl(leaf.key_count);
    for (uint i = 0; i < count; i++) {
      chordID c;
      mpz_set_rawmag_be (&c, leaf.keys[i].key, sizeof (leaf.keys[i].key));
      keylist.insert (New merkle_key (c));
    }
  } else {
    children = New array<u_int32_t, 64> ();
    hashes = New array<merkle_hash_id, 64> ();

    merkle_internal_node internal;
    int seekval = fseek (_internal, _block_no*sizeof (merkle_internal_node), 
			 SEEK_SET);
    assert (seekval == 0);
    fread (&internal, sizeof (merkle_internal_node), 1, _internal);

    count = ntohl(internal.key_count);
    for (uint i = 0; i < 64; i++) {
      mpz_set_rawmag_be (&((*hashes)[i].id), internal.hashes[i].key, 
			 sizeof(internal.hashes[i].key));
      (*hashes)[i].hash = to_merkle_hash ((*hashes)[i].id);
      (*children)[i] = ntohl (internal.child_pointers[i]);
    }
  }

  rehash ();
}

merkle_node_disk::~merkle_node_disk () 
{
  keylist.deleteall_correct();
  delete children;
  delete hashes;

  // also delete all children
  for (uint i = 0; i < to_delete.size(); i++) {
    delete to_delete[i];
  }

  // not necessary, but helps catch dangling pointers
  bzero (this, sizeof (*this)); 
}

void merkle_node_disk::rehash ()
{
  hash = 0;
  if (count == 0) {
    return;
  }
  
  sha1ctx sc;
  if (isleaf ()) {
    assert (count > 0 && count <= 64);
    merkle_key *k = keylist.first();
    while (k != NULL) {
      merkle_hash h = to_merkle_hash(k->id);
      sc.update (h.bytes, h.size);
      k = keylist.next(k);
    }
  } else {
    for (uint i = 0; i < 64; i++) {
      merkle_hash h = (*hashes)[i].hash;
      sc.update (h.bytes, h.size);
    }
  }
  sc.final (hash.bytes);
}

void
merkle_node_disk::write_out ()
{
  if (isleaf ()) {
    merkle_leaf_node leaf;
    leaf.key_count = htonl (count);
    merkle_key *m = keylist.first();
    int i = 0;
    while (m != NULL) {
      mpz_get_rawmag_be (leaf.keys[i].key, sizeof(leaf.keys[i].key), &(m->id));
      m = keylist.next (m);
      i++;
    }

    int seekval = fseek (_leaf, _block_no*sizeof(merkle_leaf_node), SEEK_SET);
    assert (seekval == 0);
    fwrite (&leaf, sizeof(merkle_leaf_node), 1, _leaf);
  } else {
    merkle_internal_node internal;
    internal.key_count = htonl(count);
    for (uint i = 0; i < 64; i++) {
      mpz_get_rawmag_be (internal.hashes[i].key, 
			 sizeof (internal.hashes[i].key), &((*hashes)[i].id));
      internal.child_pointers[i] = htonl ((*children)[i]);
      //      warn << _block_no << ") writing out child " << i << ") " << ((*children)[i] >> 1) << "\n";
    }

    int seekval = fseek (_internal, _block_no*sizeof(merkle_internal_node), 
			 SEEK_SET);
    assert (seekval == 0);
    fwrite (&internal, sizeof(merkle_internal_node), 1, _internal);
  }
}

merkle_node *
merkle_node_disk::child (u_int i)
{
  assert (!isleaf ());
  assert (i >= 0 && i < 64);

  u_int32_t pointer = (*children)[i];
  MERKLE_DISK_TYPE type;
  if (pointer % 2 == 0) {
    type = MERKLE_DISK_INTERNAL;
  } else {
    type = MERKLE_DISK_LEAF;
  }
  merkle_node_disk *n = New merkle_node_disk (_internal, _leaf, type, 
					      pointer >> 1);
  to_delete.push_back (n);
  return n;
}

merkle_hash
merkle_node_disk::child_hash (u_int i)
{
  assert (!isleaf ());
  assert (i >= 0 && i < 64);
  return (*hashes)[i].hash;
}

u_int32_t
merkle_node_disk::child_ptr (u_int i)
{
  assert (!isleaf ());
  assert (i >= 0 && i < 64);
  return (*children)[i];
}

void
merkle_node_disk::set_child (merkle_node_disk *n, u_int i)
{
  assert (!isleaf ());
  assert (i >= 0 && i < 64);

  (*children)[i] = ((n->get_block_no() << 1) | (n->isleaf()?0x00000001:0));
  (*hashes)[i].hash = n->hash;
  (*hashes)[i].id = tobigint(n->hash);
}

void
merkle_node_disk::add_key (chordID key)
{
  assert (isleaf () && count < 64);
  count++;
  merkle_key *m = New merkle_key (key);
  keylist.insert (m);
}

void
merkle_node_disk::add_key (merkle_hash key)
{
  assert (isleaf () && count < 64);
  count++;
  merkle_key *m = New merkle_key(key);
  keylist.insert(m);
}

void
merkle_node_disk::internal2leaf ()
{
  assert (!isleaf ());
  _type = MERKLE_DISK_LEAF;
  delete children;
  children = NULL;
  delete hashes;
  hashes = NULL;
}

bool
merkle_node_disk::isleaf () const {
  return (_type == MERKLE_DISK_LEAF);
}

void merkle_node_disk::leaf2internal () {
  assert (isleaf ());
  _type = MERKLE_DISK_INTERNAL;
  children = New array<u_int32_t, 64>();
  hashes = New array<merkle_hash_id, 64>();
  keylist.deleteall_correct();
  assert (!isleaf ());
}

//////////////// merkle_tree_disk /////////////////

static inline str
safe_fname (str rwname)
{
  strbuf roname ("%s.ro", rwname.cstr ());
  return roname;
}

void
merkle_tree_disk::init ()
{
  strbuf safe_index_name ("%s.ro", _index_name.cstr ());
  strbuf safe_leaf_name ("%s.ro", _leaf_name.cstr ());
  strbuf safe_internal_name ("%s.ro", _internal_name.cstr ());
  if (_writer) {
    _internal = open_file (_internal_name);
    _leaf = open_file (_leaf_name);
    _index = open_file (_index_name);
    // make sure the root is created correctly:
    merkle_node *n = get_root ();
    delete n;
    write_metadata ();
  } else {
    _internal = open_file (safe_fname (_internal_name));
    _leaf = open_file (safe_fname (_leaf_name));
    _index = open_file (safe_fname (_index_name));
  }
}

void
merkle_tree_disk::close ()<commit_msg>Remove unused variables in merkle_tree_disk::init ().<commit_after>#include <chord_types.h>
#include <id_utils.h>
#include <rxx.h>
#include "merkle_tree_disk.h"
#include "dhash_common.h"

static FILE *
open_file (str name) {
  // make all the parent directories if applicable
  vec<str> dirs;
  static const rxx dirsplit("\\/");
  (void) split (&dirs, dirsplit, name);
  strbuf filestrbuf;
  for (uint i = 0; i < dirs.size()-1; i++) {
    filestrbuf << dirs[i] << "/";
    
    str dirpath (filestrbuf);
    // check for existence, make if necessary
    struct stat st;
    if (stat (dirpath, &st) < 0) {
      if (mkdir(dirpath, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH) != 0) {
	fatal << "open_file: mkdir (" << dirpath << "): " << strerror (errno) << ".\n";
      }
    }
  }

  FILE *f = fopen (name, "r+");
  if (f == NULL) {
    f = fopen (name, "w+");
  }
  if (f == NULL) {
    fatal << "open_file: fopen (" << name << "): " << strerror (errno) << ".\n"; 
  }
  return f;
}

static int
copy_file (str src, str dst)
{
  int sfd = open (src, O_RDONLY);
  if (sfd < 0) {
    perror ("copy_file: src");
    return sfd;
  }

  str tmpfile = dst << "~";
  unlink (tmpfile);
  int dfd = open (tmpfile, O_CREAT|O_WRONLY, 0644);
  if (dfd < 0) {
    perror ("copy_file: dst");
    close (sfd);
    return dfd;
  }
#define COPY_ERROR(msg) \
  do {						  \
    int oerrno = errno;				  \
    warn ("copy_file: " msg ": %m");		  \
    close (sfd); close (dfd); unlink (tmpfile);	  \
    errno = oerrno;				  \
    return -1;					  \
  } while (0)

  char buf[8192];
  ssize_t nread;
  while ((nread = read (sfd, buf, sizeof (buf))) > 0) {
    ssize_t nwrote = write (dfd, buf, nread);
    if (nwrote < 0) {
      COPY_ERROR ("bad write");
    }
    if (nwrote != nread) {
      COPY_ERROR ("short write");
    }
  }
  if (nread < 0) {
    COPY_ERROR ("read error");
  }
  close (sfd); close (dfd);
  rename (tmpfile, dst);
#undef COPY_ERROR
  return 0;
}

//////////////// merkle_node_disk /////////////////

merkle_node_disk::merkle_node_disk (FILE *internal, FILE *leaf, 
				    MERKLE_DISK_TYPE type, u_int32_t block_no) :
  merkle_node (),
  hashes (NULL),
  children (NULL),
  _internal (internal),
  _leaf (leaf), 
  _type (type),
  _block_no (block_no)
{
  if (isleaf()) {
    merkle_leaf_node leaf;
    int seekval = fseek (_leaf, _block_no*sizeof (merkle_leaf_node), SEEK_SET);
    assert (seekval == 0);
    fread (&leaf, sizeof (merkle_leaf_node), 1, _leaf);

    count = ntohl(leaf.key_count);
    for (uint i = 0; i < count; i++) {
      chordID c;
      mpz_set_rawmag_be (&c, leaf.keys[i].key, sizeof (leaf.keys[i].key));
      keylist.insert (New merkle_key (c));
    }
  } else {
    children = New array<u_int32_t, 64> ();
    hashes = New array<merkle_hash_id, 64> ();

    merkle_internal_node internal;
    int seekval = fseek (_internal, _block_no*sizeof (merkle_internal_node), 
			 SEEK_SET);
    assert (seekval == 0);
    fread (&internal, sizeof (merkle_internal_node), 1, _internal);

    count = ntohl(internal.key_count);
    for (uint i = 0; i < 64; i++) {
      mpz_set_rawmag_be (&((*hashes)[i].id), internal.hashes[i].key, 
			 sizeof(internal.hashes[i].key));
      (*hashes)[i].hash = to_merkle_hash ((*hashes)[i].id);
      (*children)[i] = ntohl (internal.child_pointers[i]);
    }
  }

  rehash ();
}

merkle_node_disk::~merkle_node_disk () 
{
  keylist.deleteall_correct();
  delete children;
  delete hashes;

  // also delete all children
  for (uint i = 0; i < to_delete.size(); i++) {
    delete to_delete[i];
  }

  // not necessary, but helps catch dangling pointers
  bzero (this, sizeof (*this)); 
}

void merkle_node_disk::rehash ()
{
  hash = 0;
  if (count == 0) {
    return;
  }
  
  sha1ctx sc;
  if (isleaf ()) {
    assert (count > 0 && count <= 64);
    merkle_key *k = keylist.first();
    while (k != NULL) {
      merkle_hash h = to_merkle_hash(k->id);
      sc.update (h.bytes, h.size);
      k = keylist.next(k);
    }
  } else {
    for (uint i = 0; i < 64; i++) {
      merkle_hash h = (*hashes)[i].hash;
      sc.update (h.bytes, h.size);
    }
  }
  sc.final (hash.bytes);
}

void
merkle_node_disk::write_out ()
{
  if (isleaf ()) {
    merkle_leaf_node leaf;
    leaf.key_count = htonl (count);
    merkle_key *m = keylist.first();
    int i = 0;
    while (m != NULL) {
      mpz_get_rawmag_be (leaf.keys[i].key, sizeof(leaf.keys[i].key), &(m->id));
      m = keylist.next (m);
      i++;
    }

    int seekval = fseek (_leaf, _block_no*sizeof(merkle_leaf_node), SEEK_SET);
    assert (seekval == 0);
    fwrite (&leaf, sizeof(merkle_leaf_node), 1, _leaf);
  } else {
    merkle_internal_node internal;
    internal.key_count = htonl(count);
    for (uint i = 0; i < 64; i++) {
      mpz_get_rawmag_be (internal.hashes[i].key, 
			 sizeof (internal.hashes[i].key), &((*hashes)[i].id));
      internal.child_pointers[i] = htonl ((*children)[i]);
      //      warn << _block_no << ") writing out child " << i << ") " << ((*children)[i] >> 1) << "\n";
    }

    int seekval = fseek (_internal, _block_no*sizeof(merkle_internal_node), 
			 SEEK_SET);
    assert (seekval == 0);
    fwrite (&internal, sizeof(merkle_internal_node), 1, _internal);
  }
}

merkle_node *
merkle_node_disk::child (u_int i)
{
  assert (!isleaf ());
  assert (i >= 0 && i < 64);

  u_int32_t pointer = (*children)[i];
  MERKLE_DISK_TYPE type;
  if (pointer % 2 == 0) {
    type = MERKLE_DISK_INTERNAL;
  } else {
    type = MERKLE_DISK_LEAF;
  }
  merkle_node_disk *n = New merkle_node_disk (_internal, _leaf, type, 
					      pointer >> 1);
  to_delete.push_back (n);
  return n;
}

merkle_hash
merkle_node_disk::child_hash (u_int i)
{
  assert (!isleaf ());
  assert (i >= 0 && i < 64);
  return (*hashes)[i].hash;
}

u_int32_t
merkle_node_disk::child_ptr (u_int i)
{
  assert (!isleaf ());
  assert (i >= 0 && i < 64);
  return (*children)[i];
}

void
merkle_node_disk::set_child (merkle_node_disk *n, u_int i)
{
  assert (!isleaf ());
  assert (i >= 0 && i < 64);

  (*children)[i] = ((n->get_block_no() << 1) | (n->isleaf()?0x00000001:0));
  (*hashes)[i].hash = n->hash;
  (*hashes)[i].id = tobigint(n->hash);
}

void
merkle_node_disk::add_key (chordID key)
{
  assert (isleaf () && count < 64);
  count++;
  merkle_key *m = New merkle_key (key);
  keylist.insert (m);
}

void
merkle_node_disk::add_key (merkle_hash key)
{
  assert (isleaf () && count < 64);
  count++;
  merkle_key *m = New merkle_key(key);
  keylist.insert(m);
}

void
merkle_node_disk::internal2leaf ()
{
  assert (!isleaf ());
  _type = MERKLE_DISK_LEAF;
  delete children;
  children = NULL;
  delete hashes;
  hashes = NULL;
}

bool
merkle_node_disk::isleaf () const {
  return (_type == MERKLE_DISK_LEAF);
}

void merkle_node_disk::leaf2internal () {
  assert (isleaf ());
  _type = MERKLE_DISK_INTERNAL;
  children = New array<u_int32_t, 64>();
  hashes = New array<merkle_hash_id, 64>();
  keylist.deleteall_correct();
  assert (!isleaf ());
}

//////////////// merkle_tree_disk /////////////////

static inline str
safe_fname (str rwname)
{
  strbuf roname ("%s.ro", rwname.cstr ());
  return roname;
}

void
merkle_tree_disk::init ()
{
  if (_writer) {
    _internal = open_file (_internal_name);
    _leaf = open_file (_leaf_name);
    _index = open_file (_index_name);
    // make sure the root is created correctly:
    merkle_node *n = get_root ();
    delete n;
    write_metadata ();
  } else {
    _internal = open_file (safe_fname (_internal_name));
    _leaf = open_file (safe_fname (_leaf_name));
    _index = open_file (safe_fname (_index_name));
  }
}

void
merkle_tree_disk::close ()
{'fraction_non_alphanumeric': 0.17409987629402957, 'fraction_numerical': 0.012631030666058989, 'mean_word_length': 3.031496062992126, 'pattern_counts': {'":': 2, '<': 135, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6927580', 'n_tokens_mistral': 6336, 'n_tokens_neox': 6159, 'n_words': 1381}