<commit_before>#define BLOCKSIZE 16384
// supports files up to (BLOCKSIZE-260)/20 * BLOCKSIZE/20 * BLOCKSIZE
// currently:  10815307776 bytes

#include <chord.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <string.h>
#include <stdio.h>
#include <dhash.h>
#include <dhash_common.h>
#include <dhashclient.h>

dhashclient *dhash;
int inflight = 0;
FILE *outfile;

// utility functions ----------------------------------

chordID compute_hash(char *buf, int len) {
  char hashbytes[sha1::hashsize];
  chordID ID;

  sha1_hash (hashbytes, buf, len);
  mpz_set_rawmag_be (&ID, hashbytes, sizeof (hashbytes));  // For big endian

  return ID;
}

void insert_cb(dhash_stat s, ptr<insert_info>) {
  if(s != DHASH_OK)
    warn << "bad store\n";

  inflight--;
  if(inflight == 0)
    exit(0);
}

chordID write_block(char *buf, int len) {
  chordID ID = compute_hash(buf, len);
  dhash->insert (ID, buf, len, wrap(&insert_cb));
  inflight++;

  return ID;
}


// indirect block -----------------------------------

struct indirect {
  vec<chordID> hs;
  indirect *parent;

  void add_hash(chordID h) {
    hs.push_back(h);

    if(full()) {
      // write out indirect block
      write_out();
    }
  }

  virtual void print(char *buf, int l) {
    if(len() > l) {
      warnx << len() << "\n";
      fatal("buf too small\n");
    }

    for(unsigned int i=0; i<hs.size(); i++) {
      mpz_get_raw (buf, sha1::hashsize, &hs[i]);
      buf += sha1::hashsize;
    }
  }

  chordID write_out(void) {
    chordID ID;
    char buf[BLOCKSIZE];
    
    if(len() == 0)
      return 0; // xxx bug for max-sized file?

    print(buf, BLOCKSIZE);
    warnx << "len " << len() << "\n";
    ID = write_block(buf, len());
    clear();

    if(parent) {
      warnx << "p\n";
      parent->add_hash(ID);
    }

    return ID;
  }

  void clear(void) {
    hs.clear();
  }

  virtual int len(void) {
    return hs.size() * sha1::hashsize;
  }

  bool full(void) {
    return (len() + sha1::hashsize) > BLOCKSIZE;
  }

  indirect(indirect *p) : parent(p) {}
  virtual ~indirect() {}
};


// inode block -------------------------------------

struct inode : indirect {
  char filename[256];
  int filelen;
  static const int extralen = sizeof(inode::filename) + sizeof(inode::filelen);

  inode(char *aname, int alen) : indirect(NULL) {
    strncpy(filename, aname, sizeof(filename));
    filelen = alen;
  }

  int len(void) {
    return indirect::len() + extralen;
  }

  void print(char *buf, int l) {
    if(len() > l) {
      warnx << len() << "\n";
      fatal("buf too small\n");
    }

    memcpy(buf, filename, sizeof(filename));
    buf += sizeof(filename);
    memcpy(buf, &filelen, sizeof(filelen));
    buf += sizeof(filelen);

    indirect::print(buf, l - extralen);
  }

  void write_out(void) {
    warnx << indirect::write_out() << "\n";
  }
};


// callback for printing out inode info --------------------------

void list_cb(dhash_stat st, ptr<dhash_block> bl, vec<chordID> vc) {
  if(st != DHASH_OK)
    fatal("lost inode\n");

  str fns(bl->data, 256);
  char filename[256];
  strcpy(filename, fns);
  int filelen;
  memcpy(&filelen, bl->data+256, sizeof(filelen));

  warnx << filename << ": " << filelen << " bytes\n";
  exit(0);
}


// a chain of callbacks for retrieving indirect and file data blocks,
// to retrieve the file ----------------------------------------------

void gotblock_cb(int len, dhash_stat st, ptr<dhash_block> bl,
		 vec<chordID> vc) {
  if(st != DHASH_OK) {
    warnx << "at " << len;
    fatal(" lost block\n");
  }

  if(fseek(outfile, len, SEEK_SET) != 0)
    fatal("fseek failure\n");
  if(fwrite(bl->data, 1, bl->len, outfile) != bl->len)
    fatal("write failure\n");

  inflight--;
  if(inflight == 0)
    exit(0);
}

void gotindirect_cb(int len, dhash_stat st, ptr<dhash_block> bl,
		    vec<chordID> vc) {
  if(st != DHASH_OK) {
    warnx << "at " << len;
    fatal(" lost indirect\n");
  }

  char *buf = bl->data;
  chordID ID;
  for(unsigned int i=0; i<bl->len; i+=20) {
    mpz_set_rawmag_be (&ID, buf+i, sha1::hashsize);  // For big endian
    dhash->retrieve(ID, wrap(&gotblock_cb, len+BLOCKSIZE*i/20));
    warnx << "retrieve " << ID << "\n";
    inflight++;
  }
}

void gotinode_cb(dhash_stat st, ptr<dhash_block> bl, vec<chordID> vc) {
  if(st != DHASH_OK)
    fatal("lost inode\n");

  str fns(bl->data, 256);
  char filename[256];
  strcpy(filename, fns);
  int filelen;
  memcpy(&filelen, bl->data+256, sizeof(filelen));

  outfile = fopen(filename, "w");
  if(outfile == NULL)
    fatal("can't open file for writing\n");

  char *buf = bl->data+inode::extralen;
  chordID ID;
  for(unsigned int i=0; i<(bl->len-inode::extralen); i+=20) {
    mpz_set_rawmag_be(&ID, buf+i, sha1::hashsize);  // For big endian
    dhash->retrieve(ID, wrap(&gotindirect_cb, (BLOCKSIZE/20)*BLOCKSIZE*i/20));
    warnx << "retrieve " << ID << "\n";
  }
}

void store(char *name) {
  struct stat st;
  if(stat(name, &st) == -1)
    fatal("couldn't stat file\n");<commit_msg>fix to build with newer gcc<commit_after>#define BLOCKSIZE 16384
// supports files up to (BLOCKSIZE-260)/20 * BLOCKSIZE/20 * BLOCKSIZE
// currently:  10815307776 bytes

#include <chord.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <string.h>
#include <stdio.h>
#include <dhash.h>
#include <dhash_common.h>
#include <dhashclient.h>

dhashclient *dhash;
int inflight = 0;
FILE *outfile;

// utility functions ----------------------------------

chordID compute_hash(char *buf, int len) {
  char hashbytes[sha1::hashsize];
  chordID ID;

  sha1_hash (hashbytes, buf, len);
  mpz_set_rawmag_be (&ID, hashbytes, sizeof (hashbytes));  // For big endian

  return ID;
}

void insert_cb(dhash_stat s, ptr<insert_info>) {
  if(s != DHASH_OK)
    warn << "bad store\n";

  inflight--;
  if(inflight == 0)
    exit(0);
}

chordID write_block(char *buf, int len) {
  chordID ID = compute_hash(buf, len);
  dhash->insert (ID, buf, len, wrap(&insert_cb));
  inflight++;

  return ID;
}


// indirect block -----------------------------------

struct indirect {
  vec<chordID> hs;
  indirect *parent;

  void add_hash(chordID h) {
    hs.push_back(h);

    if(full()) {
      // write out indirect block
      write_out();
    }
  }

  virtual void print(char *buf, int l) {
    if(len() > l) {
      warnx << len() << "\n";
      fatal("buf too small\n");
    }

    for(unsigned int i=0; i<hs.size(); i++) {
      mpz_get_raw (buf, sha1::hashsize, &hs[i]);
      buf += sha1::hashsize;
    }
  }

  chordID write_out(void) {
    chordID ID;
    char buf[BLOCKSIZE];
    
    if(len() == 0)
      return 0; // xxx bug for max-sized file?

    print(buf, BLOCKSIZE);
    warnx << "len " << len() << "\n";
    ID = write_block(buf, len());
    clear();

    if(parent) {
      warnx << "p\n";
      parent->add_hash(ID);
    }

    return ID;
  }

  void clear(void) {
    hs.clear();
  }

  virtual int len(void) {
    return hs.size() * sha1::hashsize;
  }

  bool full(void) {
    return (len() + sha1::hashsize) > BLOCKSIZE;
  }

  indirect(indirect *p) : parent(p) {}
  virtual ~indirect() {}
};


// inode block -------------------------------------

struct inode : indirect {
  char filename[256];
  int filelen;
  int extralen;

  inode(char *aname, int alen) : indirect(NULL) {
    strncpy(filename, aname, sizeof(filename));
    filelen = alen;
    extralen = sizeof(filename) + sizeof(filelen);
  }

  int len(void) {
    return indirect::len() + extralen;
  }

  void print(char *buf, int l) {
    if(len() > l) {
      warnx << len() << "\n";
      fatal("buf too small\n");
    }

    memcpy(buf, filename, sizeof(filename));
    buf += sizeof(filename);
    memcpy(buf, &filelen, sizeof(filelen));
    buf += sizeof(filelen);

    indirect::print(buf, l - extralen);
  }

  void write_out(void) {
    warnx << indirect::write_out() << "\n";
  }
};


// callback for printing out inode info --------------------------

void list_cb(dhash_stat st, ptr<dhash_block> bl, vec<chordID> vc) {
  if(st != DHASH_OK)
    fatal("lost inode\n");

  str fns(bl->data, 256);
  char filename[256];
  strcpy(filename, fns);
  int filelen;
  memcpy(&filelen, bl->data+256, sizeof(filelen));

  warnx << filename << ": " << filelen << " bytes\n";
  exit(0);
}


// a chain of callbacks for retrieving indirect and file data blocks,
// to retrieve the file ----------------------------------------------

void gotblock_cb(int len, dhash_stat st, ptr<dhash_block> bl,
		 vec<chordID> vc) {
  if(st != DHASH_OK) {
    warnx << "at " << len;
    fatal(" lost block\n");
  }

  if(fseek(outfile, len, SEEK_SET) != 0)
    fatal("fseek failure\n");
  if(fwrite(bl->data, 1, bl->len, outfile) != bl->len)
    fatal("write failure\n");

  inflight--;
  if(inflight == 0)
    exit(0);
}

void gotindirect_cb(int len, dhash_stat st, ptr<dhash_block> bl,
		    vec<chordID> vc) {
  if(st != DHASH_OK) {
    warnx << "at " << len;
    fatal(" lost indirect\n");
  }

  char *buf = bl->data;
  chordID ID;
  for(unsigned int i=0; i<bl->len; i+=20) {
    mpz_set_rawmag_be (&ID, buf+i, sha1::hashsize);  // For big endian
    dhash->retrieve(ID, wrap(&gotblock_cb, len+BLOCKSIZE*i/20));
    warnx << "retrieve " << ID << "\n";
    inflight++;
  }
}

void gotinode_cb(dhash_stat st, ptr<dhash_block> bl, vec<chordID> vc) {
  if(st != DHASH_OK)
    fatal("lost inode\n");

  str fns(bl->data, 256);
  char filename[256];
  strcpy(filename, fns);
  int filelen;
  memcpy(&filelen, bl->data+256, sizeof(filelen));

  outfile = fopen(filename, "w");
  if(outfile == NULL)
    fatal("can't open file for writing\n");

  char *buf = bl->data+260;
  chordID ID;
  for(unsigned int i=0; i<(bl->len-260); i+=20) {
    mpz_set_rawmag_be(&ID, buf+i, sha1::hashsize);  // For big endian
    dhash->retrieve(ID, wrap(&gotindirect_cb, (BLOCKSIZE/20)*BLOCKSIZE*i/20));
    warnx << "retrieve " << ID << "\n";
  }
}

void store(char *name) {
  struct stat st;
  if(stat(name, &st) == -1)
    fatal("couldn't stat file\n");
{'fraction_non_alphanumeric': 0.20316411334735154, 'fraction_numerical': 0.015820566736757784, 'mean_word_length': 2.7282568122433744, 'pattern_counts': {'":': 2, '<': 145, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5477353', 'n_tokens_mistral': 3906, 'n_tokens_neox': 3711, 'n_words': 881}