<commit_before>#include <vector>
#include <iostream>
#include <algorithm>
#include <limits>

using namespace std;

#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <string.h>

#include "treehash/boosted-treehash.hh"
extern "C" {
#include "timers.h"
}

void fill_random(uint64_t * data, size_t n) {
  const int rfd = open("/dev/urandom", O_RDONLY);
  if (-1 == rfd) {
    const int err = errno;
    fprintf(stderr, "%s\n", strerror(err));
    exit(err);
  }
  char * const cdata = (char *)data;
  for (size_t i = 0; i < n; i += read(rfd, &cdata[i], sizeof(uint64_t)*n-i));
  (void)close(rfd);
}

uint64_t * alloc_random(size_t n) {
  uint64_t * ans = (uint64_t *)malloc(sizeof(uint64_t) * n);
  if (!ans) {
    fprintf(stderr, "Failed allocation of %zd words\n", n);
    exit(1);
  }
  fill_random(ans, n);
  return ans;
}

uint64_t sum = 0;

template<size_t N>
ticks bench(const void * r, const uint64_t * data, const size_t i) {
  const ticks start = startRDTSC();
  sum += boosted_treehash<N>(r, data, i);
  const ticks finish = startRDTSC();
  return finish-start;
}

int main() {
  size_t max_len = 2048;
  const uint64_t * const data = alloc_random(max_len);
  const void * const r64 = alloc_random(128);
  size_t iters = 10000;
  const size_t max_depth = 9;
  vector<vector<ticks> > cycles(max_depth, vector<ticks>(iters));
  size_t samples = 100;
  for (size_t i = 1; i < max_len; i = 1 + 1.01*i) {
    for (size_t j = 0; j < iters; ++j) {<commit_msg>Add comment at top of output explaining what the data is<commit_after>#include <vector>
#include <iostream>
#include <algorithm>
#include <limits>

using namespace std;

#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <string.h>

#include "treehash/boosted-treehash.hh"
extern "C" {
#include "timers.h"
}

void fill_random(uint64_t * data, size_t n) {
  const int rfd = open("/dev/urandom", O_RDONLY);
  if (-1 == rfd) {
    const int err = errno;
    fprintf(stderr, "%s\n", strerror(err));
    exit(err);
  }
  char * const cdata = (char *)data;
  for (size_t i = 0; i < n; i += read(rfd, &cdata[i], sizeof(uint64_t)*n-i));
  (void)close(rfd);
}

uint64_t * alloc_random(size_t n) {
  uint64_t * ans = (uint64_t *)malloc(sizeof(uint64_t) * n);
  if (!ans) {
    fprintf(stderr, "Failed allocation of %zd words\n", n);
    exit(1);
  }
  fill_random(ans, n);
  return ans;
}

uint64_t sum = 0;

template<size_t N>
ticks bench(const void * r, const uint64_t * data, const size_t i) {
  const ticks start = startRDTSC();
  sum += boosted_treehash<N>(r, data, i);
  const ticks finish = startRDTSC();
  return finish-start;
}

int main() {
  size_t max_len = 2048;
  const uint64_t * const data = alloc_random(max_len);
  const void * const r64 = alloc_random(128);
  size_t iters = 10000;
  const size_t max_depth = 9;
  vector<vector<ticks> > cycles(max_depth, vector<ticks>(iters));
  size_t samples = 100;
  cout << "# length percentile optimal-treeboost " << endl;
  for (size_t i = 1; i < max_len; i = 1 + 1.01*i) {
    for (size_t j = 0; j < iters; ++j) {
{'fraction_non_alphanumeric': 0.15931766977792083, 'fraction_numerical': 0.029610556807209528, 'mean_word_length': 3.925515055467512, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5754201', 'n_tokens_mistral': 1315, 'n_tokens_neox': 1245, 'n_words': 333}