<commit_before>#include <algorithm>
#include <chrono>
#include <unistd.h>
#include <vector>

#include <jtree.h>
#include <stdafx.h>

int main(int argc, char* argv[]) {
  char const *output_filename = "";

  bool verbose = false;
  bool do_faqs = false;
  bool do_distr = false;

  opterr = 0;
  int opt;
  while ((opt = getopt(argc, argv, "o:vfd")) != -1) {
    switch (opt) {
      case 'o':
        output_filename = optarg;
        break;
      case 'v':
        verbose = !verbose;
        break;
      case 'f':
        do_faqs = !do_faqs;
        break;
      case 'd':
        do_distr = !do_distr;
        break;
      case '?':
        if (optopt == 'o')
          printf("Option -%c requires a string.\n", optopt);
        else
          printf("Unknown option character '\\x%x'.\n", optopt);
        return 1;
      default:
        abort();
    }
  }

  if (optind + 1 >= argc) {
    printf("USAGE: merge_trees [options ...] first.tree second.tree ...\n");
    return 1;
  }
   
  auto start_point = std::chrono::steady_clock::now();

  JNodeTable lhs(argv[optind]);
  JNodeTable rhs(argv[optind + 1]);

  auto load_duration = std::chrono::duration_cast<std::chrono::milliseconds>(
      std::chrono::steady_clock::now() - start_point);
  if (verbose) printf("Loaded in: %lums\n", load_duration.count());

  JNodeTable jnodes = strcmp(output_filename, "") == 0 ?
    JNodeTable(lhs.size(), false, 0) :
    JNodeTable(output_filename, lhs.size(), false, 0);
  jnodes.merge(lhs, rhs, false);

  if (do_faqs) {
    JNodeTable::Facts faq = jnodes.getFacts();
    faq.print();
    if (do_distr)
      faq.print_distribution();
  }

  auto run_duration = std::chrono::duration_cast<std::chrono::milliseconds>(
      std::chrono::steady_clock::now() - start_point);
  if (verbose) printf("Finished in: %lums\n", run_duration.count());

  auto build_duration = std::chrono::duration_cast<std::chrono::milliseconds>(
      std::chrono::steady_clock::now() - start_point);
  if (verbose) printf("Built in: %lums\n", build_duration.count());

  return 0;
}
<commit_msg>fixed a string typo in merge_trees.cpp<commit_after>#include <algorithm>
#include <chrono>
#include <unistd.h>
#include <vector>

#include <jtree.h>
#include <stdafx.h>

int main(int argc, char* argv[]) {
  char const *output_filename = "";

  bool verbose = false;
  bool do_faqs = false;
  bool do_distr = false;

  opterr = 0;
  int opt;
  while ((opt = getopt(argc, argv, "o:vfd")) != -1) {
    switch (opt) {
      case 'o':
        output_filename = optarg;
        break;
      case 'v':
        verbose = !verbose;
        break;
      case 'f':
        do_faqs = !do_faqs;
        break;
      case 'd':
        do_distr = !do_distr;
        break;
      case '?':
        if (optopt == 'o')
          printf("Option -%c requires a string.\n", optopt);
        else
          printf("Unknown option character '\\x%x'.\n", optopt);
        return 1;
      default:
        abort();
    }
  }

  if (optind + 1 >= argc) {
    printf("USAGE: merge_trees [options ...] first.tree second.tree\n");
    return 1;
  }
   
  auto start_point = std::chrono::steady_clock::now();

  JNodeTable lhs(argv[optind]);
  JNodeTable rhs(argv[optind + 1]);

  auto load_duration = std::chrono::duration_cast<std::chrono::milliseconds>(
      std::chrono::steady_clock::now() - start_point);
  if (verbose) printf("Loaded in: %lums\n", load_duration.count());

  JNodeTable jnodes = strcmp(output_filename, "") == 0 ?
    JNodeTable(lhs.size(), false, 0) :
    JNodeTable(output_filename, lhs.size(), false, 0);
  jnodes.merge(lhs, rhs, false);

  if (do_faqs) {
    JNodeTable::Facts faq = jnodes.getFacts();
    faq.print();
    if (do_distr)
      faq.print_distribution();
  }

  auto run_duration = std::chrono::duration_cast<std::chrono::milliseconds>(
      std::chrono::steady_clock::now() - start_point);
  if (verbose) printf("Finished in: %lums\n", run_duration.count());

  auto build_duration = std::chrono::duration_cast<std::chrono::milliseconds>(
      std::chrono::steady_clock::now() - start_point);
  if (verbose) printf("Built in: %lums\n", build_duration.count());

  return 0;
}

{'fraction_non_alphanumeric': 0.1643835616438356, 'fraction_numerical': 0.004806536890170632, 'mean_word_length': 3.060487804878049, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3167193', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1469, 'n_words': 322}