<commit_before>#include <iostream>
#include <fstream>
#include <string>
#include <boost/archive/xml_iarchive.hpp>
#include <boost/program_options.hpp>

#include "Regression.h"

static std::string SourceDir;
static std::string DestinationDir;
static std::string ParametersFile;
static float Epsilon;
static boost::optional<bool> UseCuda;

static std::string ParseOptions(int argc, char **argv)
{
  using namespace boost::program_options;
  std::string mode;
  
  options_description all_desc("Mode options");
  all_desc.add_options()
    ("gen-ref", "Generate reference results from images in the source directory")
    ("gen-test", "Generate test results based on settings from reference results in the source directory")
    ("compare", "Compare reference results in the source directory with results in the destination directory")
    ("use-cuda", value<bool>()->notifier([](bool v) { UseCuda = v; }),
      "Overrides implementation specified by parameters")
    ("help", "Print help");
  
  options_description data_desc("Data specification options");
  data_desc.add_options()
    ("src-dir", value<std::string>(&SourceDir), "Source directory")
    ("dst-dir", value<std::string>(&DestinationDir), "Destination directory [WARNING: current contents will be lost]")
    ("parameters", value<std::string>(&ParametersFile), "Detection parameters file")
    ("epsilon", value<float>(&Epsilon)->default_value(0.5f), "Position tolerance for x/y coordinates");
  
  all_desc.add(data_desc);
  
  variables_map vm;
  store(parse_command_line(argc, argv, all_desc), vm);
  
  if (vm.count("help")) {
    std::cout << all_desc << std::endl;
    std::cout << "WARNING: 'generate' modes overwrite data in the destination directory!" << std::endl;
    std::cout << std::endl;
    exit(0);
  }
  
  if (vm.count("gen-ref")) {
    if (!vm.count("src-dir") || !vm.count("dst-dir") || !vm.count("parameters"))
      throw error("generate-reference: input-dir, output-dir and parameters are mandatory");
    mode = "gen-ref";
  }
  
  if (vm.count("gen-test")) {
    if (!mode.empty())
      throw error("only one mode option can be specified");
    if (vm.count("parameters"))
      throw error("Cannot specify parameters for gen-test");
    if (!vm.count("src-dir") || !vm.count("dst-dir"))
      throw error("gen-test: src-dir and dst-dir are mandatory");
    mode = "gen-test";
  }
  
  if (vm.count("compare")) {
    if (!mode.empty())
      throw error("only one mode option can be specified");
    if (vm.count("parameters"))
      throw error("Cannot specify parameters for compare");
    if (!vm.count("src-dir") || !vm.count("dst-dir"))
      throw error("check: src-dir and dst-dir are mandatory");
    mode = "compare";
  }
  
  if (mode.empty())
    throw error("exactly one mode option must be specified");
  
  notify(vm);
  return mode;
}

static void GenerateReference()
{
  TestRunner testRunner(SourceDir, DestinationDir, UseCuda);
  cctag::Parameters parameters;
  
  {
    std::ifstream ifs(ParametersFile);
    boost::archive::xml_iarchive ia(ifs);
    ia >> boost::serialization::make_nvp("CCTagsParams", parameters);
  }

  testRunner.generateReferenceResults(parameters);
}

static bool ReportChecks()
{
  TestChecker testChecker(SourceDir, DestinationDir, Epsilon);
  bool ok = testChecker.check();
  
  if (ok) std::clog << "All checks PASSED" << std::endl;
  else std::clog << "Some checks FAILED" << std::endl;
  
  std::clog << "Performance difference report:\n";
  std::clog << "  time,    mean=" << testChecker.elapsedTimeDifferenceMean() << ",stdev=" << testChecker.elapsedTimeDifferenceStdev() << std::endl;
  std::clog << "  quality, mean=" << testChecker.qualityDifferenceMean() << ",stdev=" << testChecker.qualityDifferenceStdev() << std::endl;
}

int main(int argc, char **argv)
{
  try {
    auto mode = ParseOptions(argc, argv);
    
    if (UseCuda) std::clog << "CUDA override SET to: " << *UseCuda << std::endl;
    else std::clog << "CUDA override NOT SET; will use parameters" << std::endl;
    
    if (mode == "gen-ref") {
      GenerateReference();
      return 0;
    }
    
    if (mode == "gen-test") {
      TestRunner testRunner(SourceDir, DestinationDir, UseCuda);<commit_msg>Hyperthreading fix: set # of omp threads to #cpus/2.<commit_after>#include <iostream>
#include <fstream>
#include <string>
#include <boost/archive/xml_iarchive.hpp>
#include <boost/program_options.hpp>
#include <omp.h>
#include "Regression.h"

static std::string SourceDir;
static std::string DestinationDir;
static std::string ParametersFile;
static float Epsilon;
static boost::optional<bool> UseCuda;

static std::string ParseOptions(int argc, char **argv)
{
  using namespace boost::program_options;
  std::string mode;
  
  options_description all_desc("Mode options");
  all_desc.add_options()
    ("gen-ref", "Generate reference results from images in the source directory")
    ("gen-test", "Generate test results based on settings from reference results in the source directory")
    ("compare", "Compare reference results in the source directory with results in the destination directory")
    ("use-cuda", value<bool>()->notifier([](bool v) { UseCuda = v; }),
      "Overrides implementation specified by parameters")
    ("help", "Print help");
  
  options_description data_desc("Data specification options");
  data_desc.add_options()
    ("src-dir", value<std::string>(&SourceDir), "Source directory")
    ("dst-dir", value<std::string>(&DestinationDir), "Destination directory [WARNING: current contents will be lost]")
    ("parameters", value<std::string>(&ParametersFile), "Detection parameters file")
    ("epsilon", value<float>(&Epsilon)->default_value(0.5f), "Position tolerance for x/y coordinates");
  
  all_desc.add(data_desc);
  
  variables_map vm;
  store(parse_command_line(argc, argv, all_desc), vm);
  
  if (vm.count("help")) {
    std::cout << all_desc << std::endl;
    std::cout << "WARNING: 'generate' modes overwrite data in the destination directory!" << std::endl;
    std::cout << std::endl;
    exit(0);
  }
  
  if (vm.count("gen-ref")) {
    if (!vm.count("src-dir") || !vm.count("dst-dir") || !vm.count("parameters"))
      throw error("generate-reference: input-dir, output-dir and parameters are mandatory");
    mode = "gen-ref";
  }
  
  if (vm.count("gen-test")) {
    if (!mode.empty())
      throw error("only one mode option can be specified");
    if (vm.count("parameters"))
      throw error("Cannot specify parameters for gen-test");
    if (!vm.count("src-dir") || !vm.count("dst-dir"))
      throw error("gen-test: src-dir and dst-dir are mandatory");
    mode = "gen-test";
  }
  
  if (vm.count("compare")) {
    if (!mode.empty())
      throw error("only one mode option can be specified");
    if (vm.count("parameters"))
      throw error("Cannot specify parameters for compare");
    if (!vm.count("src-dir") || !vm.count("dst-dir"))
      throw error("check: src-dir and dst-dir are mandatory");
    mode = "compare";
  }
  
  if (mode.empty())
    throw error("exactly one mode option must be specified");
  
  notify(vm);
  return mode;
}

static void GenerateReference()
{
  TestRunner testRunner(SourceDir, DestinationDir, UseCuda);
  cctag::Parameters parameters;
  
  {
    std::ifstream ifs(ParametersFile);
    boost::archive::xml_iarchive ia(ifs);
    ia >> boost::serialization::make_nvp("CCTagsParams", parameters);
  }

  testRunner.generateReferenceResults(parameters);
}

static bool ReportChecks()
{
  TestChecker testChecker(SourceDir, DestinationDir, Epsilon);
  bool ok = testChecker.check();
  
  if (ok) std::clog << "All checks PASSED" << std::endl;
  else std::clog << "Some checks FAILED" << std::endl;
  
  std::clog << "Performance difference report:\n";
  std::clog << "  time,    mean=" << testChecker.elapsedTimeDifferenceMean() << ",stdev=" << testChecker.elapsedTimeDifferenceStdev() << std::endl;
  std::clog << "  quality, mean=" << testChecker.qualityDifferenceMean() << ",stdev=" << testChecker.qualityDifferenceStdev() << std::endl;
}

int main(int argc, char **argv)
{
  try {
    auto mode = ParseOptions(argc, argv);
    
    if (UseCuda) std::clog << "CUDA override SET to: " << *UseCuda << std::endl;
    else std::clog << "CUDA override NOT SET; will use parameters" << std::endl;
    
    {
      // Disable hyperthreading
      int num_procs = omp_get_num_procs() / 2;
      omp_set_num_threads(num_procs);
      std::clog << "OMP #threads set to: " << num_procs << std::endl;
    }
    
    if (mode == "gen-ref") {
      GenerateReference();
      return 0;
    }
    
    if (mode == "gen-test") {
      TestRunner testRunner(SourceDir, DestinationDir, UseCuda);
{'fraction_non_alphanumeric': 0.15648371878955242, 'fraction_numerical': 0.0011506155793349442, 'mean_word_length': 4.198564593301436, 'pattern_counts': {'":': 0, '<': 132, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '349684', 'n_tokens_mistral': 2695, 'n_tokens_neox': 2701, 'n_words': 708}