<commit_before>#include <cstdlib>
#include <iostream>
#include <ios>

#include "hext/parser.h"
#include "hext/matcher.h"
#include "hext/print.h"


int main(int argc, char ** argv)
{
  std::ios_base::sync_with_stdio(false);

  // todo: boost::program_options
  if( argc < 3 )
  {
    std::cout << "Usage: " << argv[0]
              << " [rules.hext] [target.html]" << std::endl;
    return EXIT_SUCCESS;
  }

  try
  {
    auto rules = hext::parser::parse_file(argv[1]);
    hext::matcher m(argv[2]);

    for(const auto& r : rules)
    {
      hext::print_rule(r);
      std::unique_ptr<hext::match_tree> mt = m.match(&r);
      assert(mt.get() != nullptr);
      //hext::print_match_tree(mt.get());
      mt->to_json(std::cout);
    }
  }
  catch( std::ios_base::failure& e )
  {
    std::cerr << e.what() << '\n';
    return EXIT_FAILURE;
  }

  return EXIT_SUCCESS;
}
<commit_msg>removed redundant unique_ptr::get in assert<commit_after>#include <cstdlib>
#include <iostream>
#include <ios>

#include "hext/parser.h"
#include "hext/matcher.h"
#include "hext/print.h"


int main(int argc, char ** argv)
{
  std::ios_base::sync_with_stdio(false);

  // todo: boost::program_options
  if( argc < 3 )
  {
    std::cout << "Usage: " << argv[0]
              << " [rules.hext] [target.html]" << std::endl;
    return EXIT_SUCCESS;
  }

  try
  {
    auto rules = hext::parser::parse_file(argv[1]);
    hext::matcher m(argv[2]);

    for(const auto& r : rules)
    {
      hext::print_rule(r);
      std::unique_ptr<hext::match_tree> mt = m.match(&r);
      assert(mt != nullptr);
      //hext::print_match_tree(mt.get());
      mt->to_json(std::cout);
    }
  }
  catch( std::ios_base::failure& e )
  {
    std::cerr << e.what() << '\n';
    return EXIT_FAILURE;
  }

  return EXIT_SUCCESS;
}

{'fraction_non_alphanumeric': 0.188268156424581, 'fraction_numerical': 0.004469273743016759, 'mean_word_length': 3.1267281105990783, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '707937', 'n_tokens_mistral': 712, 'n_tokens_neox': 694, 'n_words': 122}