<commit_before>// Copyright 2015 <NAME>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "Parser.h"

#include "hext/AppendPipe.h"
#include "hext/AttributeCapture.h"
#include "hext/AttributeCountMatch.h"
#include "hext/AttributeMatch.h"
#include "hext/BeginsWithTest.h"
#include "hext/Builtins.h"
#include "hext/Capture.h"
#include "hext/CasePipe.h"
#include "hext/ChildCountMatch.h"
#include "hext/CollapseWsPipe.h"
#include "hext/ContainsTest.h"
#include "hext/ContainsWordsTest.h"
#include "hext/EndsWithTest.h"
#include "hext/EqualsTest.h"
#include "hext/FunctionCapture.h"
#include "hext/FunctionValueMatch.h"
#include "hext/Match.h"
#include "hext/NegateMatch.h"
#include "hext/NegateTest.h"
#include "hext/NthChildMatch.h"
#include "hext/OnlyChildMatch.h"
#include "hext/PrependPipe.h"
#include "hext/RegexPipe.h"
#include "hext/RegexReplacePipe.h"
#include "hext/RegexTest.h"
#include "hext/TrimPipe.h"
#include "hext/ValueTest.h"

#include "PatternValues.h"
#include "StringUtil.h"

#include <vector>
#include <sstream>
#include <utility>
#include <algorithm>

#include <gumbo.h>


namespace hext {


/// The ragel namespace holds ragel's static data.
namespace ragel {
  /// Embed the ragel state machine.
  %%{
    machine hext;
    include "hext-machine.rl";
    write data;
  }%%
} // namespace ragel


Parser::Parser(const char * begin, const char * end) noexcept
: rule_stack_()
, top_rule_(nullptr)
, p_begin_(begin)
, p(begin)
, pe(end)
, eof(end)
, cs(0)
{
}

Parser::~Parser() = default;
Parser::Parser(Parser&&) = default;
Parser& Parser::operator=(Parser&&) = default;

Rule Parser::parse()
{
  // Allow ragel to access its namespace.
  using namespace ragel;

  // Hide clang's warning -Wunused-const-variable.
  (void)hext_first_final;
  (void)hext_error;
  (void)hext_en_main;

  // When calling Parser::parse repeatedly, ensure we are always in a valid
  // state.
  this->p = this->p_begin_;
  this->rule_stack_.clear();
  this->top_rule_ = nullptr;

  // All values required to construct Matches and Captures.
  PatternValues pv;

  const char * tok_begin = nullptr;
  const char * tok_end = nullptr;
  std::string tok = "";

  // Convenience lambda to store the start of a token. Used within the hext
  // machine definition.
  auto tk_start = [this, &tok_begin, &tok_end](){
    tok_begin = this->p;
    tok_end = nullptr;
  };

  // Convenience lambda to complete a token. Used within the hext
  // machine definition.
  auto tk_stop = [this, &tok, &tok_begin, &tok_end](){
    assert(tok_begin);
    assert(this->p);
    tok_end = this->p;
    tok = std::string(
      tok_begin,
      static_cast<std::string::size_type>(std::distance(tok_begin, tok_end))
    );
  };

  // Hide warnings generated by ragel code.
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-conversion"
#pragma GCC diagnostic ignored "-Wold-style-cast"
#pragma GCC diagnostic ignored "-Wpragmas"
#pragma GCC diagnostic ignored "-Wunreachable-code-break"
  %%{
    machine hext;
    include "hext-machine.rl";
    write init;
    write exec;
  }%%
#pragma GCC diagnostic pop

  // Throw error if there are missing closing tags.
  if( this->rule_stack_.size() )
    this->throw_missing_tag(this->rule_stack_.back().get_tag());

  if( this->top_rule_ )
  {
    return *this->top_rule_;
  }
  else
  {
    return Rule();
  }
}

Rule& Parser::cur_rule()
{
  assert(this->rule_stack_.size());
  return this->rule_stack_.back();
}

void Parser::push_rule()
{
  this->rule_stack_.emplace_back();
}
<commit_msg>libhext: Parser: explicitly move parsed Rule to caller to avoid copy<commit_after>// Copyright 2015, 2016 <NAME>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "Parser.h"

#include "hext/AppendPipe.h"
#include "hext/AttributeCapture.h"
#include "hext/AttributeCountMatch.h"
#include "hext/AttributeMatch.h"
#include "hext/BeginsWithTest.h"
#include "hext/Builtins.h"
#include "hext/Capture.h"
#include "hext/CasePipe.h"
#include "hext/ChildCountMatch.h"
#include "hext/CollapseWsPipe.h"
#include "hext/ContainsTest.h"
#include "hext/ContainsWordsTest.h"
#include "hext/EndsWithTest.h"
#include "hext/EqualsTest.h"
#include "hext/FunctionCapture.h"
#include "hext/FunctionValueMatch.h"
#include "hext/Match.h"
#include "hext/NegateMatch.h"
#include "hext/NegateTest.h"
#include "hext/NthChildMatch.h"
#include "hext/OnlyChildMatch.h"
#include "hext/PrependPipe.h"
#include "hext/RegexPipe.h"
#include "hext/RegexReplacePipe.h"
#include "hext/RegexTest.h"
#include "hext/TrimPipe.h"
#include "hext/ValueTest.h"

#include "PatternValues.h"
#include "StringUtil.h"

#include <vector>
#include <sstream>
#include <utility>
#include <algorithm>

#include <gumbo.h>


namespace hext {


/// The ragel namespace holds ragel's static data.
namespace ragel {
  /// Embed the ragel state machine.
  %%{
    machine hext;
    include "hext-machine.rl";
    write data;
  }%%
} // namespace ragel


Parser::Parser(const char * begin, const char * end) noexcept
: rule_stack_()
, top_rule_(nullptr)
, p_begin_(begin)
, p(begin)
, pe(end)
, eof(end)
, cs(0)
{
}

Parser::~Parser() = default;
Parser::Parser(Parser&&) = default;
Parser& Parser::operator=(Parser&&) = default;

Rule Parser::parse()
{
  // Allow ragel to access its namespace.
  using namespace ragel;

  // Hide clang's warning -Wunused-const-variable.
  (void)hext_first_final;
  (void)hext_error;
  (void)hext_en_main;

  // When calling Parser::parse repeatedly, ensure we are always in a valid
  // state.
  this->p = this->p_begin_;
  this->rule_stack_.clear();
  this->top_rule_ = nullptr;

  // All values required to construct Matches and Captures.
  PatternValues pv;

  const char * tok_begin = nullptr;
  const char * tok_end = nullptr;
  std::string tok = "";

  // Convenience lambda to store the start of a token. Used within the hext
  // machine definition.
  auto tk_start = [this, &tok_begin, &tok_end](){
    tok_begin = this->p;
    tok_end = nullptr;
  };

  // Convenience lambda to complete a token. Used within the hext
  // machine definition.
  auto tk_stop = [this, &tok, &tok_begin, &tok_end](){
    assert(tok_begin);
    assert(this->p);
    tok_end = this->p;
    tok = std::string(
      tok_begin,
      static_cast<std::string::size_type>(std::distance(tok_begin, tok_end))
    );
  };

  // Hide warnings generated by ragel code.
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-conversion"
#pragma GCC diagnostic ignored "-Wold-style-cast"
#pragma GCC diagnostic ignored "-Wpragmas"
#pragma GCC diagnostic ignored "-Wunreachable-code-break"
  %%{
    machine hext;
    include "hext-machine.rl";
    write init;
    write exec;
  }%%
#pragma GCC diagnostic pop

  // Throw error if there are missing closing tags.
  if( this->rule_stack_.size() )
    this->throw_missing_tag(this->rule_stack_.back().get_tag());

  if( this->top_rule_ )
  {
    return std::move(*this->top_rule_);
  }
  else
  {
    return Rule();
  }
}

Rule& Parser::cur_rule()
{
  assert(this->rule_stack_.size());
  return this->rule_stack_.back();
}

void Parser::push_rule()
{
  this->rule_stack_.emplace_back();
}

{'fraction_non_alphanumeric': 0.1364974245768948, 'fraction_numerical': 0.002698062300711307, 'mean_word_length': 3.9786324786324787, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6583679', 'n_tokens_mistral': 2843, 'n_tokens_neox': 2778, 'n_words': 753}