<commit_before>dir = File.dirname(__FILE__)
require "#{dir}/../spec_helper"

# require 'test/unit'
require 'rubygems'
require 'treetop'
require "erector/erected"  # pull this out so we don't recreate the grammar every time

module ParserTestHelper
  def assert_evals_to_self(input)
    assert_evals_to(input, input)
  end

  def parse(input)
    result = @parser.parse(input)
    if result
      result.set_indent(0) if result.respond_to? :set_indent
    else
      puts @parser.failure_reason
      puts @parser.terminal_failures.join("\n")
      result.should_not be_nil
    end
    result
  end
end

describe RhtmlParser do
  include ParserTestHelper
  
  before :each do
    @parser = RhtmlParser.new
  end
  
  it "converts text" do
    parse("hello").convert.should == "text 'hello'\n"
    parse("hello maude!").convert.should == "text 'hello maude!'\n"
    parse(" hello ").convert.should == "text 'hello'\n"
  end
  
  it "unescapes HTML entities in text" do
    pending do
      parse("&lt;").convert.should == "text '<'"
      parse("5 &gt; 2").convert.should == "text '5 > 2'"
    end
  end

  it "converts self-closing tags" do
    parse("<br/>").convert.should == "br\n"
    parse("<br />").convert.should == "br\n"
  end

  it "converts open tag" do
    parse("<div>").convert.should == "div do\n"
    parse("<h1>").convert.should == "h1 do\n"<commit_msg>remove pending block<commit_after>dir = File.dirname(__FILE__)
require "#{dir}/../spec_helper"

# require 'test/unit'
require 'rubygems'
require 'treetop'
require "erector/erected"  # pull this out so we don't recreate the grammar every time

module ParserTestHelper
  def assert_evals_to_self(input)
    assert_evals_to(input, input)
  end

  def parse(input)
    result = @parser.parse(input)
    if result
      result.set_indent(0) if result.respond_to? :set_indent
    else
      puts @parser.failure_reason
      puts @parser.terminal_failures.join("\n")
      result.should_not be_nil
    end
    result
  end
end

describe RhtmlParser do
  include ParserTestHelper
  
  before :each do
    @parser = RhtmlParser.new
  end
  
  it "converts text" do
    parse("hello").convert.should == "text 'hello'\n"
    parse("hello maude!").convert.should == "text 'hello maude!'\n"
    parse(" hello ").convert.should == "text 'hello'\n"
  end
  
  it "unescapes HTML entities in text" do
      parse("&lt;").convert.should == "text '<'\n"
      parse("5 &gt; 2").convert.should == "text '5 > 2'\n"
  end

  it "converts self-closing tags" do
    parse("<br/>").convert.should == "br\n"
    parse("<br />").convert.should == "br\n"
  end

  it "converts open tag" do
    parse("<div>").convert.should == "div do\n"
    parse("<h1>").convert.should == "h1 do\n"
{'fraction_non_alphanumeric': 0.144534115920763, 'fraction_numerical': 0.005135730007336757, 'mean_word_length': 3.2476635514018692, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4905397', 'n_tokens_mistral': 950, 'n_tokens_neox': 929, 'n_words': 241}