<commit_before>#!/usr/bin/env ruby
# This is Dome, a pure Ruby HTML DOM parser with very simple XPath support.
#
# If you want to find out more or need a tutorial, go to
# http://dome.rubyforge.org/
# You'll find a nice wiki there!
#
# Author::      <NAME> (karottenreibe)
# Copyright::   Copyright (c) 2008 <NAME>
# License::     Boost Software License 1.0
#               For further information regarding this license, you can go to
#               http://www.boost.org/LICENSE_1_0.txt
# Homepage::    http://dome.rubyforge.org/
# Git repo::    http://rubyforge.org/scm/?group_id=7589
#

require 'test/unit'
require 'rubygems'
require 'spectre/spectre'
require 'spectre/std/string'

class Tests < Test::Unit::TestCase
    include Spectre
    include Spectre::StringParsing

    def testTag
        element = Grammar.new do ||
            var :tagname   => ( ~char('>') & ~char(' ') ).+,
                :parser    => char('<') >> :tagname >> string('></') >> :tagname >> char('>')
        end

        ret = parse '<foo></foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 11, ret.length

        ret = parse '<foo></bar>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 11, ret.length
    end

    def testTagClosure
        element = Grammar.new do ||
            var :tagname   => ( ~char('>') & ~char(' ') ).+,
                :element   => char('<') >> sym(:tagname)[:tag] >> string('></') >> closed(:tag) >> char('>'),
                :parser    => close(:element)
        end

        ret = parse '<foo></foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 11, ret.length

        ret = parse '<foo></bar>', element
        assert_kind_of NilClass, ret
    end

    def testData
        element = Grammar.new do ||
            var :tagname   => ( ~char('>') & ~char(' ') ).+,
                :element   => char('<') >> sym(:tagname)[:tag] >> char('>') >> :data >> string('</') >> closed(:tag) >> char('>'),
                :data      => ( ~char('<') ).*,
                :parser    => close(:element)
        end

        ret = parse '<foo></foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 11, ret.length

        ret = parse '<foo>data</foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 15, ret.length

        ret = parse '<foo><</foo>', element
        assert_kind_of NilClass, ret

        ret = parse '<foo></bar>', element
        assert_kind_of NilClass, ret
    end

    class Element
        attr_accessor :tag, :attributes, :data
    end

    class Data
        attr_accessor :data
    end

    def testNodeAction
        val = 1

        newelement_a = lambda { |match, closure|
            closure[:element] = Element.new
            closure[:element].tag = match.value
            val += val
        }

        element = Grammar.new do ||
            var :tagname   => ( ~char('>') & ~char(' ') ).+,
                :element   => char('<') >> sym(:tagname)[:tag][newelement_a] >> char('>') >> :data >> string('</') >> closed(:tag) >> char('>'),
                :data      => ( ~char('<') ).*,
                :parser    => close(:element)
        end

        ret = parse '<foo></foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 11, ret.length

        ret = parse '<foo>data</foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 15, ret.length

        ret = parse '<foo><</foo>', element
        assert_kind_of NilClass, ret

        ret = parse '<foo></bar>', element
        assert_kind_of NilClass, ret
        
        assert_equal 16, val
    end

    def testDataAction
        val = 1

        newelement_a = lambda { |match, closure|
            closure[:element] = Element.new
            closure[:element].tag = match.value
            closure[:element].tag = match.value
            val += val
        }

        element = Grammar.new do ||
            var :tagname   => ( ~char('>') & ~char(' ') ).+,
                :element   => char('<') >> sym(:tagname)[:tag] >> char('>') >> sym(:data)[:data] >> string('</') >> closed(:tag) >> char('>'),
                :data      => ( ~char('<') ).*,
                :parser    => close(:element)[newelement_a]
        end

        ret = parse '<foo></foo>', element
        assert_kind_of Spectre::Match, ret<commit_msg>refactored that test again<commit_after>#!/usr/bin/env ruby
# This is Dome, a pure Ruby HTML DOM parser with very simple XPath support.
#
# If you want to find out more or need a tutorial, go to
# http://dome.rubyforge.org/
# You'll find a nice wiki there!
#
# Author::      <NAME> (karottenreibe)
# Copyright::   Copyright (c) 2008 <NAME>
# License::     Boost Software License 1.0
#               For further information regarding this license, you can go to
#               http://www.boost.org/LICENSE_1_0.txt
# Homepage::    http://dome.rubyforge.org/
# Git repo::    http://rubyforge.org/scm/?group_id=7589
#

require 'test/unit'
require 'rubygems'
require 'spectre/spectre'
require 'spectre/std/string'

class Tests < Test::Unit::TestCase
    include Spectre
    include Spectre::StringParsing

    def testTag
        element = Grammar.new do ||
            var :tagname   => ( ~char('>') & ~char(' ') ).+,
                :parser    => char('<') >> :tagname >> string('></') >> :tagname >> char('>')
        end

        ret = parse '<foo></foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 11, ret.length

        ret = parse '<foo></bar>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 11, ret.length
    end

    def testTagClosure
        element = Grammar.new do ||
            var :tagname   => ( ~char('>') & ~char(' ') ).+,
                :element   => char('<') >> sym(:tagname)[:tag] >> string('></') >> closed(:tag) >> char('>'),
                :parser    => close(:element)
        end

        ret = parse '<foo></foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 11, ret.length

        ret = parse '<foo></bar>', element
        assert_kind_of NilClass, ret
    end

    def testData
        element = Grammar.new do ||
            var :tagname   => ( ~char('>') & ~char(' ') ).+,
                :element   => char('<') >> sym(:tagname)[:tag] >> char('>') >> :data >> string('</') >> closed(:tag) >> char('>'),
                :data      => ( ~char('<') ).*,
                :parser    => close(:element)
        end

        ret = parse '<foo></foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 11, ret.length

        ret = parse '<foo>data</foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 15, ret.length

        ret = parse '<foo><</foo>', element
        assert_kind_of NilClass, ret

        ret = parse '<foo></bar>', element
        assert_kind_of NilClass, ret
    end

    class Element
        attr_accessor :tag, :attributes, :data
    end

    class Data
        attr_accessor :data
    end

    def testNodeAction
        val = 1

        newelement_a = lambda { |match, closure|
            closure[:element] = Element.new
            closure[:element].tag = match.value
            val += val
        }

        element = Grammar.new do ||
            var :tagname   => ( ~char('>') & ~char(' ') ).+,
                :element   => char('<') >> sym(:tagname)[:tag][newelement_a] >> char('>') >> :data >> string('</') >> closed(:tag) >> char('>'),
                :data      => ( ~char('<') ).*,
                :parser    => close(:element)
        end

        ret = parse '<foo></foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 11, ret.length

        ret = parse '<foo>data</foo>', element
        assert_kind_of Spectre::Match, ret
        assert_equal 15, ret.length

        ret = parse '<foo><</foo>', element
        assert_kind_of NilClass, ret

        ret = parse '<foo></bar>', element
        assert_kind_of NilClass, ret
        
        assert_equal 16, val
    end

    def testDataAction
        val = 1

        newelement_a = lambda { |match, closure|
            closure[:element] = Element.new
            closure[:element].tag = closure[:tag]
            closure[:element].data = closure[:data]
            val += val
        }

        element = Grammar.new do ||
            var :tagname   => ( ~char('>') & ~char(' ') ).+,
                :element   => char('<') >> sym(:tagname)[:tag] >> char('>') >> sym(:data)[:data] >> string('</') >> closed(:tag) >> char('>'),
                :data      => ( ~char('<') ).*,
                :parser    => close(:element)[newelement_a]
        end

        ret = parse '<foo></foo>', element
        assert_kind_of Spectre::Match, ret
{'fraction_non_alphanumeric': 0.16210693608809174, 'fraction_numerical': 0.006811215802020661, 'mean_word_length': 1.9864406779661017, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 0, '>': 227, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4854310', 'n_tokens_mistral': 2712, 'n_tokens_neox': 2595, 'n_words': 676}