<commit_before>#!/usr/bin/env ruby
require 'test/unit'
require 'lib/dome/parser'
require 'lib/dome/xpath'

module Dome
    class XPath
        attr_reader :path

        class NodeParser
            attr_reader :tag, :attr_parsers
        end

        class AttrParser
            attr_reader :attr, :value
        end
    end
end

class XPathTests < Test::Unit::TestCase
    include Dome

    def testSingleNode
        path = '/div'
        xpath = XPath.new path

        assert_equal 1, xpath.path.length
        assert_equal 'div', xpath.path[0].tag
            assert_equal 0, xpath.path[0].attr_parsers.length
    end

    def testMultipleNodes
        path = '/div/p/span/em/a'
        xpath = XPath.new path

        assert_equal 5, xpath.path.length
        assert_equal 'div', xpath.path[0].tag
            assert_equal 0, xpath.path[0].attr_parsers.length
        assert_equal 'p', xpath.path[1].tag
            assert_equal 0, xpath.path[1].attr_parsers.length
        assert_equal 'span', xpath.path[2].tag
            assert_equal 0, xpath.path[2].attr_parsers.length
        assert_equal 'em', xpath.path[3].tag
            assert_equal 0, xpath.path[3].attr_parsers.length
        assert_equal 'a', xpath.path[4].tag
            assert_equal 0, xpath.path[4].attr_parsers.length
    end

    def testSingleAttribute
        path = "/div[@foo='bar']/p/span/em/a"
        xpath = XPath.new path

        assert_equal 5, xpath.path.length
        assert_equal 'div', xpath.path[0].tag
            assert_equal 1, xpath.path[0].attr_parsers.length
            assert_equal 'foo', xpath.path[0].attr_parsers[0].attr
            assert_equal 'bar', xpath.path[0].attr_parsers[0].value
        assert_equal 'p', xpath.path[1].tag
            assert_equal 0, xpath.path[1].attr_parsers.length
        assert_equal 'span', xpath.path[2].tag
            assert_equal 0, xpath.path[2].attr_parsers.length
        assert_equal 'em', xpath.path[3].tag
            assert_equal 0, xpath.path[3].attr_parsers.length
        assert_equal 'a', xpath.path[4].tag
            assert_equal 0, xpath.path[4].attr_parsers.length
    end

    def testMultipleAttributes
        path = "/div[@foo='bar']/p/span[@chunky='bacon']/em/a"
        xpath = XPath.new path

        assert_equal 5, xpath.path.length
        assert_equal 'div', xpath.path[0].tag
            assert_equal 1, xpath.path[0].attr_parsers.length
            assert_equal 'foo', xpath.path[0].attr_parsers[0].attr
            assert_equal 'bar', xpath.path[0].attr_parsers[0].value
        assert_equal 'p', xpath.path[1].tag
            assert_equal 0, xpath.path[1].attr_parsers.length
        assert_equal 'span', xpath.path[2].tag
            assert_equal 1, xpath.path[2].attr_parsers.length
            assert_equal 'chunky', xpath.path[2].attr_parsers[0].attr
            assert_equal 'bacon', xpath.path[2].attr_parsers[0].value
        assert_equal 'em', xpath.path[3].tag
            assert_equal 0, xpath.path[3].attr_parsers.length
        assert_equal 'a', xpath.path[4].tag
            assert_equal 0, xpath.path[4].attr_parsers.length
    end

    def testEscapedAttributes
        path = "/div[@foo='b\\'ar']/p/span[@chunky='ba\\\\con']/em/a"
        xpath = XPath.new path

        assert_equal 5, xpath.path.length
        assert_equal 'div', xpath.path[0].tag
            assert_equal 1, xpath.path[0].attr_parsers.length
            assert_equal 'foo', xpath.path[0].attr_parsers[0].attr
            assert_equal 'b\'ar', xpath.path[0].attr_parsers[0].value
        assert_equal 'p', xpath.path[1].tag
            assert_equal 0, xpath.path[1].attr_parsers.length
        assert_equal 'span', xpath.path[2].tag
            assert_equal 1, xpath.path[2].attr_parsers.length
            assert_equal 'chunky', xpath.path[2].attr_parsers[0].attr
            assert_equal 'ba\\con', xpath.path[2].attr_parsers[0].value
        assert_equal 'em', xpath.path[3].tag
            assert_equal 0, xpath.path[3].attr_parsers.length
        assert_equal 'a', xpath.path[4].tag
            assert_equal 0, xpath.path[4].attr_parsers.length
    end
end
<commit_msg>separated xpath construction and scraping tests<commit_after>#!/usr/bin/env ruby
require 'test/unit'
require 'lib/dome/parser'
require 'lib/dome/xpath'

module Dome
    class XPath
        attr_reader :path

        class NodeParser
            attr_reader :tag, :attr_parsers
        end

        class AttrParser
            attr_reader :attr, :value
        end
    end
end

class XPathConstructionTests < Test::Unit::TestCase
    include Dome

    def testSingleNode
        path = '/div'
        xpath = XPath.new path

        assert_equal 1, xpath.path.length
        assert_equal 'div', xpath.path[0].tag
            assert_equal 0, xpath.path[0].attr_parsers.length
    end

    def testMultipleNodes
        path = '/div/p/span/em/a'
        xpath = XPath.new path

        assert_equal 5, xpath.path.length
        assert_equal 'div', xpath.path[0].tag
            assert_equal 0, xpath.path[0].attr_parsers.length
        assert_equal 'p', xpath.path[1].tag
            assert_equal 0, xpath.path[1].attr_parsers.length
        assert_equal 'span', xpath.path[2].tag
            assert_equal 0, xpath.path[2].attr_parsers.length
        assert_equal 'em', xpath.path[3].tag
            assert_equal 0, xpath.path[3].attr_parsers.length
        assert_equal 'a', xpath.path[4].tag
            assert_equal 0, xpath.path[4].attr_parsers.length
    end

    def testSingleAttribute
        path = "/div[@foo='bar']/p/span/em/a"
        xpath = XPath.new path

        assert_equal 5, xpath.path.length
        assert_equal 'div', xpath.path[0].tag
            assert_equal 1, xpath.path[0].attr_parsers.length
            assert_equal 'foo', xpath.path[0].attr_parsers[0].attr
            assert_equal 'bar', xpath.path[0].attr_parsers[0].value
        assert_equal 'p', xpath.path[1].tag
            assert_equal 0, xpath.path[1].attr_parsers.length
        assert_equal 'span', xpath.path[2].tag
            assert_equal 0, xpath.path[2].attr_parsers.length
        assert_equal 'em', xpath.path[3].tag
            assert_equal 0, xpath.path[3].attr_parsers.length
        assert_equal 'a', xpath.path[4].tag
            assert_equal 0, xpath.path[4].attr_parsers.length
    end

    def testMultipleAttributes
        path = "/div[@foo='bar']/p/span[@chunky='bacon']/em/a"
        xpath = XPath.new path

        assert_equal 5, xpath.path.length
        assert_equal 'div', xpath.path[0].tag
            assert_equal 1, xpath.path[0].attr_parsers.length
            assert_equal 'foo', xpath.path[0].attr_parsers[0].attr
            assert_equal 'bar', xpath.path[0].attr_parsers[0].value
        assert_equal 'p', xpath.path[1].tag
            assert_equal 0, xpath.path[1].attr_parsers.length
        assert_equal 'span', xpath.path[2].tag
            assert_equal 1, xpath.path[2].attr_parsers.length
            assert_equal 'chunky', xpath.path[2].attr_parsers[0].attr
            assert_equal 'bacon', xpath.path[2].attr_parsers[0].value
        assert_equal 'em', xpath.path[3].tag
            assert_equal 0, xpath.path[3].attr_parsers.length
        assert_equal 'a', xpath.path[4].tag
            assert_equal 0, xpath.path[4].attr_parsers.length
    end

    def testEscapedAttributes
        path = "/div[@foo='b\\'ar']/p/span[@chunky='ba\\\\con']/em/a"
        xpath = XPath.new path

        assert_equal 5, xpath.path.length
        assert_equal 'div', xpath.path[0].tag
            assert_equal 1, xpath.path[0].attr_parsers.length
            assert_equal 'foo', xpath.path[0].attr_parsers[0].attr
            assert_equal 'b\'ar', xpath.path[0].attr_parsers[0].value
        assert_equal 'p', xpath.path[1].tag
            assert_equal 0, xpath.path[1].attr_parsers.length
        assert_equal 'span', xpath.path[2].tag
            assert_equal 1, xpath.path[2].attr_parsers.length
            assert_equal 'chunky', xpath.path[2].attr_parsers[0].attr
            assert_equal 'ba\\con', xpath.path[2].attr_parsers[0].value
        assert_equal 'em', xpath.path[3].tag
            assert_equal 0, xpath.path[3].attr_parsers.length
        assert_equal 'a', xpath.path[4].tag
            assert_equal 0, xpath.path[4].attr_parsers.length
    end
end

class XPathScrapingTests < Test::Unit::TestCase
    include Dome

end
{'fraction_non_alphanumeric': 0.12130248091603053, 'fraction_numerical': 0.02099236641221374, 'mean_word_length': 2.3728881737731293, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4773960', 'n_tokens_mistral': 2877, 'n_tokens_neox': 2789, 'n_words': 501}