<commit_before>require 'minitest/autorun'
require 'harvestman'

class TestHarvestman < MiniTest::Test
	def test_namespace
		assert Harvestman.is_a?(Module)
	end

	def test_single_page
		Harvestman.crawl "test/example1.html" do
			assert_equal "Foobar", css("title").inner_text
		end
	end

	def test_scraping
		results = []

		Harvestman.crawl "test/example*.html", (1..3), :plain do
			r = {
				:title => css("head title").inner_text,
				:header => css("header div.title h1").inner_text,
				:footer => css("footer span a").inner_text,
				:list => []
			}

			css "div.main ul" do
				r[:list] << css("li").inner_text
			end

			results << r
		end

		results.each_with_index do |r, i|
			assert_equal(r[:title], "ex#{i+1}")
			assert_equal(r[:header], "#{r[:title]}_header_h1")
			assert_equal(r[:footer], "#{r[:title]}_footer_span_a")
			#assert_equal(r[:list].count, 3)
		end
	end
end<commit_msg>Rename test case<commit_after>require 'minitest/autorun'
require 'harvestman'

class TestHarvestman < MiniTest::Test
	def test_namespace
		assert Harvestman.is_a?(Module)
	end

	def test_single_page
		Harvestman.crawl "test/example1.html" do
			assert_equal "Foobar", css("title").inner_text
		end
	end

	def test_multiple_pages
		results = []

		Harvestman.crawl "test/example*.html", (1..3), :plain do
			r = {
				:title => css("head title").inner_text,
				:header => css("header div.title h1").inner_text,
				:footer => css("footer span a").inner_text,
				:list => []
			}

			css "div.main ul" do
				r[:list] << css("li").inner_text
			end

			results << r
		end

		results.each_with_index do |r, i|
			assert_equal(r[:title], "ex#{i+1}")
			assert_equal(r[:header], "#{r[:title]}_header_h1")
			assert_equal(r[:footer], "#{r[:title]}_footer_span_a")
			#assert_equal(r[:list].count, 3)
		end
	end
end
{'fraction_non_alphanumeric': 0.1623412479293208, 'fraction_numerical': 0.007730535615681944, 'mean_word_length': 4.047353760445683, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1272730', 'n_tokens_mistral': 786, 'n_tokens_neox': 685, 'n_words': 137}