<commit_before>require 'json'
require 'open-uri'
require 'uri'

BASE_URI = "http://w3c.github.io/csvw/tests/"
BASE_PATH = File.join(File.dirname(__FILE__), "..", "fixtures", "csvw")
FEATURE_FILE_PATH = File.join(File.dirname(__FILE__), "..", "csvw_validation_tests.feature")
SCRIPT_FILE_PATH = File.join(File.dirname(__FILE__), "..", "..", "bin", "run-csvw-tests")

Dir.mkdir(BASE_PATH) unless Dir.exist?(BASE_PATH)

def cache_file(filename)
	file = File.join(BASE_PATH, filename)
	uri = URI.join(BASE_URI, filename)
	unless File.exist?(file)
		if filename.include? "/"
			levels = filename.split("/")[0..-2]
			for i in 0..levels.length
				dir = File.join(BASE_PATH, levels[0..i].join("/"))
				Dir.mkdir(dir) unless Dir.exist?(dir)
			end
		end
		STDERR.puts("storing #{file} locally")
		File.open(file, 'wb') do |f|
			f.puts open(uri, 'rb').read
		end
	end
	return uri, file
end

File.open(SCRIPT_FILE_PATH, 'w') do |file|
	File.chmod(0755, SCRIPT_FILE_PATH)
	manifest = JSON.parse( open("http://w3c.github.io/csvw/tests/manifest-validation.jsonld").read )
	manifest["entries"].each do |entry|
		type = "valid"
		case entry["type"] 
		when "csvt:WarningValidationTest" 
			type = "warnings" 
		when "csvt:NegativeValidationTest" 
			type = "errors" 
		end
		file.puts "echo \"#{entry["id"].split("#")[-1]}: entry[\"name\"]\""
		file.puts "echo \"#{type}: #{entry["comment"].gsub("\"", "\\\"").gsub("`", "'")}\""
		if entry["action"].end_with?(".json")
			file.puts "csvlint --schema=features/fixtures/csvw/#{entry["action"]}"
		elsif entry["option"] && entry["option"]["metadata"]
			file.puts "csvlint features/fixtures/csvw/#{entry["action"]} --schema=features/fixtures/csvw/#{entry["option"]["metadata"]}"
		else
			file.puts "csvlint features/fixtures/csvw/#{entry["action"]}"
		end
		file.puts "echo"
	end
end unless File.exist? SCRIPT_FILE_PATH
<commit_msg>duh, not useful to have entry[name] given as name of each test<commit_after>require 'json'
require 'open-uri'
require 'uri'

BASE_URI = "http://w3c.github.io/csvw/tests/"
BASE_PATH = File.join(File.dirname(__FILE__), "..", "fixtures", "csvw")
FEATURE_FILE_PATH = File.join(File.dirname(__FILE__), "..", "csvw_validation_tests.feature")
SCRIPT_FILE_PATH = File.join(File.dirname(__FILE__), "..", "..", "bin", "run-csvw-tests")

Dir.mkdir(BASE_PATH) unless Dir.exist?(BASE_PATH)

def cache_file(filename)
	file = File.join(BASE_PATH, filename)
	uri = URI.join(BASE_URI, filename)
	unless File.exist?(file)
		if filename.include? "/"
			levels = filename.split("/")[0..-2]
			for i in 0..levels.length
				dir = File.join(BASE_PATH, levels[0..i].join("/"))
				Dir.mkdir(dir) unless Dir.exist?(dir)
			end
		end
		STDERR.puts("storing #{file} locally")
		File.open(file, 'wb') do |f|
			f.puts open(uri, 'rb').read
		end
	end
	return uri, file
end

File.open(SCRIPT_FILE_PATH, 'w') do |file|
	File.chmod(0755, SCRIPT_FILE_PATH)
	manifest = JSON.parse( open("http://w3c.github.io/csvw/tests/manifest-validation.jsonld").read )
	manifest["entries"].each do |entry|
		type = "valid"
		case entry["type"] 
		when "csvt:WarningValidationTest" 
			type = "warnings" 
		when "csvt:NegativeValidationTest" 
			type = "errors" 
		end
		file.puts "echo \"#{entry["id"].split("#")[-1]}: #{entry["name"]}\""
		file.puts "echo \"#{type}: #{entry["comment"].gsub("\"", "\\\"").gsub("`", "'")}\""
		if entry["action"].end_with?(".json")
			file.puts "csvlint --schema=features/fixtures/csvw/#{entry["action"]}"
		elsif entry["option"] && entry["option"]["metadata"]
			file.puts "csvlint features/fixtures/csvw/#{entry["action"]} --schema=features/fixtures/csvw/#{entry["option"]["metadata"]}"
		else
			file.puts "csvlint features/fixtures/csvw/#{entry["action"]}"
		end
		file.puts "echo"
	end
end unless File.exist? SCRIPT_FILE_PATH

{'fraction_non_alphanumeric': 0.20549392498679345, 'fraction_numerical': 0.005810882197569995, 'mean_word_length': 4.60207100591716, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5084014', 'n_tokens_mistral': 1536, 'n_tokens_neox': 1411, 'n_words': 255}