<commit_before>require 'nokogiri'
require 'csv'
require 'json'


#  Normalize dates, e.g. "c. 1865" becomes "1865", and "1901-1902" becomes "1902".
def normalizeDate(input)
	if input.nil?
		return nil
	else
		normalized = input.first.slice(/(\d*)(\D|$)/,1).to_i
		if normalized == 0
			return nil
		else
			return normalized
		end
	end
end

# Returns a given number of topics from SAAM's hierarchical schema, e.g. "Study\botanical study\American mistletoe" can be reduced to "Study\botanical study" or "Study"
def cleanTopic(input)
	if input.nil?
		return nil
	else
		topics = input.split("\\")
		case topics.count
		when 1
			return topics[0]
		when 2...10
			return "#{topics[0]} - #{topics[1]}"
		else
		end
	end
end

# Set input and output files
INPUT = "output.json"
OUTPUT = "normalized_saam_two_topics.json"
raw_data = JSON.parse(File.read(INPUT))
normalized_data = Hash.new

# Loop through raw input
raw_data.each do |id, data|
	# Only include records that have both "Date" and "Topic"
	unless data["Date"].nil? or data["Topic"].nil?

		# Normalize date
		normalized_date = normalizeDate(data["Date"])

		# Normalize topics
		normalized_topics = Array.new
		data["Topics"].each do |topic|
			normalized_topics << cleanTopic(topic)
		end

		# Check that normalized values are still valid, e.g. items that had a Date value of "n.d" will no longer be valid after cleaning, and must be discarded
		if normalized_date.nil? or normalized_topics.empty?
			# Skips record
		else
			# Replace original values with normalized values, and store in normalized_data hash
			data["Date"] = normalized_date
			data["Topics"] = normalized_topics
			normalized_data[id] = data
		end
	else
		# Skips record
	end
end

######### Write JSON file #########
puts "Writing JSON..."
File.open(OUTPUT,"w") do |file|
	file.write(JSON.pretty_generate(normalized_data))
end
puts "Finished."<commit_msg>Successfully filters dates AND topics<commit_after>require 'nokogiri'
require 'csv'
require 'json'

######### Private methods #########

#  Normalize dates, e.g. "c. 1865" becomes "1865", and "1901-1902" becomes "1902".
def normalizeDate(input)
	if input.nil?
		return nil
	else
		normalized = input.first.slice(/(\d*)(\D|$)/,1).to_i
		if normalized == 0
			return nil
		else
			return normalized
		end
	end
end

# Returns a given number of topics from SAAM's hierarchical schema, e.g. "Study\botanical study\American mistletoe" can be reduced to "Study\botanical study" or "Study"
def cleanTopic(input)
	topics = input.split("\\")
	case topics.count
	when 1
		return topics[0]
	when 2...10
		return "#{topics[0]} - #{topics[1]}"
	else
	end
end

######### Set input and output files #########
INPUT = "output.json"
OUTPUT = "normalized_saam_two_topics.json"
raw_data = JSON.parse(File.read(INPUT))
normalized_data = Hash.new

puts "Number of raw records: #{raw_data.count}"

puts "Cleaning records"
######### Loop through raw input #########
# Only including records that have both "Date" and "Topic"
raw_data.select {|k,v| v.has_key?("Date") && v.has_key?("Topic")}.each do |id, data|
	
	# Normalize date
	normalized_date = normalizeDate(data["Date"])

	# Normalize topics
	normalized_topics = Array.new
	data["Topic"].each do |topic|
		normalized_topics << cleanTopic(topic)
	end
	
	# Check that normalized values are still valid, e.g. items that had a Date value of "n.d" will no longer be valid after cleaning, and must be discarded
	if normalized_date.nil?
		# Skips record
	else
		# Replace original values with normalized values, and store in normalized_data hash
		data["Date"] = normalized_date
		data["Topic"] = normalized_topics
		normalized_data[id] = data
	end
end

######### Write JSON file #########

puts "Number of normalied records: #{normalized_data.count}"

puts "Writing JSON..."
File.open(OUTPUT,"w") do |file|
	file.write(JSON.pretty_generate(normalized_data))
end
puts "Finished."
{'fraction_non_alphanumeric': 0.1341055341055341, 'fraction_numerical': 0.014929214929214929, 'mean_word_length': 4.323287671232877, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1941884', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1288, 'n_words': 409}