<commit_before>module GoCart
class DataUtils

	def self.extract_value(type, value)
		return nil if value.nil?
		value = value.strip
		return nil if value.empty?

		case type
		when :string
			return value
		when :text
			return value
		when :integer
			return value.to_i
		when :float
			return Float(value)
		when :decimal
			return Float(value)
		when :datetime
			return value
		when :timestamp
			return value
		when :time
			return value
		when :date
			return extract_date(value)
		when :binary
			return value
		when :boolean
			return extract_boolean(value)
		else
			return value
		end
	end

	def self.extract_boolean(value)
    return true if value == '1'
    return false if value == '0'
		if ['y', 't'].include? value.downcase[0]
			return true
		elsif ['n', 'f'].include? value.downcase[0]
			return false
    else
      raise "Invalid boolean value: #{value}"
		end
	end

	def self.extract_date(value)
		# Null/Empty
		return nil if value == '00000000' || value =~ /^00\W00\W0000$/ || value =~ /^0000\W00\W00$/
		return nil if value == '99999999' || value =~ /^99\W99\W9999$/ || value =~ /^9999\W99\W99$/

		if value =~ /^(.+)\s+\d{1,2}:\d{2}:\d{2}/<commit_msg>Loosened null date check to ignore month/year for 0000/9999<commit_after>module GoCart
class DataUtils

	def self.extract_value(type, value)
		return nil if value.nil?
		value = value.strip
		return nil if value.empty?

		case type
		when :string
			return value
		when :text
			return value
		when :integer
			return value.to_i
		when :float
			return Float(value)
		when :decimal
			return Float(value)
		when :datetime
			return value
		when :timestamp
			return value
		when :time
			return value
		when :date
			return extract_date(value)
		when :binary
			return value
		when :boolean
			return extract_boolean(value)
		else
			return value
		end
	end

	def self.extract_boolean(value)
    return true if value == '1'
    return false if value == '0'
		if ['y', 't'].include? value.downcase[0]
			return true
		elsif ['n', 'f'].include? value.downcase[0]
			return false
    else
      raise "Invalid boolean value: #{value}"
		end
	end

	def self.extract_date(value)
		# Null/Empty
		return nil if value == '00000000' || value =~ /^\d\d\W\d\d\W0000$/ || value =~ /^0000\W\d\d\W\d\d$/
		return nil if value == '99999999' || value =~ /^\d\d\W\d\d\W9999$/ || value =~ /^9999\W\d\d\W\d\d$/

		# Remove time if present
		if value =~ /^(.+)\s+\d{1,2}:\d{2}:\d{2}/
{'fraction_non_alphanumeric': 0.13079130791307914, 'fraction_numerical': 0.04264042640426404, 'mean_word_length': 2.961038961038961, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5492516', 'n_tokens_mistral': 1052, 'n_tokens_neox': 857, 'n_words': 259}