<commit_before>class CsvlintValidateService

	require 'csvlint'

	def self.validate_csv(file)
		csv_string = get_s3_string(file)
		validator = lint_csv(csv_string)
		update_database_attributes(validator, file)
	end

	def self.get_validated_csv(file)
		csv_string = get_s3_string(file)
		lint_csv(csv_string)
	end

	def self.lint_csv(csv_string)
		return Csvlint::Validator.new(csv_string)
	end

	def self.generate_badge(file)
		!file.validation ? "invalid" : generate_badge_valid_file(file)
	end

	private

	def self.get_s3_string(file)
		return FileStorageService.get_string_io(file.storage_key)
	end

	def self.update_database_attributes(csv, file)
		csv.valid? ? file.update(validation: true) : file.update(validation: false)
	end

	def self.generate_badge_valid_file(file)
		csv_string = get_s3_string(file)
		csv = lint_csv(csv_string)

		csv.warnings.count > 0 ? "warnings" : "valid"
	end

end<commit_msg>Schema being passed to CSVlint - needs refactoring and error guarding<commit_after>class CsvlintValidateService

	require 'csvlint'

	def self.validate_csv(file)
		csv = get_s3_string(file)
		# schema = Csvlint::Schema.load_from_string(URI.escape(file.dataset_file_schema.url), file.dataset_file_schema.schema)
		# tempfile = Tempfile.new 'uploaded'
		# tempcsv = File.new(file.tempfile)

		# The schema object below is correct for passing to validator
		schema = file.dataset_file_schema.parsed_schema
		# file_validation = Csvlint::Validator.new(csv, {}, schema)

		validator = lint_csv(csv, schema)
		update_database_attributes(validator, file)
	end

	def self.get_validated_csv(file)
		csv_string = get_s3_string(file)
		schema = file.dataset_file_schema.parsed_schema
		lint_csv(csv_string, schema)
	end

	def self.lint_csv(csv, schema)
		return Csvlint::Validator.new(csv, {}, schema)
	end

	def self.generate_badge(file)
		!file.validation ? "invalid" : generate_badge_valid_file(file)
	end

	private

	def self.get_s3_string(file)
		return FileStorageService.get_string_io(file.storage_key)
	end

	def self.update_database_attributes(csv, file)
		csv.valid? ? file.update(validation: true) : file.update(validation: false)
	end

	def self.generate_badge_valid_file(file)
		csv_string = get_s3_string(file)
		schema = file.dataset_file_schema.parsed_schema
		csv = lint_csv(csv_string, schema)

		csv.warnings.count > 0 ? "warnings" : "valid"
	end

end
{'fraction_non_alphanumeric': 0.09535864978902954, 'fraction_numerical': 0.004219409282700422, 'mean_word_length': 5.5497237569060776, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '398570', 'n_tokens_mistral': 896, 'n_tokens_neox': 846, 'n_words': 154}