<commit_before>require 'lightmodels/serialization'

module LightModels

module ModelBuilding

class << self
	attr_accessor :verbose
end

def self.generate_models_in_dir(src,dest,src_extension,dest_extension,max_nesting=500,&model_creator)
	puts "== #{src} -> #{dest} ==" if LightModels::ModelBuilding.verbose
	Dir["#{src}/*"].each do |fd|		
		if File.directory? fd
			basename = File.basename(fd)
			generate_models_in_dir("#{src}/#{basename}","#{dest}/#{basename}",src_extension,dest_extension,&model_creator)
		else
			if File.extname(fd)==".#{src_extension}"
				translated_simple_name = "#{File.basename(fd, ".#{src_extension}")}.#{dest_extension}"
				translated_name = "#{dest}/#{translated_simple_name}"
				puts "* #{fd} --> #{translated_name}" if LightModels::ModelBuilding.verbose
				generate_model_per_file(fd,translated_name,max_nesting,&model_creator)
			end
		end
	end
end

def self.generate_model_per_file(src,dest,max_nesting=500,&models_generator)
	if not File.exist? dest 
		puts "<Model from #{src}>"
	
		m = models_generator.call(src)

		LightModels::Serialization.save_model(m,dest,max_nesting)
	else
		puts "skipping #{src} because #{dest} found" if LightModels::ModelBuilding.verbose 
	end
end

end

end<commit_msg>Error handling for model generation<commit_after>require 'lightmodels/serialization'

module LightModels

module ModelBuilding

class << self
	attr_accessor :verbose
end

def self.generate_models_in_dir(src,dest,src_extension,dest_extension,max_nesting=500,error_handler=nil,&model_creator)
	puts "== #{src} -> #{dest} ==" if LightModels::ModelBuilding.verbose
	Dir["#{src}/*"].each do |fd|		
		if File.directory? fd
			basename = File.basename(fd)
			generate_models_in_dir("#{src}/#{basename}","#{dest}/#{basename}",src_extension,dest_extension,max_nesting,error_handler,&model_creator)
		else
			if File.extname(fd)==".#{src_extension}"
				translated_simple_name = "#{File.basename(fd, ".#{src_extension}")}.#{dest_extension}"
				translated_name = "#{dest}/#{translated_simple_name}"
				puts "* #{fd} --> #{translated_name}" if LightModels::ModelBuilding.verbose
				generate_model_per_file(fd,translated_name,max_nesting,error_handler,&model_creator)
			end
		end
	end
end

def self.generate_model_per_file(src,dest,max_nesting=500,error_handler=nil,&models_generator)
	if not File.exist? dest 
		puts "<Model from #{src}>"
		
		if error_handler
			begin
				m = models_generator.call(src)
				LightModels::Serialization.save_model(m,dest,max_nesting)
			rescue Exception => e
				error_handler.call(src,e)
			rescue
				error_handler.call(src,nil)
			end
		else
			m = models_generator.call(src)
			LightModels::Serialization.save_model(m,dest,max_nesting)
		end
	else
		puts "skipping #{src} because #{dest} found" if LightModels::ModelBuilding.verbose 
	end
end

end

end
{'fraction_non_alphanumeric': 0.1375886524822695, 'fraction_numerical': 0.00425531914893617, 'mean_word_length': 6.123737373737374, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3608152', 'n_tokens_mistral': 1076, 'n_tokens_neox': 970, 'n_words': 160}