<commit_before># encoding: UTF-8

require "open-uri"


require "./jma-info/get-uri-list"
require "./jma-info/get-info"

def file_write_and_print str
	puts str
	open("jma-info.log", "a"){|f|f.puts(str)}
end

def puts_info uris
	uris
		.lazy # 少しずつ表示していく
		.map{|u|get_info(u)}
		.select{|s|!s.nil?}
		.each{|s|file_write_and_print s}
end

def sleep_up_to_even_number_minutes
	loop_sec = 30 # 60で割り切れるように
	sleep(loop_sec-(Time.now.sec%loop_sec))
end

old_date = Time.now-120
loop do
	new_date = Time.now
	uri_list = get_uri_list(old_date, new_date)
	file_write_and_print new_date unless uri_list.empty?
	puts_info(uri_list)
	old_date = new_date
	sleep_up_to_even_number_minutes
end<commit_msg>例外を保存するように<commit_after># encoding: UTF-8

require "open-uri"


require "./jma-info/get-uri-list"
require "./jma-info/get-info"

def file_write_and_print str
	puts str
	open("jma-info.log", "a"){|f|f.puts(str)}
end

def puts_info uris
	uris
		.lazy # 少しずつ表示していく
		.map{|u|get_info(u)}
		.select{|s|!s.nil?}
		.each{|s|file_write_and_print s}
end

def sleep_up_to_even_number_minutes
	loop_sec = 30 # 60で割り切れるように
	sleep(loop_sec-(Time.now.sec%loop_sec))
end

def app
	old_date = Time.now-120
	loop do
		new_date = Time.now
		uri_list = get_uri_list(old_date, new_date)
		file_write_and_print new_date unless uri_list.empty?
		puts_info(uri_list)
		old_date = new_date
		sleep_up_to_even_number_minutes
	end
end

begin
	app()
rescue Exception => e
	file_write_and_print(e)
end
{'fraction_non_alphanumeric': 0.12150170648464163, 'fraction_numerical': 0.010921501706484642, 'mean_word_length': 4.864, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6361632', 'n_tokens_mistral': 713, 'n_tokens_neox': 668, 'n_words': 115}