<commit_before>#coding: utf-8
#version v8
#download with thread
#multi thread access unvisited queue and visited queue
require 'thread'
require 'mechanize'
require 'uri'
require 'thread'
require 'thwait'
require 'fileutils'
require 'logger'
require 'net/smtp'

Thread.abort_on_exception=true

trap('INT'){
	#logger cannot write in trap context
	$stderr.puts('exiting.............\n')
	exit 1
}

class Crawler
	VERSION = '0.0.1'
	attr_accessor :start_urls,				#crawler entrances
		:logger,												#the path to save the logger,it defalut to STDOUT
		:filters,												#the urls filters
		:thread_limit,                  #the Crawler thread limit
		:delay,													#specify how long to start next request
		:encoding,											#the website encoding,the default is nil.
		:consumer_wait_queue_limit,     #when the Consumer queue size great the this threshold,the Crawler will sleep 
		:dbname,												#the Redis database index
		:consumer,											#A Consumer which consume the 'Page' object.As you might expect the Crawler is a Producter
		#for email
		:enable_email,									#enable the email,if some problems led to the Crawler down,it will send a email to you
		:email,													#recipient's email
		:from,													#sender's email
		:address,												#the SMTP server ip address or hostname,the default is 'localhost'
		:port,													#the SMTP server port, the default is 25
		:domain,												#the HELO domain provided by the client to the server,to default to 'localhost'
		:username,											#the SMTP authentication account name
		:password,                      #the SMTP authentication password
		:authentication,                #the authentication type, one of :plain,:login,:cram_md5
		:enable_starttls_auto           #enables SMTP/TLS

	def initialize(start_urls=[],            #Crawler entrances
								 filters={},          #urls filter
								 options={
									 thread_limit:1, #crawler thread limit
									 delay:0,
									 dbname:1,        #redis index
									 consumer_wait_queue_limit:200, 
									 encoding:nil,    #default encoding
								 },
								 &block
								)

		@start_urls=start_urls
		@filters=filters
		#the max Thread limit
		@thread_limit=        options[:thread_limit]
		@delay=       options[:delay]
		@encoding=            options[:encoding]
		@consumer_wait_queue_limit=options[:consumer_wait_queue_limit]
		@dbname=              options[:dbname]
		@consumer= Consumer::Parser.new()

		@logger=STDOUT

		#for email
		@enable_email=false
		@email=''
		@from=@email
		@address='localhost'
		@port=25
		@domain='localhost'
		@username=nil
		@password=<PASSWORD>
		@authentication='plain'
		@enable_starttls_auto=true

		instance_eval(&block) if block_given?
		#initialize logger
		$logger=Logger.new(@logger)
		$logger.datetime_format="%Y:%m:%m %H:%M:%S"
		$logger.formatter=proc{|serverity,datetime,programname,msg|
			"[#{datetime}]-[#{serverity}]-#{msg}"
		}


<commit_msg>add some comment<commit_after>#coding: utf-8
#version v8
#download with thread
#multi thread access unvisited queue and visited queue
require 'thread'
require 'mechanize'
require 'uri'
require 'thread'
require 'thwait'
require 'fileutils'
require 'logger'
require 'net/smtp'

Thread.abort_on_exception=true

trap('INT'){
	#logger cannot write in trap context
	$stderr.puts('exiting.............\n')
	exit 1
}

class Crawler
	VERSION = '0.0.1'
	attr_accessor :start_urls,				#crawler entrances
		:logger,												#the path to save the logger,it defalut to STDOUT
		:filters,												#the urls filters
		:thread_limit,                  #the Crawler thread limit
		:delay,													#specify how long to start next request
		:encoding,											#the website encoding,the default is nil.
		:consumer_wait_queue_limit,     #when the Consumer queue size great the this threshold,the Crawler will sleep 
		:dbname,												#the Redis database index
		:consumer,											#A Consumer which consume the 'Page' object.As you might expect the Crawler is a Producter
		#for email
		:enable_email,									#enable the email,if some problems led to the Crawler down,it will send a email to you
		:email,													#recipient's email
		:from,													#sender's email
		:address,												#the SMTP server ip address or hostname,the default is 'localhost'
		:port,													#the SMTP server port, the default is 25
		:domain,												#the HELO domain provided by the client to the server,to default to 'localhost'
		:username,											#the SMTP authentication account name
		:password,                      #the <PASSWORD>
		:authentication,                #the authentication type, one of :plain,:login,:cram_md5
		:enable_starttls_auto           #enables SMTP/TLS

	def initialize(start_urls=[],            #Crawler entrances
								 filters={},          #urls filter
								 options={
									 thread_limit:1, #crawler thread limit
									 delay:0,
									 dbname:1,        #redis index
									 consumer_wait_queue_limit:200, 
									 encoding:nil,    #default encoding
								 }
								)

		@start_urls=start_urls
		@filters=filters
		#the max Thread limit
		@thread_limit=        options[:thread_limit]
		@delay=       options[:delay]
		@encoding=            options[:encoding]
		@consumer_wait_queue_limit=options[:consumer_wait_queue_limit]
		@dbname=              options[:dbname]
		@consumer= Consumer::Parser.new()

		@logger=STDOUT

		#for email
		@enable_email=false
		@email=''
		@from=@email
		@address='localhost'
		@port=25
		@domain='localhost'
		@username=nil
		@password=<PASSWORD>
		@authentication='plain'
		@enable_starttls_auto=true

		yield self if block_given?
		#initialize logger
		$logger=Logger.new(@logger)
		$logger.datetime_format="%Y:%m:%m %H:%M:%S"
		$logger.formatter=proc{|serverity,datetime,programname,msg|
			"[#{datetime}]-[#{serverity}]-#{msg}"
		}



{'fraction_non_alphanumeric': 0.10018710665079095, 'fraction_numerical': 0.005783296478993026, 'mean_word_length': 2.798449612403101, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6908939', 'n_tokens_mistral': 2339, 'n_tokens_neox': 1792, 'n_words': 520}