<commit_before># encoding: UTF-8
# Assume all files read in are UTF-8 format
if Object.const_defined? "Encoding"
	Encoding.default_external = Encoding.default_internal = 'UTF-8'
end

# Wicked monkey patch to avoid File.join verbosity everywhere
class String
	def / ( other )
		File.join( self, other )
	end
end

require 'fileutils'

module FileUtils
	def self.win_path( path )
		path.gsub( '/', '\\' )
	end
end

module DocuBot
	VERSION = '0.4'
	DIR     = File.expand_path( File.dirname( __FILE__ ) )
	
	TEMPLATE_DIR = DIR / 'docubot/templates'
	SHELL_DIR    = DIR / 'docubot/shells'
	Dir.chdir( SHELL_DIR ){ SHELLS = Dir['*'] }
	
	def self.id_from_text( text )
		"#" << text.strip.gsub(/[^\w.:-]+/,'-').gsub(/^[^a-z]+|-+$/i,'')
	end
end

require 'docubot/link_tree'
require 'docubot/metasection'
require 'docubot/snippet'
require 'docubot/converter'
require 'docubot/writer'
require 'docubot/page'
require 'docubot/glossary'
require 'docubot/index'<commit_msg>Bump version number for release<commit_after># encoding: UTF-8
# Assume all files read in are UTF-8 format
if Object.const_defined? "Encoding"
	Encoding.default_external = Encoding.default_internal = 'UTF-8'
end

# Wicked monkey patch to avoid File.join verbosity everywhere
class String
	def / ( other )
		File.join( self, other )
	end
end

require 'fileutils'

module FileUtils
	def self.win_path( path )
		path.gsub( '/', '\\' )
	end
end

module DocuBot
	VERSION = '0.5'
	DIR     = File.expand_path( File.dirname( __FILE__ ) )
	
	TEMPLATE_DIR = DIR / 'docubot/templates'
	SHELL_DIR    = DIR / 'docubot/shells'
	Dir.chdir( SHELL_DIR ){ SHELLS = Dir['*'] }
	
	def self.id_from_text( text )
		"#" << text.strip.gsub(/[^\w.:-]+/,'-').gsub(/^[^a-z]+|-+$/i,'')
	end
end

require 'docubot/link_tree'
require 'docubot/metasection'
require 'docubot/snippet'
require 'docubot/converter'
require 'docubot/writer'
require 'docubot/page'
require 'docubot/glossary'
require 'docubot/index'
{'fraction_non_alphanumeric': 0.14860681114551083, 'fraction_numerical': 0.005159958720330237, 'mean_word_length': 4.089238845144357, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5953774', 'n_tokens_mistral': 778, 'n_tokens_neox': 727, 'n_words': 173}