<commit_before>require "nokogiri"

class Wriggler
	def initialize(tags=[], directory="", subdirectories=true)
		@content = Hash[tags.map {|k| [k, []]}]		#Hash with content
		@subdirectories = subdirectories					#Default true for the existence of subdirs
		@directory = directory 										#Directory to grab files from

		navigate_directory
		# Writer.write_to_csv(@content)
	end

  private

  def navigate_directory
 		#Set the cwd to the given dir send to gather all nested files from there
 		Dir.chdir(@directory) 
 		gather_files
  end

  def gather_files
  	#Gathers all of the HTML or XML files from this and all subdirectories
    open_next_file("test1.xml")
    puts "=============="
    puts "1:"
    p @content
    puts "=============="
    puts ""
    open_next_file("test2.xml")
    puts "=============="
    puts "2:"
    p @content
    puts "=============="
    puts ""
    open_next_file("test3.xml")
    puts "=============="
    puts "3:"
    p @content
    puts "=============="
    puts ""
  end

  def open_next_file(file)
  	#Opens the next file on the list, depending on the extension passes it to HTML or XML
  	f = File.open(file)

  	if is_html?(file)
  		set_HTML(f)
  	elsif is_xml?(file)
  		set_XML(f)
  	end
  end

  def is_html?(file)
  	#Determines, using a regex check, if it is an HTML file
  	file =~ /.html/
  end

  def is_xml?(file)
  	#Determines, using a regex check, if it is an XML file
  	file =~ /.xml/
  end

  def set_HTML(file)
  	#Set the HTML file into Nokogiri for crawling
  	doc = Nokogiri::HTML(file)
  	crawl_file(doc)
  end

  def set_XML(file)
  	#Set the XML file into Nokogiri for crawling
  	doc = Nokogiri::XML(file)
  	crawl_file(doc)
  end

  def crawl_file(doc)
  	#Crawl the Nokogiri Object for the file
  	@content.each_key do |key|
  		if !doc.xpath("//#{key}").empty?				#Returns an empty array if tag is not present
  			doc.xpath("//#{key}").map{ |tag| @content.fetch(key) << sanitize(tag.text) }
  		end
  	end
  end

  def sanitize(text)
  	#Removes any escaped quotes, replaces them
  	text.gsub(/"/, "'")					
  end
end

test = Wriggler.new(["character", "content", "name", "title"], "/Users/47900/Desktop/Ruby/wriggler/dirtest", false)<commit_msg>Change crawl_file method to sort gathered tags by file<commit_after>require "nokogiri"

class Wriggler
	def initialize(tags=[], directory="", subdirectories=true)
		@content = Hash[tags.map {|k| [k, []]}]		#Hash with content
		@subdirectories = subdirectories					#Default true for the existence of subdirs
		@directory = directory 										#Directory to grab files from

		navigate_directory
		# Writer.write_to_csv(@content)
	end

  private

  def navigate_directory
 		#Set the cwd to the given dir send to gather all nested files from there
 		Dir.chdir(@directory) 
 		gather_files
  end

  def gather_files
  	#Gathers all of the HTML or XML files from this and all subdirectories
    open_next_file("test1.xml")
    puts "=============="
    puts "1:"
    p @content
    puts "=============="
    puts ""
    open_next_file("test2.xml")
    puts "=============="
    puts "2:"
    p @content
    puts "=============="
    puts ""
    open_next_file("test3.xml")
    puts "=============="
    puts "3:"
    p @content
    puts "=============="
    puts ""
  end

  def open_next_file(file)
  	#Opens the next file on the list, depending on the extension passes it to HTML or XML
  	f = File.open(file)

  	if is_html?(file)
  		set_HTML(f)
  	elsif is_xml?(file)
  		set_XML(f)
  	end
  end

  def is_html?(file)
  	#Determines, using a regex check, if it is an HTML file
  	file =~ /.html/
  end

  def is_xml?(file)
  	#Determines, using a regex check, if it is an XML file
  	file =~ /.xml/
  end

  def set_HTML(file)
  	#Set the HTML file into Nokogiri for crawling
  	doc = Nokogiri::HTML(file)
  	crawl_file(doc)
  end

  def set_XML(file)
  	#Set the XML file into Nokogiri for crawling
  	doc = Nokogiri::XML(file)
  	crawl_file(doc)
  end

  def crawl_file(doc)
  	#Crawl the Nokogiri Object for the file
  	@content.each_key do |key|
      arr = []
  		if !doc.xpath("//#{key}").empty?				#Returns an empty array if tag is not present
  			doc.xpath("//#{key}").map{ |tag| arr << sanitize(tag.text) }
  		end
      @content.fetch(key) << arr
  	end
  end

  def sanitize(text)
  	#Removes any escaped quotes, replaces them
  	text.gsub(/"/, "'")					
  end
end

test = Wriggler.new(["character", "content", "name", "title"], "/Users/47900/Desktop/Ruby/wriggler/dirtest", false)
{'fraction_non_alphanumeric': 0.1441222050033208, 'fraction_numerical': 0.004870489262785035, 'mean_word_length': 3.073940486925158, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6209982', 'n_tokens_mistral': 1722, 'n_tokens_neox': 1643, 'n_words': 474}