<commit_before>require "wriggler/version"
require "nokogiri"
require "find"

module Wriggler
	attr_reader :content, :directory

  def self.crawl(tags=[], directory="")
    @content = Hash[tags.map {|k| [k, []]}]   #Hash with content
    @directory = directory                    #Current top-level directory

    navigate_directory
    Writer.write(@content)
  end

  private

  def self.navigate_directory
 		#Set the cwd to the given dir send to gather all nested files from there
 		Dir.chdir(@directory) 
 		open_files(gather_files)
  end

  def self.gather_files
  	#Gathers all of the HTML or XML files from this and all subdirectories into an array
    file_array = []
    Find.find(@directory) do |file|
      file_array << f if f.match(/\.xml\Z/) || f.match(/\.html\Z/)
    end
    file_array
  end

  def self.open_files
    #Opens all the files in the file_array
    file_array.each do |file|
      open_next_file(file)
    end
  end

  def self.open_next_file(file)
  	#Opens the next file on the list, depending on the extension passes it to HTML or XML
  	f = File.open(file)
<commit_msg>Fix bug on f vs. file<commit_after>require "wriggler/version"
require "nokogiri"
require "find"

module Wriggler
	attr_reader :content, :directory

  def self.crawl(tags=[], directory="")
    @content = Hash[tags.map {|k| [k, []]}]   #Hash with content
    @directory = directory                    #Current top-level directory

    navigate_directory
    Writer.write(@content)
  end

  private

  def self.navigate_directory
 		#Set the cwd to the given dir send to gather all nested files from there
 		Dir.chdir(@directory) 
 		open_files(gather_files)
  end

  def self.gather_files
  	#Gathers all of the HTML or XML files from this and all subdirectories into an array
    file_array = []
    Find.find(@directory) do |file|
      file_array << file if file.match(/\.xml\Z/) || file.match(/\.html\Z/)
    end
    file_array
  end

  def self.open_files
    #Opens all the files in the file_array
    file_array.each do |file|
      open_next_file(file)
    end
  end

  def self.open_next_file(file)
  	#Opens the next file on the list, depending on the extension passes it to HTML or XML
  	f = File.open(file)

{'fraction_non_alphanumeric': 0.09140027014858172, 'fraction_numerical': 0.0, 'mean_word_length': 3.0036036036036036, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3462526', 'n_tokens_mistral': 745, 'n_tokens_neox': 728, 'n_words': 237}