<commit_before>require 'yaml'

module Clayoven
  class ConfigData
    attr_accessor :rootpath, :rcpath, :ignorepath, :rc, :ignore

    def initialize
      @rootpath = ".clayoven"
      @rcpath = File.expand_path "~/.clayovenrc"
      @ignorepath = "#{rootpath}/ignore"

      # Most common patterns that should sit in .clayoven/ignore.
      # Written to the file when it doesn't exist.
      @ignore = ["\\.html$", "~$", "^.\#", "^\#.*\#$",
                 "^\\.git$", "^\\.gitignore$", "^\\.htaccess$"]
      @rc = nil

      Dir.mkdir @rootpath if not Dir.exists? @rootpath
      if File.exists? @ignorepath
        @ignore = IO.read(@ignorepath).split("\n")
      else
        File.open(@ignorepath, "w") do |ignoreio|
          ignoreio.write @ignore.join("\n")
        end
        puts "[NOTE] #{@ignorepath} populated with sane defaults"
      end
      @rc = YAML.load_file @rcpath if File.exists? @rcpath
    end
  end
end<commit_msg>config: break up ConfigData.initalize<commit_after>require 'yaml'

module Clayoven
  class ConfigData
    attr_accessor :rootpath, :rcpath, :ignorepath, :rc, :ignore

    def initialize
      @rootpath = ".clayoven"
      Dir.mkdir @rootpath if not Dir.exists? @rootpath

      initialize_ignore
      initialize_rc
    end

    def initialize_ignore
      @ignorepath = "#{rootpath}/ignore"

      # Most common patterns that should sit in .clayoven/ignore.
      # Written to the file when it doesn't exist.
      @ignore = ["\\.html$", "~$", "^.\#", "^\#.*\#$",
                 "^\\.git$", "^\\.gitignore$", "^\\.htaccess$"]

      if File.exists? @ignorepath
        @ignore = IO.read(@ignorepath).split("\n")
      else
        File.open(@ignorepath, "w") do |ignoreio|
          ignoreio.write @ignore.join("\n")
        end
        puts "[NOTE] #{@ignorepath} populated with sane defaults"
      end
    end

    def initialize_rc
      @rcpath = File.expand_path "~/.clayovenrc"
      @rc = nil
      @rc = YAML.load_file @rcpath if File.exists? @rcpath
    end
  end
end
{'fraction_non_alphanumeric': 0.15209876543209877, 'fraction_numerical': 0.0, 'mean_word_length': 2.3543046357615895, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7020971', 'n_tokens_mistral': 677, 'n_tokens_neox': 643, 'n_words': 170}