<commit_before>require_relative "languages.rb"
require_relative "version.rb"

module Docks
  class Cache
    PARSE_RESULT_TYPES = [
      Docks::Types::Languages::STYLE,
      Docks::Types::Languages::MARKUP,
      Docks::Types::Languages::SCRIPT
    ]

    DIR = "docks_cache"
    GROUP_FILE = "docks_cache_groups"
    META_FILE = "docks_meta"

    def self.pattern_for(pattern)
      pattern = pattern.to_s
      cache_file = File.join(Docks.config.cache_location, pattern)

      unless File.exists?(cache_file)
        raise Docks::NoPatternError, "No pattern by the name of '#{pattern}' exists. Make sure you have a script, markup, or style file with that filename that is included in your 'docks_config' source directories."
      end

      Docks::Containers::Pattern.new(YAML::load_file(cache_file))
    end

    def self.pattern_groups
      patterns = {}

      if File.exists?(group_cache_file)
        YAML::load_file(group_cache_file).each_value do |details|
          group = details.delete(:group)
          patterns[group] ||= []
          patterns[group] << details
        end
      end

      patterns
    end

    private

    def self.group_cache_file
      File.join(Docks.config.cache_location, GROUP_FILE)
    end

    def self.meta_file
      File.join(Docks.config.cache_location, META_FILE)
    end



    public

    def initialize
      FileUtils.mkdir_p(Docks.config.cache_location)
      load_metadata
      load_group_cache
    end

    def clear
      FileUtils.rm_rf Dir[Docks.config.cache_location + "*"]
    end

    def <<(parse_result)
      return unless pattern_is_valid?(parse_result)

      id = parse_result[:name].to_s

      File.open(Docks.config.cache_location + id, 'w') do |file|
        file.write(parse_result.to_yaml)
      end

      @new_group_cache[id] = group_details(parse_result)
    end

    # When a pattern exists but did not need to be re-parsed, this needs to
    # be communicated so that we can keep track of what patterns still exist.
    def no_update(id)
      id = id.to_s
      @new_group_cache[id] = @old_group_cache[id]
    end


    def dump
      File.open(self.class.group_cache_file, "w") do |file|
        file.write(@new_group_cache.to_yaml)
      end

      File.open(self.class.meta_file, "w") do |file|
        file.write(@metadata.to_yaml)
      end

      # Clear out anything that didn't get written to the new cache
      @old_group_cache.keys.each do |id|
        FileUtils.rm_rf(Docks.config.cache_location + id.to_s) if @new_group_cache[id].nil?
      end
    end

    private

    def pattern_is_valid?(parse_results)
      !parse_results[:pattern].empty? || PARSE_RESULT_TYPES.any? { |parse_result_type| !parse_results[parse_result_type].empty? }
    end

    def load_metadata
      if File.exists?(self.class.meta_file)
        @metadata = YAML::load_file(self.class.meta_file) || Hash.new
      else
        @metadata = Hash.new
      end

      clear if !@metadata[:version].nil? && @metadata[:version] != VERSION
      @metadata[:version] = VERSION
    end
<commit_msg>Improve pattern validation<commit_after>require_relative "languages.rb"
require_relative "version.rb"

module Docks
  class Cache
    PARSE_RESULT_TYPES = [
      Docks::Types::Languages::STYLE,
      Docks::Types::Languages::MARKUP,
      Docks::Types::Languages::SCRIPT
    ]

    DIR = "docks_cache"
    GROUP_FILE = "docks_cache_groups"
    META_FILE = "docks_meta"

    def self.pattern_for(pattern)
      pattern = pattern.to_s
      cache_file = File.join(Docks.config.cache_location, pattern)

      unless File.exists?(cache_file)
        raise Docks::NoPatternError, "No pattern by the name of '#{pattern}' exists. Make sure you have a script, markup, or style file with that filename that is included in your 'docks_config' source directories."
      end

      Docks::Containers::Pattern.new(YAML::load_file(cache_file))
    end

    def self.pattern_groups
      patterns = {}

      if File.exists?(group_cache_file)
        YAML::load_file(group_cache_file).each_value do |details|
          group = details.delete(:group)
          patterns[group] ||= []
          patterns[group] << details
        end
      end

      patterns
    end

    private

    def self.group_cache_file
      File.join(Docks.config.cache_location, GROUP_FILE)
    end

    def self.meta_file
      File.join(Docks.config.cache_location, META_FILE)
    end



    public

    def initialize
      FileUtils.mkdir_p(Docks.config.cache_location)
      load_metadata
      load_group_cache
    end

    def clear
      FileUtils.rm_rf Dir[Docks.config.cache_location + "*"]
    end

    def <<(parse_result)
      return unless pattern_is_valid?(parse_result)

      id = parse_result[:name].to_s

      File.open(Docks.config.cache_location + id, 'w') do |file|
        file.write(parse_result.to_yaml)
      end

      @new_group_cache[id] = group_details(parse_result)
    end

    # When a pattern exists but did not need to be re-parsed, this needs to
    # be communicated so that we can keep track of what patterns still exist.
    def no_update(id)
      id = id.to_s
      @new_group_cache[id] = @old_group_cache[id]
    end


    def dump
      File.open(self.class.group_cache_file, "w") do |file|
        file.write(@new_group_cache.to_yaml)
      end

      File.open(self.class.meta_file, "w") do |file|
        file.write(@metadata.to_yaml)
      end

      # Clear out anything that didn't get written to the new cache
      @old_group_cache.keys.each do |id|
        FileUtils.rm_rf(Docks.config.cache_location + id.to_s) if @new_group_cache[id].nil?
      end
    end

    private

    def pattern_is_valid?(parse_results)
      !parse_results[:description].nil? || PARSE_RESULT_TYPES.any? { |parse_result_type| !parse_results[parse_result_type].empty? }
    end

    def load_metadata
      if File.exists?(self.class.meta_file)
        @metadata = YAML::load_file(self.class.meta_file) || Hash.new
      else
        @metadata = Hash.new
      end

      clear if !@metadata[:version].nil? && @metadata[:version] != VERSION
      @metadata[:version] = VERSION
    end

{'fraction_non_alphanumeric': 0.09864886863096207, 'fraction_numerical': 0.0, 'mean_word_length': 2.823273179838208, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6299454', 'n_tokens_mistral': 2033, 'n_tokens_neox': 1969, 'n_words': 428}