<commit_before>class WhiskeyDisk
  class Config
    class EnvironmentScopeFilter
      attr_reader :config
      
      def initialize(config)
        @config = config
      end
      
      def repository_depth(data, depth = 0)
        raise 'no repository found' unless data.respond_to?(:has_key?)
        return depth if data.has_key?('repository')
        repository_depth(data.values.first, depth + 1)
      end

      # is this data hash a bottom-level data hash without an environment name?
      def needs_environment_scoping?(data)
        repository_depth(data) == 0
      end
      
      def filter(data)
        return data unless needs_environment_scoping?(data)
        { config.environment_name => data }
      end
    end
    
    class ProjectScopeFilter
      attr_reader :config
      
      def initialize(config)
        @config = config
      end
      
      def repository_depth(data, depth = 0)
        raise 'no repository found' unless data.respond_to?(:has_key?)
        return depth if data.has_key?('repository')
        repository_depth(data.values.first, depth + 1)
      end
      
      # is this data hash an environment data hash without a project name?
      def needs_project_scoping?(data)
        repository_depth(data) == 1
      end
      
      def filter(data)
        return data unless needs_project_scoping?(data)
        config.override_project_name!(data)
        { config.project_name => data }
      end
    end
    
    class Filter
      attr_reader :config
  
      def initialize(config)
        @config = config
      end
  
      def filter_data(data)
        current = EnvironmentScopeFilter.new(config).filter(data.clone)
        current = ProjectScopeFilter.new(config).filter(current)
        current = config.normalize_domains(current)
        current = config.select_project_and_environment(current)
        current = config.add_environment_name(current)<commit_msg>extract common Filter functionality to a base class<commit_after>class WhiskeyDisk
  class Config
    class AbstractFilter
      attr_reader :config
      
      def initialize(config)
        @config = config
      end
    end
    
    class EnvironmentScopeFilter < AbstractFilter      
      def repository_depth(data, depth = 0)
        raise 'no repository found' unless data.respond_to?(:has_key?)
        return depth if data.has_key?('repository')
        repository_depth(data.values.first, depth + 1)
      end

      # is this data hash a bottom-level data hash without an environment name?
      def needs_environment_scoping?(data)
        repository_depth(data) == 0
      end
      
      def filter(data)
        return data unless needs_environment_scoping?(data)
        { config.environment_name => data }
      end
    end
    
    class ProjectScopeFilter < AbstractFilter
      def repository_depth(data, depth = 0)
        raise 'no repository found' unless data.respond_to?(:has_key?)
        return depth if data.has_key?('repository')
        repository_depth(data.values.first, depth + 1)
      end
      
      # is this data hash an environment data hash without a project name?
      def needs_project_scoping?(data)
        repository_depth(data) == 1
      end
      
      def filter(data)
        return data unless needs_project_scoping?(data)
        config.override_project_name!(data)
        { config.project_name => data }
      end
    end
    
    class Filter
      attr_reader :config
  
      def initialize(config)
        @config = config
      end
  
      def filter_data(data)
        current = EnvironmentScopeFilter.new(config).filter(data.clone)
        current = ProjectScopeFilter.new(config).filter(current)
        current = config.normalize_domains(current)
        current = config.select_project_and_environment(current)
        current = config.add_environment_name(current)
{'fraction_non_alphanumeric': 0.07063389391979301, 'fraction_numerical': 0.0031047865459249676, 'mean_word_length': 2.424269264836138, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4090745', 'n_tokens_mistral': 1040, 'n_tokens_neox': 1030, 'n_words': 272}