<commit_before>module OptimusPrime
  class Config
    require 'yaml'

    # parse YAML file and make available as attributes or via getter method
    attr_accessor :file_path, :sources, :transforms, :destinations, :unique_ids

    def initialize(file_path: nil)
      # add error checking to confirm that file_path is indeed a file

      raise 'file not found' unless File.file?(file_path)

      method(__method__).parameters.each do |type, k|
        next unless type == :key
        v = eval(k.to_s)
        instance_variable_set("@#{k}", v) unless v.nil?
      end

      @unique_ids = { 
        source: {}, 
        transform: {}, 
        destination: {}
      }

      @sources = Array.new

      self.parse_config
    end

    def parse_config
      yaml = YAML.load_file(self.file_path)

      yaml.each do |settings|
        if settings.has_key?("type")
          formatted = get_format_by_type(settings['type'])
          formatted.each do |key|
            raise "#{key} not found" unless settings.has_key?(key)
          end
          @sources.push settings if settings['type'] == 'source'
        else
          raise "yaml doesn't contain type" 
        end
      end
      self.check_for_duplicates(yaml)
    end

    def check_for_duplicates(yaml)
      yaml.each do |item|
        unless identifier_unique?(item['type'], item['unique_identifier'])
          raise "#{item['unique_identifier']} of #{item['type']} duplicate!" 
        end
        self.mark_as_seen(item['type'], item['unique_identifier'])
      end
    end

    def get_sources
      @sources
    end

    def get_source_by_id(unique_id)
      @sources.each do |source|
        return source if source['unique_identifier'] == unique_id
      end
      raise "#{unique_id} not exist"
    end

    def mark_as_seen(type, item)
      @unique_ids[type.to_sym][item] = true
    end

    def identifier_unique?(type, id)
      @unique_ids[type.to_sym][id].nil?
    end

    def get_format_by_type(type)
      case type
      when 'source' 
        ['unique_identifier', 'type', 'class', 'file_path', 'columns']
      when 'transform'
        ['unique_identifier', 'type', 'class', 'columns']
      when 'destination'
        ['unique_identifier', 'type', 'class', 'columns']
      else
        raise "type not found"
      end
        
    end

  end
end
<commit_msg>remove file_path from required key<commit_after>module OptimusPrime
  class Config
    require 'yaml'

    # parse YAML file and make available as attributes or via getter method
    attr_accessor :file_path, :sources, :transforms, :destinations, :unique_ids

    def initialize(file_path: nil)
      # add error checking to confirm that file_path is indeed a file

      raise 'file not found' unless File.file?(file_path)

      method(__method__).parameters.each do |type, k|
        next unless type == :key
        v = eval(k.to_s)
        instance_variable_set("@#{k}", v) unless v.nil?
      end

      @unique_ids = { 
        source: {}, 
        transform: {}, 
        destination: {}
      }

      @sources = Array.new

      self.parse_config
    end

    def parse_config
      yaml = YAML.load_file(self.file_path)

      yaml.each do |settings|
        if settings.has_key?("type")
          formatted = get_format_by_type(settings['type'])
          formatted.each do |key|
            raise "#{key} not found" unless settings.has_key?(key)
          end
          @sources.push settings if settings['type'] == 'source'
        else
          raise "yaml doesn't contain type" 
        end
      end
      self.check_for_duplicates(yaml)
    end

    def check_for_duplicates(yaml)
      yaml.each do |item|
        unless identifier_unique?(item['type'], item['unique_identifier'])
          raise "#{item['unique_identifier']} of #{item['type']} duplicate!" 
        end
        self.mark_as_seen(item['type'], item['unique_identifier'])
      end
    end

    def get_sources
      @sources
    end

    def get_source_by_id(unique_id)
      @sources.each do |source|
        return source if source['unique_identifier'] == unique_id
      end
      raise "#{unique_id} not exist"
    end

    def mark_as_seen(type, item)
      @unique_ids[type.to_sym][item] = true
    end

    def identifier_unique?(type, id)
      @unique_ids[type.to_sym][id].nil?
    end

    def get_format_by_type(type)
      case type
      when 'source' 
        ['unique_identifier', 'type', 'class', 'columns']
      when 'transform'
        ['unique_identifier', 'type', 'class', 'columns']
      when 'destination'
        ['unique_identifier', 'type', 'class', 'columns']
      else
        raise "type not found"
      end
        
    end

  end
end

{'fraction_non_alphanumeric': 0.10825180774138664, 'fraction_numerical': 0.0, 'mean_word_length': 2.3689111747851004, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2983492', 'n_tokens_mistral': 1446, 'n_tokens_neox': 1397, 'n_words': 371}