<commit_before># frozen_string_literal: true

module Gitlab
  module ImportExport
    class AttributeCleaner
      ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id']

      def self.clean(*args)
        new(*args).clean
      end

      def initialize(relation_hash:, relation_class:, excluded_keys: [])
        @relation_hash = relation_hash
        @relation_class = relation_class
        @excluded_keys = excluded_keys
      end

      def clean
        @relation_hash.reject do |key, _value|
          prohibited_key?(key) || !@relation_class.attribute_method?(key) || excluded_key?(key)
        end.except('id')
      end

      private

      def prohibited_key?(key)
        return false if allowed_reference?(key)

        return true if 'cached_markdown_version'.equal?(key)

        prohibited_suffices = %w(_id _html)
        prohibited_suffices.each do |suffix|
          return true if key.end_with?(suffix)
        end
        false
      end

      def allowed_reference?(key)
        ALLOWED_REFERENCES.include?(key)<commit_msg>Use English instead of Latin<commit_after># frozen_string_literal: true

module Gitlab
  module ImportExport
    class AttributeCleaner
      ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id']

      def self.clean(*args)
        new(*args).clean
      end

      def initialize(relation_hash:, relation_class:, excluded_keys: [])
        @relation_hash = relation_hash
        @relation_class = relation_class
        @excluded_keys = excluded_keys
      end

      def clean
        @relation_hash.reject do |key, _value|
          prohibited_key?(key) || !@relation_class.attribute_method?(key) || excluded_key?(key)
        end.except('id')
      end

      private

      def prohibited_key?(key)
        return false if allowed_reference?(key)

        return true if 'cached_markdown_version'.equal?(key)

        prohibited_suffixes = %w(_id _html)
        prohibited_suffixes.each do |suffix|
          return true if key.end_with?(suffix)
        end
        false
      end

      def allowed_reference?(key)
        ALLOWED_REFERENCES.include?(key)
{'fraction_non_alphanumeric': 0.08570139828597204, 'fraction_numerical': 0.0, 'mean_word_length': 2.804459691252144, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7650542', 'n_tokens_mistral': 667, 'n_tokens_neox': 663, 'n_words': 135}