<commit_before>require 'stringio'

module Setup
  class FileDataType < DataType
    include RailsAdmin::Models::Setup::FileDataTypeAdmin

    validates_presence_of :namespace

    build_in_data_type.referenced_by(:namespace, :name).with(:namespace, :name, :title, :slug, :_type, :validators, :schema_data_type, :events, :before_save_callbacks, :records_methods, :data_type_methods)
    build_in_data_type.and({
                             properties: {
                               slug: {
                                 type: 'string'
                               }
                             }
                           }.deep_stringify_keys)

    allow :new, :import, :pull_import, :bulk_cross, :simple_cross, :bulk_expand, :simple_expand, :download_file, :copy, :switch_navigation

    shared_deny :simple_delete_data_type, :bulk_delete_data_type

    has_and_belongs_to_many :validators, class_name: Setup::Validator.to_s, inverse_of: nil
    belongs_to :schema_data_type, class_name: Setup::JsonDataType.to_s, inverse_of: nil

    before_save :validate_configuration

    def validate_configuration
      self.title = self.name if title.blank?
      validators_classes = Hash.new { |h, k| h[k] = [] }
      if validators.present?
        validators.each { |validator| validators_classes[validator.class] << validator }
        validators_classes.delete(Setup::AlgorithmValidator)
        if validators_classes.size == 1 &&
          (validators = validators_classes.values.first).size == 1 &&
          validators[0].is_a?(Setup::EdiValidator)
          self.schema_data_type = validators[0].first.schema_data_type
        else
          if schema_data_type.present?
            errors.add(:schema_data_type, 'is not allowed if no format validator is defined')
            self.schema_data_type = nil
          end
          if validators_classes.count > 1
            errors.add(:validators, "include validators of exclusive types: #{validators_classes.keys.to_a.collect(&:to_s).to_sentence}")
          end
          validators_classes.each do |validator_class, validators|
            errors.add(:validators, "include multiple validators of the same exclusive type #{validator_class}: #{validators.collect(&:name).to_sentence}") if validators.count > 1
          end
        end
      else
        errors.add(:schema_data_type, 'is not allowed if no format validator is defined') if schema_data_type.present?
        self.schema_data_type = nil
      end
      errors.blank?
    end

    def format_validator
      @format_validator ||= validators.detect { |validator| validator.is_a?(Setup::FormatValidator) }
    end

    def data_type_storage_collection_name
      "#{super}.files"
    end

    def chunks_storage_collection_name
      data_type_storage_collection_name.gsub(/files\Z/, 'chunks')
    end

    def all_data_type_storage_collections_names
      [data_type_storage_collection_name, chunks_storage_collection_name]
    end

    def mongoff_model_class
      Mongoff::GridFs::FileModel
    end

    def schema
      @schema ||= Mongoff::GridFs::FileModel::SCHEMA.merge('edi' => { 'segment' => name })
    end

    def validate_file(file)
      errors = []
      validators.each do |v|
        next if errors.present?
        errors += v.validate_file_record(file)
      end
      errors
    end

    def validate_file!(file)
      if (errors = validate_file(file)).present?
        raise Exception.new('Invalid file data: ' + errors.to_sentence)
      end
    end

    def new_from(string_or_readable, options = {})
      if options[:data_type_parser]
        super
      else
        options.reverse_merge!(default_attributes)
        file = records_model.new
        file.data = string_or_readable
        file<commit_msg>No default EDI segments for file data types<commit_after>require 'stringio'

module Setup
  class FileDataType < DataType
    include RailsAdmin::Models::Setup::FileDataTypeAdmin

    validates_presence_of :namespace

    build_in_data_type.referenced_by(:namespace, :name).with(:namespace, :name, :title, :slug, :_type, :validators, :schema_data_type, :events, :before_save_callbacks, :records_methods, :data_type_methods)
    build_in_data_type.and({
                             properties: {
                               slug: {
                                 type: 'string'
                               }
                             }
                           }.deep_stringify_keys)

    allow :new, :import, :pull_import, :bulk_cross, :simple_cross, :bulk_expand, :simple_expand, :download_file, :copy, :switch_navigation

    shared_deny :simple_delete_data_type, :bulk_delete_data_type

    has_and_belongs_to_many :validators, class_name: Setup::Validator.to_s, inverse_of: nil
    belongs_to :schema_data_type, class_name: Setup::JsonDataType.to_s, inverse_of: nil

    before_save :validate_configuration

    def validate_configuration
      self.title = self.name if title.blank?
      validators_classes = Hash.new { |h, k| h[k] = [] }
      if validators.present?
        validators.each { |validator| validators_classes[validator.class] << validator }
        validators_classes.delete(Setup::AlgorithmValidator)
        if validators_classes.size == 1 &&
          (validators = validators_classes.values.first).size == 1 &&
          validators[0].is_a?(Setup::EdiValidator)
          self.schema_data_type = validators[0].first.schema_data_type
        else
          if schema_data_type.present?
            errors.add(:schema_data_type, 'is not allowed if no format validator is defined')
            self.schema_data_type = nil
          end
          if validators_classes.count > 1
            errors.add(:validators, "include validators of exclusive types: #{validators_classes.keys.to_a.collect(&:to_s).to_sentence}")
          end
          validators_classes.each do |validator_class, validators|
            errors.add(:validators, "include multiple validators of the same exclusive type #{validator_class}: #{validators.collect(&:name).to_sentence}") if validators.count > 1
          end
        end
      else
        errors.add(:schema_data_type, 'is not allowed if no format validator is defined') if schema_data_type.present?
        self.schema_data_type = nil
      end
      errors.blank?
    end

    def format_validator
      @format_validator ||= validators.detect { |validator| validator.is_a?(Setup::FormatValidator) }
    end

    def data_type_storage_collection_name
      "#{super}.files"
    end

    def chunks_storage_collection_name
      data_type_storage_collection_name.gsub(/files\Z/, 'chunks')
    end

    def all_data_type_storage_collections_names
      [data_type_storage_collection_name, chunks_storage_collection_name]
    end

    def mongoff_model_class
      Mongoff::GridFs::FileModel
    end

    def schema
      @schema ||= Mongoff::GridFs::FileModel::SCHEMA
    end

    def validate_file(file)
      errors = []
      validators.each do |v|
        next if errors.present?
        errors += v.validate_file_record(file)
      end
      errors
    end

    def validate_file!(file)
      if (errors = validate_file(file)).present?
        raise Exception.new('Invalid file data: ' + errors.to_sentence)
      end
    end

    def new_from(string_or_readable, options = {})
      if options[:data_type_parser]
        super
      else
        options.reverse_merge!(default_attributes)
        file = records_model.new
        file.data = string_or_readable
        file
{'fraction_non_alphanumeric': 0.0891286970423661, 'fraction_numerical': 0.0015987210231814548, 'mean_word_length': 2.7255583126550866, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5917168', 'n_tokens_mistral': 2198, 'n_tokens_neox': 2177, 'n_words': 444}