<commit_before>module ActiveRecordImporter
  class OptionsBuilder
    def self.build(options = {})
      ImporterOptions.new(options)
    end
  end

  class CsvOptions
    include Virtus.value_object

    values do
      attribute :convert_values_to_numeric, Hash[Symbol => Array],
                default: { only: [:metric_amount] }
      attribute :remove_empty_values, Boolean, default: false
      attribute :comment_regexp, Regexp, default: Regexp.new(/^#=>/)
      attribute :force_utf8, Boolean, default: true
    end
  end

  class ImporterOptions
    include Virtus.model

    attribute :required_attributes, Array
    attribute :find_options, Array
    attribute :exclude_from_find_options, Array
    attribute :import_method, Symbol, default: :create
    attribute :scope, Symbol
    attribute :importable_columns, Array
    attribute :default_attributes, Hash
    attribute :csv_opts, CsvOptions, default: CsvOptions.new
    attribute :find_assoc_opts, Hash
    attribute :before_assign_of_attrs, Proc
    attribute :after_save_callbacks, Array
    attribute :state_machine_attr, Array
  end
end<commit_msg>add and remove import options<commit_after>module ActiveRecordImporter
  class OptionsBuilder
    def self.build(options = {})
      ImporterOptions.new(options)
    end
  end

  #
  # SmarterCSV is used to process the csv file
  # https://github.com/tilo/smarter_csv
  # I used some of it's options
  # Please refer to it's documentation
  #
  class CsvOptions
    include Virtus.value_object

    values do
      attribute :convert_values_to_numeric
      attribute :value_converters, Hash, default: nil
      attribute :remove_empty_values, Boolean, default: false
      attribute :comment_regexp, Regexp, default: Regexp.new(/^#=>/)
      attribute :force_utf8, Boolean, default: true
      attribute :chunk_size, Integer, default: 500
      attribute :col_sep, String, default: ','
    end
  end

  #
  # Import Option for every IMPORTABLE model
  #
  class ImporterOptions
    include Virtus.model

    attribute :find_options, Array
    attribute :exclude_from_find_options, Array
    attribute :scope, Symbol
    attribute :importable_columns, Array
    attribute :default_attributes, Hash
    attribute :csv_opts, CsvOptions, default: CsvOptions.new
    attribute :find_assoc_opts, Hash
    attribute :before_save
    attribute :after_save
    attribute :state_machine_attr, Array
  end
end
{'fraction_non_alphanumeric': 0.06947890818858561, 'fraction_numerical': 0.0020678246484698098, 'mean_word_length': 3.687984496124031, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3823990', 'n_tokens_mistral': 709, 'n_tokens_neox': 687, 'n_words': 198}