<commit_before>class UserImportFile < ActiveRecord::Base
  include Statesman::Adapters::ActiveRecordModel
  include ImportFile
  default_scope {order('user_import_files.id DESC')}
  scope :not_imported, -> {in_state(:pending)}
  scope :stucked, -> {in_state(:pending).where('created_at < ? AND state = ?', 1.hour.ago)}

  if Setting.uploaded_file.storage == :s3
    has_attached_file :user_import, :storage => :s3,
      :s3_credentials => "#{Setting.amazon}",
      :s3_permissions => :private
  else
    has_attached_file :user_import,
      :path => ":rails_root/private/system/:class/:attachment/:id_partition/:style/:filename"
  end
  validates_attachment_content_type :user_import, :content_type => [
    'text/csv',
    'text/plain',
    'text/tab-separated-values',
    'application/octet-stream',
    'application/vnd.ms-excel'
  ]
  validates_attachment_presence :user_import
  belongs_to :user, :validate => true
  has_many :user_import_results

  has_many :user_import_file_transitions

  def state_machine
    @state_machine ||= UserImportFileStateMachine.new(self, transition_class: UserImportFileTransition)
  end

  delegate :can_transition_to?, :transition_to!, :transition_to, :current_state,
    to: :state_machine

  def import_start
    case edit_mode
    when 'create'
      import
    when 'update'
      modify
    when 'destroy'
      remove
    else
      import
    end
  end

  def import
    transition_to!(:started)
    num = {:user_imported => 0, :user_found => 0, :failed => 0}
    rows = open_import_file
    row_num = 2

    field = rows.first
    if [field['username']].reject { |f| f.to_s.strip == "" }.empty?
      raise "username column is not found"
    end

    rows.each do |row|
      next if row['dummy'].to_s.strip.present?
      import_result = UserImportResult.create!(
        :user_import_file_id => id, :body => row.fields.join("\t")
      )

      username = row['username']
      new_user = User.where(:username => username).first
      if new_user
        import_result.user = new_user
        import_result.save
        num[:user_found] += 1
      else
        new_user = User.new
        new_user.role = Role.where(:name => row['role']).first
        if new_user.role
          unless user.has_role?(new_user.role.name)
            num[:failed] += 1
            next
          end
        end
        new_user.operator = user<commit_msg>set default role<commit_after>class UserImportFile < ActiveRecord::Base
  include Statesman::Adapters::ActiveRecordModel
  include ImportFile
  default_scope {order('user_import_files.id DESC')}
  scope :not_imported, -> {in_state(:pending)}
  scope :stucked, -> {in_state(:pending).where('created_at < ? AND state = ?', 1.hour.ago)}

  if Setting.uploaded_file.storage == :s3
    has_attached_file :user_import, :storage => :s3,
      :s3_credentials => "#{Setting.amazon}",
      :s3_permissions => :private
  else
    has_attached_file :user_import,
      :path => ":rails_root/private/system/:class/:attachment/:id_partition/:style/:filename"
  end
  validates_attachment_content_type :user_import, :content_type => [
    'text/csv',
    'text/plain',
    'text/tab-separated-values',
    'application/octet-stream',
    'application/vnd.ms-excel'
  ]
  validates_attachment_presence :user_import
  belongs_to :user, :validate => true
  has_many :user_import_results

  has_many :user_import_file_transitions

  def state_machine
    @state_machine ||= UserImportFileStateMachine.new(self, transition_class: UserImportFileTransition)
  end

  delegate :can_transition_to?, :transition_to!, :transition_to, :current_state,
    to: :state_machine

  def import_start
    case edit_mode
    when 'create'
      import
    when 'update'
      modify
    when 'destroy'
      remove
    else
      import
    end
  end

  def import
    transition_to!(:started)
    num = {:user_imported => 0, :user_found => 0, :failed => 0}
    rows = open_import_file
    row_num = 2

    field = rows.first
    if [field['username']].reject { |f| f.to_s.strip == "" }.empty?
      raise "username column is not found"
    end

    rows.each do |row|
      next if row['dummy'].to_s.strip.present?
      import_result = UserImportResult.create!(
        :user_import_file_id => id, :body => row.fields.join("\t")
      )

      username = row['username']
      new_user = User.where(:username => username).first
      if new_user
        import_result.user = new_user
        import_result.save
        num[:user_found] += 1
      else
        new_user = User.new
        new_user.role = Role.where(:name => row['role']).first
        if new_user.role
          unless user.has_role?(new_user.role.name)
            num[:failed] += 1
            next
          end
        else
          new_user.role = Role.find(2) # User
        end
        new_user.operator = user
{'fraction_non_alphanumeric': 0.11767134599504542, 'fraction_numerical': 0.004748142031379026, 'mean_word_length': 2.981101068200493, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '219231', 'n_tokens_mistral': 1576, 'n_tokens_neox': 1545, 'n_words': 313}