<commit_before># == Schema Information
#
# Table name: source_files
#
#  id                  :integer(4)      not null, primary key
#  dataset_id          :integer(4)
#  source_file_name    :string(255)
#  source_content_type :string(50)
#  source_file_size    :integer(4)
#  source_updated_at   :datetime
#  status              :string(20)
#  header_rows_count   :integer(4)
#  data_rows_count     :integer(4)
#  imported_at         :datetime
#  error_message       :string(255)
#  error_at            :datetime
#  created_at          :datetime        not null
#  updated_at          :datetime        not null
#

require 'csv'

class SourceFile < ActiveRecord::Base
  belongs_to :dataset

  has_attached_file :source
  validates_attachment_presence :source
  validates_presence_of :dataset_id
  validates_uniqueness_of :source_file_name, :scope => :dataset_id, :message => "should be unique"

  attr_accessible :source

  state_machine :status, :initial => :new do
    state :new
    state :importing
    state :imported
    state :replaced
    state :error

    event :start_import do
      transition [:new, :replaced] => :importing, :if => :file_type_valid?
    end
    before_transition :on => :start_import, :do => :setup_dataset_table

    event :finish_import do
      transition :importing => :imported
    end
    before_transition :on => :finish_import, :do => :set_imported_at

    event :finish_with_error do
      transition :importing => :error
    end
    before_transition :on => :finish_with_error, :do => :set_error_at

    event :reset_new do
      transition any => :new
    end
  end

  scope :recent, order('created_at DESC')

  def preview
    @preview ||= begin
      load_preview_header_and_rows
      {
        :rows => @rows,
        :columns => preview_columns
      }
    end
  end

  AVAILABLE_DATA_TYPES = %w(string integer decimal date datetime)

  def self.available_data_types
    AVAILABLE_DATA_TYPES.map do |data_type|
      [data_type, data_type]
    end
  end

  def update_dataset_columns(column_overrides)
    load_preview_header_and_rows
    columns = preview_columns.map.with_index do |preview_column, i|
      if column_override = column_overrides[i]
        preview_column.merge column_override.symbolize_keys
      else
        preview_column
      end
    end
    dataset.update_columns columns
  end

  def import!
    start_import!
    perform_import
  end

  def perform_import
    # turn off Paperclip logging to avoid many
    #   [paperclip] Saving attachments.
    # in log file after each SourceFile save call
    paperclip_log = Paperclip.options[:log]
    Paperclip.options[:log] = false
    # do not trace import SQLs with with New Relic
    import_all_rows_result = disable_all_tracing do
      import_all_rows
    end
    if import_all_rows_result
      finish_import!
    else
      finish_with_error!
    end
  ensure
    Paperclip.options[:log] = paperclip_log
  end

  def destroy_and_delete_data
    deleted_rows = 0
    if !new?
      deleted_rows = dataset.delete_source_data(:type => 'file', :id => id)
    end
    destroy
    deleted_rows
  end

  def send_file_path
    file_path = source.path
    # modify file path for Apache + mod_xsendfile + Passenger
    file_path.gsub(%r{/releases/\d+/}, "/current/")
  end

  private

  def file_type_valid?
    source_content_type == 'text/csv'
  end<commit_msg>check if file has .csv extension if content type is not text/csv<commit_after># == Schema Information
#
# Table name: source_files
#
#  id                  :integer(4)      not null, primary key
#  dataset_id          :integer(4)
#  source_file_name    :string(255)
#  source_content_type :string(50)
#  source_file_size    :integer(4)
#  source_updated_at   :datetime
#  status              :string(20)
#  header_rows_count   :integer(4)
#  data_rows_count     :integer(4)
#  imported_at         :datetime
#  error_message       :string(255)
#  error_at            :datetime
#  created_at          :datetime        not null
#  updated_at          :datetime        not null
#

require 'csv'

class SourceFile < ActiveRecord::Base
  belongs_to :dataset

  has_attached_file :source
  validates_attachment_presence :source
  validates_presence_of :dataset_id
  validates_uniqueness_of :source_file_name, :scope => :dataset_id, :message => "should be unique"

  attr_accessible :source

  state_machine :status, :initial => :new do
    state :new
    state :importing
    state :imported
    state :replaced
    state :error

    event :start_import do
      transition [:new, :replaced] => :importing, :if => :file_type_valid?
    end
    before_transition :on => :start_import, :do => :setup_dataset_table

    event :finish_import do
      transition :importing => :imported
    end
    before_transition :on => :finish_import, :do => :set_imported_at

    event :finish_with_error do
      transition :importing => :error
    end
    before_transition :on => :finish_with_error, :do => :set_error_at

    event :reset_new do
      transition any => :new
    end
  end

  scope :recent, order('created_at DESC')

  def preview
    @preview ||= begin
      load_preview_header_and_rows
      {
        :rows => @rows,
        :columns => preview_columns
      }
    end
  end

  AVAILABLE_DATA_TYPES = %w(string integer decimal date datetime)

  def self.available_data_types
    AVAILABLE_DATA_TYPES.map do |data_type|
      [data_type, data_type]
    end
  end

  def update_dataset_columns(column_overrides)
    load_preview_header_and_rows
    columns = preview_columns.map.with_index do |preview_column, i|
      if column_override = column_overrides[i]
        preview_column.merge column_override.symbolize_keys
      else
        preview_column
      end
    end
    dataset.update_columns columns
  end

  def import!
    start_import!
    perform_import
  end

  def perform_import
    # turn off Paperclip logging to avoid many
    #   [paperclip] Saving attachments.
    # in log file after each SourceFile save call
    paperclip_log = Paperclip.options[:log]
    Paperclip.options[:log] = false
    # do not trace import SQLs with with New Relic
    import_all_rows_result = disable_all_tracing do
      import_all_rows
    end
    if import_all_rows_result
      finish_import!
    else
      finish_with_error!
    end
  ensure
    Paperclip.options[:log] = paperclip_log
  end

  def destroy_and_delete_data
    deleted_rows = 0
    if !new?
      deleted_rows = dataset.delete_source_data(:type => 'file', :id => id)
    end
    destroy
    deleted_rows
  end

  def send_file_path
    file_path = source.path
    # modify file path for Apache + mod_xsendfile + Passenger
    file_path.gsub(%r{/releases/\d+/}, "/current/")
  end

  private

  def file_type_valid?
    source_content_type == 'text/csv' || source_file_name =~ /\.csv$/i
  end
{'fraction_non_alphanumeric': 0.07830796252927401, 'fraction_numerical': 0.00468384074941452, 'mean_word_length': 2.9180045871559632, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1971158', 'n_tokens_mistral': 2216, 'n_tokens_neox': 2163, 'n_words': 542}