<commit_before># encoding: UTF-8'
require 'sequel'
require 'fileutils'
require 'uuidtools'
require_relative './user'
require_relative './table'
require_relative '../../lib/cartodb/errors'
require_relative '../../lib/cartodb/metrics'
require_relative '../../lib/cartodb/github_reporter'
require_relative '../../lib/cartodb_stats'
require_relative '../../services/track_record/track_record/log'
require_relative '../../config/initializers/redis'
require_relative '../../services/importer/lib/importer'

class DataImport < Sequel::Model
  REDIS_LOG_KEY_PREFIX          = 'importer'
  REDIS_LOG_EXPIRATION_IN_SECS  = 3600 * 24 * 2 # 2 days

  attr_reader   :log

  PUBLIC_ATTRIBUTES = %W{ id user_id table_id data_type table_name state
    error_code queue_id get_error_text tables_created_count }

  def after_initialize
    instantiate_log
    self.results  = []
    self.state    ||= 'uploading'
  end #after_initialize

  def before_save
    self.logger = self.log.id unless self.logger.present?
    self.updated_at = Time.now
  end

  def public_values
    Hash[PUBLIC_ATTRIBUTES.map{ |attribute| [attribute, send(attribute)] }]
      .merge!("queue_id" => id)
      .merge!(success: success) if (state == 'complete' || state == 'failure')
  end

  def set_unsupported_file_error
    self.error_code = 1002
    self.state      = 'failure'
    save
  end #set_unsupported_file_error

  def run_import!
    success = !!dispatch
    if self.results.empty?
      set_unsupported_file_error
      return self
    end
    success ? handle_success : handle_failure
    Rails.logger.debug log.to_s
    self
  rescue => exception
    log.append "Exception: #{exception.to_s}"
    log.append exception.backtrace
    handle_failure
    self
  end

  def set_error_code(code)
    self.error_code = code
    self.save
  end

  def get_error_text
    return CartoDB::IMPORTER_ERROR_CODES[99999] if self.error_code.blank? 
    return CartoDB::IMPORTER_ERROR_CODES[self.error_code]
  end

  def raise_error_if_over_quota(number_of_tables)
    if !current_user.remaining_table_quota.nil? && <commit_msg>Report general import failure to Rollbar<commit_after># encoding: UTF-8'
require 'sequel'
require 'fileutils'
require 'uuidtools'
require_relative './user'
require_relative './table'
require_relative '../../lib/cartodb/errors'
require_relative '../../lib/cartodb/metrics'
require_relative '../../lib/cartodb/github_reporter'
require_relative '../../lib/cartodb_stats'
require_relative '../../services/track_record/track_record/log'
require_relative '../../config/initializers/redis'
require_relative '../../services/importer/lib/importer'

class DataImport < Sequel::Model
  REDIS_LOG_KEY_PREFIX          = 'importer'
  REDIS_LOG_EXPIRATION_IN_SECS  = 3600 * 24 * 2 # 2 days

  attr_reader   :log

  PUBLIC_ATTRIBUTES = %W{ id user_id table_id data_type table_name state
    error_code queue_id get_error_text tables_created_count }

  def after_initialize
    instantiate_log
    self.results  = []
    self.state    ||= 'uploading'
  end #after_initialize

  def before_save
    self.logger = self.log.id unless self.logger.present?
    self.updated_at = Time.now
  end

  def public_values
    Hash[PUBLIC_ATTRIBUTES.map{ |attribute| [attribute, send(attribute)] }]
      .merge!("queue_id" => id)
      .merge!(success: success) if (state == 'complete' || state == 'failure')
  end

  def set_unsupported_file_error
    self.error_code = 1002
    self.state      = 'failure'
    save
  end #set_unsupported_file_error

  def run_import!
    success = !!dispatch
    if self.results.empty?
      set_unsupported_file_error
      return self
    end
    success ? handle_success : handle_failure
    Rails.logger.debug log.to_s
    self
  rescue => exception
    log.append "Exception: #{exception.to_s}"
    log.append exception.backtrace
    stacktrace = exception.to_s + exception.backtrace.join
    Rollbar.report_message("Import error", "error", error_info: stacktrace)
    handle_failure
    self
  end

  def set_error_code(code)
    self.error_code = code
    self.save
  end

  def get_error_text
    return CartoDB::IMPORTER_ERROR_CODES[99999] if self.error_code.blank? 
    return CartoDB::IMPORTER_ERROR_CODES[self.error_code]
  end

  def raise_error_if_over_quota(number_of_tables)
    if !current_user.remaining_table_quota.nil? && 
{'fraction_non_alphanumeric': 0.10539667896678967, 'fraction_numerical': 0.008302583025830259, 'mean_word_length': 3.6335470085470085, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7428071', 'n_tokens_mistral': 1459, 'n_tokens_neox': 1414, 'n_words': 291}