<commit_before>class DocumentUpload < ApplicationRecord
  belongs_to :user, optional: true
  belongs_to :collection, optional: true

  attr_accessible :file, :collection_id, :status, :preserve_titles

  validates :collection_id, :file, :presence => true

  mount_uploader :file, DocumentUploader

  module Status
    NEW = 'new'
    QUEUED = 'queued'
    PROCESSING = 'processing'
    FINISHED = 'finished'
  end

  def submit_process
    self.status = Status::QUEUED
    self.save
    rake_call = "#{RAKE} fromthepage:process_document_upload[#{self.id}]  --trace 2>&1 >> #{log_file} &"

    # Nice-up the rake call if settings are present
    rake_call = "nice -n #{NICE_RAKE_LEVEL} " << rake_call if NICE_RAKE_ENABLED

    logger.info rake_call
    system(rake_call)
  end

  def log_file
    File.join(upload_dir, "process.log")
  end

  def name
    File.basename(self.file.to_s)
  end

private
  def upload_dir
    if self.file && self.file.path
      File.dirname(self.file.path)
    else
      "/tmp/fromthepage_rake.log"
    end
  end

end<commit_msg>document_upload: indent<commit_after>class DocumentUpload < ApplicationRecord
  belongs_to :user, optional: true
  belongs_to :collection, optional: true

  attr_accessible :file, :collection_id, :status, :preserve_titles

  validates :collection_id, :file, :presence => true

  mount_uploader :file, DocumentUploader

  module Status
    NEW = 'new'
    QUEUED = 'queued'
    PROCESSING = 'processing'
    FINISHED = 'finished'
  end

  def submit_process
    self.status = Status::QUEUED
    self.save
    rake_call = "#{RAKE} fromthepage:process_document_upload[#{self.id}]  --trace 2>&1 >> #{log_file} &"

    # Nice-up the rake call if settings are present
    rake_call = "nice -n #{NICE_RAKE_LEVEL} " << rake_call if NICE_RAKE_ENABLED

    logger.info rake_call
    system(rake_call)
  end

  def log_file
    File.join(upload_dir, "process.log")
  end

  def name
    File.basename(self.file.to_s)
  end

  private
  def upload_dir
    if self.file && self.file.path
      File.dirname(self.file.path)
    else
      "/tmp/fromthepage_rake.log"
    end
  end

end
{'fraction_non_alphanumeric': 0.10093896713615023, 'fraction_numerical': 0.0018779342723004694, 'mean_word_length': 3.402892561983471, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '943492', 'n_tokens_mistral': 760, 'n_tokens_neox': 734, 'n_words': 172}