<commit_before>require 'kernel'
require 'open3'

class Job < ActiveRecord::Base
  include Ansible

  belongs_to :project
  belongs_to :user
  belongs_to :task

  default_scope order('created_at DESC')
  default_scope where(:deleted_at => nil, :visible => true)

  validate :precense => :task, :if => :visible

  def self.deleted
    self.unscoped.where 'deleted_at IS NOT NULL'
  end

  def perform
    Rails.logger.info "Performing Job #{id}"
    if visible?
      Project::PullRepo.perform(project_id) unless project.pull_in_progress?

      success = true

      FileUtils.chdir project.repo.path do
        out, status = Open3.capture2e(task.name)

        puts out

        success = status.success?
      end

    else
      # perform system task
      case notes
      when 'PullRepo'
        Project::PullRepo.perform project_id
      when 'CloneRepo'
        Project::CloneRepo.perform project_id
      when 'RemoveRepo'
        Project::RemoveRepo.perform project_id
      end
    end
    update_attributes :completed_at => Time.now, :success => success
  end

  def complete?
    !completed_at.nil?
  end

  def command
    "#{stage} #{task}"
  end

  def puts(msg)
    update_attribute :results, (results_before_type_cast || '') + msg unless msg.blank?
  end

  def results
    unless (res = read_attribute(:results)).blank?
      escape_to_html res
    end
  end

  private

    def full_command
      %W(-f #{Rails.root.join('Capfile.repos')} -f Capfile -Xx#{verbosity}) + branch_setting + command.split(' ')
    end

    def branch_setting
      %W(-s branch=#{branch}) unless branch.blank?
    end

end<commit_msg>Now job execution log updated on the fly<commit_after>require 'kernel'
require 'open3'

class Job < ActiveRecord::Base
  include Ansible

  belongs_to :project
  belongs_to :user
  belongs_to :task

  default_scope order('created_at DESC')
  default_scope where(:deleted_at => nil, :visible => true)

  validate :precense => :task, :if => :visible

  def self.deleted
    self.unscoped.where 'deleted_at IS NOT NULL'
  end

  def perform
    Rails.logger.info "Performing Job #{id}"
    if visible?
      Project::PullRepo.perform(project_id) unless project.pull_in_progress?

      success = true

      FileUtils.chdir project.repo.path do
        Open3.popen2e(task.name) do |input, output_and_error, wait_thread|
          input.close
          while !output_and_error.eof?
            msg = output_and_error.readline
            update_attribute :results, (results_before_type_cast || '') + msg unless msg.blank?
          end

          success = wait_thread.value.success?
        end
      end
    else
      # perform system task
      case notes
      when 'PullRepo'
        Project::PullRepo.perform project_id
      when 'CloneRepo'
        Project::CloneRepo.perform project_id
      when 'RemoveRepo'
        Project::RemoveRepo.perform project_id
      end
    end
    update_attributes :completed_at => Time.now, :success => success
  end

  def complete?
    !completed_at.nil?
  end

  def command
    "#{stage} #{task}"
  end

  def results
    unless (res = read_attribute(:results)).blank?
      escape_to_html res
    end
  end

  private

    def full_command
      %W(-f #{Rails.root.join('Capfile.repos')} -f Capfile -Xx#{verbosity}) + branch_setting + command.split(' ')
    end

    def branch_setting
      %W(-s branch=#{branch}) unless branch.blank?
    end

end
{'fraction_non_alphanumeric': 0.09611451942740286, 'fraction_numerical': 0.0017528483786152498, 'mean_word_length': 2.775082690187431, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6913203', 'n_tokens_mistral': 1082, 'n_tokens_neox': 1071, 'n_words': 274}