<commit_before># encoding: utf-8

require 'libis/ingester'
require 'fileutils'

module Libis
  module Ingester

    class Run < ::Libis::Workflow::Mongoid::Run

      field :producer, type: Hash
      field :material_flow
      field :ingest_dir

      belongs_to :ingest_model, inverse_of: :runs, class_name:  Libis::Ingester::IngestModel.to_s

      set_callback(:destroy, :before) do |document|
        dir = document.ingest_dir
        FileUtils.rmtree dir if dir && !dir.blank? && Dir.exist?(dir)
      end

      set_callback(:destroy, :after) do |document|
        job = document.job
        # noinspection RubyResolve
        job.runs.delete(document)
        job.save!
      end

      def workflow
        self.job.workflow
      end

      def producer
        result = self[:producer] || self.job.producer.key_symbols_to_strings
        self[:producer] ||= result unless self.frozen?
        result.key_strings_to_symbols
      end

      def material_flow
        result = self[:material_flow] || self.job.material_flow
        self[:material_flow] ||= result unless self.frozen?
        result
      end

      def ingest_dir
        result = self[:ingest_dir] || File.join(self.job.ingest_dir, self.ingest_sub_dir)
        self[:ingest_dir] ||= result unless self.frozen?
        result
      end

      def ingest_sub_dir
        self.name
      end

      def execute(options = {})
        action = options.delete('action') || :run
        options.each { |key,value| self.send(key, value) }
        case action.to_sym
          when :run, :restart
            self.action = :run
            self.remove_work_dir
            self.remove_items
            self.run :run
          when :retry
            self.action = :retry
            self.run :retry<commit_msg>Fix Run#execute: grab job's options<commit_after># encoding: utf-8

require 'libis/ingester'
require 'fileutils'

module Libis
  module Ingester

    class Run < ::Libis::Workflow::Mongoid::Run

      field :producer, type: Hash
      field :material_flow
      field :ingest_dir

      belongs_to :ingest_model, inverse_of: :runs, class_name:  Libis::Ingester::IngestModel.to_s

      set_callback(:destroy, :before) do |document|
        dir = document.ingest_dir
        FileUtils.rmtree dir if dir && !dir.blank? && Dir.exist?(dir)
      end

      set_callback(:destroy, :after) do |document|
        job = document.job
        # noinspection RubyResolve
        job.runs.delete(document)
        job.save!
      end

      def workflow
        self.job.workflow
      end

      def producer
        result = self[:producer] || self.job.producer.key_symbols_to_strings
        self[:producer] ||= result unless self.frozen?
        result.key_strings_to_symbols
      end

      def material_flow
        result = self[:material_flow] || self.job.material_flow
        self[:material_flow] ||= result unless self.frozen?
        result
      end

      def ingest_dir
        result = self[:ingest_dir] || File.join(self.job.ingest_dir, self.ingest_sub_dir)
        self[:ingest_dir] ||= result unless self.frozen?
        result
      end

      def ingest_sub_dir
        self.name
      end

      def execute(options = {})
        action = options.delete('action') || :run
        if self.options.empty?
          self.options = self.job.input
          self.save!
        end
        options.each { |key,value| self.send(key, value) }
        case action.to_sym
          when :run, :restart
            self.action = :run
            self.remove_work_dir
            self.remove_items
            self.run :run
          when :retry
            self.action = :retry
            self.run :retry
{'fraction_non_alphanumeric': 0.09663409337676439, 'fraction_numerical': 0.0005428881650380022, 'mean_word_length': 2.112331081081081, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2611748', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1117, 'n_words': 249}