<commit_before>require 'bricolage/application'
require 'bricolage/context'
require 'bricolage/jobnet'
require 'bricolage/taskqueue'
require 'bricolage/job'
require 'bricolage/jobresult'
require 'bricolage/datasource'
require 'bricolage/variables'
require 'bricolage/eventhandlers'
require 'bricolage/logger'
require 'bricolage/exception'
require 'bricolage/version'
require 'pathname'
require 'optparse'

module Bricolage

  class JobNetRunner
    def JobNetRunner.main
      new.main
    end

    def initialize
      Signal.trap('PIPE', 'IGNORE')
      @hooks = ::Bricolage
      @jobnet_id = nil
      @jobnet_start_time = Time.now
      @log_path = nil
    end

    EXIT_SUCCESS = JobResult::EXIT_SUCCESS
    EXIT_FAILURE = JobResult::EXIT_FAILURE
    EXIT_ERROR = JobResult::EXIT_ERROR

    def main
      opts = Options.new(self)
      @hooks.run_before_option_parsing_hooks(opts)
      opts.parse ARGV
      @ctx = Context.for_application(nil, opts.jobnet_file, environment: opts.environment, global_variables: opts.global_variables)
      @jobnet_id = "#{opts.jobnet_file.dirname.basename}/#{opts.jobnet_file.basename('.jobnet')}"
      @log_path = opts.log_path
      jobnet = RootJobNet.load(@ctx, opts.jobnet_file)
      queue = get_queue(opts)
      if queue.locked?
        raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help}"
      end
      unless queue.queued?
        enqueue_jobs jobnet, queue
        logger.info "jobs are queued." if opts.queue_exist?
      end
      if opts.list_jobs?
        list_jobs queue
        exit EXIT_SUCCESS
      end
      check_jobs queue
      if opts.check_only?
        puts "OK"
        exit EXIT_SUCCESS
      end
      run_queue queue
      exit EXIT_SUCCESS
    rescue OptionError => ex
      raise if $DEBUG
      usage_exit ex.message, opts.help
    rescue ApplicationError => ex
      raise if $DEBUG
      error_exit ex.message
    end

    def logger
      @ctx.logger
    end

    def get_queue(opts)
      if opts.queue_path
        FileTaskQueue.restore_if_exist(opts.queue_path)
      else
        TaskQueue.new
      end
    end

    def enqueue_jobs(jobnet, queue)
      seq = 1
      jobnet.sequential_jobs.each do |ref|
        queue.enq JobTask.new(ref)
        seq += 1
      end
      queue.save
    end

    def list_jobs(queue)
      queue.each do |task|
        puts task.job
      end
    end

    def check_jobs(queue)
      queue.each do |task|
        Job.load_ref(task.job, @ctx).compile
      end
    end

    def run_queue(queue)
      @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@jobnet_id, queue))
      queue.consume_each do |task|
        result = execute_job(task.job, queue)
        unless result.success?
          logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
          logger.error "[job #{task.job}] #{result.message}"
          @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(false, queue))
          exit result.status
        end
      end
      @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(true, queue))
      logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
      logger.info "status all green"
    end

    def execute_job(ref, queue)
      logger.debug "job #{ref}"
      job = Job.load_ref(ref, @ctx)
      job.compile
      @hooks.run_before_job_hooks(BeforeJobEvent.new(ref))
      result = job.execute_in_process(make_log_path(ref))
      @hooks.run_after_job_hooks(AfterJobEvent.new(result))
      result
    rescue Exception => ex
      logger.exception ex
      logger.error "unexpected error: #{ref} (#{ex.class}: #{ex.message})"
      JobResult.error(ex)
    end

    def make_log_path(job_ref)
      return nil unless @log_path
      start_time = Time.now
      @log_path.gsub(/%\{\w+\}/) {|var|
        case var
        when '%{jobnet_start_date}' then @jobnet_start_time.strftime('%Y%m%d')
        when '%{jobnet_start_time}' then @jobnet_start_time.strftime('%Y%m%d_%H%M%S%L')
        when '%{job_start_date}' then start_time.strftime('%Y%m%d')
        when '%{job_start_time}' then start_time.strftime('%Y%m%d_%H%M%S%L')
        when '%{jobnet}', '%{net}', '%{jobnet_id}', '%{net_id}', '%{flow}', '%{flow_id}' then @jobnet_id.gsub('/', '::')
        when '%{subsystem}' then job_ref.subsystem
        when '%{job}', '%{job_id}' then job_ref.name
        else
          raise ParameterError, "bad log path variable: #{var}"
        end
      }
    end

    def usage_exit(msg, usage)
      print_error msg
      $stderr.puts usage
      exit 1
    end

    def error_exit(msg)
      print_error msg
      exit 1
    end

    def print_error(msg)
      $stderr.puts "#{program_name}: error: #{msg}"
    end<commit_msg>[FIX] --help and --version option did not work<commit_after>require 'bricolage/application'
require 'bricolage/context'
require 'bricolage/jobnet'
require 'bricolage/taskqueue'
require 'bricolage/job'
require 'bricolage/jobresult'
require 'bricolage/datasource'
require 'bricolage/variables'
require 'bricolage/eventhandlers'
require 'bricolage/logger'
require 'bricolage/exception'
require 'bricolage/version'
require 'pathname'
require 'optparse'

module Bricolage

  class JobNetRunner
    def JobNetRunner.main
      new.main
    end

    def initialize
      Signal.trap('PIPE', 'IGNORE')
      @hooks = ::Bricolage
      @jobnet_id = nil
      @jobnet_start_time = Time.now
      @log_path = nil
    end

    EXIT_SUCCESS = JobResult::EXIT_SUCCESS
    EXIT_FAILURE = JobResult::EXIT_FAILURE
    EXIT_ERROR = JobResult::EXIT_ERROR

    def main
      opts = Options.new(self)
      @hooks.run_before_option_parsing_hooks(opts)
      opts.parse ARGV
      @ctx = Context.for_application(nil, opts.jobnet_file, environment: opts.environment, global_variables: opts.global_variables)
      @jobnet_id = "#{opts.jobnet_file.dirname.basename}/#{opts.jobnet_file.basename('.jobnet')}"
      @log_path = opts.log_path
      jobnet = RootJobNet.load(@ctx, opts.jobnet_file)
      queue = get_queue(opts)
      if queue.locked?
        raise ParameterError, "Job queue is still locked. If you are sure to restart jobnet, #{queue.unlock_help}"
      end
      unless queue.queued?
        enqueue_jobs jobnet, queue
        logger.info "jobs are queued." if opts.queue_exist?
      end
      if opts.list_jobs?
        list_jobs queue
        exit EXIT_SUCCESS
      end
      check_jobs queue
      if opts.check_only?
        puts "OK"
        exit EXIT_SUCCESS
      end
      run_queue queue
      exit EXIT_SUCCESS
    rescue OptionError => ex
      raise if $DEBUG
      usage_exit ex.message, opts.help
    rescue ApplicationError => ex
      raise if $DEBUG
      error_exit ex.message
    end

    def logger
      @ctx.logger
    end

    def get_queue(opts)
      if opts.queue_path
        FileTaskQueue.restore_if_exist(opts.queue_path)
      else
        TaskQueue.new
      end
    end

    def enqueue_jobs(jobnet, queue)
      seq = 1
      jobnet.sequential_jobs.each do |ref|
        queue.enq JobTask.new(ref)
        seq += 1
      end
      queue.save
    end

    def list_jobs(queue)
      queue.each do |task|
        puts task.job
      end
    end

    def check_jobs(queue)
      queue.each do |task|
        Job.load_ref(task.job, @ctx).compile
      end
    end

    def run_queue(queue)
      @hooks.run_before_all_jobs_hooks(BeforeAllJobsEvent.new(@jobnet_id, queue))
      queue.consume_each do |task|
        result = execute_job(task.job, queue)
        unless result.success?
          logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
          logger.error "[job #{task.job}] #{result.message}"
          @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(false, queue))
          exit result.status
        end
      end
      @hooks.run_after_all_jobs_hooks(AfterAllJobsEvent.new(true, queue))
      logger.elapsed_time 'jobnet total: ', (Time.now - @jobnet_start_time)
      logger.info "status all green"
    end

    def execute_job(ref, queue)
      logger.debug "job #{ref}"
      job = Job.load_ref(ref, @ctx)
      job.compile
      @hooks.run_before_job_hooks(BeforeJobEvent.new(ref))
      result = job.execute_in_process(make_log_path(ref))
      @hooks.run_after_job_hooks(AfterJobEvent.new(result))
      result
    rescue Exception => ex
      logger.exception ex
      logger.error "unexpected error: #{ref} (#{ex.class}: #{ex.message})"
      JobResult.error(ex)
    end

    def make_log_path(job_ref)
      return nil unless @log_path
      start_time = Time.now
      @log_path.gsub(/%\{\w+\}/) {|var|
        case var
        when '%{jobnet_start_date}' then @jobnet_start_time.strftime('%Y%m%d')
        when '%{jobnet_start_time}' then @jobnet_start_time.strftime('%Y%m%d_%H%M%S%L')
        when '%{job_start_date}' then start_time.strftime('%Y%m%d')
        when '%{job_start_time}' then start_time.strftime('%Y%m%d_%H%M%S%L')
        when '%{jobnet}', '%{net}', '%{jobnet_id}', '%{net_id}', '%{flow}', '%{flow_id}' then @jobnet_id.gsub('/', '::')
        when '%{subsystem}' then job_ref.subsystem
        when '%{job}', '%{job_id}' then job_ref.name
        else
          raise ParameterError, "bad log path variable: #{var}"
        end
      }
    end

    public :puts
    public :exit

    def usage_exit(msg, usage)
      print_error msg
      $stderr.puts usage
      exit 1
    end

    def error_exit(msg)
      print_error msg
      exit 1
    end

    def print_error(msg)
      $stderr.puts "#{program_name}: error: #{msg}"
    end
{'fraction_non_alphanumeric': 0.10916301398455437, 'fraction_numerical': 0.0008348987685243164, 'mean_word_length': 2.6758726505561947, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1658358', 'n_tokens_mistral': 3232, 'n_tokens_neox': 3237, 'n_words': 688}