<commit_before>require 'json'

class TurbotDockerRunner
  @queue = :turbot_docker_runs

  def self.perform(bot_name, run_id, run_uid, run_type)
    runner = TurbotDockerRunner.new(bot_name, run_id, run_uid, :run_type => run_type)
    runner.run
  end

  def initialize(bot_name, run_id, run_uid, run_params={})
    @bot_name = bot_name
    if run_id == 'draft'
      @run_id = 'draft'
    else
      @run_id = run_id.to_i
    end
    @run_uid = run_uid
    @run_params = run_params
    @run_ended = false
  end

  def run
    set_up
    status_code = run_in_container

    process_output
    metrics = read_metrics

    if !config['incremental'] && !config['manually_end_run'] # the former is legacy
      @run_ended = true
      send_run_ended_to_angler
    end

    report_run_ended(status_code, metrics)
  ensure
    clean_up
  end

  def set_up
    connect_to_rabbitmq
    set_up_directory(data_path)
    set_up_directory(output_path)
    synchronise_repo
    write_runtime_config

    clear_saved_vars if @run_params[:run_type] == 'first_of_scrape'

    @stdout_file = File.open(stdout_path, 'wb')
    @stdout_file.sync = true
    @stderr_file = File.open(stderr_path, 'wb')
    @stderr_file.sync = true
  end

  def clean_up
    @stdout_file.close if (@stdout_file && !@stdout_file.closed?)
    @stderr_file.close if (@stderr_file && !@stderr_file.closed?)
  end

  def connect_to_rabbitmq
    return if Hutch.connected?
    Rails.logger.info('Connecting to RabbitMQ')
    Hutch.connect({}, HutchConfig)
  end

  def set_up_directory(path)
    Rails.logger.info("Setting up #{path}")
    FileUtils.mkdir_p(path)
    FileUtils.chmod(0777, path)
  end

  def synchronise_repo
    if Dir.exists?(repo_path)
      Rails.logger.info("Pulling into #{repo_path}")
      Git.open(repo_path).pull
    else
      Rails.logger.info("Cloning #{git_url} into #{repo_path}")
      Git.clone(git_url, repo_path)
    end
  end

  def write_runtime_config
    runtime_config = {
      :name => @bot_name,
      :run_params => @run_params,
    }

    File.open(File.join(repo_path, 'runtime.json'), 'w') do |f|
      f.write(runtime_config.to_json)
    end
  end

  def clear_saved_vars
    FileUtils.rm_f(File.join(data_path, '_vars.yml'))
  end

  def run_in_container
    container = create_container

    begin
      binds = [
        "#{local_root_path}/#{repo_path}:/repo:ro",
        "#{local_root_path}/#{data_path}:/data",
        "#{local_root_path}/utils:/utils:ro"
      ]

      if Rails.env.production?
        # In production, output_path is an absolute path
        binds << "#{output_path}:/output"
      else
        binds << "#{local_root_path}/#{output_path}:/output"
      end

      Rails.logger.info("Starting container with bindings: #{binds}")
      container.start('Binds' => binds)

      container.attach do |stream, chunk|<commit_msg>Get bind paths right<commit_after>require 'json'

class TurbotDockerRunner
  @queue = :turbot_docker_runs

  def self.perform(bot_name, run_id, run_uid, run_type)
    runner = TurbotDockerRunner.new(bot_name, run_id, run_uid, :run_type => run_type)
    runner.run
  end

  def initialize(bot_name, run_id, run_uid, run_params={})
    @bot_name = bot_name
    if run_id == 'draft'
      @run_id = 'draft'
    else
      @run_id = run_id.to_i
    end
    @run_uid = run_uid
    @run_params = run_params
    @run_ended = false
  end

  def run
    set_up
    status_code = run_in_container

    process_output
    metrics = read_metrics

    if !config['incremental'] && !config['manually_end_run'] # the former is legacy
      @run_ended = true
      send_run_ended_to_angler
    end

    report_run_ended(status_code, metrics)
  ensure
    clean_up
  end

  def set_up
    connect_to_rabbitmq
    set_up_directory(data_path)
    set_up_directory(output_path)
    synchronise_repo
    write_runtime_config

    clear_saved_vars if @run_params[:run_type] == 'first_of_scrape'

    @stdout_file = File.open(stdout_path, 'wb')
    @stdout_file.sync = true
    @stderr_file = File.open(stderr_path, 'wb')
    @stderr_file.sync = true
  end

  def clean_up
    @stdout_file.close if (@stdout_file && !@stdout_file.closed?)
    @stderr_file.close if (@stderr_file && !@stderr_file.closed?)
  end

  def connect_to_rabbitmq
    return if Hutch.connected?
    Rails.logger.info('Connecting to RabbitMQ')
    Hutch.connect({}, HutchConfig)
  end

  def set_up_directory(path)
    Rails.logger.info("Setting up #{path}")
    FileUtils.mkdir_p(path)
    FileUtils.chmod(0777, path)
  end

  def synchronise_repo
    if Dir.exists?(repo_path)
      Rails.logger.info("Pulling into #{repo_path}")
      Git.open(repo_path).pull
    else
      Rails.logger.info("Cloning #{git_url} into #{repo_path}")
      Git.clone(git_url, repo_path)
    end
  end

  def write_runtime_config
    runtime_config = {
      :name => @bot_name,
      :run_params => @run_params,
    }

    File.open(File.join(repo_path, 'runtime.json'), 'w') do |f|
      f.write(runtime_config.to_json)
    end
  end

  def clear_saved_vars
    FileUtils.rm_f(File.join(data_path, '_vars.yml'))
  end

  def run_in_container
    container = create_container

    begin
      binds = [
        "#{repo_path}:/repo:ro",
        "#{data_path}:/data",
        "#{local_root_path}/utils:/utils:ro",
        "#{output_path}:/output"
      ]

      Rails.logger.info("Starting container with bindings: #{binds}")
      container.start('Binds' => binds)

      container.attach do |stream, chunk|
{'fraction_non_alphanumeric': 0.1119810633648944, 'fraction_numerical': 0.0014566642388929353, 'mean_word_length': 3.1676783004552354, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '939932', 'n_tokens_mistral': 1996, 'n_tokens_neox': 1938, 'n_words': 370}