<commit_before>module Morph
  class DockerRunner
    ALL_CONFIG_FILENAMES = ["Gemfile", "Gemfile.lock", "Procfile", "requirements.txt", "runtime.txt", "composer.json", "composer.lock", "cpanfile"]

    # options: repo_path, container_name, data_path, env_variables
    def self.compile_and_run(options)
      wrapper = Multiblock.wrapper
      yield(wrapper)

      i = compile_step1 do |s,c|
        wrapper.call(:log, s, c)
      end
      i2 = Dir.mktmpdir("morph") do |dest|
        write_all_config_with_defaults_to_directory(options[:repo_path], dest)
        compile_step2(i, dest) do |s,c|
          wrapper.call(:log, s, c)
        end
      end
      i3 = compile_step3(i2) do |s,c|
        wrapper.call(:log, s, c)
      end

      # If something went wrong during the compile and it couldn't finish
      if i3.nil?
        # TODO: Return the status for a compile error
        return 255;
      end

      i4 = Dir.mktmpdir("morph") do |dest|
        write_all_run_to_directory(options[:repo_path], dest)
        compile_step4(i3, dest) do |s,c|
          wrapper.call(:log, s, c)
        end
      end

      command = Metric.command("/start scraper", "/data/" + Run.time_output_filename)

      status_code = run(
        command: command,
        image_name: i4.id,
        container_name: options[:container_name],
        data_path: options[:data_path],
        env_variables: options[:env_variables]
      ) do |on|
          on.log { |s,c| wrapper.call(:log, s, c)}
          on.ip_address {|ip| wrapper.call(:ip_address, ip)}
      end

      # There's a potential race condition here where we are trying to delete something
      # that might be used elsewhere. Do the most crude thing and just ignore any errors
      # that deleting might throw up.
      begin
        i4.delete("noprune" => 1)
      # TODO When docker-api gem gets updated Docker::Error::ConfictError will be
      # changed to Docker::Error::ConflictError
      rescue Docker::Error::ConfictError
      end
      status_code
    end

    def self.stop(container_name)
      if container_exists?(container_name)
        c = Docker::Container.get(container_name)
        c.kill
      end
    end

    def self.write_all_config_with_defaults_to_directory(source, dest)
      ALL_CONFIG_FILENAMES.each do |config_filename|
        path = File.join(source, config_filename)
        FileUtils.cp(path, dest) if File.exists?(path)
      end

      # We don't need to check that the language is recognised because
      # the compiler is never called if the language isn't valid
      add_config_defaults_to_directory(dest, Morph::Language.language(source))

      fix_modification_times(dest)
    end

    def self.copy_directory_contents(source, dest)
      FileUtils.cp_r File.join(source, "."), dest
    end

    def self.write_all_run_to_directory(source, dest)
      copy_directory_contents(source, dest)

      ALL_CONFIG_FILENAMES.each do |path|
        FileUtils.rm_f(File.join(dest, path))
      end

      remove_hidden_directories(dest)

      # TODO I don't think I need to this step here
      fix_modification_times(dest)
    end

    # Set an arbitrary & fixed modification time on everything in a directory
    # This ensures that if the content is the same docker will cache
    def self.fix_modification_times(dir)
      Find.find(dir) do |entry|
        FileUtils.touch(entry, mtime: Time.new(2000,1,1))
      end
    end

    def self.update_docker_image!
      Morph::DockerUtils.pull_docker_image("openaustralia/buildstep")
    end

    def self.remove_stopped_containers!
      containers = Docker::Container.all(:all => true)
      containers = containers.select do |c|
        running = c.json["State"]["Running"]
        # Time ago in seconds that this finished
        finished_ago = Time.now - Time::iso8601(c.json["State"]["FinishedAt"])
        # Only show containers that have been stopped for more than 5 minutes
        !running && finished_ago > 5 * 60
      end<commit_msg>We only need to fix the modification time just before we tar up<commit_after>module Morph
  class DockerRunner
    ALL_CONFIG_FILENAMES = ["Gemfile", "Gemfile.lock", "Procfile", "requirements.txt", "runtime.txt", "composer.json", "composer.lock", "cpanfile"]

    # options: repo_path, container_name, data_path, env_variables
    def self.compile_and_run(options)
      wrapper = Multiblock.wrapper
      yield(wrapper)

      i = compile_step1 do |s,c|
        wrapper.call(:log, s, c)
      end
      i2 = Dir.mktmpdir("morph") do |dest|
        write_all_config_with_defaults_to_directory(options[:repo_path], dest)
        compile_step2(i, dest) do |s,c|
          wrapper.call(:log, s, c)
        end
      end
      i3 = compile_step3(i2) do |s,c|
        wrapper.call(:log, s, c)
      end

      # If something went wrong during the compile and it couldn't finish
      if i3.nil?
        # TODO: Return the status for a compile error
        return 255;
      end

      i4 = Dir.mktmpdir("morph") do |dest|
        write_all_run_to_directory(options[:repo_path], dest)
        compile_step4(i3, dest) do |s,c|
          wrapper.call(:log, s, c)
        end
      end

      command = Metric.command("/start scraper", "/data/" + Run.time_output_filename)

      status_code = run(
        command: command,
        image_name: i4.id,
        container_name: options[:container_name],
        data_path: options[:data_path],
        env_variables: options[:env_variables]
      ) do |on|
          on.log { |s,c| wrapper.call(:log, s, c)}
          on.ip_address {|ip| wrapper.call(:ip_address, ip)}
      end

      # There's a potential race condition here where we are trying to delete something
      # that might be used elsewhere. Do the most crude thing and just ignore any errors
      # that deleting might throw up.
      begin
        i4.delete("noprune" => 1)
      # TODO When docker-api gem gets updated Docker::Error::ConfictError will be
      # changed to Docker::Error::ConflictError
      rescue Docker::Error::ConfictError
      end
      status_code
    end

    def self.stop(container_name)
      if container_exists?(container_name)
        c = Docker::Container.get(container_name)
        c.kill
      end
    end

    def self.write_all_config_with_defaults_to_directory(source, dest)
      ALL_CONFIG_FILENAMES.each do |config_filename|
        path = File.join(source, config_filename)
        FileUtils.cp(path, dest) if File.exists?(path)
      end

      # We don't need to check that the language is recognised because
      # the compiler is never called if the language isn't valid
      add_config_defaults_to_directory(dest, Morph::Language.language(source))
    end

    def self.copy_directory_contents(source, dest)
      FileUtils.cp_r File.join(source, "."), dest
    end

    def self.write_all_run_to_directory(source, dest)
      copy_directory_contents(source, dest)

      ALL_CONFIG_FILENAMES.each do |path|
        FileUtils.rm_f(File.join(dest, path))
      end

      remove_hidden_directories(dest)
    end

    # Set an arbitrary & fixed modification time on everything in a directory
    # This ensures that if the content is the same docker will cache
    def self.fix_modification_times(dir)
      Find.find(dir) do |entry|
        FileUtils.touch(entry, mtime: Time.new(2000,1,1))
      end
    end

    def self.update_docker_image!
      Morph::DockerUtils.pull_docker_image("openaustralia/buildstep")
    end

    def self.remove_stopped_containers!
      containers = Docker::Container.all(:all => true)
      containers = containers.select do |c|
        running = c.json["State"]["Running"]
        # Time ago in seconds that this finished
        finished_ago = Time.now - Time::iso8601(c.json["State"]["FinishedAt"])
        # Only show containers that have been stopped for more than 5 minutes
        !running && finished_ago > 5 * 60
      end
{'fraction_non_alphanumeric': 0.09451604751074046, 'fraction_numerical': 0.0075815011372251705, 'mean_word_length': 2.7618821292775664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6313672', 'n_tokens_mistral': 2472, 'n_tokens_neox': 2380, 'n_words': 657}