<commit_before># typed: strict
# frozen_string_literal: true

module Morph
  # Utility methods for manipulating docker containers and preparing data
  # to inject into docker containers
  class DockerUtils
    extend T::Sig

    sig { params(image: String).returns(Docker::Image) }
    def self.pull_docker_image(image)
      Docker::Image.create("fromImage" => image) do |chunk|
        data = JSON.parse(chunk)
        Rails.logger.info "#{data['status']} #{data['id']} #{data['progress']}"
      end
    end

    # Returns temporary file which it is your responsibility
    # to remove after you're done with it
    sig { params(directory: String).returns(Tempfile) }
    def self.create_tar_file(directory)
      temp = Tempfile.new("morph_tar", Dir.tmpdir, encoding: "ascii-8bit")
      # We used to use Archive::Tar::Minitar but that doesn't support
      # symbolic links in the tar file. So, using tar from the command line
      # instead.
      `tar cf #{temp.path} -C #{directory} .`
      temp
    end

    sig { params(directory: String).returns(T.nilable(String)) }
    def self.create_tar(directory)
      temp = create_tar_file(directory)
      content = temp.read
      path = temp.path
      FileUtils.rm_f(path) if path
      content
    end

    sig { params(path: String, directory: String).void }
    def self.extract_tar_file(path, directory)
      # Quick and dirty
      `tar xf #{path} -C #{directory}`
    end

    sig { params(content: String, directory: String).void }
    def self.extract_tar(content, directory)
      # Use ascii-8bit as the encoding to ensure that the binary data isn't
      # changed on saving
      tmp = Tempfile.new("morph.tar", Dir.tmpdir, encoding: "ASCII-8BIT")
      tmp << content
      tmp.close
      extract_tar_file(T.must(tmp.path), directory)
      tmp.unlink
    end

    # If image is present locally use that. If it isn't then pull it from
    # the hub. This makes initial setup easier
    sig { params(name: String).returns(Docker::Image) }
    def self.get_or_pull_image(name)
      Docker::Image.get(name)
    rescue Docker::Error::NotFoundError
      Docker::Image.create("fromImage" => name) do |chunk|
        chunk.split("\n").each do |c|
          data = JSON.parse(c)
          Rails.logger.info "#{data['status']} #{data['id']} #{data['progress']}\n"
        end
      end
    end

    sig { params(container: Docker::Container, label_key: String).returns(T.nilable(String)) }
    def self.label_value(container, label_key)
      container.info["Labels"][label_key] if container.info && container.info["Labels"]
    end

    sig { params(container: Docker::Container, key: String, value: String).returns(T::Boolean) }
    def self.container_has_label_value?(container, key, value)
      label_value(container, key) == value
    end

    # Finds the first matching container
    # Returns nil otherwise
    sig { params(key: String, value: String).returns(T.nilable(Docker::Container)) }
    def self.find_container_with_label(key, value)
      # TODO: We can use the docker api to do this search
      Docker::Container.all(all: true).find do |container|
        container_has_label_value?(container, key, value)
      end
    end

    sig { params(key: String).returns(T::Array[Docker::Container]) }
    def self.find_all_containers_with_label(key)
      Docker::Container.all(all: true, filters: { label: [key.to_s] }.to_json)
    end

    sig { params(key: String, value: String).returns(T::Array[Docker::Container]) }
    def self.find_all_containers_with_label_and_value(key, value)
      Docker::Container.all(all: true, filters: { label: ["#{key}=#{value}"] }.to_json)
    end

    sig { params(source: String, dest: String).void }
    def self.copy_directory_contents(source, dest)
      FileUtils.cp_r File.join(source, "."), dest
    end

    # Set an arbitrary & fixed modification time on everything in a directory
    # This ensures that if the content is the same docker will cache
    sig { params(dir: String).void }
    def self.fix_modification_times(dir)
      Find.find(dir) do |entry|
        FileUtils.touch(entry, mtime: Time.zone.local(2000, 1, 1).time)
      end
    end

    # Copy a single file from a container. Returns a temp file with the contents
    # of the file from the container. Obviously need to provide a filesystem
    # path within the container
    sig { params(container: Docker::Container, path: String).returns(T.nilable(Tempfile)) }
    def self.copy_file(container, path)
      # Use ascii-8bit as the encoding to ensure that the binary data isn't
      # changed on saving
      # Saving everything directly to a temporary file so we don't have to fill
      # up our memory
      tmp = Tempfile.new("morph.tar", Dir.tmpdir, encoding: "ASCII-8BIT")
      begin
        container.archive_out(path) { |chunk| tmp << chunk }
      rescue Docker::Error::NotFoundError
        # If the path isn't found
        return nil
      end
      tmp.close

      # Now extract the tar file and return the contents of the file
      Dir.mktmpdir("morph") do |dest|
        extract_tar_file(T.must(tmp.path), dest)
        tmp.unlink

        path2 = File.join(dest, Pathname.new(path).basename.to_s)
        tmp = Tempfile.new("morph-file")
        FileUtils.cp(path2, T.must(tmp.path))
      end
      tmp
    end

    # Get a set of files from a container and return them as a hash of
    # local temporary files
    sig { params(container: Docker::Container, paths: T::Array[String]).returns(T::Hash[String, T.nilable(Tempfile)]) }
    def self.copy_files(container, paths)
      data = {}
      paths.each do |path|
        data[path] = copy_file(container, path)
      end
      data
    end

    sig { returns(T::Array[Docker::Container]) }
    def self.stopped_containers
      Docker::Container.all(all: true).reject do |c|
        c.json["State"]["Running"]
      end
    end

    sig { returns(T::Array[Docker::Container]) }
    def self.running_containers
      Docker::Container.all
    end

    sig { params(dir: String, connection_options: T::Hash[Symbol, Integer], block: T.proc.params(text: String).void).returns(T.nilable(Docker::Image)) }
    def self.docker_build_from_dir(dir, connection_options, &block)
      # How does this connection get closed?
      connection = docker_connection(connection_options)
      temp = create_tar_file(dir)
      buffer = +""
      Docker::Image.build_from_tar(
        temp, { "forcerm" => 1 }, connection
      ) do |chunk|
        # Sometimes a chunk contains multiple lines of json
        chunk.split("\n").each do |line|
          parsed_line = JSON.parse(line)
          next unless parsed_line.key?("stream")

          buffer << parsed_line["stream"]
          # Buffer output until an end-of-line is detected. This
          # makes line output more consistent across platforms.
          # Make sure that buffer can't grow out of control by limiting
          # it's size around 256 bytes
          if buffer[-1..-1] == "\n" || buffer.length >= 256
            block.call buffer
            buffer = +""
          end
        end
        block.call buffer if buffer != ""
      end
    # This exception gets thrown if there is an error during the build (for
    # example if the compile fails). In this case we just want to return nil
    rescue Docker::Error::UnexpectedResponseError
      nil
    end

    sig { params(options: T::Hash[Symbol, Integer]).returns(Docker::Connection) }
    def self.docker_connection(options)
      Docker::Connection.new(Docker.url, Docker.env_options.merge(options))
    end

    # Copy the contents of "src" to the directory dest in the container c
    sig { params(container: Docker::Container, src: String, dest: String).void }
    def self.insert_contents_of_directory(container, src, dest)
      # Rather than using archive_in we're doing this more roundabout way
      # because archive_in seems to have very broken handling of directories
      # TODO Submit a fix to the docker-api gem to fix this
      # In the meantime get something more long-winded working here

      tar_file = Docker::Util.create_dir_tar(src).path
      File.open(tar_file, "rb") do |tar|
        container.archive_in_stream(dest) do
          tar.read(Excon.defaults[:chunk_size]).to_s
        end
      end
      File.delete(tar_file)
    end

    # Inserts a single file into a container.
    # Not using archive_in because that doesn't maintain file permissions
    sig { params(container: Docker::Container, src: String, dest: String).void }
    def self.insert_file(container, src, dest)
      # This is very roundabout
      Dir.mktmpdir("morph") do |tmp_dir|
        FileUtils.cp(src, tmp_dir)
        insert_contents_of_directory(container, tmp_dir, dest)
      end
    end

    # TODO: There's probably a more sensible way of doing this
    sig { params(image: Docker::Image, image_base: Docker::Image).returns(T::Boolean) }
    def self.image_built_on_other_image?(image, image_base)
      index = image.history.find_index { |l| l["Id"] == image_base.id }
      index&.nonzero?
    end

    # This returns the total size of all the layers down to but not include the<commit_msg>Fix actual returned type for method<commit_after># typed: strict
# frozen_string_literal: true

module Morph
  # Utility methods for manipulating docker containers and preparing data
  # to inject into docker containers
  class DockerUtils
    extend T::Sig

    sig { params(image: String).returns(Docker::Image) }
    def self.pull_docker_image(image)
      Docker::Image.create("fromImage" => image) do |chunk|
        data = JSON.parse(chunk)
        Rails.logger.info "#{data['status']} #{data['id']} #{data['progress']}"
      end
    end

    # Returns temporary file which it is your responsibility
    # to remove after you're done with it
    sig { params(directory: String).returns(Tempfile) }
    def self.create_tar_file(directory)
      temp = Tempfile.new("morph_tar", Dir.tmpdir, encoding: "ascii-8bit")
      # We used to use Archive::Tar::Minitar but that doesn't support
      # symbolic links in the tar file. So, using tar from the command line
      # instead.
      `tar cf #{temp.path} -C #{directory} .`
      temp
    end

    sig { params(directory: String).returns(T.nilable(String)) }
    def self.create_tar(directory)
      temp = create_tar_file(directory)
      content = temp.read
      path = temp.path
      FileUtils.rm_f(path) if path
      content
    end

    sig { params(path: String, directory: String).void }
    def self.extract_tar_file(path, directory)
      # Quick and dirty
      `tar xf #{path} -C #{directory}`
    end

    sig { params(content: String, directory: String).void }
    def self.extract_tar(content, directory)
      # Use ascii-8bit as the encoding to ensure that the binary data isn't
      # changed on saving
      tmp = Tempfile.new("morph.tar", Dir.tmpdir, encoding: "ASCII-8BIT")
      tmp << content
      tmp.close
      extract_tar_file(T.must(tmp.path), directory)
      tmp.unlink
    end

    # If image is present locally use that. If it isn't then pull it from
    # the hub. This makes initial setup easier
    sig { params(name: String).returns(Docker::Image) }
    def self.get_or_pull_image(name)
      Docker::Image.get(name)
    rescue Docker::Error::NotFoundError
      Docker::Image.create("fromImage" => name) do |chunk|
        chunk.split("\n").each do |c|
          data = JSON.parse(c)
          Rails.logger.info "#{data['status']} #{data['id']} #{data['progress']}\n"
        end
      end
    end

    sig { params(container: Docker::Container, label_key: String).returns(T.nilable(String)) }
    def self.label_value(container, label_key)
      container.info["Labels"][label_key] if container.info && container.info["Labels"]
    end

    sig { params(container: Docker::Container, key: String, value: String).returns(T::Boolean) }
    def self.container_has_label_value?(container, key, value)
      label_value(container, key) == value
    end

    # Finds the first matching container
    # Returns nil otherwise
    sig { params(key: String, value: String).returns(T.nilable(Docker::Container)) }
    def self.find_container_with_label(key, value)
      # TODO: We can use the docker api to do this search
      Docker::Container.all(all: true).find do |container|
        container_has_label_value?(container, key, value)
      end
    end

    sig { params(key: String).returns(T::Array[Docker::Container]) }
    def self.find_all_containers_with_label(key)
      Docker::Container.all(all: true, filters: { label: [key.to_s] }.to_json)
    end

    sig { params(key: String, value: String).returns(T::Array[Docker::Container]) }
    def self.find_all_containers_with_label_and_value(key, value)
      Docker::Container.all(all: true, filters: { label: ["#{key}=#{value}"] }.to_json)
    end

    sig { params(source: String, dest: String).void }
    def self.copy_directory_contents(source, dest)
      FileUtils.cp_r File.join(source, "."), dest
    end

    # Set an arbitrary & fixed modification time on everything in a directory
    # This ensures that if the content is the same docker will cache
    sig { params(dir: String).void }
    def self.fix_modification_times(dir)
      Find.find(dir) do |entry|
        FileUtils.touch(entry, mtime: Time.zone.local(2000, 1, 1).time)
      end
    end

    # Copy a single file from a container. Returns a temp file with the contents
    # of the file from the container. Obviously need to provide a filesystem
    # path within the container
    sig { params(container: Docker::Container, path: String).returns(T.nilable(Tempfile)) }
    def self.copy_file(container, path)
      # Use ascii-8bit as the encoding to ensure that the binary data isn't
      # changed on saving
      # Saving everything directly to a temporary file so we don't have to fill
      # up our memory
      tmp = Tempfile.new("morph.tar", Dir.tmpdir, encoding: "ASCII-8BIT")
      begin
        container.archive_out(path) { |chunk| tmp << chunk }
      rescue Docker::Error::NotFoundError
        # If the path isn't found
        return nil
      end
      tmp.close

      # Now extract the tar file and return the contents of the file
      Dir.mktmpdir("morph") do |dest|
        extract_tar_file(T.must(tmp.path), dest)
        tmp.unlink

        path2 = File.join(dest, Pathname.new(path).basename.to_s)
        tmp = Tempfile.new("morph-file")
        FileUtils.cp(path2, T.must(tmp.path))
      end
      tmp
    end

    # Get a set of files from a container and return them as a hash of
    # local temporary files
    sig { params(container: Docker::Container, paths: T::Array[String]).returns(T::Hash[String, T.nilable(Tempfile)]) }
    def self.copy_files(container, paths)
      data = {}
      paths.each do |path|
        data[path] = copy_file(container, path)
      end
      data
    end

    sig { returns(T::Array[Docker::Container]) }
    def self.stopped_containers
      Docker::Container.all(all: true).reject do |c|
        c.json["State"]["Running"]
      end
    end

    sig { returns(T::Array[Docker::Container]) }
    def self.running_containers
      Docker::Container.all
    end

    sig { params(dir: String, connection_options: T::Hash[Symbol, Integer], block: T.proc.params(text: String).void).returns(T.nilable(Docker::Image)) }
    def self.docker_build_from_dir(dir, connection_options, &block)
      # How does this connection get closed?
      connection = docker_connection(connection_options)
      temp = create_tar_file(dir)
      buffer = +""
      Docker::Image.build_from_tar(
        temp, { "forcerm" => 1 }, connection
      ) do |chunk|
        # Sometimes a chunk contains multiple lines of json
        chunk.split("\n").each do |line|
          parsed_line = JSON.parse(line)
          next unless parsed_line.key?("stream")

          buffer << parsed_line["stream"]
          # Buffer output until an end-of-line is detected. This
          # makes line output more consistent across platforms.
          # Make sure that buffer can't grow out of control by limiting
          # it's size around 256 bytes
          if buffer[-1..-1] == "\n" || buffer.length >= 256
            block.call buffer
            buffer = +""
          end
        end
        block.call buffer if buffer != ""
      end
    # This exception gets thrown if there is an error during the build (for
    # example if the compile fails). In this case we just want to return nil
    rescue Docker::Error::UnexpectedResponseError
      nil
    end

    sig { params(options: T::Hash[Symbol, Integer]).returns(Docker::Connection) }
    def self.docker_connection(options)
      Docker::Connection.new(Docker.url, Docker.env_options.merge(options))
    end

    # Copy the contents of "src" to the directory dest in the container c
    sig { params(container: Docker::Container, src: String, dest: String).void }
    def self.insert_contents_of_directory(container, src, dest)
      # Rather than using archive_in we're doing this more roundabout way
      # because archive_in seems to have very broken handling of directories
      # TODO Submit a fix to the docker-api gem to fix this
      # In the meantime get something more long-winded working here

      tar_file = Docker::Util.create_dir_tar(src).path
      File.open(tar_file, "rb") do |tar|
        container.archive_in_stream(dest) do
          tar.read(Excon.defaults[:chunk_size]).to_s
        end
      end
      File.delete(tar_file)
    end

    # Inserts a single file into a container.
    # Not using archive_in because that doesn't maintain file permissions
    sig { params(container: Docker::Container, src: String, dest: String).void }
    def self.insert_file(container, src, dest)
      # This is very roundabout
      Dir.mktmpdir("morph") do |tmp_dir|
        FileUtils.cp(src, tmp_dir)
        insert_contents_of_directory(container, tmp_dir, dest)
      end
    end

    # TODO: There's probably a more sensible way of doing this
    sig { params(image: Docker::Image, image_base: Docker::Image).returns(T::Boolean) }
    def self.image_built_on_other_image?(image, image_base)
      index = image.history.find_index { |l| l["Id"] == image_base.id }
      !index.nil? && index != 0
    end

    # This returns the total size of all the layers down to but not include the
{'fraction_non_alphanumeric': 0.10706521739130435, 'fraction_numerical': 0.0024456521739130437, 'mean_word_length': 2.9538031800601634, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2335285', 'n_tokens_mistral': 5528, 'n_tokens_neox': 5380, 'n_words': 1638}