<commit_before>module Morph
  # High level API for running morph scraper. Handles the setting up of default
  # configuration if things like Gemfiles are not included (for Ruby)
  class Runner
    include Sync::Actions
    attr_accessor :run

    def initialize(run)
      @run = run
    end

    # The main section of the scraper running that is run in the background
    def synch_and_go!
      # If this run belongs to a scraper that has just been deleted
      # or if the run has already been marked as finished then
      # don't do anything
      return if run.scraper.nil? || run.finished?

      run.scraper.synchronise_repo
      go_with_logging
    end

    def go_with_logging
      go do |timestamp, s, c|
        log(timestamp, s, c)
        yield timestamp, s, c if block_given?
      end
    end

    def log(timestamp, stream, text)
      # TODO Do something with the timestamp
      puts "#{stream}: #{text}" if Rails.env.development?
      # Not using create on association to try to avoid memory bloat
      line = LogLine.create!(run: run, stream: stream.to_s, text: text)
      sync_new line, scope: run unless Rails.env.test?
    end

    def go
      # If container already exists we just attach to it
      c = container_for_run
      # TODO Use new feature of docker api to skip over log lines based on the timestamp
      if c.nil?
        c = compile_and_start_run do |s, c|
          # TODO Could we get sensible timestamps out at this stage too?
          yield nil, s, c
        end
        lines_to_skip = 0
      else
        # Figure out how many log lines we want to skip
        lines_to_skip = run.log_lines.where("stream = 'stdout' OR stream = 'stderr'").count
      end
      count = 0
      attach_to_run_and_finish(c) do |timestamp, s, c|
        count += 1
        if count > lines_to_skip
          yield timestamp, s, c
        end
      end
    end

    def compile_and_start_run
      #puts "Starting...\n"
      run.database.backup
      run.update_attributes(started_at: Time.now,
                            git_revision: run.current_revision_from_repo)
      sync_update run.scraper if run.scraper
      FileUtils.mkdir_p run.data_path
      FileUtils.chmod 0777, run.data_path

      unless run.language && run.language.supported?
        supported_scraper_files =
          Morph::Language.languages_supported.map(&:scraper_filename)
        m = "Can't find scraper code. Expected to find a file called " +
            supported_scraper_files.to_sentence(last_word_connector: ', or ') +
            ' in the root directory'
        yield 'stderr', m
        run.update_attributes(status_code: 999, finished_at: Time.now)
        return
      end

      c, image = Dir.mktmpdir('morph') do |defaults|
        Morph::Runner.add_config_defaults_to_directory(run.repo_path, defaults)
        Morph::Runner.remove_hidden_directories(defaults)
        Morph::Runner.add_sqlite_db_to_directory(run.data_path, defaults)

        Morph::DockerRunner.compile_and_start_run(
          defaults, run.env_variables, docker_container_labels) do |s, c|
          yield(s, c)
        end
      end

      # Record ip address of running container
      ip_address = c.json['NetworkSettings']['IPAddress'] if c
      # The image id here is a short one. Not sure why.
      # TODO: Investigate
      docker_image = image.id if image
      run.update_attributes(ip_address: ip_address, docker_image: docker_image)
      c
    end

    def attach_to_run_and_finish(c)
      if c.nil?
        # TODO: Return the status for a compile error
        result = Morph::RunResult.new(255, {}, {})
      else
        result = Morph::DockerRunner.attach_to_run_and_finish(
          c, ['data.sqlite']) do |timestamp, s, c|
          yield(timestamp, s, c)
        end
      end

      # Only copy back database if it's there and has something in it<commit_msg>Skip log lines in a much more efficient way<commit_after>module Morph
  # High level API for running morph scraper. Handles the setting up of default
  # configuration if things like Gemfiles are not included (for Ruby)
  class Runner
    include Sync::Actions
    attr_accessor :run

    def initialize(run)
      @run = run
    end

    # The main section of the scraper running that is run in the background
    def synch_and_go!
      # If this run belongs to a scraper that has just been deleted
      # or if the run has already been marked as finished then
      # don't do anything
      return if run.scraper.nil? || run.finished?

      run.scraper.synchronise_repo
      go_with_logging
    end

    def go_with_logging
      go do |timestamp, s, c|
        log(timestamp, s, c)
        yield timestamp, s, c if block_given?
      end
    end

    def log(timestamp, stream, text)
      puts "#{stream}: #{text}" if Rails.env.development?
      # Not using create on association to try to avoid memory bloat
      line = LogLine.create!(run: run, timestamp: timestamp, stream: stream.to_s, text: text)
      sync_new line, scope: run unless Rails.env.test?
    end

    def go
      # If container already exists we just attach to it
      c = container_for_run
      if c.nil?
        c = compile_and_start_run do |s, c|
          # TODO Could we get sensible timestamps out at this stage too?
          yield nil, s, c
        end
        since = nil
      else
        # The timestamp of the last log line we've already captured
        since = run.log_lines.maximum(:timestamp)
        # We add a microsecond to compensate for rounding error as
        # part of the time being stored in the database. The true time
        # gets truncated to the lower microsecond. So, likely the true
        # time happens *after* the recorded time. So, we add a microsecond
        # to compensate for this and ensure that the "since" time occurs
        # slightly after the true time.
        since += 1e-6 if since
      end
      attach_to_run_and_finish(c, since) do |timestamp, s, c|
        yield timestamp, s, c
      end
    end

    def compile_and_start_run
      #puts "Starting...\n"
      run.database.backup
      run.update_attributes(started_at: Time.now,
                            git_revision: run.current_revision_from_repo)
      sync_update run.scraper if run.scraper
      FileUtils.mkdir_p run.data_path
      FileUtils.chmod 0777, run.data_path

      unless run.language && run.language.supported?
        supported_scraper_files =
          Morph::Language.languages_supported.map(&:scraper_filename)
        m = "Can't find scraper code. Expected to find a file called " +
            supported_scraper_files.to_sentence(last_word_connector: ', or ') +
            ' in the root directory'
        yield 'stderr', m
        run.update_attributes(status_code: 999, finished_at: Time.now)
        return
      end

      c, image = Dir.mktmpdir('morph') do |defaults|
        Morph::Runner.add_config_defaults_to_directory(run.repo_path, defaults)
        Morph::Runner.remove_hidden_directories(defaults)
        Morph::Runner.add_sqlite_db_to_directory(run.data_path, defaults)

        Morph::DockerRunner.compile_and_start_run(
          defaults, run.env_variables, docker_container_labels) do |s, c|
          yield(s, c)
        end
      end

      # Record ip address of running container
      ip_address = c.json['NetworkSettings']['IPAddress'] if c
      # The image id here is a short one. Not sure why.
      # TODO: Investigate
      docker_image = image.id if image
      run.update_attributes(ip_address: ip_address, docker_image: docker_image)
      c
    end

    def attach_to_run_and_finish(c, since)
      if c.nil?
        # TODO: Return the status for a compile error
        result = Morph::RunResult.new(255, {}, {})
      else
        result = Morph::DockerRunner.attach_to_run_and_finish(
          c, ['data.sqlite'], since) do |timestamp, s, c|
          yield(timestamp, s, c)
        end
      end

      # Only copy back database if it's there and has something in it
{'fraction_non_alphanumeric': 0.07300636465743167, 'fraction_numerical': 0.003119930113565456, 'mean_word_length': 2.4029723991507432, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4510872', 'n_tokens_mistral': 2407, 'n_tokens_neox': 2319, 'n_words': 793}