<commit_before># typed: strict
# frozen_string_literal: true

module Morph
  # High level API for running morph scraper. Handles the setting up of default
  # configuration if things like Gemfiles are not included (for Ruby)
  class Runner
    extend T::Sig

    include RenderSync::Actions
    sig { returns(Run) }
    attr_accessor :run

    sig { params(run: Run).void }
    def initialize(run)
      @run = run
    end

    # TODO: Move this to a configuration somewhere
    sig { returns(Integer) }
    def self.default_max_lines
      10_000
    end

    sig { returns(Integer) }
    def self.total_slots
      SiteSetting.maximum_concurrent_scrapers
    end

    # This includes stopped containers too
    sig { returns(Integer) }
    def self.used_slots
      Morph::DockerUtils.find_all_containers_with_label(run_label_key).count
    end

    sig { returns(Integer) }
    def self.available_slots
      total_slots - used_slots
    end

    # The main section of the scraper running that is run in the background
    sig { params(block: T.nilable(T.proc.params(timestamp: T.nilable(Time), stream: Symbol, text: String).void)).void }
    def synch_and_go_with_logging!(&block)
      synch_and_go! do |timestamp, s, c|
        log(timestamp, s, c, &block)
      end
    end

    sig { params(max_lines: Integer, block: T.nilable(T.proc.params(timestamp: T.nilable(Time), stream: Symbol, text: String).void)).void }
    def go_with_logging(max_lines = Runner.default_max_lines, &block)
      go(max_lines) do |timestamp, s, c|
        log(timestamp, s, c, &block)
      end
    end

    sig { params(timestamp: T.nilable(Time), stream: Symbol, text: String, block: T.nilable(T.proc.params(timestamp: T.nilable(Time), stream: Symbol, text: String).void)).void }
    def log(timestamp, stream, text, &block)
      Rails.logger.info "#{stream}: #{text}" if Rails.env.development?
      # Not using create on association to try to avoid memory bloat
      # Truncate text so that it fits in the database
      # Note that mysql TEXT is limited to 65535 bytes so we have to be
      # particularly careful with unicode.
      line = LogLine.create!(run: run, timestamp: timestamp, stream: stream.to_s, text: text.mb_chars.limit(65535).to_s)
      sync_new line, scope: run unless Rails.env.test?
      block.call timestamp, stream, text if block_given?
    end

    sig { params(text: String, status_code: Integer, block: T.nilable(T.proc.params(timestamp: T.nilable(Time), stream: Symbol, text: String).void)).void }
    def error(text:, status_code:, &block)
      block.call(nil, :internalerr, text) if block_given?
      run.update(status_code: status_code, finished_at: Time.zone.now)
      sync_update run.scraper if run.scraper
    end

    sig { params(block: T.nilable(T.proc.params(timestamp: T.nilable(Time), stream: Symbol, text: String).void)).void }
    def synch_and_go!(&block)
      scraper = run.scraper
      # If this run belongs to a scraper that has just been deleted
      # or if the run has already been marked as finished then
      # don't do anything
      return if scraper.nil? || run.finished?

      # TODO: Indicate that scraper is running before we do the synching
      error = SynchroniseRepoService.call(scraper)
      case error
      when nil
        nil
      when Morph::Github::NoAppInstallationForOwner
        error(
          status_code: 999,
          text: "Please install the Morph Github App on #{T.must(scraper.owner).nickname} so that Morph can access this repository on GitHub. Please go to #{T.must(scraper.owner).app_install_url}",
          &block
        )
        return
      when Morph::Github::AppInstallationNoAccessToRepo
        error(
          status_code: 999,
          text: "The Morph Github App installed on #{T.must(scraper.owner).nickname} needs access to the repository #{scraper.name}. Please go to #{T.must(scraper.owner).app_install_url}",
          &block
        )
        return
      when Morph::Github::SynchroniseRepoError, Morph::Github::NoAccessToRepo
        error(text: "There was a problem getting the latest scraper code from GitHub", status_code: 999, &block)
        return
      when SynchroniseRepoService::RepoNeedsToBePublic
        error(
          status_code: 999,
          text: "The repository #{scraper.full_name} needs to be made public",
          &block
        )
      when SynchroniseRepoService::RepoNeedsToBePrivate
        error(
          status_code: 999,
          text: "The repository #{scraper.full_name} needs to be made private",
          &block
        )
      else
        T.absurd(error)
      end<commit_msg>Stop correctly if there's an error<commit_after># typed: strict
# frozen_string_literal: true

module Morph
  # High level API for running morph scraper. Handles the setting up of default
  # configuration if things like Gemfiles are not included (for Ruby)
  class Runner
    extend T::Sig

    include RenderSync::Actions
    sig { returns(Run) }
    attr_accessor :run

    sig { params(run: Run).void }
    def initialize(run)
      @run = run
    end

    # TODO: Move this to a configuration somewhere
    sig { returns(Integer) }
    def self.default_max_lines
      10_000
    end

    sig { returns(Integer) }
    def self.total_slots
      SiteSetting.maximum_concurrent_scrapers
    end

    # This includes stopped containers too
    sig { returns(Integer) }
    def self.used_slots
      Morph::DockerUtils.find_all_containers_with_label(run_label_key).count
    end

    sig { returns(Integer) }
    def self.available_slots
      total_slots - used_slots
    end

    # The main section of the scraper running that is run in the background
    sig { params(block: T.nilable(T.proc.params(timestamp: T.nilable(Time), stream: Symbol, text: String).void)).void }
    def synch_and_go_with_logging!(&block)
      synch_and_go! do |timestamp, s, c|
        log(timestamp, s, c, &block)
      end
    end

    sig { params(max_lines: Integer, block: T.nilable(T.proc.params(timestamp: T.nilable(Time), stream: Symbol, text: String).void)).void }
    def go_with_logging(max_lines = Runner.default_max_lines, &block)
      go(max_lines) do |timestamp, s, c|
        log(timestamp, s, c, &block)
      end
    end

    sig { params(timestamp: T.nilable(Time), stream: Symbol, text: String, block: T.nilable(T.proc.params(timestamp: T.nilable(Time), stream: Symbol, text: String).void)).void }
    def log(timestamp, stream, text, &block)
      Rails.logger.info "#{stream}: #{text}" if Rails.env.development?
      # Not using create on association to try to avoid memory bloat
      # Truncate text so that it fits in the database
      # Note that mysql TEXT is limited to 65535 bytes so we have to be
      # particularly careful with unicode.
      line = LogLine.create!(run: run, timestamp: timestamp, stream: stream.to_s, text: text.mb_chars.limit(65535).to_s)
      sync_new line, scope: run unless Rails.env.test?
      block.call timestamp, stream, text if block_given?
    end

    sig { params(text: String, status_code: Integer, block: T.nilable(T.proc.params(timestamp: T.nilable(Time), stream: Symbol, text: String).void)).void }
    def error(text:, status_code:, &block)
      block.call(nil, :internalerr, text) if block_given?
      run.update(status_code: status_code, finished_at: Time.zone.now)
      sync_update run.scraper if run.scraper
    end

    sig { params(block: T.nilable(T.proc.params(timestamp: T.nilable(Time), stream: Symbol, text: String).void)).void }
    def synch_and_go!(&block)
      scraper = run.scraper
      # If this run belongs to a scraper that has just been deleted
      # or if the run has already been marked as finished then
      # don't do anything
      return if scraper.nil? || run.finished?

      # TODO: Indicate that scraper is running before we do the synching
      error = SynchroniseRepoService.call(scraper)
      case error
      when nil
        nil
      when Morph::Github::NoAppInstallationForOwner
        error(
          status_code: 999,
          text: "Please install the Morph Github App on #{T.must(scraper.owner).nickname} so that Morph can access this repository on GitHub. Please go to #{T.must(scraper.owner).app_install_url}",
          &block
        )
        return
      when Morph::Github::AppInstallationNoAccessToRepo
        error(
          status_code: 999,
          text: "The Morph Github App installed on #{T.must(scraper.owner).nickname} needs access to the repository #{scraper.name}. Please go to #{T.must(scraper.owner).app_install_url}",
          &block
        )
        return
      when Morph::Github::SynchroniseRepoError, Morph::Github::NoAccessToRepo
        error(text: "There was a problem getting the latest scraper code from GitHub", status_code: 999, &block)
        return
      when SynchroniseRepoService::RepoNeedsToBePublic
        error(
          status_code: 999,
          text: "The repository #{scraper.full_name} needs to be made public",
          &block
        )
        return
      when SynchroniseRepoService::RepoNeedsToBePrivate
        error(
          status_code: 999,
          text: "The repository #{scraper.full_name} needs to be made private",
          &block
        )
        return
      else
        T.absurd(error)
      end
{'fraction_non_alphanumeric': 0.09329039184111648, 'fraction_numerical': 0.00644122383252818, 'mean_word_length': 2.9077181208053693, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '625052', 'n_tokens_mistral': 2884, 'n_tokens_neox': 2792, 'n_words': 854}