<commit_before>require 'thread'
require 'villein'
require 'mamiya/logger'

require 'mamiya/steps/fetch'
require 'mamiya/agent/fetcher'

require 'mamiya/agent/handlers/fetch'

module Mamiya
  class Agent
    def initialize(config, logger: Mamiya::Logger.new)
      @config = config
      @serf = init_serf
      @fetcher = Mamiya::Agent::Fetcher.new(destination: config[:packages_dir])
      @logger = logger['agent']
    end

    attr_reader :config, :serf, :logger, :fetcher

    def run!
      logger.info "Starting..."

      serf_start
      fetcher_start

      logger.info "Started."

      loop do
        sleep 10
      end
    end

    def update_tags
      # TODO:
      # serf.tags.update(
    end

    def status
    end

    def releases
    end

    private

    def init_serf
      Villein::Agent.new(**config[:serf][:agent]).tap do |serf|
        serf.on_user_event do |event|
          user_event_handler(event)
        end
      end
    end

    def serf_start
      logger.debug "Starting serf"

      @serf.start!
      @serf.auto_stop
    end

    def fetcher_start
      logger.debug "Starting fetcher"

      @fetcher.start!
    end

    def user_event_handler(event)
      type, payload = event.user_event, JSON.parse(event.payload)
      class_name = type.sub(/^mamiya-/,'').capitalize.gsub(/-./) { |_| _[1].upcase }

      logger.debug "Received user event #{type}"
      logger.debug payload.inspect

      if Handlers.const_defined?(class_name)
        Handlers.const_get(class_name).new(self, event).run!
      else
        logger.warn("Discarded event[#{event.user_event}] because we don't handle it")
      end
    rescue Exception => e
      logger.fatal("Error during handling event: #{e.inspect}")
      e.backtrace.each do |line|
        logger.fatal line.prepend("\t")
      end

      raise e if $0.end_with?('rspec')
    rescue JSON::ParserError
      logger.warn("Discarded event[#{event.user_event}] with invalid payload (unable to parse as json)")
    end

    def handle_fetch_event(payload, event)

    end

  end
end
<commit_msg>Remove unused empty method<commit_after>require 'thread'
require 'villein'
require 'mamiya/logger'

require 'mamiya/steps/fetch'
require 'mamiya/agent/fetcher'

require 'mamiya/agent/handlers/fetch'

module Mamiya
  class Agent
    def initialize(config, logger: Mamiya::Logger.new)
      @config = config
      @serf = init_serf
      @fetcher = Mamiya::Agent::Fetcher.new(destination: config[:packages_dir])
      @logger = logger['agent']
    end

    attr_reader :config, :serf, :logger, :fetcher

    def run!
      logger.info "Starting..."

      serf_start
      fetcher_start

      logger.info "Started."

      loop do
        sleep 10
      end
    end

    def update_tags
      # TODO:
      # serf.tags.update(
    end

    def status
    end

    def releases
    end

    private

    def init_serf
      Villein::Agent.new(**config[:serf][:agent]).tap do |serf|
        serf.on_user_event do |event|
          user_event_handler(event)
        end
      end
    end

    def serf_start
      logger.debug "Starting serf"

      @serf.start!
      @serf.auto_stop
    end

    def fetcher_start
      logger.debug "Starting fetcher"

      @fetcher.start!
    end

    def user_event_handler(event)
      type, payload = event.user_event, JSON.parse(event.payload)
      class_name = type.sub(/^mamiya-/,'').capitalize.gsub(/-./) { |_| _[1].upcase }

      logger.debug "Received user event #{type}"
      logger.debug payload.inspect

      if Handlers.const_defined?(class_name)
        Handlers.const_get(class_name).new(self, event).run!
      else
        logger.warn("Discarded event[#{event.user_event}] because we don't handle it")
      end
    rescue Exception => e
      logger.fatal("Error during handling event: #{e.inspect}")
      e.backtrace.each do |line|
        logger.fatal line.prepend("\t")
      end

      raise e if $0.end_with?('rspec')
    rescue JSON::ParserError
      logger.warn("Discarded event[#{event.user_event}] with invalid payload (unable to parse as json)")
    end
  end
end

{'fraction_non_alphanumeric': 0.11084043848964677, 'fraction_numerical': 0.00194884287454324, 'mean_word_length': 2.494468085106383, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6468768', 'n_tokens_mistral': 1381, 'n_tokens_neox': 1329, 'n_words': 307}