<commit_before>require 'date'

module Bipbip

  TIMESTAMP_REGEXP = '^\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}'

  class Plugin::LogParser < Plugin

    def metrics_schema
      [
          {:name => config['name'], :type => 'gauge', :unit => 'Boolean'}
      ]
    end

    def monitor
      {
          config['name'] => match_count
      }
    end

    private

    def match_count
      lines.reject { |line| line.match(Regexp.new(config['regexp_text'])).nil? }.length
    end

    def log_time_min
      @log_time_min ||= Time.now - @frequency.to_i
    end

    def log_time_min=(time)
      @log_time_min = time
    end

    def regexp_timestamp
      @regexp_timestamp ||= Regexp.new(config.fetch('regexp_timestamp', TIMESTAMP_REGEXP))
    end



    def lines
      [].tap do |b|

        buffer_size = 65536

        File.open(config['path']) do |file|
          file.seek(0, File::SEEK_END)

          _time = log_time_min
          while file.pos >= 0
            if file.pos < buffer_size
              buffer_size = file.pos
            end

            file.seek(-buffer_size, File::SEEK_CUR)

            line_list = file.read(buffer_size).split("\n")

            # Remove first line as can be incomplete
            # due to seeking backward with
            # buffer_size steps
            first_line = line_list.shift
            file.seek(-buffer_size + first_line.length, File::SEEK_CUR)

            a = line_list.select do |line|
              timestamp = line.match(regexp_timestamp)
              unless timestamp.nil?
                time = DateTime.parse(timestamp[0]).to_time
                if time >= log_time_min
                  1 == 1
                end
                if time > _time
                  _time = time
                end
              end
            end

            b.push(*line_list)

            break if line_list.empty?
          end

          self.log_time_min = _time
        end
      end
    end
  end
end
<commit_msg>Split "reading lines backwards" and "parsing lines with timestamp" into two functions<commit_after>require 'date'

module Bipbip

  TIMESTAMP_REGEXP = '^\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}'

  class Plugin::LogParser < Plugin

    def metrics_schema
      [
          {:name => config['name'], :type => 'gauge', :unit => 'Boolean'}
      ]
    end

    def monitor
      {
          config['name'] => match_count
      }
    end

    private

    def match_count
      time_first = nil

      lines = lines_backwards.take_while do |line|
        timestamp_match = line.match(regexp_timestamp)
        raise "Line doesn't match `#{regexp_timestamp}`: `#{line}`" if timestamp_match.nil?
        time = DateTime.parse(timestamp_match[0]).to_time
        time_first ||= time
        (time > log_time_min)
      end

      self.log_time_min = time_first unless time_first.nil?

      lines.reject { |line| line.match(Regexp.new(config['regexp_text'])).nil? }.length
    end

    def log_time_min
      @log_time_min ||= Time.now - @frequency.to_i
    end

    def log_time_min=(time)
      @log_time_min = time
    end

    def regexp_timestamp
      @regexp_timestamp ||= Regexp.new(config.fetch('regexp_timestamp', TIMESTAMP_REGEXP))
    end

    def lines_backwards
      buffer_size = 65536

      Enumerator.new do |yielder|
        File.open(config['path']) do |file|
          file.seek(0, File::SEEK_END)

          while file.pos > 0
            buffer_size = file.pos if file.pos < buffer_size

            file.seek(-buffer_size, File::SEEK_CUR)
            buffer = file.read(buffer_size)
            file.seek(-buffer_size, File::SEEK_CUR)

            line_list = buffer.each_line.entries

            if file.pos != 0
              # Remove first line as can be incomplete
              # due to seeking backward with buffer_size steps
              first_line = line_list.shift
              raise "Line length exceeds buffer size `#{buffer_size}`" if first_line.length == buffer_size
              file.seek(first_line.length, File::SEEK_CUR)
            end

            line_list.reverse.each do |line|
              yielder.yield(line)
            end
          end
        end
      end
    end

  end
end

{'fraction_non_alphanumeric': 0.10380952380952381, 'fraction_numerical': 0.007380952380952381, 'mean_word_length': 1.8156836461126005, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4416931', 'n_tokens_mistral': 1344, 'n_tokens_neox': 1272, 'n_words': 287}