<commit_before>module Kafka

  # An ordered sequence of messages fetched from a Kafka partition.
  class FetchedBatch
    # @return [String]
    attr_reader :topic

    # @return [Integer]
    attr_reader :partition

    # @return [Integer] the offset of the most recent message in the partition.
    attr_reader :highwater_mark_offset

    # @return [Array<Kafka::FetchedMessage>]
    attr_reader :messages

    def initialize(topic:, partition:, highwater_mark_offset:, messages:)
      @topic = topic
      @partition = partition
      @highwater_mark_offset = highwater_mark_offset
      @messages = messages
    end

    def empty?
      @messages.empty?
    end

    def last_offset
      messages.last.offset
    end

    def offset_lag
      if empty?
        0
      else
        highwater_mark_offset - last_offset
      end
    end<commit_msg>Make sure FetchedBatch#last_offset always returns something sane<commit_after>module Kafka

  # An ordered sequence of messages fetched from a Kafka partition.
  class FetchedBatch
    # @return [String]
    attr_reader :topic

    # @return [Integer]
    attr_reader :partition

    # @return [Integer] the offset of the most recent message in the partition.
    attr_reader :highwater_mark_offset

    # @return [Array<Kafka::FetchedMessage>]
    attr_reader :messages

    def initialize(topic:, partition:, highwater_mark_offset:, messages:)
      @topic = topic
      @partition = partition
      @highwater_mark_offset = highwater_mark_offset
      @messages = messages
    end

    def empty?
      @messages.empty?
    end

    def last_offset
      if empty?
        highwater_mark_offset - 1
      else
        messages.last.offset
      end
    end

    def offset_lag
      if empty?
        0
      else
        highwater_mark_offset - last_offset
      end
    end
{'fraction_non_alphanumeric': 0.06168122270742358, 'fraction_numerical': 0.0016375545851528383, 'mean_word_length': 2.601178781925344, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5294001', 'n_tokens_mistral': 541, 'n_tokens_neox': 518, 'n_words': 155}