<commit_before>module Rird
  module Blocks
    Line = Struct.new(:string, :lineno, :tag, :lead_space, :headchar, :inner_space)

    class Lexer
      def initialize(src, filename, initial_lineno = 1)
        @src      = src
        @filename = filename
        @lineno   = initial_lineno
        @enum     = lex()
        @buf      = []
      end

      def read
        @enum.next
      end

      private

      # !!! This method never ends
      def lex
        return enum_for(__method__) unless block_given?
        @src.each_line do |line|
          yield match(line)
          @lineno += 1
        end
        eof = Line.new("", @lineno, :eof, "", "", "")
        loop { yield eof }
      end

      BLOCKS_RE = {
        whiteline:  /^\s*$/,<commit_msg>impl unread<commit_after>module Rird
  module Blocks
    Line = Struct.new(:string, :lineno, :tag, :lead_space, :headchar, :inner_space) do
      def inner_indent
        "#{lead_space}#{' ' * headchar.size}#{inner_space}"
      end
    end

    class Lexer
      def initialize(src, filename, initial_lineno = 1)
        @src      = src
        @filename = filename
        @lineno   = initial_lineno
        @enum     = lex()
        @buf      = []
      end

      def read
        @buf.empty? ? @enum.next : @buf.pop
      end

      def unread(line, raw: false)
        return nil if line.tag == :eof
        line = match(line) if raw
        @buf.push line
        nil
      end

      private

      # !!! This method never ends
      def lex
        return enum_for(__method__) unless block_given?
        @src.each_line do |line|
          yield match(line)
          @lineno += 1
        end
        eof = Line.new("", @lineno, :eof, "", "", "")
        loop { yield eof }
      end

      BLOCKS_RE = {
        whiteline:  /^\s*$/,
{'fraction_non_alphanumeric': 0.11480865224625623, 'fraction_numerical': 0.0022185246810870773, 'mean_word_length': 1.629737609329446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4705358', 'n_tokens_mistral': 592, 'n_tokens_neox': 567, 'n_words': 150}