<commit_before>class Token
  attr_reader :line, :start_column, :end_column, :text

  def initialize(line, start_column)
    @line = line
    @start_column = start_column
    @end_column = start_column
    @text = ""
  end

  def add_character(character)
    @end_column += 1
    @text += character
  end
end

class EndOfFile
  attr_reader :line

  def initialize(line, column)
    @line = line
    @column = column
  end

  def start_column
    @column
  end

  def end_column
    @column
  end

  def text
    nil
  end
end

class Tokenizer
  def initialize(stream)
    @stream = stream
    @line = 1
    @column = 1
    @token = nil
  end

  def read
    while true
      if @stream.eof?
        # TODO: why this and the nil case both?
        @token ||= EndOfFile.new(@line, @column)
        return
      end

      character = @stream.readchar
      case character
        when ' ', "\t", "\n"
          # do nothing
        when nil
          @token ||= EndOfFile.new(@line, @column)
        else
          @token ||= Token.new(@line, @column)
          @token.add_character(character)
      end
      @column += 1
    end
  end<commit_msg>Line, column, and stream seem to want to go together.<commit_after>class Token
  attr_reader :line, :start_column, :end_column, :text

  def initialize(line, start_column)
    @line = line
    @start_column = start_column
    @end_column = start_column
    @text = ""
  end

  def add_character(character)
    @end_column += 1
    @text += character
  end
end

class EndOfFile
  attr_reader :line

  def initialize(line, column)
    @line = line
    @column = column
  end

  def start_column
    @column
  end

  def end_column
    @column
  end

  def text
    nil
  end
end

class LineAndColumnCountingStream
  def initialize(stream)
    @stream = stream
    @line = 1
    @column = 1
  end

  def read_character
    if @stream.eof?
      [@line, @column, nil]
    else
      character = @stream.readchar
      column = @column
      @column += 1
      [@line, column, character]
    end
  end
end

class Tokenizer
  def initialize(stream)
    @stream = LineAndColumnCountingStream.new(stream)
    @token = nil
  end

  def read
    while true
      line, column, character = @stream.read_character
      case character
        when ' ', "\t", "\n"
          # do nothing
        when nil
          @token ||= EndOfFile.new(line, column)
          return
        else
          @token ||= Token.new(line, column)
          @token.add_character(character)
      end
    end
  end
{'fraction_non_alphanumeric': 0.08986539984164688, 'fraction_numerical': 0.003167062549485352, 'mean_word_length': 2.1706398996235885, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5755079', 'n_tokens_mistral': 824, 'n_tokens_neox': 797, 'n_words': 235}