<commit_before>class RLispGrammar
  def initialize(input, file_name="example.rl")
    if input.is_a? String
      @input = nil
      @buf = input.dup
      @eof = true
    else
      @input = input
      @buf = ""
      @eof = false
    end
    @tokens = []
    @file_name = file_name
    @line    = 1
    @column  = 0
  end
  def get_chars
    return if @eof
    c = @input.read(1)
    if c
      @buf << c
    else
      @eof = true
    end
  end
  def each_expr
    while true
      @tokens << get_token if @tokens.empty?
      return if @tokens[0][0] == :eof
      yield expr
    end
  end
  
  def expr
    if @tokens.empty?
       tok = get_token
    else
       tok = @tokens.shift
    end
    
    case tok[0]
    when :expr
      return tok[1]
    when :quote, :unquote, :quasiquote, :"unquote-splicing"
      res = [tok[0], expr]
      res.debug_info = [@file_name, tok[2], tok[3]]
      return res
    when :open
      res = []
      res.debug_info = [@file_name, tok[2], tok[3]]
      while true
         @tokens << get_token
         if @tokens[0][0] == :close
           @tokens.shift
           return res
         else
           res << expr
         end
      end
    when :sqopen
      res = [:send]
      res.debug_info = [@file_name, tok[2], tok[3]]
      while true
         @tokens << get_token
         if @tokens[0][0] == :sqclose
           @tokens.shift
           res[2] = [:quote, res[2]] unless res.size <= 2
           return res
         else
           res << expr
         end
      end
    when :istr_beg
      res = [:str, tok[1]]
      res.debug_info = [@file_name, tok[2], tok[3]]
      while true
         @tokens << get_token
         if @tokens[0][0] == :istr_end
           res << @tokens.shift[1]
           return res
         elsif @tokens[0][0] == :istr_mid
           res << @tokens.shift[1]
         else
           # We don't really know how many exprs will there be inside #{ ... }
           res << expr
         end
      end
    else
      raise "Expected expr, got #{tok[0]} at #{@file_name}:#{tok[2]}:#{tok[3]}"
    end
  end

  def eat(sz)
     eaten = @buf[0...sz]
     @buf = @buf[sz..-1]
     # eaten.each_char if we had it ...
     eaten.each_byte{|c|
     if c == ?\n
       @column = 0
       @line += 1
     else
       @column += 1
     end
     }
  end
  
  # Possible tokens:
  # * expr
  # * open, close
  # * istr_beg, istr_mid, istr_end
  # * sqopen, sqclose
  # * quote, quasiquote<commit_msg>Line numbers in debug info fixed for 1.9, backtraces still fail<commit_after>class RLispGrammar
  def initialize(input, file_name="example.rl")
    if input.is_a? String
      @input = nil
      @buf = input.dup
      @eof = true
    else
      @input = input
      @buf = ""
      @eof = false
    end
    @tokens = []
    @file_name = file_name
    @line    = 1
    @column  = 0
  end
  def get_chars
    return if @eof
    c = @input.read(1)
    if c
      @buf << c
    else
      @eof = true
    end
  end
  def each_expr
    while true
      @tokens << get_token if @tokens.empty?
      return if @tokens[0][0] == :eof
      yield expr
    end
  end
  
  def expr
    if @tokens.empty?
       tok = get_token
    else
       tok = @tokens.shift
    end
    
    case tok[0]
    when :expr
      return tok[1]
    when :quote, :unquote, :quasiquote, :"unquote-splicing"
      res = [tok[0], expr]
      res.debug_info = [@file_name, tok[2], tok[3]]
      return res
    when :open
      res = []
      res.debug_info = [@file_name, tok[2], tok[3]]
      while true
         @tokens << get_token
         if @tokens[0][0] == :close
           @tokens.shift
           return res
         else
           res << expr
         end
      end
    when :sqopen
      res = [:send]
      res.debug_info = [@file_name, tok[2], tok[3]]
      while true
         @tokens << get_token
         if @tokens[0][0] == :sqclose
           @tokens.shift
           res[2] = [:quote, res[2]] unless res.size <= 2
           return res
         else
           res << expr
         end
      end
    when :istr_beg
      res = [:str, tok[1]]
      res.debug_info = [@file_name, tok[2], tok[3]]
      while true
         @tokens << get_token
         if @tokens[0][0] == :istr_end
           res << @tokens.shift[1]
           return res
         elsif @tokens[0][0] == :istr_mid
           res << @tokens.shift[1]
         else
           # We don't really know how many exprs will there be inside #{ ... }
           res << expr
         end
      end
    else
      raise "Expected expr, got #{tok[0]} at #{@file_name}:#{tok[2]}:#{tok[3]}"
    end
  end

  def eat(sz)
    eaten = @buf[0...sz]
    @buf = @buf[sz..-1]
    # eaten.each_char if we had it ...
    eaten.each_byte{|c|
      if c == 10 # ?\n
        @column = 0
        @line += 1
      else
        @column += 1
      end
    }
  end
  
  # Possible tokens:
  # * expr
  # * open, close
  # * istr_beg, istr_mid, istr_end
  # * sqopen, sqclose
  # * quote, quasiquote
{'fraction_non_alphanumeric': 0.12670682730923694, 'fraction_numerical': 0.01606425702811245, 'mean_word_length': 1.5582948125321006, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4079627', 'n_tokens_mistral': 1759, 'n_tokens_neox': 1733, 'n_words': 468}