<commit_before>module Parslet::Atoms
  module Precedence
    prec = 0
    BASE       = (prec+=1)    # everything else
    LOOKAHEAD  = (prec+=1)    # &SOMETHING
    REPETITION = (prec+=1)    # 'a'+, 'a'?
    SEQUENCE   = (prec+=1)    # 'a' 'b'
    ALTERNATE  = (prec+=1)    # 'a' / 'b'
    OUTER      = (prec+=1)    # printing is done here.
  end
  
  # This is raised when the parse failed to match or to consume all its input.
  # It contains the message that should be presented to the user. If you want
  # to display more error explanation, you can print the #error_tree that is
  # stored in the parslet. This is a graphical representation of what went
  # wrong. 
  #
  # Example: 
  #    
  #   begin
  #     parslet.parse(str)
  #   rescue Parslet::ParseFailed => failure
  #     puts parslet.error_tree.ascii_tree
  #   end
  #
  class ParseFailed < Exception
  end
  
  # Base class for all parslets, handles orchestration of calls and implements
  # a lot of the operator and chaining methods.
  #
  class Base
    def parse(io)
      if io.respond_to? :to_str
        io = StringIO.new(io)
      end
      
      result = apply(io)
      
      # If we haven't consumed the input, then the pattern doesn't match. Try
      # to provide a good error message (even asking down below)
      unless io.eof?
        # Do we know why we stopped matching input? If yes, that's a good
        # error to fail with. Otherwise just report that we cannot consume the
        # input.
        if cause 
          raise ParseFailed, cause
        else
          error(io, "Don't know what to do with #{io.string[io.pos,100]}") 
        end
      end
      
      return flatten(result)
    end
    
    def apply(io)
      # p [:start, self, io.string[io.pos, 10]]<commit_msg>. better failure on overhanging input<commit_after>module Parslet::Atoms
  module Precedence
    prec = 0
    BASE       = (prec+=1)    # everything else
    LOOKAHEAD  = (prec+=1)    # &SOMETHING
    REPETITION = (prec+=1)    # 'a'+, 'a'?
    SEQUENCE   = (prec+=1)    # 'a' 'b'
    ALTERNATE  = (prec+=1)    # 'a' / 'b'
    OUTER      = (prec+=1)    # printing is done here.
  end
  
  # This is raised when the parse failed to match or to consume all its input.
  # It contains the message that should be presented to the user. If you want
  # to display more error explanation, you can print the #error_tree that is
  # stored in the parslet. This is a graphical representation of what went
  # wrong. 
  #
  # Example: 
  #    
  #   begin
  #     parslet.parse(str)
  #   rescue Parslet::ParseFailed => failure
  #     puts parslet.error_tree.ascii_tree
  #   end
  #
  class ParseFailed < Exception
  end
  
  # Base class for all parslets, handles orchestration of calls and implements
  # a lot of the operator and chaining methods.
  #
  class Base
    def parse(io)
      if io.respond_to? :to_str
        io = StringIO.new(io)
      end
      
      result = apply(io)
      
      # If we haven't consumed the input, then the pattern doesn't match. Try
      # to provide a good error message (even asking down below)
      unless io.eof?
        # Do we know why we stopped matching input? If yes, that's a good
        # error to fail with. Otherwise just report that we cannot consume the
        # input.
        if cause 
          raise ParseFailed, "Unconsumed input, maybe because of this: #{cause}"
        else
          error(io, "Don't know what to do with #{io.string[io.pos,100]}") 
        end
      end
      
      return flatten(result)
    end
    
    def apply(io)
      # p [:start, self, io.string[io.pos, 10]]
{'fraction_non_alphanumeric': 0.0883489784649365, 'fraction_numerical': 0.006626173384870237, 'mean_word_length': 2.158674803836094, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4782900', 'n_tokens_mistral': 1167, 'n_tokens_neox': 1119, 'n_words': 424}