<commit_before>#!/usr/bin/python

"""
Top-down operator precendence parser library.

This is based on code from:
http://eli.thegreenplace.net/2010/01/02/top-down-operator-precedence-parsing/

Concepts:

 - token type: an object representing the type of the token.
   Paser.END_TOKEN is a special token type representing the end of the
   input stream.

 - token: a tuple of (token_type, token_content, token_line,
   token_column).  token_conent is the text content of the token.

 - action map: used to map token types to handler functions called when
   the token is encountered.
"""

class Assoc:
    """
    Defines constants for left/right associative binary oprators, used
    by ActionMap.add_binary_op().

    The values are added to the operator precedence value to obtain the
    final right binding power - do not change the values.
    """
    LEFT = 0
    RIGHT = -1

class Token:
    """
    Represents a token in the input stream.  It has the following
    attributes:

    type -- the type of token.  This can be any object including an
    constant integer or string.

    content -- the text content of the token

    filename -- optional, the name of the input file where the token was
    found

    line_pos -- optional, the line number in the input file where the
    token was found

    column_pos -- optional, the column number in the input file where
    the token was found
    """
    def __init__(self, type, content, filename = None, line_pos = None, column_pos = None):
        self.type = type
        self.content = content
        self.filename = filename
        self.line_pos = line_pos
        self.column_pos = column_pos

class ParseError(Exception):
    """
    Raised when an error occurs in parsing.  The exception has two
    attributes:

    token -- the token that was being parsed when the error occured

    message -- a descriptive message
    """
    def __init__(self, token, message):
        self.token = token
        self.message = message

    def __str__(self):
        m = self.message
        t = self.token
        if t.filename:
            m += " in " + t.filename
        if t.line_pos:
            m += " at line " + str(t.line_pos)
            if t.column_pos != None:
                m += " column " + str(t.column_pos)
        return m

class ActionMap:
    """
    Determines what action to take when a token is encountered in the
    input stream.  This determines the expressions that can be parsed.

    Initialised by the client by calling the add_* methods.
    """

    def __init__(self):
        self.prefix_actions = {}
        self.infix_actions = {}
    
    ##################################################################
    # Internal implementation
    ##################################################################

    def prefix(self, parser, token):
        """
        Internal - called by the parser when a token is encountered in<commit_msg>Allow ActionMaps to include the actions from existing maps<commit_after>#!/usr/bin/python

"""
Top-down operator precendence parser library.

This is based on code from:
http://eli.thegreenplace.net/2010/01/02/top-down-operator-precedence-parsing/

Concepts:

 - token type: an object representing the type of the token.
   Paser.END_TOKEN is a special token type representing the end of the
   input stream.

 - token: a tuple of (token_type, token_content, token_line,
   token_column).  token_conent is the text content of the token.

 - action map: used to map token types to handler functions called when
   the token is encountered.
"""

class Assoc:
    """
    Defines constants for left/right associative binary oprators, used
    by ActionMap.add_binary_op().

    The values are added to the operator precedence value to obtain the
    final right binding power - do not change the values.
    """
    LEFT = 0
    RIGHT = -1

class Token:
    """
    Represents a token in the input stream.  It has the following
    attributes:

    type -- the type of token.  This can be any object including an
    constant integer or string.

    content -- the text content of the token

    filename -- optional, the name of the input file where the token was
    found

    line_pos -- optional, the line number in the input file where the
    token was found

    column_pos -- optional, the column number in the input file where
    the token was found
    """
    def __init__(self, type, content, filename = None, line_pos = None, column_pos = None):
        self.type = type
        self.content = content
        self.filename = filename
        self.line_pos = line_pos
        self.column_pos = column_pos

class ParseError(Exception):
    """
    Raised when an error occurs in parsing.  The exception has two
    attributes:

    token -- the token that was being parsed when the error occured

    message -- a descriptive message
    """
    def __init__(self, token, message):
        self.token = token
        self.message = message

    def __str__(self):
        m = self.message
        t = self.token
        if t.filename:
            m += " in " + t.filename
        if t.line_pos:
            m += " at line " + str(t.line_pos)
            if t.column_pos != None:
                m += " column " + str(t.column_pos)
        return m

class ActionMap:
    """
    Determines what action to take when a token is encountered in the
    input stream.  This determines the expressions that can be parsed.

    include -- optional, if specified then initialise the new action map
    by including all the actions defined by this one

    Initialised by the client by calling the add_* methods.
    """

    def __init__(self, include = None):
        if include:
            self.prefix_actions = dict(include.prefix_actions)
            self.infix_actions = dict(include.infix_actions)
        else:
            self.prefix_actions = {}
            self.infix_actions = {}
    
    ##################################################################
    # Internal implementation
    ##################################################################

    def prefix(self, parser, token):
        """
        Internal - called by the parser when a token is encountered in
{'fraction_non_alphanumeric': 0.11095054592164419, 'fraction_numerical': 0.0032113037893384713, 'mean_word_length': 2.511273957158963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '579649', 'n_tokens_mistral': 1681, 'n_tokens_neox': 1607, 'n_words': 675}