<commit_before>from vint.ast.traversing import traverse
from vint.ast.node_type import NodeType


IDENTIFIER_ATTRIBUTE = 'VINT:identifier_attribute'
IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG = 'is_definition'
IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG = 'is_dynamic'
IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG = 'is_member_of_subscript'
IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG = 'is_function'
REFERENCING_IDENTIFIERS = 'VINT:referencing_identifiers'
DECLARING_IDENTIFIERS = 'VINT:declaring_identifiers'


DeclarativeNodeTypes = {
    NodeType.LET: True,
    NodeType.FUNCTION: True,
    NodeType.FOR: True,
}

IdentifierTerminateNodeTypes = {
    NodeType.IDENTIFIER: True,
    NodeType.ENV: True,
    NodeType.REG: True,
    NodeType.OPTION: True,
    NodeType.CURLYNAME: True,
}

AccessorLikeNodeTypes = {
    NodeType.DOT: True,
    NodeType.SUBSCRIPT: True,
    NodeType.SLICE: True,
}

# These node types is identifier-like when:
#   let object['name'] = 0
AnalyzableSubScriptChildNodeTypes = {
    NodeType.NUMBER: True,
    NodeType.STRING: True,
}


class IdentifierClassifier(object):
    """ A class for identifier classifiers.
    This class classify nodes by 3 flags:

    - is dynamic: True if the identifier name can be determined by static analysis.
    - is member: True if the identifier is a member of a subscription/dot/slice node.
    - is declaring: True if the identifier is used to declare.
    """

    class IdentifierCollector(object):
        """ A class for identifier node collectors.
        Only static and not member nodes will be collected and the nodes will
        be grouped by 2 types; declaring or referencing.
        """

        def collect_identifiers(self, ast):
            self.static_referencing_identifiers = []
            self.static_declaring_identifiers = []

            # TODO: Make more performance efficiency.
            traverse(ast, on_enter=self._enter_handler)

            return {
                'static_declaring_identifiers': self.static_declaring_identifiers,
                'static_referencing_identifiers': self.static_referencing_identifiers,
            }


        def _enter_handler(self, node):
            is_identifier_like_node = IDENTIFIER_ATTRIBUTE in node

            if not is_identifier_like_node:
                return

            id_attr = node[IDENTIFIER_ATTRIBUTE]
            if id_attr[IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG] or \
                    id_attr[IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG]:
                return

            if id_attr[IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG]:
                self.static_declaring_identifiers.append(node)
            else:
                self.static_referencing_identifiers.append(node)


    def attach_identifier_attributes(self, ast):
        """ Attach 3 flags to the AST.

        - is dynamic: True if the identifier name can be determined by static analysis.
        - is member: True if the identifier is a member of a subscription/dot/slice node.
        - is declaring: True if the identifier is used to declare.
        - is function: True if the identifier is a function. Vim distinguish
                       between function identifiers and variable identifiers.
        """
        traverse(ast, on_enter=self._enter_handler)
        return ast


    def _set_identifier_attribute(self, node, is_definition=None, is_dynamic=None,
                                  is_member_of_subscript=None, is_function=None):

        id_attr = node.setdefault(IDENTIFIER_ATTRIBUTE, {
            IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG: False,
            IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG: False,
            IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG: False,
            IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG: False,
        })

        if is_definition is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG] = is_definition

        if is_dynamic is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG] = is_dynamic

        if is_member_of_subscript is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG] = is_member_of_subscript

        if is_function is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG] = is_function


    def _enter_handler(self, node):
        node_type = NodeType(node['type'])

        if node_type in IdentifierTerminateNodeTypes:
            # Attach identifier attributes to all IdentifierTerminateNodeTypes.
            self._set_identifier_attribute(node)

        if node_type in AccessorLikeNodeTypes:
            self._pre_mark_accessor_children(node)

        if node_type in DeclarativeNodeTypes:
            self._enter_declarative_node(node)

        if node_type is NodeType.CALL:
            self._enter_call_node(node)


    def _pre_mark_accessor_children(self, node):
        node_type = NodeType(node['type'])
        dict_node = node['left']

        if NodeType(dict_node['type']) in AccessorLikeNodeTypes:
            self._pre_mark_accessor_children(dict_node)

        if node_type is NodeType.SLICE:
            for member_node in node['rlist']:
                self._pre_mark_member_node(member_node)
        else:
            member_node = node['right']
            self._pre_mark_member_node(member_node)


    def _pre_mark_member_node(self, member_node):
        member_node_type = NodeType(member_node['type'])

        if member_node_type in IdentifierTerminateNodeTypes or \
                member_node_type in AnalyzableSubScriptChildNodeTypes:
            self._set_identifier_attribute(member_node,
                                           is_member_of_subscript=True)


    def _enter_identifier_like_node(self, id_like_node, is_definition=None,<commit_msg>Fix a bug when list[1:] style SLICE node arrived<commit_after>from vint.ast.traversing import traverse
from vint.ast.node_type import NodeType


IDENTIFIER_ATTRIBUTE = 'VINT:identifier_attribute'
IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG = 'is_definition'
IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG = 'is_dynamic'
IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG = 'is_member_of_subscript'
IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG = 'is_function'
REFERENCING_IDENTIFIERS = 'VINT:referencing_identifiers'
DECLARING_IDENTIFIERS = 'VINT:declaring_identifiers'


DeclarativeNodeTypes = {
    NodeType.LET: True,
    NodeType.FUNCTION: True,
    NodeType.FOR: True,
}

IdentifierTerminateNodeTypes = {
    NodeType.IDENTIFIER: True,
    NodeType.ENV: True,
    NodeType.REG: True,
    NodeType.OPTION: True,
    NodeType.CURLYNAME: True,
}

AccessorLikeNodeTypes = {
    NodeType.DOT: True,
    NodeType.SUBSCRIPT: True,
    NodeType.SLICE: True,
}

# These node types is identifier-like when:
#   let object['name'] = 0
AnalyzableSubScriptChildNodeTypes = {
    NodeType.NUMBER: True,
    NodeType.STRING: True,
}


class IdentifierClassifier(object):
    """ A class for identifier classifiers.
    This class classify nodes by 3 flags:

    - is dynamic: True if the identifier name can be determined by static analysis.
    - is member: True if the identifier is a member of a subscription/dot/slice node.
    - is declaring: True if the identifier is used to declare.
    """

    class IdentifierCollector(object):
        """ A class for identifier node collectors.
        Only static and not member nodes will be collected and the nodes will
        be grouped by 2 types; declaring or referencing.
        """

        def collect_identifiers(self, ast):
            self.static_referencing_identifiers = []
            self.static_declaring_identifiers = []

            # TODO: Make more performance efficiency.
            traverse(ast, on_enter=self._enter_handler)

            return {
                'static_declaring_identifiers': self.static_declaring_identifiers,
                'static_referencing_identifiers': self.static_referencing_identifiers,
            }


        def _enter_handler(self, node):
            is_identifier_like_node = IDENTIFIER_ATTRIBUTE in node

            if not is_identifier_like_node:
                return

            id_attr = node[IDENTIFIER_ATTRIBUTE]
            if id_attr[IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG] or \
                    id_attr[IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG]:
                return

            if id_attr[IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG]:
                self.static_declaring_identifiers.append(node)
            else:
                self.static_referencing_identifiers.append(node)


    def attach_identifier_attributes(self, ast):
        """ Attach 3 flags to the AST.

        - is dynamic: True if the identifier name can be determined by static analysis.
        - is member: True if the identifier is a member of a subscription/dot/slice node.
        - is declaring: True if the identifier is used to declare.
        - is function: True if the identifier is a function. Vim distinguish
                       between function identifiers and variable identifiers.
        """
        traverse(ast, on_enter=self._enter_handler)
        return ast


    def _set_identifier_attribute(self, node, is_definition=None, is_dynamic=None,
                                  is_member_of_subscript=None, is_function=None):

        id_attr = node.setdefault(IDENTIFIER_ATTRIBUTE, {
            IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG: False,
            IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG: False,
            IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG: False,
            IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG: False,
        })

        if is_definition is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG] = is_definition

        if is_dynamic is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG] = is_dynamic

        if is_member_of_subscript is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG] = is_member_of_subscript

        if is_function is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG] = is_function


    def _enter_handler(self, node):
        node_type = NodeType(node['type'])

        if node_type in IdentifierTerminateNodeTypes:
            # Attach identifier attributes to all IdentifierTerminateNodeTypes.
            self._set_identifier_attribute(node)

        if node_type in AccessorLikeNodeTypes:
            self._pre_mark_accessor_children(node)

        if node_type in DeclarativeNodeTypes:
            self._enter_declarative_node(node)

        if node_type is NodeType.CALL:
            self._enter_call_node(node)


    def _pre_mark_accessor_children(self, node):
        node_type = NodeType(node['type'])
        dict_node = node['left']

        if NodeType(dict_node['type']) in AccessorLikeNodeTypes:
            self._pre_mark_accessor_children(dict_node)

        if node_type is NodeType.SLICE:
            for member_node in node['rlist']:
                # In VimLParser spec, an empty array means null.
                #   list[1:] => {rlist: [node, []]}
                if type(member_node) is list:
                    continue

                self._pre_mark_member_node(member_node)
        else:
            member_node = node['right']
            self._pre_mark_member_node(member_node)


    def _pre_mark_member_node(self, member_node):
        member_node_type = NodeType(member_node['type'])

        if member_node_type in IdentifierTerminateNodeTypes or \
                member_node_type in AnalyzableSubScriptChildNodeTypes:
            self._set_identifier_attribute(member_node,
                                           is_member_of_subscript=True)


    def _enter_identifier_like_node(self, id_like_node, is_definition=None,
{'fraction_non_alphanumeric': 0.0631768953068592, 'fraction_numerical': 0.0008595495960116899, 'mean_word_length': 2.7423608877452557, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7387285', 'n_tokens_mistral': 3429, 'n_tokens_neox': 3305, 'n_words': 755}