<commit_before># Copyright 2016 by <NAME> Technologies Corp.  All Rights Reserved.

"""
Check that the variable names referenced in a function are
defined within the scope of that function.

This is a useful heuristic in some cases, but is contrary
to the Python programming model, which permits this to be
more dynamic, using things like decorators and **kwargs
to sneak names into the local scope in a way that can be
impossible to analyze without considering the context in
which the function is called.

It is also possible to explicitly undefine a variable name,
and we don't take that into account either: once a variable
has been given a name, we assume that it remains until it
falls out of scope.

A more difficult (and more typical) problem is that whether
a name is or is not bound to a variable may depend on details
of the execution.  We don't have a good answer for this
yet.  For example:

        # y is not bound yet
        if x:
            y = 'something'

        print(y)

If x is non-False, then y will be bound, but if not, then
y will be unbound and the "print(y)" will fail with an
exception.  (there are also other, less obvious situations
where variables may or may not be bound after a statement
is executed, such as loops with empty iterators, or
assignments within orelse or except blocks)

We're going to start with simple heuristics:

1) If a name appears as in the formal parameters of the
    function definition, it is assumed to be in-scope

2) If a name appears as an lval (or part of an lval tuple)
    prior to its reference, it is assumed to be in-scope

    "prior to its reference" means "prior in the preorder walk
    through the AST" and there's no guarantee that the
    code that defines the symbol is executed prior to the
    code referencing the symbol.

3) If a name appears anywhere else (not the lval) in a
    statement, and is not in the global scope nor a formal
    parameter nor has appeared in an earlier assignment
    (in order of appearance) then consider it an potential
    error.

4) If a name first appears at a nesting level X and then
    is referenced at any nesting level smaller than X,
    consider it a potential error.

5) Redefinitions of builtins (either by use as a formal
    parameter, or explicitly assigned) is considered a
    potential error.
"""

import ast
import builtins

from pyqgl2.ast_util import NodeError
from pyqgl2.inline import NameFinder


def scope_check(function_def, module_names=None):
    """
    Convenience function to create a CheckScoping instance and
    use it to check the scoping of variables within an ast.FunctionDef.

    The caller should use NodeError.error_detected() and/or
    NodeError.halt_on_error() to check if any errors were detected.
    Ordinarily this function has no effect except to generate
    warning or error messages if it detects problems.
    """

    # Something is seriously wrong if we get anything other than
    # an ast.FunctionDef
    #
    assert isinstance(function_def, ast.FunctionDef)

    CheckScoping(module_names=module_names).visit(function_def)


class CheckScoping(ast.NodeVisitor):
    """
    """

    BUILTIN_SCOPE = -2
    MODULE_SCOPE = -1
    PARAM_SCOPE = 0
    LOCAL_SCOPE = 1

    def __init__(self, module_names=None):

        self.formal_names = set()

        # mapping from name to nesting level (the nesting level
        # is 1-based: formal parameters are at nesting level 0).
        # Python builtins are -2, which means that we shouldn't
        # redefine them (although this is permitted).
        #
        self.local_names = dict()

        # This method of enumerating the builtins is probably
        # not portable
        #
        for name in builtins.__dict__:
            self.local_names[name] = CheckScoping.BUILTIN_SCOPE

        if module_names:
            for name in module_names:
                self.local_names[name] = CheckScoping.MODULE_SCOPE

        # How deeply we're nested.  Formal parameters are defined
        # at nesting level PARAM_SCOPE, and assignments in the body
        # of the function are at level LOCAL_SCOPE or greater.
        #
        self.nesting_depth = CheckScoping.LOCAL_SCOPE

        self.name_finder = NameFinder()

    def visit_FunctionDef(self, node):
        """
        The usual entry point: insert the names used by the
        formal parameters, and then process the body
        """

        for arg in node.args.args:
            name = arg.arg
            if name not in self.local_names:
                self.local_names[arg.arg] = CheckScoping.PARAM_SCOPE
            else:
                NodeError.warning_msg(
                        node,
                        'formal parameter masks a builtin symbol [%s]' % name)

        self.do_body(node.body)

    def visit_If(self, node):
        self.visit(node.test)
        self.do_body(node.body)
        self.do_body(node.orelse)

    def visit_For(self, node):
        self.visit(node.iter)
        self.do_lval(node.target)
        self.do_body(node.body)
        self.do_body(node.orelse)

    def visit_Attribute(self, node):
        """
        We don't examine attributes (yet).  We assume that
        they're OK without actually checking the namespace.

        TODO: check the collapsed name of the attribute<commit_msg>differentiate between module and builtin symbols<commit_after># Copyright 2016 by Raytheon BBN Technologies Corp.  All Rights Reserved.

"""
Check that the variable names referenced in a function are
defined within the scope of that function.

This is a useful heuristic in some cases, but is contrary
to the Python programming model, which permits this to be
more dynamic, using things like decorators and **kwargs
to sneak names into the local scope in a way that can be
impossible to analyze without considering the context in
which the function is called.

It is also possible to explicitly undefine a variable name,
and we don't take that into account either: once a variable
has been given a name, we assume that it remains until it
falls out of scope.

A more difficult (and more typical) problem is that whether
a name is or is not bound to a variable may depend on details
of the execution.  We don't have a good answer for this
yet.  For example:

        # y is not bound yet
        if x:
            y = 'something'

        print(y)

If x is non-False, then y will be bound, but if not, then
y will be unbound and the "print(y)" will fail with an
exception.  (there are also other, less obvious situations
where variables may or may not be bound after a statement
is executed, such as loops with empty iterators, or
assignments within orelse or except blocks)

We're going to start with simple heuristics:

1) If a name appears as in the formal parameters of the
    function definition, it is assumed to be in-scope

2) If a name appears as an lval (or part of an lval tuple)
    prior to its reference, it is assumed to be in-scope

    "prior to its reference" means "prior in the preorder walk
    through the AST" and there's no guarantee that the
    code that defines the symbol is executed prior to the
    code referencing the symbol.

3) If a name appears anywhere else (not the lval) in a
    statement, and is not in the global scope nor a formal
    parameter nor has appeared in an earlier assignment
    (in order of appearance) then consider it an potential
    error.

4) If a name first appears at a nesting level X and then
    is referenced at any nesting level smaller than X,
    consider it a potential error.

5) Redefinitions of builtins (either by use as a formal
    parameter, or explicitly assigned) is considered a
    potential error.
"""

import ast
import builtins

import pyqgl2.inline

from pyqgl2.ast_util import NodeError


def scope_check(function_def, module_names=None):
    """
    Convenience function to create a CheckScoping instance and
    use it to check the scoping of variables within an ast.FunctionDef.

    The caller should use NodeError.error_detected() and/or
    NodeError.halt_on_error() to check if any errors were detected.
    Ordinarily this function has no effect except to generate
    warning or error messages if it detects problems.

    Returns True to indicate success.  Right now, there's no way
    for this function to fail (short of crashing), but eventually
    we might decide that some of the warnings are really errors.
    """

    # Something is seriously wrong if we get anything other than
    # an ast.FunctionDef
    #
    assert isinstance(function_def, ast.FunctionDef)

    checker = CheckScoping(module_names=module_names)
    checker.visit(function_def)

    return True


class CheckScoping(ast.NodeVisitor):
    """
    """

    BUILTIN_SCOPE = -2
    MODULE_SCOPE = -1
    PARAM_SCOPE = 0
    LOCAL_SCOPE = 1

    def __init__(self, module_names=None):

        # mapping from name to nesting level (the nesting level
        # is 1-based: formal parameters are at nesting level 0).
        # Python builtins are -2, which means that we shouldn't
        # redefine them (although this is permitted).
        #
        self.local_names = dict()

        # This method of enumerating the builtins is probably
        # not portable
        #
        for name in builtins.__dict__:
            self.local_names[name] = CheckScoping.BUILTIN_SCOPE

        if module_names:
            for name in module_names:
                self.local_names[name] = CheckScoping.MODULE_SCOPE

        # How deeply we're nested.  Formal parameters are defined
        # at nesting level PARAM_SCOPE, and assignments in the body
        # of the function are at level LOCAL_SCOPE or greater.
        #
        self.nesting_depth = CheckScoping.LOCAL_SCOPE

        self.name_finder = pyqgl2.inline.NameFinder()

    def visit_FunctionDef(self, node):
        """
        The usual entry point: insert the names used by the
        formal parameters, and then process the body
        """

        for arg in node.args.args:
            name = arg.arg
            if name not in self.local_names:
                self.local_names[name] = CheckScoping.PARAM_SCOPE
            elif self.local_names[name] == CheckScoping.MODULE_SCOPE:
                NodeError.warning_msg(
                        node,
                        'formal parameter masks a module symbol [%s]' % name)
            else:
                NodeError.warning_msg(
                        node,
                        'formal parameter masks a builtin symbol [%s]' % name)

        self.do_body(node.body)

    def visit_If(self, node):
        self.visit(node.test)
        self.do_body(node.body)
        self.do_body(node.orelse)

    def visit_For(self, node):
        self.visit(node.iter)
        self.do_lval(node.target)
        self.do_body(node.body)
        self.do_body(node.orelse)

    def visit_Attribute(self, node):
        """
        We don't examine attributes (yet).  We assume that
        they're OK without actually checking the namespace.

        TODO: check the collapsed name of the attribute
{'fraction_non_alphanumeric': 0.05387872781331651, 'fraction_numerical': 0.0033336336606901522, 'mean_word_length': 2.657331136738056, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5256374', 'n_tokens_mistral': 3085, 'n_tokens_neox': 2909, 'n_words': 1379}