<commit_before>'''
This module provides a few code analysis for the pythran language.
    * LocalDeclarations gathers declarations local to a node
    * GlobalDeclarations gathers top-level declarations
    * Locals computes the value of locals()
    * Globals computes the value of globals()
    * ImportedIds gathers identifiers imported by a node
    * ConstantExpressions gathers constant expression
    * Aliases gather aliasing informations
    * Identifiers gathers all identifiers used in a module
    * YieldPoints gathers all yield points from a node
    * BoundedExpressions gathers temporary objects
'''

from tables import modules, builtin_constants, builtin_constructors
import ast
import networkx as nx
import metadata
from passmanager import NodeAnalysis, FunctionAnalysis, ModuleAnalysis


##

class LocalDeclarations(NodeAnalysis):
    """Gathers all local symbols from a function"""
    def __init__(self):
        self.result=set()
        NodeAnalysis.__init__(self)

    def visit_Assign(self, node):
        for t in node.targets:
            assert isinstance(t, ast.Name) or isinstance(t, ast.Subscript)
            if isinstance(t, ast.Name) and not metadata.get(t, metadata.LocalVariable):
                self.result.add(t)

    def visit_For(self, node):
        assert isinstance(node.target, ast.Name)
        self.result.add(node.target)
        [ self.visit(n) for n in node.body ]

##
class GlobalDeclarations(ModuleAnalysis):
    """Generates a function name -> function node binding"""
    def __init__(self):
        self.result=dict()
        ModuleAnalysis.__init__(self)

    def visit_Import(self, node):
        for alias in node.names:
            self.result[alias.name]=alias

    def visit_ImportFrom(self, node):
        for alias in node.names:
            self.result[alias.asname if alias.asname else alias.name ]=alias
        return None

    def visit_FunctionDef(self, node):
        self.result[node.name]=node

##
class Locals(ModuleAnalysis):
    """Statically compute the value of locals() before each statement"""
    def __init__(self):
        self.result=dict()
        self.locals=set()
        ModuleAnalysis.__init__(self, GlobalDeclarations)
        # statements that do not define a new variable
        self.visit_Return=self.visit_Yield=self.visit_TryExcept=self.visit_AugAssign=self.visit_Print=self.visit_While=self.visit_If=self.Raise=self.Assert=self.Expr=self.Pass=self.Break=self.Continue=self.store_and_visit

    def generic_visit(self, node):
        ModuleAnalysis.generic_visit(self, node)
        if node not in self.result:
            self.result[node]=self.result[self.expr_parent]

    def store_and_visit(self, node):
        self.expr_parent=node
        self.result[node]=self.locals.copy()
        self.generic_visit(node)

    def visit_FunctionDef(self, node):
        self.expr_parent=node
        self.result[node]=self.locals.copy()
        parent_locals=self.locals.copy()
        self.locals.update(arg.id for arg in node.args.args)
        [ self.visit(n) for n in node.body ]
        self.locals=parent_locals

    def visit_Assign(self, node):
        self.expr_parent=node
        self.result[node]=self.locals.copy()
        self.visit(node.value)
        self.locals.update(t.id for t in node.targets if isinstance(t, ast.Name))

    def visit_For(self, node):
        self.expr_parent=node
        self.result[node]=self.locals.copy()
        self.visit(node.iter)
        self.locals.add(node.target.id)
        [self.visit(n) for n in node.body]
        [self.visit(n) for n in node.orelse]

    def visit_Import(self, node):
        self.result[node]=self.locals.copy()
        self.locals.update(alias.name for alias in node.names)

    def visit_ImportFrom(self, node):
        self.result[node]=self.locals.copy()
        self.locals.update(alias.name for alias in node.names)<commit_msg>Fix bug in Locals analysis.<commit_after>'''
This module provides a few code analysis for the pythran language.
    * LocalDeclarations gathers declarations local to a node
    * GlobalDeclarations gathers top-level declarations
    * Locals computes the value of locals()
    * Globals computes the value of globals()
    * ImportedIds gathers identifiers imported by a node
    * ConstantExpressions gathers constant expression
    * Aliases gather aliasing informations
    * Identifiers gathers all identifiers used in a module
    * YieldPoints gathers all yield points from a node
    * BoundedExpressions gathers temporary objects
'''

from tables import modules, builtin_constants, builtin_constructors
import ast
import networkx as nx
import metadata
from passmanager import NodeAnalysis, FunctionAnalysis, ModuleAnalysis


##

class LocalDeclarations(NodeAnalysis):
    """Gathers all local symbols from a function"""
    def __init__(self):
        self.result=set()
        NodeAnalysis.__init__(self)

    def visit_Assign(self, node):
        for t in node.targets:
            assert isinstance(t, ast.Name) or isinstance(t, ast.Subscript)
            if isinstance(t, ast.Name) and not metadata.get(t, metadata.LocalVariable):
                self.result.add(t)

    def visit_For(self, node):
        assert isinstance(node.target, ast.Name)
        self.result.add(node.target)
        [ self.visit(n) for n in node.body ]

##
class GlobalDeclarations(ModuleAnalysis):
    """Generates a function name -> function node binding"""
    def __init__(self):
        self.result=dict()
        ModuleAnalysis.__init__(self)

    def visit_Import(self, node):
        for alias in node.names:
            self.result[alias.name]=alias

    def visit_ImportFrom(self, node):
        for alias in node.names:
            self.result[alias.asname if alias.asname else alias.name ]=alias
        return None

    def visit_FunctionDef(self, node):
        self.result[node.name]=node

##
class Locals(ModuleAnalysis):
    """Statically compute the value of locals() before each statement"""
    def __init__(self):
        self.result=dict()
        self.locals=set()
        ModuleAnalysis.__init__(self, GlobalDeclarations)
        # statements that do not define a new variable
        self.visit_Return=self.visit_Yield=self.visit_TryExcept=self.visit_AugAssign=self.visit_Print=self.visit_While=self.visit_If=self.Raise=self.Assert=self.Expr=self.Pass=self.Break=self.Continue=self.store_and_visit

    def generic_visit(self, node):
        ModuleAnalysis.generic_visit(self, node)
        if node not in self.result:
            self.result[node]=self.result[self.expr_parent]

    def store_and_visit(self, node):
        self.expr_parent=node
        self.result[node]=self.locals.copy()
        self.generic_visit(node)

    def visit_Module(self, node):
        self.expr_parent=node
        self.result[node]=self.locals
        [ self.visit(n) for n in node.body]

    def visit_FunctionDef(self, node):
        self.expr_parent=node
        self.result[node]=self.locals.copy()
        parent_locals=self.locals.copy()
        self.locals.update(arg.id for arg in node.args.args)
        [ self.visit(n) for n in node.body ]
        self.locals=parent_locals

    def visit_Assign(self, node):
        self.expr_parent=node
        self.result[node]=self.locals.copy()
        self.visit(node.value)
        self.locals.update(t.id for t in node.targets if isinstance(t, ast.Name))

    def visit_For(self, node):
        self.expr_parent=node
        self.result[node]=self.locals.copy()
        self.visit(node.iter)
        self.locals.add(node.target.id)
        [self.visit(n) for n in node.body]
        [self.visit(n) for n in node.orelse]

    def visit_Import(self, node):
        self.result[node]=self.locals.copy()
        self.locals.update(alias.name for alias in node.names)

    def visit_ImportFrom(self, node):
        self.result[node]=self.locals.copy()
        self.locals.update(alias.name for alias in node.names)
{'fraction_non_alphanumeric': 0.09754871150219988, 'fraction_numerical': 0.0, 'mean_word_length': 3.044738179969497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2534750', 'n_tokens_mistral': 2365, 'n_tokens_neox': 2314, 'n_words': 583}