<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from redbaron import RedBaron
from redbaron import StringNode, IntNode, FloatNode, ComplexNode, FloatExponantNode ,StarNode
from redbaron import NameNode
from redbaron import AssignmentNode
from redbaron import CommentNode, EndlNode
from redbaron import ComparisonNode
from redbaron import ComparisonOperatorNode
from redbaron import UnitaryOperatorNode
from redbaron import BinaryOperatorNode, BooleanOperatorNode
from redbaron import AssociativeParenthesisNode
from redbaron import DefNode
from redbaron import ClassNode
from redbaron import TupleNode, ListNode
from redbaron import CommaProxyList
from redbaron import LineProxyList
from redbaron import NodeList
from redbaron import DotProxyList
from redbaron import ReturnNode
from redbaron import PassNode
from redbaron import DefArgumentNode
from redbaron import ForNode
from redbaron import PrintNode
from redbaron import DelNode
from redbaron import DictNode, DictitemNode
from redbaron import WhileNode
from redbaron import IfelseblockNode, IfNode, ElseNode, ElifNode
from redbaron import DotNode, AtomtrailersNode
from redbaron import CallNode
from redbaron import CallArgumentNode
from redbaron import AssertNode
from redbaron import ExceptNode
from redbaron import FinallyNode
from redbaron import RaiseNode
from redbaron import TryNode
from redbaron import YieldNode
from redbaron import YieldAtomNode
from redbaron import BreakNode
from redbaron import GetitemNode, SliceNode
from redbaron import ImportNode, FromImportNode
from redbaron import DottedAsNameNode
from redbaron import NameAsNameNode
from redbaron import LambdaNode
from redbaron import WithNode

from pyccel.ast import NativeInteger, NativeFloat, NativeDouble, NativeComplex
from pyccel.ast import NativeBool
from pyccel.ast import NativeRange
from pyccel.ast import NativeIntegerList
from pyccel.ast import NativeFloatList
from pyccel.ast import NativeDoubleList
from pyccel.ast import NativeComplexList
from pyccel.ast import NativeList
from pyccel.ast import NativeSymbol
from pyccel.ast import String
from pyccel.ast import datatype, DataTypeFactory
from pyccel.ast import Nil
from pyccel.ast import Variable
from pyccel.ast import DottedName, DottedVariable
from pyccel.ast import Assign, AliasAssign, SymbolicAssign, AugAssign ,Assigns
from pyccel.ast import Return
from pyccel.ast import Pass
from pyccel.ast import FunctionCall, MethodCall, ConstructorCall
from pyccel.ast import FunctionDef, Interface, PythonFunction, SympyFunction
from pyccel.ast import ClassDef
from pyccel.ast import GetDefaultFunctionArg
from pyccel.ast import For
from pyccel.ast import If
from pyccel.ast import While
from pyccel.ast import Print
from pyccel.ast import SymbolicPrint
from pyccel.ast import Del
from pyccel.ast import Assert
from pyccel.ast import Comment, EmptyLine, NewLine
from pyccel.ast import Break
from pyccel.ast import Slice, IndexedVariable, IndexedElement
from pyccel.ast import FunctionHeader, ClassHeader, MethodHeader
from pyccel.ast import VariableHeader, InterfaceHeader
from pyccel.ast import MetaVariable
from pyccel.ast import Concatinate
from pyccel.ast import ValuedVariable
from pyccel.ast import Argument, ValuedArgument
from pyccel.ast import Is
from pyccel.ast import Import, TupleImport
from pyccel.ast import AsName
from pyccel.ast import AnnotatedComment
from pyccel.ast import With
from pyccel.ast import Range
from pyccel.ast import List
from pyccel.ast import builtin_function as pyccel_builtin_function
from pyccel.ast import builtin_import as pyccel_builtin_import
from pyccel.ast import builtin_import_registery as pyccel_builtin_import_registery

from pyccel.parser.utilities import omp_statement, acc_statement
from pyccel.parser.utilities import fst_move_directives
from pyccel.parser.utilities import is_valid_filename_pyh, is_valid_filename_py
from pyccel.parser.utilities import read_file

from pyccel.parser.syntax.headers import parse as hdr_parse
from pyccel.parser.syntax.openmp import parse as omp_parse
from pyccel.parser.syntax.openacc import parse as acc_parse

from pyccel.parser.errors import (Errors, PyccelSyntaxError,
                                  PyccelSemanticError)

# TODO - remove import * and only import what we need
#      - use OrderedDict whenever it is possible
from pyccel.parser.messages import *


from sympy import Symbol, sympify
from sympy import Tuple
from sympy import NumberSymbol, Number
from sympy import Integer, Float
from sympy import Add, Mul, Pow, floor, Mod
from sympy import FunctionClass
from sympy import Lambda
from sympy.core.expr import Expr
from sympy.core.relational import Eq, Ne, Lt, Le, Gt, Ge
from sympy.core.containers import Dict
from sympy.core.function import Function, FunctionClass
from sympy.logic.boolalg import And, Or
from sympy.logic.boolalg import true, false
from sympy.logic.boolalg import Not
from sympy.logic.boolalg import Boolean, BooleanTrue, BooleanFalse
from sympy.tensor import Idx, Indexed, IndexedBase
from sympy.utilities.iterables import iterable

from collections import OrderedDict

import traceback
import importlib
import os
import sys


# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3


#  ... useful functions for imports
# TODO installed modules. must ask python (working version) where the module is
#      installed
def get_filename_from_import(module):
    """Returns a valid filename with absolute path, that corresponds to the
    definition of module.
    The priority order is:
        - header files (extension == pyh)
        - python files (extension == py)
    """
    filename_pyh = '{}.pyh'.format(module)
    filename_py  = '{}.py'.format(module)

    if is_valid_filename_pyh(filename_pyh): return os.path.abspath(filename_pyh)
    if is_valid_filename_py(filename_py): return os.path.abspath(filename_py)

    source = module
    if len(module.split('.')) > 1:
        # we remove the last entry, since it can be a pyh file
        source = '.'.join(i for i in module.split('.')[:-1])
        _module = module.split('.')[-1]
        filename_pyh = '{}.pyh'.format(_module)
        filename_py  = '{}.py'.format(_module)

    try:
        package = importlib.import_module(source)
        package_dir = str(package.__path__[0])
    except:
        errors = Errors()
        errors.report(PYCCEL_UNFOUND_IMPORTED_MODULE,
                      symbol=source,
                      severity='fatal')

    filename_pyh = os.path.join(package_dir, filename_pyh)
    filename_py  = os.path.join(package_dir, filename_py)

    if os.path.isfile(filename_pyh):
        return filename_pyh
    elif os.path.isfile(filename_py):
        return filename_py

    errors = Errors()
    errors.report(PYCCEL_UNFOUND_IMPORTED_MODULE,
                  symbol=module,
                  severity='fatal')
#  ...

#  ...
def _get_variable_name(var):
    """."""

    if isinstance(var, (Variable, IndexedVariable)):
        return str(var)
    elif isinstance(var, IndexedElement):
        return str(var.base)

    raise NotImplementedError('Uncovered type {dtype}'.format(dtype=type(var)))
#  ...


class Parser(object):

    """ Class for a Parser."""

    def __init__(self, inputs, debug=False, headers=None, show_traceback=True):
        """Parser constructor.

        inputs: str
            filename or code to parse as a string

        debug: bool
            True if in debug mode.

        headers: list, tuple
            list of headers to append to the namespace

        show_traceback: bool
            prints Tracebacke exception if True

        """
        self._fst = None
        self._ast = None
        self._filename = None
        self._metavars = {}
        self._namespace = {}
        self._namespace['imports'] = OrderedDict()
        self._namespace['variables'] = {}
        self._namespace['classes'] = {}
        self._namespace['functions'] = {}
        self._namespace['cls_constructs'] = {}
        self._namespace['symbolic_functions'] = {}
        self._namespace['python_functions'] = {}
        self._scope = {}  # represent the namespace of a function
        self._current_class = None
        self._current = None  # we use it to detect the current method or function
        # we use it to store the imports
        self._imports = {}
        # a Parser can have parents, who are importing it. imports are then its sons.
        self._parents = []
        self._sons = []
        self._d_parsers = OrderedDict()
        # the following flags give us a status on the parsing stage
        self._syntax_done = False
        self._semantic_done = False
        # current position for errors
        self._bounding_box = None
        # flag for blocking errors. if True, an error with this flag will cause
        # Pyccel to stop
        # TODO ERROR must be passed to the Parser __init__ as argument
        self._blocking = False
        # printing exception
        self._show_traceback = show_traceback

        # TODO use another name for headers
        #      => reserved keyword, or use __

        self._namespace['headers'] = {}
        if headers:
            if not isinstance(headers, dict):
                raise TypeError('Expecting a dict of headers')

            for (key, value) in list(headers.items()):
                self._namespace['headers'][key] = value

        # check if inputs is a file
        code = inputs
        if os.path.isfile(inputs):
            # we don't use is_valid_filename_py since it uses absolute path
            # file extension
            ext = inputs.split('.')[-1]
            if not(ext in ['py', 'pyh']):
                errors = Errors()
                errors.report(INVALID_FILE_EXTENSION,
                              symbol=ext,
                              severity='fatal')
                errors.check()
                raise SystemExit(0)

            code = read_file(inputs)
            self._filename = inputs

        self._code = code

        try:
            red = RedBaron(code)
        except Exception as e:
            errors = Errors()
            errors.report(INVALID_PYTHON_SYNTAX,
                          symbol='\n'+str(e),
                          severity='fatal')
            errors.check()
            raise SystemExit(0)

        red = fst_move_directives(red)
        self._fst = red

    @property
    def namespace(self):
        return self._namespace

    @property
    def headers(self):
        return self.namespace['headers']

    @property
    def imports(self):
        return self.namespace['imports']

    @property
    def filename(self):
        return self._filename

    @property
    def code(self):
        return self._code

    @property
    def fst(self):
        return self._fst

    @property
    def ast(self):
        if self._ast is None:
            self._ast = self.parse()
        return self._ast

    @property
    def metavars(self):
        return self._metavars

    @property
    def current_class(self):
        return self._current_class

    @property
    def syntax_done(self):
        return self._syntax_done

    @property
    def semantic_done(self):
        return self._semantic_done

    @property
    def parents(self):
        """Returns the parents parser."""
        return self._parents

    @property
    def sons(self):
        """Returns the sons parser."""
        return self._sons

    @property
    def d_parsers(self):
        """Returns the d_parsers parser."""
        return self._d_parsers

    @property
    def is_header_file(self):
        """Returns True if we are treating a header file."""
        if self.filename:
            return self.filename.split('.')[-1] == 'pyh'
        else:
            return False

    @property
    def bounding_box(self):
        return self._bounding_box

    @property
    def blocking(self):
        return self._blocking

    @property
    def show_traceback(self):
        return self._show_traceback

    def append_parent(self, parent):
        """."""
        # TODO check parent is not in parents
        self._parents.append(parent)

    def append_son(self, son):
        """."""
        # TODO check son is not in sons
        self._sons.append(son)

    def parse(self, d_parsers=None, verbose=False):
        """converts redbaron fst to sympy ast."""
        if self.syntax_done:
            return self.ast

        # TODO - add settings to Errors
        #      - filename

        errors = Errors()
        if self.filename:
            errors.set_target(self.filename, 'file')
        errors.set_parser_stage('syntax')

        # we add the try/except to allow the parser to find all possible errors
        try:
            ast = self._fst_to_ast(self.fst)
        except Exception as e:
            errors.check()
            if self.show_traceback:
                traceback.print_exc()
            raise SystemExit(0)

        self._ast = ast

        errors.check()
        self._syntax_done = True

        if d_parsers is None:
            d_parsers = OrderedDict()
        self._d_parsers = self._parse_sons(d_parsers, verbose=verbose)

        return ast

    def annotate(self, **settings):
        """."""
        if self.semantic_done:
            return self.ast

        # TODO - add settings to Errors
        #      - filename

        errors = Errors()
        if self.filename:
            errors.set_target(self.filename, 'file')
        errors.set_parser_stage('semantic')

        # we first treat all sons to get imports
        verbose = settings.pop('verbose', False)
        self._annotate_parents(verbose=verbose)

        # then we treat the current file
        ast = self.ast
        # we add the try/except to allow the parser to find all possible errors
        try:
            ast = self._annotate(ast, **settings)
        except Exception as e:
            errors.check()
            if self.show_traceback:
                traceback.print_exc()
            raise SystemExit(0)

        self._ast = ast

        # in the case of a header file, we need to convert all headers to
        # FunctionDef etc ...
        if self.is_header_file:
            for name,v in list(self.headers.items()):
                if isinstance(v, FunctionHeader) and not isinstance(v, MethodHeader):
                    F = self.get_function(name)
                    if F is None:
                        interfaces = v.create_definition()
                        for F in interfaces:
                            self.insert_function(F)
                    else:
                        errors.report(IMPORTING_EXISTING_IDENTIFIED,
                                      symbol=name,
                                      blocker=True,
                                      severity='fatal')

#        print('++++++++++++++')
#        print(errors.error_info_map)
        errors.check()
        self._semantic_done = True

        return ast

    def _parse_sons(self, d_parsers, verbose=False):
        """Recursive algorithm for syntax analysis on a given file and its
        dependencies.
        This function always terminates with an OrderedDict that contains parsers
        for all involved files.
        """
        treated = set(d_parsers.keys())
        imports = set(self.imports.keys())
        imports = imports.difference(treated)
        if not imports:
            return d_parsers

        for source in imports:
            if verbose:
                print('>>> treating :: {}'.format(source))

            # get the absolute path corresponding to source
            filename = get_filename_from_import(source)

            q = Parser(filename)
            q.parse(d_parsers=d_parsers)
            d_parsers[source] = q

        # link self to its sons
        imports = list(self.imports.keys())
        for source in imports:
            d_parsers[source].append_parent(self)
            self.append_son(d_parsers[source])

        return d_parsers

    def _annotate_parents(self, **settings):

        verbose = settings.pop('verbose', False)

        # ...
        def _update_from_son(p):
            # TODO - only import what is needed
            #      - use insert_variable etc
            for entry in ['variables', 'classes', 'functions',
                          'cls_constructs']:
                d_self = self._namespace[entry]
                d_son  = p.namespace[entry]
                for k,v in list(d_son.items()):
                    d_self[k] = v
        # ...

        # we first treat sons that have no imports
        for p in self.sons:
            if not p.sons:
                if verbose:
                    print('>>> treating :: {}'.format(p.filename))
                p.annotate(**settings)

        # finally we treat the remaining sons recursively
        for p in self.sons:
            if p.sons:
                if verbose:
                    print('>>> treating :: {}'.format(p.filename))
                p.annotate(**settings)


    def print_namespace(self):
        # TODO improve spacing
        print('------- namespace -------')
        for (k, v) in self.namespace.items():
            print('{var} \t :: \t {dtype}'.format(var=k, dtype=type(v)))
        print('-------------------------')

    def view_namespace(self, entry):
        # TODO improve
        try:
            from tabulate import tabulate

            table = []
            for (k, v) in self.namespace[entry].items():
                k_str = '{}'.format(k)
                if entry == 'imports':
                    if v is None:
                        v_str = '*'
                    else:
                        v_str = '{}'.format(v)
                elif entry == 'variables':
                    v_str = '{}'.format(type(v))
                else:
                    raise NotImplementedError('TODO')

                line = [k_str, v_str]
                table.append(line)
            headers = ['module', 'target']
#            txt = tabulate(table, headers, tablefmt="rst")
            txt = tabulate(table, tablefmt="rst")
            print(txt)

        except:
            print('------- namespace.{} -------'.format(entry))
            for (k, v) in self.namespace[entry].items():
                print('{var} \t :: \t {value}'.format(var=k, value=v))
            print('-------------------------')


    def dot(self, filename):
        """Exports sympy AST using graphviz then convert it to an image."""

        expr = self.ast
        graph_str = dotprint(expr)

        f = open(filename, 'w')
        f.write(graph_str)
        f.close()

        # name without path

        name = os.path.basename(filename)

        # name without extension

        name = os.path.splitext(name)[0]
        cmd = 'dot -Tps {name}.gv -o {name}.ps'.format(name=name)
        os.system(cmd)

    def insert_import(self, expr):
        """."""
        # TODO improve

        if not isinstance(expr, Import):
            raise TypeError('Expecting Import expression')

        # if source is not specified, imported things are treated as sources
        # TODO test if builtin import
        source = expr.source
        if source is None:
            for t in expr.target:
                name = str(t)
                self._namespace['imports'][name] = None
        else:
            source = str(source)
            if not(source in pyccel_builtin_import_registery):
                for t in expr.target:
                    name = str(t)
                    self._namespace['imports'][source] = name


    def get_variable(self, name):
        """."""

        var = None
        if self.current_class:
            for i in self._current_class.attributes:
                if str(i.name) == name:
                    var = i
                    return var
        if self._current:
            if name in self._scope[self._current]['variables']:
                var = self._scope[self._current]['variables'][name]
        if isinstance(self._current, DottedName):
            if name in self._scope[self._current.name[0]]['variables']:
                var = self._scope[self._current.name[0]]['variables'
                        ][name]
        if name in self._namespace['variables']:
            var = self._namespace['variables'][name]
        return var

    def get_variables(self, source=None):
        if source == 'parent':<commit_msg>convert only the headers needed in imports<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from redbaron import RedBaron
from redbaron import StringNode, IntNode, FloatNode, ComplexNode, FloatExponantNode ,StarNode
from redbaron import NameNode
from redbaron import AssignmentNode
from redbaron import CommentNode, EndlNode
from redbaron import ComparisonNode
from redbaron import ComparisonOperatorNode
from redbaron import UnitaryOperatorNode
from redbaron import BinaryOperatorNode, BooleanOperatorNode
from redbaron import AssociativeParenthesisNode
from redbaron import DefNode
from redbaron import ClassNode
from redbaron import TupleNode, ListNode
from redbaron import CommaProxyList
from redbaron import LineProxyList
from redbaron import NodeList
from redbaron import DotProxyList
from redbaron import ReturnNode
from redbaron import PassNode
from redbaron import DefArgumentNode
from redbaron import ForNode
from redbaron import PrintNode
from redbaron import DelNode
from redbaron import DictNode, DictitemNode
from redbaron import WhileNode
from redbaron import IfelseblockNode, IfNode, ElseNode, ElifNode
from redbaron import DotNode, AtomtrailersNode
from redbaron import CallNode
from redbaron import CallArgumentNode
from redbaron import AssertNode
from redbaron import ExceptNode
from redbaron import FinallyNode
from redbaron import RaiseNode
from redbaron import TryNode
from redbaron import YieldNode
from redbaron import YieldAtomNode
from redbaron import BreakNode
from redbaron import GetitemNode, SliceNode
from redbaron import ImportNode, FromImportNode
from redbaron import DottedAsNameNode
from redbaron import NameAsNameNode
from redbaron import LambdaNode
from redbaron import WithNode

from pyccel.ast import NativeInteger, NativeFloat, NativeDouble, NativeComplex
from pyccel.ast import NativeBool
from pyccel.ast import NativeRange
from pyccel.ast import NativeIntegerList
from pyccel.ast import NativeFloatList
from pyccel.ast import NativeDoubleList
from pyccel.ast import NativeComplexList
from pyccel.ast import NativeList
from pyccel.ast import NativeSymbol
from pyccel.ast import String
from pyccel.ast import datatype, DataTypeFactory
from pyccel.ast import Nil
from pyccel.ast import Variable
from pyccel.ast import DottedName, DottedVariable
from pyccel.ast import Assign, AliasAssign, SymbolicAssign, AugAssign ,Assigns
from pyccel.ast import Return
from pyccel.ast import Pass
from pyccel.ast import FunctionCall, MethodCall, ConstructorCall
from pyccel.ast import FunctionDef, Interface, PythonFunction, SympyFunction
from pyccel.ast import ClassDef
from pyccel.ast import GetDefaultFunctionArg
from pyccel.ast import For
from pyccel.ast import If
from pyccel.ast import While
from pyccel.ast import Print
from pyccel.ast import SymbolicPrint
from pyccel.ast import Del
from pyccel.ast import Assert
from pyccel.ast import Comment, EmptyLine, NewLine
from pyccel.ast import Break
from pyccel.ast import Slice, IndexedVariable, IndexedElement
from pyccel.ast import FunctionHeader, ClassHeader, MethodHeader
from pyccel.ast import VariableHeader, InterfaceHeader
from pyccel.ast import MetaVariable
from pyccel.ast import Concatinate
from pyccel.ast import ValuedVariable
from pyccel.ast import Argument, ValuedArgument
from pyccel.ast import Is
from pyccel.ast import Import, TupleImport
from pyccel.ast import AsName
from pyccel.ast import AnnotatedComment
from pyccel.ast import With
from pyccel.ast import Range
from pyccel.ast import List
from pyccel.ast import builtin_function as pyccel_builtin_function
from pyccel.ast import builtin_import as pyccel_builtin_import
from pyccel.ast import builtin_import_registery as pyccel_builtin_import_registery

from pyccel.parser.utilities import omp_statement, acc_statement
from pyccel.parser.utilities import fst_move_directives
from pyccel.parser.utilities import is_valid_filename_pyh, is_valid_filename_py
from pyccel.parser.utilities import read_file

from pyccel.parser.syntax.headers import parse as hdr_parse
from pyccel.parser.syntax.openmp import parse as omp_parse
from pyccel.parser.syntax.openacc import parse as acc_parse

from pyccel.parser.errors import (Errors, PyccelSyntaxError,
                                  PyccelSemanticError)

# TODO - remove import * and only import what we need
#      - use OrderedDict whenever it is possible
from pyccel.parser.messages import *


from sympy import Symbol, sympify
from sympy import Tuple
from sympy import NumberSymbol, Number
from sympy import Integer, Float
from sympy import Add, Mul, Pow, floor, Mod
from sympy import FunctionClass
from sympy import Lambda
from sympy.core.expr import Expr
from sympy.core.relational import Eq, Ne, Lt, Le, Gt, Ge
from sympy.core.containers import Dict
from sympy.core.function import Function, FunctionClass
from sympy.logic.boolalg import And, Or
from sympy.logic.boolalg import true, false
from sympy.logic.boolalg import Not
from sympy.logic.boolalg import Boolean, BooleanTrue, BooleanFalse
from sympy.tensor import Idx, Indexed, IndexedBase
from sympy.utilities.iterables import iterable

from collections import OrderedDict

import traceback
import importlib
import os
import sys


# Useful for very coarse version differentiation.
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3


#  ... useful functions for imports
# TODO installed modules. must ask python (working version) where the module is
#      installed
def get_filename_from_import(module):
    """Returns a valid filename with absolute path, that corresponds to the
    definition of module.
    The priority order is:
        - header files (extension == pyh)
        - python files (extension == py)
    """
    filename_pyh = '{}.pyh'.format(module)
    filename_py  = '{}.py'.format(module)

    if is_valid_filename_pyh(filename_pyh): return os.path.abspath(filename_pyh)
    if is_valid_filename_py(filename_py): return os.path.abspath(filename_py)

    source = module
    if len(module.split('.')) > 1:
        # we remove the last entry, since it can be a pyh file
        source = '.'.join(i for i in module.split('.')[:-1])
        _module = module.split('.')[-1]
        filename_pyh = '{}.pyh'.format(_module)
        filename_py  = '{}.py'.format(_module)

    try:
        package = importlib.import_module(source)
        package_dir = str(package.__path__[0])
    except:
        errors = Errors()
        errors.report(PYCCEL_UNFOUND_IMPORTED_MODULE,
                      symbol=source,
                      severity='fatal')

    filename_pyh = os.path.join(package_dir, filename_pyh)
    filename_py  = os.path.join(package_dir, filename_py)

    if os.path.isfile(filename_pyh):
        return filename_pyh
    elif os.path.isfile(filename_py):
        return filename_py

    errors = Errors()
    errors.report(PYCCEL_UNFOUND_IMPORTED_MODULE,
                  symbol=module,
                  severity='fatal')
#  ...

#  ...
def _get_variable_name(var):
    """."""

    if isinstance(var, (Variable, IndexedVariable)):
        return str(var)
    elif isinstance(var, IndexedElement):
        return str(var.base)

    raise NotImplementedError('Uncovered type {dtype}'.format(dtype=type(var)))
#  ...


class Parser(object):

    """ Class for a Parser."""

    def __init__(self, inputs, debug=False, headers=None, show_traceback=True):
        """Parser constructor.

        inputs: str
            filename or code to parse as a string

        debug: bool
            True if in debug mode.

        headers: list, tuple
            list of headers to append to the namespace

        show_traceback: bool
            prints Tracebacke exception if True

        """
        self._fst = None
        self._ast = None
        self._filename = None
        self._metavars = {}
        self._namespace = {}
        self._namespace['imports'] = OrderedDict()
        self._namespace['variables'] = {}
        self._namespace['classes'] = {}
        self._namespace['functions'] = {}
        self._namespace['cls_constructs'] = {}
        self._namespace['symbolic_functions'] = {}
        self._namespace['python_functions'] = {}
        self._scope = {}  # represent the namespace of a function
        self._current_class = None
        self._current = None  # we use it to detect the current method or function
        # we use it to store the imports
        self._imports = {}
        # a Parser can have parents, who are importing it. imports are then its sons.
        self._parents = []
        self._sons = []
        self._d_parsers = OrderedDict()
        # the following flags give us a status on the parsing stage
        self._syntax_done = False
        self._semantic_done = False
        # current position for errors
        self._bounding_box = None
        # flag for blocking errors. if True, an error with this flag will cause
        # Pyccel to stop
        # TODO ERROR must be passed to the Parser __init__ as argument
        self._blocking = False
        # printing exception
        self._show_traceback = show_traceback

        # TODO use another name for headers
        #      => reserved keyword, or use __

        self._namespace['headers'] = {}
        if headers:
            if not isinstance(headers, dict):
                raise TypeError('Expecting a dict of headers')

            for (key, value) in list(headers.items()):
                self._namespace['headers'][key] = value

        # check if inputs is a file
        code = inputs
        if os.path.isfile(inputs):
            # we don't use is_valid_filename_py since it uses absolute path
            # file extension
            ext = inputs.split('.')[-1]
            if not(ext in ['py', 'pyh']):
                errors = Errors()
                errors.report(INVALID_FILE_EXTENSION,
                              symbol=ext,
                              severity='fatal')
                errors.check()
                raise SystemExit(0)

            code = read_file(inputs)
            self._filename = inputs

        self._code = code

        try:
            red = RedBaron(code)
        except Exception as e:
            errors = Errors()
            errors.report(INVALID_PYTHON_SYNTAX,
                          symbol='\n'+str(e),
                          severity='fatal')
            errors.check()
            raise SystemExit(0)

        red = fst_move_directives(red)
        self._fst = red

    @property
    def namespace(self):
        return self._namespace

    @property
    def headers(self):
        return self.namespace['headers']

    @property
    def imports(self):
        return self.namespace['imports']

    @property
    def filename(self):
        return self._filename

    @property
    def code(self):
        return self._code

    @property
    def fst(self):
        return self._fst

    @property
    def ast(self):
        if self._ast is None:
            self._ast = self.parse()
        return self._ast

    @property
    def metavars(self):
        return self._metavars

    @property
    def current_class(self):
        return self._current_class

    @property
    def syntax_done(self):
        return self._syntax_done

    @property
    def semantic_done(self):
        return self._semantic_done

    @property
    def parents(self):
        """Returns the parents parser."""
        return self._parents

    @property
    def sons(self):
        """Returns the sons parser."""
        return self._sons

    @property
    def d_parsers(self):
        """Returns the d_parsers parser."""
        return self._d_parsers

    @property
    def is_header_file(self):
        """Returns True if we are treating a header file."""
        if self.filename:
            return self.filename.split('.')[-1] == 'pyh'
        else:
            return False

    @property
    def bounding_box(self):
        return self._bounding_box

    @property
    def blocking(self):
        return self._blocking

    @property
    def show_traceback(self):
        return self._show_traceback

    def append_parent(self, parent):
        """."""
        # TODO check parent is not in parents
        self._parents.append(parent)

    def append_son(self, son):
        """."""
        # TODO check son is not in sons
        self._sons.append(son)

    def parse(self, d_parsers=None, verbose=False):
        """converts redbaron fst to sympy ast."""
        if self.syntax_done:
            return self.ast

        # TODO - add settings to Errors
        #      - filename

        errors = Errors()
        if self.filename:
            errors.set_target(self.filename, 'file')
        errors.set_parser_stage('syntax')

        # we add the try/except to allow the parser to find all possible errors
        try:
            ast = self._fst_to_ast(self.fst)
        except Exception as e:
            errors.check()
            if self.show_traceback:
                traceback.print_exc()
            raise SystemExit(0)

        self._ast = ast

        errors.check()
        self._syntax_done = True

        if d_parsers is None:
            d_parsers = OrderedDict()
        self._d_parsers = self._parse_sons(d_parsers, verbose=verbose)

        return ast

    def annotate(self, **settings):
        """."""
        if self.semantic_done:
            return self.ast

        # TODO - add settings to Errors
        #      - filename

        errors = Errors()
        if self.filename:
            errors.set_target(self.filename, 'file')
        errors.set_parser_stage('semantic')

        # we first treat all sons to get imports
        verbose = settings.pop('verbose', False)
        self._annotate_parents(verbose=verbose)

        # then we treat the current file
        ast = self.ast
        # we add the try/except to allow the parser to find all possible errors
        try:
            ast = self._annotate(ast, **settings)
        except Exception as e:
            errors.check()
            if self.show_traceback:
                traceback.print_exc()
            raise SystemExit(0)

        self._ast = ast

        # in the case of a header file, we need to convert all headers to
        # FunctionDef etc ...
        
        if self.is_header_file:
            target = []
            for parent in self.parents:
                for  key,item in parent.imports.items():
                      if get_filename_from_import(key)==self.filename:
                          target += item
            target = set(target)
            target = target.intersection(self.headers.keys())
#            print(target)
  
            for name in list(target):
                v = self.headers[name]
                if isinstance(v, FunctionHeader) and not isinstance(v, MethodHeader):
                    F = self.get_function(name)
                    if F is None:
                        interfaces = v.create_definition()
                        for F in interfaces:
                            self.insert_function(F)
            
                    else:
                        errors.report(IMPORTING_EXISTING_IDENTIFIED,
                                      symbol=name,
                                      blocker=True,
                                      severity='fatal')    
#        print('++++++++++++++')
#        print(errors.error_info_map)
        errors.check()
        self._semantic_done = True

        return ast

    def _parse_sons(self, d_parsers, verbose=False):
        """Recursive algorithm for syntax analysis on a given file and its
        dependencies.
        This function always terminates with an OrderedDict that contains parsers
        for all involved files.
        """
        treated = set(d_parsers.keys())
        imports = set(self.imports.keys())
        imports = imports.difference(treated)
        if not imports:
            return d_parsers

        for source in imports:
            if verbose:
                print('>>> treating :: {}'.format(source))

            # get the absolute path corresponding to source
            filename = get_filename_from_import(source)

            q = Parser(filename)
            q.parse(d_parsers=d_parsers)
            d_parsers[source] = q

        # link self to its sons
        imports = list(self.imports.keys())
        for source in imports:
            d_parsers[source].append_parent(self)
            self.append_son(d_parsers[source])

        return d_parsers

    def _annotate_parents(self, **settings):

        verbose = settings.pop('verbose', False)

        # ...
        def _update_from_son(p):
            # TODO - only import what is needed
            #      - use insert_variable etc
            for entry in ['variables', 'classes', 'functions',
                          'cls_constructs']:
                d_self = self._namespace[entry]
                d_son  = p.namespace[entry]
                for k,v in list(d_son.items()):
                    d_self[k] = v
        # ...

        # we first treat sons that have no imports
        for p in self.sons:
            if not p.sons:
                if verbose:
                    print('>>> treating :: {}'.format(p.filename))
                p.annotate(**settings)

        # finally we treat the remaining sons recursively
        for p in self.sons:
            if p.sons:
                if verbose:
                    print('>>> treating :: {}'.format(p.filename))
                p.annotate(**settings)


    def print_namespace(self):
        # TODO improve spacing
        print('------- namespace -------')
        for (k, v) in self.namespace.items():
            print('{var} \t :: \t {dtype}'.format(var=k, dtype=type(v)))
        print('-------------------------')

    def view_namespace(self, entry):
        # TODO improve
        try:
            from tabulate import tabulate

            table = []
            for (k, v) in self.namespace[entry].items():
                k_str = '{}'.format(k)
                if entry == 'imports':
                    if v is None:
                        v_str = '*'
                    else:
                        v_str = '{}'.format(v)
                elif entry == 'variables':
                    v_str = '{}'.format(type(v))
                else:
                    raise NotImplementedError('TODO')

                line = [k_str, v_str]
                table.append(line)
            headers = ['module', 'target']
#            txt = tabulate(table, headers, tablefmt="rst")
            txt = tabulate(table, tablefmt="rst")
            print(txt)

        except:
            print('------- namespace.{} -------'.format(entry))
            for (k, v) in self.namespace[entry].items():
                print('{var} \t :: \t {value}'.format(var=k, value=v))
            print('-------------------------')


    def dot(self, filename):
        """Exports sympy AST using graphviz then convert it to an image."""

        expr = self.ast
        graph_str = dotprint(expr)

        f = open(filename, 'w')
        f.write(graph_str)
        f.close()

        # name without path

        name = os.path.basename(filename)

        # name without extension

        name = os.path.splitext(name)[0]
        cmd = 'dot -Tps {name}.gv -o {name}.ps'.format(name=name)
        os.system(cmd)

    def insert_import(self, expr):
        """."""
        # TODO improve
        
        if not isinstance(expr, Import):
            raise TypeError('Expecting Import expression')

        # if source is not specified, imported things are treated as sources
        # TODO test if builtin import
        source = expr.source
        if source is None:
            for t in expr.target:
                name = str(t)
                self._namespace['imports'][name] = None
        else:
            source = str(source)
            if not(source in pyccel_builtin_import_registery):
                for t in expr.target:
                    name = [str(t)]
                    if not source in self._namespace['imports'].keys():
                        self._namespace['imports'][source] = []
                    self._namespace['imports'][source] += name



    def get_variable(self, name):
        """."""

        var = None
        if self.current_class:
            for i in self._current_class.attributes:
                if str(i.name) == name:
                    var = i
                    return var
        if self._current:
            if name in self._scope[self._current]['variables']:
                var = self._scope[self._current]['variables'][name]
        if isinstance(self._current, DottedName):
            if name in self._scope[self._current.name[0]]['variables']:
                var = self._scope[self._current.name[0]]['variables'
                        ][name]
        if name in self._namespace['variables']:
            var = self._namespace['variables'][name]
        return var

    def get_variables(self, source=None):
        if source == 'parent':
{'fraction_non_alphanumeric': 0.08688939814256423, 'fraction_numerical': 0.0009649017006392474, 'mean_word_length': 2.1525475285171103, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '82417', 'n_tokens_mistral': 11539, 'n_tokens_neox': 11136, 'n_words': 3413}