<commit_before>"""
This module provides the generic functionality of tracing code by
modifying its AST. Eventually this will become a separate package.
This is similar to the standard library module bdb, while birdseye
itself would correspond to pdb.
Most of the work is in TreeTracerBase.
"""

from __future__ import print_function, division, absolute_import

from future import standard_library

standard_library.install_aliases()

import ast
import inspect
from collections import namedtuple
from copy import deepcopy
from functools import partial, update_wrapper
from itertools import takewhile
from typing import List, Dict, Any, Optional, NamedTuple, Tuple, Iterator, Callable, cast, Union
from types import FrameType, TracebackType, CodeType, FunctionType

from birdseye.utils import of_type, safe_next, PY3, Type, is_lambda, lru_cache, read_source_file, is_ipython_cell


class TracedFile(object):
    """
    An instance of this class corresponds to a single .py file.
    It contains some useful data in the following attributes:

    - filename: name of the source file
    - source: textual contents of the file
    - root: root of the original Abstract Syntax Tree (AST) of the source,
            where the nodes of this tree have an additional handy attribute:
            - parent: parent of the node, so this node is a child node of its parent
    - tracer: instance of TreeTracerBase
    - code: executable code object compiled from the modified AST
    """

    is_ipython_cell = False

    def __init__(self, tracer, source, filename, flags):
        # type: (TreeTracerBase, str, str, int) -> None
        # Here the source code is parsed, modified, and compiled
        root = ast.parse(source, filename)  # type: ast.Module

        self.nodes = []  # type: List[ast.AST]

        def set_basic_node_attributes():
            self.nodes = []  # type: List[ast.AST]
            for node in ast.walk(root):  # type: ast.AST
                for child in ast.iter_child_nodes(node):
                    child.parent = node
                node._tree_index = len(self.nodes)
                self.nodes.append(node)

            # Mark __future__ imports and anything before (i.e. module docstrings)
            # to be ignored by the AST transformer
            for i, stmt in enumerate(root.body):
                if isinstance(stmt, ast.ImportFrom) and stmt.module == "__future__":
                    for s in root.body[:i + 1]:
                        for node in ast.walk(s):
                            node._visit_ignore = True

        set_basic_node_attributes()

        new_root = tracer.parse_extra(root, source, filename)
        if new_root is not None:
            root = new_root
<commit_msg>Use future flags when parsing AST<commit_after>"""
This module provides the generic functionality of tracing code by
modifying its AST. Eventually this will become a separate package.
This is similar to the standard library module bdb, while birdseye
itself would correspond to pdb.
Most of the work is in TreeTracerBase.
"""

from __future__ import print_function, division, absolute_import

from future import standard_library

standard_library.install_aliases()

import ast
import inspect
from collections import namedtuple
from copy import deepcopy
from functools import partial, update_wrapper
from itertools import takewhile
from typing import List, Dict, Any, Optional, NamedTuple, Tuple, Iterator, Callable, cast, Union
from types import FrameType, TracebackType, CodeType, FunctionType

from birdseye.utils import of_type, safe_next, PY3, Type, is_lambda, lru_cache, read_source_file, is_ipython_cell


class TracedFile(object):
    """
    An instance of this class corresponds to a single .py file.
    It contains some useful data in the following attributes:

    - filename: name of the source file
    - source: textual contents of the file
    - root: root of the original Abstract Syntax Tree (AST) of the source,
            where the nodes of this tree have an additional handy attribute:
            - parent: parent of the node, so this node is a child node of its parent
    - tracer: instance of TreeTracerBase
    - code: executable code object compiled from the modified AST
    """

    is_ipython_cell = False

    def __init__(self, tracer, source, filename, flags):
        # type: (TreeTracerBase, str, str, int) -> None
        # Here the source code is parsed, modified, and compiled
        root = compile(source, filename, 'exec', ast.PyCF_ONLY_AST | flags, dont_inherit=True)  # type: ast.Module

        self.nodes = []  # type: List[ast.AST]

        def set_basic_node_attributes():
            self.nodes = []  # type: List[ast.AST]
            for node in ast.walk(root):  # type: ast.AST
                for child in ast.iter_child_nodes(node):
                    child.parent = node
                node._tree_index = len(self.nodes)
                self.nodes.append(node)

            # Mark __future__ imports and anything before (i.e. module docstrings)
            # to be ignored by the AST transformer
            for i, stmt in enumerate(root.body):
                if isinstance(stmt, ast.ImportFrom) and stmt.module == "__future__":
                    for s in root.body[:i + 1]:
                        for node in ast.walk(s):
                            node._visit_ignore = True

        set_basic_node_attributes()

        new_root = tracer.parse_extra(root, source, filename)
        if new_root is not None:
            root = new_root

{'fraction_non_alphanumeric': 0.06752353367125272, 'fraction_numerical': 0.000724112961622013, 'mean_word_length': 2.7006028131279303, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5729096', 'n_tokens_mistral': 1498, 'n_tokens_neox': 1445, 'n_words': 595}