<jupyter_start><jupyter_text>Explore ASTBill Ingram   October 5, 2021   This notebook generates an abstract syntax tree (AST) from a given Python program.<jupyter_code>import ast
# with open('execution_log.py', 'r') as source:
#     tree = ast.parse(source.read())
    
def parse_file(filename):
    with open(filename) as f:
        return ast.parse(f.read(), filename=filename)
tree = parse_file('test2.py')
dump = ast.dump(tree, indent=2)
print(dump)<jupyter_output>Module(
  body=[
    Assign(
      targets=[
        Name(id='n', ctx=Store())],
      value=Constant(value=3)),
    Assign(
      targets=[
        Name(id='arr', ctx=Store())],
      value=List(
        elts=[
          UnaryOp(
            op=USub(),
            operand=Constant(value=4)),
          Constant(value=3),
          Constant(value=2)],
        ctx=Load())),
    Assign(
      targets=[
        Name(id='i', ctx=Store())],
      value=Constant(value=1)),
    Assign(
      targets=[
        Name(id='z', ctx=Store())],
      value=Constant(value=0)),
    While(
      test=Compare(
        left=Name(id='i', ctx=Load()),
        ops=[
          LtE()],
        comparators=[
          Name(id='n', ctx=Load())]),
      body=[
        Assign(
          targets=[
            Name(id='x', ctx=Store())],
          value=Subscript(
            value=Name(id='arr', ctx=Load()),
            slice=BinOp(
              left=Name(id='i', ctx=Load()),
              op=Sub(),
              r[...]<jupyter_text>Find the assignment that is the closest to a given line number.    Credit: Finding definitions from a source file and a line number in Python. <NAME>.  Retrieved October 11, 2021 from [https://julien.danjou.info/finding-definitions-from-a-source-file-and-a-line-number-in-python/](https://julien.danjou.info/finding-definitions-from-a-source-file-and-a-line-number-in-python/)<jupyter_code>def filename_and_lineno_to_def(filename, lineno):
    candidate = None
    for item in ast.walk(parse_file(filename)):
#         if hasattr(item, 'lineno'):
#             print(item.lineno)
#             print(item)
#         if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Assign)):
        if hasattr(item, 'lineno'):
            if item.lineno == lineno:
                candidate = item
#             if item.lineno > lineno:
#                 # Ignore whatever is after our line
#                 continue
#             if candidate:
#                 distance = lineno - item.lineno
#                 if distance < (lineno - candidate.lineno):
#                     candidate = item
#             else:
#                 candidate = item

    if candidate:
        return candidate
stmt = filename_and_lineno_to_def('execution_log.py', 100) 
print(ast.dump(stmt, indent=2))<jupyter_output>Name(id='km', ctx=Load())<jupyter_text>**New Approach**<jupyter_code>def mid(x,y,z):
    m = z
    if y < z:
        if x < y:
            m = y
        elif x < z:
            m = x
    else:
        if x > y:
            m = y
        elif x > z:
            m = x
    print(m)
    return m

import inspect

tree = ast.parse(inspect.getsource(mid))
import showast

def show_ast(tree):
    showast.show_ast(tree)
    
show_ast(tree)
from types import FunctionType, FrameType, TracebackType
from typing import cast, Dict, Any, Tuple, Callable, Optional, Type

class StackInspector:
    """Provide functions to inspect the stack"""

    def caller_frame(self) -> FrameType:
        """Return the frame of the caller."""

        # Walk up the call tree until we leave the current class
        frame = cast(FrameType, inspect.currentframe())

        while self.our_frame(frame):
            frame = cast(FrameType, frame.f_back)

        return frame

    def our_frame(self, frame: FrameType) -> bool:
        """Return true if `frame` is in the current (inspecting) class."""
        return isinstance(frame.f_locals.get('self'), self.__class__)

class DataTracker(StackInspector):
    """Track data accesses during execution"""

    def __init__(self, log: bool = False) -> None:
        """Constructor. If `log` is set, turn on logging."""
        self.log = log
        
class DataTracker(DataTracker):
    def get(self, name: str, value: Any) -> Any:
        """Track getting `value` from `name`."""

        if self.log:
            caller_func, lineno = self.caller_location()
            print(f"{caller_func.__name__}:{lineno}: getting {name}")

        return value

from ast import NodeTransformer, NodeVisitor, Name, AST, \
    Module, Load, Store, \
    Attribute, With, withitem, keyword, Call, Expr, \
    Assign, AugAssign, AnnAssign, Assert

class TrackGetTransformer(NodeTransformer):
    def visit_Name(self, node: Name) -> AST:
        self.generic_visit(node)

        if is_internal(node.id):
            # Do not change built-in names and types
            return node

        if node.id == DATA_TRACKER:
            # Do not change own accesses
            return node

        if not isinstance(node.ctx, Load):
            # Only change loads (not stores, not deletions)
            return node

        new_node = make_get_data(node.id)
        ast.copy_location(new_node, node)
        return new_node
import typing 

def is_internal(id: str) -> bool:
    """Return True if `id` is a built-in function or type"""
    return (id in dir(__builtins__) or id in dir(typing))
print(dir(typing))
print(dir(__builtins__))
DATA_TRACKER = '_data'
def make_get_data(id: str, method: str = 'get') -> Call:
    return Call(func=Attribute(value=Name(id=DATA_TRACKER, ctx=Load()), 
                               attr=method, ctx=Load()),
                args=[ast.Str(s=id), Name(id=id, ctx=Load())],
                keywords=[])
show_ast(Module(body=[make_get_data("m")]))
print(ast.dump(ast.parse("_data.get('m', m)")))

TrackGetTransformer().visit(tree);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1308616895209074, 'fraction_numerical': 0.003893685457931268, 'mean_word_length': 2.3954022988505748, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 22, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '352242', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1853, 'n_tokens_neox': 1800, 'n_words': 460, 'path': 'explore_ast.ipynb', 'repo_name': 'waingram/python-program-slicing', 'size': '5914'}