<jupyter_start><jupyter_text>Create Node Hierarchy<jupyter_code>root_path = '/Users/rljacobson/Google Drive/Development/Mathematica-IntelliJ-Plugin/src/de/halirutan/mathematica/lang/psi/api/'
lst = glob.glob(root_path + '**/*.java', recursive=True)
# Cut off root_path and extension.
def remove_ends(text, prefix, suffix):
    if prefix:
        if text.startswith(prefix):
            text = text[len(prefix):]
    if suffix:
        if text.endswith(suffix):
            text = text[:-len(suffix)]
    return text

lst = [remove_ends(t, root_path, '.java') for t in lst]

removals = ['MathematicaPsiFile', 
            'OperatorNameProvider', 
            'graph/UndirectedEdge', 
            'StringifiedSymbol']
for t in removals:
    lst.remove(t)
replacements = {'lists/MList': 'lists/List', 'string/MString': 'string/String'}

for i, name in enumerate(lst):
    if name in replacements:
        lst[i] = replacements[name]
dirs = set()
file_list = []
root_path = 'FoxySheep/AST/'

current_dir = root_path

for t in lst:
    parts = t.split('/')
    if len(parts) == 2:
        d, f = parts
        total_path = root_path + d
        if total_path != current_dir:
            dirs.add(d)
            file_list.append((d, total_path + '/__init__.py'))
            current_dir = total_path
        file_list.append((d, '{rp}{f}Node.py'.format(rp=root_path, f=t)))
    else:
        file_list.append((None, '{rp}{f}Node.py'.format(rp=root_path, f=t)))
class_template = """from AST import ASTNode


class {name}(ASTNode):
    pass

"""

AST_init_template = '\nfrom .{module} import {name}'

init_template = 'from .{name} import {name}\n'

AST_init_filename = 'FoxySheep/AST/__init__.py'

current_dir = None
current_py_init_file = AST_init_filename

with open(AST_init_filename, 'a') as init_file:
    current_py_init_file = None
    
    for (d, py_file_path) in file_list:
        # Get the class name without path.
        base_path, py_name = os.path.split(py_file_path)
        py_name = py_name[:-3]
        
        # Create the subdirectory if it does not exist.
        if current_dir != d:
            # Create the sub
            if not os.path.exists(base_path):
                os.makedirs(base_path)
                
            # Close previous init file if exists.
            if current_py_init_file and not current_py_init_file.closed:
                current_py_init_file.close()
                
            # Delete the current init file if exists.
            current_py_init_file_path = base_path + '/__init__.py'
            try:
                os.remove(current_py_init_file_path)
            except OSError:
                pass
            # Create new init file.
            current_py_init_file = open(current_py_init_file_path, 'w')
            current_dir = d
        
        # Create/overwrite the file.
        if py_name != '__init__':
            # Create a new python file
            with open(py_file_path, 'w') as py_file:
                py_file.write(class_template.format(name=py_name))
            # Append to init file.
            if current_py_init_file:
                current_py_init_file.write(init_template.format(name=py_name))
            # Also append to AST init file.
            if d is not None:
                module = '{d}.{name}'.format(d=d, name=py_name)
            else:
                module = py_name
            init_file.write(AST_init_template.format(module=module, name=py_name))
        
    # Close any open init files.
    if current_py_init_file and not current_py_init_file.closed:
        current_py_init_file.close()
        
    # Write a final newline at the end of the file.
    init_file.write('\n')
print('Done!')
glob.glob('./FoxySheep/AST/**/*.py', recursive=True)<jupyter_output><empty_output><jupyter_text>Write __init__ function<jupyter_code>txt="""identifier: str,
                 own_values: list = None,
                 down_values: list = None,
                 up_values: list = None,
                 default_values: list = None,
                 options: set = None,
                 attributes: set = None,
                 messages: set = None"""
lst = txt.split('\n')
lst = [t.strip() for t in lst]
lst = [t.split(' =')[0] for t in lst]
lst = [t.split(': ') for t in lst]
lst
init_txt = """self._{name} \t= {name}"""

for name, typename in lst:
    print(init_txt.format(name=name, typename=typename))<jupyter_output>self._identifier 	= identifier
self._own_values 	= own_values
self._down_values 	= down_values
self._up_values 	= up_values
self._default_values 	= default_values
self._options 	= options
self._attributes 	= attributes
self._messages 	= messages
{'fraction_non_alphanumeric': 0.11230175739391342, 'fraction_numerical': 0.0006429489927132448, 'mean_word_length': 2.3599712023038157, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '460236', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1492, 'n_words': 345, 'path': 'python_target/tools/CodeWriter.ipynb', 'repo_name': 'rljacobson/FoxySheep', 'size': '4666'}