<jupyter_start><jupyter_text>Graphvizlibrary with tutorial: https://pypi.org/project/graphviz/python doc: https://www.graphviz.org/pdf/gv.3python.pdfanother doc: https://github.com/xflr6/graphvizpaper: https://www.graphviz.org/pdf/dotguide.pdf<jupyter_code>from graphviz import Digraph
import re
import os
import json<jupyter_output><empty_output><jupyter_text>an example<jupyter_code>dot = Digraph('testing')
dot.node('sf__opportunity_products', 'sf__opportunity_products')
dot.edge('sf__opportunities', 'sf__opportunities_extended')
dot.edge('sf__opportunities_extended', 'sf__accounts', label='${sf__opportunities_extended.account_id} = ${sf__accounts.id}')
dot.edge('sf__accounts', 'sf__users', label='${sf__accounts.owner_id} = ${sf__users.id}')
print(dot.source)

dot.render('sf_model.gv', view=True)<jupyter_output><empty_output><jupyter_text>getting real<jupyter_code>with open('explores/etl_jobs/status.json', 'r') as f:
    model = json.load(f)
s = model['explore']
for n, l in enumerate(s): print(n,l)
num0 = 0
counter = num0+1
counter_list = []
while counter < len(s): 
    num0 = counter
    counter = num0 + 1
    num1 = next((num for num, line in enumerate(s[counter:]) if 'join:' in line), len(s))

    if num1 != len(s):
        counter += num1
        counter_list.append(counter)
    else:
        counter = len(s)
    
    
    
counter_list
grouped_explore = []
grouped_explore.append(s[:counter_list[0]])
for r in range(len(counter_list)-1):
    grouped_explore.append(s[counter_list[r]: counter_list[r+1]])
grouped_explore.append(s[counter_list[-1]:])
grouped_explore
grouped_explore[0]
explore_tree = dict()

explore_base = list(filter(None, grouped_explore[0][0].split(' ')))[1]
for line in grouped_explore[0][1:]:
    if 'from:' in line:
        explore_base = list(filter(None, line.split(' ')))[1]
explore_tree['explore_base'] = explore_base

joins = []
for clause in grouped_explore[1:]:
    join_name = list(filter(None, clause[0].split(' ')))[1]
    
    for line in clause[1:]:
        if 'from:' in line:
            join_name = list(filter(None, line.split(' ')))[1]
    joins.append(join_name)
explore_tree['joins'] = joins
explore_tree
def trace_tree(grouped_explore):
    explore_tree = dict()

    explore_tree['explore_base'] = trace_base(grouped_explore[0])

    joins = []
    for clause in grouped_explore[1:]:
        joins.append(trace_base(clause))

    explore_tree['joins'] = joins
    return explore_tree
explore_tree = trace_tree(grouped_explore)
def trace_base(clause):
    for line in clause[1:]:
        base = list(filter(None, clause[0].split(' ')))[1]
        if 'from:' in line:
            base = list(filter(None, line.split(' ')))[1]
    return base    
explore_tree<jupyter_output><empty_output><jupyter_text>testing main<jupyter_code>import re
import os
import json
import logging


def divider(explore_list):
    """
    This function returns a list of location number that points to the joins in the explore list.

    :param explore_list: the list of explore, each element is a line from the explore, with possible joins.

    :type explore_list: list

    :return: a list of integers
    """
    num0 = 0
    counter = num0+1
    counter_list = []
    while counter < len(explore_list): 
        num0 = counter
        counter = num0 + 1
        num1 = next((num for num, line in enumerate(explore_list[counter:]) if 'join:' in line), len(explore_list))

        if num1 != len(explore_list):
            counter += num1
            counter_list.append(counter)
        else:
            counter = len(explore_list)

    return counter_list


def parser(explore_list, loc_list):
    """
    This function parses a list of explore lines into a grouped structure of explore lines.

    :param explore_list: the list representing raw explore file.

    :type explore_list: list

    :param loc_list: the list of dividers, each divider is the number of join in the explore list

    :type loc_list: list

    :return: a grouped and nested list representing the explore structure with joins.
    """
    grouped_explore = []
    grouped_explore.append(explore_list[:loc_list[0]])
    for r in range(len(loc_list)-1):
        grouped_explore.append(explore_list[loc_list[r]: loc_list[r+1]])
    grouped_explore.append(explore_list[loc_list[-1]:])
    return grouped_explore


def trace_base(clause):
    """
    This function traces down a join or explore clause back to the base view name.

    :param clause: a list of one Lookml clause, that is either an explore level or join level.

    :type clause: list

    :return: the base view name. string type
    """
    if len(clause) > 1:
        for line in clause[1:]:
            base = list(filter(None, clause[0].split(' ')))[1]
            if 'from:' in line:
                base = list(filter(None, line.split(' ')))[1]
    else:
        base = list(filter(None, clause[0].split(' ')))[1]
    return base   


def trace_joins(grouped_explore):
    """
    This function generates a dictionary of the explore tree.

    :param grouped_explore: a list representing one explore, with each element being either the explore base view details, or the joined view details.

    :type grouped_explore: list

    :return: a list representing all the joined base view names.
    """
    joins = []
    for clause in grouped_explore:
        joins.append(trace_base(clause))

    return joins
with open(f'explores/salesforce/sf__cases.json', 'r') as f:
    model = json.load(f)

explore_name = list(filter(None, model['explore'][0].split(' ')))[1]
explore_list = model['explore']
# logging.info(model_name, explore_name)

# find the divider of explore level and join level clauses
loc_list = divider(explore_list)

if len(loc_list) > 1:
    # parse the raw list, generate a nested and well grouped list representing the explore and join structure
    grouped_explore = parser(explore_list, loc_list)
    # generate a list of all joined base view names
    explore_joins = trace_joins(grouped_explore)
else: 
    explore_joins = trace_joins([explore_list])

explore_dict = dict()
explore_dict['explore_name'] = explore_name
explore_dict['explore_joins'] = explore_joins
trace_joins(grouped_explore)
grouped_explore
joins = []
for clause in grouped_explore:
    joins.append(trace_base(clause))
grouped_explore[4]
for line in grouped_explore[4][1:]:

    base = list(filter(None, grouped_explore[4][0].split(' ')))[1]
    
    if 'from:' in line:
        print('yes')
        base = list(filter(None, line.split(' ')))[1]
        break
        
line = grouped_explore[4][1]
list(filter(None, line.split(' ')))[1]
'from:' in line
line<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1216439178041098, 'fraction_numerical': 0.010199490025498725, 'mean_word_length': 3.899338721528288, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '394604', 'license': 'MIT', 'n_tokens_mistral': 2278, 'n_tokens_neox': 2214, 'n_words': 570, 'path': 'src/notebooks/treeeeeeeeeeeee.ipynb', 'repo_name': 'vickykwan92/gazetteer', 'size': '6667'}