<jupyter_start><jupyter_text>single derivation; "expr local to global"<jupyter_code>this_deriv_df = df[df['derivation name']==list_of_derivs[0]]
def get_expr_tuples_for_df(this_deriv_df):
    list_of_expr_tuples = []
    for indx, ths_row in this_deriv_df.iterrows():
        row_as_dict = ths_row.to_dict()
        if row_as_dict['input type'] == 'expression':
            expr_tuple = (str(row_as_dict['in local ID']), row_as_dict['in global ID'])
            list_of_expr_tuples.append(expr_tuple)
        if row_as_dict['output type'] == 'expression':
            expr_tuple = (str(row_as_dict['out local ID']), row_as_dict['out global ID'])
            list_of_expr_tuples.append(expr_tuple)
    return list(set(list_of_expr_tuples))
list_of_expr_tuples = get_expr_tuples_for_df(this_deriv_df)
list_of_expr_tuples<jupyter_output><empty_output><jupyter_text>single derivation; "derivations"<jupyter_code>list_of_derivs[0]
this_deriv_df = df[df['derivation name']==list_of_derivs[0]]
def get_step_IDs(this_deriv_df):
    list_of_step_IDs = []
    for indx, ths_row in this_deriv_df.iterrows():
        row_as_dict = ths_row.to_dict()
        if row_as_dict['input type']=='infrule':
            list_of_step_IDs.append(row_as_dict['in local ID'])
        elif row_as_dict['output type']=='infrule':
            list_of_step_IDs.append(row_as_dict['out local ID'])

    return list(set(list_of_step_IDs))
list_of_step_IDs = get_step_IDs(this_deriv_df)
list_of_step_IDs
def extract_printable_steps(list_of_step_IDs, this_deriv_df):
    dict_of_steps = {}
    for step_ID in list_of_step_IDs:
        step_str=""
        #print('step_ID =', step_ID)
        list_of_dicts_for_this_step = []
        for indx, ths_row in this_deriv_df.iterrows():
            #print(ths_row.to_dict())
            row_as_dict = ths_row.to_dict()
            if row_as_dict['input type'] == 'infrule' and row_as_dict['in local ID']==step_ID:
                list_of_dicts_for_this_step.append(row_as_dict)
            if row_as_dict['output type'] == 'infrule' and row_as_dict['out local ID']==step_ID:
                list_of_dicts_for_this_step.append(row_as_dict)

    #    print(list_of_dicts_for_this_step)

        for sub_dict in list_of_dicts_for_this_step:
            if sub_dict['input type']=='infrule':
                inf_rule = sub_dict['in global ID']
            if sub_dict['output type']=='infrule':
                inf_rule = sub_dict['out global ID']
        step_str+=("     '"+str(step_ID)+"': {'inf rule': '"+inf_rule+"',\n")
        
        list_of_input_expr_local_ID = []
        for sub_dict in list_of_dicts_for_this_step:
            if sub_dict['input type']=='expression':
                list_of_input_expr_local_ID.append(sub_dict['in local ID'])
        list_of_input_expr_local_ID = list(set(list_of_input_expr_local_ID))
        # https://stackoverflow.com/questions/3590165/join-a-list-of-items-with-different-types-as-string-in-python
        list_of_input_expr_local_ID = ', '.join("'"+str(x)+"'" for x in list_of_input_expr_local_ID)
        step_str+=("                 'inputs':  ["+str(list_of_input_expr_local_ID)+"],\n" )
        
        list_of_input_expr_local_ID = []
        for sub_dict in list_of_dicts_for_this_step:
            if sub_dict['input type']=='feed':
                list_of_input_expr_local_ID.append(sub_dict['in local ID'])
        list_of_input_expr_local_ID = list(set(list_of_input_expr_local_ID))
        list_of_input_expr_local_ID = ', '.join("'"+str(x)+"'" for x in list_of_input_expr_local_ID)
        step_str+=("                 'feeds':   ["+str(list_of_input_expr_local_ID)+"],\n" )

        list_of_input_expr_local_ID = []
        for sub_dict in list_of_dicts_for_this_step:
            if sub_dict['output type']=='expression':
                list_of_input_expr_local_ID.append(sub_dict['out local ID'])
        list_of_input_expr_local_ID = list(set(list_of_input_expr_local_ID))
        list_of_input_expr_local_ID = ', '.join("'"+str(x)+"'" for x in list_of_input_expr_local_ID)
        step_str+=("                 'outputs': ["+list_of_input_expr_local_ID+"],\n" )

        step_str+=("                 'linear index': "+str(list_of_dicts_for_this_step[0]['linear index'])+"},")

        dict_of_steps[list_of_dicts_for_this_step[0]['linear index']] = step_str

    return dict_of_steps
dict_of_steps = extract_printable_steps(list_of_step_IDs, this_deriv_df)
list_of_linindx = list(dict_of_steps.keys())
list_of_linindx.sort()
for linindx in list_of_linindx:
    print(dict_of_steps[linindx])<jupyter_output><empty_output><jupyter_text>all the derivations<jupyter_code>for this_d in list_of_derivs:
    this_deriv_df = df[df['derivation name']==this_d]
    list_of_expr_tuples = get_expr_tuples_for_df(this_deriv_df)
    print('# '+this_d)
    for this_tup in list_of_expr_tuples:
        print("        '"+this_tup[0]+"': '"+this_tup[1]+"',")
for this_d in list_of_derivs:
    this_deriv_df = df[df['derivation name']==this_d]
    list_of_step_IDs = get_step_IDs(this_deriv_df)
    dict_of_steps = extract_printable_steps(list_of_step_IDs, this_deriv_df)
    list_of_linindx = list(dict_of_steps.keys())
    list_of_linindx.sort()
    print("  '"+this_d+"': {")
    for linindx in list_of_linindx:
        print(dict_of_steps[linindx])
    print("      },")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12231959724035055, 'fraction_numerical': 0.002610479209397725, 'mean_word_length': 3.7807486631016043, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '653372', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 2039, 'n_tokens_neox': 1979, 'n_words': 272, 'path': 'v7_pickle_web_interface/flask/convert_v3_csv_derivations_to_dict.ipynb', 'repo_name': 'carlosal1015/proofofconcept', 'size': '5363'}