<jupyter_start><jupyter_text>Transform the data, easier to plot<jupyter_code>def create_df(x, y):
    try:
        num_sites = len(x["Evidence Ratios"]["constrained"][0])
        print("# Number of sites (BUSTEDS-MH):", num_sites)
    except:
        return 1
    pass

    res = [x+1 for x in range(num_sites)]

    # BUSTEDS-MH
    bs = ["BUSTEDS-MH"] * num_sites
    data_tuples = list(zip(res, x["Evidence Ratios"]["constrained"][0], bs))
    # Site, ER_Value, Model
    df1 = pd.DataFrame(data_tuples, columns=['Site','Evidence Ratio', "Method"])

    # BUSTEDS
    bs = ["BUSTEDS"] * num_sites
    data_tuples = list(zip(res, y["Evidence Ratios"]["constrained"][0], bs))
    # Site, ER_Value, Model
    df3 = pd.DataFrame(data_tuples, columns=['Site','Evidence Ratio', "Method"])
    
    #result = pd.concat(df_list)
    result = pd.concat([df1, df3])

    return result


def create_df_log10(x, y):
    try:
        num_sites = len(x["Evidence Ratios"]["constrained"][0])
        print("# Number of sites (BUSTEDS-MH):", num_sites)
    except:
        return 1
    pass

    res = [x+1 for x in range(num_sites)]

    # BUSTEDS-MH
    bs = ["BUSTEDS-MH"] * num_sites
    data_tuples = list(zip(res, np.log10(x["Evidence Ratios"]["constrained"][0]), bs))
    
    # Site, ER_Value, Model
    df1 = pd.DataFrame(data_tuples, columns=['Site','Evidence Ratio', "Method"])

    # BUSTEDS
    bs = ["BUSTEDS"] * num_sites
    data_tuples = list(zip(res, np.log10(y["Evidence Ratios"]["constrained"][0]), bs))
    
    # Site, ER_Value, Model
    df3 = pd.DataFrame(data_tuples, columns=['Site','Evidence Ratio', "Method"])
    
    result = pd.concat([df1, df3])

    return result<jupyter_output><empty_output><jupyter_text>Pure ER's between MH, BUSTEDS, and BUSTEDS-MH  Visualize<jupyter_code>def plot(result, JSON_FILE_BUSTEDS_MH, title_text, outfile,clamp=False):
    source = result
    #title_text = os.path.basename(JSON_FILE_BUSTEDS_MH).replace(".BUSTEDS-MH.json", "")
    
    print("# Plotting:", title_text)
    
    if clamp == False:
        line = alt.Chart(source, title=title_text).mark_area().encode(
            x='Site',
            y='sum(Evidence Ratio)',
            color='Method'
        ).properties(
            width=1200,
            height=800,
            title=title_text).configure_axis(
        labelFontSize=18,
        titleFontSize=18).configure_title(fontSize=24)
        #line
        #chart.save('chart.png')
        #line.save(title_text + ".png")
        line.save(outfile)
        
    if clamp == True:
        line = alt.Chart(source, title=title_text).mark_area().encode(
            x='Site',
            y=alt.Y('sum(Evidence Ratio)', scale=alt.Scale(domain=(-2, 10), clamp=True)),
            color='Method'
        ).properties(
            width=1200,
            height=800,
            title=title_text).configure_axis(
        labelFontSize=18,
        titleFontSize=18).configure_title(fontSize=24)
        
        line.save(outfile)
        
#plot(result)<jupyter_output><empty_output><jupyter_text>Main<jupyter_code>def main(JSON_FILE_BUSTEDS_MH, JSON_FILE_BUSTEDS):
    
    title_text = os.path.basename(JSON_FILE_BUSTEDS_MH).replace(".BUSTEDS-MH.json", "")
    
    x = read_json(JSON_FILE_BUSTEDS_MH)
    y = read_json(JSON_FILE_BUSTEDS)
    
    result = create_df(x, y)
    result_log10 = create_df_log10(x, y)
    
    if type(result) != pd.core.frame.DataFrame:
        print("# No ER data")
        print()
        return # Error
    else:
        #print(result)
        outfile = title_text + ".png"
        plot(result, JSON_FILE_BUSTEDS_MH, title_text, outfile)
        outfile_log10 = title_text + "_log10.png"
        plot(result_log10, JSON_FILE_BUSTEDS_MH, title_text, outfile_log10)
        
        outfile_log10 = title_text + "_log10_clamp.png"
        plot(result_log10, JSON_FILE_BUSTEDS_MH, title_text, outfile_log10, True)
    #end if
    
#end method\

for BUSTEDSMH_File in BUSTEDS_MH_DIR_FILES:
    basename = os.path.basename(BUSTEDSMH_File).replace(".nex.BUSTEDS-MH.json", "")
    # Find BUSTEDS File
    BUSTEDS_File = os.path.join(BUSTEDS_DIR, basename + ".nex.BUSTEDS.json")
    print(BUSTEDSMH_File, BUSTEDS_File)
    
    main(BUSTEDSMH_File, BUSTEDS_File)
type(result)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13839701770736254, 'fraction_numerical': 0.01700838769804287, 'mean_word_length': 2.993488372093023, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '408654', 'license': 'MIT', 'n_tokens_mistral': 1654, 'n_tokens_neox': 1548, 'n_words': 261, 'path': 'notebooks/Analysis_SingleGene-ER_Plot.ipynb', 'repo_name': 'aglucaci/BUSTEDS-MH', 'size': '4292'}