<jupyter_start><jupyter_text>Distribution ComparisonsKS & Anderson-Darling tests<jupyter_code>onek_fp = "Biology/Data/KEGG_Samples/KEGG_fragments_1000cpds_occurances.csv"
onek_df = pd.read_csv(onek_fp)

twok_fp = "Biology/Data/KEGG_Samples/KEGG_fragments_2000cpds_occurances.csv"
twok_df = pd.read_csv(twok_fp)

threek_fp = "Biology/Data/KEGG_Samples/KEGG_fragments_3000cpds_occurances.csv"
threek_df = pd.read_csv(threek_fp)

fourk_fp = "Biology/Data/KEGG_Samples/KEGG_fragments_4000cpds_occurances.csv"
fourk_df = pd.read_csv(fourk_fp)

fivek_fp = "Biology/Data/KEGG_Samples/KEGG_fragments_5000cpds_occurances.csv"
fivek_df = pd.read_csv(fivek_fp)
#1k and 4k
print("KS")
print(stats.ks_2samp(onek_df["Occurances"], fourk_df["Occurances"]))
print("<NAME>")
print(stats.anderson_ksamp([onek_df["Occurances"].tolist(), fourk_df["Occurances"].tolist()]))
#3k and 4k
print("KS")
print(stats.ks_2samp(threek_df["Occurances"], fourk_df["Occurances"]))
print("<NAME>")
print(stats.anderson_ksamp([threek_df["Occurances"].tolist(), fourk_df["Occurances"].tolist()]))
#4k and 5k
print("KS")
print(stats.ks_2samp(fourk_df["Occurances"], fivek_df["Occurances"]))
print("<NAME>")
print(stats.anderson_ksamp([fourk_df["Occurances"].tolist(), fivek_df["Occurances"].tolist()]))<jupyter_output>KS
Ks_2sampResult(statistic=0.03057877608518339, pvalue=5.059552959783144e-83)
<NAME>
Anderson_ksampResult(statistic=389.37909519917815, critical_values=array([0.325, 1.226, 1.961, 2.718, 3.752]), significance_level=inf)<jupyter_text>Timeout Test comparisons<jupyter_code>#Compare variations within timeout parameters - using Welch's t-test & KS test on random combinations
t0_001_files = []
t0_01_files =[]
t0_1_files = []
t1_files = []
for f in os.listdir("Biology/Data/Tests/Timeout/"):
    if "0_001_" in f and f.endswith(".csv"):
        t0_001_files.append("Biology/Data/Tests/Timeout/" + f)
    elif "0_01_" in f and f.endswith(".csv"):
        t0_01_files.append("Biology/Data/Tests/Timeout/" + f)
    elif "0_1_" in f and f.endswith(".csv"):
        t0_1_files.append("Biology/Data/Tests/Timeout/" + f)
    elif "t1_" in f and f.endswith(".csv"):
        t1_files.append("Biology/Data/Tests/Timeout/" + f)
        
            
#print(t0_001_files)
print(t1_files)
#             df = pd.read_csv("Biology/Data/Tests/Timeout/" + f)
#             occurances = df["Occurances"].tolist()
#             print(len(occurances))
            
#             #x & y vals, all normalized to 1
#             yvals = sorted(list(map(int, occurances)), reverse=True)
#             yvals = minmax_scale(yvals) #Normalize yvals to 1
#             print(len(yvals))
#             xvals = np.linspace(0, 1, num=len(yvals))
#             print(len(xvals))
def load_occurances(fp):
    df = pd.read_csv(fp)
    return df["Occurances"].tolist()
#Get random combinations of files
for i in range(10):
    f1, f2 = random.sample(t0_01_files, 2)
    print(f1, f2)
    o1 = load_occurances(f1)
    o2 = load_occurances(f2)
    
    s, p = stats.ttest_ind(o1, o2, equal_var=False)
    ks_s, ks_p = stats.ks_2samp(o1, o1)
    print(s, p)
    print(ks_s, ks_p)
#Differences between avg distributions for all timeout parameters (especially between 0.001 and others)
f_001 = random.sample(t0_001_files, 1)
o_001 = load_occurances(f_001[0])

f_01 = random.sample(t0_01_files, 1)
o_01 = load_occurances(f_01[0])

f_1 = random.sample(t0_1_files, 1)
o_1 = load_occurances(f_1[0])

f1 = random.sample(t1_files, 1)
o1 = load_occurances(f1[0])
#KS between 0.001 and 1
print(stats.ks_2samp(o_001, o1))

#Between 0.001 and 0.1
print(stats.ks_2samp(o_001, o_1))

#Between 0.001 and 0.01
print(stats.ks_2samp(o_001, o_01))
#KS between 0.01 and 1
print(stats.ks_2samp(o_01, o1))

#Between 0.01 and 0.1
print(stats.ks_2samp(o_01, o_1))
#KS between 0.1 and 1
print(stats.ks_2samp(o_1, o1))<jupyter_output>Ks_2sampResult(statistic=0.04818362428297451, pvalue=2.7974592049033237e-33)
{'fraction_non_alphanumeric': 0.14602446483180428, 'fraction_numerical': 0.07619775739041794, 'mean_word_length': 4.832095096582466, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '138103', 'license': 'MIT', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1594, 'n_words': 256, 'path': 'Fragments/Stats_V2.ipynb', 'repo_name': 'jfmalloy1/ChemAsLanguage', 'size': '3964'}