<commit_before><commit_msg>Create scripts making downstream aggragation and stats easier.<commit_after>import camoco as co
import glob
import pandas as pd
import os

def read_FDR(glob_path):
    dfs = []
    for x in glob.glob(glob_path):
        df = pd.read_table(x,sep=',')
        net,gwas,win,flank,*junk = os.path.basename(x).replace('.','_').split('_')
        if 'WindowSize' not in df.columns:
            df.insert(0,'WindowSize',win)
        if 'NumFlank' not in df.columns:
            df.insert(0,'NumFlank',flank)
        dfs.append(df)
    df = pd.concat(dfs)
    # I guess we forgot this before
    df.insert(4,'TraitType','Element')
    df.loc[[x.startswith('Log') for x in df.Term],'TraitType'] = 'Log'
    df.loc[[x.startswith('PCA') for x in df.Term],'TraitType'] = 'PCA'
    df.loc[[x.startswith('Trans') for x in df.Term],'TraitType'] = 'Trans'
    return df


def groupedFDR(df,by='term'):
    def grouped_agg(x):
        return pd.DataFrame(
            {
                'Tot':  sum(x.numReal),
                'FDR10':sum(x[x.FDR<=0.1].numReal),
                'FDR35':sum(x[x.FDR<=0.35].numReal),
                'FDR50':sum(x[x.FDR<=.5].numReal)
            },index=[None]
        )
    if by == 'term':
        groups = ['Ontology','COB','WindowSize','NumFlank','TraitType','Term']
    elif by == 'trait':
        groups = ['Ontology','COB','WindowSize','NumFlank','TraitType']
    return df.reset_index().groupby(groups).apply(grouped_agg)
{'fraction_non_alphanumeric': 0.17687074829931973, 'fraction_numerical': 0.01020408163265306, 'mean_word_length': 2.7814910025706943, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3605249', 'n_tokens_mistral': 538, 'n_tokens_neox': 525, 'n_words': 96}