<commit_before>#!/usr/bin/env python3

import sys
import os
import copy

import numpy as np
import scipy as sp
import scipy.stats

from collections import OrderedDict

import camoco as co
import pandas as pd
import matplotlib.pylab as plt
import statsmodels.api as sm

lowess = sm.nonparametric.lowess
co.cf.logging.log_level = 'quiet'

def mean_confidence_interval(data):
    return np.mean(data), confidence_interval(data)

def confidence_interval(data, confidence=0.95):
    a = 1.0*np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * sp.stats.t._ppf((1+confidence)/2., n-1)
    return 1.96*se

class NearestDict(OrderedDict):
    '''
        This extension overrides the get item method 
        of dict where if a key does not exist, it returns
        the nearst key which does exist.
    '''
    def __getitem__(self,key):
        'Returns the nearest key which exists'
        return dict.__getitem__(self,min(self.keys(),key=lambda x: abs(x-key)))

def locality(args):
    # Initiate for args 
    # Generate output dirs

    if os.path.exists("{}_Locality.csv".format(args.out.replace('.csv',''))):
        print(
            "{}_Locality.csv exist! Skipping!".format(
                args.out.replace('.csv','')
            )
        )
        return

    # Grab the COB object
    cob = co.COB(args.cob)
    gwas = co.GWAS(args.gwas)

    # If all, grab a generater
    if 'all' in args.terms:
        terms = gwas.iter_terms()
    else:
        # Otherwise get the term out of the GWAS
        terms = [gwas[x] for x in args.terms] 

    term_localities = []
    # Add in text for axes
    for term in terms:
        #fig,(ax1,ax2,ax3) = plt.subplots(1,3,figsize=(24,8))
        # Change the relevent things in the permuted args 
        # Generate data using permuted arguments
        loc,bsloc,fdr = generate_data(cob,gwas,term,args) 
        # Add extra Columns 
        # Plot the data
        #plot_data(args,loc,bsloc,fdr,ax1)
        #plot_scatter(args,loc,bsloc,fdr,ax2)
        #plot_fdr(args,loc,bsloc,fdr,ax3)
        #plt.tight_layout()
        #plt.savefig("{}_{}.png".format(
        #    args.out,
        #    term.id
        #))
        #plt.close()
        # Keep track of this shit
        term_localities.append(loc)
        term_localities.append(bsloc)
        term_localities.append(fdr)
    # Add parameters to the data frame
    term_localities = pd.concat(term_localities)
    term_localities.insert(0,'COB',cob.name)
    term_localities.insert(0,'Ontology',gwas.name)
    term_localities.insert(0,'WindowSize',args.candidate_window_size)
    term_localities.insert(0,'FlankLimit',args.candidate_flank_limit)

    # Calculate global FDR and number of candidates discovered
    global_fdr = []
    for (gwas,cob,term),df in term_localities.groupby(['Ontology','COB','Term']):
        # Get Z score
        zscores = list(np.arange(0,8))
        for zscore in zscores:
            zdf = df[df.zscore >= zscore]
            if len(zdf) > 0:
                fdr_indices = [x.startswith('fdr') for x in zdf.iter_name]
                if sum(fdr_indices) > 0:
                    num_random = zdf.loc[
                        fdr_indices     
                    ].groupby('iter_name').apply(len).mean()
                else:
                    num_random = 0
                emp_indices = [x.startswith('emp') for x in zdf.iter_name]
                if sum(emp_indices) > 0:
                    num_real = len(zdf.loc[
                        emp_indices
                    ])
                if num_real != 0 and num_random != 0:
                    fdr = num_random/num_real
                else:
                    fdr = 1
                global_fdr.append([gwas,cob,term,zscore,num_random,num_real,fdr])
    global_fdr  = pd.DataFrame(
        global_fdr,
        columns=['Ontology','COB','Term','zscore','numRandom','numReal','FDR']
    )

    # Output the Locality Measures
    term_localities.to_csv(
        "{}_Locality.csv".format(args.out.replace('.csv',''))        
    )
    global_fdr.to_csv(
        "{}_Locality_FDR.csv".format(args.out.replace('.csv','')),
        index=None
    )
        
def generate_data(cob,gwas,term,args):
    '''
        Generates the data according to parameters in args
    '''
    if args.snp2gene == 'effective':
        loci = sorted(term.effective_loci(
            window_size=args.candidate_window_size<commit_msg>Cleaner output, fixed carry along bug<commit_after>#!/usr/bin/env python3

import sys
import os
import copy

import numpy as np
import scipy as sp
import scipy.stats

from collections import OrderedDict

import camoco as co
import pandas as pd
import matplotlib.pylab as plt
import statsmodels.api as sm

lowess = sm.nonparametric.lowess
co.cf.logging.log_level = 'quiet'

def mean_confidence_interval(data):
    return np.mean(data), confidence_interval(data)

def confidence_interval(data, confidence=0.95):
    a = 1.0*np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * sp.stats.t._ppf((1+confidence)/2., n-1)
    return 1.96*se

class NearestDict(OrderedDict):
    '''
        This extension overrides the get item method 
        of dict where if a key does not exist, it returns
        the nearst key which does exist.
    '''
    def __getitem__(self,key):
        'Returns the nearest key which exists'
        return dict.__getitem__(self,min(self.keys(),key=lambda x: abs(x-key)))

def locality(args):
    # Initiate for args 
    # Generate output dirs

    if os.path.dirname(args.out) != '':
        os.makedirs(os.path.dirname(args.out),exist_ok=True)
    if os.path.exists("{}_Locality.csv".format(args.out.replace('.csv',''))):
        print(
            "{}_Locality.csv exist! Skipping!".format(
                args.out.replace('.csv','')
            )
        )
        return

    # Grab the COB object
    cob = co.COB(args.cob)

    if args.sig_edge_zscore is not None:
        cob.set_sig_edge_zscore(args.sig_edge_zscore)
    gwas = co.GWAS(args.gwas)

    # If all, grab a generater
    if 'all' in args.terms:
        terms = gwas.iter_terms()
    else:
        # Otherwise get the term out of the GWAS
        terms = [gwas[x] for x in args.terms] 

    term_localities = []
    # Add in text for axes
    for term in terms:
        loc,bsloc,fdr = generate_data(cob,gwas,term,args) 
        if args.plot:
            fig,(ax1,ax2,ax3) = plt.subplots(1,3,figsize=(24,8))
            # Change the relevent things in the permuted args 
            # Generate data using permuted arguments
            # Add extra Columns 
            # Plot the data
            plot_data(args,loc,bsloc,fdr,ax1)
            plot_scatter(args,loc,bsloc,fdr,ax2)
            plot_fdr(args,loc,bsloc,fdr,ax3)
            plt.tight_layout()
            plt.savefig("{}_{}.png".format(
                args.out,
                term.id
            ))
            plt.close()
            # Keep track of this shit
        term_localities.append(loc)
        term_localities.append(bsloc)
        term_localities.append(fdr)
    # Add parameters to the data frame
    term_localities = pd.concat(term_localities)
    term_localities.insert(0,'COB',cob.name)
    term_localities.insert(0,'Ontology',gwas.name)
    term_localities.insert(0,'WindowSize',args.candidate_window_size)
    term_localities.insert(0,'FlankLimit',args.candidate_flank_limit)

    # Calculate global FDR and number of candidates discovered
    global_fdr = []
    for (gwas,cob,term),df in term_localities.groupby(['Ontology','COB','Term']):
        # Get Z score
        zscores = list(np.arange(0,8))
        for zscore in zscores:
            zdf = df[df.zscore >= zscore]
            if len(zdf) > 0:
                fdr_indices = [x.startswith('fdr') for x in zdf.iter_name]
                if sum(fdr_indices) > 0:
                    num_random = zdf.loc[
                        fdr_indices     
                    ].groupby('iter_name').apply(len).mean()
                else:
                    num_random = 0
                emp_indices = [x.startswith('emp') for x in zdf.iter_name]
                if sum(emp_indices) > 0:
                    num_real = len(zdf.loc[
                        emp_indices
                    ])
                else:
                    num_real = 0
                if num_real != 0 and num_random != 0:
                    fdr = num_random/num_real
                else:
                    fdr = 1
                global_fdr.append(
                    [
                        gwas, cob,
                        args.candidate_window_size,
                        args.candidate_flank_limit,
                        term, zscore, num_random, num_real, fdr
                    ]
                )
    global_fdr  = pd.DataFrame(
        global_fdr,
        columns=[
            'Ontology','COB','WindowSize','FlankLimit',
            'Term','zscore','numRandom','numReal','FDR'
        ]
    )

    # Output the Locality Measures
    term_localities[term_localities.iter_name=='emp'].to_csv(
        "{}_Locality.csv".format(args.out.replace('.csv',''))        
    )
    # Output the FDR 
    term_localities[term_localities.iter_name=='fdr'].to_csv(
        "{}_Locality_BS.csv".format(args.out.replace('.csv',''))        
    )
    global_fdr.to_csv(
        "{}_Locality_FDR.csv".format(args.out.replace('.csv','')),
        index=None
    )
        
def generate_data(cob,gwas,term,args):
    '''
        Generates the data according to parameters in args
    '''
    if args.snp2gene == 'effective':
        loci = sorted(term.effective_loci(
            window_size=args.candidate_window_size
{'fraction_non_alphanumeric': 0.1137347906784904, 'fraction_numerical': 0.007733553309960816, 'mean_word_length': 2.0366311834690043, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1186296', 'n_tokens_mistral': 3219, 'n_tokens_neox': 3086, 'n_words': 654}