<commit_before>import matplotlib
from mpl_toolkits.axes_grid1 import make_axes_locatable

import matplotlib.pylab as plt
matplotlib.style.use('ggplot')
#matplotlib.use('Agg')

import glob as glob
import camoco as co
import pandas as pd
import numpy as np
pd.set_option('display.width',300)

from ggplot import *

tableau20 = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120),    
             (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150),    
             (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148),    
             (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199),    
             (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)]    
  
# Scale the RGB values to the [0, 1] range, which is the format matplotlib accepts.    
for i in range(len(tableau20)):    
    r, g, b = tableau20[i]    
    tableau20[i] = (r / 255., g / 255., b / 255.)

class SimulationAnalysis(object):

    def __init__(self,dir='./',sep='\t'):
        self.sep = sep
        self.build_data(dir=dir)

    # Read in Raw data
    def build_data(self,dir='./'):
        self.df = pd.concat(
            [pd.read_table(x,sep=self.sep) \
                for x in glob.glob(dir+'*GWASSim.csv')]
        )
        if 'MCR' not in self.df.columns:
            self.df['MCR'] = 0
        if 'FCR' not in self.df.columns:
            self.df['FCR'] = 0
        self.df = pd.pivot_table(
            self.df,
            index=['COB','GO','WindowSize','FlankLimit','MCR','FCR'],
            aggfunc=np.mean
        )
        self.df = self.df.reset_index()
        # Add an id column
        self.df['id'] = self.df.COB+'/'+self.df.GO
        self.df['window_id'] = ['{}/{}'.format(x,y) for x,y in zip(self.df.FlankLimit,self.df.WindowSize)]
    
    def terms_with_signal(self, max_pval=0.05, min_pval=0,
			  pval_col='Density_pval'):
        '''
            Return only terms with a starting pvalue between the 
            specified argument.
        '''
        # Split the data in a few informative ways
        # We only want the True FCR
        fdr = self.df.set_index(['COB','GO','FCR'],drop=False).sort_index()
        # Split out groups
        terms_with_signal = fdr.query(
            'FCR==0 and MCR==0 and FlankLimit==0 and {}<{} and {}>={}'.format(pval_col,max_pval,pval_col,min_pval)
        )
        # Only return terms that were significant at FCR of 0 
        return pd.concat(
            [fdr.loc[x,y,:] \
                for x,y in zip(
                    terms_with_signal.COB,
                    terms_with_signal.GO
                )
            ]
        )

    def plot_signal_vs_noise(self,filename=None, figsize=(16,8), alpha=0.05,
                             figaxes=None, label=None, noise_source='FCR',
                             pval_col='Density_pval', max_pval=0.05, 
                             min_pval=0, normalize_num_sig=False):
        '''
            Plot the number of terms significant at varying FCR/MCR levels.

            THIS IS FOR WHEN YOU HAVE THE SAME TERMS AND YOU VARY THEIR
            FCR/MCR OR NOISE LEVEL!!!
        '''
        df = self.terms_with_signal(max_pval=max_pval,min_pval=min_pval)
        # Aggregate by COB and by noise source and calculate the 
        # number of GO terms that have a sig pval below alpha
        breakdown = df.reset_index(drop=True)\
            .groupby(['COB',noise_source])\
            .apply(lambda x: sum(x[pval_col] <= alpha))\
            .reset_index()\
            .rename(columns={0:'num_sig'})
        # Get the number of unique cobs in the dataset
        cobs = breakdown.COB.unique()
        if figaxes == None:
            fig, axes = plt.subplots(1,len(cobs),figsize=figsize,sharey=True)
        else:
            fig,axes = figaxes
            if len(cobs) != len(axes):
                raise ValueError('Cannot overlay plots with different number of COBs')<commit_msg>Add option to plot sig vs noise stratified by term size<commit_after>import matplotlib
from mpl_toolkits.axes_grid1 import make_axes_locatable

import matplotlib.pylab as plt
matplotlib.style.use('ggplot')
#matplotlib.use('Agg')

import glob as glob
import camoco as co
import pandas as pd
import numpy as np
pd.set_option('display.width',300)

from ggplot import *

tableau20 = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120),    
             (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150),    
             (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148),    
             (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199),    
             (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)]    
  
# Scale the RGB values to the [0, 1] range, which is the format matplotlib accepts.    
for i in range(len(tableau20)):    
    r, g, b = tableau20[i]    
    tableau20[i] = (r / 255., g / 255., b / 255.)

class SimulationAnalysis(object):

    def __init__(self,dir='./',sep='\t'):
        self.sep = sep
        self.build_data(dir=dir)

    # Read in Raw data
    def build_data(self,dir='./'):
        self.df = pd.concat(
            [pd.read_table(x,sep=self.sep) \
                for x in glob.glob(dir+'*GWASSim.csv')]
        ).fillna(0)
        for col in ['MCR','FCR','WindowSize','FlankLimit','COB','GO']:
            if col not in self.df.columns:
                self.df[col] = 0
        self.df = pd.pivot_table(
            self.df,
            index=['COB','GO','WindowSize','FlankLimit','MCR','FCR'],
            dropna=True,
            aggfunc=np.mean
        )
        self.df = self.df.reset_index()
        # Add an id column
        self.df['id'] = self.df.COB+'/'+self.df.GO
        self.df['window_id'] = ['{}/{}'.format(x,y) for x,y in zip(self.df.FlankLimit,self.df.WindowSize)]
    
    def terms_with_signal(self, max_pval=0.05, min_pval=0,
			  pval_col='Density_pval',min_term_size=0,max_term_size=10e10):
        '''
            Return only terms with a starting pvalue between the 
            specified argument.
        '''
        # Split the data in a few informative ways
        # We only want the True FCR
        fdr = self.df.set_index(['COB','GO','FCR'],drop=False).sort_index()
        # Split out groups
        terms_with_signal = fdr.query(
               'FCR==0 and MCR==0 and FlankLimit==0 '
               'and {}<{} and {}>={}'
               'and NumCandidates>={} and NumCandidates<={}'.format(
                pval_col,max_pval,
                pval_col,min_pval,
                min_term_size,max_term_size
            )
        )
        # Only return terms that were significant at FCR of 0 
        return pd.concat(
            [fdr.loc[x,y,:] \
                for x,y in zip(
                    terms_with_signal.COB,
                    terms_with_signal.GO
                )
            ]
        )

    def plot_signal_vs_noise(self,filename=None, figsize=(16,8), alpha=0.05,
                             figaxes=None, label=None, noise_source='FCR',
                             pval_col='Density_pval', max_pval=0.05, 
                             min_pval=0, normalize_num_sig=False,
                             min_term_size=0,max_term_size=10e10):
        '''
            Plot the number of terms significant at varying FCR/MCR levels.

            THIS IS FOR WHEN YOU HAVE THE SAME TERMS AND YOU VARY THEIR
            FCR/MCR OR NOISE LEVEL!!!
        '''
        df = self.terms_with_signal(
            max_pval=max_pval, min_pval=min_pval, 
            min_term_size=min_term_size, max_term_size=max_term_size
        )
        # Aggregate by COB and by noise source and calculate the 
        # number of GO terms that have a sig pval below alpha
        breakdown = df.reset_index(drop=True)\
            .groupby(['COB',noise_source])\
            .apply(lambda x: sum(x[pval_col] <= alpha))\
            .reset_index()\
            .rename(columns={0:'num_sig'}).copy()
        # Get the number of unique cobs in the dataset
        cobs = breakdown.COB.unique()
        if figaxes == None:
            fig, axes = plt.subplots(1,len(cobs),figsize=figsize,sharey=True)
        else:
            fig,axes = figaxes
            if len(cobs) != len(axes):
                raise ValueError('Cannot overlay plots with different number of COBs')
{'fraction_non_alphanumeric': 0.1312357937552339, 'fraction_numerical': 0.05263787534394066, 'mean_word_length': 1.9634881247784473, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5601822', 'n_tokens_mistral': 3169, 'n_tokens_neox': 2727, 'n_words': 703}