<commit_before>import numpy as np
import pandas as pd
import io, sys, collections
from tuba_seq.functions import LN_Mean_P_values, percentiles, LN_mean, inerts 

def LN_Mean_Summary(S, inerts=inerts, min_FWER=0.0001, correction='Bonferroni'):
    """Hypothesis test of increased growth for sgRNAs within the screen, assuming 
a Lognormal distribution of tumor sizes. 

This function is similar to tuba_seq.functions.LN_Mean_P_values, however it 
corrects for multiple-hypothesis testing, and generates a more descriptive table. 
This function uses the bootstrap sampling technique to estimate a null sampling 
distribution from the sgInert tumors sizes. 

Note: This hypothesis test was not used in Rogers et al. 2017. This test 
*assumes* that sgInert tumors represent a null sampling distribution and, 
therefore, cannot test whether any sgInert guide deviates in growth from the 
other inerts. 
    
This test tends to be more conservative in P-value estimates because it attempts
to account for the typical amount of off-target endonuclease activity insofar as 
it alters cell fitness. For example, sgNeo2 tumors tend to grow slightly slower 
than other inerts. By estimating the mean size of sgNeo2 tumors separate from 
the other inerts (and by generating a sampling distribution from all sgInerts 
separately), this unintended change in fitness is incorporated into the null 
sampling distribution, widening the null distribution and, thus, reducing the 
likelihood that target sgRNAs deviate significantly from the null distribution. 

This test (not the Nat Methods test) is recommended. However, if one of the 
`inerts` within the screen changes tumor growth inordinately, then you will not 
observe any statistically-significant growth effects. So if you are using a new 
sgInert, you may want to first excluded it from `inerts` to affirm that it 
minimally alters tumor growth. 
    
Parameters:
-----------

inerts : sgRNAs that are not expected to alter tumor sizes to be used as the 
    null sampling distribution (default: ['Neo1-3', 'NT1', 'NT3'])

min_FWER : Determines number of bootstrap samples to draw. Family-Wise Error Rate
    (FWER) estimates below min_FWER are not reliably--more bootstrapping samples 
    need to be drawn. The default resolution can take hours to run even on a 
    modern CPU architecture. See bootstrap.sample for details. (default: 0.0001). 

correction : Correction method for FWER (default: 'Bonferroni', other options: 
    'Sidak', 'Holm', 'Hochberg'. See tuba_seq.bootstrap for details. The # of
    hypotheses is estimated from the non-inert sgRNAs in the input array.
"""
   out = S.groupby(level='target').agg(LN_mean)
   from 
   min_pval = #####
   df = pd.concat({
        'LN Mean (absolute cell no.)':out, 
        'LN Mean (Relative to sgInerts)':out.groupby(level='target').transform(inert_normalize)},
        'One-Sided raw P-value': LN_mean_P_values(S, inert=inerts, min_pvalue=min_pvalue))



def best_power_law_fit(S, resolution=400, percentile_min=10, percentile_max=99, sigma_threshold=0.05):
    """Returns best-fitting powerlaw.Distribution object of the data.<commit_msg>added summary of LN_mean statistics<commit_after>import numpy as np
import pandas as pd
import io, sys, collections
from tuba_seq.functions import LN_Mean_P_values, percentiles, LN_mean, inerts, inert_normalize 

def LN_Mean_Summary(S, inerts=inerts, min_FWER=0.0001):
    """Hypothesis test of increased growth for sgRNAs within the screen, assuming 
a Lognormal distribution of tumor sizes. 

This function is similar to tuba_seq.functions.LN_Mean_P_values, however it 
corrects for multiple-hypothesis testing, and generates a more descriptive table. 
This function uses the bootstrap sampling technique to estimate a null sampling 
distribution from the sgInert tumors sizes. 

Note: This hypothesis test was not used in Rogers et al. 2017. This test 
*assumes* that sgInert tumors represent a null sampling distribution and, 
therefore, cannot test whether any sgInert guide deviates in growth from the 
other inerts. 
    
This test tends to be more conservative in P-value estimates because it attempts
to account for the typical amount of off-target endonuclease activity insofar as 
it alters cell fitness. For example, sgNeo2 tumors tend to grow slightly slower 
than other inerts. By estimating the mean size of sgNeo2 tumors separate from 
the other inerts (and by generating a sampling distribution from all sgInerts 
separately), this unintended change in fitness is incorporated into the null 
sampling distribution, widening the null distribution and, thus, reducing the 
likelihood that target sgRNAs deviate significantly from the null distribution. 

This test (not the Nat Methods test) is recommended. However, if one of the 
`inerts` within the screen changes tumor growth inordinately, then you will not 
observe any statistically-significant growth effects. So if you are using a new 
sgInert, you may want to first excluded it from `inerts` to affirm that it 
minimally alters tumor growth. 
    
Parameters:
-----------

inerts : sgRNAs that are not expected to alter tumor sizes to be used as the 
    null sampling distribution (default: ['Neo1', 'Neo2', 'Neo3', 'NT1', 'NT3'])

min_FWER : Determines number of bootstrap samples to draw. Family-Wise Error Rate
    (FWER) estimates below min_FWER are not reliably--more bootstrapping samples 
    need to be drawn. The default resolution can take hours to run even on a 
    modern CPU architecture. See bootstrap.sample for details. (default: 0.0001). 

correction : Correction method for FWER (default: 'Bonferroni', other options: 
    'Sidak', 'Holm', 'Hochberg'. See tuba_seq.bootstrap for details. The # of
    hypotheses is estimated from the non-inert sgRNAs in the input array.
"""
    out = S.groupby(level='target').agg(LN_mean)
    m = (~out.index.isin(inerts)).sum()
    min_pval = min_FWER/m
    df = pd.DataFrame({
        'LN Mean (absolute cell no.)':out, 
        'LN Mean (Relative to sgInerts)':out.groupby(level='target').transform(inert_normalize),
        'One-Sided raw P-value': LN_Mean_P_values(S, inert=inerts, min_pvalue=min_pval)})
    df['Two-Sided Bonferroni-Corrected P-Value'] = df['One-Sided raw P-value'].apply(lambda p: min(p, 1 - p)/(2*m))
    return df

def best_power_law_fit(S, resolution=400, percentile_min=10, percentile_max=99, sigma_threshold=0.05):
    """Returns best-fitting powerlaw.Distribution object of the data.
{'fraction_non_alphanumeric': 0.0688964241676942, 'fraction_numerical': 0.00971023427866831, 'mean_word_length': 4.376139188069594, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7614174', 'n_tokens_mistral': 1983, 'n_tokens_neox': 1778, 'n_words': 813}