<jupyter_start><jupyter_text>Entropy - binomial distributionWe calculate an entropy per each L100 gene set. The entropy is calculated with probabilities based on the binomial distribution for each given L100 gene set, we calculate the binomial probability for each of the target gene sets (number of success = number of overlap genes; number of trials is the number of genes in the given L100 gene set; and probability is size of target gs set over total count of all genes in the target gene sets).<jupyter_code>import pandas as pd
import re
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
def getGS(fname):
    df = pd.read_csv(fname, delimiter='_', header=None)
    df['gs'] = df.loc[:,2].str.replace('^\d+\t\t','').str.split('\t')
    df.drop([0,1,2], inplace=True, axis=1)
    return df.copy()

df_assoc = getGS('../out/20.0216 feat/reg_univariate_rf/network/gs.txt') #get the reg univariate-rf results
df_infer = getGS('../out/20.0216 feat/reg_rf_boruta/network/gs.txt') #get the rf-boruta results

# L100 results
df_L100 = pd.read_csv('../19.1013 tight cluster/landmarks_n100_k100.csv')
df_assoc['gs_size'] = df_assoc.apply(lambda x: len(x[0]), axis=1)
df_infer['gs_size'] = df_infer.apply(lambda x: len(x[0]), axis=1)
df_infer.head()
df_assoc.head()
sns.distplot(df_assoc.gs_size)
sns.distplot(df_infer.gs_size)
# retrieve the genes L100
df_L100['gs'] = df_L100.genes.apply(lambda x: re.findall('(\w*)\.\.', x))
df_L100['gs'] = df_L100.apply(lambda x: re.findall('(\w*)\s', x['landmark'])+x['gs'], axis=1)
df_L100['gs_size'] = df_L100.apply(lambda x: len(x['gs']), axis=1)
df_L100.head()
sns.distplot(df_L100.gs_size)<jupyter_output><empty_output><jupyter_text>calculate binomial probability, and entropy<jupyter_code>from scipy.stats import binom

def calcOverlap(df_source, df_target):
    score = 1
    gs_target = df_target['gs']
    gs_target_sizes = df_target['gs_size']
    gs_source = df_source['gs']

    overlap_n = []
    overlap_prob = []
    for gs in gs_target:
        gs_op_n = len(set(gs).intersection(set(gs_source))) # number of genes overlap for given target gs
        overlap_n.append(gs_op_n)
        gs_op_prob = binom.pmf(gs_op_n, n=len(gs_source),p=len(gs)/sum(gs_target_sizes))
        overlap_prob.append(gs_op_prob)

    entropy = -np.sum(overlap_prob* np.log(overlap_prob))
    return entropy

entropy_assoc = df_L100.apply(calcOverlap, df_target=df_assoc, axis=1)
entropy_infer = df_L100.apply(calcOverlap, df_target=df_infer, axis=1)
df_entropy = pd.DataFrame({'assoc': entropy_assoc,
                        'infer': entropy_infer,})
df_entropy.describe()
sns.distplot(df_entropy.infer)
sns.distplot(df_entropy.assoc)

plt.legend(title='list',loc='upper right', labels=['infer entropy', 'assoc entropy'])
plt.xlabel('Entropy')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13872220261207202, 'fraction_numerical': 0.030356512530885987, 'mean_word_length': 5.081545064377682, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '417461', 'license': 'MIT', 'n_tokens_mistral': 1126, 'n_tokens_neox': 1047, 'n_words': 235, 'path': 'notebooks/03c-entropy-binom.ipynb', 'repo_name': 'pritchardlabatpsu/cga', 'size': '2833'}