<jupyter_start><jupyter_text>How many time each of the components are chosen as the top component?<jupyter_code>cnt = collections.Counter([
    d_all.get_kth_pc_for_phe_by_index(index=x, k=1) + 1
    for x in range(d_all.d['n_phes'])
])
plt.bar(
    cnt.keys(),
    cnt.values()
)
cnt<jupyter_output><empty_output><jupyter_text>How similar the GREAT enrichments are across 100 PCs?<jupyter_code>def get_topk_terms(great_df, topk = 5):
    return list(great_df['# ID'])[:topk]
great_plot_dfs = [
    read_great_PC(x + 1, BFold=2, BPval=float('5e-7')) 
    for x in range(100)
]
great_raw_dfs = [
    read_great_PC(x + 1, BPval=float('5e-6')) 
    for x in range(100)
]
top1_plot = [
    get_topk_terms(great_df, topk = 1) 
    for great_df in great_plot_dfs
]
top5_plot = [
    get_topk_terms(great_df, topk = 5) 
    for great_df in great_plot_dfs
]
top1_raw = [
    get_topk_terms(great_df, topk = 1) 
    for great_df in great_raw_dfs
]
top5_raw = [
    get_topk_terms(great_df, topk = 5) 
    for great_df in great_raw_dfs
]<jupyter_output><empty_output><jupyter_text>let's count how many times do we get the same term (top 1)<jupyter_code>plt.hist(
    list(collections.Counter([
        x for x in itertools.chain.from_iterable(top1_plot)
    ]).values()), bins=range(1, 11)
)<jupyter_output><empty_output><jupyter_text>let's count how many times do we get the same term (top 5)<jupyter_code>plt.hist(
    list(collections.Counter([
        x for x in itertools.chain.from_iterable(top5_plot)
    ]).values()), bins=range(1, 31)
)
def compute_jaccard_index_mat(topk_terms_list):
    return np.array([
        [
            jaccard_index_onto_terms(x, y) 
            for y in topk_terms_list
        ] for x in topk_terms_list
    ])
        
def plot_jaccard_index_mat(mat, title, save=None):
    fig = plt.figure(figsize=(6,6))
    gs = gridspec.GridSpec(1, 1)
    fig_axs = [fig.add_subplot(sp) for sp in gs]
    ax=fig_axs[0]
    im = ax.imshow(mat)
    cbar = ax.figure.colorbar(im, ax=ax)
    cbar.ax.set_ylabel('Jaccard index (pairwise gene set similarity)', rotation=-90, va="bottom")
    ax.set_xlabel('DeGAs components')
    ax.set_ylabel('DeGAs components')
    ticks = np.arange(0, mat.shape[0], 20) + 19
    ticklabels = ['{}'.format(x + 1) for x in ticks]
    ax.set_xticks(ticks)
    ax.set_yticks(ticks)
    ax.set_xticklabels(ticklabels)
    ax.set_yticklabels(ticklabels)
    ax.set_title(title)
#     gs.tight_layout(fig, rect=[0, 0, 1, 1])  
    if(save is not None):
        save_dirname=os.path.dirname(save)
        if(not os.path.exists(save_dirname)):
            os.mkdir(save_dirname)
        for ext in ['png', 'pdf']:
            fig.savefig(
                '{}.{}'.format(save, ext), 
                bbox_inches="tight", 
                pad_inches=0.0
            )
    
plot_jaccard_index_mat(
    compute_jaccard_index_mat(top5_plot),
    'Similarity of the top enriched terms \nfor each DeGAs component',
    save=os.path.join(out_dir, 'Heatmap_top5terms_plot')
)
np.median(compute_jaccard_index_mat(top5_plot)[np.triu_indices(100, 1)])<jupyter_output><empty_output><jupyter_text>the followings are backup figures<jupyter_code>plot_jaccard_index_mat(
    compute_jaccard_index_mat(top1_plot),
    'Similarity of the top enriched terms \nfor each DeGAs component',
    save=os.path.join(out_dir, 'other_params', 'Heatmap_top1terms_plot')
)
plot_jaccard_index_mat(
    compute_jaccard_index_mat(top1_raw),
    'Similarity of the top enriched terms \nfor each DeGAs component',
    save=os.path.join(out_dir, 'other_params', 'Heatmap_top1terms_raw')
)
plot_jaccard_index_mat(
    compute_jaccard_index_mat(top5_raw),
    'Similarity of the top enriched terms \nfor each DeGAs component',
    save=os.path.join(out_dir, 'other_params', 'Heatmap_top1terms_raw')
)<jupyter_output>2019-01-30 17:31:15,158 matplotlib.backends.backend_pdf DEBUG    Assigning font /b'F1' = '/Users/yosuke/anaconda3/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf/Helvetica.ttf'
2019-01-30 17:31:15,201 matplotlib.backends.backend_pdf DEBUG    Embedding font /Users/yosuke/anaconda3/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf/Helvetica.ttf.
2019-01-30 17:31:15,203 matplotlib.backends.backend_pdf DEBUG    Writing TrueType font.
2019-01-30 17:31:15,338 matplotlib.backends.backend_pdf DEBUG    Assigning font /b'F1' = '/Users/yosuke/anaconda3/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf/Helvetica.ttf'
2019-01-30 17:31:15,390 matplotlib.backends.backend_pdf DEBUG    Embedding font /Users/yosuke/anaconda3/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf/Helvetica.ttf.
2019-01-30 17:31:15,392 matplotlib.backends.backend_pdf DEBUG    Writing TrueType font.
{'fraction_non_alphanumeric': 0.12166172106824925, 'fraction_numerical': 0.039423484527342095, 'mean_word_length': 3.988372093023256, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '140315', 'license': 'MIT', 'n_tokens_mistral': 2019, 'n_tokens_neox': 1826, 'n_words': 327, 'path': 'uk_biobank/DeGAs/notebook/GREAT_specificity.ipynb', 'repo_name': 'bschilder/public-resources', 'size': '4718'}