<jupyter_start><jupyter_text>REMOVING NON-ACTIVE NODES<jupyter_code># finding non-active pathways (according to std values)
pathway_std_less_than_one = df_activation_pred.groupby('cell_type').std().sum()[df_activation_pred.groupby('cell_type').std().sum()<1].index
# printing information about non-active nodes
print('The number of non-active nodes is ', len(pathway_std_less_than_one))
print('The non-active nodes are; ')
print(pathway_std_less_than_one)<jupyter_output>The number of non-active nodes is  49
The non-active nodes are; 
Index(['hsa04010', 'hsa04012', 'hsa04014', 'hsa04015', 'hsa04062', 'hsa04064',
       'hsa04071', 'hsa04072', 'hsa04110', 'hsa04150', 'hsa04151', 'hsa04210',
       'hsa04213', 'hsa04218', 'hsa04261', 'hsa04270', 'hsa04310', 'hsa04380',
       'hsa04510', 'hsa04530', 'hsa04540', 'hsa04610', 'hsa04611', 'hsa04621',
       'hsa04623', 'hsa04662', 'hsa04664', 'hsa04666', 'hsa04710', 'hsa04713',
       'hsa04722', 'hsa04724', 'hsa04727', 'hsa04728', 'hsa04750', 'hsa04810',
       'hsa04913', 'hsa04915', 'hsa04916', 'hsa04917', 'hsa04922', 'hsa04925',
       'hsa04960', 'hsa04962', 'hsa04970', 'hsa04972', 'hsa04973', 'hsa04976',
       'hsa05100'],
      dtype='object')<jupyter_text>EXPORTING ACTIVE NODES' ACTIVATION SCORE<jupyter_code># export activation scores of each pathways for each sample
df_active = df_activation_pred.iloc[:, ~df_activation_pred.columns.isin(pathway_std_less_than_one)]
df_active.to_csv(os.path.join(loc_output, 'activation_score.csv'), index=False)
df_active<jupyter_output><empty_output><jupyter_text>EXPORTING CELL TYPE - PATHWAY RELATION and CELL TYPE - FREQUENT GENE INFORMATION<jupyter_code>no_of_nodes_in_pathways=6
cell_path_info = [ohe.categories_[0][x] for x in np.argmax(model.layers[2].get_weights()[0], axis=-1)]
df_cell_path_info = pd.DataFrame()
df_gene_freq = pd.DataFrame()
for i_cell in df_raw['cell_type'].unique():
    print(i_cell)
    cell_path_no = [i for i, x in enumerate(cell_path_info) if x == i_cell]
    # realted pathways with cell types
    df_temp = pd.DataFrame(df_bio_filtered[df_bio_filtered.columns[cell_path_no]].sum(axis=0), columns=['total_gene']).reset_index()
    df_temp['cell_type'] = i_cell
    df_cell_path_info = df_cell_path_info.append(df_temp)
    # the most frequent gene
    gene_freq = df_bio_filtered[df_bio_filtered.columns[cell_path_no]].sum(axis=1)[df_bio_filtered[df_bio_filtered.columns[cell_path_no]].sum(axis=1)>no_of_nodes_in_pathways].index
    df_temp = pd.DataFrame(gene_freq)
    df_temp['cell_type'] = i_cell
    df_gene_freq = df_gene_freq.append(df_temp)
    
df_cell_path_info.to_csv(os.path.join(loc_output,'pathway_cell_type_relation_fig3g.csv'), index=False)
df_gene_freq.to_csv(os.path.join(loc_output,'gene_cell_type_relation_fig3g.csv'), index=False)<jupyter_output>NK
T cell
Monocyte
DC
B cell
Macrophage
Plasma B
Mast
ILC<jupyter_text>EXPORTING MOST FREQUENT GENE LIST FOR OVERALL DATASET<jupyter_code># frequent more than {len(cell type)-1} cell type
df_freq_for_cell_type = df_gene_freq['symbol'].value_counts()[df_gene_freq['symbol'].value_counts()>=len(df_raw['cell_type'].unique())-1]
# pd.DataFrame(df_freq_for_cell_type.index, columns=['gene']).to_csv(os.path.join(loc_output,'gene_most_freq_fig3g.csv'), index=False)
pd.DataFrame(df_gene_freq['symbol'].value_counts().index, columns=['gene']).to_csv(os.path.join(loc_output,'gene_most_freq_fig3g.csv'), index=False)
df_gene_freq['symbol'].value_counts().index<jupyter_output><empty_output><jupyter_text>TOP 10 ACTIVE PATHWAYS for EACH CELL TYPES<jupyter_code>top_active_pathway = 15
df_freq_pathway = pd.DataFrame()
for i in range(len(df_active)):
#     cell_type_info = df_active.iloc[i, -1]
    pathway_active = pd.DataFrame(pd.DataFrame(df_active.iloc[i, :-1][df_active.iloc[i, :-1]>0]).sort_values(by=i, ascending=False)[:top_active_pathway].index
                                  , columns=['pathways'])
    pathway_active['cell_type'] = df_active.iloc[i, -1]
    df_freq_pathway = df_freq_pathway.append(pathway_active)
df_freq_pathway_count = pd.DataFrame(df_freq_pathway.groupby(['pathways', 'cell_type']).size(), columns=['count']).reset_index()
# df_freq_pathway_count
df_freq_pathway_pivot = df_freq_pathway_count.pivot(index='pathways',columns='cell_type', values='count').fillna(0).reset_index()

top_n = 10
df_freq_pathway_top_n = pd.DataFrame()
for i_ct in df_freq_pathway['cell_type'].unique():
    df_temp = df_freq_pathway_pivot[['pathways',i_ct]].sort_values(by=i_ct,ascending=False)[:top_n][['pathways']]
    df_temp['cell_type']= i_ct
    df_freq_pathway_top_n = df_freq_pathway_top_n.append(df_temp)
    
df_freq_pathway_top_n.to_csv(os.path.join(loc_output,'freq_pathways_top'+str(top_n)+'_fig3g.csv'), index=False)
len(df_freq_pathway_top_n)<jupyter_output><empty_output><jupyter_text>PLOT DISTRIBUTION of PATHWAYS ACTIVITIES<jupyter_code>!pwd
import matplotlib.pyplot as plt
import seaborn as sns
# to visualize the distribution for each pathway, pd.melt operation applied into df_plot dataset
# visualization of the activities of all pathway according to cell_type
order_train = sorted(list(set(df_raw['cell_type'].values)))
plt.figure(figsize=(20,6))
sns.boxplot(x='variable', y='value', data=pd.melt(df_active, id_vars='cell_type'), hue='cell_type', hue_order=dict(zip(order_train, range(len(order_train)))))
plt.xticks(rotation=45);
plt.title('Distribution of activation of each cell type')
plt.xlabel('pathways')
plt.ylabel('activation score')
plt.tight_layout()
# exporting the graph
plt.savefig(os.path.join(loc_output, 'distribution_activities_fig3g.png'), dpi=300, bbox_inches = 'tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14237707817474354, 'fraction_numerical': 0.050229925716307036, 'mean_word_length': 7.516566265060241, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '446089', 'license': 'RSA-MD', 'n_tokens_mistral': 2408, 'n_tokens_neox': 2180, 'n_words': 325, 'path': 'notebooks/5.0-pg-pbk-analysis.ipynb', 'repo_name': 'pelingundogdu/signalization_prior_knowledge_based_nn', 'size': '5654'}