<jupyter_start><jupyter_text>Load data and filter columns with too many empty values<jupyter_code>dfCore = pd.read_csv('https://raw.githubusercontent.com/meidelien/Metabolic-network-layout-using-biochemical-coordinates/main/data/Chemical_descriptors.csv', usecols=[2,5,6] + list(range(10,218)), index_col=0)
df1515 = pd.read_csv("https://raw.githubusercontent.com/meidelien/Metabolic-network-layout-using-biochemical-coordinates/main/data/Chemical_descriptors1515.csv", usecols=[2,5,6] + list(range (10,218)), index_col=0)
dfCheck = pd.read_csv("https://raw.githubusercontent.com/meidelien/Metabolic-network-layout-using-biochemical-coordinates/main/data/Chemical_descriptors1515.csv",usecols = [1,2] + list(range(5,217)), index_col = 1)


# usecols = [0,1] + range(3,217)
# remove all columns that have less than 90% of non-zero entries
dfCore = dfCore.loc[:,((dfCore != 0).sum() > dfCore.shape[0] * 0.9)]
df1515 = df1515.loc[:,((df1515 !=0).sum() >df1515.shape[0]*0.9 )]
dfCheck = dfCheck.loc[:,((dfCheck !=0).sum() >dfCheck.shape[0]*0.9 )]

dfCore.shape<jupyter_output><empty_output><jupyter_text>Remove NANs in df1515 with 0's<jupyter_code>import numpy as np

# df1515.replace(np.nan, 0) # fml

df1515["BCUT2D_MWHI"] = df1515["BCUT2D_MWHI"].replace(np.nan, 0)

df1515["BCUT2D_MWLOW"] = df1515["BCUT2D_MWLOW"].replace(np.nan, 0)

df1515["BCUT2D_CHGHI"] = df1515["BCUT2D_CHGHI"].replace(np.nan, 0)

df1515["BCUT2D_LOGPHI"] = df1515["BCUT2D_LOGPHI"].replace(np.nan, 0)

df1515["BCUT2D_LOGPLOW"] = df1515["BCUT2D_LOGPLOW"].replace(np.nan, 0)

df1515["BCUT2D_CHGLO"] = df1515["BCUT2D_CHGLO"].replace(np.nan, 0)

df1515["BCUT2D_MRHI"] = df1515["BCUT2D_MRHI"].replace(np.nan, 0)

df1515["BCUT2D_MRLOW"] = df1515["BCUT2D_MRLOW"].replace(np.nan, 0)

df1515["MaxPartialCharge"] = df1515["MaxPartialCharge"].replace(np.nan, 0)

df1515["MinPartialCharge"] = df1515["MinPartialCharge"].replace(np.nan, 0)

df1515["MaxAbsPartialCharge"] = df1515["MaxAbsPartialCharge"].replace(np.nan, 0)

df1515["MinAbsPartialCharge"] = df1515["MinAbsPartialCharge"].replace(np.nan, 0)
#df.sample(5)
#df1515.sample(5)<jupyter_output><empty_output><jupyter_text>Analyse the distribution of values for each descriptor<jupyter_code>import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Plot e.coli core metabolite feature distributions<jupyter_code>f, axs = plt.subplots(6, 8, figsize=(20, 12))
axs = axs.flatten()

for i in range(48):
    dfCore.iloc[:,i].plot.hist(ax=axs[i], bins=20)
    axs[i].set_xlabel(dfCore.columns[i])
    axs[i].set_ylabel('')
    
f.tight_layout()<jupyter_output><empty_output><jupyter_text>Plot e.coli iML1515 metabolite feature distributions<jupyter_code>f, axs = plt.subplots(6, 8, figsize=(29, 12))
axs = axs.flatten()
title = plt.title("iML1515", loc ="center")

for i in range(48):
    df1515.iloc[:,i].plot.hist(ax=axs[i], bins=20)
    axs[i].set_xlabel(df1515.columns[i])
    axs[i].set_ylabel('')
    
f.tight_layout()<jupyter_output><empty_output><jupyter_text>Estimate Spearman correlation between all pairs of descriptors<jupyter_code>from scipy.stats import spearmanr
correlation = [(d1, d2, spearmanr(dfCore.loc[:,d1], dfCore.loc[:,d2])[0]) for d1 in dfCore.columns for d2 in dfCore.columns]
correlation1515 = [(d1_1515, d2_1515, spearmanr(df1515.loc[:,d1_1515], df1515.loc[:,d2_1515])[0]) for d1_1515 in df1515.columns for d2_1515 in df1515.columns]

dfCore_2 = pd.DataFrame(correlation, columns=['d1', 'd2', 'value'])
df1515_2 = pd.DataFrame(correlation1515, columns =["d1_1515", "d2_1515","value1515"])

dfCore_3 = dfCore_2.pivot_table(index='d1', columns='d2', values='value')
df1515_3 = df1515_2.pivot_table(index ="d1_1515", columns = "d2_1515", values = "value1515")

import seaborn as sns
sns.clustermap(dfCore_3, vmin=-1, vmax=1, cmap='PiYG', figsize=(16,16))
sns.clustermap(df1515_3, vmin=-1, vmax=1, cmap='PiYG', figsize=(16,16))<jupyter_output><empty_output><jupyter_text>Do scatter plots for pairs of features with low correlation<jupyter_code>lowest = dfCore_2.query("abs(value) < 0.1 and d1 < d2")  # d1 < d2 is just to avoid plotting (a,b) and (b,a)
lowest_1515 = df1515_2.query("abs(value1515) < 0.1 and d1_1515 < d2_1515")  # d1 < d2 is just to avoid plotting (a,b) and (b,a)
f, axs = plt.subplots(6, 10, figsize=(20, 12))
axs = axs.flatten()

for i in range(60):
    dfCore.plot.scatter(lowest.iloc[i,0], lowest.iloc[i,1], ax=axs[i])
    axs[i].set_xticks([])
    axs[i].set_yticks([])
    
f.tight_layout()
f, axs = plt.subplots(6, 10, figsize=(20, 12))
axs = axs.flatten()

for i in range(60):
    df1515.plot.scatter(lowest_1515.iloc[i,0], lowest_1515.iloc[i,1], ax=axs[i])
    axs[i].set_xticks([])
    axs[i].set_yticks([])
    
f.tight_layout()
import pyvis as pv
from pyvis.network import Network
import networkx as nx
from tqdm.auto import tqdm
from matplotlib import cm, colors
from IPython.display import IFrame
net = Network(height ="750px", width ="100%", bgcolor="#222222", font_color= "white")
net.barnes_hut()

dfCorr = df1515_3.corr()<jupyter_output><empty_output><jupyter_text>Stuff<jupyter_code>from pyvis.network import Network
import pandas as pd

got_net = Network(height='750px', width='100%', bgcolor='#222222', font_color='white')

# set the physics layout of the network
got_net.barnes_hut()
got_data = pd.read_csv('https://www.macalester.edu/~abeverid/data/stormofswords.csv')

sources = got_data['Source']
targets = got_data['Target']
weights = got_data['Weight']

edge_data = zip(sources, targets, weights)

for e in edge_data:
    src = e[0]
    dst = e[1]
    w = e[2]

    got_net.add_node(src, src, title=src)
    got_net.add_node(dst, dst, title=dst)
    got_net.add_edge(src, dst, value=w)

neighbor_map = got_net.get_adj_list()

# add neighbor data to node hover data
for node in got_net.nodes:
    node['title'] += ' Neighbors:<br>' + '<br>'.join(neighbor_map[node['id']])
    node['value'] = len(neighbor_map[node['id']])

got_net.show('gameofthrones.html')<jupyter_output><empty_output><jupyter_text>Pyvis stuff<jupyter_code>from pyvis.network import Network
import pandas as pd

net2 = Network(height='750px', width='100%', bgcolor='#222222', font_color='white')

# set the physics layout of the network
net2.barnes_hut()
data = df1515_2

sources = data["d1_1515"]
targets = data['d2_1515']
weights = data['value1515']

edge_data = zip(sources, targets, weights)

for e in edge_data:
    src = e[0]
    dst = e[1]
    w = e[2]

    net2.add_node(src, src, title=src)
    net2.add_node(dst, dst, title=dst)
    net2.add_edge(src, dst, value=w)

neighbor_map = net2.get_adj_list()

# add neighbor data to node hover data
for node in net2.nodes:
    node['title'] += ' Neighbors:<br>' + '<br>'.join(neighbor_map[node['id']])
    node['value'] = len(neighbor_map[node['id']])

net2.show('Metabolic_network.html')
df1515.to_csv("1515_for_PCA.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.166520979020979, 'fraction_numerical': 0.07706876456876458, 'mean_word_length': 6.2876857749469215, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 44, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '525804', 'license': 'MIT', 'n_tokens_mistral': 3125, 'n_tokens_neox': 2852, 'n_words': 484, 'path': 'Archive/descriptors_analysis_OLD.ipynb', 'repo_name': 'meidelien/Metabolic-network-layout-using-biochemical-coordinates', 'size': '6864'}