<jupyter_start><jupyter_text>GoalUnderstand the independent variable that will be used for subsequent ML applications<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

import pubchempy as pcp


%reload_ext autoreload
%autoreload 2
%matplotlib inline

colnames = ['PUBCHEM_RESULT_TAG', 'PUBCHEM_SID', 'PUBCHEM_CID',
       'PUBCHEM_ACTIVITY_OUTCOME', 'PUBCHEM_ACTIVITY_SCORE',
       'PUBCHEM_ACTIVITY_URL', 'PUBCHEM_ASSAYDATA_COMMENT', 'Phenotype',
       'Potency', 'Efficacy', 'Analysis Comment', 'Curve_Description',
       'Fit_LogAC50', 'Fit_HillSlope', 'Fit_R2', 'Fit_InfiniteActivity',
       'Fit_ZeroActivity', 'Fit_CurveClass', 'Excluded_Points', 'Max_Response',
       'Activity at 0.0000368000 uM', 'Activity at 0.0001704127 uM',
       'Activity at 0.0003680000 uM', 'Activity at 0.0008534482 uM',
       'Activity at 0.00184 uM', 'Activity at 0.00427 uM',
       'Activity at 0.00920 uM', 'Activity at 0.021 uM',
       'Activity at 0.046 uM', 'Activity at 0.107 uM', 'Activity at 0.230 uM',
       'Activity at 0.534 uM', 'Activity at 1.150 uM', 'Activity at 2.666 uM',
       'Activity at 5.750 uM', 'Activity at 12.80 uM', 'Activity at 28.70 uM',
       'Compound QC']

data = pd.read_csv('../data/raw/csv/compounds_malaria_all.csv', skiprows=6, header=None, names=colnames)
pd.options.display.max_columns = 999
data.head(n=20)
# Get a subset of the data for quicker exploration
subset = data.sample(n=100000)
# Pubchem does some classification on it's own
# unfortunately, a lot of these are 'inconclusive'
sns.countplot(subset.PUBCHEM_ACTIVITY_OUTCOME)
# What are the numbers for the full dataset?
sns.countplot(data.PUBCHEM_ACTIVITY_OUTCOME)

# 200k inactive
# 25k active
# What does the potency distribution look like for the active compounds?
sns.distplot(subset[subset.PUBCHEM_ACTIVITY_OUTCOME == 'Active'].Fit_LogAC50)
sns.distplot(subset[subset.PUBCHEM_ACTIVITY_OUTCOME == 'Inconclusive'].Fit_LogAC50)

# Are they all defined?
active = subset[subset.PUBCHEM_ACTIVITY_OUTCOME == 'Active']
inconcl = subset[subset.PUBCHEM_ACTIVITY_OUTCOME == 'Inconclusive']
inactive = subset[subset.PUBCHEM_ACTIVITY_OUTCOME == 'Inactive']
# print the number of NANs / total 
print(np.count_nonzero(~np.isnan(active.Fit_LogAC50)), np.count_nonzero(np.isnan(active.Fit_LogAC50)))
print(np.count_nonzero(~np.isnan(inconcl.Fit_LogAC50)),  np.count_nonzero(np.isnan(inconcl.Fit_LogAC50)))
print(np.count_nonzero(~np.isnan(inactive.Fit_LogAC50)),  np.count_nonzero(np.isnan(inactive.Fit_LogAC50)))

# Un-logged
sns.distplot(10**subset[subset.PUBCHEM_ACTIVITY_OUTCOME == 'Active'].Fit_LogAC50)
sns.distplot(10**subset[subset.PUBCHEM_ACTIVITY_OUTCOME == 'Inconclusive'].Fit_LogAC50)
#What's the activity score?
sns.distplot(active.PUBCHEM_ACTIVITY_SCORE)
sns.distplot(inconcl.PUBCHEM_ACTIVITY_SCORE)
sns.distplot(active.PUBCHEM_ACTIVITY_SCORE)# What are potency and efficacy?
sns.distplot(subset.Potency.dropna())
ax = sns.distplot(subset.Efficacy.dropna())
ax = sns.distplot(subset.Efficacy.dropna()[subset.Efficacy < 1000])
ax.set(xlim=(0, 1000))
ax = sns.distplot(subset.Efficacy.dropna()[subset.Efficacy < 200])
ax.set(xlim=(0, 200))
ax = sns.scatterplot(subset.Fit_LogAC50, subset.Efficacy.dropna(), alpha=0.05)
ax.set(ylim=(0, 250))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1269554753309266, 'fraction_numerical': 0.05174488567990373, 'mean_word_length': 5.884057971014493, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '197516', 'license': 'MIT', 'n_tokens_mistral': 1420, 'n_tokens_neox': 1262, 'n_words': 252, 'path': 'notebooks/0.2-DateExploration_MalariaActivity.ipynb', 'repo_name': 'afederation/chem-dl', 'size': '3324'}