<jupyter_start><jupyter_text>DescriptionThis notebook is used to process and evalutate the initial universe of generated smiles, and then retain the network applying techniques and principles from both transfer learning and genetic algorithms to increasingly improve molecule generation for the specific task of binding with cornovirus protease.  First process initial generated smiles for PyRx analysis<jupyter_code>import pandas as pd
from rdkit import Chem, DataStructs
import random
import numpy as np
import rdkit.Chem.PropertyMol
gen0_table = pd.read_csv('./generations/gen0_smiles.smi',sep=',', header=None)
gen0 = list(gen0_table[0])[0:10000]
len(gen0)
def validate_mols(list_of_smiles):
    valid_mols = []
    for smi in list_of_smiles:
        mol = Chem.MolFromSmiles(smi)
        if mol is not None:
            valid_mols.append(mol)
    return valid_mols

def convert_mols_to_smiles(list_of_mols):
    valid_smiles = [Chem.MolToSmiles(mol) for mol in list_of_mols]
    return valid_smiles
gen0_mols = validate_mols(gen0)
len(gen0_mols)
'''Intakes a list of smiles, randomly shuffles them, then adds first thirty,
then sets a max-similarity threshold between any new molecule and existing list
and iteratively increases the treshold until X components are picked to ensure diveristy'''

def initialize_generation_from_mols(list_of_mols,desired_length):  
    assert desired_length >30
    random.shuffle(list_of_mols)
    random.shuffle(list_of_mols)
    
    #Prepare fingerprints for similarity calcs
    mol_fingerprints = []
    for mol in list_of_mols:
        mol_fingerprints.append(Chem.RDKFingerprint(mol))
    
    selected_mols = list_of_mols[0:30]
    selected_fingerprints = mol_fingerprints[0:30]
    remaining_mols = list_of_mols[30:]
    remaining_fingerprints = mol_fingerprints[30:]
    
    similarity_threshold = .05   
    while len(selected_mols) < desired_length:
        for fingerprint, mol in zip(remaining_fingerprints, remaining_mols):
            max_similarity = np.max(DataStructs.BulkTanimotoSimilarity(fingerprint,selected_fingerprints))
            if (max_similarity <= similarity_threshold) and (max_similarity < 1):
                selected_fingerprints.append(fingerprint)
                selected_mols.append(mol)
        print("Completed loop with threshold at: ", similarity_threshold, ". Length is currently: ", len(selected_mols))
        similarity_threshold += .05
    return selected_mols
gen0_mols = initialize_generation_from_mols(gen0_mols,1000)
print(len(gen0_mols))
master_table = pd.read_csv('./generations/master_results_table.csv',sep=',')
master_table.shape[0]
'''Certainly not opimized and not strictly necessary, but in the PyRx GUI
molecule names would sort oddly when in any numeric order, so ordering
molcules by a four letter code. This function iterates the four letter code.'''
def iterate_alpha(alpha_code):
    numbers = []
    for letter in alpha_code:
        number = ord(letter)
        numbers.append(number)
    
    if numbers[3]+1 > 90:
        if numbers[2]+1 > 90:
            if numbers[1]+1 > 90:
                if numbers[0]+1 > 90:
                    raise ValueError('Too long for alpha code')
                else:
                    numbers[3] = 65
                    numbers[2] = 65
                    numbers[1] = 65
                    numbers[0] = numbers[0] + 1
            else:
                numbers[3] = 65
                numbers[2] = 65
                numbers[1] = numbers[1] + 1
        else:
            numbers[3] = 65
            numbers[2] = numbers[2] + 1
    else:
        numbers[3] = numbers[3] + 1
    

    new_code = ""
    for number in numbers:
        new_code += chr(number)
    return new_code
iterate_alpha('AAAA')
def append_to_tracking_table(master_table,mols_to_append, source, generation):
    # Assign IDs for tracking to each mol, and assign a pandas table entry for each
    mols_to_export = []
    rows_list = []
    
    master_table_gen = master_table[master_table['gen'] == generation]
    if master_table_gen.shape[0] == 0:
        id_code = 'AAAA'
    else:
        master_table_gen_ids = master_table_gen.sort_values('id', ascending=True)
        master_table_gen_max_id = master_table_gen_ids.tail(1)
        key = master_table_gen_max_id['id'].keys()[0]
        id_code = iterate_alpha(str(master_table_gen_max_id['id'][key]))
        
    training_data = pd.read_csv('./datasets/all_smiles_clean.smi', header=None)
    training_set = set(list(training_data[0]))
    
    for mol in mols_to_append:
        pm = Chem.PropertyMol.PropertyMol(mol)
        title = 'id' + str(id_code) + 'gen'+ str(generation)
        print(title)
        # Enables for tracking which molecule is which in PyRx GUI and PyRx results export
        pm.SetProp('Title', title)
        mols_to_export.append(pm)

        #And track in pandas
        mol_dict = {}
        mol_dict['id'] = id_code
        mol_dict['gen'] = generation
        smile = Chem.MolToSmiles(mol)
        assert type(smile) == type('string')
        mol_dict['smile'] = smile

        if (source!= 'hiv' and source != 'manual' and source != 'baseline') and (smile in training_set):
            mol_dict['source'] = 'training'
        else:
            mol_dict['source'] = source
        mol_dict['score'] = 99.9

        rows_list.append(mol_dict)
        id_code = iterate_alpha(id_code)
        
    df = pd.DataFrame(rows_list)
    return df, mols_to_export
new_mols_to_test = append_to_tracking_table(master_table,gen0_mols, 'generated', 0)
mols_for_pd = new_mols_to_test[0]
mols_for_export = new_mols_to_test[1]
master_table = master_table.append(mols_for_pd)
len(mols_for_export)
master_table = master_table.reset_index(drop=True)
master_table.to_csv(r'./generations/master_results_table.csv', index=False)
# Add HIV inhibitors manually into the table
hiv_smiles = pd.read_csv('./datasets/hiv_inhibitors_cleaned.smi',sep=',', header=None)
hiv_smiles = list(hiv_smiles[0])
hiv_mols = validate_mols(hiv_smiles)

master_table = pd.read_csv('./generations/master_results_table.csv',sep=',')
new_mols_to_test = append_to_tracking_table(master_table,hiv_mols, 'hiv', 0)
mols_for_pd = new_mols_to_test[0]
mols_for_export = mols_for_export + new_mols_to_test[1]

master_table = master_table.append(mols_for_pd)
master_table = master_table.reset_index(drop=True)
master_table.to_csv(r'./generations/master_results_table.csv', index=False)
# Add a few other smiles manually into the table ("control group" of training smiles)
manual_smiles = pd.read_csv('./datasets/manual_testing_cleaned.smi',sep=',', header=None)
manual_smiles = list(manual_smiles[0])
manual_mols = validate_mols(hiv_smiles)

master_table = pd.read_csv('./generations/master_results_table.csv',sep=',')
new_mols_to_test = append_to_tracking_table(master_table,manual_mols, 'manual', 0)
mols_for_pd = new_mols_to_test[0]
mols_for_export = mols_for_export + new_mols_to_test[1]

master_table = master_table.append(mols_for_pd)
master_table = master_table.reset_index(drop=True)
master_table.to_csv(r'./generations/master_results_table.csv', index=False)
def write_gen_to_sdf(mols_for_export, generation, batch_size):
    if len(mols_for_export) > batch_size:
        batches = (len(mols_for_export) // 1000)+1
        for i in range(0,batches):
            batch_to_export = mols_for_export[i*batch_size:(i+1)*batch_size]
            w = Chem.SDWriter('./generations/gen' +str(generation) + '_batch_' + str(i+1) + '.sdf')
            for m in batch_to_export: w.write(m)
    else:
        w = Chem.SDWriter('./generations/gen' +str(generation) + '.sdf')
        for m in mols_for_export:
            w.write(m)
    
    # Noticed an issue where the very last line item of an sdf write is not written correctly until another arbitary write is made
    w = Chem.SDWriter('./generations/junk/test.sdf')
    w.write(m)
    
    return mols_for_export
write_gen_to_sdf(mols_for_export, 0, 2000)
print('ok')<jupyter_output><empty_output><jupyter_text>NOW GO TO PyRx: Analyze the SDF file and create a csv of binding score results  Afterwards, process binding simulation results to 'evolve' the molecules<jupyter_code>'''This number must be MANUALLY iterated each generation. I did not write the entire process into a smooth function or loop but that would be the next steps.''' 
GLOBAL_GENERATION = 11
master_table = pd.read_csv('./generations/master_results_table_gen' + str(GLOBAL_GENERATION-1) + '.csv',sep=',')
master_table.tail()
new_scores = pd.read_csv('./generations/results/results_gen' + str(GLOBAL_GENERATION-1) + '.csv',sep=',')
new_scores.head()
new_scores = new_scores.groupby("Ligand").min()["Binding Affinity"].reset_index()
new_scores['id'] = new_scores['Ligand'].str.split("_").str[1].str.split("gen").str[0].str.split("id").str[1]
new_scores['gen'] = new_scores['Ligand'].str.split("_").str[1].str.split("gen").str[1]
new_scores['score'] = new_scores["Binding Affinity"]
new_scores = new_scores[['id','gen','score']]
new_scores.head()
new_scores.id = new_scores.id.astype(str)
new_scores.gen = new_scores.gen.astype(int)
master_table.id = master_table.id.astype(str)
master_table.gen = master_table.gen.astype(int)
new_table = pd.merge(master_table, new_scores, on=['id','gen'], suffixes=('_old','_new'), how='left')
new_table['score'] = np.where(new_table['score_new'].isnull(), new_table['score_old'], new_table['score_new'])
new_table = new_table.drop(['score_old','score_new'], axis=1)
new_table['weight'] = new_table['smile'].apply(lambda x: Chem.Descriptors.MolWt(Chem.MolFromSmiles(x)))
new_table = new_table.sort_values('score', ascending=True)
new_table.head()
new_table.to_csv(r'./generations/master_results_table_gen' + str(GLOBAL_GENERATION-1) + '.csv', index=False)
# Select top X ranked by score for training data to refine the molecule generator RNN
training_smiles = list(set(list(new_table.head(35)['smile'])))
len(training_smiles)
training_fingerprints = []
for smile in training_smiles:
    training_fingerprints.append(Chem.RDKFingerprint(Chem.MolFromSmiles(smile)))

def calc_similarity_score(row):
    fingerprint = Chem.RDKFingerprint(Chem.MolFromSmiles(row['smile']))
    similarity = np.max(DataStructs.BulkTanimotoSimilarity(fingerprint,training_fingerprints))
    adj_factor = (1 / similarity) **.333
    adj_score = row['score'] * adj_factor
    return adj_score

similarity_adjusted = new_table.copy(deep=True)
similarity_adjusted = similarity_adjusted[similarity_adjusted['weight'] < 900]
similarity_adjusted['similarity_adj_score'] = similarity_adjusted.apply(calc_similarity_score, axis=1)
similarity_adjusted = similarity_adjusted.sort_values('similarity_adj_score', ascending=True)
similarity_adjusted.head()
# Select top X ranked by similarity adjusted score for training data to refine the molecule generator RNN (ensures diverity)
training_smiles += list(similarity_adjusted.head(5)['smile'])
len(training_smiles)
def calc_weight_score(row):
    adj_factor = (900 / row['weight']) ** .333
    if adj_factor < 1:
        adj_score = 0
    else:
        adj_score = row['score'] * adj_factor
    return adj_score

weight_adjusted = new_table.copy(deep=True)
weight_adjusted['weight_adj_score'] = weight_adjusted.apply(calc_weight_score, axis=1)
weight_adjusted = weight_adjusted.sort_values('weight_adj_score', ascending=True)
weight_adjusted.head()
# Select top X ranked by similarity adjusted score for training data to refine the molecule generator RNN (ensures diverity)
training_smiles += list(weight_adjusted.head(5)['smile'])
len(training_smiles)
import tensorflow
tensorflow.test.is_gpu_available()
import numpy as np
from copy import copy

import keras

from lstm_chem.utils.config import process_config
from lstm_chem.model import LSTMChem
from lstm_chem.generator import LSTMChemGenerator
from lstm_chem.trainer import LSTMChemTrainer
from lstm_chem.data_loader import DataLoader
# Generate some with the base original model
CONFIG_FILE = 'experiments/2019-12-23/LSTM_Chem/config.json'
config = process_config(CONFIG_FILE)
modeler = LSTMChem(config, session='generate')
generator = LSTMChemGenerator(modeler)
sample_number = 20
base_generated = generator.sample(num=sample_number)
base_generated_mols = validate_mols(base_generated)
base_generated_smiles = convert_mols_to_smiles(base_generated_mols)
random.shuffle(base_generated_smiles)
random.shuffle(base_generated_smiles)
# Select X for training data to refine the molecule generator RNN (ensures diverity)
training_smiles += base_generated_smiles[0:5]
len(training_smiles)
master_table = pd.read_csv('./generations/master_results_table_gen' + str(GLOBAL_GENERATION-1) + '.csv',sep=',')
master_table.head()
# Save the list of smiles to train on
with open('./generations/training/gen' + str(GLOBAL_GENERATION) + '_training.smi', 'w') as f:
    for item in training_smiles:
        f.write("%s\n" % item)<jupyter_output><empty_output><jupyter_text>Retrain the network to create molecules more like those selected above<jupyter_code>from lstm_chem.finetuner import LSTMChemFinetuner
config = process_config('experiments/2019-12-23/LSTM_Chem/config.json')
config['model_weight_filename'] = 'experiments/2019-12-23/LSTM_Chem/checkpoints/finetuned_gen' + str(GLOBAL_GENERATION-1) + '.hdf5'
config['finetune_data_filename'] = './generations/training/gen' + str(GLOBAL_GENERATION) + '_training.smi'
print(config)
modeler = LSTMChem(config, session='finetune')
finetune_dl = DataLoader(config, data_type='finetune')

finetuner = LSTMChemFinetuner(modeler, finetune_dl)
finetuner.finetune()
finetuner.model.save_weights('experiments/2019-12-23/LSTM_Chem/checkpoints/finetuned_gen' + str(GLOBAL_GENERATION) + '.hdf5')
config['model_weight_filename'] = 'experiments/2019-12-23/LSTM_Chem/checkpoints/finetuned_gen' + str(GLOBAL_GENERATION) + '.hdf5'
modeler = LSTMChem(config, session='generate')
generator = LSTMChemGenerator(modeler)
print(config)
sample_number = 5000
sampled_smiles = generator.sample(num=sample_number)
valid_mols = []
for smi in sampled_smiles:
    mol = Chem.MolFromSmiles(smi)
    if mol is not None:
        valid_mols.append(mol)
# low validity
print('Validity: ', f'{len(valid_mols) / sample_number:.2%}')

valid_smiles = [Chem.MolToSmiles(mol) for mol in valid_mols]
# high uniqueness
print('Uniqueness: ', f'{len(set(valid_smiles)) / len(valid_smiles):.2%}')

# Of valid smiles generated, how many are truly original vs ocurring in the training data
import pandas as pd
training_data = pd.read_csv('./datasets/all_smiles_clean.smi', header=None)
training_set = set(list(training_data[0]))
original = []
for smile in list(set(valid_smiles)):
    if not smile in training_set:
        original.append(smile)
print('Originality: ', f'{len(set(original)) / len(set(valid_smiles)):.2%}')
valid_smiles = list(set(valid_smiles))
len(valid_smiles)
#take the valid smiles from above and run them through process to add to tracking table and to generate next PyRx testing data
mols_for_next_generation = validate_mols(valid_smiles)

master_table = pd.read_csv('./generations/master_results_table_gen' + str(GLOBAL_GENERATION-1) +'.csv',sep=',')
new_mols_to_test = append_to_tracking_table(master_table,mols_for_next_generation, 'generated', GLOBAL_GENERATION)
mols_for_pd = new_mols_to_test[0]
mols_for_export = new_mols_to_test[1]

master_table = master_table.append(mols_for_pd)
master_table = master_table.reset_index(drop=True)
master_table.to_csv(r'./generations/master_results_table_gen' + str(GLOBAL_GENERATION) + '.csv', index=False)
len(mols_for_export)
write_gen_to_sdf(mols_for_export, GLOBAL_GENERATION, 2000)
print('ok')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11050041541509555, 'fraction_numerical': 0.013740653160350226, 'mean_word_length': 4.904905660377358, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '90702', 'license': 'Unlicense', 'n_tokens_mistral': 5481, 'n_tokens_neox': 5201, 'n_words': 1083, 'path': 'Evaluation and Refinement.ipynb', 'repo_name': 'PhysicsTeacher13/Deep-Learning---Coronavirus-Cure-Project', 'size': '15647'}