<jupyter_start><jupyter_text>***Tokenize all of the training strings*** *Tokenize the training set*<jupyter_code># traverse through the training smiles and tokenize, and accumulate the alphabet
tkn_smls_train = []
train_alphabet = set()
for sml in train_srp53.smiles:
    toks = atomwise_tokenizer(sml)
    train_alphabet = train_alphabet.union(set(toks))
    tkn_smls_train.append(toks)
tkn_smls_test = []
for sml in test_srp53.smiles:
    toks = atomwise_tokenizer(sml)
    tkn_smls_test.append(toks)
print(len(tkn_smls_train))
print(len(train_alphabet))
print(len(tkn_smls_test))<jupyter_output>8629
127
268<jupyter_text>***Preprocessing sequence data*** *Trying to deal with unseen tokens in test set*<jupyter_code># copied from https://stackoverflow.com/questions/21057621/sklearn-labelencoder-with-never-seen-before-values
from sklearn.preprocessing import LabelEncoder
class LabelEncoderExt(object):
    def __init__(self):
        """
        It differs from LabelEncoder by handling new classes and providing a value for it [Unknown]
        Unknown will be added in fit and transform will take care of new item. It gives unknown class id
        """
        self.label_encoder = LabelEncoder()
        # self.classes_ = self.label_encoder.classes_

    def fit(self, data_list):
        """
        This will fit the encoder for all the unique values and introduce unknown value
        :param data_list: A list of string
        :return: self
        """
        self.label_encoder = self.label_encoder.fit(list(data_list) + ['Unknown'])
        self.classes_ = self.label_encoder.classes_

        return self

    def transform(self, data_list):
        """
        This will transform the data_list to id list where the new values get assigned to Unknown class
        :param data_list:
        :return:
        """
        new_data_list = list(data_list)
        for unique_item in np.unique(data_list):
            if unique_item not in self.label_encoder.classes_:
                new_data_list = ['Unknown' if x==unique_item else x for x in new_data_list]

        return self.label_encoder.transform(new_data_list)

## EXAMPLE OF UNSEEN TOKEN PROCESSING
# country_list = ['Argentina', 'Australia', 'Canada', 'France', 'Italy', 'Spain', 'US', 'Canada', 'Argentina, ''US']

# label_encoder = LabelEncoderExt()

# label_encoder.fit(country_list)
# print(label_encoder.classes_) # you can see new class called Unknown
# print(label_encoder.transform(country_list))


# new_country_list = ['Canada', 'France', 'Italy', 'Spain', 'US', 'India', 'Pakistan', 'South Africa']
# print(label_encoder.transform(new_country_list))
# using above object LabelEncoderExt
le = LabelEncoderExt()
# fit a label encoder on the alphabet from our training data
alphabet_enc = le.fit(list(train_alphabet))
alphabet_enc.transform(tkn_smls_train[0])
# traverse through the tokenized training samples and store a numpy array of integer-encoded samples
train_encoded = [alphabet_enc.transform(sm) for sm in tkn_smls_train]
train_encoded[0]
test_encoded = [alphabet_enc.transform(sm) for sm in tkn_smls_test]
test_encoded[0]
import tensorflow as tf

padded_train = tf.keras.preprocessing.sequence.pad_sequences(
    train_encoded, padding="post"
)
padded_test = tf.keras.preprocessing.sequence.pad_sequences(
    test_encoded, padding="post", maxlen=padded_train.shape[1]
)
padded_train
print(padded_train.shape)
print(padded_test.shape)<jupyter_output>(8629, 240)
(268, 240)<jupyter_text>***Training data****Going to split this for the baseline -- the test file from the NIH doesn't have a target variable*<jupyter_code># X_train = tf.one_hot(padded_inputs, depth=len(train_alphabet))
X_train = padded_train
X_test = padded_test

y_train = train_srp53.target
y_test = test_srp53.target
X_train
X_train.shape
print(X_train.shape)
print(X_test.shape)
print(y_train.shape)
print(y_test.shape)<jupyter_output>(8629, 240)
(268, 240)
(8629,)
(268,)<jupyter_text>Baseline model<jupyter_code>from tensorflow import keras
from keras.models import Sequential
from tensorflow.keras import layers
from tensorflow.keras.layers import Dense, LSTM, SimpleRNN, Embedding

# initialize weights and biases project
wandb.init(project="SMILES_baseline")
embedding_vector_length = len(train_alphabet) + 1
model = Sequential()
model.add(tf.keras.layers.Masking(mask_value=0., input_shape=(X_train.shape[1],)))
model.add(Embedding((len(train_alphabet)+1), embedding_vector_length, input_length=240))
model.add(SimpleRNN(100, input_shape=(X_train.shape[0],X_train.shape[1])))
# model.add(LSTM(100, input_shape=(X_train.shape[0],X_train.shape[1])))
model.add(Dense(1, activation='sigmoid'))

# common metric for the binary classification problem: AUC
auc_metric = tf.keras.metrics.AUC()

opt = keras.optimizers.SGD(lr = 0.0001)
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy',auc_metric])

print(model.summary())

# need the wandb callback
callbacks = [wandb.keras.WandbCallback()]

model.fit(X_train, 
          y_train, 
          validation_data=(X_test, y_test), 
          epochs=10, 
          batch_size=32,
          callbacks = callbacks)
wandb.finish()<jupyter_output><empty_output><jupyter_text>... and the moral is: Because toxicity is less common than non-toxicity, data will be very skewed in favor of non-tox. To combat this and get an informative judge of classification quality, we use AUROC (resilient to skewed data)<jupyter_code>(unique, counts) = np.unique(y_train, return_counts=True)
frequencies = np.asarray((unique, counts)).T

print(frequencies[0][1]/len(y_train)) #<--- proportion in class 0
(unique, counts) = np.unique(y_test, return_counts=True)
frequencies = np.asarray((unique, counts)).T

print(frequencies[0][1]/len(y_test)) #<--- proportion in class 0<jupyter_output>0.8955223880597015
{'fraction_non_alphanumeric': 0.11006018916595013, 'fraction_numerical': 0.01943250214961307, 'mean_word_length': 5.020703933747412, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '89944', 'license': 'MIT', 'n_tokens_mistral': 2027, 'n_tokens_neox': 1907, 'n_words': 464, 'path': 'Colab_Notebooks/SMILES_baseline.ipynb', 'repo_name': 'Molecular-Exploration/toxicity-classification', 'size': '5815'}