Upload 10 files

.gitattributes

@@ -1,35 +1,2 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Auto detect text files and perform LF normalization
+* text=auto

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Azmine Toushik Wasi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Model/data/__init__.py

@@ -0,0 +1 @@
+from .dataset import get_loaders_sequence, get_loaders_n_gram

Model/data/dataset.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from collections import Counter
+from itertools import product
+
+import numpy as np
+import pandas as pd
+import torch
+from torch.nn.utils.rnn import pad_sequence
+from torch.utils.data import DataLoader, Dataset
+
+
+def read_csv(
+    csv_file,
+    x_col="smiles",
+    y_col="tags",
+):
+    df = pd.read_csv(csv_file)
+
+    all_y = set()
+    all_x = set()
+
+    # drop multi columns
+    df = df[~df[y_col].str.contains(" ")]
+
+    x = df[x_col]
+    y = df[y_col]
+
+    # find all y
+    for item_y in y:
+        all_y.update(item_y.split(" "))
+
+    # make y mapping
+    mapping_y = {val: index for index, val in enumerate(sorted(list(all_y)))}
+
+    # find all x
+    for item_x in x:
+        all_x.update(set(item_x))
+
+    # make x mapping
+    mapping_x = {val: index + 1 for index, val in enumerate(sorted(list(all_x)))}
+    mapping_x["<pad>"] = 0
+
+    # encode y
+    ys = [mapping_y[i] for i in y]
+    ys = np.array(ys)
+
+    # encode x
+    xs = []
+    for item_x in x:
+        encoded_item = [mapping_x[c] for c in item_x]
+        xs.append(encoded_item)
+    xs = [np.array(item) for item in xs]
+
+    to_return = {
+        "x": {"raw": x.values, "data": xs},
+        "y": {"data": ys},
+        "mapping": {"x": mapping_x, "y": mapping_y},
+    }
+    return to_return
+
+
+def split_data(data, ratio_dev=0.1, ratio_test=0.1, seed=None):
+    # random number generator
+    rng = np.random.default_rng(seed=seed)
+
+    # dataset sizes
+    size_total = len(data["y"]["data"])
+    ratios = {"dev": ratio_dev, "test": ratio_test}
+    sizes = {}
+    for split, ratio in ratios.items():
+        sizes[split] = int(ratio * size_total)
+    sizes["train"] = size_total - sum(sizes.values())
+
+    # split
+    index = np.arange(size_total)
+    rng.shuffle(index)
+
+    indices = {}
+    start = 0
+    for split, size in sizes.items():
+        indices[split] = index[start : start + size]
+        start += size
+
+    splits = {}
+    for split, index in indices.items():
+        x_data = data["x"]
+        x_data = {k: [v[i] for i in index] for k, v in x_data.items()}
+
+        y_data = data["y"]
+        y_data = {k: v[index] for k, v in y_data.items()}
+
+        splits[split] = {"x": x_data, "y": y_data}
+
+    return splits
+
+
+def make_n_gram_mapping(mapping, n):
+    values = mapping.keys()
+    combos = product(values, repeat=n)
+    mapping = {"".join(v): i for i, v in enumerate(sorted(combos))}
+    return mapping
+
+
+def count_n_grams(text, n):
+    len_gram = len(text) + 1 - n
+    n_grams = [text[i : i + n] for i in range(len_gram)]
+    return Counter(n_grams)
+
+
+def get_topk_n_grams(data, n, topk=1000):
+    counters = [count_n_grams(text, n) for text in data]
+    counter = Counter()
+    for c in counters:
+        counter += c
+    results = [w for w, _ in counter.most_common(topk)]
+    return results
+
+
+def sequence_collate(batch):
+    x, y = zip(*batch)
+    x = [torch.LongTensor(item) for item in x]
+    lens = torch.LongTensor([len(i) for i in x])
+    x_padded = pad_sequence(x, batch_first=True, padding_value=0)
+    y = torch.LongTensor(np.array(y))
+    _, perm_idx = lens.sort(0, descending=True)
+    return x_padded[perm_idx], y[perm_idx], lens[perm_idx]
+
+
+class NgramDataset(Dataset):
+    """
+    Encoder based on n grams
+    """
+
+    def __init__(self, x, y, top_grams=None, n=1, topk=1000):
+        data_x = x["raw"]
+        data_y = y["data"]
+        if top_grams is None:
+            top_grams = get_topk_n_grams(data_x, n, topk=topk)
+
+        all_grams = []
+        for item_x in data_x:
+            unk = 0  # other tokens
+            grams = count_n_grams(item_x, n)
+            item = [grams[g] for g in top_grams]
+            unk = [v for k, v in grams.items() if k not in top_grams]  # unk
+            unk = sum(unk)
+            item.append(unk)
+            all_grams.append(item)
+
+        self.top_grams = top_grams
+        self.x = np.array(all_grams, dtype="float32")
+        self.x_raw = data_x
+        self.y = np.array(data_y, dtype="long")
+
+    def __getitem__(self, index):
+        item_x = self.x[index]
+        item_y = self.y[index]
+
+        return item_x, item_y
+
+    def __len__(self):
+        return len(self.x)
+
+
+class SequenceDataset(Dataset):
+    """
+    Encode each character in sequence.
+    0: padding
+    """
+
+    def __init__(self, x, y, mapping_x, mapping_y, n=1):
+        data_x = x["data"]
+        data_y = y["data"]
+
+        self.x = data_x
+
+        self.x_raw = x["raw"]
+        self.y = np.array(data_y, dtype="int64")
+
+        self.mapping_x = mapping_x
+        self.mapping_x_inverse = {v: k for k, v in self.mapping_x.items()}
+        self.mapping_y = mapping_y
+        self.mapping_y_inverse = {v: k for k, v in self.mapping_y.items()}
+
+    def __getitem__(self, index):
+        item_x = np.array(self.x[index], dtype="int64")
+        item_y = self.y[index]
+
+        return item_x, item_y
+
+    def __len__(self):
+        return len(self.x)
+
+
+def get_loaders_n_gram(
+    csv_file, n=1, topk=20, ratio_dev=0.1, ratio_test=0.1, batch_size=32, seed=None
+):
+    data = read_csv(csv_file)
+    mapping_x = data["mapping"]["x"]
+    mapping_y = data["mapping"]["y"]
+    splits = split_data(
+        data,
+        ratio_dev=ratio_dev,
+        ratio_test=ratio_test,
+        seed=seed,
+    )
+
+    # make train sets
+    split_train = splits.pop("train")
+    dataset_train = NgramDataset(split_train["x"], split_train["y"], n=n, topk=topk)
+    top_grams = dataset_train.top_grams
+
+    datasets = {
+        k: NgramDataset(v["x"], v["y"], n=n, top_grams=top_grams)
+        for k, v in splits.items()
+    }
+    datasets["train"] = dataset_train
+    # batch size * 2 for train
+    batch_sizes = {
+        k: batch_size if k == "train" else batch_size * 2 for k in datasets.keys()
+    }
+    # shuffle only the train set
+    shuffle = {k: True if k == "train" else False for k in datasets.keys()}
+    # make loaders
+    loaders = {
+        k: DataLoader(v, batch_size=batch_sizes[k], shuffle=shuffle[k])
+        for k, v in datasets.items()
+    }
+    # find sizes
+    size_x = len(top_grams) + 1
+    size_y = len(mapping_y)
+    return {"loaders": loaders, "sizes": {"x": size_x, "y": size_y}}
+
+
+def get_loaders_sequence(
+    csv_file,
+    ratio_dev=0.1,
+    ratio_test=0.1,
+    batch_size=32,
+    seed=None,
+):
+    data = read_csv(csv_file)
+    mapping_x = data["mapping"]["x"]
+    mapping_y = data["mapping"]["y"]
+    splits = split_data(
+        data,
+        ratio_dev=ratio_dev,
+        ratio_test=ratio_test,
+        seed=seed,
+    )
+
+    datasets = {
+        k: SequenceDataset(v["x"], v["y"], mapping_x, mapping_y)
+        for k, v in splits.items()
+    }
+    # batch size * 2 for train
+    batch_sizes = {
+        k: batch_size if k == "train" else batch_size * 2 for k in datasets.keys()
+    }
+    # shuffle only the train set
+    shuffle = {k: True if k == "train" else False for k in datasets.keys()}
+    # make loaders
+    loaders = {
+        k: DataLoader(
+            v,
+            batch_size=batch_sizes[k],
+            shuffle=shuffle[k],
+            collate_fn=sequence_collate,
+        )
+        for k, v in datasets.items()
+    }
+    # find sizes
+    size_x = len(mapping_x)
+    size_y = len(mapping_y)
+    return {"loaders": loaders, "sizes": {"x": size_x, "y": size_y}}

Model/methods/MLP.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from torch import nn
+
+class MLP(nn.Module):
+    """
+    Multi layer perceptron.
+    """
+    def __init__(self, size_in, size_out, size_hidden=None, dropout=0.0):
+        super().__init__()
+        if size_hidden is None:
+            size_hidden = []
+        sizes = [size_in] + size_hidden + [size_out]
+
+        net = []
+        for i in range(len(sizes) - 2):
+            net.append(nn.Linear(sizes[i], sizes[i+1]))
+            net.append(nn.ReLU())
+            net.append(nn.Dropout(dropout))
+
+        net.append(nn.Linear(sizes[-2], sizes[-1]))
+        net = nn.Sequential(*net)
+        self.net = net
+
+    def forward(self, x):
+        """
+        Forward method.
+        """
+        x = self.net(x)
+        return x

Model/methods/__init__.py

@@ -0,0 +1 @@
+from .MLP import MLP

Model/train-ngram.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import copy
+
+import torch
+from torch import nn
+from torch.optim import Adam
+from tqdm import tqdm
+
+from data import get_loaders_n_gram
+from methods import MLP
+
+
+def train(loader_train, loader_dev, model, device, optimizer, n_epochs):
+    acc_best = 0
+    model_best = None
+    criterion = nn.CrossEntropyLoss()
+
+    bar_epochs = tqdm(range(n_epochs), leave=False)
+    for epoch in bar_epochs:
+        # train
+        bar_epoch = tqdm(loader_train, disable=True, leave=False)
+        model.train()
+        for x, y in bar_epoch:
+            x = x.to(device)
+            y = y.to(device)
+            y_out = model(x)
+            loss = criterion(y_out, y.type(torch.LongTensor))
+            loss.backward()
+            optimizer.step()
+            loss_iter = loss.item()
+            bar_epoch.set_postfix({"loss": loss_iter})
+        bar_epoch.close()
+
+        bar_dev = tqdm(loader_dev, disable=True, leave=False)
+        model.eval()
+
+        # val
+        ys_pred, ys_true = [], []
+        with torch.no_grad():
+            for x, y in bar_dev:
+                x = x.to(device)
+                y = y.to(device)
+                y_out = model(x)
+                y_pred = torch.argmax(y_out, axis=1)
+                ys_pred.append(y_pred.cpu())
+                ys_true.append(y.cpu())
+        bar_dev.close()
+        ys_pred = torch.cat(ys_pred)
+        ys_true = torch.cat(ys_true)
+        acc = (ys_pred == ys_true).float().mean()
+        acc = acc.item() * 100
+        if acc > acc_best:
+            acc_best = acc
+            model_best = copy.deepcopy(model)
+        bar_epochs.set_postfix({"acc_best": acc_best})
+
+    return model_best
+
+
+def test(loader_test, model, device):
+    model.eval()
+    ys_pred, ys_true = [], []
+    bar_test = tqdm(loader_test, leave=False)
+    with torch.no_grad():
+        for x, y in bar_test:
+            x = x.to(device)
+            y = y.to(device)
+            y_pred = model(x)
+            y_pred = torch.argmax(y_pred, axis=1)
+            ys_pred.append(y_pred.cpu())
+            ys_true.append(y.cpu())
+
+    bar_test.close()
+
+    ys_pred = torch.cat(ys_pred)
+    ys_true = torch.cat(ys_true)
+
+    return ys_pred, ys_true
+
+
+def run(
+    csv_file,
+    seed,
+    n=5,
+    topk=1000,
+    ratio_dev=0.1,
+    ratio_test=0.1,
+    batch_size=32,
+    size_hidden=None,
+    dropout=0.1,
+    n_epochs=50,
+    lr=3e-4,
+    weight_decay=0,
+):
+    # data settings
+    ratio_dev = ratio_dev
+    ratio_test = ratio_test
+    batch_size = batch_size
+    n = n
+    data = get_loaders_n_gram(
+        csv_file,
+        n=n,
+        topk=topk,
+        ratio_dev=ratio_dev,
+        ratio_test=ratio_test,
+        seed=seed,
+        batch_size=batch_size,
+    )
+    size_x = data["sizes"]["x"]
+    size_y = data["sizes"]["y"]
+    loader_train = data["loaders"]["train"]
+    loader_dev = data["loaders"]["dev"]
+    loader_test = data["loaders"]["test"]
+    # device
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    # model settings
+    if size_hidden is None:
+        size_hidden = [size_x // 2, size_x // 4]
+    size_hidden = [size_x] + size_hidden
+    dropout = dropout
+    model = MLP(
+        size_in=size_x,
+        size_out=size_y,
+        size_hidden=size_hidden,
+        dropout=dropout,
+    )
+    model = model.to(device)
+
+    # training settings
+    n_epochs = n_epochs
+    lr = lr
+    weight_decay = weight_decay
+    optimizer = Adam(
+        model.parameters(),
+        lr=lr,
+        weight_decay=weight_decay,
+    )
+
+    # train
+    model_best = train(loader_train, loader_dev, model, device, optimizer, n_epochs)
+    return test(loader_test, model_best, device)
+
+
+if __name__ == "__main__":
+    # data dir
+    csv_file = "./_DATA/all_chem_df.csv"
+    # number of trials
+    n_trials = 5
+    seeds = list(range(n_trials))
+    # data settings
+    topk = 1000
+    ratio_dev = 0.1
+    ratio_test = 0.2
+    batch_size = 32
+    # model settings
+    n = 5
+    dropout = 0.1
+    size_hidden = [512, 256, 128, 32]
+    # training settings
+    n_epochs = 200
+    lr = 3e-5
+    weight_decay = 0
+
+    
+
+    for seed in seeds:
+        y_pred, y_true = run(
+            csv_file,
+            seed,
+            n,
+            topk,
+            ratio_dev,
+            ratio_test,
+            batch_size,
+            size_hidden,
+            dropout,
+            n_epochs,
+            lr,
+        )
+        log_file = f"./scores/MLP/{seed}-seed--{n}-gram--topk-{topk}--lr-{lr}.csv"
+        with open(log_file, "a") as f:
+            f.write("pred,true\n")
+            for p, t in zip(y_pred, y_true):
+                f.write(f"{p},{t}\n")

README.md

@@ -1,3 +1,48 @@
----
-license: mit
----
+# ***When SMILES have Language*: Drug Classification using Text Classification Methods on Drug SMILES Strings**
+- **Authors:** Azmine Toushik Wasi, Šerbetar Karlo, Raima Islam, Taki Hasan Rafi, Dong-Kyu Chae
+- Accepted (***invited to present***) to the **The Second Tiny Papers Track at ICLR 2024**!
+- Read full paper in [arXiv](https://arxiv.org/abs/2403.12984).
+---
+
+<p align="center">
+  <img src="Fig/model.png" width="1000"/>
+</p>
+
+**Abstract**: Complex chemical structures, like drugs, are usually defined by SMILES strings as a sequence of molecules and bonds. These SMILES strings are used in different complex machine learning-based drug-related research and representation works. Escaping from complex representation, in this work, we pose a single question: What if we treat drug SMILES as conventional sentences and engage in text classification for drug classification? Our experiments affirm the possibility with very competitive scores. The study explores the notion of viewing each atom and bond as sentence components, employing basic NLP methods to categorize drug types, proving that complex problems can also be solved with simpler perspectives.
+
+---
+
+# Setup and run
+- Data is available at `./Model/_DATA_`
+- Dataloader is available at `./Model/data`
+- To run the training script, place the dataset from DrugBank, go to `./Model/` folder and run: `python train-ngram.py`
+- To change parameters, you can check and edit `145-165` no lines of `./Model/train-ngram.py`
+
+# Experimental Results
+
+| Model          | Accuracy | Precision | Recall | F1 (Weighted) | F1 (Macro) | ROC-AUC |
+|----------------|----------|-----------|--------|----------------|-------------|---------|
+| 1-gram+MLP     | 0.622    | 0.610     | 0.622  | 0.604          | 0.406       | 0.760   |
+| 2-gram+MLP     | 0.669    | 0.700     | 0.669  | 0.672          | 0.445       | 0.810   |
+| 3-gram+MLP     | **0.737**| **0.764** | **0.737**| **0.744**     | 0.553       | **0.848**|
+| 4-gram+MLP     | 0.726    | 0.758     | 0.726  | 0.731          | 0.524       | 0.841   |
+| 5-gram+MLP     | 0.728    | 0.740     | 0.728  | 0.730          | **0.563**   | 0.838   |
+| AtomPair+MLP   | 0.799    | 0.804     | 0.800  | 0.799          | 0.702       | 0.876   |
+| MACCS+MLP      | 0.797    | 0.801     | 0.797  | 0.796          | 0.702       | 0.873   |
+| Morgan+MLP     | **0.800**| **0.804** | **0.800**| **0.799**     | **0.703**   | **0.876**|
+|
+
+
+
+
+# Citation
+```
+@inproceedings{wasi2024drug_nlp,,
+  author       = {Azmine Toushik Wasi and Šerbetar Karlo and Raima Islam and Taki Hasan Rafi and Dong-Kyu Chae},
+  title        = {When SMILES have Language: Drug Classification using Text Classification Methods on Drug SMILES Strings},
+  booktitle    = {The Second Tiny Papers Track at {ICLR} 2024, Tiny Papers @ {ICLR} 2024, Vienna Austria, May 11, 2024},
+  publisher    = {OpenReview.net},
+  year         = {2023},
+  url          = {https://openreview.net/forum?id=VUYCyH8fCw}
+}
+```