<jupyter_start><jupyter_text>Quora Similar Questions Dataset<jupyter_code>! pip install colorlog --quiet
! pip install gdown==3.13.0 --quiet
! pip install spacy==3.0.6 --quiet
! pip install pytorch-lightning --quiet
! pip install --pre torch==1.8.0 torchtext==0.9.0 -f https://download.pytorch.org/whl/nightly/cu111/torch_nightly.html
! python -m spacy download en_core_web_sm<jupyter_output><empty_output><jupyter_text>NOTE: Torchtext 0.9.0 and TorchText Nightly are the two version of torchtext working for this notebook<jupyter_code>import colorlog

logger = colorlog.getLogger()
logger.setLevel(colorlog.colorlog.logging.DEBUG)

handler = colorlog.StreamHandler()
handler.setFormatter(
    colorlog.ColoredFormatter(
        '%(log_color)s [%(asctime)s] %(levelname)s [%(filename)s.%(funcName)s:%(lineno)d] %(message)s', datefmt='%a, %d %b %Y %H:%M:%S',
        log_colors={
                'DEBUG':    'cyan',
                'INFO':     'green',
                'WARNING':  'yellow',
                'ERROR':    'red',
                'CRITICAL': 'red,bg_white',
            },
        )
    )
logger.addHandler(handler)
! wget http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv
import pandas as pd
! cat quora_duplicate_questions.tsv | head
quora_df = pd.read_csv('quora_duplicate_questions.tsv', sep="\t")
quora_df
duplicate_df = quora_df[quora_df['is_duplicate'] == 1]
duplicate_df
duplicate_df.to_csv('quora_duplicate_only_questions.csv')<jupyter_output><empty_output><jupyter_text>Quora Similar Questions Dataset<jupyter_code>! gdown https://drive.google.com/uc?id=1g2YqSPXBWdCU1SjkCb69ENUuoEuxPbLg
import pandas as pd
pd.read_csv('/content/quora_duplicate_only_questions.csv', index_col=0)
import torch
import torchtext
import gdown

import pytorch_lightning as pl

from torch.utils.data import Dataset, DataLoader, random_split

from torchtext.data.utils import get_tokenizer
# from torchtext.vocab import build_vocab_from_iterator

import torchtext.experimental.functional as text_f

from sklearn.model_selection import train_test_split

from pathlib import Path
from functools import partial
from tqdm.auto import tqdm

from collections import Counter
from torchtext.vocab import Vocab

import pandas as pd

from typing import *
def build_vocab_from_iterator(iterator, num_lines=None, *args, **kwargs):
    """
    Build a Vocab from an iterator.

    Args:
        iterator: Iterator used to build Vocab. Must yield list or iterator of tokens.
        num_lines: The expected number of elements returned by the iterator.
            (Default: None)
            Optionally, if known, the expected number of elements can be passed to
            this factory function for improved progress reporting.
    """

    counter = Counter()
    with tqdm(unit_scale=0, unit='lines', total=num_lines) as t:
        for tokens in iterator:
            counter.update(tokens)
            t.update(1)
    word_vocab = Vocab(counter, *args, **kwargs)
    return word_vocab
class QuoraSQDataset(Dataset):
    """
    Quora Similar Questions Dataset
    """

    URL = 'https://drive.google.com/uc?id=1g2YqSPXBWdCU1SjkCb69ENUuoEuxPbLg'
    OUTPUT = 'quora_duplicate_only_questions.csv'
 

    def __init__(self, root, split='train', vocab=None, vectors=None, text_transforms=None, label_transforms=None, ngrams=1):
        """Initiate dataset.
        Args:
            vocab: Vocabulary object used for dataset.
        """

        super(self.__class__, self).__init__()

        if vectors:
            raise NotImplementedError(f'vectors not supported for this dataset as of now') 

        if split not in ['train', 'test']:
            raise ValueError(f'split must be either "train" or "test" unknown split {split}')

        if vocab and vectors:
            raise ValueError(f'both vocab and vectors cannot be provided')

        self.vocab = vocab
        self.vectors = vectors

        gdown.cached_download(self.URL, Path(root) / self.OUTPUT)

        self.generate_tweet_dataset(Path(root) / self.OUTPUT)

        self.train_dset, self.test_dset = train_test_split(self.full_dataset_, test_size=0.3, random_state=42)

        if split == 'train':
            self.dataset = self.train_dset
        elif split == 'test':
            self.dataset = self.test_dset
        else:
            raise ValueError("What did you do? you stupid potato?")

        # create the tokenizer, here we use spacy
        tokenizer = get_tokenizer("spacy", language="en_core_web_sm")
        self.tokenizer = tokenizer

        # the text transform can only work at the sentence level
        # the rest of tokenization and vocab is done by this class
        self.text_transform = text_f.sequential_transforms(tokenizer, text_f.ngrams_func(ngrams))

        self.vocab_transforms = text_f.sequential_transforms()
        self.vector_transforms = text_f.sequential_transforms()

        def build_vocab(data, transforms):
            def apply_transforms(data):
                for line in data:
                    yield transforms(line)
            return build_vocab_from_iterator(apply_transforms(data), len(data), specials=['<unk>', '<pad>', '<bos>', '<eos>'])

        if self.vectors:
            self.vector_transforms = text_f.sequential_transforms(
                partial(vectors.get_vecs_by_tokens, lower_case_backup=True)
            )
        elif self.vocab is None:
            self.vocab = build_vocab(
                pd.concat([self.train_dset['question1'], self.train_dset['question2']]),
                self.text_transform
            )
            self.PAD_IDX = self.vocab['<pad>']
            self.BOS_IDX = self.vocab['<bos>']
            self.EOS_IDX = self.vocab['<eos>']

    
        # if the user is using vocab, instead of vector
        if self.vocab:
            self.vocab_transforms = text_f.sequential_transforms(
                text_f.vocab_func(self.vocab), text_f.totensor(dtype=torch.long)
            )

        # label transform is similar to text_transform for this dataset except this does not have vectors
        self.label_transform = text_f.sequential_transforms(
            self.text_transform, self.vocab_transforms
        )

        if text_transforms is not None:
            self.text_transform = text_f.sequential_transforms(
                self.text_transform, text_transforms, self.vocab_transforms, self.vector_transforms 
            )
        else:
            self.text_transform = text_f.sequential_transforms(
                self.text_transform, self.vocab_transforms, self.vector_transforms
            )
        

    def generate_tweet_dataset(self, dataset_file):
        self.full_dataset_ = pd.read_csv(dataset_file)

    def __getitem__(self, idx):
        text = self.text_transform(self.dataset['question1'].iloc[idx])
        label = self.label_transform(self.dataset['question2'].iloc[idx])
        return label, text

    def __len__(self):
        return len(self.dataset)

    def get_vocab(self):
        return self.vocab

    def get_vectors(self):
        return self.vectors

    def batch_sampler_fn(self):
        raise NotImplementedError(f'batch_sampler function WIP')

        def batch_sampler():
            indices = [(i, len(self.tokenizer(s[1]))) for i, s in enumerate(train_list)]
            random.shuffle(indices)
            pooled_indices = []
            # create pool of indices with similar lengths 
            for i in range(0, len(indices), batch_size * 100):
                pooled_indices.extend(sorted(indices[i:i + batch_size * 100], key=lambda x: x[1]))

            pooled_indices = [x[0] for x in pooled_indices]

            # yield indices for current batch
            for i in range(0, len(pooled_indices), batch_size):
                yield pooled_indices[i:i + batch_size]
        return batch_sampler()


    def collator_fn(self):
        def collate_fn(batch):
            
            targets, sequences = zip(*batch)

            lengths = torch.LongTensor([len(sequence) for sequence in sequences])

            targets = [torch.cat([torch.tensor([self.BOS_IDX]), item, torch.tensor([self.EOS_IDX])]) for item in targets]
            sequences = [torch.cat([torch.tensor([self.BOS_IDX]), item, torch.tensor([self.EOS_IDX])]) for item in sequences]

            if not self.vectors:
                pad_idx = self.PAD_IDX
                sequences = torch.nn.utils.rnn.pad_sequence(sequences, 
                                                            padding_value = pad_idx,
                                                            batch_first=True
                                                            )
                targets = torch.nn.utils.rnn.pad_sequence(targets, 
                                            padding_value = pad_idx,
                                            batch_first=True
                                            )
            
            return targets, sequences, lengths
        
        return collate_fn
dataset = QuoraSQDataset(root='.', split='train')
dataset[0]
loader = DataLoader(
            dataset,
            batch_size=16,
            shuffle=False,
            num_workers=2,
            pin_memory=True,
            collate_fn=dataset.collator_fn()
        )
targets, sequences, lengths = next(iter(loader))
targets.shape, sequences.shape, len(lengths)
' '.join(dataset.get_vocab().itos[x] for x in sequences[0])
' '.join(dataset.get_vocab().itos[x] for x in targets[0])
class QuoraSQDataModule(pl.LightningDataModule):
    """
    DataModule for Quora Similar Questions, train, val, test splits and transforms
    """

    name = "quora similar questions"

    def __init__(
        self,
        data_dir: str = '.',
        num_workers: int = 2,
        batch_size: int = 128,
        *args,
        **kwargs,
    ):
        """
        Args:
            data_dir: where to save/load the data
            val_split: how many of the training images to use for the validation split
            num_workers: how many workers to use for loading data
            normalize: If true applies image normalize
            batch_size: desired batch size.
        """
        super().__init__(*args, **kwargs)

        self.data_dir = data_dir
        self.num_workers = num_workers
        self.batch_size = batch_size

        self.dataset_train = ...
        self.dataset_val = ...
        self.dataset_test = ...
        self.vectors = ...

        self.Dataset = QuoraSQDataset

    def prepare_data(self):
        """Saves IMDB files to `data_dir`"""
        self.Dataset(self.data_dir)

    def setup(self, stage: Optional[str] = None):
        """Split the train and valid dataset"""

        self.dataset_train = self.Dataset(self.data_dir, split='train')
        self.dataset_test = self.Dataset(self.data_dir, split='test')

    def train_dataloader(self):
        loader = DataLoader(
            self.dataset_train,
            batch_size=self.batch_size,
            shuffle=True,
            num_workers=self.num_workers,
            pin_memory=True,
            collate_fn=self.collator_fn
        )
        return loader

    def val_dataloader(self):
        loader = DataLoader(
            self.dataset_test,
            batch_size=self.batch_size,
            shuffle=False,
            num_workers=self.num_workers,
            pin_memory=True,
            collate_fn=self.collator_fn
        )
        return loader

    def test_dataloader(self):
        loader = DataLoader(
            self.dataset_test,
            batch_size=self.batch_size,
            shuffle=False,
            num_workers=self.num_workers,
            pin_memory=True,
            collate_fn=self.collator_fn
        )
        return loader

    def get_vocab(self):
        return self.dataset_train.get_vocab()
    
    def get_vectors(self):
        return self.dataset_train.get_vectors()

    @property
    def default_transforms(self):
        raise NotImplementedError(f'default_transforms not implemented')
        train_transforms = {
            'text_transforms': text_f.sequential_transforms(
                random_deletion,
                random_swap
            ),
            'label_transforms': None
        }
        test_transforms = {
            'text_transforms': None,
            'label_transforms': None
        }

        return train_transforms, test_transforms

    @property
    def collator_fn(self):
        return self.dataset_train.collator_fn()
datamodule = QuoraSQDataModule(batch_size=16)
datamodule.setup()
train_loader = datamodule.train_dataloader()
val_loader = datamodule.val_dataloader()
test_loader = datamodule.test_dataloader()
len(train_loader), len(val_loader), len(test_loader)
a, b, c = next(iter(train_loader))
a.shape, b.shape, len(c)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09475082018434619, 'fraction_numerical': 0.005077331666927042, 'mean_word_length': 2.346314688970204, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '433124', 'license': 'MIT', 'n_tokens_mistral': 3931, 'n_tokens_neox': 3796, 'n_words': 840, 'path': '07_Seq2Seq/Quora_Question_Dataset.ipynb', 'repo_name': 'extensive-nlp/TSAI-DeepNLP-END2.0', 'size': '12802'}