<jupyter_start><jupyter_text><jupyter_code>!nvidia-smi<jupyter_output>Fri Oct 16 05:08:53 2020       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 455.23.05    Driver Version: 418.67       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla P4            Off  | 00000000:00:04.0 Off |                    0 |
| N/A   42C    P8     7W /  75W |      0MiB /  7611MiB |      0%      Default |
|                               |                      |                 ERR! |
+-------------------------------+----------------------+----------------------+
                                                                               
+-------[...]<jupyter_text>Load the training data- 200,000 pairs of parallel corpus<jupyter_code>pwd
import pandas as pd

df = pd.read_csv('/content/drive/My Drive/Colab Notebooks/news_200k.csv')
df = df[['kor', 'eng']]

print(df.shape)
df.sample(10)<jupyter_output><empty_output><jupyter_text>Preprocessing the training data - konlpy 라이브러리의 한국어 형태소분석기 Mecab 설치<jupyter_code>!pip install konlpy
! sudo apt-get install curl git
! bash <(curl -s https://raw.githubusercontent.com/konlpy/konlpy/master/scripts/mecab.sh)<jupyter_output>Collecting konlpy
[?25l  Downloading https://files.pythonhosted.org/packages/85/0e/f385566fec837c0b83f216b2da65db9997b35dd675e107752005b7d392b1/konlpy-0.5.2-py2.py3-none-any.whl (19.4MB)
[K     |████████████████████████████████| 19.4MB 1.3MB/s 
[?25hRequirement already satisfied: numpy>=1.6 in /usr/local/lib/python3.6/dist-packages (from konlpy) (1.18.5)
Collecting beautifulsoup4==4.6.0
[?25l  Downloading https://files.pythonhosted.org/packages/9e/d4/10f46e5cfac773e22707237bfcd51bbffeaf0a576b0a847ec7ab15bd7ace/beautifulsoup4-4.6.0-py3-none-any.whl (86kB)
[K     |████████████████████████████████| 92kB 10.5MB/s 
[?25hCollecting tweepy>=3.7.0
  Downloading https://files.pythonhosted.org/packages/bb/7c/99d51f80f3b77b107ebae2634108717362c059a41384a1810d13e2429a81/tweepy-3.9.0-py2.py3-none-any.whl
Requirement already satisfied: lxml>=4.1.0 in /usr/local/lib/python3.6/dist-packages (from konlpy) (4.2.6)
Collecting colorama
  Downloading https://files.pythonhosted.org/packages/44/98/5b86[...]<jupyter_text>- spacy 라이브러리의 영어 형태소분석기 설치<jupyter_code>!python -m spacy download en
from konlpy.tag import Mecab
import spacy

mecab = Mecab()
spacy_en = spacy.load('en')

def tokenize_en(text):
    return [tok.text for tok in spacy_en.tokenizer(text)]

print('tokenization test with sample texts')
print('tokenizing for Korean: ', mecab.morphs('안녕하세요 저는 건국대학교에 재학 중인 박도준입니다.'))
print('tokenizing for English: ', tokenize_en('Hello, I am <NAME>, a student at Konkuk University.'))
from torchtext.data import Field, TabularDataset, BucketIterator

korean = Field(tokenize=mecab.morphs, lower=True, init_token='<sos>', eos_token='<eos>')
english = Field(tokenize=tokenize_en, lower=True, init_token='<sos>', eos_token='<eos>')<jupyter_output>/usr/local/lib/python3.6/dist-packages/torchtext/data/field.py:150: UserWarning: Field class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.
  warnings.warn('{} class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.'.format(self.__class__.__name__), UserWarning)<jupyter_text>- training, test, vaildation data의 비율을 6:2:2로 분할<jupyter_code>from sklearn.model_selection import train_test_split

ko_train, ko_test, en_train, en_test = train_test_split(df['kor'], df['eng'], test_size=0.2, random_state=1)
ko_train, ko_val, en_train, en_val = train_test_split(ko_train, en_train, test_size=0.25, random_state=1)
from sklearn.model_selection import train_test_split

train_data, test_data = train_test_split(df, test_size=0.2, random_state=1)
train_data, valid_data = train_test_split(train_data, test_size=0.25, random_state=1)
print('number of train_data: {}, {:.1f}%'.format(len(train_data), len(train_data)/(len(train_data)+len(test_data)+len(valid_data))))
print('number of test_data: {}, {:.1f}%'.format(len(test_data), len(test_data)/(len(train_data)+len(test_data)+len(valid_data))))
print('number of valid_data: {}, {:.1f}%'.format(len(valid_data), len(valid_data)/(len(train_data)+len(test_data)+len(valid_data))))<jupyter_output>number of train_data: 120006, 0.6%
number of test_data: 40003, 0.2%
number of valid_data: 40002, 0.2%<jupyter_text>- make vocab list<jupyter_code>korean.build_vocab(train_data['kor'], max_size=10000, min_freq=2)
english.build_vocab(train_data['eng'], max_size=10000, min_freq=2)<jupyter_output><empty_output><jupyter_text>Model designing<jupyter_code>import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.tensorboard import SummaryWriter


class Transformer(nn.Module):
    def __init__(
        self,
        embedding_size,
        src_vocab_size,
        trg_vocab_size,
        src_pad_idx,
        num_heads,
        num_encoder_layers,
        num_decoder_layers,
        forward_expansion,
        dropout,
        max_length,
        device):

        super(Transformer, self).__init__()
        self.src_word_embedding = nn.Embedding(src_vocab_size, embedding_size)
        self.src_position_embedding = nn.Embedding(max_len, embedding_size)
        self.trg_word_embedding = nn.Embedding(trg_vocab_size, embedding_size)
        self.trg_position_embedding = nn.Embedding(max_len, embedding_size)
        self.device = device
        self.Transformer = nn.Transformer(
            embedding_size,
            num_heads,
            num_encoder_layers,
            num_decoder_layers,
            forward_expansion,
            dropout)
        self.fc_out = nn.Linear(embedding_size, trg_vocab_size)
        self.dropout = nn.Dropout(dropout)
        self.src_pad_idx = src_pad_idx

    def make_src_mask(self, src):
        # src shape: (src_len, N)
        src_mask = src.transpose(0, 1) == self.src_pad_idx
        # (N, src_len)
        return src_mask

    def forward(self, src, trg):
        src_seq_length, N = src.shape
        src_seq_length, N = trg.shape

        src_positions = (
            torch.arange(0, src_seq_length).unsqueeze(1).expand(scr_seq_length, N)
            .to(self.device)
        )

        trg_positions = (
            torch.arange(0, trg_seq_length).unsqueeze(1).expand(trg_seq_length, N)
            .to(self.device)
        )

        embed_src = self.dropout(
            (self.src_word_embedding(src) + self.src_position_embedding(src_positions))
        )

        embed_trg = self.dropout(
            (self.trg_word_embedding(trg) + self.trg_position_embedding(trg_positions))
        )

        src_padding_mask = self.make_src_mask(src)
        trg_mask = self.transformer.generate_square_subsequenct_mask(trg_seg_length).to(
            self.device)

        out = self.trnsformer(
            embed_src,
            embed_trg,
            src_key_padding_mask = src_padding_mask,
            tgt_mask = trg_mask
        )
        out = self.fc_out(out)

        return out
# Setup the training phase
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
load_model = False
save_model = True

# Training hyperparameters
num_epochs = 5
learning_rate = 3e-4
batch_size = 32

# Model hyperparameters
src_vocab_size = len(korean.vocab)
trg_vocab_size = len(english.vocab)
embedding_size = 512
num_heads = 8
num_encoder_layers = 3  # in the paper 6
num_decoder_layers = 3
dropout = 0.10
max_len = 100
forward_expansion = 4
src_pad_idx = english.vocab.stoi['<pad>']

# Tensorboard for nice plots
write = SummaryWriter('runs/loss_plot')
step = 0

train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
    (train_data, valid_data, test_data),
    batch_size = batch_size,
    sort_within_batch = True,
    sort_key = lambda x: len(x.src),
    device = device
)

model = Transformer(
    embedding_size,
    src_vocab_size,
    trg_vocab_size,
    src_pad_idx,
    num_heads,
    num_encoder_layers,
    num_decoder_layers,
    forward_expansion,
    dropout,
    max_len,
    device).to(device)

optimizer = optim.Adam(model.parameters(), lr=learning_rate)
pad_idx = english.vocab.stoi['<pad>']
criterion = nn.CrossEntropyLoss(ignore_index = pad_idx)<jupyter_output>/usr/local/lib/python3.6/dist-packages/torchtext/data/iterator.py:48: UserWarning: BucketIterator class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.
  warnings.warn('{} class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.'.format(self.__class__.__name__), UserWarning)<jupyter_text>Test functions<jupyter_code>!pip install --upgrade torchtext
!pip install --upgrade torch
import torch
import spacy
from torchtext.data.metrics import bleu_score
import sys


def translate_sentence(model, sentence, korean, english, device, max_length=50):
    # print(sentence)

    # sys.exit()

    # Load german tokenizer

    # Create tokens using spacy and everything in lower case (which is what our vocab is)
    tokens = mecab.morphs(sentence)

    # print(tokens)

    # sys.exit()
    # Add <SOS> and <EOS> in beginning and end respectively
    tokens.insert(0, korean.init_token)
    tokens.append(korean.eos_token)

    # Go through each korean token and convert to an index
    text_to_indices = [korean.vocab.stoi[token] for token in tokens]

    # Convert to Tensor
    sentence_tensor = torch.LongTensor(text_to_indices).unsqueeze(1).to(device)

    # Build encoder hidden, cell state
    with torch.no_grad():
        hidden, cell = model.encoder(sentence_tensor)

    outputs = [english.vocab.stoi["<sos>"]]

    for _ in range(max_length):
        previous_word = torch.LongTensor([outputs[-1]]).to(device)

        with torch.no_grad():
            output, hidden, cell = model.decoder(previous_word, hidden, cell)
            best_guess = output.argmax(1).item()

        outputs.append(best_guess)

        # Model predicts it's the end of the sentence
        if output.argmax(1).item() == english.vocab.stoi["<eos>"]:
            break

    translated_sentence = [english.vocab.itos[idx] for idx in outputs]

    # remove start token
    return translated_sentence[1:]


def bleu(data, model, korean, english, device):
    targets = []
    outputs = []

    for example in data:
        src = vars(example)["src"]
        trg = vars(example)["trg"]

        prediction = translate_sentence(model, src, korean, english, device)
        prediction = prediction[:-1]  # remove <eos> token

        targets.append([trg])
        outputs.append(prediction)

    return bleu_score(outputs, targets)


def save_checkpoint(state, filename="my_checkpoint.pth.tar"):
    print("=> Saving checkpoint")
    torch.save(state, filename)


def load_checkpoint(checkpoint, model, optimizer):
    print("=> Loading checkpoint")
    model.load_state_dict(checkpoint["state_dict"])
    optimizer.load_state_dict(checkpoint["optimizer"])<jupyter_output><empty_output><jupyter_text>Training the model<jupyter_code>sentence = '이 지역의 많은 공장들이 다른 곳에 외주를 주고 있다.'  # df['kor'][117]

for epoch in range(num_epochs):
    print(f'[Epoch {epoch} / {num_epochs}]')

    if save_model:
        checkpoint = {
            'state_dict' : model.state_dict(),
            'optimizer' : optimizer.state_dict()
            }

        save_checkpoint(checkpoint)

    model.eval()
    translated_sentence = sentence
    # translate_sentence(
    #     model, sentence, korean, english, device, max_length = 100
    # )

    print(f'Translated exsample sentence \n {translated_sentence}')
    model.train()
    print('진행 완료')

    for batch_idx, batch in enumerate(train_iterator):
        print('진행 완료')
        inp_data = batch.src.to(device)
        target = batch.trg.to(device)

        # forward prop
        output = model(input_data, target[:-1])
        out = output.reshape(-1, output.shape[2])
        target = target[1:].reshape(-1)
        optimizer.zero_grad()

        loss = criterion(output, target)
        loss.backward()

        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1)

        optimizer.step()

        writer.add_scalar('Training loss', loss, global_step=step)
        step += 1

score = bleu(test_data, model, korean, english, device)
print(f'Bleu score {score*100:.2f}')
for i, j in enumerate(train_iterator):
    print(i, j)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1400820369940407, 'fraction_numerical': 0.03057038928875474, 'mean_word_length': 3.1310741687979537, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 49, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '542202', 'license': 'MIT', 'n_tokens_mistral': 4805, 'n_tokens_neox': 4516, 'n_words': 861, 'path': '20201016_transformer modeling.ipynb', 'repo_name': 'DojunPark/TIL', 'size': '12925'}