<jupyter_start><jupyter_text>colab 설정<jupyter_code>from google.colab import auth
auth.authenticate_user()

from google.colab import drive
drive.mount('/content/gdrive', force_remount=False)
import os
from pathlib import Path

folder = "colab/kobert_practice" ## 자기 드라이브 경로 입력

base_path = Path("/content/gdrive/My Drive/")
project_path = base_path / folder
os.chdir(project_path)
for x in list(project_path.glob("*")):
    if x.is_dir():
        dir_name = str(x.relative_to(project_path))
        os.rename(dir_name, dir_name.split(" ", 1)[0])
print(f"{os.getcwd()}")<jupyter_output>/content/gdrive/My Drive/colab/kobert_practice<jupyter_text>패키지 설치<jupyter_code>!git clone https://github.com/SKTBrain/KoBERT.git
cd KoBERT
!pip install .
!pip install -r requirements.txt
!pip install mxnet-cu101
!pip install gluonnlp pandas tqdm
!pip install transformers==2.1.1
!pip install torch==1.3.1
cd ..<jupyter_output>/content/gdrive/My Drive/colab/kobert_practice<jupyter_text>패키지 로드, class 정의<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import gluonnlp as nlp
import numpy as np
from tqdm import tqdm, tqdm_notebook
from kobert.pytorch_kobert import get_pytorch_kobert_model
import pandas as pd
import numpy as np
from kobert.utils import get_tokenizer
from transformers import AdamW
from transformers.optimization import WarmupLinearSchedule
import random
device = 'cuda' if torch.cuda.is_available() else 'cpu'
bertmodel, vocab = get_pytorch_kobert_model()
tokenizer = get_tokenizer()
tok = nlp.data.BERTSPTokenizer(tokenizer, vocab, lower=False)
train_data = pd.read_csv('data/ChatbotData .csv')
class ChatDataset(Dataset):
  def __init__(self, dataset, q_idx, a_idx, bert_tokenizer, max_len, pad, pair):
    transform = nlp.data.BERTSentenceTransform(
        bert_tokenizer, max_seq_length=max_len, pad=pad, pair=pair)
    
    self.question = [transform([dataset.loc[i][q_idx]]) for i in range(len(dataset))]
    self.answer = [transform([dataset.loc[i][a_idx]]) for i in range(len(dataset))]

  def __getitem__(self, i):
    return (self.question[i] + (self.answer[i][0],self.answer[i][1]))
  
  def __len__(self):
    return (len(self.answer))
## 하이퍼 파라미터 설정
max_len = 100
batch_size = 32
warmup_ratio = 0.1
num_epochs = 160
num_workers = 0
max_grad_norm = 1
learning_rate =  5e-5
print_every = 150
save_every = 20
embed_dim = 100 
vocab_size = len(vocab.idx_to_token)
train_dataset = ChatDataset(train_data, "Q", "A", tok, max_len, True, False)
train_loader = DataLoader(train_dataset, batch_size=batch_size, num_workers=num_workers, shuffle=True)
class Encoder(nn.Module):
  def __init__(self, bert):
    super(Encoder, self).__init__()
    self.bert = bert
    
  def gen_attention_mask(self, token_ids, valid_length):
    ## masked attenion, 패딩에 패널티 부여, 학습 x
    attention_mask = torch.zeros_like(token_ids)
    for i, v in enumerate(valid_length):
        attention_mask[i][:v] = 1
    return attention_mask.float()
  
  def forward(self, token_ids, valid_length, segment_ids):
    attention_mask = self.gen_attention_mask(token_ids, valid_length)

    output, hidden = self.bert(input_ids = token_ids, token_type_ids = segment_ids.long(), attention_mask = attention_mask.float().to(token_ids.device))

    return output, hidden
class Attention(nn.Module):
  def __init__(self,
               enc_hid_dim,
               dec_hid_dim,
               attn_dim):
    super().__init__()

    self.enc_hid_dim = enc_hid_dim
    self.dec_hid_dim = dec_hid_dim
    self.attn_in = enc_hid_dim + dec_hid_dim
    self.attn = nn.Linear(self.attn_in, attn_dim)
  
  def forward(self, decoder_hidden, encoder_outputs):
    src_len = encoder_outputs.shape[1]

    repeated_decoder_hidden = decoder_hidden.unsqueeze(1).repeat(1, src_len, 1)

    energy = torch.tanh(self.attn(torch.cat((
        repeated_decoder_hidden,
        encoder_outputs),
        dim = 2)))

    attention = torch.sum(energy, dim=2)

    return F.softmax(attention, dim=1) 

class Decoder(nn.Module):
  def __init__(self,
               enc_hid_dim,
               hidden_dim,
               emb_dim,
               vocab_size,
               attention,
               max_length,
               dropout_rate = 0.1):
    super(Decoder, self).__init__()

    self.hidden_dim = hidden_dim
    self.dropout_rate = dropout_rate
    self.max_length = max_length
    self.vocab_size = vocab_size
    self.emb_dim = emb_dim
    self.enc_hid_dim = enc_hid_dim

    self.attention = attention

    self.embedding = nn.Embedding(self.vocab_size, self.emb_dim)
    self.gru = nn.GRU(self.enc_hid_dim + self.emb_dim, self.hidden_dim, batch_first=True)
    self.out = nn.Linear(self.attention.attn_in, self.vocab_size)
  
  def _weighted_encoder_rep(self,decoder_hidden, encoder_outputs):
    a = self.attention(decoder_hidden, encoder_outputs)

    a = a.unsqueeze(1)

    weighted_encoder_rep = torch.bmm(a, encoder_outputs)

    return weighted_encoder_rep

  def forward(self, input, decoder_hidden, encoder_outputs):
    
    input = input.unsqueeze(1).long()
    embedded = self.embedding(input)
    weighted_encoder_rep = self._weighted_encoder_rep(decoder_hidden, encoder_outputs)
 
    input = torch.cat((embedded, weighted_encoder_rep), dim=2)
    
    output, decoder_hidden = self.gru(input, decoder_hidden.unsqueeze(0))

    embedded = embedded.squeeze(1)
    output = output.squeeze(1)
    weighted_encoder_rep = weighted_encoder_rep.squeeze(1)

    output = self.out(torch.cat((output, weighted_encoder_rep), dim = 1))

    return output, decoder_hidden.squeeze(0)

class Seq2Seq(nn.Module):
  def __init__(self, encoder, decoder, device):
    super().__init__()
    self.encoder = encoder
    self.decoder = decoder
    self.device = device

    self.vocab_size = self.decoder.vocab_size

  def forward(self, src, valid_length, segment_ids, trg, teacher_forcing_ratio = 0.5):
    batch_size = src.shape[0]
    max_len = trg.shape[1]

    encoder_outputs, hidden = self.encoder(src, valid_length, segment_ids)

    outputs = torch.zeros(max_len, batch_size, self.vocab_size).to(self.device)

    output = trg[:,0].long()

    for t in range(1, max_len):
      output, hidden = self.decoder(output, hidden, encoder_outputs)
      outputs[t] = output
      teacher_force = random.random() < teacher_forcing_ratio
      top1 = output.max(1)[1]
      output = (trg[:,t] if teacher_force else top1)
    
    return outputs.permute(1,2,0)
## 모델 초기화
encoder = Encoder(bertmodel)
attention = Attention(768, 768, 100)
decoder = Decoder(768, 768, embed_dim, vocab_size, attention, max_len)
model = Seq2Seq(encoder, decoder, device).to(device)
# Prepare optimizer and schedule (linear warmup and decay)
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
    {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
    {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
## 손실함수, optimizer 초기화
optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)
criterion = nn.CrossEntropyLoss(ignore_index=0).to(device)
## linear scheduler 초기화
t_total = len(train_loader) * num_epochs
warmup_step = int(t_total * warmup_ratio)
scheduler = WarmupLinearSchedule(optimizer, warmup_steps=warmup_step, t_total=t_total)
def save_checkpoint(epoch, model, optimizer, scheduler, loss_list, PATH):
  torch.save({
            'epoch': epoch,
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
            'scheduler_state_dict': scheduler.state_dict(),
            'loss_list': loss_list
            }, PATH)
def translate_example(inputs, input_length, outputs, label, label_length):
  _, topi = outputs.topk(1)
  for x in [1,2,3,4,5]:
    print("Example #"+str(x))
    s = ""
    for y in inputs[x][:input_length[x]]:
      s += vocab.idx_to_token[y.item()]
    print("input :", s)
    s = ""
    for y in topi[x]:
      s += vocab.idx_to_token[y.item()]
      # if y.item() in [1,3]:
      #   break
    print("pred:", s)
    s = ""
    for y in label[x][:label_length[x]]:
      s += vocab.idx_to_token[y.item()]
    print("label:", s)<jupyter_output><empty_output><jupyter_text>모델 학습, 불러오기, 결과<jupyter_code>save_path = '/content/gdrive/My Drive/colab/kobert_practice/model/checkpoint_2_sample.tar'
checkpoint = torch.load(save_path)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
loss_list = []
print('Training..')
## checkpoint 불러와서 학습시킬 경우 range 범위 바꿔주기
for epoch in range(0, num_epochs):

    model.train()

    for i, (token_ids, valid_length, segment_ids, label, label_length) in enumerate(tqdm_notebook(train_loader)):
        optimizer.zero_grad()

        token_ids, valid_length, segment_ids, label = token_ids.long().to(device), valid_length.to(device), segment_ids.long().to(device), label.long().to(device)

        outputs = model(token_ids, valid_length, segment_ids, label)
  
        loss = criterion(outputs, label)

        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)
        optimizer.step() 
        scheduler.step()

        ## accuracy 계산
        _, topi = outputs.permute(0,2,1).topk(1)
        topi = topi.squeeze()
        accuracy = (label == topi).float().mean()

        if (i+1) % print_every == 0: ## 상태 출력
            print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, Accuracy: {:.2f}%'.format(
                epoch+1, num_epochs, i+1, len(train_loader), loss.item(), accuracy.item() * 100))
            loss_list.append(loss.item())
            translate_example(token_ids, valid_length, outputs.permute(0,2,1), label, label_length)

        
    ## checkpoint 저장
    if (epoch+1) % save_every == 0:
       if not os.path.exists("model"):
         os.makedirs("model")
       save_checkpoint(epoch+1, model, optimizer, scheduler, loss_list, "model/checkpoint_2_"+str(epoch+1)+".tar")
       print("checkpoint saved!")
# for i, (a,b,c,d, e) in enumerate(train_loader):
#   break
# model.eval()
# seq = model(a.long().to(device),b.to(device),c.long().to(device))
# translate_example(a, b, seq, d, e)
def translate(s):
  
  transform = nlp.data.BERTSentenceTransform(tok, max_seq_length=max_len, pad=True, pair=False)
  token, length, segment = transform([s])
  token = torch.from_numpy(token).to(device)
  length = torch.from_numpy(length).to(device)
  segment = torch.from_numpy(segment).to(device)
  model.eval()
  token = token.view(1,-1)
  length = length.view(1, -1)
  segment = segment.view(1, -1)
  label = torch.zeros(1, max_len).to(device)
  label.fill_(2)
  seq = model(token.long(), length, segment.long(), label, teacher_forcing_ratio=0)
  seq = seq.permute(0,2,1)
  _, topi = seq.topk(1)
  answer = ""
  for x in topi[0]:
    answer += vocab.idx_to_token[x.item()]
    if x.item() in [3]:
        break

  return answer
translate("넌 뭐야?")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.127588989509549, 'fraction_numerical': 0.012194028512507846, 'mean_word_length': 3.959537572254335, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '465812', 'license': 'Apache-2.0', 'n_tokens_mistral': 4172, 'n_tokens_neox': 4116, 'n_words': 798, 'path': 'ybigta_chatbot.ipynb', 'repo_name': 'ksyu0508/ybigta_chatbot', 'size': '11153'}