<jupyter_start><jupyter_text>NarrativeKoGPT2 학습  1.Google Drive 연동- 모델 파일과 학습 데이터가 저장 되어있는 구글 드라이브의 디렉토리와 Colab을 연동.  - 좌측상단 메뉴에서 런타임-> 런타임 유형 변경 -> 하드웨어 가속기 -> GPU 선택 후 저장  1.1 GPU 연동 확인<jupyter_code>!nvidia-smi<jupyter_output>Fri Mar 20 10:06:07 2020       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 440.64.00    Driver Version: 418.67       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |
| N/A   38C    P0    27W / 250W |      0MiB / 16280MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU  [...]<jupyter_text>1.2 Google Drive 연동아래 코드를 실행후 나오는 URL을 클릭하여 나오는 인증 코드 입력<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
!ls<jupyter_output><empty_output><jupyter_text>**Colab 디렉토리 아래 NarrativeKoGPT2 경로 확인**<jupyter_code>!ls drive/'My Drive'/'Colab Notebooks'/<jupyter_output>BERT_X				  KorQuAD-beginner   NarrativeKoGPT2.ipynb
'Copy of NarrativeKoGPT2.ipynb'   NarrativeKoGPT2<jupyter_text>**필요 패키지들 설치**<jupyter_code>!pip install -r drive/'My Drive'/'Colab Notebooks'/NarrativeKoGPT2/requirements.txt
import os

import sys
sys.path.append('drive/My Drive/Colab Notebooks/')
print(os.getcwd())<jupyter_output>/content<jupyter_text>2.KoGPT2 Transfer Learning  2.1.Import Package<jupyter_code>import random
import torch
from torch.utils.data import DataLoader # 데이터로더
from gluonnlp.data import SentencepieceTokenizer 
from NarrativeKoGPT2.kogpt2.utils import get_tokenizer
from NarrativeKoGPT2.kogpt2.utils import download, tokenizer
from NarrativeKoGPT2.model.torch_gpt2 import GPT2Config, GPT2LMHeadModel
from NarrativeKoGPT2.util.data import NovelDataset
import gluonnlp
from tqdm import tqdm<jupyter_output><empty_output><jupyter_text>**torch GPU 확인**<jupyter_code>print(torch.cuda.device_count())  # GPU deviec count check<jupyter_output>1<jupyter_text>2.2. koGPT-2 Config<jupyter_code>ctx= 'cuda'#'cuda' #'cpu' #학습 Device CPU or GPU. colab의 경우 GPU 사용
cachedir='~/kogpt2/' # KoGPT-2 모델 다운로드 경로
epoch =200  # 학습 epoch
save_path = 'drive/My Drive/Colab Notebooks/NarrativeKoGPT2/checkpoint/'
#use_cuda = True # Colab내 GPU 사용을 위한 값

pytorch_kogpt2 = {
    'url':
    'https://kobert.blob.core.windows.net/models/kogpt2/pytorch/pytorch_kogpt2_676e9bcfa7.params',
    'fname': 'pytorch_kogpt2_676e9bcfa7.params',
    'chksum': '676e9bcfa7'
}
kogpt2_config = {
    "initializer_range": 0.02,
    "layer_norm_epsilon": 1e-05,
    "n_ctx": 1024,
    "n_embd": 768,
    "n_head": 12,
    "n_layer": 12,
    "n_positions": 1024,
    "vocab_size": 50000
}<jupyter_output><empty_output><jupyter_text>2.3 Model and Vocab Download<jupyter_code># download model
model_info = pytorch_kogpt2
model_path = download(model_info['url'],
                       model_info['fname'],
                       model_info['chksum'],
                       cachedir=cachedir)
# download vocab
vocab_info = tokenizer
vocab_path = download(vocab_info['url'],
                       vocab_info['fname'],
                       vocab_info['chksum'],
                       cachedir=cachedir)<jupyter_output>[██████████████████████████████████████████████████]
[██████████████████████████████████████████████████]<jupyter_text>2.4.KoGPT-2 Model Vocab<jupyter_code># KoGPT-2 언어 모델 학습을 위한 GPT2LMHeadModel 선언
kogpt2model = GPT2LMHeadModel(config=GPT2Config.from_dict(kogpt2_config))
# model_path로부터 다운로드 받은 내용을 load_state_dict으로 업로드
kogpt2model.load_state_dict(torch.load(model_path))

device = torch.device(ctx)
kogpt2model.to(device)

# kogpt2model.eval()
# 추가로 학습하기 위해 .train() 사용
kogpt2model.train()
vocab_b_obj = gluonnlp.vocab.BERTVocab.from_sentencepiece(vocab_path,
                                                     mask_token=None,
                                                     sep_token=None,
                                                     cls_token=None,
                                                     unknown_token='<unk>',
                                                     padding_token='<pad>',
                                                     bos_token='<s>',
                                                     eos_token='</s>')<jupyter_output><empty_output><jupyter_text>2.5. Get Batch Data using DataLoader<jupyter_code>tok_path = get_tokenizer()
model, vocab = kogpt2model, vocab_b_obj
sentencepieceTokenizer = SentencepieceTokenizer(tok_path)

#os.chdir("../")
data_file_path = 'drive/My Drive/Colab Notebooks/NarrativeKoGPT2/data/backmyo_novel_1/untokenized_bm_data.txt'
batch_size = 2
novel_dataset = NovelDataset(data_file_path, vocab,sentencepieceTokenizer)
novel_data_loader = DataLoader(novel_dataset, batch_size=batch_size, shuffle=True, pin_memory=True)<jupyter_output>using cached model
(905,)<jupyter_text>2.6. Learning rate, Loss function, Adam Optimizer<jupyter_code>learning_rate = 1e-5
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

import subprocess

def get_gpu_memory_map():
    """Get the current gpu usage.

    Returns
    -------
    usage: dict
        Keys are device ids as integers.
        Values are memory usage as integers in MB.
    """
    result = subprocess.check_output(
        [
            'nvidia-smi', '--query-gpu=memory.used',
            '--format=csv,nounits,noheader'
        ], encoding='utf-8')
    # Convert lines into a dictionary
    gpu_memory = [int(x) for x in result.strip().split('\n')]
    gpu_memory_map = dict(zip(range(len(gpu_memory)), gpu_memory))
    return gpu_memory_map<jupyter_output><empty_output><jupyter_text>2.7. KoGPT-2 Transfer Laerning<jupyter_code>torch.cuda.

print('KoGPT-2 Transfer Learning Start')
epoch=200
for epoch in range(epoch):
  count = 0
  for data in novel_data_loader:
    optimizer.zero_grad()
    data = torch.stack(data) # list of Tensor로 구성되어 있기 때문에 list를 stack을 통해 변환해준다.

    data= data.transpose(1,0)
    data= data.to(ctx)
    
    outputs = model(data, labels=data)
    loss, logits = outputs[:2]
    loss.backward()
    optimizer.step()
    if count %10 ==0:
      print('epoch no.{} train no.{}  loss = {}' . format(epoch, count+1, loss))
      # torch.save(model,save_path+'checkpoint_{}_{}.tar'.format(epoch,count))
      # 추론 및 학습 재개를 위한 일반 체크포인트 저장하기
    if (count >0 and count%100==0) or (len(data) < batch_size):
      torch.save({
        'epoch': epoch,
        'train_no': count,
        'model_state_dict': model.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
        'loss':loss
      }, save_path+'narrativeKoGPT2_checkpoint.tar')

    count += 1<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17168092740822521, 'fraction_numerical': 0.02649737786364891, 'mean_word_length': 2.502658289028516, 'pattern_counts': {'":': 8, '<': 48, '<?xml version=': 0, '>': 51, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '134390', 'license': 'MIT', 'n_tokens_mistral': 2844, 'n_tokens_neox': 2835, 'n_words': 466, 'path': 'NarrativeKoGPT2.ipynb', 'repo_name': 'shbictai/narrativeKoGPT2', 'size': '7246'}