<jupyter_start><jupyter_text>Data Load<jupyter_code>import pandas as pd

# 수정된 파일
path = '/content/lyrics_droped.csv'
df = pd.read_csv(path, encoding='utf-8')
df.head()

# 각 훈련마다 해당 세대만 사용할 경우 gen column n세대만 사용
# df1 = df[df['gen'] == '1세대']
# df1.head()
len(df) # 8000개는 되어야 제대로 훈련할 수 있을듯; 2000은 너무 적다
# 세대별 토큰 넣어주기

df['<'] = '<'
df['>'] = '> '
df['lyrics'] = df['<'] + df['gen'] + df['>'] + df['lyrics']
df.head()<jupyter_output><empty_output><jupyter_text>Tokenizer<jupyter_code># !pip install transformers
from transformers import BertTokenizerFast#, GPT2LMHeadModel
# 스폐셜 토큰 설정 안하고 하는 것의 차이?
gens=["<1세대>", "<2세대>", "<3세대>"] # 세대를 설정안하면 < 도 다 학습해버림.. 그래서 오류 난듯

tokenizer_gpt3 = BertTokenizerFast.from_pretrained("kykim/gpt3-kor-small_based_on_gpt2", 
                                                   bos_token='<|startoftext|>', 
                                                   eos_token='<|endoftext|>', 
                                                   pad_token='<|pad|>',
                                                   additional_special_tokens=gens)
 
tokenizer_gpt3.add_tokens("<br>")

# input_ids = tokenizer_gpt3.encode("text to tokenize")[1:]  # remove cls token
print("The max model length is {} for this model, although the actual embedding size for GPT small is 768".format(tokenizer_gpt3.model_max_length))
print("The beginning of sequence token {} token has the id {}".format(tokenizer_gpt3.convert_ids_to_tokens(tokenizer_gpt3.bos_token_id), tokenizer_gpt3.bos_token_id))
print("The end of sequence token {} has the id {}".format(tokenizer_gpt3.convert_ids_to_tokens(tokenizer_gpt3.eos_token_id), tokenizer_gpt3.eos_token_id))
print("The padding token {} has the id {}".format(tokenizer_gpt3.convert_ids_to_tokens(tokenizer_gpt3.pad_token_id), tokenizer_gpt3.pad_token_id))<jupyter_output>The max model length is 2048 for this model, although the actual embedding size for GPT small is 768
The beginning of sequence token <|startoftext|> token has the id 42000
The end of sequence token <|endoftext|> has the id 42001
The padding token <|pad|> has the id 42002<jupyter_text>Pytorch Dataset & Dataloaders 1. br은 그대로 (or not)2. 세대 토큰 추가할 것 - 모든 세대 대상 다 돌릴 것이므로<jupyter_code>import re
def preprocess(text):
  # <br> -> 양 옆에 스페이스 추가 
  text = re.sub(r"<br>", " <br> ", text)

  # 한영, <br>, 숫자 제외 특수부호 제거 
  text = re.sub(r"[^ <>0-9a-zA-Zㄱ-힗]+", "", text)

  # 영어는 lowercase로
  text = text.lower()

  return text
# 예시 문장
ex = preprocess(df['lyrics'][0])
print(ex)
print(tokenizer_gpt3.encode('<|startoftext|>'+ex, 
      truncation=True, max_length=768, padding="max_length")[1:]) #이런 식으로!!
tok_ex = tokenizer_gpt3(ex)
print(tok_ex)
print(tok_ex.keys())
print(len(tok_ex['input_ids']))
print(len(tok_ex['token_type_ids']))
print(len(tok_ex['attention_mask']))
len(tokenizer_gpt3.vocab)
random.randint(1,3)
f"<{random.randint(1,3)}세대> " + get_key(random.randint(1, 42000)) 
def get_key(val):
    for key, value in tokenizer_gpt3.vocab.items():
         if val == value:
             return key
             
# 예시로 뽑은 것들이 잘 되었는지 확인!
print(get_key(42000))
print(get_key(14567))
print(get_key(2))
print(get_key(20958))<jupyter_output><|startoftext|>
새로운
[CLS]
br<jupyter_text>* 토치 데이터셋 커스텀<jupyter_code>import torch
from torch.utils.data import Dataset, DataLoader, random_split, RandomSampler, SequentialSampler
torch.manual_seed(42)

from transformers import GPT2LMHeadModel, GPT2Config
from transformers import AdamW, get_linear_schedule_with_warmup
class GPT2Dataset(Dataset):

  def __init__(self, txt_list, tokenizer, gpt2_type="gpt2", max_length=768):

    self.tokenizer = tokenizer
    self.input_ids = []
    self.attn_masks = []

    for txt in txt_list:

      encodings_dict = tokenizer('<|startoftext|>'+ txt + '<|endoftext|>', 
                                        truncation=True, max_length=max_length, 
                                        padding="max_length") 

      self.input_ids.append(torch.tensor(encodings_dict['input_ids'][1:])) # cls 제거
      self.attn_masks.append(torch.tensor(encodings_dict['attention_mask'][1:])) # cls 제거
    
  def __len__(self):
    return len(self.input_ids)

  def __getitem__(self, idx):
    return self.input_ids[idx], self.attn_masks[idx] 
# 전처리한 가사 리스트
lyrics_list = df['lyrics'].apply(preprocess)

# 파이토치 데이터셋 생성
dataset = GPT2Dataset(lyrics_list, tokenizer_gpt3, max_length=768)

# Split into training and validation sets
train_size = int(0.9 * len(dataset))
val_size = len(dataset) - train_size

train_dataset, val_dataset = random_split(dataset, [train_size, val_size])

print('{:>5,} training samples'.format(train_size))
print('{:>5,} validation samples'.format(val_size))
# 토치 데이터 로더
batch_size = 2
# GPT2 is a large model. Increasing the batch size above 2 has lead to out of memory problems. 
# This can be mitigated by accumulating the gradients but that is out of scope here.

# Create the DataLoaders for our training and validation datasets.
# We'll take training samples in random order. 
train_dataloader = DataLoader(
            train_dataset,  # The training samples.
            sampler = RandomSampler(train_dataset), # Select batches randomly
            batch_size = batch_size # Trains with this batch size.
        )

# For validation the order doesn't matter, so we'll just read them sequentially.
validation_dataloader = DataLoader(
            val_dataset, # The validation samples.
            sampler = SequentialSampler(val_dataset), # Pull out batches sequentially.
            batch_size = batch_size # Evaluate with this batch size.
        )<jupyter_output><empty_output><jupyter_text>Fine-tuning<jupyter_code># # I'm not really doing anything with the config buheret
configuration = GPT2Config.from_pretrained('kykim/gpt3-kor-small_based_on_gpt2', output_hidden_states=False)

# # instantiate the model
model = GPT2LMHeadModel.from_pretrained("kykim/gpt3-kor-small_based_on_gpt2")
import seaborn as sns
import numpy as np
import random
import os

# this step is necessary because I've added some tokens (bos_token, etc) to the embeddings
# otherwise the tokenizer and model tensors won't match up
model.resize_token_embeddings(len(tokenizer_gpt3))

# Tell pytorch to run this model on the GPU.
device = torch.device("cuda")
model.cuda()

# Set the seed value all over the place to make this reproducible.
seed_val = 42

random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)
# some parameters I cooked up that work reasonably well

epochs = 5
learning_rate = 5e-4
warmup_steps = 1e2
epsilon = 1e-8

# this produces sample output every 100 steps
sample_every = 100
# Note: AdamW is a class from the huggingface library (as opposed to pytorch) 
optimizer = AdamW(model.parameters(),
                  lr = learning_rate,
                  eps = epsilon
                )
# Total number of training steps is [number of batches] x [number of epochs]. 
# (Note that this is not the same as the number of training samples).
total_steps = len(train_dataloader) * epochs

# Create the learning rate scheduler.
# This changes the learning rate as the training loop progresses
scheduler = get_linear_schedule_with_warmup(optimizer, 
                                            num_warmup_steps = warmup_steps, 
                                            num_training_steps = total_steps)
import time
import datetime

def format_time(elapsed):
    return str(datetime.timedelta(seconds=int(round((elapsed)))))
total_t0 = time.time()

training_stats = []

model = model.to(device)

for epoch_i in range(0, epochs):

    # ========================================
    #               Training
    # ========================================

    print("")
    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
    print('Training...')

    t0 = time.time()

    total_train_loss = 0

    model.train()

    for step, batch in enumerate(train_dataloader):

        b_input_ids = batch[0].to(device)
        b_labels = batch[0].to(device)
        b_masks = batch[1].to(device)

        model.zero_grad()        

        outputs = model(  b_input_ids,
                          labels=b_labels, 
                          attention_mask = b_masks,
                          token_type_ids=None 
                        )

        loss = outputs[0]  

        batch_loss = loss.item()
        total_train_loss += batch_loss

        # Get sample every x batches.
        if step % sample_every == 0 and not step == 0:

            elapsed = format_time(time.time() - t0)
            print('  Batch {:>5,}  of  {:>5,}. Loss: {:>5,}.   Elapsed: {:}.'.format(step, len(train_dataloader), batch_loss, elapsed))

            model.eval()

            sample_outputs = model.generate( # 세대 토큰 넣어서 해봄
                                    bos_token_id=random.randint(1,30000),
                                    do_sample=True,   
                                    top_k=50, 
                                    max_length = 200,
                                    top_p=0.95, 
                                    num_return_sequences=1,
                                    repetition_penalty=2.0,
                                
                                )
            for i, sample_output in enumerate(sample_outputs):
                  print("{}: {}".format(i, tokenizer_gpt3.decode(sample_output, skip_special_tokens=True)))
            
            model.train()

        loss.backward()

        optimizer.step()

        scheduler.step()

    # Calculate the average loss over all of the batches.
    avg_train_loss = total_train_loss / len(train_dataloader)       
    
    # Measure how long this epoch took.
    training_time = format_time(time.time() - t0)

    print("")
    print("  Average training loss: {0:.2f}".format(avg_train_loss))
    print("  Training epoch took: {:}".format(training_time))
        
    # ========================================
    #               Validation
    # ========================================

    print("")
    print("Running Validation...")

    t0 = time.time()

    model.eval()

    total_eval_loss = 0
    nb_eval_steps = 0

    # Evaluate data for one epoch
    for batch in validation_dataloader:
        
        b_input_ids = batch[0].to(device)
        b_labels = batch[0].to(device)
        b_masks = batch[1].to(device)
        
        with torch.no_grad():        

            outputs  = model(b_input_ids, 
#                            token_type_ids=None, 
                             attention_mask = b_masks,
                            labels=b_labels)
          
            loss = outputs[0]  
            
        batch_loss = loss.item()
        total_eval_loss += batch_loss        

    avg_val_loss = total_eval_loss / len(validation_dataloader)
    
    validation_time = format_time(time.time() - t0)    

    print("  Validation Loss: {0:.2f}".format(avg_val_loss))
    print("  Validation took: {:}".format(validation_time))

    # Record all statistics from this epoch.
    training_stats.append(
        {
            'epoch': epoch_i + 1,
            'Training Loss': avg_train_loss,
            'Valid. Loss': avg_val_loss,
            'Training Time': training_time,
            'Validation Time': validation_time
        }
    )

print("")
print("Training complete!")
print("Total training took {:} (h:mm:ss)".format(format_time(time.time()-total_t0)))
# Display floats with two decimal places.
pd.set_option('precision', 2)

# Create a DataFrame from our training statistics.
df_stats = pd.DataFrame(data=training_stats)

# Use the 'epoch' as the row index.
df_stats = df_stats.set_index('epoch')

# A hack to force the column headers to wrap.
#df = df.style.set_table_styles([dict(selector="th",props=[('max-width', '70px')])])

# Display the table.
df_stats
import matplotlib.pyplot as plt

# Use plot styling from seaborn.
sns.set(style='darkgrid')

# Increase the plot size and font size.
sns.set(font_scale=1.5)
plt.rcParams["figure.figsize"] = (12,6)

# Plot the learning curve.
plt.plot(df_stats['Training Loss'], 'b-o', label="Training")
plt.plot(df_stats['Valid. Loss'], 'g-o', label="Validation")

# Label the plot.
plt.title("Training & Validation Loss")
plt.xlabel("Epoch")
plt.ylabel("Loss")
plt.legend()
plt.xticks([1, 2, 3, 4])

plt.show()
# 파라미터 표시
params = list(model.named_parameters())

print('The GPT-2 model has {:} different named parameters.\n'.format(len(params)))

print('==== Embedding Layer ====\n')

for p in params[0:2]:
    print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))

print('\n==== First Transformer ====\n')

for p in params[2:14]:
    print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))

print('\n==== Output Layer ====\n')

for p in params[-2:]:
    print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size()))))
# 모델 저장

output_dir = '/content/drive/MyDrive/model_save'

# 디렉토리 만들기
if not os.path.exists(output_dir):
    os.makedirs(output_dir)

print("Saving model to %s" % output_dir)

# 모델, 토크나이저 등 저장
model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
model_to_save.save_pretrained(output_dir)
tokenizer_gpt3.save_pretrained(output_dir)
!ls -l --block-size=K /content/drive/MyDrive/model_save
!ls -l --block-size=M /content/drive/MyDrive/model_save/pytorch_model.bin
# !cp -r /content/drive/MyDrive/model_save/ $data_dir
# 구글 드라이브에서 모델, 토크나이저 불러오기

model = GPT2LMHeadModel.from_pretrained('/content/drive/MyDrive/model_save')
# gpt3-kor-small_based_on_gpt2
from transformers import BertTokenizerFast, GPT2LMHeadModel
tokenizer = BertTokenizerFast.from_pretrained('/content/drive/MyDrive/model_save')
input_ids = tokenizer.encode("text to tokenize")[1:]  # remove cls token<jupyter_output><empty_output><jupyter_text>Generate Text * generate 함수<jupyter_code>device = torch.device("cuda")
model.cuda()

# Set the seed value all over the place to make this reproducible.
seed_val = 42

random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)
model = model.to(device)
model.eval()

prompt = "<|startoftext|> <2세대> 배고프다"
generated = torch.tensor(tokenizer_gpt3.encode(prompt)[1:]).unsqueeze(0)
generated = generated.to(device)

print(generated)

sample_outputs = model.generate(
                                generated, 
                                #bos_token_id=random.randint(1,30000),
                                do_sample=True,   
                                top_k=50, 
                                max_length = 300,
                                top_p=0.95, 
                                num_return_sequences=3,
                                repetition_penalty=1.2
                                )

for i, sample_output in enumerate(sample_outputs):
  print("{}: {}\n\n".format(i, tokenizer_gpt3.decode(sample_output, skip_special_tokens=True)))<jupyter_output>Setting `pad_token_id` to `eos_token_id`:3 for open-end generation.<jupyter_text>1세대 테스트<jupyter_code>prompt = "<|startoftext|> <1세대> 배고프다 "
generated = torch.tensor(tokenizer_gpt3.encode(prompt)[1:]).unsqueeze(0)
generated = generated.to(device)

print(generated)

sample_outputs = model.generate(
                                generated, 
                                #bos_token_id=random.randint(1,30000),
                                do_sample=True,   
                                top_k=50, 
                                max_length = 300,
                                top_p=0.95, 
                                num_return_sequences=3,
                                repetition_penalty=1.2
                                )

for i, sample_output in enumerate(sample_outputs):
  result = "{}: {}\n\n".format(i, tokenizer_gpt3.decode(sample_output, skip_special_tokens=True))
  result = re.sub(r"<br>", "\n", result)
  print(result)<jupyter_output>Setting `pad_token_id` to `eos_token_id`:3 for open-end generation.<jupyter_text>2세대 테스트<jupyter_code>prompt = "<|startoftext|> <2세대> 배고프다 "
generated = torch.tensor(tokenizer_gpt3.encode(prompt)[1:]).unsqueeze(0)
generated = generated.to(device)

print(generated)

sample_outputs = model.generate(
                                generated, 
                                #bos_token_id=random.randint(1,30000),
                                do_sample=True,   
                                top_k=50, 
                                max_length = 300,
                                top_p=0.95, 
                                num_return_sequences=3,
                                repetition_penalty=1.2
                                )

for i, sample_output in enumerate(sample_outputs):
  result = "{}: {}\n\n".format(i, tokenizer_gpt3.decode(sample_output, skip_special_tokens=True))
  result = re.sub(r"<br>", "\n", result)
  print(result)<jupyter_output>Setting `pad_token_id` to `eos_token_id`:3 for open-end generation.<jupyter_text>3세대 테스트<jupyter_code>prompt = "<|startoftext|> <3세대> 배고프다 "
generated = torch.tensor(tokenizer_gpt3.encode(prompt)[1:]).unsqueeze(0)
generated = generated.to(device)

print(generated)

sample_outputs = model.generate(
                                generated, 
                                #bos_token_id=random.randint(1,30000),
                                do_sample=True,   
                                top_k=50, 
                                max_length = 300,
                                top_p=0.95, 
                                num_return_sequences=3,
                                repetition_penalty=1
                                )

for i, sample_output in enumerate(sample_outputs):
  result = "{}: {}\n\n".format(i, tokenizer_gpt3.decode(sample_output, skip_special_tokens=True))
  result = re.sub(r"<br>", "\n", result)
  print(result)<jupyter_output>Setting `pad_token_id` to `eos_token_id`:3 for open-end generation.<jupyter_text>생성기<jupyter_code>def lyric_generator(gen, lyric):
  prompt = f"<|startoftext|> <{gen}세대> {lyric}"
  generated = torch.tensor(tokenizer_gpt3.encode(prompt)[1:]).unsqueeze(0)
  generated = generated.to(device)

  print(generated)

  sample_outputs = model.generate(
                                  generated, 
                                  #bos_token_id=random.randint(1,30000),
                                  do_sample=True,   
                                  top_k=50, 
                                  max_length = 300,
                                  top_p=0.95, 
                                  num_return_sequences=3,
                                  repetition_penalty=1.1
                                  )

  for i, sample_output in enumerate(sample_outputs):
    result = "{}: {}\n\n".format(i, tokenizer_gpt3.decode(sample_output, skip_special_tokens=True))
    result = re.sub(r"<br>", "\n", result)
    print(result)
  return
lyric_generator(2, '배고프다')<jupyter_output>Setting `pad_token_id` to `eos_token_id`:3 for open-end generation.
{'fraction_non_alphanumeric': 0.12406608439440177, 'fraction_numerical': 0.018520467220877617, 'mean_word_length': 2.532899628252788, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 29, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '423464', 'license': 'MIT', 'n_tokens_mistral': 6680, 'n_tokens_neox': 6536, 'n_words': 1306, 'path': '2. Model/Lyrics_Generation_Model.ipynb', 'repo_name': 'chloesung/Song-Lyrics-Generator', 'size': '19006'}