<jupyter_start><jupyter_text>Section 1. 개발 테스트 환경 구축  구글 드라이브의 데이터와 모델을 불러올 수 있도록 연동하는 과정<jupyter_code>#!pip install pytorch-transformers # For Colab 
import pandas as pd
from google.colab import drive
drive.mount('/content/drive')
!ls drive/'My Drive'/'Colab Notebooks'
PATH = "drive/My Drive/Colab Notebooks/amazon_cells_labelled.txt"<jupyter_output><empty_output><jupyter_text>Section 2. 알고리즘 구동을 위한 라이브러리 설치<jupyter_code>!pip install transformers
from transformers import *
#from google.colab import files
#uploaded = files.upload()
import torch
# from pytorch_transformers import XLNetTokenizer,XLNetForSequenceClassification
# from pytorch_transformers import XLNetTokenizer
from transformers import XLNetForSequenceClassification
from sklearn.model_selection import train_test_split
#from pytorch_transformers import AdamW
import matplotlib.pyplot as plt
from keras.preprocessing.sequence import pad_sequences
from torch.utils.data import TensorDataset,DataLoader,RandomSampler,SequentialSampler<jupyter_output><empty_output><jupyter_text>Dataset 불러오기 및 샘플 데이터 확인<jupyter_code>fd = pd.read_csv(PATH,sep='\t')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
fd.head()<jupyter_output><empty_output><jupyter_text>데이터셋 전처리<jupyter_code>fd.columns = ['sentence','value']
fd.head()
sentences  = []
for sentence in fd['sentence']:
  sentence = sentence+"[SEP] [CLS]"
  sentences.append(sentence)
sentences[0:3]<jupyter_output><empty_output><jupyter_text>Section 3. 학습 및 분류(prediction) 정확도 출력  1. XLNet 사전학습 모델을 기본으로한 tokenizer를 이용하여 텍스트 정보를 token으로 전환한다. 이때 XLNet’s vocabulary 셋트를 이용하였다.  2. 정수로 조합된 token의 정보들이 인덱스 정보로 저장되어 모델 학습 및 테스트용으로 사용된다. Load pre-trained model tokenizer<jupyter_code>tokenizer  = XLNetTokenizer.from_pretrained('xlnet-base-cased',do_lower_case=True) #xlnet-base-cased
tokenized_text = [tokenizer.tokenize(sent) for sent in sentences]<jupyter_output><empty_output><jupyter_text>sample text<jupyter_code>tokenized_text[100]
ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_text]
print(ids[100])
labels = fd['value'].values
print(labels[100])<jupyter_output>[8689, 39, 3676, 19, 44, 121, 13179, 356, 203, 151, 18, 8976, 9, 10849, 23, 3882, 3158, 4145, 11974, 23, 3158]
0<jupyter_text>Token 길이 통일을 위한 Padding - 가장 긴 길이의 token을 기준으로 나머지 token의 길이를 맞춰줌<jupyter_code>max1 = len(ids[0])
for i in ids:
  if(len(i)>max1):
    max1=len(i)
print(max1)
MAX_LEN = max1<jupyter_output>54<jupyter_text>패딩(padding) 수행<jupyter_code>#input_ids2 = pad_sequences(ids,maxlen=MAX_LEN,dtype="long",truncating="post",padding="post")
import numpy as np
input_ids2 = np.load('drive/My Drive/Colab Notebooks/ids_93.npy')<jupyter_output><empty_output><jupyter_text>학습과 테스트를 위한 데이터 분류작업을 진행한다.<jupyter_code>#xtrain,xtest,ytrain,ytest = train_test_split(input_ids2,labels,test_size=0.15)
xtrain = np.load('drive/My Drive/Colab Notebooks/xtrain_95_bs3_ep7.npy')
ytrain = np.load('drive/My Drive/Colab Notebooks/ytrain_95_bs3_ep7.npy')
xtest = np.load('drive/My Drive/Colab Notebooks/xtest_95_bs3_ep7.npy')
ytest = np.load('drive/My Drive/Colab Notebooks/ytest_95_bs3_ep7.npy')
ytest[0:10]
# np.save('drive/My Drive/Colab Notebooks/ids_93.npy', input_ids2) # x_save.npy
# 동일한 index를 이용하여 다시 테스트하기 위한 index저장 코드
PATH2 = "drive/My Drive/Colab Notebooks/model/XLnet-based-Custom-96"
Xtrain = torch.tensor(xtrain)
Ytrain = torch.tensor(ytrain)
Xtest = torch.tensor(xtest)
Ytest = torch.tensor(ytest)
# np.save('drive/My Drive/Colab Notebooks/xtrain_93_bs3_ep7.npy', xtrain) # x_save.npy
# np.save('drive/My Drive/Colab Notebooks/ytrain_93_bs3_ep7.npy', ytrain) # x_save.npy
# np.save('drive/My Drive/Colab Notebooks/ytest_93_bs3_ep7.npy', ytest) # x_save.npy
# np.save('drive/My Drive/Colab Notebooks/xtest_93_bs3_ep7.npy', xtest) # x_save.npy<jupyter_output><empty_output><jupyter_text>사전 학습 모델 load 및 학습 설정 및 학습진행 한번의 iteration에 들어갈 데이터의 양(bs)과, 학습할 레이어의 개수와 얼마나 학습할 지(epoch)를 선택한다. batch_size :(default) 3<jupyter_code>batch_size = 5 #3
no_train = 0
epochs = 5 # 3
train_data = TensorDataset(Xtrain,Ytrain)
test_data = TensorDataset(Xtest,Ytest)
loader = DataLoader(train_data,batch_size=batch_size)
test_loader = DataLoader(test_data,batch_size=batch_size)<jupyter_output><empty_output><jupyter_text>사전 학습된 모델로 성능 테스트 진행<jupyter_code>def flat_accuracy(preds,labels):  # A function to predict Accuracy
  correct=0
  for i in range(0,len(labels)):
    if(preds[i]==labels[i]):
      correct+=1
  return (correct/len(labels))*100
model = XLNetForSequenceClassification.from_pretrained("xlnet-base-cased",num_labels=2) # xlnet-base(large)-cased
torch.cuda.empty_cache()
model.cuda()
optimizer = AdamW(model.parameters(),lr=2e-5) # initial learning rate 0.00002

checkpoint = torch.load(PATH2)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
loss = checkpoint['loss']

model.eval()
acc = []
lab = []
t = 0
for inp,lab1 in test_loader:
  inp.to(device)
  lab1.to(device)
  t+=lab1.size(0)
  outp1 = model(inp.to(device))
  [acc.append(p1.item()) for p1 in torch.argmax(outp1[0],axis=1).flatten() ]
  [lab.append(z1.item()) for z1 in lab1]
print("Total Examples : {} Accuracy {}".format(t,flat_accuracy(acc,lab)))<jupyter_output>/usr/local/lib/python3.6/dist-packages/transformers/configuration_xlnet.py:211: FutureWarning: This config doesn't use attention memories, a core feature of XLNet. Consider setting `men_len` to a non-zero value, for example `xlnet = XLNetLMHeadModel.from_pretrained('xlnet-base-cased'', mem_len=1024)`, for accurate training performance as well as an order of magnitude faster inference. Starting from version 3.5.0, the default parameter will be 1024, following the implementation in https://arxiv.org/abs/1906.08237
  FutureWarning,
Some weights of the model checkpoint at xlnet-base-cased were not used when initializing XLNetForSequenceClassification: ['lm_loss.weight', 'lm_loss.bias']
- This IS expected if you are initializing XLNetForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing XLNetForSeque[...]<jupyter_text>Section 4. 학습과정 증명 자료학습 모델을 다운로드 하고, 여기에 parameter를 설정하여 학습과정을 증명한다. 모델 초기화<jupyter_code>model = XLNetForSequenceClassification.from_pretrained("xlnet-base-cased",num_labels=2) # xlnet-base(large)-cased
torch.cuda.empty_cache()
model.cuda()
optimizer = AdamW(model.parameters(),lr=2e-5) # initial learning rate 0.00002
import torch.nn as nn
criterion = nn.CrossEntropyLoss()
# def flat_accuracy(preds,labels):  # A function to predict Accuracy
#   correct=0
#   for i in range(0,len(labels)):
#     if(preds[i]==labels[i]):
#       correct+=1
#   return (correct/len(labels))*100<jupyter_output><empty_output><jupyter_text>Transfer learning을 시작한다.<jupyter_code>for epoch in range(epochs):
  model.train()
  loss1 = []
  steps = 0
  train_loss = []
  l = []
  for inputs,labels1 in loader :
    inputs.to(device)
    labels1.to(device)
    optimizer.zero_grad()
    outputs = model(inputs.to(device))
    loss = criterion(outputs[0],labels1.to(device)).to(device)
    logits = outputs[1]
    #ll=outp(loss)
    [train_loss.append(p.item()) for p in torch.argmax(outputs[0],axis=1).flatten() ]#our predicted 
    [l.append(z.item()) for z in labels1]# real labels
    loss.backward()
    optimizer.step()
    loss1.append(loss.item())
    no_train += inputs.size(0)
    steps += 1
  print("Current Loss is : {} Step is : {} number of Example : {} Accuracy : {}".format(loss.item(),epoch,no_train,flat_accuracy(train_loss,l)))<jupyter_output>Current Loss is : 0.6339799165725708 Step is : 0 number of Example : 849 Accuracy : 51.9434628975265
Current Loss is : 0.8297128677368164 Step is : 1 number of Example : 1698 Accuracy : 86.45465253239105
Current Loss is : 0.020124394446611404 Step is : 2 number of Example : 2547 Accuracy : 91.51943462897526
Current Loss is : 0.007085510529577732 Step is : 3 number of Example : 3396 Accuracy : 97.1731448763251
Current Loss is : 0.005526750348508358 Step is : 4 number of Example : 4245 Accuracy : 98.70435806831567<jupyter_text>테스트 결과<jupyter_code>model.eval()#Testing our Model
acc = []
lab = []
t = 0
for inp,lab1 in test_loader:
  inp.to(device)
  lab1.to(device)
  t+=lab1.size(0)
  outp1 = model(inp.to(device))
  [acc.append(p1.item()) for p1 in torch.argmax(outp1[0],axis=1).flatten() ]
  [lab.append(z1.item()) for z1 in lab1]
print("Total Examples : {} Accuracy {}".format(t,flat_accuracy(acc,lab)))

#!pip freeze > 'drive/My Drive/Colab Notebooks/requirement.txt'<jupyter_output><empty_output><jupyter_text>테스트 결과 모델 저장<jupyter_code>PATH2 = "drive/My Drive/Colab Notebooks/model/XLnet-based-Custom-96"
# torch.save(model.state_dict(), PATH2)
# torch.save({
#             'epoch': epoch,
#             'model_state_dict': model.state_dict(),
#             'optimizer_state_dict': optimizer.state_dict(),
#             'loss': loss,
#             }, PATH2)<jupyter_output><empty_output><jupyter_text>학습된 모델 재구동 테스트<jupyter_code># model = TheModelClass(*args, **kwargs)
# optimizer = TheOptimizerClass(*args, **kwargs)

checkpoint = torch.load(PATH2)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
loss = checkpoint['loss']

model.eval()
acc = []
lab = []
t = 0
for inp,lab1 in test_loader:
  inp.to(device)
  lab1.to(device)
  t+=lab1.size(0)
  outp1 = model(inp.to(device))
  [acc.append(p1.item()) for p1 in torch.argmax(outp1[0],axis=1).flatten() ]
  [lab.append(z1.item()) for z1 in lab1]
print("Total Examples : {} Accuracy {}".format(t,flat_accuracy(acc,lab)))<jupyter_output>Total Examples : 150 Accuracy 96.66666666666667
{'fraction_non_alphanumeric': 0.12653724972050004, 'fraction_numerical': 0.04858217298505946, 'mean_word_length': 5.905263157894737, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 62, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '288181', 'license': 'MIT', 'n_tokens_mistral': 4272, 'n_tokens_neox': 4219, 'n_words': 775, 'path': 'test/XLnet_sentiment.ipynb', 'repo_name': 'sbseong/sbseong.github.io', 'size': '9839'}