<jupyter_start><jupyter_text>kcbert Fine-tune<jupyter_code>import pandas as pd
import torch
from torch.nn import functional as F
from torch.utils.data import DataLoader, Dataset
from transformers import AutoTokenizer, ElectraForSequenceClassification, AdamW, BertForSequenceClassification
from tqdm.notebook import tqdm
if torch.cuda.is_available():    
    device = torch.device("cuda")

    print('There are %d GPU(s) available.' % torch.cuda.device_count())
    print('We will use the GPU:', torch.cuda.get_device_name(0))

else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")<jupyter_output>There are 1 GPU(s) available.
We will use the GPU: GeForce RTX 2070<jupyter_text>data load - fine-tuning 할 데이터를 불러옵니다.- 네이버 쇼핑 리뷰 데이터    - 언어: 한국어    - 출처: 네이버 쇼핑 (https://shopping.naver.com/)    - 수집 기간: 2020.06~2020.07    - 데이터 건수: 20만 건    - 데이터 출처: (https://github.com/bab2min/corpus/tree/master/sentiment)<jupyter_code>import os 
import csv

file_path = os.getenv('HOME') + '/Projects/crawling_data'
ns_data = pd.read_csv(file_path + '/naver_shopping.txt', delimiter = '\t', names =['label', 'text'])
len(ns_data)
ns_data.head()<jupyter_output><empty_output><jupyter_text>- label 5, 4 = 긍정    -> 1- label 1, 2 = 부정    -> 0<jupyter_code>ns_data['label'] = ns_data['label'].apply(lambda row : 1 if row == 5 or row == 4 else 0)
ns_data.head()
ns_data.iloc[4, :].values
ns_data.label.unique()
class createDataset(Dataset):
  
    def __init__(self, csv_file):
        # 일부 값중에 NaN이 있음...
        self.dataset = csv_file.dropna(axis=0) 
        # 중복제거
        self.dataset.drop_duplicates(subset=['text'], inplace=True)
        self.tokenizer = AutoTokenizer.from_pretrained("beomi/kcbert-base")

        print(self.dataset.describe())
  
    def __len__(self):
        return len(self.dataset)
  
    def __getitem__(self, idx):
        row = self.dataset.iloc[idx, :].values
        y = row[0]
        text = row[1]

        inputs = self.tokenizer(
            text, 
            return_tensors='pt',
            truncation=True,
            max_length=64,
            pad_to_max_length=True,
            add_special_tokens=True
            )

        input_ids = inputs['input_ids'][0]
        attention_mask = inputs['attention_mask'][0]

        return input_ids, attention_mask, y
train_dataset = createDataset(ns_data)
model = BertForSequenceClassification.from_pretrained(
    "beomi/kcbert-base", num_labels = 3).to(device)
# 모델 레이어 보기
model
epochs = 5
batch_size = 64
optimizer = AdamW(model.parameters(), lr=1e-5)
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
losses = []
accuracies = []

for i in range(epochs):
    total_loss = 0.0
    correct = 0
    total = 0
    batches = 0

    model.train()

    for input_ids_batch, attention_masks_batch, y_batch in tqdm(train_loader):
        optimizer.zero_grad()
        y_batch = y_batch.to(device)
        y_pred = model(input_ids_batch.to(device), attention_mask=attention_masks_batch.to(device))[0]
        loss = F.cross_entropy(y_pred, y_batch)
        loss.backward()
        optimizer.step()

        total_loss += loss.item()

        _, predicted = torch.max(y_pred, 1)
        correct += (predicted == y_batch).sum()
        total += len(y_batch)

        batches += 1
        if batches % 100 == 0:
            print("Batch Loss:", total_loss, "Accuracy:", correct.float() / total)
  
    losses.append(total_loss)
    accuracies.append(correct.float() / total)
    print("Train Loss:", total_loss, "Accuracy:", correct.float() / total)
    acc = correct.float() / total
    
    # 모델 저장하기
    torch.save(model.state_dict(), file_path + f"/model_{i}_{acc}acc.pt")
losses, accuracies
os.listdir(file_path)
pwd<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12794701986754967, 'fraction_numerical': 0.015099337748344372, 'mean_word_length': 3.1955555555555555, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '254492', 'license': 'MIT', 'n_tokens_mistral': 1430, 'n_tokens_neox': 1424, 'n_words': 277, 'path': 'kcbertFinetuning.ipynb', 'repo_name': 'vg-rlo/SentimentAnalysis', 'size': '3775'}