<jupyter_start><jupyter_text>필요한 필수 새팅 작업<jupyter_code>!ls
!pip install -r drive/'My Drive'/'KoGPT2-FineTuning_pre'/requirements.txt
import os
import sys
sys.path.append('drive/My Drive/KoGPT2-FineTuning_pre')
logs_base_dir = "runs"
from jupyter_main_auto import main
ctx= 'cuda'
cachedir='~/kogpt2/'
load_path = './gdrive/My Drive/KoGPT2-FineTuning_pre/checkpoint/KoGPT2_checkpoint_640000.tar' # 이어서 학습시킬 모델 경로
save_path = './gdrive/My Drive/KoGPT2-FineTuning_pre/checkpoint/' # 학습한 모델을 저장시킬 경로
data_file_path = './gdrive/My Drive/KoGPT2-FineTuning_pre/dataset/dataset.csv' # 학습할 데이터셋 경로<jupyter_output><empty_output><jupyter_text>모델 학습 시작<jupyter_code># 저장 잘 되는지 테스트
drive.mount('/content/gdrive')

f = open(save_path+ 'KoGPT2_checkpoint_' + str(142) + '.tar', 'w')
f.write("가자")
f.close()
main(load_path = load_path, data_file_path = data_file_path, save_path = './gdrive/My Drive/KoGPT2-FineTuning_pre/checkpoint/', summary_url = './gdrive/My Drive/KoGPT2-FineTuning_pre/runs/2020-07-20/', text_size = 500, new = 1, batch_size = 1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14460093896713616, 'fraction_numerical': 0.03004694835680751, 'mean_word_length': 6.0131578947368425, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '668468', 'license': 'Apache-2.0', 'n_tokens_mistral': 536, 'n_tokens_neox': 531, 'n_words': 75, 'path': 'Colab.ipynb', 'repo_name': 'andrewlaikhT/KoGPT2', 'size': '1065'}