<jupyter_start><jupyter_text>Table of Contents前処理データセット読み込みと文字IDベクトルへの変換ハイパーパラメータ設定とデータ前処理model, optimizer, trainerインスタンス作成学習実行推論結果の保存確信度の取得  seq2seqモデルの学習と結果の保存 ディレクトリ構成：  ```./├dataset/│   └(読み込むCSVファイル)├models/│   └(モデルパラメータキャッシュファイル)├notebooks/│   └train_seq2seq.ipynb(このファイル)├results/│   └(実行結果CSVと学習曲線グラフ画像)└src/   ├common/   │ └(共用クラス、関数)   ├data/   │ └(データ読み書き用コード)   ├attention_layer.py   ├attention_seq2seq.py   ├peeky_seq2seq.py   └seq2seq.py```Cupyを利用するときは、`src/common/config.py`を編集して```pythonGPU = True```とする<jupyter_code># 変数の全消去
%reset
# coding: utf-8
import sys
sys.path.append("../src")
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from pathlib import Path
from datetime import datetime
from data.sequence import TextSequence
from common.optimizer import Adam
from common.trainer import Trainer
from seq2seq import Seq2seq, Seq2seqBiLSTM
from peeky_seq2seq import PeekySeq2seq, PeekySeq2seqBiLSTM
from attention_seq2seq import AttentionSeq2seq, AttentionSeq2seqBiLSTM
# matplotlibのインライン表示
%matplotlib inline
# モジュールの更新時に自動で再読み込み
%reload_ext autoreload
%autoreload 2
# ディレクトリパスの設定
dataset_dir =Path('../dataset')
result_dir = Path('../results')
model_dir = Path('../models')
# ファイルパスの設定
# 最初に全てのデータセットを読み込み、ボキャブラリ辞書を作成する
vocab_csv = [dataset_dir / "translator_gene-con_rev3.csv", dataset_dir / "translator_tori_rev2.csv"]
# source_csv = dataset_dir / "translator_gene-con_rev3.csv"
source_csv = dataset_dir / "translator_tori_rev2.csv"
dataset_name = source_csv.stem
encoding = "utf-8"<jupyter_output><empty_output><jupyter_text>前処理 データセット読み込みと文字IDベクトルへの変換<jupyter_code># 全てのデータセットを読み込み、Vocabularyを作成する
seq = TextSequence()
for path in vocab_csv:
    seq.read_csv(path)
char_to_id, id_to_char = seq.vocab
# 学習対象のデータセットを読み込み
seq.read_csv(source_csv)<jupyter_output><empty_output><jupyter_text>ハイパーパラメータ設定とデータ前処理<jupyter_code># ハイパーパラメータ
vocab_size = len(char_to_id)
wordvec_size = 128
hidden_size = 128
batch_size = 64
max_epoch = 150
max_grad = 5.0
vocab_size
# データセット分割
x_train, x_test, t_train, t_test = seq.split_data(seed=1, test_size=0.3)<jupyter_output><empty_output><jupyter_text>学習実行<jupyter_code># 保存ファイルのファイル名生成
modelname = 'PeekySeq2seq'
timestamp = datetime.now().strftime("_%y%m%d_%H%M")
save_dir = result_dir / ("survey_acc_" + dataset_name + timestamp)
os.makedirs(save_dir, exist_ok=True)
divisor = 5
unit = len(x_train) // divisor
print("Target dataset:", dataset_name)
start_id = seq.start_id
sample_size = seq.sample_size
for i in range(1, divisor+1):
    x_train_part = x_train[:i*unit]
    t_train_part = t_train[:i*unit]
    # Model, Optimizer, Trainerの初期化
    model = PeekySeq2seq(vocab_size, wordvec_size, hidden_size)
    optimizer = Adam(lr=0.002)
    trainer = Trainer(model, optimizer)
    # Train
    trainer.fit(x_train_part, t_train_part, x_test, t_test,
                max_epoch=max_epoch,
                batch_size=batch_size,
                max_grad=max_grad)
    # Inference
    print("Inference:")
    guess_train = model.generate(x_train_part, start_id, sample_size)
    guess_test = model.generate(x_test, start_id, sample_size)
    # Save result as csv
    result_train_csv = save_dir /  ("result_{}per{}_{}_{}_train.csv".format(i, divisor, dataset_name, modelname))
    result_test_csv = save_dir / ("result_{}per{}_{}_{}_test.csv".format(i, divisor, dataset_name, modelname))
    df_result_train = seq.result_to_csv(result_train_csv, x_train_part, t_train_part, guess_train, encoding=encoding)
    df_result_test = seq.result_to_csv(result_test_csv, x_test, t_test, guess_test, encoding=encoding)
    # Save result as csv as utf-8 with BOM
    result_train_csv = save_dir /  ("result_{}per{}_{}_{}_train_bom.csv".format(i, divisor, dataset_name, modelname))
    result_test_csv = save_dir / ("result_{}per{}_{}_{}_test_bom.csv".format(i, divisor, dataset_name, modelname))
    df_result_train = seq.result_to_csv(result_train_csv, x_train_part, t_train_part, guess_train, encoding="utf_8_sig")
    df_result_test = seq.result_to_csv(result_test_csv, x_test, t_test, guess_test, encoding="utf_8_sig")
    # Plot learning curve and save it as png image
    image_path = save_dir / ("result_{}per{}_{}_{}.png".format(i, divisor, dataset_name, modelname))
    trainer.plot(image_path=image_path, log_scale=False)<jupyter_output>Target dataset: translator_tori_rev2
Start training:
model = PeekySeq2seq
optimizer = Adam
batch size = 64
train epochs = 150
| epoch 1 | time/total 0/0[s] | loss = 6.05605
| epoch 1 | time/total 0/0[s] | train error = 1.00000
| epoch 1 | time/total 0/0[s] | test error  = 1.00000
| epoch 2 | time/total 0/0[s] | loss = 6.01042
| epoch 2 | time/total 0/0[s] | train error = 1.00000
| epoch 2 | time/total 0/0[s] | test error  = 1.00000
| epoch 3 | time/total 0/0[s] | loss = 5.94583
| epoch 3 | time/total 0/1[s] | train error = 1.00000
| epoch 3 | time/total 0/1[s] | test error  = 1.00000
| epoch 4 | time/total 0/1[s] | loss = 5.83386
| epoch 4 | time/total 0/1[s] | train error = 1.00000
| epoch 4 | time/total 0/1[s] | test error  = 1.00000
| epoch 5 | time/total 0/1[s] | loss = 5.60856
| epoch 5 | time/total 0/1[s] | train error = 1.00000
| epoch 5 | time/total 0/1[s] | test error  = 1.00000
| epoch 6 | time/total 0/2[s] | loss = 5.12723
| epoch 7 | time/total 0/2[s] | loss = 4.19006
| epo[...]<jupyter_text>ゼネコン向けデータセットの精度予測<jupyter_code>from sklearn.linear_model import LinearRegression
x_g = np.arange(unit, unit*divisor+1, unit).reshape(-1, 1)
y_g = np.array([0.29794, 0.69322, 0.81711, 0.89381, 0.9174])
# def sigmoid(x):
#     return 1 / (1 - np.exp(-x))
model_g = LinearRegression(fit_intercept=True)
x_inv = 1 / x_g
model_g.fit(x_inv, y_g)
model_g.coef_
x_hat = np.arange(100, 2000).reshape(-1, 1)
plt.scatter(x_g, y_g, color='gold', label='gene-con')
plt.plot(x_hat, model_g.predict(1 / x_hat)-0.025, color='gold', linestyle='--')
plt.axhline(y=1, linestyle='--', color='red')
plt.ylim((0.0, 1.05))
plt.xlabel('the number of data')
plt.ylabel('test accuracy')
plt.legend()
plt.savefig(result_dir / "predict_accuracy_gene-con.png")<jupyter_output><empty_output><jupyter_text>トリデータセットの精度予測<jupyter_code>from sklearn.linear_model import LinearRegression
x_t = np.arange(unit, unit*divisor+1, unit).reshape(-1, 1)
y_t = np.array([0.092233, 0.373786, 0.572816, 0.674757, 0.742718])
# def sigmoid(x):
#     return 1 / (1 - np.exp(-x))
model_t = LinearRegression(fit_intercept=True)
x_inv = np.c_[1 / x, np.sqrt(x)]
x_log = np.c_[np.log(x_t), x_t]
model_t.fit(x_log, y_t)
model.coef_
x_hat = np.arange(100, 2000).reshape(-1, 1)
plt.scatter(x_t, y_t, color='orange', label='tori')
plt.plot(x_hat, model_t.predict(np.c_[np.log(x_hat), x_hat])-0.025, color='orange', linestyle='--')
plt.axhline(y=1, linestyle='--', color='red')
plt.ylim((0.0, 1.05))
plt.xlabel('the number of data')
plt.ylabel('test accuracy')
plt.legend()
plt.savefig(result_dir / "predict_accuracy_tori.png")
plt.scatter(x_t, y_t, color='orange')
plt.scatter(x_g, y_g, color='yellow')
plt.plot(x_hat, model_t.predict(np.c_[np.log(x_hat), x_hat])-0.025)
plt.axhline(y=1, linestyle='--', color='red')
plt.ylim((0.0, 1.05))
plt.xlabel('the number of data')
plt.ylabel('test accuracy')
x = np.r_[x_g, x_t]
y = np.r_[y_g, y_t]
x_inverse = 1 / x
model = LinearRegression(fit_intercept=True)
model.fit(x_inverse, y)
model.coef_
model.predict([[1/2000]])
plt.scatter(x_t, y_t, color='orange', label='tori')
plt.scatter(x_g, y_g, color='gold', label='gene-con')
# plt.plot(x_hat, model.predict(1 / x_hat))
plt.plot(x_hat, model_g.predict(1 / x_hat)-0.025, color='gold', linestyle='--')
plt.plot(x_hat, model_t.predict(np.c_[np.log(x_hat), x_hat])-0.025, color='orange', linestyle='--')
plt.axhline(y=1, linestyle='--', color='red')
plt.ylim((0.0, 1.05))
plt.xlabel('the number of data')
plt.ylabel('test accuracy')
plt.legend()
plt.savefig(result_dir / "predict_accuracy.png")
plt.scatter(x_t, y_t, color='orange', label='tori')
plt.scatter(x_g, y_g, color='gold', label='gene-con')
plt.plot(x_hat, model.predict(1 / x_hat), linestyle='--')
# plt.plot(x_hat, model_g.predict(1 / x_hat)-0.025, color='gold', linestyle='--')
# plt.plot(x_hat, model_t.predict(np.c_[np.log(x_hat), x_hat])-0.025, color='orange', linestyle='--')
plt.axhline(y=1, linestyle='--', color='red')
plt.ylim((0.0, 1.05))
plt.xlabel('the number of data')
plt.ylabel('test accuracy')
plt.legend()
plt.savefig(result_dir / "predict_accuracy_total.png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15565031982942432, 'fraction_numerical': 0.04406538734896944, 'mean_word_length': 5.376888217522659, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '203812', 'license': 'MIT', 'n_tokens_mistral': 3985, 'n_tokens_neox': 3735, 'n_words': 633, 'path': 'notebooks/predict_accuracy-depends.ipynb', 'repo_name': 'bbbnodx/seq2seq', 'size': '8442'}