<jupyter_start><jupyter_text>[View in Colaboratory](https://colab.research.google.com/github/sleepless-se/Abstract/blob/master/stock_filter_018.ipynb) バリエーションだけを抽出する<jupyter_code>!apt-get install mecab
!apt-get install libmecab-dev
!apt-get install mecab-ipadic-utf8
!pip install mecab-python3
import sys
import MeCab
import pickle
import os
import sys
import MeCab
import pickle
from keras.datasets import boston_housing
from keras.models import Model,model_from_json
from keras.callbacks import ModelCheckpoint
from keras.utils import to_categorical
from keras.preprocessing.text import Tokenizer
from sklearn.model_selection import train_test_split
import keras
from keras.utils import to_categorical
from keras.datasets import mnist
from keras import models
from keras import layers
import time
import numpy as np
from google.colab import files
import re
from sklearn.model_selection import train_test_split
from keras.callbacks import EarlyStopping
from keras.metrics import categorical_accuracy
import pickle

    
mecab = MeCab.Tagger ("-Owakati")<jupyter_output>Using TensorFlow backend.<jupyter_text>Tokenizerの読み書き用関数<jupyter_code>def save_text_tokenizer(tokenizer,file_name):
  # saving
  with open(file_name+".pickle", 'wb') as handle:
      pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)

def load_text_tokenizer(file_name):
  # loading
  with open(file_name+".pickle", 'rb') as handle:
      return pickle.load(handle)<jupyter_output><empty_output><jupyter_text>ドライブをマウント<jupyter_code># Load the Drive helper and mount
from google.colab import drive

# This will prompt for authorization.
drive.mount('/content/gdrive')

# After executing the cell above, Drive
# files will be present in "/content/drive/My Drive".
base_dir = "/content/gdrive/My Drive/data/stock_filter/018/"
!ls "/content/gdrive/My Drive/data/stock_filter/018/"<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code

Enter your authorization code:
··········
Mounted at /content/gdrive
alt_token.pickle    model.json	     none_label.csv	 vari_label.csv
href_token.pickle   none_label0.csv  src_token.pickle	 weights.best.hdf5
iid_token.pickle    none_label2.csv  style_token.pickle
label_token.pickle  none_label3.csv  tag_token.pickle
model.h5	    none_label4.csv  text_token.pickle<jupyter_text>学習データの読み込み<jupyter_code>import pandas as pd

# dtype ={"x":int,"p_x":int,"pp_x":int,"y":int,"p_y":int,"pp_y":int,"width":int,"p_width":int,"pp_width":int,"height":int,"p_height":int,"pp_height":int,"iid": str,"style": str, "text": str, "p_text": str} 
dtype ={"style": str, "text": str, "p_text": str} 
file_name = 'vari_label.csv'
stock_df = pd.read_csv(base_dir+file_name,sep=',',error_bad_lines=False, dtype=dtype,engine='python')
# print(stock_df.head(10))
print("vari shape",stock_df.shape)

file_name = 'none_label5.csv'
null_df = pd.read_csv(base_dir+file_name,sep=',',error_bad_lines=False, dtype=dtype,engine='python')
null_df["label"] = "skip"
print("null shape",null_df.shape)

df = stock_df.append(null_df, ignore_index=True)
df.columns
df.shape
# print(df.head(10))<jupyter_output>vari shape (1489, 31)
null shape (3000, 31)<jupyter_text>分かち書きをしたりするテキストの前処理<jupyter_code>def to_df(token,se):
  matrix = token.texts_to_matrix(se,mode='count')
  temp_df = pd.DataFrame(matrix)
  return temp_df

def add_space_front_uppercase(text:str)->str:
  return ''.join(' ' + char if char.isupper() else char.strip() for char in text).strip()

def add_space_color_size(text:str)->str:
  return text.replace("colour","color").replace("color"," color ").replace("size"," size ")

def wakati(text)->str:
  text = str(text)
  if len(text)>3:
    text = mecab.parse(text)
  return replace_numbers_to_0(text)

def replace_numbers_to_0(text:str)->str:
  return re.sub(r'[0-9]+', "0",str(text))

def pre_text(df):
  wakachi_fileds = ['alt', 'href', 'iid','src', 'style', 'tag', 'text',
                    'p_iid', 'p_style', 'p_tag','p_text', 'pp_iid', 'pp_style', 'pp_tag']
  df[wakachi_fileds] = df[wakachi_fileds].applymap(lambda x : wakati(x)).fillna("")

  add_space_fileds = ["iid","style","p_iid","p_style","pp_iid","pp_style"]
  df[add_space_fileds] = df[add_space_fileds].applymap(lambda x : add_space_front_uppercase(x))
  df[add_space_fileds] = df[add_space_fileds].applymap(lambda x : add_space_color_size(x))
  return df                

df = pre_text(df)
df.dtypes<jupyter_output><empty_output><jupyter_text>学習データ全体の前処理<jupyter_code>def prepare_data(data):
  df = pd.DataFrame(data)
  df = pre_text(df)

  fields = ['height', 'width', 'x', 'y',  'p_x', 'p_y', 'p_width','p_height','pp_height', 'pp_width', 'pp_x', 'pp_y']
  X = df[fields]

  X = pd.concat([X, to_df(tag_token,df["tag"])], axis=1)
  X = pd.concat([X, to_df(tag_token,df["p_tag"])], axis=1)
  X = pd.concat([X, to_df(tag_token,df["pp_tag"])], axis=1)
  X = pd.concat([X, to_df(style_token,df["style"])], axis=1)
  X = pd.concat([X, to_df(style_token,df["p_style"])], axis=1)
  X = pd.concat([X, to_df(style_token,df["pp_style"])], axis=1)
  X = pd.concat([X, to_df(iid_token,df["iid"])], axis=1)
  X = pd.concat([X, to_df(iid_token,df["p_iid"])], axis=1)
  X = pd.concat([X, to_df(iid_token,df["pp_iid"])], axis=1)
  X = pd.concat([X, to_df(alt_token,df["alt"])], axis=1)
  X = pd.concat([X, to_df(src_token,df["src"])], axis=1)
  X = pd.concat([X, to_df(href_token,df["href"])], axis=1)
  X = pd.concat([X, to_df(text_token,df["text"])], axis=1)
  X = pd.concat([X, to_df(text_token,df["p_text"])], axis=1)
  X = X.fillna(0)
  return X<jupyter_output><empty_output><jupyter_text>tokenの作成<jupyter_code>temp_df = df["tag"].append(df["p_tag"]).append(df["pp_tag"])
tag_token = Tokenizer(lower=True)
tag_token.fit_on_texts(temp_df)

temp_df = df["iid"].append(df["p_iid"]).append(df["pp_iid"])
iid_token = Tokenizer(100,lower=True)
iid_token.fit_on_texts(temp_df)

temp_df = df["style"].append(df["p_style"]).append(df["pp_style"])
style_token = Tokenizer(100,lower=True)
style_token.fit_on_texts(temp_df)

alt_token = Tokenizer(100,lower=True)
alt_token.fit_on_texts(df["alt"])
src_token = Tokenizer(100,lower=True)
src_token.fit_on_texts(df["src"])
href_token = Tokenizer(100,lower=True)
href_token.fit_on_texts(df["href"])

temp_df = df["text"].append(df["p_text"])
text_token = Tokenizer(500,lower=True)
text_token.fit_on_texts(df["text"])<jupyter_output><empty_output><jupyter_text>学習データを作成<jupyter_code>X = prepare_data(df)
print(X.shape)<jupyter_output>(4489, 2050)
height       int64
width        int64
x            int64
y            int64
p_x          int64
p_y          int64
p_width      int64
p_height     int64
pp_height    int64
pp_width     int64
pp_x         int64
pp_y         int64
dtype: object<jupyter_text>ラベルを作成<jupyter_code>label_token = Tokenizer()
label_token.fit_on_texts(df["label"])
save_text_tokenizer(label_token,base_dir+"label_token")
y = label_token.texts_to_matrix(df["label"])
print(y.shape)<jupyter_output>(4489, 3)<jupyter_text>データ(型)を確認<jupyter_code>print(X[["height","width","x","y","p_x","p_y","p_width","p_height","pp_height","pp_width","pp_x","pp_y"]].dtypes)<jupyter_output>height       int64
width        int64
x            int64
y            int64
p_x          int64
p_y          int64
p_width      int64
p_height     int64
pp_height    int64
pp_width     int64
pp_x         int64
pp_y         int64
dtype: object<jupyter_text>ラベルの変換用ディクショナリの作成<jupyter_code>label_dic= {v: k for k, v in label_token.word_index.items()}
print(label_dic)<jupyter_output>{1: 'skip', 2: 'vari'}<jupyter_text>学習データをシャッフル<jupyter_code>seed = 7
np.random.seed(seed)
X_train, X_test, y_train, y_test  = train_test_split(X, y, test_size=0.33, random_state=seed)<jupyter_output><empty_output><jupyter_text>コールバックの設定<jupyter_code># stop when score is not improve
early_stopping = EarlyStopping(monitor='val_loss', patience=3)

# save best score
filepath = base_dir+"weights.best.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_categorical_accuracy', verbose=1, save_best_only=True, mode='max')
callbacks_list = [checkpoint]<jupyter_output><empty_output><jupyter_text>学習開始<jupyter_code>EPOCHS = 100
BATCH_SIZE = 275


model = models.Sequential()
model.add(layers.Dense(1024, activation='relu',
                       input_shape=(X_train.shape[1],)))
model.add(layers.Dropout(0.3))

model.add(layers.Dense(160, activation='relu'))
model.add(layers.Dropout(0.3))


model.add(layers.Dense(1024, activation='relu'))
model.add(layers.Dropout(0.3))

model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dropout(0.3))


model.add(layers.Dense(1024, activation='relu'))
model.add(layers.Dropout(0.3))

model.add(layers.Dense(100, activation='relu'))
model.add(layers.Dropout(0.3))

model.add(layers.Dense(y_train.shape[1],activation='softmax'))

model.compile(loss='binary_crossentropy', optimizer='Adam', metrics=['categorical_accuracy'])



history = model.fit(X_train, y_train,verbose=1, validation_data=(X_test,y_test), epochs=EPOCHS, batch_size=BATCH_SIZE,callbacks=callbacks_list)
score = model.evaluate(X_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])


import matplotlib.pyplot as plt

print(history.history.keys())
# summarize history for loss
plt.plot(history.history['val_loss'])
plt.plot(history.history['loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['val_loss', 'loss'], loc='upper left')
plt.show()

plt.plot(history.history['val_categorical_accuracy'])
plt.plot(history.history['categorical_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['val_acc', 'acc'], loc='upper left')
plt.show()<jupyter_output>Train on 3007 samples, validate on 1482 samples
Epoch 1/100
3007/3007 [==============================] - 2s 792us/step - loss: 0.9204 - categorical_accuracy: 0.7083 - val_loss: 0.2821 - val_categorical_accuracy: 0.8927

Epoch 00001: val_categorical_accuracy improved from -inf to 0.89271, saving model to /content/gdrive/My Drive/data/stock_filter/018/weights.best.hdf5
Epoch 2/100
3007/3007 [==============================] - 2s 582us/step - loss: 0.3320 - categorical_accuracy: 0.8766 - val_loss: 0.2097 - val_categorical_accuracy: 0.8974

Epoch 00002: val_categorical_accuracy improved from 0.89271 to 0.89744, saving model to /content/gdrive/My Drive/data/stock_filter/018/weights.best.hdf5
Epoch 3/100
3007/3007 [==============================] - 2s 582us/step - loss: 0.2531 - categorical_accuracy: 0.9162 - val_loss: 0.1699 - val_categorical_accuracy: 0.9480

Epoch 00003: val_categorical_accuracy improved from 0.89744 to 0.94804, saving model to /content/gdrive/My Drive/data/stock_filter/01[...]<jupyter_text>```model = models.Sequential()model.add(layers.Dense(1024, activation='relu',                       input_shape=(X_train.shape[1],)))model.add(layers.Dropout(0.3))model.add(layers.Dense(160, activation='relu'))model.add(layers.Dropout(0.3))model.add(layers.Dense(1024, activation='relu'))model.add(layers.Dropout(0.3))model.add(layers.Dense(64, activation='relu'))model.add(layers.Dropout(0.3))model.add(layers.Dense(1024, activation='relu'))model.add(layers.Dropout(0.3))model.add(layers.Dense(100, activation='relu'))model.add(layers.Dropout(0.3))model.add(layers.Dense(y_train.shape[1],activation='softmax'))```Test accuracy: 0.9949977262391997  tokenの保存<jupyter_code>save_text_tokenizer(tag_token,base_dir+"tag_token")
save_text_tokenizer(iid_token,base_dir+"iid_token")
save_text_tokenizer(style_token,base_dir+"style_token")
save_text_tokenizer(href_token,base_dir+"href_token")
save_text_tokenizer(src_token,base_dir+"src_token")
save_text_tokenizer(alt_token,base_dir+"alt_token")
save_text_tokenizer(text_token,base_dir+"text_token")<jupyter_output><empty_output><jupyter_text>Tokenisor を読み込む<jupyter_code>tag_token = load_text_tokenizer(base_dir+"tag_token")
iid_token = load_text_tokenizer(base_dir+"iid_token")
style_token = load_text_tokenizer(base_dir+"style_token")
href_token = load_text_tokenizer(base_dir+"href_token")
src_token = load_text_tokenizer(base_dir+"src_token")
alt_token = load_text_tokenizer(base_dir+"alt_token")
text_token = load_text_tokenizer(base_dir+"text_token")<jupyter_output><empty_output><jupyter_text>モデルと重みを保存する モデルを保存<jupyter_code>model_json = model.to_json()
with open(base_dir + "model.json", "w") as json_file:
    json_file.write(model_json)
    
with open(base_dir + "model.h5",'w') as model_file: 
  model.save_weights(base_dir+"model.h5")<jupyter_output><empty_output><jupyter_text>モデルを読み込み<jupyter_code># load
with open(base_dir+'model.json', 'r') as json_file:
  loaded_model_json = json_file.read()
  loaded_model = model_from_json(loaded_model_json)

with open(base_dir+"model.h5",'r') as model_file: 
  loaded_model.load_weights(base_dir+"model.h5")
  print("Loaded model from disk")

# compile
loaded_model.compile(loss='binary_crossentropy', optimizer='Adam', metrics=['categorical_accuracy'])<jupyter_output><empty_output><jupyter_text>モデルの復元が正確かを確認<jupyter_code>score = loaded_model.evaluate(X_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])<jupyter_output><empty_output><jupyter_text>実際のデータでテスト<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15705269338030237, 'fraction_numerical': 0.03493321591075884, 'mean_word_length': 7.130668257756564, 'pattern_counts': {'":': 19, '<': 73, '<?xml version=': 0, '>': 78, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '68077', 'license': 'Apache-2.0', 'n_tokens_mistral': 5714, 'n_tokens_neox': 5415, 'n_words': 761, 'path': 'stock_filter_018.ipynb', 'repo_name': 'sleepless-se/Abstract', 'size': '13626'}