<jupyter_start><jupyter_text>0. import 모음<jupyter_code>from google.colab import drive
import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split
from tensorflow import keras
from keras.layers import LSTM
from keras.models import Sequential
from keras.layers import Dense
import keras.backend as K
from keras.callbacks import EarlyStopping
from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score<jupyter_output><empty_output><jupyter_text>1. 구글마운트<jupyter_code># drive.mount('/gdrive')<jupyter_output>Drive already mounted at /gdrive; to attempt to forcibly remount, call drive.mount("/gdrive", force_remount=True).<jupyter_text>2. 데이터셋 만들기  데이터 위치로 이동<jupyter_code># %cd '/gdrive/MyDrive/EM_LSTM/data'<jupyter_output>/gdrive/MyDrive/EM_LSTM/data<jupyter_text>file list 가져오기<jupyter_code>path_dir = './'
file_list = os.listdir(path_dir)<jupyter_output><empty_output><jupyter_text>file 불러오기<jupyter_code>em_x = []
em_y = []
for i in file_list :
  tempstr = './'+i
  em_x.append(np.array(pd.read_csv(tempstr)))
  #파일을 불러오면서 맨 앞 글자를 가져와 y 로 씀
  em_y.append(i[0])<jupyter_output><empty_output><jupyter_text>- 파일 갯수 확인<jupyter_code>print(len(em_x), len(em_y))
# 데이터를 다시 불러와야 하거나 로우데이터로 뭔가 확인을 해야할 수 있으므로 em_x 는 그대로 두고 em_x2 생성
# 하는 김에 np.array로 해서 shape 도 봄.
em_x2 = np.array(em_x)
em_x2[0].shape<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
  This is separate from the ipykernel package so we can avoid doing imports until<jupyter_text>y 값 인코딩<jupyter_code>reshape_ha = list(em_y)
enc = {'a': 0, 'b': 1, 'c': 2}
lst_e = [enc[i] for i in reshape_ha]<jupyter_output><empty_output><jupyter_text>- em_x 와 마찬가지로 em_y 보존을 위해 em_y2 생성<jupyter_code>em_y2 = lst_e<jupyter_output><empty_output><jupyter_text>y 값 원핫 인코딩<jupyter_code>ohe=np.eye( len(enc.keys() ))
ohe
temp_elst = [ohe[i] for i in em_y2]
em_y2 = np.array(temp_elst)<jupyter_output><empty_output><jupyter_text>EDA  학습형태로 만들어주기- 100개 씩 끊어서 LSTM 에서 학습할 수 있게 만들어 줌.<jupyter_code># 앞에서 끊어주기
# for 문을 통해 frames 단위로 학습을 하는 부분임.
npx = []
npy = []
frames = 100
for fileidx in range(0,len(em_x2)):
  # 파일당 최대 갯수를 지정(C 데이터가 158 이 max 인 파일이 있어서 150을 기준으로 잡음.)
  endpoint = 150
  for i in range(0,endpoint,1):
    # 100단위로 쪼개서 하기 위해 인덱스 지정
    idx = (i+frames) # i ~ idx 까지 잡아줌.
    idx2 = (idx+1)
    if idx2 >= endpoint :
      break;
    else:
      npx.append(np.array(em_x2[fileidx][i:idx]))
      npy.append(np.array(em_y2[fileidx]))
npx = np.array(npx)
npy = np.array(npy)
print(npx.shape, npy.shape)
npx.shape
npy.shape<jupyter_output><empty_output><jupyter_text>정답 분포 확인<jupyter_code>pdy = pd.DataFrame(npy)
pdy.value_counts()<jupyter_output><empty_output><jupyter_text>셔플링<jupyter_code># shuffle = True
X_train, X_temptest, y_train, y_temptest = train_test_split(npx,  npy, 
                                                    test_size=0.0001, 
                                                    shuffle=True, 
                                                    random_state=1004)
print (len(X_train),len(X_temptest),len(y_train),len(y_temptest))<jupyter_output>2057 1 2057 1<jupyter_text>TEST data 생성<jupyter_code>%cd '/gdrive/MyDrive/EM_LSTM/test'
em_x2[0].shape[1]
path_dir = './'
file_list = os.listdir(path_dir)
em_x = []
em_y = []
for i in file_list :
  tempstr = './'+i
  em_x.append(np.array(pd.read_csv(tempstr)))
  #파일을 불러오면서 맨 앞 글자를 가져와 y 로 씀
  em_y.append(i[0])
# 데이터를 다시 불러와야 하거나 로우데이터로 뭔가 확인을 해야할 수 있으므로 em_x 는 그대로 두고 em_x2 생성
# 하는 김에 np.array로 해서 shape 도 봄.
em_x2 = np.array(em_x)
em_x2[0].shape
reshape_ha = list(em_y)
new_list = []
for v in reshape_ha:
    if v not in new_list:
        new_list.append(v)
enc = {'a': 0, 'b': 1, 'c': 2}
lst_e = [enc[i] for i in reshape_ha]
em_y2 = lst_e
# 앞에서 끊어주기
# for 문을 통해 frames 단위로 학습을 하는 부분임.
npx = []
npy = []
frames = 100
for fileidx in range(0,len(em_x2)):
  endpoint = 150
  for i in range(0,endpoint,1):
    # 1 개씩 하는 것이 시간이 오래 걸리는 관계로 100단위로 쪼개서 하기 위해 인덱스 지정
    idx = (i+frames) # i ~ idx 까지 잡아줌.
    idx2 = (idx+1)
    if idx2 >= endpoint :
      break;
    else:
      npx.append(np.array(em_x2[fileidx][i:idx]))
      npy.append(np.array(em_y2[fileidx]))
npx = np.array(npx)
npy = np.array(npy)
print(npx.shape, npy.shape)
ohe=np.eye( len(enc.keys() ))
temp_elst = [ohe[i] for i in npy]
npy = np.array(temp_elst)
# shuffle = True
from sklearn.model_selection import train_test_split
tempx_train, X_test, tempy_train, y_test = train_test_split(npx, npy, 
                                                    test_size=0.99, 
                                                    shuffle=True, 
                                                    random_state=1004)
print (len(tempx_train),len(X_test),len(tempy_train),len(y_test))<jupyter_output>1 146 1 146<jupyter_text>4. LSTM 모델 만들기  모델 구성- 들어오는 데이터의 약 2배 이상의 히든레이어 생성- y 값에 맞춰 3 개를 softmax 형태로 아웃풋 지정<jupyter_code>K.clear_session()
model = Sequential() # Sequeatial Model 
model.add(LSTM(200, input_shape=( frames,  em_x2[0].shape[1]),activation='sigmoid')) # (timestep, feature) 
model.add(Dense(3,activation='softmax')) # output = 3
admax = keras.optimizers.Adamax(learning_rate=0.012, beta_1=0.95, beta_2=0.999, epsilon=None, decay=0.0)
model.compile(loss='mse', optimizer=admax)

model.summary()<jupyter_output>Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm (LSTM)                  (None, 200)               166400    
_________________________________________________________________
dense (Dense)                (None, 3)                 603       
=================================================================
Total params: 167,003
Trainable params: 167,003
Non-trainable params: 0
_________________________________________________________________<jupyter_text>학습 실행<jupyter_code>history = model.fit(X_train, y_train, batch_size=30, epochs=180, verbose=1, validation_data=(X_test, y_test))<jupyter_output>Epoch 1/180
69/69 [==============================] - 15s 198ms/step - loss: 0.2938 - val_loss: 0.2786
Epoch 2/180
69/69 [==============================] - 13s 191ms/step - loss: 0.2564 - val_loss: 0.2248
Epoch 3/180
69/69 [==============================] - 13s 191ms/step - loss: 0.2219 - val_loss: 0.2247
Epoch 4/180
69/69 [==============================] - 13s 188ms/step - loss: 0.2195 - val_loss: 0.2262
Epoch 5/180
69/69 [==============================] - 13s 188ms/step - loss: 0.2146 - val_loss: 0.2400
Epoch 6/180
69/69 [==============================] - 13s 191ms/step - loss: 0.2002 - val_loss: 0.2673
Epoch 7/180
69/69 [==============================] - 13s 189ms/step - loss: 0.1870 - val_loss: 0.2616
Epoch 8/180
69/69 [==============================] - 13s 186ms/step - loss: 0.1837 - val_loss: 0.2638
Epoch 9/180
69/69 [==============================] - 13s 189ms/step - loss: 0.1777 - val_loss: 0.2787
Epoch 10/180
69/69 [==============================] - 14s 197ms/step - loss: 0.186[...]<jupyter_text>모델 저장<jupyter_code>%cd '/gdrive/MyDrive/EM_LSTM'
model.save('./emo.h5')<jupyter_output>/gdrive/MyDrive/EM_LSTM
/gdrive/MyDrive/EM_LSTM<jupyter_text>평가  학습 History<jupyter_code>pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='test')
pyplot.legend()
pyplot.show()<jupyter_output><empty_output><jupyter_text>Predict 진행<jupyter_code>y_true = np.array(y_test)
x_scaled = np.array(X_test)
y_pred = model.predict(x_scaled)<jupyter_output><empty_output><jupyter_text>argmax 로 예측된 값 정리<jupyter_code>y_pred2 = np.argmax(y_pred, axis=1)
y_true2 = np.argmax(y_true, axis=1)<jupyter_output><empty_output><jupyter_text>Confusion matrix<jupyter_code>labels = ["Bob", "Jook", "Mium"]

cm = confusion_matrix(y_true2, y_pred2)

disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)

disp.plot(cmap=plt.cm.Blues)
plt.show()<jupyter_output><empty_output><jupyter_text>Accuracy<jupyter_code>print(accuracy_score(y_true2, y_pred2))<jupyter_output>0.9383561643835616
{'fraction_non_alphanumeric': 0.16725559481743227, 'fraction_numerical': 0.04946996466431095, 'mean_word_length': 4.2220172201722015, 'pattern_counts': {'":': 0, '<': 76, '<?xml version=': 0, '>': 78, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '207125', 'license': 'MIT', 'n_tokens_mistral': 3938, 'n_tokens_neox': 3845, 'n_words': 696, 'path': '2.emo_LSTM_letsgo_v3.ipynb', 'repo_name': 'nowage/Emotion-recognition', 'size': '8490'}