<jupyter_start><jupyter_text><jupyter_code>!curl -O https://raw.githubusercontent.com/e9t/nsmc/master/ratings_train.txt
!curl -O https://raw.githubusercontent.com/e9t/nsmc/master/ratings_test.txt
import pandas as pd
train_data = pd.read_table('./ratings_train.txt')
train_data.info()
train_data.head(3)
train_data.dropna(inplace=True)
train_data.info()
train_data['label'].value_counts()
!python -m pip install konlpy
import konlpy
stopwords = ['의','가','이','은','들','는','좀','잘','걍','과','도','를','으로','자','에','와','한','하다']
import pickle
pickle.dump(stopwords, open('./stopwords.pkl','wb'))
okt = konlpy.tag.Okt()
okt.morphs('와 이런 것도 영화라고 차라리 뮤직비디오를 만드는 게 나을 뻔', stem = True)
train_data_small = train_data[0:300] 
# train_data_small = train_data #need to modify
Y_train = train_data['label'][0:300]
print(Y_train )
x_train = list()
okt = konlpy.tag.Okt()
for sentence in train_data_small['document']:
  temp_x = okt.morphs(sentence, stem=True)
  words = list()
  for tok in temp_x:
    if tok not in stopwords:
      words.append(tok)
  x_train.append(words)

x_train[3:5]
import tensorflow as tf
tokenizer = tf.keras.preprocessing.text.Tokenizer()
tokenizer.fit_on_texts(x_train)

print(tokenizer.word_index)
len(tokenizer.word_index)
vocab_size =1000       #need to 
tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=vocab_size)  #num_words 사전사이즈 제약
tokenizer.fit_on_texts(x_train) # 사전이 줄음
pickle.dump(tokenizer,open('./tokenizer.pkl','wb'))
X_train = tokenizer.texts_to_sequences(x_train) # 사전기반 숫자화
print(X_train[3:5])
len(X_train[0]), len(X_train[1])
hist = list()
for sent in X_train:
  hist.append(len(sent))
print(hist)
import matplotlib.pyplot as plt
plt.hist(hist)
X_train = tf.keras.preprocessing.sequence.pad_sequences(X_train, maxlen=50)
X_train[3:5]

Y_train.shape, X_train.shape
import numpy as np
np.unique(Y_train)
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Embedding(vocab_size, 50)) # input layer
model.add(tf.keras.layers.LSTM(64, activation='tanh')) #hidden layer
model.add(tf.keras.layers.Dense(1, activation='sigmoid')) # ouptut layer

model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc']) # gadget
hist = model.fit(X_train, Y_train, epochs=5, batch_size=256, validation_split=0.3)<jupyter_output>Epoch 1/5
1/1 [==============================] - 4s 4s/step - loss: 0.6932 - acc: 0.4810 - val_loss: 0.6922 - val_acc: 0.5556
Epoch 2/5
1/1 [==============================] - 0s 152ms/step - loss: 0.6916 - acc: 0.6238 - val_loss: 0.6910 - val_acc: 0.6889
Epoch 3/5
1/1 [==============================] - 0s 167ms/step - loss: 0.6899 - acc: 0.7143 - val_loss: 0.6898 - val_acc: 0.8000
Epoch 4/5
1/1 [==============================] - 0s 160ms/step - loss: 0.6880 - acc: 0.7667 - val_loss: 0.6884 - val_acc: 0.7556
Epoch 5/5
1/1 [==============================] - 0s 158ms/step - loss: 0.6859 - acc: 0.7952 - val_loss: 0.6868 - val_acc: 0.7556<jupyter_text>evaluation  Service<jupyter_code>model.save('./NaverSentimentAnalysis_LSTM.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2040210942649967, 'fraction_numerical': 0.05767963085036256, 'mean_word_length': 5.5835140997830806, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '482464', 'license': 'Apache-2.0', 'n_tokens_mistral': 1369, 'n_tokens_neox': 1286, 'n_words': 221, 'path': 'NaverSentimentAnalysis_LSTM.ipynb', 'repo_name': 'junanote/test_deeplearning', 'size': '3034'}