<jupyter_start><jupyter_text><jupyter_code>!curl -O https://raw.githubusercontent.com/e9t/nsmc/master/ratings_train.txt
!curl -O https://raw.githubusercontent.com/e9t/nsmc/master/ratings_test.txt
import pandas as pd
train_data = pd.read_table('./ratings_train.txt')
train_data.info()
train_data.head(3)
train_data.dropna(inplace=True)
train_data.info()
train_data['label'].value_counts()
!python -m pip install konlpy
import konlpy
stopwords = ['의','가','이','은','들','는','좀','잘','걍','과','도','를','으로','자','에','와','한','하다']
import pickle
pickle.dump(stopwords, open('./stopwords.pkl', 'wb'))
okt = konlpy.tag.Okt()
okt.morphs('와 이런 것도 영화라고 차라리 뮤직비디오를 만드는 게 나을 뻔', stem = True)
train_data_small = train_data[0:300]    # need to modify
Y_train = train_data['label'][0:300]
print(Y_train)
x_train = list()
okt = konlpy.tag.Okt()
for sentence in train_data_small['document']:
  temp_x = okt.morphs(sentence, stem=True)
  words = list()
  for tok in temp_x:
    if tok not in stopwords:
      words.append(tok)
  x_train.append(words)

x_train[3:5]
import tensorflow as tf
tokenizer = tf.keras.preprocessing.text.Tokenizer()
tokenizer.fit_on_texts(x_train)
print(tokenizer.word_index)
len(tokenizer.word_index)
vocab_size = 1000
tokenizer = tf.keras.preprocessing.text.Tokenizer(vocab_size)
tokenizer.fit_on_texts(x_train)
pickle.dump(tokenizer, open('./tokenizer.pkl', 'wb'))
X_train = tokenizer.texts_to_sequences(x_train)
print(X_train[3:5])
len(X_train[0]), len(X_train[1])
hist = list()
for sent in X_train:
  hist.append(len(sent))

print(hist)
import matplotlib.pyplot as plt
plt.hist(hist)
X_train = tf.keras.preprocessing.sequence.pad_sequences(X_train, maxlen=50)
X_train[3:5]
Y_train.shape, X_train.shape
import numpy as np
np.unique(Y_train)
model = tf.keras.models.Sequential()

model.add(tf.keras.layers.Embedding(vocab_size,50)) # input layer
model.add(tf.keras.layers.LSTM(64, activation='tanh')) # hidden layer
model.add(tf.keras.layers.Dense(1, activation='sigmoid')) # output layer

model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc']) #gadget
hist = model.fit(X_train, Y_train, epochs=5, batch_size=256, validation_split=0.3)<jupyter_output>Epoch 1/5
1/1 [==============================] - 4s 4s/step - loss: 0.6929 - acc: 0.5333 - val_loss: 0.6922 - val_acc: 0.5778
Epoch 2/5
1/1 [==============================] - 0s 161ms/step - loss: 0.6917 - acc: 0.6143 - val_loss: 0.6913 - val_acc: 0.6000
Epoch 3/5
1/1 [==============================] - 0s 168ms/step - loss: 0.6903 - acc: 0.6619 - val_loss: 0.6904 - val_acc: 0.6778
Epoch 4/5
1/1 [==============================] - 0s 157ms/step - loss: 0.6887 - acc: 0.7143 - val_loss: 0.6894 - val_acc: 0.6889
Epoch 5/5
1/1 [==============================] - 0s 163ms/step - loss: 0.6869 - acc: 0.7333 - val_loss: 0.6883 - val_acc: 0.6889<jupyter_text>evaluation  Service<jupyter_code>model.save('./NaverSentimentAnalysis_LSTM.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20816326530612245, 'fraction_numerical': 0.05952380952380952, 'mean_word_length': 5.63882618510158, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '212140', 'license': 'Apache-2.0', 'n_tokens_mistral': 1315, 'n_tokens_neox': 1220, 'n_words': 212, 'path': 'NaverSentimentAnalysis_LSTM.ipynb', 'repo_name': 'fonslucens/test_deeplearning', 'size': '2940'}