<jupyter_start><jupyter_text>NLP<jupyter_code>x_temp = x_train
x_temp.str.replace('[가-힣ㄱ-ㅎㅠ ]','')
x_temp01 = x_train
x_temp01.str.replace('[^가-힣ㄱ-ㅎㅠ ]','')
x_train.str.replace('[^가-힣ㄱ-ㅎㅠ ]','') # [a-zA-Z ]
from konlpy.tag import Mecab
mecab = Mecab()
print(mecab.morphs('와 이런 것도 상품이라고 차라리 내가 만드는게 나을 것 같다.'))
x_train_small = x_train[0:5000]
sentance = list()
stopwords = ['도', '는', '다', '의', '가', '이', '은', '한', '에', '하', '고', '을', '를', '인', '듯', '과', '와', '네', '들', '듯', '지', '임', '게']

for tok in x_train_small :
  encoded = mecab.morphs(tok)
  sentance.append([item for item in encoded if item not in stopwords])
        
sentance<jupyter_output><empty_output><jupyter_text>Tokenizer<jupyter_code>print(sentance)
import tensorflow as tf
tokenizer = tf.keras.preprocessing.text.Tokenizer()
tokenizer.fit_on_texts(sentance)
tokenizer.word_index
tokenizer.word_counts
total_cnt = len(tokenizer.word_index)

rare_cnt = 0
total_freq, rare_freq = 0, 0

for key, value in tokenizer.word_counts.items():
  total_freq = total_freq + value # 전체 단어 수
  if(value <= 2):
    rare_cnt = rare_cnt + 1
    rare_freq = rare_freq + value # 2미만인 단어 수
total_cnt, rare_cnt, (rare_cnt/total_cnt)*100, (rare_freq/total_freq)*100
vocab_size = total_cnt - rare_cnt
tokenizer = tf.keras.preprocessing.text.Tokenizer(vocab_size, oov_token='OOV')
tokenizer.fit_on_texts(sentance)<jupyter_output><empty_output><jupyter_text>['후기', '엄청', '맛있', '다고', [], '구입', '했', [], '기대', '커서', '그런가', [], [], [], '번', '안', '먹', '것', '같', '아요']<jupyter_code>tokenizer.index_word
x_train_small = tokenizer.texts_to_sequences(sentance)
len(x_train_small[0]), len(x_train_small[40]), len(x_train_small[50])
hist_len = [len(words) for words in x_train_small]
import matplotlib.pyplot as plt
plt.hist(hist_len, bins=50)
sum(hist_len) / len(x_train_small)
x_train_small = tf.keras.preprocessing.sequence.pad_sequences(x_train_small, maxlen=50)
y_train_small = y_train[0:5000]
np.unique(y_train_small)
import numpy as np
y_train_small = np.array(y_train_small)-1
np.unique(y_train_small)
# y_train_small[6] = 3
# y_train_small[6]
# y_train_small[0:6]
len(x_train_small), len(y_train_small)
# y_train_onehot = tf.keras.utils.to_categorical(y_train_small)
# len(y_train_onehot[5])
# len(y_train_onehot[5]), y_train_onehot[5]<jupyter_output><empty_output><jupyter_text>make model<jupyter_code>model = tf.keras.models.Sequential()

model.add(tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=30, input_length=50)) # input layer
# model.add(tf.keras.layers.LSTM(128)) # hidden layer
model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(128))) # hidden layer
# model.add(tf.keras.layers.GRU(128)) # hidden layer

model.add(tf.keras.layers.Dense(5, activation='softmax')) # output layer

model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['acc']) # gadget
hist = model.fit(x_train_small, y_train_small, epochs=2, batch_size=256, validation_split=0.3, shuffle=True)
# model.evaluate(x_train_small, y_train_small) # LSTM(128) - loss: 0.9170 - acc: 0.8500
# model.evaluate(x_train_small, y_train_small) # GRU - loss: 0.9265 - acc: 0.8436
# model.evaluate(x_train_small, y_train_small) # Bidirectional(LSTM(128)) - loss: 0.8787 - acc: 0.8382<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16748466257668712, 'fraction_numerical': 0.03067484662576687, 'mean_word_length': 6.654929577464789, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '416896', 'license': 'Apache-2.0', 'n_tokens_mistral': 1505, 'n_tokens_neox': 1482, 'n_words': 228, 'path': 'NaverShopping_BiLSTM_GRU.ipynb', 'repo_name': 'SanghunOh/test_deeplearning', 'size': '3260'}