<jupyter_start><jupyter_text><jupyter_code>import numpy as np
index_inputs = np.load(open('./train_inputs.npy','rb'))
index_outputs = np.load(open('./train_outputs.npy','rb'))
index_targets = np.load(open('./train_targets.npy','rb'))
type(index_inputs),type(index_outputs),type(index_targets)
index_inputs[3:5]
! python -m pip install konlpy
import preprocess as pp
prepro_configs = pp.json.load(open('./data_configs.json','r'))
type(prepro_configs)
prepro_configs.keys()<jupyter_output><empty_output><jupyter_text>make model<jupyter_code>import tensorflow as tf<jupyter_output><empty_output><jupyter_text>encode model<jupyter_code>vocab_size = prepro_configs['vocab_size']
latent_dim=60
encode_input = tf.keras.Input(shape=(None,))
encode_hidden = tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=latent_dim)(encode_input)
encode_output, state_h, state_c = tf.keras.layers.LSTM(50, return_state=True)(encode_hidden)
encode_states = [state_h, state_c]<jupyter_output><empty_output><jupyter_text>decode model<jupyter_code>decode_input = tf.keras.Input(shape=(None,))
decode_hidden = tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=latent_dim)(decode_input)
decode_lstm, _, _ = tf.keras.layers.LSTM(50,return_state=True,return_sequences=True)(decode_hidden, initial_state=encode_states)
decode_output = tf.keras.layers.Dense(vocab_size, activation='softmax')(decode_lstm)
model = tf.keras.models.Model(inputs=[encode_input,decode_input], outputs=decode_output)
tf.keras.utils.plot_model(model)
model.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['acc'])<jupyter_output><empty_output><jupyter_text>fit<jupyter_code># index_inputs,index_outputs,index_targets
hist = model.fit(x=[index_inputs, index_outputs], y=index_targets, batch_size=128, epochs=50)<jupyter_output>Epoch 1/50
1/1 [==============================] - 4s 4s/step - loss: 4.6939 - acc: 0.0100
Epoch 2/50
1/1 [==============================] - 0s 33ms/step - loss: 4.6801 - acc: 0.1780
Epoch 3/50
1/1 [==============================] - 0s 30ms/step - loss: 4.6656 - acc: 0.8180
Epoch 4/50
1/1 [==============================] - 0s 29ms/step - loss: 4.6500 - acc: 0.8180
Epoch 5/50
1/1 [==============================] - 0s 37ms/step - loss: 4.6331 - acc: 0.8180
Epoch 6/50
1/1 [==============================] - 0s 37ms/step - loss: 4.6143 - acc: 0.8180
Epoch 7/50
1/1 [==============================] - 0s 44ms/step - loss: 4.5934 - acc: 0.8180
Epoch 8/50
1/1 [==============================] - 0s 44ms/step - loss: 4.5698 - acc: 0.8180
Epoch 9/50
1/1 [==============================] - 0s 43ms/step - loss: 4.5427 - acc: 0.8180
Epoch 10/50
1/1 [==============================] - 0s 45ms/step - loss: 4.5111 - acc: 0.8180
Epoch 11/50
1/1 [==============================] - 0s 49ms/step - loss: 4.4735 - [...]<jupyter_text>refer : https://github.com/NLP-kr/tensorflow-ml-nlp-tf2/blob/master/6.CHATBOT/6.4.seq2seq.ipynb<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2515151515151515, 'fraction_numerical': 0.07138047138047138, 'mean_word_length': 6.777486910994765, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '215746', 'license': 'Apache-2.0', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1114, 'n_words': 166, 'path': 'seq2seq.ipynb', 'repo_name': 'SolbiChoi/test_deeplearning', 'size': '2970'}