<jupyter_start><jupyter_text><jupyter_code>!python -m pip install konlpy
# from preprocess import *
import preprocess as pp
path_csv = './ChatBotData.csv_short'
inputs , outputs = pp.load_data(path_csv)
type(inputs) , type(outputs)
inputs , outputs
path_vocab  = './vocabulary.txt'
char2idx , idx2char , vocab_size = pp.load_vocabulary(path_csv,path_vocab,)
type(char2idx) , type(idx2char) , type(vocab_size)
print(char2idx)
print(idx2char)
print(vocab_size) 
idx_inputs , input_seq_len = pp.enc_processing(inputs , char2idx)

type(idx_inputs) , type(input_seq_len) 

print(idx_inputs[3:5])

idx_outputs , output_seq_len = pp.dec_output_processing(outputs, char2idx)

type(idx_outputs) , type(output_seq_len)
idx_outputs[3:5]
idx_targets = pp.dec_target_processing(outputs,char2idx)
type(idx_targets) , type(idx_targets)
print(idx_targets[3:5])
data_configs = dict()
data_configs['char2idx'] = char2idx
data_configs['idx2char'] = idx2char
data_configs['vocab_size'] = vocab_size
import numpy as np
np.save(open('./train_inputs.npy','wb'),idx_inputs)
!ls ./train_inputs.npy
np.save(open('./train_outputs.npy','wb'),idx_outputs)
np.save(open('./train_targets.npy','wb'),idx_targets)
data_configs['char2idx'] = char2idx
data_configs['idx2char'] = idx2char
data_configs['vocab_size'] = vocab_size

data_configs['pad_symbol'] = '<PAD>'
data_configs['std_symbol'] = '<STD>'
data_configs['end_symbol'] = '<END>'
data_configs['unk_symbol'] = '<UNK>'
pp.json.dump(data_configs,open('./data_configs.json','w'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15294117647058825, 'fraction_numerical': 0.015032679738562092, 'mean_word_length': 8.751592356687897, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '307783', 'license': 'Apache-2.0', 'n_tokens_mistral': 605, 'n_tokens_neox': 595, 'n_words': 84, 'path': 'NLP / seq2seq_preprocess.ipynb', 'repo_name': 'Ethan-Jeong/test_deeplearning', 'size': '1530'}