<jupyter_start><jupyter_text>Initial global var<jupyter_code>## 미리 Global 변수를 지정하자. 파일 명, 파일 위치, 디렉토리 등이 있다.

DATA_IN_PATH = './data_in/'
DATA_OUT_PATH = './data_out/'

TRAIN_Q1_DATA_FILE = 'train_q1.npy'
TRAIN_Q2_DATA_FILE = 'train_q2.npy'
TRAIN_LABEL_DATA_FILE = 'train_label.npy'
DATA_CONFIGS = 'data_configs.json'

## 학습에 필요한 파라메터들에 대해서 지정하는 부분이다.

BATCH_SIZE = 16
EPOCH = 2
HIDDEN = 64
BUFFER_SIZE = 10000

NUM_LAYERS = 3
DROPOUT_RATIO = 0.3

TEST_SPLIT = 0.1
RNG_SEED = 1<PASSWORD>
EMBEDDING_DIM = 128
MAX_SEQ_LEN = 31<jupyter_output><empty_output><jupyter_text>Load Dataset<jupyter_code>## 데이터를 불러오는 부분이다. 효과적인 데이터 불러오기를 위해, 미리 넘파이 형태로 저장시킨 데이터를 로드한다.

q1_data = np.load(open(DATA_IN_PATH + TRAIN_Q1_DATA_FILE, 'rb'))
q2_data = np.load(open(DATA_IN_PATH + TRAIN_Q2_DATA_FILE, 'rb'))
labels = np.load(open(DATA_IN_PATH + TRAIN_LABEL_DATA_FILE, 'rb'))
prepro_configs = None

with open(DATA_IN_PATH + DATA_CONFIGS, 'r') as f:
    prepro_configs = json.load(f)
VOCAB_SIZE = prepro_configs['vocab_size']<jupyter_output><empty_output><jupyter_text>Split train and test dataset<jupyter_code>q1_data_len = np.array([min(len(x), MAX_SEQ_LEN) for x in q1_data])
q2_data_len = np.array([min(len(x), MAX_SEQ_LEN) for x in q2_data])
## 데이터를 나누어 저장하자. sklearn의 train_test_split을 사용하면 유용하다. 하지만, 쿼라 데이터의 경우는
## 입력이 1개가 아니라 2개이다. 따라서, np.stack을 사용하여 두개를 하나로 쌓은다음 활용하여 분류한다.

X = np.stack((q1_data, q2_data), axis=1)
y = labels
train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=TEST_SPLIT, random_state=RNG_SEED)

train_Q1 = train_X[:,0]
train_Q2 = train_X[:,1]
test_Q1 = test_X[:,0]
test_Q2 = test_X[:,1]
def rearrange(base, hypothesis, labels):
    features = {"base": base, "hypothesis": hypothesis}
    return features, labels

def train_input_fn():
    dataset = tf.data.Dataset.from_tensor_slices((train_Q1, train_Q2, train_y))
    dataset = dataset.shuffle(buffer_size=len(train_Q1))
    dataset = dataset.batch(BATCH_SIZE)
    dataset = dataset.map(rearrange)
    dataset = dataset.repeat(EPOCH)
    iterator = dataset.make_one_shot_iterator()
    
    return iterator.get_next()

def eval_input_fn():
    dataset = tf.data.Dataset.from_tensor_slices((test_Q1, test_Q2, test_y))
    dataset = dataset.batch(BATCH_SIZE)
    dataset = dataset.map(rearrange)
    iterator = dataset.make_one_shot_iterator()
    
    return iterator.get_next()<jupyter_output><empty_output><jupyter_text>Model setup<jupyter_code>def Malstm(features, labels, mode):
        
    TRAIN = mode == tf.estimator.ModeKeys.TRAIN
    EVAL = mode == tf.estimator.ModeKeys.EVAL
    PREDICT = mode == tf.estimator.ModeKeys.PREDICT
            
    embedding = tf.keras.layers.Embedding(VOCAB_SIZE, EMBEDDING_DIM)
    
    base_embedded_matrix = embedding(features['base'])
    hypothesis_embedded_matrix = embedding(features['hypothesis'])
    
    # Question 1에 대한 LSTM모델
    q_lstm_fw_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units = 64, activation = tf.nn.tanh)
    q_lstm_bw_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units = 64, activation = tf.nn.tanh)
    _, q_output_states = tf.nn.bidirectional_dynamic_rnn(cell_fw = q_lstm_fw_cell,
                                                       cell_bw = q_lstm_bw_cell,
                                                       inputs = base_embedded_matrix,
                                                       dtype = tf.float32,
                                                      scope='query')
    # 마지막 state 값을 뽑아 추출한다
    q_final_state = tf.concat([q_output_states[0].h, q_output_states[1].h], axis=1)

    # Question 2 (유사한 쿼리)에 대한 LSTM모델
    s_lstm_fw_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units = 64, activation = tf.nn.tanh)
    s_lstm_bw_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units = 64, activation = tf.nn.tanh)
    _, s_output_states = tf.nn.bidirectional_dynamic_rnn(cell_fw = s_lstm_fw_cell,
                                                       cell_bw = s_lstm_bw_cell,
                                                       inputs = hypothesis_embedded_matrix,
                                                        dtype = tf.float32,
                                                      scope='sim_query')
    #LSTM의 마지막 state 값을 추출한다.
    sim_final_state = tf.concat([s_output_states[0].h, s_output_states[1].h], axis=1)
        
#     merged_matrix = tf.concat([base_sementic_matrix, hypothesis_sementic_matrix], -1)
#     logit_layer = tf.keras.layers.dot([base_sementic_matrix, hypothesis_sementic_matrix], axes=1, normalize=True)    

    with tf.variable_scope('output_layer'):
#     logit_layer = K.exp(-K.sum(K.abs(base_sementic_matrix - hypothesis_sementic_matrix), axis=1, keepdims=True))
        logit_layer = tf.exp(-tf.reduce_sum(tf.abs(q_final_state - sim_final_state), axis=1, keepdims=True))
        logit_layer = tf.squeeze(logit_layer, axis=-1)
                
    if PREDICT:
        return tf.estimator.EstimatorSpec(
                  mode=mode,
                  predictions={
                      'is_duplicate':logit_layer
                  })
    
    #prediction 진행 시, None
    if labels is not None:
        labels = tf.to_float(labels)
    
#     loss = tf.reduce_mean(tf.keras.metrics.binary_crossentropy(y_true=labels, y_pred=logit_layer))
    loss = tf.losses.mean_squared_error(labels=labels, predictions=logit_layer)
#     loss = tf.reduce_mean(tf.losses.sigmoid_cross_entropy(labels, logit_layer))
    
    if EVAL:
        accuracy = tf.metrics.accuracy(labels, tf.round(logit_layer))
        eval_metric_ops = {'acc': accuracy}
        return tf.estimator.EstimatorSpec(
                  mode=mode,
                  eval_metric_ops= eval_metric_ops,
                  loss=loss)

    elif TRAIN:

        global_step = tf.train.get_global_step()
        train_op = tf.train.AdamOptimizer(1e-3).minimize(loss, global_step)

        return tf.estimator.EstimatorSpec(
                  mode=mode,
                  train_op=train_op,
                  loss=loss)<jupyter_output><empty_output><jupyter_text>Training & Eval<jupyter_code># os.environ["CUDA_VISIBLE_DEVICES"]="0" #For TEST  

model_dir = os.path.join(os.getcwd(), DATA_OUT_PATH + "/checkpoint/rnn/")
os.makedirs(model_dir, exist_ok=True)

config_tf = tf.estimator.RunConfig(save_checkpoints_steps=500,
                                save_checkpoints_secs=None,
                                  keep_checkpoint_max=2,
                                  log_step_count_steps=200)

lstm_est = tf.estimator.Estimator(Malstm, model_dir=model_dir, config=config_tf)
lstm_est.train(train_input_fn)
lstm_est.evaluate(eval_input_fn)<jupyter_output><empty_output><jupyter_text>Load test dataset & create submit dataset to kaggle<jupyter_code>TEST_Q1_DATA_FILE = 'test_q1.npy'
TEST_Q2_DATA_FILE = 'test_q2.npy'
TEST_ID_DATA_FILE = 'test_id.npy'

test_q1_data = np.load(open(DATA_IN_PATH + TEST_Q1_DATA_FILE, 'rb'))
test_q2_data = np.load(open(DATA_IN_PATH + TEST_Q2_DATA_FILE, 'rb'))
test_id_data = np.load(open(DATA_IN_PATH + TEST_ID_DATA_FILE, 'rb'))
predict_input_fn = tf.estimator.inputs.numpy_input_fn(x={"base":test_q1_data, 
                                                         "hypothesis":test_q2_data}, 
                                                      shuffle=False)
predictions = np.array([p['is_duplicate'] for p in lstm_est.predict(input_fn=predict_input_fn)])
print(len(predictions)) #2345796

output = pd.DataFrame( data={"test_id":test_id_data, "is_duplicate": list(predictions)} )
output.to_csv( "rnn_predict.csv", index=False, quoting=3 )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10675496688741722, 'fraction_numerical': 0.013774834437086093, 'mean_word_length': 2.926677067082683, 'pattern_counts': {'":': 6, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '500193', 'license': 'Apache-2.0', 'n_tokens_mistral': 2958, 'n_tokens_neox': 2999, 'n_words': 428, 'path': '5.TEXT_SIM/OLD/5.3.3_Quora_LSTM.ipynb', 'repo_name': 'JITH0423/tensorflow-ml-nlp-tf2', 'size': '7547'}