<jupyter_start><jupyter_text>hyperparameters<jupyter_code>valid_size = 0.33
random_seed = 42
max_iterations = 1000
kp = 1
lr = 0.01<jupyter_output><empty_output><jupyter_text>load data<jupyter_code>train = pd.read_csv("../data/water/csv/train2017.csv")
test = pd.read_csv("../data/water/csv/test2017.csv")
X_train = train.values[:, 0:-1]
y_train = train.values[:, -1]
X_test = test.values[:, 0:-1]
y_test = train.values[:, -1]<jupyter_output><empty_output><jupyter_text>train valid split<jupyter_code>X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size = valid_size, 
                                                      stratify = y_train, random_state = random_seed)
print("1 in train:", sum(y_train == 1) / len(y_train))
print("1 in valid", sum(y_valid == 1) / len(y_valid))<jupyter_output>1 in train: 0.0157049727924142
1 in valid 0.01569678407350689<jupyter_text>oversampling<jupyter_code>X_train_oversampled, y_train_oversampled = Smoter(X_train, y_train, is_random=True)

print("============ SMOTE ============")
print("train: %d, contains %.4f of 0 , after SMOTE: train: %d contains %.4f of 1" 
      %(X_train.shape[0], 
        (y_train == 0).sum()/y_train.shape[0], 
        X_train_oversampled.shape[0], 
        (y_train_oversampled == 0).sum()/y_train_oversampled.shape[0]))<jupyter_output>============ SMOTE ============
train: 74244, contains 0.9843 of 0 , after SMOTE: train: 146156 contains 0.5000 of 1<jupyter_text>normalize<jupyter_code>clean_pipeline = Pipeline([('imputer', preprocessing.Imputer(missing_values='NaN',strategy="median")),
                           ('std_scaler', preprocessing.StandardScaler()),])
X_train_oversampled = clean_pipeline.fit_transform(X_train_oversampled)
X_valid = clean_pipeline.fit_transform(X_valid)
X_test = clean_pipeline.fit_transform(X_test)<jupyter_output><empty_output><jupyter_text>transfer y into probability vector<jupyter_code>y_train_oversampled_pro = np.zeros([y_train_oversampled.shape[0], 2])
for i in range(len(y_train_oversampled)):
    if y_train_oversampled[i] == 1:
        y_train_oversampled_pro[i] = np.array([0, 1])
    else:
        y_train_oversampled_pro[i] = np.array([1, 0])
y_train_oversampled = y_train_oversampled_pro

y_valid_pro = np.zeros([y_valid.shape[0], 2])
for i in range(len(y_valid)):
    if y_valid[i] == 1:
        y_valid_pro[i] = np.array([0, 1])
    else:
        y_valid_pro[i] = np.array([1, 0])
y_valid = y_valid_pro


y_test_pro = np.zeros([y_test.shape[0], 2])
for i in range(len(y_test)):
    if y_test[i] == 1:
        y_test_pro[i] = np.array([0, 1])
    else:
        y_test_pro[i] = np.array([1, 0])
y_test = y_test_pro


# y_valid_nonezero = np.count_nonzero(np.argmax(y_valid, 1))
# print("y_valid: 1 contains: ", y_valid_nonezero, "/",len(y_valid))
keep_prob = tf.placeholder(tf.float32)
xs = tf.placeholder(tf.float32, [None, 9])
ys = tf.placeholder(tf.float32, [None, 2])
learning_rate = tf.placeholder(tf.float32)
X_input = tf.reshape(xs,[-1,9,1]) # [n_samples, 9 ,1]    -1 具体是多少由导入数据决定（多少组数据） 
def MLP():
    ## func1 layer ##
    W_fc1 = weight_variable([9,2],name="W_fc1")
    b_fc1 = bias_variable([2],name="b_fc1")

    X_input_flat = tf.reshape(X_input, [-1,9])
    h_fc1 = tf.nn.softmax(tf.matmul(X_input_flat, W_fc1)+b_fc1)
    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
    prediction = h_fc1_drop

    var_dict = {'W_fc1': W_fc1, 
                'b_fc1': b_fc1}
    return prediction, var_dict
# output
prediction, var_dict = MLP()

# the error between prediction and real data
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=prediction, labels=ys))
train_op = tf.train.AdamOptimizer(learning_rate).minimize(cost)
valid_acc = []
valid_f1 = []
valid_loss = []

train_acc = []
train_f1 = []
train_loss = []

X_train_oversampled = np.array(X_train_oversampled, dtype=np.float32)
y_train_oversampled = np.array(y_train_oversampled, dtype=np.float32)
X_valid = np.array(X_valid, dtype=np.float32)
y_valid = np.array(y_valid, dtype=np.float32)

saver = tf.train.Saver(var_dict)<jupyter_output><empty_output><jupyter_text>train<jupyter_code>with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
            
    true_iteration = 1
    for i in range(max_iterations):
        feed = {xs : X_train_oversampled, ys : y_train_oversampled, keep_prob : kp, learning_rate : lr}
        
        # train
        # print("++++++++++++++++++ train ++++++++++++++++++")
        loss, _ = sess.run([cost, train_op],feed_dict=feed)
        precision, recall, f1_score, accuracy = evaluate(X_train_oversampled, y_train_oversampled, sess)
        
        train_loss.append(loss)
        train_f1.append(f1_score)
        train_acc.append(accuracy)
        
        # valid cost
        loss_v = sess.run(cost, feed_dict={xs: X_valid, ys: y_valid, keep_prob: 1})
        valid_loss.append(loss_v)
        # valid evaluationX_testX_testX_testX_test
        # print("++++++++++++++++++ valid ++++++++++++++++++")
        valid_precision, valid_recall, valid_f1_score, valid_accuracy = evaluate(X_valid, y_valid, sess)
        valid_f1.append(valid_f1_score)
        valid_acc.append(valid_accuracy)

        if valid_f1_score >= 0.98:
            break
        print("Iteration: {}/{}\n".format(true_iteration, max_iterations),
              "Train loss: {:6f}".format(loss),
              "Train acc: {:.6f}".format(accuracy),
              "Train f1: {:.6f}\n".format(f1_score),
              "Valid loss: {:6f}".format(loss_v),
              "Valid acc: {:.6f}".format(valid_accuracy),
              "Valid f1: {:.6f}".format(valid_f1_score))
        true_iteration += 1
            
#     save_path = saver.save(sess,"mlp_2017/2017_save_net.ckpt")
#     print("Save to path:", save_path)

# Plot training and valid loss
t = np.arange(true_iteration - 1)

plt.figure(figsize = (6,6))
plt.plot(t, np.array(train_loss), 'r-', t, np.array(valid_loss), 'b-')
plt.xlabel("iteration")
plt.ylabel("Loss")
plt.legend(['train', 'valid'], loc='upper right')
plt.savefig("../img/loss_before_ensemble.png")
plt.show()
# Plot Accuracies
plt.figure(figsize = (6,6))

plt.plot(t, np.array(train_acc), 'r-', t, valid_acc, 'b-')
plt.xlabel("iteration")
plt.ylabel("Accuray")
plt.legend(['train', 'valid'], loc='upper right')
plt.savefig("../img/acc_before_ensemble.png")
plt.show()
# Plot F1
plt.figure(figsize = (6,6))

plt.plot(t, np.array(train_f1), 'r-', t, valid_f1, 'b-')
plt.xlabel("iteration")
plt.ylabel("F1")
plt.legend(['train', 'valid'], loc='upper right')
plt.savefig("../img/f1_before_ensemble.png")
plt.show()
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
            
    true_iteration = 1
    for i in range(max_iterations):
        feed = {xs : X_train_oversampled, ys : y_train_oversampled, keep_prob : kp, learning_rate : lr}
        
        # train
        # print("++++++++++++++++++ train ++++++++++++++++++")
        loss, _ = sess.run([cost, train_op],feed_dict=feed)
        precision, recall, f1_score, accuracy = evaluate(X_train_oversampled, y_train_oversampled, sess)
        
        train_loss.append(loss)
        train_f1.append(f1_score)
        train_acc.append(accuracy)
        
        # valid cost
        loss_v = sess.run(cost, feed_dict={xs: X_valid, ys: y_valid, keep_prob: 1})
        valid_loss.append(loss_v)
        # valid evaluationX_testX_testX_testX_test
        # print("++++++++++++++++++ valid ++++++++++++++++++")
        valid_precision, valid_recall, valid_f1_score, valid_accuracy = evaluate(X_valid, y_valid, sess)
        valid_f1.append(valid_f1_score)
        valid_acc.append(valid_accuracy)

        if valid_f1_score >= 0.98:
            break
        print("Iteration: {}/{}\n".format(true_iteration, max_iterations),
              "Train loss: {:6f}".format(loss),
              "Train acc: {:.6f}".format(accuracy),
              "Train f1: {:.6f}\n".format(f1_score),
              "Valid loss: {:6f}".format(loss_v),
              "Valid acc: {:.6f}".format(valid_accuracy),
              "Valid f1: {:.6f}".format(valid_f1_score))
        true_iteration += 1
            
#     save_path = saver.save(sess,"mlp_2017/2017_save_net.ckpt")
#     print("Save to path:", save_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16620364778354874, 'fraction_numerical': 0.029592946007971976, 'mean_word_length': 3.728726442033124, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '255425', 'license': 'MIT', 'n_tokens_mistral': 3337, 'n_tokens_neox': 3108, 'n_words': 504, 'path': 'notebook/demo.ipynb', 'repo_name': 'chenkangyang/EnsembleDNN', 'size': '8279'}