<jupyter_start><jupyter_text>Now I want to stack output from genetic model, neural network, xgboost, lightbgm and randomforest<jupyter_code># also can use sklearn.LabelEncoder to do these

def processdata(data):
    # Sex
    data.drop(['Ticket', 'Name'], inplace = True, axis = 1)
    data.Sex.fillna('0', inplace = True)
    data.loc[data.Sex != 'male', 'Sex'] = 0
    data.loc[data.Sex == 'male', 'Sex'] = 1
    # Cabin
    data.Cabin.fillna('0', inplace = True)
    data.loc[data.Cabin.str[0] == 'A', 'Cabin'] = 1
    data.loc[data.Cabin.str[0] == 'B', 'Cabin'] = 2
    data.loc[data.Cabin.str[0] == 'C', 'Cabin'] = 3
    data.loc[data.Cabin.str[0] == 'D', 'Cabin'] = 4
    data.loc[data.Cabin.str[0] == 'E', 'Cabin'] = 5
    data.loc[data.Cabin.str[0] == 'F', 'Cabin'] = 6
    data.loc[data.Cabin.str[0] == 'G', 'Cabin'] = 7
    data.loc[data.Cabin.str[0] == 'T', 'Cabin'] = 8
    # Embarked
    data.loc[data.Embarked == 'C', 'Embarked'] = 1
    data.loc[data.Embarked == 'Q', 'Embarked'] = 2
    data.loc[data.Embarked == 'S', 'Embarked'] = 3
    data.Embarked.fillna(0, inplace = True)
    # fill negative result for NA
    data.fillna(-1, inplace = True)
    return data.astype(float)
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
train.head()
train_processed = processdata(train)
test_processed = processdata(test)
def GeneticFunction(data):
    return ((np.minimum( ((((0.058823499828577 + data["Sex"]) - np.cos((data["Pclass"] / 2.0))) * 2.0)),  ((0.885868))) * 2.0) +
            np.maximum( ((data["SibSp"] - 2.409090042114258)),  ( -(np.minimum( (data["Sex"]),  (np.sin(data["Parch"]))) * data["Pclass"]))) +
            (0.138462007045746 * ((np.minimum( (data["Sex"]),  (((data["Parch"] / 2.0) / 2.0))) * data["Age"]) - data["Cabin"])) +
            np.minimum( ((np.sin((data["Parch"] * ((data["Fare"] - 0.720430016517639) * 2.0))) * 2.0)),  ((data["SibSp"] / 2.0))) +
            np.maximum( (np.minimum( ( -np.cos(data["Embarked"])),  (0.138462007045746))),  (np.sin(((data["Cabin"] - data["Fare"]) * 2.0)))) +
            -np.minimum( ((((data["Age"] * data["Parch"]) * data["Embarked"]) + data["Parch"])),  (np.sin(data["Pclass"]))) +
            np.minimum( (data["Sex"]),  ((np.sin( -(data["Fare"] * np.cos((data["Fare"] * 1.630429983139038)))) / 2.0))) +
            np.minimum( ((0.230145)),  (np.sin(np.minimum( (((67.0 / 2.0) * np.sin(data["Fare"]))),  (0.31830988618379069))))) +
            np.sin((np.sin(data["Cabin"]) * (np.sin((12.6275)) * np.maximum( (data["Age"]),  (data["Fare"]))))) +
            np.sin(((np.minimum( (data["Fare"]),  ((data["Cabin"] * data["Embarked"]))) / 2.0) *  -data["Fare"])) +
            np.minimum( (((2.675679922103882 * data["SibSp"]) * np.sin(((96) * np.sin(data["Cabin"]))))),  (data["Parch"])) +
            np.sin(np.sin((np.maximum( (np.minimum( (data["Age"]),  (data["Cabin"]))),  ((data["Fare"] * 0.31830988618379069))) * data["Cabin"]))) +
            np.maximum( (np.sin(((12.4148) * (data["Age"] / 2.0)))),  (np.sin((-3.0 * data["Cabin"])))) +
            (np.minimum( (np.sin((((np.sin(((data["Fare"] * 2.0) * 2.0)) * 2.0) * 2.0) * 2.0))),  (data["SibSp"])) / 2.0) +
            ((data["Sex"] - data["SibSp"]) * (np.cos(((data["Embarked"] - 0.730768978595734) + data["Age"])) / 2.0)) +
            ((np.sin(data["Cabin"]) / 2.0) - (np.cos(np.minimum( (data["Age"]),  (data["Embarked"]))) * np.sin(data["Embarked"]))) +
            np.minimum( (0.31830988618379069),  ((data["Sex"] * (2.212120056152344 * (0.720430016517639 - np.sin((data["Age"] * 2.0))))))) +
            (np.minimum( (np.cos(data["Fare"])),  (np.maximum( (np.sin(data["Age"])),  (data["Parch"])))) * np.cos((data["Fare"] / 2.0))) +
            np.sin((data["Parch"] * np.minimum( ((data["Age"] - 1.5707963267948966)),  ((np.cos((data["Pclass"] * 2.0)) / 2.0))))) +
            (data["Parch"] * (np.sin(((data["Fare"] * (0.623655974864960 * data["Age"])) * 2.0)) / 2.0)) +
            (0.31830988618379069 * np.cos(np.maximum( ((0.602940976619720 * data["Fare"])),  ((np.sin(0.720430016517639) * data["Age"]))))) +
            (np.minimum( ((data["SibSp"] / 2.0)),  (np.sin(((data["Pclass"] - data["Fare"]) * data["SibSp"])))) * data["SibSp"]) +
            np.tanh((data["Sex"] * np.sin((5.199999809265137 * np.sin((data["Cabin"] * np.cos(data["Fare"]))))))) +
            (np.minimum( (data["Parch"]),  (data["Sex"])) * np.cos(np.maximum( ((np.cos(data["Parch"]) + data["Age"])),  (3.1415926535897931)))) +
            (np.minimum( (np.tanh(((data["Cabin"] / 2.0) + data["Parch"]))),  ((data["Sex"] + np.cos(data["Age"])))) / 2.0) +
            (np.sin((np.sin(data["Sex"]) * (np.sin((data["Age"] * data["Pclass"])) * data["Pclass"]))) / 2.0) +
            (data["Sex"] * (np.cos(((data["Sex"] + data["Fare"]) * ((8.48635) * (63)))) / 2.0)) +
            np.minimum( (data["Sex"]),  ((np.cos((data["Age"] * np.tanh(np.sin(np.cos(data["Fare"]))))) / 2.0))) +
            (np.tanh(np.tanh( -np.cos((np.maximum( (np.cos(data["Fare"])),  (0.094339601695538)) * data["Age"])))) / 2.0) +
            (np.tanh(np.cos((np.cos(data["Age"]) + (data["Age"] + np.minimum( (data["Fare"]),  (data["Age"])))))) / 2.0) +
            (np.tanh(np.cos((data["Age"] * ((-2.0 + np.sin(data["SibSp"])) + data["Fare"])))) / 2.0) +
            (np.minimum( (((281) - data["Fare"])),  (np.sin((np.maximum( ((176)),  (data["Fare"])) * data["SibSp"])))) * 2.0) +
            np.sin(((np.maximum( (data["Embarked"]),  (data["Age"])) * 2.0) * (((785) * 3.1415926535897931) * data["Age"]))) +
            np.minimum( (data["Sex"]),  (np.sin( -(np.minimum( ((data["Cabin"] / 2.0)),  (data["SibSp"])) * (data["Fare"] / 2.0))))) +
            np.sin(np.sin((data["Cabin"] * (data["Embarked"] + (np.tanh( -data["Age"]) + data["Fare"]))))) +
            (np.cos(np.cos(data["Fare"])) * (np.sin((data["Embarked"] - ((734) * data["Fare"]))) / 2.0)) +
            ((np.minimum( (data["SibSp"]),  (np.cos(data["Fare"]))) * np.cos(data["SibSp"])) * np.sin((data["Age"] / 2.0))) +
            (np.sin((np.sin((data["SibSp"] * np.cos((data["Fare"] * 2.0)))) + (data["Cabin"] * 2.0))) / 2.0) +
            (((data["Sex"] * data["SibSp"]) * np.sin(np.sin( -(data["Fare"] * data["Cabin"])))) * 2.0) +
            (np.sin((data["SibSp"] * ((((5.428569793701172 + 67.0) * 2.0) / 2.0) * data["Age"]))) / 2.0) +
            (data["Pclass"] * (np.sin(((data["Embarked"] * data["Cabin"]) * (data["Age"] - (1.07241)))) / 2.0)) +
            (np.cos((((( -data["SibSp"] + data["Age"]) + data["Parch"]) * data["Embarked"]) / 2.0)) / 2.0) +
            (0.31830988618379069 * np.sin(((data["Age"] * ((data["Embarked"] * np.sin(data["Fare"])) * 2.0)) * 2.0))) +
            ((np.minimum( ((data["Age"] * 0.058823499828577)),  (data["Sex"])) - 0.63661977236758138) * np.tanh(np.sin(data["Pclass"]))) +
            -np.minimum( ((np.cos(((727) * ((data["Fare"] + data["Parch"]) * 2.0))) / 2.0)),  (data["Fare"])) +
            (np.minimum( (np.cos(data["Fare"])),  (data["SibSp"])) * np.minimum( (np.sin(data["Parch"])),  (np.cos((data["Embarked"] * 2.0))))) +
            (np.minimum( (((data["Fare"] / 2.0) - 2.675679922103882)),  (0.138462007045746)) * np.sin((1.5707963267948966 * data["Age"]))) +
            np.minimum( ((0.0821533)),  (((np.sin(data["Fare"]) + data["Embarked"]) - np.cos((data["Age"] * (9.89287)))))))
def Outputs(data):
    return np.round(1.-(1./(1.+np.exp(-data))))
np.mean(Outputs(GeneticFunction(train_processed)) == train_processed.Survived)<jupyter_output><empty_output><jupyter_text>93% from genetic programming is correct<jupyter_code>print(train_processed.shape)
train_processed.head()
X = tf.placeholder("float", [None, 8])
Y = tf.placeholder("float", [None, 2])
layer1 = tf.Variable(tf.random_normal([8, 256]))
layer2 = tf.Variable(tf.random_normal([256, 256]))
layer3 = tf.Variable(tf.random_normal([256, 128]))
layer4 = tf.Variable(tf.random_normal([128, 2]))
bias1 = tf.Variable(tf.random_normal([256], stddev = 0.1))
bias2 = tf.Variable(tf.random_normal([256], stddev = 0.1))
bias3 = tf.Variable(tf.random_normal([128], stddev = 0.1))
bias4 = tf.Variable(tf.random_normal([2], stddev = 0.1))
hidden1 = tf.nn.sigmoid(tf.matmul(X, layer1) + bias1)
hidden2 = tf.nn.sigmoid(tf.matmul(hidden1, layer2) + bias2)
hidden3 = tf.nn.sigmoid(tf.matmul(hidden2, layer3) + bias3)
hidden4 = tf.matmul(hidden3, layer4) + bias4
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = Y, logits = hidden4))
optimizer = tf.train.AdamOptimizer(learning_rate = 0.001).minimize(loss)
correct_pred = tf.equal(tf.argmax(hidden4, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
from sklearn.preprocessing import MinMaxScaler
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())

array_processed = train_processed.iloc[:, 2:].values
# f(x(ij)) = x(ij) - x min(j) / x max(j) - x min(j)
# iterate over column if want to do manually
normalize_train = MinMaxScaler().fit_transform(array_processed)

# 5k epochs
for i in range(5000):
    total_loss, total_acc = 0, 0
    # 20 per batch, we loss 11 information
    for k in range(0, (array_processed.shape[0] // 20) * 20, 20):
        batch_x = np.zeros((20, 8))
        batch_x = normalize_train[k: k + 20, :]
        batch_y = np.zeros((20, 2))
        for n in range(20):
            batch_y[n, int(train_processed.iloc[k + n, 1])] = 1.0
        lost, _ = sess.run([loss, optimizer], feed_dict = {X: batch_x, Y: batch_y})
        total_acc += sess.run(accuracy, feed_dict = {X: batch_x, Y: batch_y})
        total_loss += lost
    total_loss /= (array_processed.shape[0] // 20)
    total_acc /= (array_processed.shape[0] // 20)
    if (i + 1) % 100 == 0:
        print('epoch:', i, ', avg loss:', total_loss, ', avg acc:', total_acc)
y = np.zeros((train_processed.iloc[:, 1].shape[0], 2))
for i in range(train_processed.iloc[:, 1].shape[0]):
    y[i, int(train_processed.iloc[i, 1])] = 1.0
sess.run(accuracy, feed_dict = {X: normalize_train, Y: y})<jupyter_output><empty_output><jupyter_text>95% accuracy<jupyter_code>params = {
    'n_estimators' : 2000,
    'max_depth': 4,
    'min_child_weight': 2,
    'gamma': 0.9,
    'subsample': 0.8,
    'colsample_bytree': 0.8,
    'objective': 'binary:logistic'
}

watchlist = [(xgb.DMatrix(normalize_train, train_processed.iloc[:, 1].values), 'train')]
model = xgb.train(params, (xgb.DMatrix(normalize_train, train_processed.iloc[:, 1].values)), 1000,  watchlist, verbose_eval=50, early_stopping_rounds=100)
np.mean(np.around(model.predict(xgb.DMatrix(normalize_train), ntree_limit=model.best_ntree_limit+80)) == train_processed.iloc[:, 1].values)<jupyter_output><empty_output><jupyter_text>0.95% accurate! more higher!<jupyter_code>params = {
    'max_depth': 4,
    'min_child_weight': 2,
    'sub_feature': 0.8,
    'colsample_bytree': 0.8,
    'objective': 'binary'
}

d_train = gbm.Dataset(normalize_train, train_processed.iloc[:, 1].values)
watchlist = [gbm.Dataset(normalize_train, train_processed.iloc[:, 1].values)]
clf = gbm.train(params, d_train, 1000, watchlist)
np.mean(np.around(clf.predict(normalize_train)) == train_processed.iloc[:, 1].values)<jupyter_output><empty_output><jupyter_text>0.96% accuracy!<jupyter_code>test_processed.head()
output_ga = Outputs(GeneticFunction(test_processed))
normalize_test = MinMaxScaler().fit_transform(test_processed.iloc[:, 1:])
output_nn = sess.run(tf.argmax(hidden4, 1), feed_dict = {X: normalize_test})
output_xgb = np.around(model.predict(xgb.DMatrix(normalize_test), ntree_limit=model.best_ntree_limit+80))
putput_gbm = np.around(clf.predict(normalize_test))
from sklearn.ensemble import RandomForestClassifier
forest = RandomForestClassifier().fit(normalize_train, train_processed.iloc[:, 1].values)
np.mean(forest.predict(normalize_train) == train_processed.iloc[:, 1].values)
output_forest = forest.predict(normalize_test)
outputs_stack = output_ga * 0.5 + output_nn * 0.3 + output_xgb * 0.1 + putput_gbm * 0.095 + output_forest * 0.005
StackingSubmission = pd.DataFrame({ 'PassengerId': test_processed.iloc[:, 0].astype(int),
                            'Survived': np.around(outputs_stack).astype(int)})
StackingSubmission.to_csv("StackingSubmission.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.24654961208656595, 'fraction_numerical': 0.07366271947733768, 'mean_word_length': 3.8788844621513943, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '337605', 'license': 'MIT', 'n_tokens_mistral': 5573, 'n_tokens_neox': 4888, 'n_words': 746, 'path': 'stacking/1.binary/stacking.ipynb', 'repo_name': 'huseinzol05/Machine-Learning-Data-Science-Reuse', 'size': '12245'}