<jupyter_start><jupyter_text>Best Two Stage-ET+DNN<jupyter_code>def load_model():
    json_file = open('/mnt/home/spicrowdpred/Spi-Occ/Lifetime/model.json', 'r')
    loaded_model_json = json_file.read()
    json_file.close()
    loaded_model = model_from_json(loaded_model_json)
    # load weights into new model
    loaded_model.load_weights(
        "/mnt/home/spicrowdpred/Spi-Occ/Lifetime/model.h5")
    print(f"Loaded model from disk")
    return loaded_model
from sklearn.ensemble import ExtraTreesClassifier
usefulFeatures = ['Capacity', 'DaysShowedInWeek', 'Total_MoviesinWeek',
                  'WeekOfTheYear', 'ShowsInWeek', 'ShowHistory1', 'RelativeOccAtWeek0',
                  'trans_2', 'trans_3', 'OccPer', 'RelativeOccAtWeek_History_Min']
print(len(usefulFeatures), "is length of importantfeatures")
etc = ExtraTreesClassifier(n_estimators=200, min_samples_split=3,
                           min_samples_leaf=1, max_depth=None, bootstrap=False, n_jobs=-1)
etc.fit(train_clas[usefulFeatures], train_clas['Target_Clas'])

predictions_clas = (etc.predict(test_clas[usefulFeatures]))

print(classification_report(test_clas["Target_Clas"], predictions_clas))
print(confusion_matrix(test_clas["Target_Clas"], predictions_clas))
classification_confusion_matrix = classification_report(
    test_clas["Target_Clas"], predictions_clas)
test_clas['predicted'] = predictions_clas
test['Clas_Pred'] = predictions_clas
test_reg = test_clas[test_clas['predicted'] == 1]
train_reg = train[train['Target'] > 0]

seed(69)
set_random_seed(69)
np.random.seed(69)

test_clas['predicted'] = predictions_clas
test['Clas_Pred'] = predictions_clas
test_reg = test_clas[test_clas['predicted'] == 1]

test_reg = test_reg[test_reg['week_x'] != 192]

train_reg = train[train['Target'] > 0]

model = Sequential()
model.add(Dense(128, input_shape=(train_reg[features].shape[-1],)))
model.add(keras.layers.LeakyReLU())
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Dense(64))
model.add(keras.layers.LeakyReLU())
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Dense(32))
model.add(keras.layers.LeakyReLU())
model.add(BatchNormalization())
model.add(Dense(8))
model.add(keras.layers.LeakyReLU())
model.add(BatchNormalization())
model.add(Dense(1,))
opzr = RMSprop(lr=3e-3)
model.compile(optimizer=opzr, loss='mae')  # mae has 60% no error


scalerX = StandardScaler().fit(train_reg[features])  # Standard Scalar
scalery = StandardScaler().fit(np.array(train_reg['Target']).reshape(-1, 1))
X_train = scalerX.transform(train_reg[features])
y_train = scalery.transform(np.array(train_reg['Target']).reshape(-1, 1))
X_test = scalerX.transform(test_reg[features])
y_test = scalery.transform(np.array(test_reg['Target']).reshape(-1, 1))
seed(69)
set_random_seed(69)
np.random.seed(69)

# model.fit(X_train,y_train,epochs=14,batch_size=32,verbose=1)
model = load_model()
ann_pred = (model.predict(scalerX.transform(test_reg[features].values)))
ann_pred = scalery.inverse_transform(ann_pred)
print("R SQURED SCORE      :", r2_score(test_reg["Target"], ann_pred))
print("MAE                 :", mean_absolute_error(
    test_reg["Target"].values, ann_pred))
print("MSE                 :", mean_squared_error(
    test_reg["Target"].values, ann_pred))
print("RMSE                :", np.sqrt(
    mean_squared_error(test_reg["Target"].values, ann_pred)))
ann_error_df = pd.DataFrame()
ann_error_df = test_reg[["Movie", "Target",
                         "Capacity", 'Lifetime', 'Language']]
ann_error_df['Pred'] = ann_pred
ann_error_df["error"] = (abs(ann_error_df["Target"]-ann_error_df["Pred"]))
ann_error_df.loc[ann_error_df["error"] < 1, "count"] = '0 Week Error:'
ann_error_df.loc[((ann_error_df["error"] >= 1) & (
    ann_error_df["error"] < 2)), "count"] = '1 Week Error:'
ann_error_df.loc[((ann_error_df["error"] >= 2) & (
    ann_error_df["error"] < 3)), "count"] = '2 Week Error:'
ann_error_df.loc[((ann_error_df["error"] >= 3) & (
    ann_error_df["error"] < 4)), "count"] = '3 Week Error:'
ann_error_df.loc[ann_error_df["error"] >= 4, "count"] = '>3 Week Error:'
ann_error = ann_error_df["count"].value_counts()
ann_error = (ann_error*100)/sum(ann_error)
print(ann_error)<jupyter_output>11 is length of importantfeatures
              precision    recall  f1-score   support

         0.0       0.97      0.92      0.94       312
         1.0       0.97      0.99      0.98       743

   micro avg       0.97      0.97      0.97      1055
   macro avg       0.97      0.95      0.96      1055
weighted avg       0.97      0.97      0.97      1055

[[288  24]
 [ 10 733]]
Loaded model from disk
R SQURED SCORE      : 0.3073924663550127
MAE                 : 1.1610977039413048
MSE                 : 4.67174451076025
RMSE                : 2.16142187246272
0 Week Error:     64.854111
1 Week Error:     19.098143
2 Week Error:      7.824934
>3 Week Error:     4.774536
3 Week Error:      3.448276
Name: count, dtype: float64<jupyter_text>THRESHOLD (DNN)<jupyter_code>from sklearn.metrics import confusion_matrix, classification_report
test['predicted'] = predictions

test.loc[test["predicted"] >= (1/7), "predicted_Clas_1"] = 1
test.loc[test["predicted"] < (1/7), "predicted_Clas_1"] = 0

test.loc[test["predicted"] >= (2/7), "predicted_Clas_2"] = 1
test.loc[test["predicted"] < (2/7), "predicted_Clas_2"] = 0

test.loc[test["predicted"] >= (3/7), "predicted_Clas_3"] = 1
test.loc[test["predicted"] < (3/7), "predicted_Clas_3"] = 0


test.loc[test["predicted"] >= (4/7), "predicted_Clas_4"] = 1
test.loc[test["predicted"] < (4/7), "predicted_Clas_4"] = 0

test.loc[test["predicted"] >= (5/7), "predicted_Clas_5"] = 1
test.loc[test["predicted"] < (5/7), "predicted_Clas_5"] = 0

test.loc[test["predicted"] >= (6/7), "predicted_Clas_6"] = 1
test.loc[test["predicted"] < (6/7), "predicted_Clas_6"] = 0

test.loc[test["predicted"] >= (7/7), "predicted_Clas_7"] = 1
test.loc[test["predicted"] < (7/7), "predicted_Clas_7"] = 0


test.loc[test["Target"] > 0, "Target_Class"] = 1
test.loc[test["Target"] <= 0, "Target_Class"] = 0

confusion_matrices = []
for i in range(1, 8):
    confusion_matrices.append(confusion_matrix(
        test["Target_Class"], test["predicted_Clas_"+str(i)]))
    # print(confusion_matrices[i-1])
    # print(confusion_matrices[i-1][0][0])
    print("recall for 0 as", i, "/7")
    print(confusion_matrices[i-1][0][0]/sum(confusion_matrices[i-1][0]))
    print(sum(confusion_matrices[i-1][0]))
#     print("*****-------------------*************")
#     print("classification results---------------------------------")
#     print(classification_confusion_matrix)
#     print("-----------------end------------------------------------")
    print(confusion_matrices[i-1])
    print(classification_report(
        test["Target_Class"], test["predicted_Clas_"+str(i)]))
    # print(confusion_matrices(test["Target_Class"],test["predicted_Clas_"+str(i)]))
    print
    print("------------------------------")
# Similarly Thresholds were obtained for XGB and ET<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18276205376191154, 'fraction_numerical': 0.04835727492533068, 'mean_word_length': 4.228252788104089, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '577591', 'license': 'MIT', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2563, 'n_words': 403, 'path': 'Results.ipynb', 'repo_name': 'ramanathanmurugappan/MovieLifetimePrediction', 'size': '7031'}