<jupyter_start><jupyter_text>2 stages 30s hrv_len, with feature ENMO_HRV  load minute prediction<jupyter_code>num_classes = 2
feature_type ='ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
minutes_path =  os.path.join(result_folder, sub_folder, "%d_stages_minutes_summary_%s.csv"
                                % (num_classes, feature_type))
mins_df = pd.read_csv(minutes_path)
mins_df = mins_df.rename(columns={"Unnamed: 0": "algorithms"})
mins_df.head()<jupyter_output><empty_output><jupyter_text>load classifier performance prediction<jupyter_code>SUMMARY_FILE = os.path.join(result_folder, sub_folder, "%d_stages_summary_%s.csv"
                                % (num_classes, feature_type))
PICKLE_RESULT_FILE = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                      % (num_classes, feature_type))
summary = pd.read_csv(SUMMARY_FILE)
print("summary csv is loaded")
summary = summary.rename(columns={summary.columns[0]: "algs"})
summary = summary.set_index("algs")

with open(PICKLE_RESULT_FILE, "rb") as f:
    results = pickle.load(f)
print("Pickle file is loaded")
metrics = ["accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa"]
summary['algorithms'] = summary.index
summary = summary.reset_index(drop = True)

summary.head()<jupyter_output><empty_output><jupyter_text>combine predicted minutes and metrics<jupyter_code>stages_2_combined = pd.merge(summary,mins_df, on =['algorithms'])
stages_2_clf_columns= ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "Wake", "Sleep"]

stages_2_combined = stages_2_combined.reindex(columns=stages_2_clf_columns)
stages_2_combined.head(110)
readable_df = pd.DataFrame(list(readable_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
print(readable_df)
readable_algs_dict['always_0']
stages_2_baselines = ['stages', 'always_0', 'always_1']
stages_2_combined['algorithms'] = stages_2_combined['algorithms'].apply(lambda x: readable_algs_dict[x])


stages_2_combined.sample(5)
bls = readable_df.new_alg_name.unique().tolist()
# stages_2_combined<jupyter_output><empty_output><jupyter_text>this is the section to produce latex table<jupyter_code>stages_2_combined['acc_for_sort'] = stages_2_combined['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
print(bls)  
print(stages_2_combined[stages_2_combined['algorithms'].isin(bls)].sort_values(by="acc_for_sort", ascending=False)[stages_2_clf_columns].to_latex(index=False))
stages_2_combined[stages_2_combined['algorithms'].isin(bls)].sort_values(by="acc_for_sort", ascending=False)[stages_2_clf_columns]<jupyter_output><empty_output><jupyter_text>this is the 2 stages ENMO alone and HRV alone in one table<jupyter_code>num_classes = 2
stages_2_clf_columns=  ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "Sleep", "Wake"]
stage_2_df = load_task(sub_folder, num_classes=num_classes, feature_type='ENMO', stages_clf_columns=stages_2_clf_columns)

readable_2_algs_dict = {"stages":"ground truth","always_0":"always wake", "always_1":"always sleep", "CNN_20_ENMO": "CNN 20", "CNN_50_ENMO":"CNN 50", "CNN_100_ENMO":"CNN 100", "LSTM_20_ENMO":"LSTM 20", "LSTM_50_ENMO":"LSTM 50","LSTM_100_ENMO":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_2stages_df = pd.DataFrame(list(readable_2_algs_dict.items()), columns=["old_alg_name","new_alg_name"])


readable_df = pd.DataFrame(list(readable_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
stages_2_baselines = ['stages', 'always_0', 'always_1']
stage_2_df['algorithms'] = stage_2_df['algorithms'].apply(lambda x: readable_2_algs_dict[x])
bls = readable_df.new_alg_name.unique().tolist()<jupyter_output><empty_output><jupyter_text>2 stages HRV<jupyter_code>num_classes = 2
stages_2_clf_columns=  ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "Sleep", "Wake"]
stage_2_df = load_task(sub_folder, num_classes=num_classes, feature_type='HRV', stages_clf_columns=stages_2_clf_columns)

# readable_2_algs_dict = {"stages":"ground truth","always_0":"always wake", "always_1":"always sleep", "CNN_20_HRV": "CNN 20", "CNN_50_HRV":"CNN 50", "CNN_100_HRV":"CNN 100", "LSTM_20_HRV":"LSTM 20", "LSTM_50_HRV":"LSTM 50","LSTM_100_HRV":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_2stages_df = pd.DataFrame(list(readable_2_algs_dict.items()), columns=["old_alg_name","new_alg_name"])

stage_2_df
readable_df = pd.DataFrame(list(readable_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
stages_2_baselines = ['stages', 'always_0', 'always_1']
stage_2_df['algorithms'] = stage_2_df['algorithms'].apply(lambda x: readable_2_algs_dict[x])
stage_2_bls = readable_df.new_alg_name.unique().tolist()
stage_2_df['acc_for_sort'] = stage_2_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
stage_2_df.sort_values(by="acc_for_sort", ascending=False)
print(stage_2_df[stage_2_df['algorithms'].isin(stage_2_bls)].sort_values(by="acc_for_sort", ascending=False)[stages_2_clf_columns].to_latex(index=False))<jupyter_output>\begin{tabular}{lllllllll}
\toprule
          algorithms &      accuracy &   specificity &     precision &        recall &      f1-score &   cohen-kappa &           Sleep &            Wake \\
\midrule
        ground truth &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &    365.1 +- 8.7 &    187.4 +- 8.6 \\
            LSTM 100 &   79.5 +- 1.2 &   62.2 +- 2.1 &   81.8 +- 1.4 &   88.9 +- 1.3 &   84.1 +- 1.1 &   51.5 +- 2.2 &     35.3 +- 8.7 &    -35.3 +- 8.7 \\
             CNN 100 &   79.1 +- 1.2 &   57.0 +- 2.1 &   79.9 +- 1.5 &   91.0 +- 1.4 &   83.9 +- 1.3 &   49.8 +- 2.1 &     54.4 +- 9.2 &    -54.4 +- 9.2 \\
             LSTM 50 &   78.6 +- 1.2 &   61.1 +- 2.0 &   81.2 +- 1.4 &   88.2 +- 1.3 &   83.4 +- 1.2 &   49.5 +- 2.1 &     34.5 +- 8.9 &    -34.5 +- 8.9 \\
              CNN 50 &   78.2 +- 1.2 &   54.0 +- 2.1 &   78.9 +- 1.5 &   91.3 +- 1.3 &   83.5 +- 1.2 &   47.1 +- 2.1 &     61.2 +- 9.2 &    -61.2 +- 9.2 \\
             LSTM 2[...]<jupyter_text>This is the ENMO alone<jupyter_code>num_classes = 2
stages_2_clf_columns=  ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "Sleep", "Wake"]
stage_2_df = load_task(sub_folder, num_classes=num_classes, feature_type='ENMO', stages_clf_columns=stages_2_clf_columns)

readable_2_algs_dict = {"stages":"ground truth","always_0":"always wake", "always_1":"always sleep", "CNN_20_ENMO": "CNN 20", "CNN_50_ENMO":"CNN 50", "CNN_100_ENMO":"CNN 100", "LSTM_20_ENMO":"LSTM 20", "LSTM_50_ENMO":"LSTM 50","LSTM_100_ENMO":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_2stages_df = pd.DataFrame(list(readable_2_algs_dict.items()), columns=["old_alg_name","new_alg_name"])

readable_df = pd.DataFrame(list(readable_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
stages_2_baselines = ['stages', 'always_0', 'always_1']
stage_2_df['algorithms'] = stage_2_df['algorithms'].apply(lambda x: readable_2_algs_dict[x])
stage_2_bls = readable_df.new_alg_name.unique().tolist()
stage_2_df['acc_for_sort'] = stage_2_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
stage_2_df.sort_values(by="acc_for_sort", ascending=False)
print(stage_2_df[stage_2_df['algorithms'].isin(stage_2_bls)].sort_values(by="acc_for_sort", ascending=False)[stages_2_clf_columns].to_latex(index=False))<jupyter_output><empty_output><jupyter_text>======================== 3 stages =======================  This is the 3 stages ENMO + HRV<jupyter_code>num_classes = 3
stages_3_clf_columns= ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "Non-REM sleep","REM sleep","Wake"]
stage_3_df = load_task(sub_folder, num_classes=num_classes, feature_type='ENMO_HRV', stages_clf_columns=stages_3_clf_columns)
readable_3stages_algs_dict= {"stages":"ground truth","always_0":"always wake", "always_1":"always Non-REM sleep", "always_2":"always REM sleep", "CNN_20_ENMO_HRV": "CNN 20", "CNN_50_ENMO_HRV":"CNN 50", "CNN_100_ENMO_HRV":"CNN 100", "LSTM_20_ENMO_HRV":"LSTM 20", "LSTM_50_ENMO_HRV":"LSTM 50","LSTM_100_ENMO_HRV":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_3stages_df = pd.DataFrame(list(readable_3stages_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
print(stage_3_df)

print(readable_3stages_df)
stages_3_baselines = ['stages', 'always_0', 'always_1', 'always_2']
stage_3_df['algorithms'] = stage_3_df['algorithms'].apply(lambda x: readable_3stages_algs_dict[x])
# get ready the rename df
stage_3_bls = readable_3stages_df.new_alg_name.unique().tolist()
stage_3_df['acc_for_sort'] = stage_3_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
stage_3_df[['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa","Wake","REM sleep", "Non-REM sleep",'acc_for_sort']].sort_values(by="acc_for_sort", ascending=False)
print(stage_3_df[stage_3_df['algorithms'].isin(stage_3_bls)].sort_values(by="acc_for_sort", ascending=False)[stages_3_clf_columns].to_latex())<jupyter_output>\begin{tabular}{lllllllllll}
\toprule
{} &            algorithms &      accuracy &   specificity &     precision &        recall &      f1-score &   cohen-kappa &  Non-REM sleep &     REM sleep &           Wake \\
\midrule
13 &          ground truth &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &   297.5 +- 7.1 &   67.6 +- 3.0 &   187.4 +- 8.6 \\
7  &               LSTM 50 &   76.2 +- 1.0 &   85.6 +- 0.5 &   72.2 +- 1.3 &   68.8 +- 1.2 &   67.9 +- 1.3 &   58.4 +- 1.8 &    23.9 +- 7.1 &  -10.7 +- 3.8 &   -13.2 +- 6.8 \\
8  &              LSTM 100 &   76.1 +- 0.9 &   85.1 +- 0.5 &   71.9 +- 1.4 &   66.8 +- 1.2 &   66.4 +- 1.3 &   57.4 +- 1.9 &    26.5 +- 7.0 &  -23.3 +- 3.4 &    -3.2 +- 6.8 \\
5  &               CNN 100 &   76.0 +- 1.0 &   85.6 +- 0.6 &   72.2 +- 1.2 &   69.7 +- 1.3 &   68.1 +- 1.3 &   58.6 +- 1.9 &    30.2 +- 7.7 &    2.5 +- 4.5 &   -32.7 +- 7.2 \\
4  &                CNN 50 &   75.2 +- 1.1 &   85.7 +- 0.6 &   71.7 +- 1.[...]<jupyter_text>Task 2 :  3stages classification for ENMO alone<jupyter_code>num_classes = 3
stages_clf_columns= ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "Non-REM sleep","REM sleep","Wake"]
stage_3_df = load_task(sub_folder, num_classes=num_classes, feature_type='ENMO', stages_clf_columns=stages_clf_columns)
readable_3stages_algs_dict= {"stages":"ground truth","always_0":"always wake", "always_1":"always Non-REM sleep", "always_2":"always REM sleep", "CNN_20_ENMO": "CNN 20", "CNN_50_ENMO":"CNN 50", "CNN_100_ENMO":"CNN 100", "LSTM_20_ENMO":"LSTM 20", "LSTM_50_ENMO":"LSTM 50","LSTM_100_ENMO":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_3stages_df = pd.DataFrame(list(readable_3stages_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
stage_3_df['algorithms'] = stage_3_df['algorithms'].apply(lambda x: readable_3stages_algs_dict[x])
stage_3_df['acc_for_sort'] = stage_3_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
stage_3_df[['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa","Wake","REM sleep", "Non-REM sleep",'acc_for_sort']].sort_values(by="acc_for_sort", ascending=False)
bls = readable_3stages_df.new_alg_name.unique().tolist()
print(stage_3_df[stage_3_df['algorithms'].isin(bls)].sort_values(by="acc_for_sort", ascending=False)[stages_clf_columns].to_latex(index=False))<jupyter_output>\begin{tabular}{llllllllll}
\toprule
           algorithms &      accuracy &   specificity &     precision &        recall &      f1-score &   cohen-kappa &  Non-REM sleep &      REM sleep &           Wake \\
\midrule
         ground truth &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &   297.5 +- 7.1 &    67.6 +- 3.0 &   187.4 +- 8.6 \\
             LSTM 100 &   73.8 +- 1.2 &   84.3 +- 0.6 &   69.8 +- 1.5 &   66.1 +- 1.3 &   64.9 +- 1.5 &   50.0 +- 2.2 &    36.4 +- 8.1 &    -8.6 +- 4.2 &   -27.8 +- 8.5 \\
              LSTM 50 &   72.9 +- 1.1 &   83.8 +- 0.6 &   67.9 +- 1.4 &   64.1 +- 1.3 &   62.9 +- 1.4 &   45.5 +- 2.1 &    34.1 +- 8.3 &   -16.2 +- 4.3 &   -17.9 +- 8.5 \\
              CNN 100 &   71.0 +- 1.2 &   83.6 +- 0.6 &   66.3 +- 1.4 &   65.4 +- 1.4 &   62.7 +- 1.4 &   46.1 +- 2.0 &    10.6 +- 9.1 &     2.3 +- 5.0 &   -12.9 +- 9.4 \\
              LSTM 20 &   70.4 +- 1.0 &   82.3 +- 0.6 &   65.3 +- 1.4 &   60.8 +- 1.2 &   59.6[...]<jupyter_text>Task 2 :  3stages classification for HRV alone<jupyter_code>num_classes = 3
stages_3_clf_columns= ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "Non-REM sleep","REM sleep","Wake"]
stage_3_df = load_task(sub_folder, num_classes=num_classes, feature_type='HRV', stages_clf_columns=stages_3_clf_columns)
readable_3stages_algs_dict= {"stages":"ground truth","always_0":"always wake", "always_1":"always Non-REM sleep", "always_2":"always REM sleep", "CNN_20_HRV": "CNN 20", "CNN_50_HRV":"CNN 50", "CNN_100_HRV":"CNN 100", "LSTM_20_HRV":"LSTM 20", "LSTM_50_HRV":"LSTM 50","LSTM_100_HRV":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_3stages_df = pd.DataFrame(list(readable_3stages_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
stage_3_df['algorithms'] = stage_3_df['algorithms'].apply(lambda x: readable_3stages_algs_dict[x])
stage_3_df['acc_for_sort'] = stage_3_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
stage_3_df[['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa","Wake","REM sleep", "Non-REM sleep",'acc_for_sort']].sort_values(by="acc_for_sort", ascending=False)<jupyter_output><empty_output><jupyter_text>produce latex<jupyter_code>bls = readable_3stages_df.new_alg_name.unique().tolist()
print(stage_3_df[stage_3_df['algorithms'].isin(bls)].sort_values(by="acc_for_sort", ascending=False)[stages_3_clf_columns].to_latex(index=False))<jupyter_output>\begin{tabular}{llllllllll}
\toprule
           algorithms &      accuracy &   specificity &     precision &        recall &      f1-score &   cohen-kappa &  Non-REM sleep &      REM sleep &           Wake \\
\midrule
         ground truth &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &   297.5 +- 7.1 &    67.6 +- 3.0 &   187.4 +- 8.6 \\
             LSTM 100 &   73.8 +- 1.2 &   84.3 +- 0.6 &   69.8 +- 1.5 &   66.1 +- 1.3 &   64.9 +- 1.5 &   50.0 +- 2.2 &    36.4 +- 8.1 &    -8.6 +- 4.2 &   -27.8 +- 8.5 \\
              LSTM 50 &   72.9 +- 1.1 &   83.8 +- 0.6 &   67.9 +- 1.4 &   64.1 +- 1.3 &   62.9 +- 1.4 &   45.5 +- 2.1 &    34.1 +- 8.3 &   -16.2 +- 4.3 &   -17.9 +- 8.5 \\
              CNN 100 &   71.0 +- 1.2 &   83.6 +- 0.6 &   66.3 +- 1.4 &   65.4 +- 1.4 &   62.7 +- 1.4 &   46.1 +- 2.0 &    10.6 +- 9.1 &     2.3 +- 5.0 &   -12.9 +- 9.4 \\
              LSTM 20 &   70.4 +- 1.0 &   82.3 +- 0.6 &   65.3 +- 1.4 &   60.8 +- 1.2 &   59.6[...]<jupyter_text>================== Task 3 4 stages ===================  4 stages ENMO & HRV<jupyter_code>stages_4_clf_columns= ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score","cohen-kappa", "Deep sleep","Light sleep","REM sleep","Wake"]
stage_4_df = load_task(sub_folder, num_classes=4, feature_type='ENMO_HRV', stages_clf_columns=stages_4_clf_columns)
readable_4stages_algs_dict= {"stages":"ground truth","always_0":"always wake", "always_1":"always light sleep", "always_2":"always deep sleep", "always_3":"always REM sleep", "CNN_20_ENMO_HRV": "CNN 20", "CNN_50_ENMO_HRV":"CNN 50", "CNN_100_ENMO_HRV":"CNN 100", "LSTM_20_ENMO_HRV":"LSTM 20", "LSTM_50_ENMO_HRV":"LSTM 50","LSTM_100_ENMO_HRV":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_4stages_df = pd.DataFrame(list(readable_4stages_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
stages_4_baselines = ['stages', 'always_0', 'always_1', 'always_2','always_3']
stage_4_df['algorithms'] = stage_4_df['algorithms'].apply(lambda x: readable_4stages_algs_dict[x])
# get ready the rename df
bls = readable_4stages_df.new_alg_name.unique().tolist()
stage_4_df['acc_for_sort'] = stage_4_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
stage_4_df[['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score","cohen-kappa","Wake","REM sleep","Deep sleep","Light sleep","acc_for_sort"]].sort_values(by="acc_for_sort", ascending=False)

print(stage_4_df[stage_4_df['algorithms'].isin(bls)].sort_values(by="acc_for_sort", ascending=False)[stages_4_clf_columns].to_latex(index=False))<jupyter_output>\begin{tabular}{lllllllllll}
\toprule
          algorithms &      accuracy &   specificity &     precision &        recall &      f1-score &   cohen-kappa &    Deep sleep &    Light sleep &     REM sleep &           Wake \\
\midrule
        ground truth &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &   39.3 +- 3.6 &   258.2 +- 7.0 &   67.6 +- 3.0 &   187.4 +- 8.6 \\
             LSTM 50 &   70.3 +- 1.0 &   87.4 +- 0.4 &   57.9 +- 1.3 &   54.0 +- 1.0 &   51.9 +- 1.0 &   53.8 +- 1.9 &  -36.2 +- 3.5 &    42.8 +- 7.4 &   -5.6 +- 4.0 &    -1.0 +- 6.9 \\
            LSTM 100 &   70.2 +- 1.0 &   86.9 +- 0.4 &   59.9 +- 1.5 &   52.4 +- 1.0 &   51.3 +- 1.1 &   51.7 +- 1.8 &  -32.4 +- 3.5 &    76.0 +- 7.3 &  -24.7 +- 3.7 &   -18.9 +- 6.6 \\
             CNN 100 &   69.0 +- 1.0 &   87.0 +- 0.4 &   58.0 +- 1.4 &   53.7 +- 1.0 &   51.2 +- 1.1 &   51.6 +- 1.8 &  -34.5 +- 3.5 &    46.1 +- 8.1 &    4.4 +- 4.8 &   -15.9 +- 7.5 \\
             LSTM 20 & [...]<jupyter_text>Task 3 4 stages HRV single modality<jupyter_code>num_classes = 4
stages_4_clf_columns= ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "Deep sleep","Light sleep","REM sleep","Wake"]
stage_4_df = load_task(sub_folder, num_classes=num_classes, feature_type='HRV', stages_clf_columns=stages_4_clf_columns) # here to change

readable_4stages_algs_dict= {"stages":"ground truth","always_0":"always wake", "always_1":"always light sleep", "always_2":"always deep sleep", "always_3":"always REM sleep", "CNN_20_HRV": "CNN 20", "CNN_50_HRV":"CNN 50", "CNN_100_HRV":"CNN 100", "LSTM_20_HRV":"LSTM 20", "LSTM_50_HRV":"LSTM 50","LSTM_100_HRV":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_4stages_df = pd.DataFrame(list(readable_4stages_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
bls = readable_4stages_df.new_alg_name.unique().tolist()
print(stage_4_df)
print(readable_4stages_df)
stage_4_df['algorithms'] = stage_4_df['algorithms'].apply(lambda x: readable_4stages_algs_dict[x])
stage_4_df['acc_for_sort'] = stage_4_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
stage_4_df[['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa","Wake","REM sleep", "Deep sleep","Light sleep","acc_for_sort"]].sort_values(by="acc_for_sort", ascending=False)
print(stage_4_df[stage_4_df['algorithms'].isin(bls)].sort_values(by="acc_for_sort", ascending=False)[stages_4_clf_columns].to_latex(index=False))<jupyter_output>\begin{tabular}{lllllllllll}
\toprule
          algorithms &      accuracy &   specificity &     precision &        recall &      f1-score &   cohen-kappa &     Deep sleep &     Light sleep &      REM sleep &           Wake \\
\midrule
        ground truth &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &    39.3 +- 3.6 &    258.2 +- 7.0 &    67.6 +- 3.0 &   187.4 +- 8.6 \\
            LSTM 100 &   67.4 +- 1.2 &   86.2 +- 0.4 &   56.2 +- 1.6 &   51.3 +- 1.1 &   49.5 +- 1.2 &   44.6 +- 2.2 &   -33.7 +- 3.5 &     60.4 +- 8.1 &   -13.5 +- 3.8 &   -13.1 +- 8.4 \\
             LSTM 50 &   66.2 +- 1.1 &   85.6 +- 0.4 &   54.4 +- 1.5 &   49.5 +- 1.1 &   47.4 +- 1.1 &   41.2 +- 2.1 &   -36.4 +- 3.5 &     65.9 +- 7.9 &   -14.6 +- 4.1 &   -15.0 +- 8.1 \\
             CNN 100 &   64.3 +- 1.1 &   85.3 +- 0.4 &   54.4 +- 1.6 &   50.2 +- 1.1 &   47.1 +- 1.1 &   40.9 +- 2.1 &   -34.8 +- 3.5 &     49.6 +- 8.9 &     8.2 +- 5.1 &   -23.0 +- 9.5 \\
        [...]<jupyter_text>Task 3 4 stages Single modality ENMO<jupyter_code>num_classes = 4
stages_4_clf_columns= ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score","cohen-kappa", "Deep sleep","Light sleep","REM sleep","Wake"]
stage_4_df = load_task(sub_folder, num_classes=num_classes, feature_type='ENMO', stages_clf_columns=stages_4_clf_columns)
readable_4stages_algs_dict= {"stages":"ground truth","always_0":"always wake", "always_1":"always light sleep", "always_2":"always deep sleep", "always_3":"always REM sleep", "CNN_20_ENMO": "CNN 20", "CNN_50_ENMO":"CNN 50", "CNN_100_ENMO":"CNN 100", "LSTM_20_ENMO":"LSTM 20", "LSTM_50_ENMO":"LSTM 50","LSTM_100_ENMO":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_4stages_df = pd.DataFrame(list(readable_4stages_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
bls = readable_4stages_df.new_alg_name.unique().tolist()
print(stage_4_df)
print(readable_4stages_df)
stage_4_df['algorithms'] = stage_4_df['algorithms'].apply(lambda x: readable_4stages_algs_dict[x])
stage_4_df['acc_for_sort'] = stage_4_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
stage_4_df[['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score","cohen-kappa","Wake","REM sleep", "Deep sleep","Light sleep","acc_for_sort"]].sort_values(by="acc_for_sort", ascending=False)
print(stage_4_df[stage_4_df['algorithms'].isin(bls)].sort_values(by="acc_for_sort", ascending=False)[stages_4_clf_columns].to_latex(index=False))<jupyter_output>\begin{tabular}{lllllllllll}
\toprule
          algorithms &      accuracy &   specificity &     precision &        recall &      f1-score &   cohen-kappa &    Deep sleep &    Light sleep &     REM sleep &           Wake \\
\midrule
        ground truth &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &   39.3 +- 3.6 &   258.2 +- 7.0 &   67.6 +- 3.0 &   187.4 +- 8.6 \\
            LSTM 100 &   64.1 +- 1.0 &   82.9 +- 0.5 &   35.6 +- 0.7 &   39.6 +- 0.7 &   35.8 +- 0.7 &   33.5 +- 1.4 &  -39.3 +- 3.6 &   139.4 +- 8.5 &  -67.6 +- 3.0 &   -32.5 +- 7.4 \\
             CNN 100 &   63.9 +- 1.0 &   83.0 +- 0.4 &   36.3 +- 0.9 &   39.6 +- 0.7 &   35.7 +- 0.7 &   33.5 +- 1.4 &  -39.3 +- 3.6 &   133.2 +- 8.7 &  -67.5 +- 3.0 &   -26.4 +- 7.6 \\
             LSTM 50 &   63.6 +- 1.0 &   82.7 +- 0.4 &   35.6 +- 0.8 &   39.3 +- 0.7 &   35.5 +- 0.7 &   33.0 +- 1.4 &  -39.3 +- 3.6 &   143.0 +- 8.2 &  -67.3 +- 3.0 &   -36.3 +- 7.1 \\
              CNN 50 & [...]<jupyter_text>================ Task 4 5 stages ====================    ENMO&HRV<jupyter_code>num_classes=5
stages_5_clf_columns= ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "N1 sleep","N2 sleep","N3 sleep","REM sleep","Wake"]
stage_5_df = load_task(sub_folder, num_classes, feature_type='ENMO_HRV', stages_clf_columns=stages_5_clf_columns)
readable_5stages_algs_dict= {"stages":"ground truth","always_0":"always wake", "always_1":"always N1 sleep", "always_2":"always N2 sleep", "always_3":"always N3 sleep", "always_4":"always REM sleep", "CNN_20_ENMO_HRV": "CNN 20", "CNN_50_ENMO_HRV":"CNN 50", "CNN_100_ENMO_HRV":"CNN 100", "LSTM_20_ENMO_HRV":"LSTM 20", "LSTM_50_ENMO_HRV":"LSTM 50","LSTM_100_ENMO_HRV":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_5stages_df = pd.DataFrame(list(readable_5stages_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
print(readable_5stages_df)
stages_5_baselines = ['stages', 'always_0', 'always_1', 'always_2','always_3',"always_4"]
stage_5_df['algorithms'] = stage_5_df['algorithms'].apply(lambda x: readable_5stages_algs_dict[x])
# get ready the rename df
bls = readable_5stages_df.new_alg_name.unique().tolist()
stage_5_df['acc_for_sort'] = stage_5_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))

stage_5_df[['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa","Wake","REM sleep","N3 sleep","N2 sleep", "N1 sleep","acc_for_sort"]].sort_values(by="acc_for_sort", ascending=False)
print(stage_5_df[stage_5_df['algorithms'].isin(bls)].sort_values(by="acc_for_sort", ascending=False)[stages_5_clf_columns].to_latex(index=False))<jupyter_output>\begin{tabular}{llllllllllll}
\toprule
          algorithms &      accuracy &   specificity &     precision &        recall &      f1-score &   cohen-kappa &      N1 sleep &       N2 sleep &      N3 sleep &     REM sleep &           Wake \\
\midrule
        ground truth &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &   49.4 +- 3.2 &   208.7 +- 6.2 &   39.3 +- 3.6 &   67.6 +- 3.0 &   187.4 +- 8.6 \\
             LSTM 50 &   63.7 +- 1.0 &   88.7 +- 0.3 &   47.1 +- 1.4 &   43.0 +- 0.8 &   39.9 +- 0.8 &   56.3 +- 1.8 &  -46.0 +- 3.0 &    71.9 +- 7.5 &  -35.2 +- 3.5 &  -12.9 +- 3.9 &    22.2 +- 7.1 \\
            LSTM 100 &   63.6 +- 1.0 &   88.7 +- 0.3 &   47.8 +- 1.3 &   43.3 +- 0.8 &   40.5 +- 0.9 &   57.0 +- 1.8 &  -46.2 +- 3.0 &    97.7 +- 7.5 &  -32.3 +- 3.5 &  -15.9 +- 3.9 &    -3.3 +- 6.8 \\
             CNN 100 &   63.1 +- 1.1 &   88.8 +- 0.3 &   51.5 +- 1.4 &   44.7 +- 0.9 &   41.9 +- 0.9 &   56.2 +- 1.8 &  -40.2 +- 3.1 &    92.4 +[...]<jupyter_text>Task 4 5 stages, single modality ENMO<jupyter_code>num_classes = 5
stages_clf_columns= ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "N1 sleep","N2 sleep","N3 sleep","REM sleep","Wake"]
stage_5_df = load_task(sub_folder, num_classes=num_classes, feature_type='ENMO', stages_clf_columns=stages_clf_columns)
readable_5stages_algs_dict= {"stages":"ground truth","always_0":"always wake", "always_1":"always N1 sleep", "always_2":"always N2 sleep", "always_3":"always N3 sleep", "always_4":"always REM sleep", "CNN_20_ENMO": "CNN 20", "CNN_50_ENMO":"CNN 50", "CNN_100_ENMO":"CNN 100", "LSTM_20_ENMO":"LSTM 20", "LSTM_50_ENMO":"LSTM 50","LSTM_100_ENMO":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
readable_5stages_df = pd.DataFrame(list(readable_5stages_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
bls = readable_5stages_df.new_alg_name.unique().tolist()
# print(stage_5_df)
# print(readable_5stages_df)
stage_5_df['algorithms'] = stage_5_df['algorithms'].apply(lambda x: readable_5stages_algs_dict[x])
stage_5_df['acc_for_sort'] = stage_5_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
stage_5_df[['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa","Wake","REM sleep","N3 sleep","N2 sleep", "N1 sleep","acc_for_sort"]].sort_values(by="acc_for_sort", ascending=False)
print(stage_5_df[stage_5_df['algorithms'].isin(bls)].sort_values(by="acc_for_sort", ascending=False)[stages_clf_columns].to_latex(index=False))<jupyter_output>\begin{tabular}{llllllllllll}
\toprule
          algorithms &      accuracy &   specificity &     precision &        recall &      f1-score &   cohen-kappa &      N1 sleep &       N2 sleep &      N3 sleep &     REM sleep &           Wake \\
\midrule
        ground truth &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &   49.4 +- 3.2 &   208.7 +- 6.2 &   39.3 +- 3.6 &   67.6 +- 3.0 &   187.4 +- 8.6 \\
             LSTM 50 &   56.9 +- 1.0 &   85.7 +- 0.4 &   26.1 +- 0.8 &   32.2 +- 0.6 &   27.1 +- 0.7 &   46.9 +- 1.7 &  -49.4 +- 3.2 &   169.2 +- 8.5 &  -39.3 +- 3.6 &  -67.6 +- 3.0 &   -12.9 +- 7.5 \\
            LSTM 100 &   56.9 +- 1.0 &   85.7 +- 0.4 &   25.3 +- 0.7 &   32.3 +- 0.6 &   27.1 +- 0.7 &   47.1 +- 1.7 &  -49.4 +- 3.2 &   159.7 +- 8.7 &  -39.3 +- 3.6 &  -67.6 +- 3.0 &    -3.3 +- 7.5 \\
             CNN 100 &   56.8 +- 1.1 &   85.8 +- 0.3 &   27.7 +- 0.9 &   32.2 +- 0.5 &   27.2 +- 0.6 &   46.9 +- 1.7 &  -49.4 +- 3.2 &   144.7 +[...]<jupyter_text>Task 4 5 stages only HRV<jupyter_code>num_classes = 5
stages_clf_columns= ['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa", "N1 sleep","N2 sleep","N3 sleep","REM sleep","Wake"]
stage_5_df = load_task(sub_folder,num_classes=num_classes, feature_type='HRV', stages_clf_columns=stages_clf_columns)
readable_5stages_algs_dict= {"stages":"ground truth","always_0":"always wake", "always_1":"always N1 sleep", "always_2":"always N2 sleep", "always_3":"always N3 sleep", "always_4":"always REM sleep", "CNN_20_HRV": "CNN 20", "CNN_50_HRV":"CNN 50", "CNN_100_HRV":"CNN 100", "LSTM_20_HRV":"LSTM 20", "LSTM_50_HRV":"LSTM 50","LSTM_100_HRV":"LSTM 100", "SGD_hinge":"Linear SVM", "SGD_log":"Logistic Regression", "SGD_perceptron":"Perception", "Random_forest_300": "Random Forest"}
rreadable_5stages_df = pd.DataFrame(list(readable_5stages_algs_dict.items()), columns=["old_alg_name","new_alg_name"])
bls = readable_5stages_df.new_alg_name.unique().tolist()
# print(stage_5_df)
# print(readable_5stages_df)
stage_5_df['algorithms'] = stage_5_df['algorithms'].apply(lambda x: readable_5stages_algs_dict[x])
stage_5_df['acc_for_sort'] = stage_5_df['accuracy'].apply(lambda x: np.float(x.split("+-")[0]))
stage_5_df[['algorithms', "accuracy", "specificity", "precision", "recall", "f1-score", "cohen-kappa","Wake","REM sleep","N3 sleep","N2 sleep", "N1 sleep","acc_for_sort"]].sort_values(by="acc_for_sort", ascending=False)
print(stage_5_df[stage_5_df['algorithms'].isin(bls)].sort_values(by="acc_for_sort", ascending=False)[stages_clf_columns].to_latex(index=False))<jupyter_output>\begin{tabular}{llllllllllll}
\toprule
          algorithms &      accuracy &   specificity &     precision &        recall &      f1-score &   cohen-kappa &      N1 sleep &       N2 sleep &       N3 sleep &     REM sleep &           Wake \\
\midrule
        ground truth &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &  100.0 +- 0.0 &   49.4 +- 3.2 &   208.7 +- 6.2 &    39.3 +- 3.6 &   67.6 +- 3.0 &   187.4 +- 8.6 \\
              CNN 20 &   55.6 +- 1.1 &   86.4 +- 0.3 &   40.4 +- 1.2 &   37.3 +- 0.8 &   33.6 +- 0.9 &   36.2 +- 1.8 &  -44.4 +- 3.0 &   103.0 +- 9.8 &   -39.1 +- 3.6 &   -3.9 +- 5.8 &  -15.6 +- 10.1 \\
             CNN 100 &   55.6 +- 1.1 &   86.7 +- 0.3 &   44.9 +- 1.4 &   38.9 +- 0.9 &   35.9 +- 1.0 &   37.1 +- 1.8 &  -40.8 +- 3.1 &    81.2 +- 9.4 &   -29.5 +- 3.6 &  -12.0 +- 4.8 &    1.1 +- 10.7 \\
              CNN 50 &   54.2 +- 1.1 &   86.0 +- 0.3 &   41.2 +- 1.3 &   35.6 +- 0.8 &   32.1 +- 1.0 &   32.3 +- 1.9 &  -40.6 +- 3.1 &   69.[...]<jupyter_text>To print  the latex of HRV feature description<jupyter_code>hrv_description = pd.read_csv("C:/tmp/hrv_feature-description.csv")
pd.set_option('max_colwidth',500)
print(hrv_description.to_latex(index=False, float_format="%s"))<jupyter_output>\begin{tabular}{ll}
\toprule
             SDNN &                                                                                                                                                     Standard deviation of Normal-to-Normal interval \\
\midrule
             SDSD &                                                                                                                                                               Standard deviation of NNI differences \\
             NN50 &                                                                                                                                                      Number of NN interval differences greater 50ms \\
            pNN50 &                                                                                                                                                 Ratio between NN50 and total number of NN intervals \\
             NN20 &                                                                   [...]<jupyter_text>This is for task 4: 5 stages tasks multimodality HRV & ENMO<jupyter_code># ml_dl_algorithms = []
# print(summary.loc[traditional].sort_values(by="Accuracy", ascending=False)[metrics].to_latex())

# t = summary.loc[traditional].sort_values(by="Accuracy", ascending=False)[metrics]
# print(get_group_average(t, metrics))
traditionalresc = ["resc_" + alg for alg in defaultalgs]
print(summary.loc[traditionalresc].sort_values(by="Accuracy", ascending=False)[metrics].to_latex())

t = summary.loc[traditionalresc].sort_values(by="Accuracy", ascending=False)[metrics]
print(get_group_average(t, metrics))
ml = [alg for alg in defaultml]
print(summary.loc[ml].sort_values(by="Accuracy", ascending=False)[metrics].to_latex())


t = summary.loc[ml].sort_values(by="Accuracy", ascending=False)[metrics]
print(get_group_average(t, metrics))
mlresc = ["resc_" + alg for alg in defaultml]
print(summary.loc[mlresc].sort_values(by="Accuracy", ascending=False)[metrics].to_latex())

t = summary.loc[mlresc].sort_values(by="Accuracy", ascending=False)[metrics]
print(get_group_average(t, metrics))
dl = [alg for alg in defaultdl]
print(summary.loc[dl].sort_values(by="Accuracy", ascending=False)[metrics].to_latex())

t = summary.loc[dl].sort_values(by="Accuracy", ascending=False)[metrics]
print(get_group_average(t, metrics))
dlresc = ["resc_" + alg for alg in defaultdl]
print(summary.loc[dlresc].sort_values(by="Accuracy", ascending=False)[metrics].to_latex())

t = summary.loc[dlresc].sort_values(by="Accuracy", ascending=False)[metrics]
print(get_group_average(t, metrics))<jupyter_output><empty_output><jupyter_text>T test start from here   Ensemble method  to compare the F1, accuracy, and kappa between ensemble method and 2 stages ENMO_HRV  task 1<jupyter_code># setup the comparision
num_classes = 2
feature_type = 'ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 2
ENMO_HRV_PICKLE_RESULT_FILE = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type))
with open(ENMO_HRV_PICKLE_RESULT_FILE, "rb") as f:
    task_results = pickle.load(f)

# data loading part for ensemble
ENSEMBLE_PICKLE_RESULT_FILE = os.path.join(result_folder, "ensemble_all", "%d_stages_ensemble_results_%s.pkl"
                                          % (num_classes, feature_type))
with open(ENSEMBLE_PICKLE_RESULT_FILE, "rb") as f:
    ensemble_results = pickle.load(f)
print("test dataset size is: ")
print(ensemble_results['mean_ensemble'].shape)
print("non ensemble")
print(task_results.keys())
print(ensemble_results.keys())
ttest(task_results, ensemble_results,'CNN_100_ENMO_HRV', 'mean_ensemble', ['f1-score', 'accuracy','cohen-kappa'])
ttest(task_results, ensemble_results,'LSTM_100_ENMO_HRV', 'mean_ensemble', ['f1-score'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.218<jupyter_text>to compare the F1, accuracy, and kappa between ensemble method and 3 stages ENMO_HRV  task 2<jupyter_code># setup the comparision
num_classes = 3
feature_type = 'ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 2
ENMO_HRV_PICKLE_RESULT_FILE = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type))
with open(ENMO_HRV_PICKLE_RESULT_FILE, "rb") as f:
    task_results = pickle.load(f)

# data loading part for ensemble
ENSEMBLE_PICKLE_RESULT_FILE = os.path.join(result_folder, "ensemble_all", "%d_stages_ensemble_results_%s.pkl"
                                          % (num_classes, feature_type))
with open(ENSEMBLE_PICKLE_RESULT_FILE, "rb") as f:
    ensemble_results = pickle.load(f)
print("test dataset size is: ")
print(ensemble_results['mean_ensemble'].shape)
print("non ensemble")
print(task_results.keys())
print(ensemble_results.keys())
ttest(task_results, ensemble_results,'LSTM_50_ENMO_HRV', 'mean_ensemble', ['f1-score', 'accuracy','cohen-kappa'])
ttest(task_results, ensemble_results,'CNN_100_ENMO_HRV', 'mean_ensemble', ['f1-score', 'accuracy','cohen-kappa'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.065
T-Test for metric: accuracy, the p value = 0.002
T-Test for metric: cohen-kappa, the p value = 0.015<jupyter_text>to compare the F1, accuracy, and kappa between ensemble method and 4 stages ENMO_HRV  task 3<jupyter_code># setup the comparision
num_classes = 4
feature_type = 'ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 2
ENMO_HRV_PICKLE_RESULT_FILE = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type))
with open(ENMO_HRV_PICKLE_RESULT_FILE, "rb") as f:
    task_results = pickle.load(f)

# data loading part for ensemble
ENSEMBLE_PICKLE_RESULT_FILE = os.path.join(result_folder, "ensemble_all", "%d_stages_ensemble_results_%s.pkl"
                                          % (num_classes, feature_type))
with open(ENSEMBLE_PICKLE_RESULT_FILE, "rb") as f:
    ensemble_results = pickle.load(f)
print("test dataset size is: ")
print(ensemble_results['mean_ensemble'].shape)
print("non ensemble")
print(task_results.keys())
print(ensemble_results.keys())
ttest(task_results, ensemble_results,'LSTM_50_ENMO_HRV', 'mean_ensemble', ['f1-score', 'accuracy','cohen-kappa'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.772
T-Test for metric: accuracy, the p value = 0.067
T-Test for metric: cohen-kappa, the p value = 0.087<jupyter_text>to compare the F1, accuracy, and kappa between ensemble method and 5 stages ENMO_HRV  task 4<jupyter_code># setup the comparision
num_classes = 5
feature_type = 'ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 2
ENMO_HRV_PICKLE_RESULT_FILE = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type))
with open(ENMO_HRV_PICKLE_RESULT_FILE, "rb") as f:
    task_results = pickle.load(f)

# data loading part for ensemble
ENSEMBLE_PICKLE_RESULT_FILE = os.path.join(result_folder, "ensemble_all", "%d_stages_ensemble_results_%s.pkl"
                                          % (num_classes, feature_type))
with open(ENSEMBLE_PICKLE_RESULT_FILE, "rb") as f:
    ensemble_results = pickle.load(f)
print("test dataset size is: ")
print(ensemble_results['mean_ensemble'].shape)
print("non ensemble")
print(task_results.keys())
print(ensemble_results.keys())
ttest(task_results, ensemble_results,'LSTM_50_ENMO_HRV', 'mean_ensemble', ['f1-score', 'accuracy','cohen-kappa'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.028
T-Test for metric: accuracy, the p value = 0.016
T-Test for metric: cohen-kappa, the p value = 0.004<jupyter_text>Non ensemble method: Task 1   compare between combined snesing ENMO + HRV vs ENMO<jupyter_code># setup the comparision
num_classes = 2
feature_type1 = 'ENMO_HRV'
feature_type2 = 'ENMO'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 2
ENMO_HRV_PICKLE_RESULT_FILE1 = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type1))
with open(ENMO_HRV_PICKLE_RESULT_FILE1, "rb") as f:
    task_results1 = pickle.load(f)

# data loading part for ensemble
ENMO_HRV_PICKLE_RESULT_FILE2 = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type2))
with open(ENMO_HRV_PICKLE_RESULT_FILE2, "rb") as f:
    task_results2 = pickle.load(f)
print("test dataset size is: ")
print("non ensemble")
print(task_results1.keys())
print(task_results2.keys())
ttest(task_results1, task_results2,'CNN_100_ENMO_HRV', 'CNN_100_ENMO', ['f1-score', 'accuracy','cohen-kappa'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.347
T-Test for metric: accuracy, the p value = 0.499
T-Test for metric: cohen-kappa, the p value = 0.506<jupyter_text>compare between combined snesing ENMO + HRV vs HRV<jupyter_code>['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity']<jupyter_output><empty_output><jupyter_text>Non ensemble method: Task 2<jupyter_code># setup the comparision
num_classes = 3
feature_type1 = 'ENMO_HRV'
feature_type2 = 'ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 2
ENMO_HRV_PICKLE_RESULT_FILE1 = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type1))
with open(ENMO_HRV_PICKLE_RESULT_FILE1, "rb") as f:
    task_results1 = pickle.load(f)

# data loading part for ensemble
ENMO_HRV_PICKLE_RESULT_FILE2 = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type2))
with open(ENMO_HRV_PICKLE_RESULT_FILE2, "rb") as f:
    task_results2 = pickle.load(f)
print("test dataset size is: ")
print("non ensemble")
print(task_results1.keys())
print(task_results2.keys())
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'Random_forest_300', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'CNN_20_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'LSTM_100_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'LSTM_20_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'CNN_50_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'CNN_100_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.138
T-Test for metric: accuracy, the p value = 0.364
T-Test for metric: cohen-kappa, the p value = nan
T-Test for metric: precision, the p value = 0.031
T-Test for metric: recall, the p value = 0.063
T-Test for metric: specificity, the p value = 0.399<jupyter_text>Compare between multi modality task2 vs task 1 best classifier metrics<jupyter_code># setup the comparision
num_classes1 = 3
num_classes2 = 2
feature_type1 = 'ENMO_HRV'
feature_type2 = 'ENMO_HRV'
result_folder1 = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes1]
# data loading part for first group
ENMO_HRV_PICKLE_RESULT_FILE1 = os.path.join(result_folder1, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes1, feature_type1))
with open(ENMO_HRV_PICKLE_RESULT_FILE1, "rb") as f:
    task_results1 = pickle.load(f)

# data loading part for second group
result_folder2 = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes2]
ENMO_HRV_PICKLE_RESULT_FILE2 = os.path.join(result_folder2, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes2, feature_type2))
with open(ENMO_HRV_PICKLE_RESULT_FILE2, "rb") as f:
    task_results2 = pickle.load(f)
print("test dataset size is: ")
print("non ensemble")
print(task_results1.keys())
print(task_results2.keys())
ttest(task_results1, task_results2,'LSTM_50_ENMO_HRV', 'LSTM_50_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results2,'LSTM_100_ENMO_HRV', 'LSTM_100_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results2,'LSTM_20_ENMO_HRV', 'LSTM_20_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.000
T-Test for metric: accuracy, the p value = 0.000
T-Test for metric: cohen-kappa, the p value = 0.001
T-Test for metric: precision, the p value = 0.000
T-Test for metric: recall, the p value = 0.000
T-Test for metric: specificity, the p value = 0.000<jupyter_text>Compare between multi modality task3 best classifier vs the single modality on the best classifiers<jupyter_code># setup the comparision
num_classes1 = 3
num_classes2 = 3
feature_type1 = 'ENMO_HRV'
feature_type2 = 'HRV'
result_folder1 = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes1]
# data loading part for first group
ENMO_HRV_PICKLE_RESULT_FILE1 = os.path.join(result_folder1, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes1, feature_type1))
with open(ENMO_HRV_PICKLE_RESULT_FILE1, "rb") as f:
    task_results1 = pickle.load(f)

# data loading part for second group
result_folder2 = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes2]
ENMO_HRV_PICKLE_RESULT_FILE2 = os.path.join(result_folder2, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes2, feature_type2))
with open(ENMO_HRV_PICKLE_RESULT_FILE2, "rb") as f:
    task_results2 = pickle.load(f)
print("test dataset size is: ")
print("non ensemble")
print(task_results1.keys())
print(task_results2.keys())
# will combined sensing in three stages classification is better than single modality HRV? for the best classifer on each group?
ttest(task_results1, task_results2,'LSTM_50_ENMO_HRV', 'LSTM_100_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.002
T-Test for metric: accuracy, the p value = 0.002
T-Test for metric: cohen-kappa, the p value = 0.000
T-Test for metric: precision, the p value = 0.018
T-Test for metric: recall, the p value = 0.003
T-Test for metric: specificity, the p value = 0.001<jupyter_text>Non ensemble method: Task 3  Compare between multi modality task3 best classifier vs the rest<jupyter_code># setup the comparision
num_classes = 4
feature_type1 = 'ENMO_HRV'
feature_type2 = 'ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 2
ENMO_HRV_PICKLE_RESULT_FILE1 = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type1))
with open(ENMO_HRV_PICKLE_RESULT_FILE1, "rb") as f:
    task_results1 = pickle.load(f)

# data loading part for ensemble
ENMO_HRV_PICKLE_RESULT_FILE2 = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type2))
with open(ENMO_HRV_PICKLE_RESULT_FILE2, "rb") as f:
    task_results2 = pickle.load(f)
print("test dataset size is: ")
print("non ensemble")
print(task_results1.keys())
print(task_results2.keys())
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'LSTM_100_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'CNN_100_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'LSTM_20_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'CNN_20_ENMO_HRV', 'Random_forest_300', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'CNN_20_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.000
T-Test for metric: accuracy, the p value = 0.000
T-Test for metric: cohen-kappa, the p value = 0.000
T-Test for metric: precision, the p value = 0.000
T-Test for metric: recall, the p value = 0.000
T-Test for metric: specificity, the p value = 0.000<jupyter_text>Compare the 4 stages ENMO_HRV vs HRV<jupyter_code># setup the comparision
num_classes1 = 4
num_classes2 = 4
feature_type1 = 'ENMO_HRV'
feature_type2 = 'HRV'
result_folder1 = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes1]
# data loading part for first group
ENMO_HRV_PICKLE_RESULT_FILE1 = os.path.join(result_folder1, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes1, feature_type1))
with open(ENMO_HRV_PICKLE_RESULT_FILE1, "rb") as f:
    task_results1 = pickle.load(f)

# data loading part for second group
result_folder2 = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes2]
ENMO_HRV_PICKLE_RESULT_FILE2 = os.path.join(result_folder2, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes2, feature_type2))
with open(ENMO_HRV_PICKLE_RESULT_FILE2, "rb") as f:
    task_results2 = pickle.load(f)
print("test dataset size is: ")
print("non ensemble")
print(task_results1.keys())
print(task_results2.keys())
# will combined sensing in 4 stages classification is better than single modality HRV? for the best classifer on each group?
ttest(task_results1, task_results2,'LSTM_50_ENMO_HRV', 'LSTM_100_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.004
T-Test for metric: accuracy, the p value = 0.000
T-Test for metric: cohen-kappa, the p value = 0.000
T-Test for metric: precision, the p value = 0.113
T-Test for metric: recall, the p value = 0.001
T-Test for metric: specificity, the p value = 0.000<jupyter_text>Non ensemble method: Task 4<jupyter_code># setup the comparision
num_classes = 5
feature_type1 = 'ENMO_HRV'
feature_type2 = 'ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 4
ENMO_HRV_PICKLE_RESULT_FILE1 = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type1))
with open(ENMO_HRV_PICKLE_RESULT_FILE1, "rb") as f:
    task_results1 = pickle.load(f)

# data loading part for ensemble
ENMO_HRV_PICKLE_RESULT_FILE2 = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type2))
with open(ENMO_HRV_PICKLE_RESULT_FILE2, "rb") as f:
    task_results2 = pickle.load(f)
print("test dataset size is: ")
print("non ensemble")
print(task_results1.keys())
print(task_results2.keys())
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'LSTM_100_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'CNN_100_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'CNN_50_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'SGD_log', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'LSTM_20_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])
ttest(task_results1, task_results1,'LSTM_50_ENMO_HRV', 'CNN_20_ENMO_HRV', ['f1-score', 'accuracy','cohen-kappa', 'precision', 'recall', 'specificity'])<jupyter_output>T-Test for metric: f1-score, the p value = 0.067
T-Test for metric: accuracy, the p value = 0.001
T-Test for metric: cohen-kappa, the p value = 0.013
T-Test for metric: precision, the p value = 0.009
T-Test for metric: recall, the p value = 0.632
T-Test for metric: specificity, the p value = 0.055<jupyter_text>This is the data science way to T test<jupyter_code># setup the comparision
num_classes = 3
feature_type = 'ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 2
ENMO_HRV_PICKLE_RESULT_FILE = os.path.join(result_folder, sub_folder, "%d_stages_results_%s.pkl"
                                          % (num_classes, feature_type))
with open(ENMO_HRV_PICKLE_RESULT_FILE, "rb") as f:
    task_results = pickle.load(f)

# data loading part for ensemble
ENSEMBLE_PICKLE_RESULT_FILE = os.path.join(result_folder, "ensemble_all", "%d_stages_ensemble_results_%s.pkl"
                                          % (num_classes, feature_type))
with open(ENSEMBLE_PICKLE_RESULT_FILE, "rb") as f:
    ensemble_results = pickle.load(f)
print("test dataset size is: ")
print(ensemble_results['mean_ensemble'].shape)
print("non ensemble")
print(task_results.keys())
print(ensemble_results.keys())<jupyter_output>test dataset size is: 
(348, 6)
non ensemble
dict_keys(['always_0', 'always_1', 'always_2', 'CNN_20_ENMO_HRV', 'CNN_50_ENMO_HRV', 'CNN_100_ENMO_HRV', 'LSTM_20_ENMO_HRV', 'LSTM_50_ENMO_HRV', 'LSTM_100_ENMO_HRV', 'SGD_hinge', 'SGD_log', 'SGD_perceptron', 'Random_forest_300', 'stages'])
dict_keys(['stages', 'CNN_20', 'CNN_50', 'CNN_100', 'LSTM_20', 'LSTM_50', 'LSTM_100', 'max_ensemble', 'mean_ensemble'])<jupyter_text>This is the T test for time deviation<jupyter_code>num_classes = 3
feature_type1 = 'ENMO_HRV'
feature_type2 = 'ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 4
ENMO_HRV_PICKLE_RESULT_FILE1 = os.path.join(result_folder, ANALYSIS_SUB_FOLDER["recording_period"], "%d_stages_minutes_results_%s.pkl"
                                          % (num_classes, feature_type1))
with open(ENMO_HRV_PICKLE_RESULT_FILE1, "rb") as f:
    task_results1 = pickle.load(f)

# data loading part for ensemble
ENMO_HRV_PICKLE_RESULT_FILE2 = os.path.join(result_folder, ANALYSIS_SUB_FOLDER["recording_period"], "%d_stages_minutes_results_%s.pkl"
                                          % (num_classes, feature_type2))
with open(ENMO_HRV_PICKLE_RESULT_FILE2, "rb") as f:
    task_results2 = pickle.load(f)
print("test dataset size is: ")
print("non ensemble")
print(task_results1.keys())
print(task_results2.keys())
ttest(task_results1, task_results1,'CNN_100_ENMO_HRV', 'LSTM_100_ENMO_HRV', ['0','1', '2'])
num_classes = 4
feature_type1 = 'ENMO_HRV'
feature_type2 = 'ENMO_HRV'
result_folder = cfg.STAGE_OUTPUT_FOLDER_HRV30s[num_classes]
# data loading part for task 4
ENMO_HRV_PICKLE_RESULT_FILE1 = os.path.join(result_folder, ANALYSIS_SUB_FOLDER["recording_period"], "%d_stages_minutes_results_%s.pkl"
                                          % (num_classes, feature_type1))
with open(ENMO_HRV_PICKLE_RESULT_FILE1, "rb") as f:
    task_results1 = pickle.load(f)

# data loading part for ensemble
ENMO_HRV_PICKLE_RESULT_FILE2 = os.path.join(result_folder, ANALYSIS_SUB_FOLDER["recording_period"], "%d_stages_minutes_results_%s.pkl"
                                          % (num_classes, feature_type2))
with open(ENMO_HRV_PICKLE_RESULT_FILE2, "rb") as f:
    task_results2 = pickle.load(f)
print("test dataset size is: ")
print("non ensemble")
print(task_results1.keys())
print(task_results2.keys())
ttest(task_results1, task_results1,'CNN_100_ENMO_HRV', 'LSTM_50_ENMO_HRV', ['0','1', '2','3'])<jupyter_output>T-Test for metric: 0, the p value = 0.004
T-Test for metric: 1, the p value = 0.558
T-Test for metric: 2, the p value = 0.501
T-Test for metric: 3, the p value = 0.002
{'fraction_non_alphanumeric': 0.16482283148949817, 'fraction_numerical': 0.06545169508132472, 'mean_word_length': 3.8588245477365186, 'pattern_counts': {'":': 175, '<': 109, '<?xml version=': 0, '>': 109, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 10, 'word_list_counts': {'cursed_substrings.json': 91, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '33', 'dir': 'jupyter-structured-clean-dedup', 'id': '323206', 'license': 'MIT', 'n_tokens_mistral': 23901, 'n_tokens_neox': 21848, 'n_words': 3971, 'path': 'notebooks/Results Analysis.ipynb', 'repo_name': 'bzhai/multimodal_sleep_stages_benchmark', 'size': '56133'}