<jupyter_start><jupyter_text>Load Dataset<jupyter_code>def load_dataset(per_day,embedding_method,embedding_size):
    if per_day: 
        df_name='Timeseries_per_patient_per_day_LSTM_Data_embedded_'
        data_structure='per_patient'
    else : 
        df_name='Timeseries_per_patient_LSTM_Data_embedded_'
        data_structure='per_patient'
    if embedding_method=='cbow':
        df_name=df_name+'cbow_dim{}_win5_mc0'.format(embedding_size)
    if embedding_method=='skipgram':
        df_name=df_name+'skipgram_dim{}_win5_mc0'.format(embedding_size)
    print(df_name)
    with open("Cohort/Time_Series/"+df_name+'.txt', "rb") as fp:   # Unpickling
        data = pickle.load(fp)
    data_sample= data[:300]
    return data,data_sample, df_name, data_structure<jupyter_output><empty_output><jupyter_text>Load Model<jupyter_code>def load_lstm_representation(data_structure,timesteps,n_features,layer_size_1,layer_size_2,activation_func,optimizer_func,loss_func,n_epochs,n_batch_size,df_name,data):
    #create experiment Name to Load model 
    #experiment_name=df_name+'_'+data_structure+str(layer_size_1)+'_'+str(layer_size_2)+'_'+activation_func+'_'+optimizer_func+'_'+loss_func+'_'+str(n_epochs)+'_'+str(n_batch_size)
    experiment_name=df_name+'_'+data_structure+str(layer_size_1)+'_'+str(layer_size_2)+'_'+activation_func+'_'+optimizer_func+'_'+loss_func+'_'+str(n_epochs)+'_'+str(n_batch_size)


    
    model = keras.models.load_model('Cohort/LSTM_Models/'+experiment_name)
    hidden_representation = Sequential() 
    hidden_representation.add(model.layers[0]) 
    hidden_representation.add(model.layers[1])
    dim_red_data=hidden_representation.predict(data)
    print('data transformed')
    return dim_red_data<jupyter_output><empty_output><jupyter_text>kmeans<jupyter_code>def apply_kmeans(dim_red_data,cluster,ellbow_method):
    if ellbow_method==True:
        elbow_method(dim_red_data)
    kmeans = KMeans(n_clusters=cluster, init='k-means++', max_iter=5000, n_init=10, random_state=0)
    pred_y = kmeans.fit_predict(dim_red_data)
    return kmeans.labels_
def elbow_method(transformed_sample): 
    wcss = []
    for i in range(1, 11):
        kmeans = KMeans(n_clusters=i, init='k-means++', max_iter=300, n_init=10, random_state=0)
        kmeans.fit(transformed_sample)
        wcss.append(kmeans.inertia_)
    plt.plot(range(1, 11), wcss)
    plt.title('Elbow Method')
    plt.xlabel('Number of clusters')
    plt.ylabel('WCSS')
    plt.show()<jupyter_output><empty_output><jupyter_text>Hierarchical<jupyter_code>def apply_hierachical(dim_red_data,n_cluster):
    Dendogram=False
    #Dendogram: 
    if Dendogram==True:
        try: 
        #load Dendogram image
            plt.figure(figsize=(250, 7))
            experiment_name=experiment_name
            img=mpimg.imread('Cohort/LSTM/Plots/Dendograms/'+experiment_name+'.png')
            print('Dendogram loaded')
            imgplot = plt.imshow(img)
            plt.show()
        except:
            plt.figure(figsize=(10, 7))
            experiment_name=df_name+'_'+num_scaler_name+'_'+cat_scaler_name+'_'+dim_red_method+'_'+str(dimension)+'_hierachical_'
            plt.title(experiment_name)
            dend = shc.dendrogram(shc.linkage(df_dim_red, method='ward'))  
            plt.savefig('Cohort/LSTM/Plots/Dendograms/'+experiment_name+'.png')

    # setting distance_threshold=0 ensures we compute the full tree.
    model = AgglomerativeClustering(n_clusters=n_cluster, affinity='euclidean', linkage='ward',distance_threshold=None)

    
    model.fit_predict(dim_red_data)
    #scatter_plot(dim_red_data,model.labels_)
   
    
    return model.labels_<jupyter_output><empty_output><jupyter_text>Plotting of Clusters<jupyter_code>def apply_umap(dim_red_data,dimension):
    clusterable_embedding = umap.UMAP(
        n_neighbors=50,
        min_dist=0.1,
        n_components=dimension,
        random_state=42,
    )
    X_transformed=clusterable_embedding.fit_transform(dim_red_data)
    return X_transformed
def scatter_plot(df,labels):
    sns.set(style='white', rc={'figure.figsize':(10,8)})
    sns.color_palette("Set2")
    plt.scatter(df[:, 0], df[:, 1], c=labels, s=0.1, cmap='Accent');
    plt.show()<jupyter_output><empty_output><jupyter_text>Metrics for Clusters  Silhouette Coefficient<jupyter_code>def get_silhouette_Coefficient(labels,df):
    m=metrics.silhouette_score(df, labels, metric='euclidean')
    print('silhouette_score:',m)
    return m<jupyter_output><empty_output><jupyter_text>Calinski-Harabasz Index<jupyter_code>def get_calinski_harabasz(labels,df):
    m=metrics.calinski_harabasz_score(df, labels)
    print('Calinski-Harabasz Index:',m)
    return m<jupyter_output><empty_output><jupyter_text>Davies-Bouldin Index<jupyter_code>def get_davies_bouldin(labels,df):
    m=davies_bouldin_score(df, labels)
    print('Davies-Bouldin Index:',m)
    return m<jupyter_output><empty_output><jupyter_text>Cluster informations<jupyter_code>#function for statistics: 
def get_base_characteristic_value(df , characteristic , kind):    
    if kind=="mean": 
        df_mean=df[characteristic].mean()
        df_std= df[characteristic].std()
        df_max= df[characteristic].max()
        df_min= df[characteristic].min()
        if characteristic == "HF_Onset_age_in_days":
            base_characteristics_cohort=pd.DataFrame({'Variable': [characteristic+"_mean", characteristic+"_std", characteristic+"_max", characteristic+"_min"],
                                                  'Value': [(df_mean/365), (df_std/365), (df_max/365), (df_min/365)],})
        else:
            base_characteristics_cohort=pd.DataFrame({'Variable': [characteristic+"_mean", characteristic+"_std", characteristic+"_max", characteristic+"_min"],
                                                  'Value': [(df_mean), (df_std), (df_max), (df_min)],})
        
    if kind=="count":
        base_characteristics_cohort=pd.DataFrame(columns=["Variable","Value"])
        feature_value=df[characteristic].unique()
        #print(feature_value)
        for value in feature_value: 
            df_condition=df.loc[df[characteristic]==value]
            df_percent= df_condition.shape[0]/df.shape[0]
            #print(df_percent)
            new_row1 = {'Variable': value+"_total",'Value': df_condition.shape[0]}
            new_row2 = {'Variable': value+"_relation",'Value': df_percent}
            base_characteristics_cohort=base_characteristics_cohort.append(new_row1, ignore_index=True)
            base_characteristics_cohort=base_characteristics_cohort.append(new_row2, ignore_index=True)
       # print(df_condition.shape[0], df_percent)
    #print (base_characteristics_cohort)
    return base_characteristics_cohort

def get_base_characteristics(df, characteristics): 
    base_characteristics_cohort=pd.DataFrame(columns=["Variable","Value"])
    for characteristic in characteristics:
        intermediate_base_characteristics_cohort=get_base_characteristic_value(df,characteristic[0],characteristic[1])
        base_characteristics_cohort=pd.concat([base_characteristics_cohort,intermediate_base_characteristics_cohort])
    #print(base_characteristics_cohort)
    return base_characteristics_cohort

def get_cluster_information(df,dim_red_method,base_characteristics):
    
    baseline_characteristics=[]
    for cluster in df[dim_red_method].unique(): 
        cluster_characteristics=[]
        df_temp=df.loc[df[dim_red_method] == cluster]
        df_base_characteristics=get_base_characteristics(df_temp, base_characteristics)
        
        cluster_characteristics.append(cluster)
        cluster_characteristics.append(len(df_temp))
        cluster_characteristics.append(df_base_characteristics)
        baseline_characteristics.append(cluster_characteristics)
    return baseline_characteristics
def get_cluster_statistics(df,dim_red_method):
    #load inpatient and EF dataframe 
    hospitalization = pq.read_table('Cohort/Feature_Extraction/days_in_hospital.parquet').to_pandas()
    ef=pq.read_table('Cohort/Feature_Extraction/avg_EF.parquet').to_pandas()
    #merge both to the df: 
    df_cohort=pd.merge(df, hospitalization, how='left', left_index=True, right_on='medical_record_number')
    df_cohort=pd.merge(df_cohort, ef, how='left',left_index=True, right_on='medical_record_number')
    #get average days in hospital per patient per cluster
    base_characteristics=[
        [ "avg_ef","mean"],
        ["days_in_hospital","mean"],
        [ "HF_Onset_age_in_days","mean"],
        ["gender","count"]
        ]
    baseline_characteristics=get_cluster_information(df_cohort,dim_red_method,base_characteristics)
    print (baseline_characteristics)
    #calculate t-statistics: 
    cl_0=df_cohort.loc[df_cohort[dim_red_method]=='Cluster_0']
    cl_1=df_cohort.loc[df_cohort[dim_red_method]=='Cluster_1']
    cl_2=df_cohort.loc[df_cohort[dim_red_method]=='Cluster_2']
    t_test_avg_ef=ttest_ind(cl_0['avg_ef'], cl_1['avg_ef'],cl_2['avg_ef'])
    print(t_test_avg_ef)
    return baseline_characteristics
        #print(str(cluster))
        #print(len(df_temp))
        
        #print(df_temp_baseline)
#function for statistics: 
def get_base_characteristic_value(df , characteristic , kind):    
    if kind=="mean": 
        df_mean=df[characteristic].mean()
        df_std= df[characteristic].std()
        df_max= df[characteristic].max()
        df_min= df[characteristic].min()
        if characteristic == "HF_Onset_age_in_days":
            base_characteristics_cohort=pd.DataFrame({'Variable': [characteristic+"_mean", characteristic+"_std", characteristic+"_max", characteristic+"_min"],
                                                  'Value': [(df_mean/365), (df_std/365), (df_max/365), (df_min/365)],})
        else:
            base_characteristics_cohort=pd.DataFrame({'Variable': [characteristic+"_mean", characteristic+"_std", characteristic+"_max", characteristic+"_min"],
                                                  'Value': [(df_mean), (df_std), (df_max), (df_min)],})
        
    if kind=="count":
        base_characteristics_cohort=pd.DataFrame(columns=["Variable","Value"])
        feature_value=df[characteristic].unique()
        #print(feature_value)
        for value in feature_value: 
            df_condition=df.loc[df[characteristic]==value]
            df_percent= df_condition.shape[0]/df.shape[0]
            #print(df_percent)
            new_row1 = {'Variable': value+"_total",'Value': df_condition.shape[0]}
            new_row2 = {'Variable': value+"_relation",'Value': df_percent}
            base_characteristics_cohort=base_characteristics_cohort.append(new_row1, ignore_index=True)
            base_characteristics_cohort=base_characteristics_cohort.append(new_row2, ignore_index=True)
       # print(df_condition.shape[0], df_percent)
    #print (base_characteristics_cohort)
    return base_characteristics_cohort

def get_base_characteristics(df, characteristics): 
    base_characteristics_cohort=pd.DataFrame(columns=["Variable","Value"])
    for characteristic in characteristics:
        intermediate_base_characteristics_cohort=get_base_characteristic_value(df,characteristic[0],characteristic[1])
        base_characteristics_cohort=pd.concat([base_characteristics_cohort,intermediate_base_characteristics_cohort])
    #print(base_characteristics_cohort)
    return base_characteristics_cohort

def get_cluster_information(df,dim_red_method,base_characteristics):
    
    baseline_characteristics=[]
    for cluster in df[dim_red_method].unique(): 
        cluster_characteristics=[]
        df_temp=df.loc[df[dim_red_method] == cluster]
        df_base_characteristics=get_base_characteristics(df_temp, base_characteristics)
        
        cluster_characteristics.append(cluster)
        cluster_characteristics.append(len(df_temp))
        cluster_characteristics.append(df_base_characteristics)
        baseline_characteristics.append(cluster_characteristics)
    return baseline_characteristics
def get_cluster_statistics(df,dim_red_method,boxplot):
    #load inpatient and EF dataframe 
    hospitalization = pq.read_table('Cohort/Feature_Extraction/days_in_hospital.parquet').to_pandas()
    ef=pq.read_table('Cohort/Feature_Extraction/avg_EF.parquet').to_pandas()
    #merge both to the df: 
    df_cohort=pd.merge(df, hospitalization, how='left', left_index=True, right_on='medical_record_number')
    df_cohort=pd.merge(df_cohort, ef, how='left',left_index=True, right_on='medical_record_number')
    #get average days in hospital per patient per cluster
    base_characteristics=[
        [ "avg_ef","mean"],
        ["days_in_hospital","mean"],
        [ "HF_Onset_age_in_days","mean"],
        ["gender","count"]
        ]
    baseline_characteristics=get_cluster_information(df_cohort,dim_red_method,base_characteristics)
    print (baseline_characteristics)
    if boxplot == True: 
        df_boxplt=df_cohort[["avg_ef",dim_red_method]]
        df_boxplt.boxplot(by=dim_red_method)
        df_boxplt=df_cohort[[ "days_in_hospital",dim_red_method]]
        df_boxplt.boxplot(by=dim_red_method)                   


    return baseline_characteristics
        #print(str(cluster))
        #print(len(df_temp))
        
        #print(df_temp_baseline)<jupyter_output><empty_output><jupyter_text>Feature Evaluation   CHI test<jupyter_code>#https://www.pythonfordatascience.org/chi-square-test-of-independence-python/
def cat_feature_importance(df,ctransformer,sup_colums,dim_red_method,n_cluster):
    
   
    
    #replace cluster names 
    #establish two categories in all Categories 
    
   
    for cluster in (range(n_cluster)):
        cluster_name='cluster_'+str(cluster)
        df[dim_red_method].replace({cluster: cluster_name},inplace=True)
    df=df.replace(True, 'Yes')
    df=df.replace(False,'No')
    df=df.fillna('No')
    df=df.replace(1, 'Yes')
    df=df.replace(0,'No')
    df=df.fillna('No')
    col=['column name','Pearson Chi-square','Cramers V','p-value','absolute_p','compared to other']
    result_all=pd.DataFrame(columns=col)
    result_chi=[]
    for cluster in df[dim_red_method].unique():
        result_all=pd.DataFrame(columns=col)
        df['temp_cluster']=df[dim_red_method]
        df.loc[df[dim_red_method] != cluster, "temp_cluster"] = "other_cluster"
        #print(df[[dim_red_method,'temp_cluster']])     
        cat_columns=ctransformer.transformers[1][2]
        #iterate over cat columns and calculate the p-Value: 
        for cat_col in cat_columns: 
            feature=cat_col
            crosstab, test_results, expected = rp.crosstab(df[feature], df['temp_cluster'],
                                                   test= "chi-square",
                                                   expected_freqs= True,
                                                   prop= "cell")
            p=format(test_results["results"][1], '.300000000g')
            #print(p)
           # if test_results["results"][1]!=0:
            p=float(p)
            if p!=0:

                importance=abs(np.log(p))
            else: 
                importance=0
            compared=''
            if feature !='gender':
                feature_count_1=len(df.loc[df['temp_cluster']==cluster])
                feature_cluster=df.loc[df['temp_cluster']==cluster]
                feature_percentage_1=(len(feature_cluster.loc[feature_cluster[feature]=='Yes'])/feature_count_1)
                #print(feature_percentage_1)
    
                feature_count_2=len(df.loc[df['temp_cluster']=='other_cluster'])
                feature_cluster_2=df.loc[df['temp_cluster']=='other_cluster']
                feature_percentage_2=(len(feature_cluster_2.loc[feature_cluster_2[feature]=='Yes'])/feature_count_2)
                #print(feature_percentage_2)
                if feature_percentage_1 > feature_percentage_2: 
                    compared='higher'
                else:
                    compared='lower'
            row={'column name':(feature+'_'+cluster),'Pearson Chi-square':test_results["results"][0],'Cramers V':test_results["results"][2],'p-value':p,'absolute_p':importance,'compared to other':compared}
            #row={'column name':feature,'Pearson Chi-square':test_results["results"][0],'Cramers V':test_results["results"][2],'p-value':p,'absolute_p':importance}
            result_all=result_all.append(row, ignore_index=True)
        for cat_col in sup_colums: 
            #print('Calculaint Supervised features')
            feature=cat_col
            crosstab, test_results, expected = rp.crosstab(df[feature], df['temp_cluster'],
                                                   test= "chi-square",
                                                   expected_freqs= True,
                                                   prop= "cell")
            #print(crosstab)
            p=format(test_results["results"][1], '.300000000g')
            #print(p)
           # if test_results["results"][1]!=0:
            p=float(p)
            if p!=0:

                importance=abs(np.log(p))
            else: 
                importance=0
            compare=''
            if feature !='gender':
                feature_count_1=len(df.loc[df['temp_cluster']==cluster])
                feature_cluster=df.loc[df['temp_cluster']==cluster]
                feature_percentage_1=(len(feature_cluster.loc[feature_cluster[feature]=='Yes'])/feature_count_1)
               # print(feature_percentage_1)
    
                feature_count_2=len(df.loc[df['temp_cluster']=='other_cluster'])
                feature_cluster_2=df.loc[df['temp_cluster']=='other_cluster']
                feature_percentage_2=(len(feature_cluster_2.loc[feature_cluster_2[feature]=='Yes'])/feature_count_2)
               # print(feature_percentage_2)
                if feature_percentage_1 > feature_percentage_2: 
                    compared='higher'
                else:
                    compared='lower'
            row={'column name':(feature+'_'+cluster),'Pearson Chi-square':test_results["results"][0],'Cramers V':test_results["results"][2],'p-value':p,'absolute_p':importance,'compared to other':compared}
            #row={'column name':feature,'Pearson Chi-square':test_results["results"][0],'Cramers V':test_results["results"][2],'p-value':p,'absolute_p':importance}
            result_all=result_all.append(row, ignore_index=True)
        result_all=result_all.sort_values(by=['absolute_p'],ascending=False)
        result_chi.append(result_all)
    #result_all=result_all.drop_duplicates(subset='column name',keep='first', inplace=False)
    #return result_all.head(top_features)
    return result_chi<jupyter_output><empty_output><jupyter_text>T-Test<jupyter_code>def num_feature_importance_t_test(df,ctransformer,dim_red_method,n_cluster):
    df_temp=df
    #replace cluster names 
    for cluster in (range(n_cluster)):
        cluster_name='cluster_'+str(cluster)
        df[dim_red_method].replace({cluster: cluster_name},inplace=True)
    #normalize num columns
    min_max_scaler = preprocessing.MinMaxScaler()
    num_columns=ctransformer.transformers[0][2]
    #print(num_columns)
    df_temp[list(num_columns)] = min_max_scaler.fit_transform(df_temp[list(num_columns)])
    #iterate over num columns and calculate the p-Value: 
    col=['column name','T-Statistics','p-value','absolute_p','compared to other']
    result_all=pd.DataFrame(columns=col) 
    result_t_test=[]
    for cluster in df_temp[dim_red_method].unique():
        result_all=pd.DataFrame(columns=col)
        df_temp['temp_cluster']=df_temp[dim_red_method]
        df_temp.loc[df[dim_red_method] != cluster, "temp_cluster"] = "other_cluster"
        for num_col in num_columns: 
            feature=num_col
            feature_value_1=df_temp.loc[df_temp['temp_cluster']==cluster][feature].values
            feature_value_2=df_temp.loc[df_temp['temp_cluster']=="other_cluster"][feature].values
            statistics,p=stats.ttest_ind(feature_value_1, feature_value_2, equal_var = False)
            mean_1=feature_value_1.mean()
            mean_2=feature_value_2.mean()
            if mean_1 > mean_2: 
                compared='higher'
            else:
                compared='lower' 
           # print(feature_value_1)
           # print(feature_value_2)
           # print(p)
            p=format(p, '.300000000g')
            p=float(p)
            if p!=0:
                importance=abs(np.log(p))
            else: 
                importance=0
            row={'column name':(feature+'_'+cluster),'T-Statistics':statistics,'p-value':p,'absolute_p':importance,'compared to other':compared}
            result_all=result_all.append(row, ignore_index=True)
        result_all=result_all.sort_values(by=['absolute_p'],ascending=False)
        result_t_test.append(result_all)        
    return result_t_test<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>def subStringCluster(string):
    a_string=string
    split_string=a_string.split('_cluster_',1)
    substring = split_string[0]
    return substring
def getTopCluster(evaluation_pandas, n_topFeature, n_cluster ): 
    topFeatures=[]
    for n in range(n_cluster):
        #print(n)
        features=[]
         #categorical features
        features=evaluation_pandas[1][n]['column name'].values
        all_features = evaluation_pandas[1][n]
        x=0
        for i in range(n_topFeature):
            feature=subStringCluster(features[x])
            
            if 'Procedure' in feature: 
               # print (feature)
                #x=x+1
                #print(subStringCluster(features[x]))
                #topFeatures.append(subStringCluster(features[x]))
                i=i-1
            elif feature != 'gender' :
                f=all_features.loc[all_features['column name']==features[x]]
                p_value=f['p-value'].values
                if p_value < 0.05 and p_value!=0.0 :
                    topFeatures.append([subStringCluster(features[x]),'categorical'])
                    #print(feature)
                
            else: 
                i=i-1
            x=x+1
        
        #numeric
        features=evaluation_pandas[2][n]['column name'].values
        all_features = evaluation_pandas[2][n]
        for i in range(n_topFeature):
            f=all_features.loc[all_features['column name']==features[i]]
            p_value=f['p-value'].values
            if p_value < 0.05 and p_value!=0.0 :
                topFeatures.append([subStringCluster(features[i]),'numeric'])
    topFeatures_tuple=set(tuple(t)for t in topFeatures)
    #print(topFeatures_tuple)
    topFeatures=[t[0] for t in topFeatures_tuple]
    #print(topFeatures)
    #topFeatures=set(topFeatures)
    #topFeatures=list(topFeatures)
    #print(topFeatures)
    return topFeatures_tuple, topFeatures
        
#https://github.com/hpi-dhc/robotehr/blob/e3673aef701aa817c74d04170986f01fa191212a/robotehr/evaluation/risk_groups.py#L70-L100
def plot_risk_groups(df, features,dim_red_method, friendly_names_converter=None, filename='', nrows=2, figsize=[12,3]):
    #features=features[:2]
    #ncols = int(len(features) / nrows)
    ncols=1
    #fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=figsize)
    fig, ax = plt.subplots(len(features),figsize=figsize)
    fig.tight_layout(pad=3.0)
    #print(len(features))
    for i in range(len(features)):
        #row_index = int(i / ncols)
        row_index=i
        #col_index = i % int(len(features) / nrows)
        col_index=0

        #current_axis = ax[row_index][col_index] if nrows > 1 else ax[col_index]
        current_axis = ax[row_index]
        if df[features[i]].min() == 0 and df[features[i]].max() == 1:
            current_axis.set_ylim(bottom=-0.5, top=1.5)
        sns.violinplot(
            x=dim_red_method,
            y=features[i],
            data=df,
            palette="muted",
            ax=current_axis,
            hue='gender'
        )
        if friendly_names_converter:
            title = friendly_names_converter.get(features[i])
        else:
            title = features[i]
        if len(title) > 50:
            title = f'{title[:50]} ...'
        current_axis.set_title(f'{title}', fontsize=20)
        current_axis.set_xlabel('')
        current_axis.set_ylabel('')
    if filename:
        fig.savefig(filename, dpi=300, bbox_inches="tight")
    return fig
def map_feature_names(feature_evaluation):
    feature_look_up=pq.read_table('Cohort/feature_look_up.parquet').to_pandas()
    feature_evaluation['human_readable']=''
    for index,r in feature_evaluation.iterrows():
        lookuplist=feature_look_up['original_feature_name'].to_list()
        if r['features'] in lookuplist:
            human_readable_row=feature_look_up.loc[feature_look_up['original_feature_name']==r['features']]
            human_readable=human_readable_row['human_readable'].values
            #print(human_readable)
            feature_evaluation.loc[feature_evaluation['features']==r['features'],'human_readable']=human_readable[0]
        else :
            feature_evaluation.loc[feature_evaluation['features']==r['features'],'human_readable']=r['features']
    return feature_evaluation
def create_overview_table(conv_df,features_tuples,features,dim_red_method): 
    feature_evaluation_df = pd.DataFrame()
    feature_evaluation_df['features']=features
    #print (feature_evaluation_df)
    for cluster in conv_df[dim_red_method].unique(): 
        feature_evaluation_df[cluster]=0
        cluster_df= conv_df.loc[conv_df[dim_red_method]==cluster]        
        for features_tuple in features_tuples:
            if features_tuple[1]=='categorical':
                sum_feature=cluster_df[features_tuple[0]].sum()
                percentage=sum_feature/len(cluster_df)
                feature_evaluation_df.loc[feature_evaluation_df['features']==features_tuple[0],cluster]=percentage
                
                #print('categorical')
            if features_tuple[1]=='numeric':
                mean_feature=cluster_df[features_tuple[0]].mean()
                median_feature=cluster_df[features_tuple[0]].median()
                feature_evaluation_df.loc[feature_evaluation_df['features']==features_tuple[0],cluster]=(str(mean_feature)+'/'+str(median_feature))
                #print('numeric') '''
   # print(feature_evaluation_df)
    return feature_evaluation_df
def plotTopFeatures(df_path,df,merge_w_supervised,dim_red_method, evaluation_results,  n_cluster, n_topFeatures): 
    '''#convert the dataframe
    df_origin=pq.read_table(df_path).to_pandas()
    #print(df_origin['gender'])
    df_origin[dim_red_method]=df[dim_red_method]
    conv_df=df_origin
    if merge_w_supervised==True:
        df_supervised_merge= pq.read_table('Cohort/Feature_Extraction/ALL_HF_cohort_supervised_only_ever_diag_drugFORMerge.parquet').to_pandas()
        conv_df.index = conv_df.index.map(str)
        df_supervised_merge.index = df_supervised_merge.index.map(str)
        conv_df=pd.merge(conv_df, df_supervised_merge, left_on='medical_record_number', right_on='medical_record_number')
     '''
    conv_df=pq.read_table(df_path).to_pandas()
    if merge_w_supervised==True:
            df_supervised_merge= pq.read_table('Cohort/Feature_Extraction/ALL_HF_cohort_supervised_only_ever_diag_drugFORMerge_wLab.parquet').to_pandas()
            conv_df.index = conv_df.index.map(str)
            df_supervised_merge.index = df_supervised_merge.index.map(str)
            sup_colums=df_supervised_merge.columns
            conv_df=pd.merge(conv_df, df_supervised_merge, left_on='medical_record_number', right_on='medical_record_number')
    
    conv_df=conv_df.replace(True, 1)
    conv_df=conv_df.replace(False,0)
    conv_df=conv_df.replace('yes', 1)
    conv_df=conv_df.replace('no',0)
    conv_df=conv_df.fillna(0)
    conv_df[dim_red_method]=df[dim_red_method]
    conv_df=conv_df.sort_values(by=[dim_red_method],ascending=True)
    #get top featrues: 
    evaluation_pandas=evaluation_results
    features_tuples,features=getTopCluster(evaluation_pandas, n_topFeatures, n_cluster)
    #plot features 
    #print (cluster_name)
   # fig_x=12*len(features)
    fig_y=8*len(features)
    plot_risk_groups(conv_df, features, dim_red_method,friendly_names_converter=None, filename='', nrows=10, figsize=[12,fig_y])
    feature_evaluation_df=create_overview_table(conv_df,features_tuples,features,dim_red_method)
    feature_evaluation_df=map_feature_names(feature_evaluation_df)
    return feature_evaluation_df<jupyter_output><empty_output><jupyter_text>Cluster Pipeline<jupyter_code>def cluster_LSTM_Pipeline(per_day,embedding_method,embedding_size,cluster_methods,n_cluster,ellbow_method,plot,boxplot,feature_evaluation,merge_w_supervised): 
    #load Data
    data,data_sample, df_name,data_structure=load_dataset(per_day,embedding_method,embedding_size)
    timesteps=len(data[0])
    n_features=len(data[0][0]) 
    #load model and compromise data dimension
    dim_red_data=load_lstm_representation(data_structure,timesteps,n_features,layer_size_1,layer_size_2,activation_func,optimizer_func,loss_func,n_epochs,n_batch_size,df_name,data)
      

    #apply cluster method: 
    if cluster_methods=='kmeans':
        labels=apply_kmeans(dim_red_data,n_cluster,ellbow_method)   
    if cluster_methods=='hierachical':
        labels=apply_hierachical(dim_red_data,n_cluster)
    #plott cluster results
    if plot==True: 
        plot_data=apply_umap(dim_red_data,2)
        scatter_plot(plot_data,None)
        scatter_plot(plot_data,labels)
    #apply metrics 
    silhouette_Coefficient= get_silhouette_Coefficient(labels,dim_red_data)
    calinski_harabasz=get_calinski_harabasz(labels,dim_red_data)
    davies_bouldin=get_davies_bouldin(labels,dim_red_data)
    if per_day== True: 
        data_structure='per_day'
    result=pq.read_table('Cohort/LSTM_Models/Evaluation_LSTM.parquet').to_pandas()
    result=result.append({'df_name':df_name,'data_structure':data_structure,'timesteps':timesteps,'n_features':n_features,'layer_size_1':layer_size_1,'layer_size_2':layer_size_2,'activation_func':activation_func,'optimizer_func':optimizer_func,'loss_func':loss_func,'n_epochs':n_epochs,'n_batch_size':n_batch_size,'cluster_method':cluster_methods,'n_cluster':n_cluster,'silhouette_Coefficient':silhouette_Coefficient,'calinski_harabasz':calinski_harabasz,'davies_bouldin':davies_bouldin}, ignore_index=True)
    result.to_parquet('Cohort/LSTM_Models/Evaluation_LSTM.parquet')
    print(result)
    if per_day== True: 
        data_structure='per_patient_per_day'
    #load static dataset for calculating features importaces
    if data_structure=='per_patient':
        time_series_static=pq.read_table('Cohort/Time_Series/time_series_per_patient_static_features.parquet').to_pandas()
    if data_structure=='per_patient_per_day':
        time_series_static=pq.read_table('Cohort/Time_Series/time_series_per_patient_per_day_static_features.parquet').to_pandas()
        print(len(time_series_static))
        print(len(labels))
    #sample
    #time_series_static=time_series_static.head(300)
    if merge_w_supervised==True:
            df_supervised_merge= pq.read_table('Cohort/Feature_Extraction/ALL_HF_cohort_supervised_only_ever_diag_drugFORMerge_wLab.parquet').to_pandas()
            time_series_static.index = time_series_static.index.map(str)
            df_supervised_merge.index = df_supervised_merge.index.map(str)
            sup_colums=df_supervised_merge.columns
            time_series_static=pd.merge(time_series_static, df_supervised_merge, left_on='medical_record_number', right_on='medical_record_number')
    else:
        sup_colums=[]  
    #attach labels to static feature set:
    time_series_static['label']=labels
    #apply age filter: 
    time_series_static.loc[(time_series_static['HF_Onset_age_in_days'] > 32850),'HF_Onset_age_in_days']=32850
    evaluation_results=[]
    cluster_information=get_cluster_statistics(time_series_static,'label',boxplot)
    evaluation_results.append(cluster_information) 
    if feature_evaluation == True:
         #load ctansformer: 
        with open('Cohort/Models/ColumnTransformer/ALL_HF_cohort_unsupervised_only_after_onset_HF_ALL_all_any_all_mean_medium_cleaned_wLab_MinMaxScaler_BinaryEncoder_woGender_woAge.pkl', 'rb') as f:
            ctransformer = pickle.load(f)
        print('ctransformer loaded')
        top_catigorical_features=cat_feature_importance(time_series_static,ctransformer,sup_colums,'label',n_cluster)
        print('Top Categorical features: \n',top_catigorical_features)
        evaluation_results.append(top_catigorical_features)
        
        
        top_numerical_features_t_test=num_feature_importance_t_test(time_series_static,ctransformer,'label',n_cluster)
        print('Top Numerical features: \n',top_numerical_features_t_test)
        evaluation_results.append(top_numerical_features_t_test) 
    return evaluation_results,time_series_static
    
per_day=False
embedding_method='cbow'
embedding_size='20'
boxplot=True
#Model
#data_structure='per_patient'#'per_patient_per_day'

layer_size_1=32
layer_size_2=16
activation_func='sigmoid'
optimizer_func='adam'
loss_func='mse'
n_epochs=300
n_batch_size=1024
#CLuster
n_cluster=3
ellbow_method=False
cluster_methods='kmeans'
plot=True
merge_w_supervised=True
feature_evaluation=True
evaluation_results,time_series_static=cluster_LSTM_Pipeline(per_day,embedding_method,embedding_size,cluster_methods,n_cluster,ellbow_method,plot,boxplot,feature_evaluation,merge_w_supervised)
if per_day==False:
        df_path='Cohort/Time_Series/time_series_per_patient_static_features.parquet'
if per_day==True:
        df_path='Cohort/Time_Series/time_series_per_patient_per_day_static_features.parquet'
        
feature_evaluation_df=plotTopFeatures(df_path,time_series_static,merge_w_supervised,'label', evaluation_results,n_cluster ,5)
feature_evaluation_df
feature_evaluation_df.to_excel("Cohort/LSTM_Models/Feature_evaluation_"+activation_func+optimizer_func+loss_func+str(layer_size_1)+str(layer_size_2)+cluster_methods+".xlsx")<jupyter_output><empty_output><jupyter_text>Calculate Inpatient p-value<jupyter_code>#load inpatient and EF dataframe 
hospitalization = pq.read_table('Cohort/Feature_Extraction/days_in_hospital.parquet').to_pandas()
ef=pq.read_table('Cohort/Feature_Extraction/avg_EF.parquet').to_pandas()
#merge both to the df: 
df_cohort=pd.merge(time_series_static, hospitalization, how='left', left_index=True, right_on='medical_record_number')
df_cohort=pd.merge(df_cohort, ef, how='left',left_index=True, right_on='medical_record_number')
df_cohort['days_in_hospital']=df_cohort['days_in_hospital'].fillna(0)
#get average days in hospital per patient per cluster
cl_0=df_cohort.loc[df_cohort['temp_cluster']=='cluster_1']
cl_1=df_cohort.loc[df_cohort['temp_cluster']=='other_cluster']
#cl_2=df_cohort.loc[df_cohort['label']=='cluster_2']
t=t_test_avg_ef=stats.ttest_ind(cl_0['days_in_hospital'], cl_1['days_in_hospital'])
t
#cl_0['days_in_hospital']
evaluation_results[2][1]
time_series_static['min__LabValue__QT']
plotTopFeatures(time_series_static,'label', evaluation_results,n_cluster ,10)
#Loop 
embedding_method='cbow'
embedding_size='20'
boxplot=False
#size= [(32,16),(128,64)]
size= [(100,50)]
arr_per_day=[ False,True]#[False,True]
#arr_activation_function=['tanh','sigmoid','relu']
arr_activation_function=['tanh','sigmoid','relu']
optimizer_func='adam'
loss_func='mse'
n_epochs=300
n_batch_size=1024
arr_n_cluster=[3,4,5]
ellbow_method=False
plot=False
merge_w_supervised=True
t=0
arr_cluster_methods=['kmeans','hierachical']
for q in arr_per_day: 
    per_day=q
    for s in size: 
        layer_size_1=s[0]
        layer_size_2=s[1]
        for activation in arr_activation_function: 
            activation_func=activation
            for c in arr_cluster_methods: 
                    cluster_methods=c
                    t=t+1
                    print(t)
                    for n in arr_n_cluster: 
                        n_cluster=n 
                        evaluation_results=cluster_LSTM_Pipeline(per_day,embedding_method,embedding_size,cluster_methods,n_cluster,ellbow_method,plot,boxplot,merge_w_supervised)
per_day=True
embedding_method='cbow'
embedding_size='20'
boxplot=True
#Model
#data_structure='per_patient'#'per_patient_per_day'

layer_size_1=32
layer_size_2=16
activation_func='relu'
optimizer_func='adam'
loss_func='mse'
n_epochs=300
n_batch_size=1024
#CLuster
n_cluster=3
ellbow_method=True
cluster_methods='kmeans'
plot=True
merge_w_supervised=True
evaluation_results=cluster_LSTM_Pipeline(per_day,embedding_method,embedding_size,cluster_methods,n_cluster,ellbow_method,plot,boxplot,feature_evaluation,merge_w_supervised)
evaluation_results<jupyter_output><empty_output><jupyter_text>Create blank evaluation excel<jupyter_code>col=['df_name','data_structure','timesteps','n_features','layer_size_1','layer_size_2','activation_func','optimizer_func','loss_func','n_epochs','n_batch_size','cluster_method','n_cluster','silhouette_Coefficient','calinski_harabasz','davies_bouldin','silhouette_Coefficient_static','calinski_harabasz_static','davies_bouldin_static']
result=pd.DataFrame(columns=col)
#result.to_parquet('Cohort/LSTM_Models/Evaluation_LSTM.parquet')
result=pq.read_table('Cohort/LSTM_Models/Evaluation_LSTM.parquet').to_pandas()
result.sort_values(by=['silhouette_Coefficient','data_structure','activation_func','layer_size_2'], ascending=False)
cm = sns.light_palette("green", as_cmap=True)
result=result.sort_values(['silhouette_Coefficient','data_structure','activation_func','layer_size_2'], ascending = False )
s = result.style.background_gradient(cmap=cm)
s
#s.to_excel("Cohort/LSTM_Models/11_04_Evaluation_cluster.xlsx")<jupyter_output><empty_output><jupyter_text>Table with LSTM Model Loss<jupyter_code>a=pq.read_table('Cohort/Metrics_LSTM.parquet').to_pandas()
a=a.sort_values(by=['data_structure','loss'])
#a=
a.loc[a['timesteps']==7,'data_structure']='per_day'
a=a.sort_values(by=['data_structure','loss'])
a = a.style.background_gradient(cmap=cm)
a
a.to_excel("Cohort/LSTM_Models/11_04_Evaluation_loss.xlsx")
s= [(32,16),(128,64)]
for e in s: 
    print(e[1])
  time_series_static=pq.read_table('Cohort/Time_Series/time_series_per_patient_static_features.parquet').to_pandas()
time_series_static<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11691490469742735, 'fraction_numerical': 0.011118633475679617, 'mean_word_length': 4.204634774359669, 'pattern_counts': {'":': 6, '<': 67, '<?xml version=': 0, '>': 71, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '530705', 'license': 'MIT', 'n_tokens_mistral': 13161, 'n_tokens_neox': 12582, 'n_words': 1380, 'path': 'Evaluation_of_LSTMs.ipynb', 'repo_name': 'Melhac/Subtyping_of_HF_Master_Thesis', 'size': '38404'}