<jupyter_start><jupyter_text>Check max and minimum for each feature  1 Based on WSS  1.1 WSS for downsampled files<jupyter_code>def scale_data(col: pd.Series,scaler) -> pd.Series:
    """
    Scale a column
    """
    X = col.values.reshape(-1, 1).copy()
    scaled_array = scaler.fit_transform(X)

    scaled_column = pd.Series(scaled_array.tolist()).explode()

    return scaled_column
def my_global_scaler(X, r_min,r_max,t_min,t_max):
    return ((X - r_min)/(r_max - r_min))*(t_max-t_min) + t_min
#https://stats.stackexchange.com/questions/281162/scale-a-number-between-a-range


r_min = -100
r_max = 100
t_min = 0
t_max = 1

# X = scaler_df.fpfh_1.apply(lambda x: my_global_scaler(x, r_min,r_max,t_min,t_max))
# np.min(X), np.max(X)
wss_columns = ['original_points','sampled_points','down_percent','wss_mean','wss_median','wss_std','wss_min','wss_max',]

wss_eda = pd.DataFrame(index=patient_id_list, columns = wss_columns )
wss_eda_scaled = pd.DataFrame(index=patient_id_list, columns = wss_columns )

if not wss_eda.empty and wss_eda_scaled.empty :
    del(wss_eda)
    del(wss_eda_scaled)
    

wss_eda.head(2)

for PATIENT_ID in patient_id_list:

    df_wss_org= pd.read_csv(os.path.join(INPUT_PATH,f"{PATIENT_ID}{WSS_FILENAME}"))
    df_wss= pd.read_csv(os.path.join(INPUT_TEMP_PATH,f"{PATIENT_ID}{WSS_DOWN_FILENME}"))
    
    #Scale the data
    min_max_scaler = MinMaxScaler() 
    df_wss["WSS_scaled"] = scale_data(df_wss["WSS"],min_max_scaler)
    
    
    wss_eda.loc[PATIENT_ID] = [df_wss_org.shape[0], df_wss.shape[0] , 
                               np.round((df_wss.shape[0]/df_wss_org.shape[0])*100,4),
                               np.mean(df_wss.WSS), np.median(df_wss.WSS), np.std(df_wss.WSS), 
                               np.min(df_wss.WSS), np.max(df_wss.WSS)
                         ]
    wss_eda_scaled.loc[PATIENT_ID] = [df_wss_org.shape[0], df_wss.shape[0],
                                    np.round((df_wss.shape[0]/df_wss_org.shape[0])*100,4),
                                      np.mean(df_wss.WSS_scaled), np.median(df_wss.WSS_scaled), np.std(df_wss.WSS_scaled), 
                                      np.min(df_wss.WSS_scaled), np.max(df_wss.WSS_scaled)
                         ]
wss_eda = wss_eda.infer_objects()
wss_eda_scaled = wss_eda_scaled.infer_objects()

wss_eda.head(2)
wss_eda
wss_eda.describe(percentiles = np.linspace(0.5,1.0,10))
wss_eda.iloc[:,3:].describe(percentiles = [0,0.05,0.25,0.5,0.75,0.85,0.9,0.95,1])
sns.distplot(wss_eda[["wss_std"]], hist= False, kde=True )
sns.distplot(wss_eda[["wss_min"]], hist= False, kde=True )
sns.boxplot(x="variable", y="value", data=pd.melt(wss_eda.iloc[:,3:-1]))

plt.show()
sns.boxplot(x="variable", y="value", data=pd.melt(wss_eda.iloc[:,7:]))

plt.show()
wss_eda.query("wss_max >= 183").index.values
wss_eda.query("wss_std >= 28").index.values
OUTLIER_WSS_PIDs = ['25_HM', '49_LE', '53_LS', '56_WA', '66_GI', '72_HRA', '82_KBB',
       '112_SM']
r_min = 1.2
r_max = 30
t_min = 0
t_max = 1

X = wss_eda.query("wss_std<@r_max").wss_std.apply(lambda x: my_global_scaler(x, r_min,r_max,t_min,t_max))
np.min(X), np.max(X)<jupyter_output><empty_output><jupyter_text>Removing Outliers<jupyter_code>patient_id_list_new = [x for x in patient_id_list if x not in OUTLIER_WSS_PIDs]
len(patient_id_list_new)
wss_columns = ['original_points','sampled_points','down_percent','wss_mean','wss_median','wss_std','wss_min','wss_max',]

if not wss_eda.empty and wss_eda_scaled.empty :
    del(wss_eda_new)
    del(wss_eda_scaled)
    

wss_eda_new = pd.DataFrame(index=patient_id_list, columns = wss_columns )
# wss_eda_scaled = pd.DataFrame(index=patient_id_list, columns = wss_columns )

wss_eda_new.head(2)

for PATIENT_ID in patient_id_list_new:

    df_wss_org= pd.read_csv(os.path.join(INPUT_PATH,f"{PATIENT_ID}{WSS_FILENAME}"))
    df_wss= pd.read_csv(os.path.join(INPUT_TEMP_PATH,f"{PATIENT_ID}{WSS_DOWN_FILENME}"))
    
    #Scale the data
    min_max_scaler = MinMaxScaler() 
    df_wss["WSS_scaled"] = scale_data(df_wss["WSS"],min_max_scaler)
    
    
    wss_eda_new.loc[PATIENT_ID] = [df_wss_org.shape[0], df_wss.shape[0] , 
                               np.round((df_wss.shape[0]/df_wss_org.shape[0])*100,4),
                               np.mean(df_wss.WSS), np.median(df_wss.WSS), np.std(df_wss.WSS), 
                               np.min(df_wss.WSS), np.max(df_wss.WSS)
                         ]
#     wss_eda_scaled.loc[PATIENT_ID] = [df_wss_org.shape[0], df_wss.shape[0],
#                                     np.round((df_wss.shape[0]/df_wss_org.shape[0])*100,4),
#                                       np.mean(df_wss.WSS_scaled), np.median(df_wss.WSS_scaled), np.std(df_wss.WSS_scaled), 
#                                       np.min(df_wss.WSS_scaled), np.max(df_wss.WSS_scaled)
#                          ]
wss_eda_new = wss_eda_new.infer_objects()
# wss_eda_scaled = wss_eda_scaled.infer_objects()

wss_eda_new.head(2)
wss_eda_new.describe(percentiles = np.linspace(0.5,1.0,10))
r_min = 1.2
r_max = 33
t_min = 0
t_max = 1

X = wss_eda_new.query("wss_std<@r_max").wss_std.apply(lambda x: my_global_scaler(x, r_min,r_max,t_min,t_max))
np.min(X), np.max(X)
sns.boxplot(x="variable", y="value", data=pd.melt(wss_eda_new.iloc[:,2:]))

plt.show()<jupyter_output><empty_output><jupyter_text>1.2 WSS percentage distribution<jupyter_code>wss_columns = ['original_points','sampled_points','min','20p','50p','70p','80p','90p','max']



wss_distribution = pd.DataFrame(index=patient_id_list, columns = wss_columns )
wss_distribution_scaled = pd.DataFrame(index=patient_id_list, columns = wss_columns )

if not wss_distribution.empty and wss_eda_scaled.empty :
    del(wss_distribution)
    del(wss_distribution_scaled)
    

wss_distribution.head(2)
def my_global_scaler(X, r_min,r_max):
    t_min = 0
    t_max = 1
    return ((X - r_min)/(r_max - r_min))*(t_max-t_min) + t_min
#https://stats.stackexchange.com/questions/281162/scale-a-number-between-a-range

for PATIENT_ID in patient_id_list_new:

    df_wss_org= pd.read_csv(os.path.join(INPUT_PATH,f"{PATIENT_ID}{WSS_FILENAME}"))
    df_wss= pd.read_csv(os.path.join(INPUT_TEMP_PATH,f"{PATIENT_ID}{WSS_DOWN_FILENME}"))
    
#     #Scale the data
#     min_max_scaler = MinMaxScaler() 
#     df_wss["WSS_scaled"] = scale_data(df_wss["WSS"],min_max_scaler)
    
    col = "WSS" 
    
    r_min = 0
    r_max = 183
    
    df = df_wss.copy()
    df.loc[df[col] < r_min] = r_min
    df.loc[df[col] > r_max] = r_max
    df[col] = df[col].apply(
        lambda x: my_global_scaler(x, r_min, r_max)
    )
    
    desc = df_wss.describe(percentiles = [0.2, 0.5, 0.7, 0.8, 0.9])['WSS'] 
    desc_scaled = df.describe(percentiles = [0.2, 0.5, 0.7, 0.8, 0.9])['WSS'] 

    
    wss_distribution.loc[PATIENT_ID] = [df_wss_org.shape[0], df_wss.shape[0] , 
                                   desc["min"],  desc["20%"],desc["50%"],desc["70%"],desc["80%"],
                                   desc["90%"], desc["max"], 
                                  ]
    
    wss_distribution_scaled.loc[PATIENT_ID] = [df_wss_org.shape[0], df_wss.shape[0] , 
                                   desc_scaled["min"],  desc_scaled["20%"],desc_scaled["50%"],desc_scaled["70%"]
                                               ,desc_scaled["80%"],
                                   desc_scaled["90%"], desc_scaled["max"], 
                                  ]

wss_distribution = wss_distribution.infer_objects()
wss_distribution_scaled = wss_distribution_scaled.infer_objects()

wss_distribution.head(2)
wss_distribution.describe(percentiles = np.linspace(0.5,1.0,10))
#95% of the files have 90% of points with WSS <0.46
wss_distribution_scaled.describe(percentiles = np.linspace(0.5,1.0,10))
sns.boxplot(x="variable", y="value", data=pd.melt(wss_distribution.iloc[:,2:]))

plt.show()
sns.boxplot(x="variable", y="value", data=pd.melt(wss_distribution_scaled.iloc[:,2:]))

plt.show()<jupyter_output><empty_output><jupyter_text>Local Scaling<jupyter_code>def scale_data(col: pd.Series,scaler) -> pd.Series:
    """
    Scale a column
    """
    X = col.values.reshape(-1, 1).copy()
    scaled_array = scaler.fit_transform(X)

    scaled_column = pd.Series(scaled_array.tolist()).explode()

    return scaled_column
wss_columns = ['original_points','sampled_points','min','20p','50p','70p','80p','90p','max']



wss_distribution = pd.DataFrame(index=patient_id_list, columns = wss_columns )
wss_distribution_scaled = pd.DataFrame(index=patient_id_list, columns = wss_columns )

if not wss_distribution.empty and wss_eda_scaled.empty :
    del(wss_distribution)
    del(wss_distribution_scaled)
    

wss_distribution.head(2)

for PATIENT_ID in patient_id_list_new:

    df_wss_org= pd.read_csv(os.path.join(INPUT_PATH,f"{PATIENT_ID}{WSS_FILENAME}"))
    df_wss= pd.read_csv(os.path.join(INPUT_TEMP_PATH,f"{PATIENT_ID}{WSS_DOWN_FILENME}"))
    
#     #Scale the data
    df = df_wss.copy()
    min_max_scaler = MinMaxScaler() 
    df["WSS"] = scale_data(df["WSS"],min_max_scaler)
    df = df.infer_objects()
#     col = "WSS" 
    
#     r_min = 0
#     r_max = 183
    
#     df = df_wss.copy()
#     df.loc[df[col] < r_min] = r_min
#     df.loc[df[col] > r_max] = r_max
#     df[col] = df[col].apply(
#         lambda x: my_global_scaler(x, r_min, r_max)
#     )
    
    desc = df_wss.describe(percentiles = [0.2, 0.5, 0.7, 0.8, 0.9])['WSS'] 
    desc_scaled = df.describe(percentiles = [0.2, 0.5, 0.7, 0.8, 0.9])['WSS'] 

    
    wss_distribution.loc[PATIENT_ID] = [df_wss_org.shape[0], df_wss.shape[0] , 
                                   desc["min"],  desc["20%"],desc["50%"],desc["70%"],desc["80%"],
                                   desc["90%"], desc["max"], 
                                  ]
    
    wss_distribution_scaled.loc[PATIENT_ID] = [df_wss_org.shape[0], df_wss.shape[0] , 
                                   desc_scaled["min"],  desc_scaled["20%"],desc_scaled["50%"],desc_scaled["70%"]
                                               ,desc_scaled["80%"],
                                   desc_scaled["90%"], desc_scaled["max"], 
                                  ]

wss_distribution = wss_distribution.infer_objects()
wss_distribution_scaled = wss_distribution_scaled.infer_objects()

wss_distribution.head(2)
#95% of the files have 90% of points with WSS <0.46
wss_distribution_scaled.describe(percentiles = np.linspace(0.5,1.0,10))
sns.boxplot(x="variable", y="value", data=pd.melt(wss_distribution_scaled.iloc[:,2:]))

plt.show()<jupyter_output><empty_output><jupyter_text>2. Based on Mean Curvature<jupyter_code>curv_columns = ['original_points','sampled_points','mean','median','std','min','max',]

curv_eda = pd.DataFrame(index=patient_id_list, columns = curv_columns )
curv_eda_scaled_global = pd.DataFrame(index=patient_id_list, columns = curv_columns )
curv_eda_scaled_local = pd.DataFrame(index=patient_id_list, columns = curv_columns )


if not curv_eda.empty and curv_eda_scaled.empty :
    del(curv_eda)
    del(curv_eda_scaled)
    

curv_eda_scaled.head(2)
for PATIENT_ID in patient_id_list:

    df_org= pd.read_csv(os.path.join(INPUT_PATH,f"{PATIENT_ID}{WSS_FILENAME}"))
    df_curv= pd.read_csv(os.path.join(INPUT_TEMP_PATH,f"{PATIENT_ID}{CURV_FILENAME}"))
    
    #Scale the data
    min_max_scaler = MinMaxScaler()
    df = df_curv.copy()
    
    df["curv_scaled"] = scale_data(df["mean_curv"],min_max_scaler)
    df = df.infer_objects()
   
    curv_eda.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0] , 
                               np.mean(df.mean_curv), np.median(df.mean_curv), np.std(df.mean_curv), 
                               np.min(df.mean_curv), np.max(df.mean_curv)
                         ]
    curv_eda_scaled_local.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0],
                                      np.mean(df.curv_scaled), np.median(df.curv_scaled), np.std(df.curv_scaled), 
                                      np.min(df.curv_scaled), np.max(df.curv_scaled)]
    col = "mean_curv" 
    
    r_min = -0.001
    r_max = 0.0008

    df = df_curv.copy()
    
    df.loc[df[col] < r_min] = r_min
    df.loc[df[col] > r_max] = r_max
    df["curv_scaled"] = df[col].apply(
        lambda x: my_global_scaler(x, r_min, r_max)
    )
    df = df.infer_objects()
    curv_eda_scaled_global.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0],
                                      np.mean(df.curv_scaled), np.median(df.curv_scaled), np.std(df.curv_scaled), 
                                      np.min(df.curv_scaled), np.max(df.curv_scaled)]
    
                         
curv_eda = curv_eda.infer_objects()
curv_eda_scaled_local = curv_eda_scaled_local.infer_objects()
curv_eda_scaled_global = curv_eda_scaled_global.infer_objects()

curv_eda_scaled.head(2)
curv_eda.describe(percentiles = np.linspace(0.5,1.0,10))<jupyter_output><empty_output><jupyter_text>Remove outliers<jupyter_code>patient_id_list_new = [x for x in patient_id_list if x not in OUTLIER_WSS_PIDs]
len(patient_id_list_new)
curv_columns = ['original_points','sampled_points','mean','median','std','min','max',]

curv_eda = pd.DataFrame(index=patient_id_list, columns = curv_columns )
curv_eda_scaled_global = pd.DataFrame(index=patient_id_list, columns = curv_columns )
curv_eda_scaled_local = pd.DataFrame(index=patient_id_list, columns = curv_columns )


if not curv_eda.empty and curv_eda_scaled.empty :
    del(curv_eda)
    del(curv_eda_scaled)
    

curv_eda_scaled.head(2)
for PATIENT_ID in patient_id_list_new:

    df_org= pd.read_csv(os.path.join(INPUT_PATH,f"{PATIENT_ID}{WSS_FILENAME}"))
    df_curv= pd.read_csv(os.path.join(INPUT_TEMP_PATH,f"{PATIENT_ID}{CURV_FILENAME}"))
    
    #Scale the data
    min_max_scaler = MinMaxScaler()
    df = df_curv.copy()
    
    df["curv_scaled"] = scale_data(df["mean_curv"],min_max_scaler)
    df = df.infer_objects()
   
    curv_eda.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0] , 
                               np.mean(df.mean_curv), np.median(df.mean_curv), np.std(df.mean_curv), 
                               np.min(df.mean_curv), np.max(df.mean_curv)
                         ]
    curv_eda_scaled_local.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0],
                                      np.mean(df.curv_scaled), np.median(df.curv_scaled), np.std(df.curv_scaled), 
                                      np.min(df.curv_scaled), np.max(df.curv_scaled)]
    col = "mean_curv" 
    
    r_min = -0.001
    r_max = 0.0008

    df = df_curv.copy()
    
    df.loc[df[col] < r_min] = r_min
    df.loc[df[col] > r_max] = r_max
    df["curv_scaled"] = df[col].apply(
        lambda x: my_global_scaler(x, r_min, r_max)
    )
    df = df.infer_objects()
    curv_eda_scaled_global.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0],
                                      np.mean(df.curv_scaled), np.median(df.curv_scaled), np.std(df.curv_scaled), 
                                      np.min(df.curv_scaled), np.max(df.curv_scaled)]
    
                         
curv_eda = curv_eda.infer_objects()
curv_eda_scaled_local = curv_eda_scaled_local.infer_objects()
curv_eda_scaled_global = curv_eda_scaled_global.infer_objects()

curv_eda_scaled.head(2)
curv_eda_scaled_global.describe(percentiles = np.linspace(0.5,1.0,10))
sns.boxplot(x="variable", y="value", data=pd.melt(curv_eda_scaled_local.iloc[:,2:]))

plt.show()
sns.boxplot(x="variable", y="value", data=pd.melt(curv_eda_scaled_global.iloc[:,2:]))

plt.show()<jupyter_output><empty_output><jupyter_text>3. Based on Gauss Curvature<jupyter_code>curv_columns = ['original_points','sampled_points','mean','median','std','min','max',]

curv_eda = pd.DataFrame(index=patient_id_list, columns = curv_columns )
curv_eda_scaled = pd.DataFrame(index=patient_id_list, columns = curv_columns )

if not curv_eda.empty and curv_eda_scaled.empty :
    del(curv_eda)
    del(curv_eda_scaled)
    

curv_eda_scaled.head(2)
for PATIENT_ID in patient_id_list:

    df_org= pd.read_csv(os.path.join(INPUT_PATH,f"{PATIENT_ID}{WSS_FILENAME}"))
    df= pd.read_csv(os.path.join(INPUT_TEMP_PATH,f"{PATIENT_ID}{CURV_FILENAME}"))
    
    #Scale the data
    min_max_scaler = MinMaxScaler() 
    df["curv_scaled"] = scale_data(df["gauss_curv"],min_max_scaler)
    
    
    curv_eda.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0] , 
                               np.mean(df.gauss_curv), np.median(df.gauss_curv), np.std(df.gauss_curv), 
                               np.min(df.gauss_curv), np.max(df.gauss_curv)
                         ]
#     curv_eda_scaled.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0],
#                                       np.mean(df.curv_scaled), np.median(df.curv_scaled), np.std(df.curv_scaled), 
#                                       np.min(df.curv_scaled), np.max(df.curv_scaled)
#                          ]
curv_eda = curv_eda.infer_objects()
# curv_eda_scaled = curv_eda_scaled.infer_objects()

curv_eda_scaled.head(2)
curv_eda.describe(percentiles = np.linspace(0.5,1.0,10))<jupyter_output><empty_output><jupyter_text>Remove outliers<jupyter_code>patient_id_list_new = [x for x in patient_id_list if x not in OUTLIER_WSS_PIDs]
len(patient_id_list_new)
curv_columns = ['original_points','sampled_points','mean','median','std','min','max',]

if not curv_eda.empty and curv_eda_scaled.empty :
    del(curv_eda_new)
    del(curv_eda_scaled)
    
curv_eda_new = pd.DataFrame(index=patient_id_list, columns = curv_columns )
curv_eda_scaled = pd.DataFrame(index=patient_id_list, columns = curv_columns )



curv_eda_new.head(2)
for PATIENT_ID in patient_id_list_new:

    df_org= pd.read_csv(os.path.join(INPUT_PATH,f"{PATIENT_ID}{WSS_FILENAME}"))
    df= pd.read_csv(os.path.join(INPUT_TEMP_PATH,f"{PATIENT_ID}{CURV_FILENAME}"))
    
    #Scale the data
    min_max_scaler = MinMaxScaler() 
    df["curv_scaled"] = scale_data(df["gauss_curv"],min_max_scaler)
    
    
    curv_eda_new.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0] , 
                               np.mean(df.gauss_curv), np.median(df.gauss_curv), np.std(df.gauss_curv), 
                               np.min(df.gauss_curv), np.max(df.gauss_curv)
                         ]
#     curv_eda_scaled.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0],
#                                       np.mean(df.curv_scaled), np.median(df.curv_scaled), np.std(df.curv_scaled), 
#                                       np.min(df.curv_scaled), np.max(df.curv_scaled)
#                          ]
curv_eda_new = curv_eda_new.infer_objects()
# curv_eda_scaled = curv_eda_scaled.infer_objects()

curv_eda_new.head(2)
curv_eda_new
curv_eda_new.describe(percentiles = np.linspace(0.5,1.0,10))<jupyter_output><empty_output><jupyter_text>3. Based on FPFH Value<jupyter_code>feat_columns = ['original_points','sampled_points','mean','median','std','min','max',]

feat_new_1 = pd.DataFrame(index=patient_id_list, columns = feat_columns )
feat_new_2= pd.DataFrame(index=patient_id_list, columns = feat_columns )

if not feat_new_1.empty and feat_new_2.empty :
    del(feat_new_1)
    del(feat_new_2)
    

feat_new_1.head(2)
for PATIENT_ID in patient_id_list_new:

    df_org= pd.read_csv(os.path.join(INPUT_PATH,f"{PATIENT_ID}{WSS_FILENAME}"))
#     df= pd.read_csv(os.path.join(INPUT_TEMP_PATH,f"{PATIENT_ID}{CURV_FILENAME}"))
    
    meshply = PlyData.read(os.path.join(OUTPUT_PCA_PATH,f"{PATIENT_ID}{PCA_FILENAME}"))
    df = pd.DataFrame()
    df['x'] = pd.Series(meshply.elements[0].data['x'])
    df['y'] = pd.Series(meshply.elements[0].data['y'])
    df['z'] = pd.Series(meshply.elements[0].data['z'])
    df[['fpfh_1','fpfh_2']] = pd.DataFrame(meshply.elements[0].data['fpfh'].tolist())
    
    #Scale the data
    min_max_scaler = MinMaxScaler() 
    df["fpfh_1_scaled"] = scale_data(df["fpfh_1"],min_max_scaler)
    df["fpfh_2_scaled"] = scale_data(df["fpfh_2"],min_max_scaler)

    
    
    feat_new_1.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0] , 
                               np.mean(df.fpfh_1), np.median(df.fpfh_1), np.std(df.fpfh_1), 
                               np.min(df.fpfh_1), np.max(df.fpfh_1)
                                 ]
    
    feat_new_2.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0] , 
                               np.mean(df.fpfh_2), np.median(df.fpfh_2), np.std(df.fpfh_2), 
                               np.min(df.fpfh_2), np.max(df.fpfh_2)
                                 ]
                                  
#     curv_eda_scaled.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0],
#                                       np.mean(df.curv_scaled), np.median(df.curv_scaled), np.std(df.curv_scaled), 
#                                       np.min(df.curv_scaled), np.max(df.curv_scaled)
#                          ]
feat_new_1 = feat_new_1.infer_objects()
feat_new_2 = feat_new_2.infer_objects()

# curv_eda_scaled = curv_eda_scaled.infer_objects()

feat_new_1.head(2)
feat_new_1.describe(percentiles = np.linspace(0.5,1.0,10))
feat_new_2.describe(percentiles = np.linspace(0.5,1.0,10))<jupyter_output><empty_output><jupyter_text>4. Based on SHOT Value<jupyter_code>feat_columns = ['original_points','sampled_points','mean','median','std','min','max',]

feat_new_1 = pd.DataFrame(index=patient_id_list, columns = feat_columns )
feat_new_2= pd.DataFrame(index=patient_id_list, columns = feat_columns )
feat_new_3= pd.DataFrame(index=patient_id_list, columns = feat_columns )

if not feat_new_1.empty and feat_new_2.empty :
    del(feat_new_1)
    del(feat_new_2)
    del(feat_new_3)

    

feat_new_1.head(2)
for PATIENT_ID in patient_id_list_new:

    df_org= pd.read_csv(os.path.join(INPUT_PATH,f"{PATIENT_ID}{WSS_FILENAME}"))
#     df= pd.read_csv(os.path.join(INPUT_TEMP_PATH,f"{PATIENT_ID}{CURV_FILENAME}"))
    
    meshply = PlyData.read(os.path.join(OUTPUT_PCA_PATH,f"{PATIENT_ID}{PCA_FILENAME}"))
    df = pd.DataFrame()
    df['x'] = pd.Series(meshply.elements[0].data['x'])
    df['y'] = pd.Series(meshply.elements[0].data['y'])
    df['z'] = pd.Series(meshply.elements[0].data['z'])
    df[['shot_1','shot_2','shot_3']] = pd.DataFrame(meshply.elements[0].data['shot'].tolist())
    
    #Scale the data
    min_max_scaler = MinMaxScaler() 
    df["shot_1_scaled"] = scale_data(df["shot_1"],min_max_scaler)
    df["shot_2_scaled"] = scale_data(df["shot_2"],min_max_scaler)
    df["shot_3_scaled"] = scale_data(df["shot_3"],min_max_scaler)


    
    feat_new_1.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0] , 
                               np.mean(df.shot_1), np.median(df.shot_1), np.std(df.shot_1), 
                               np.min(df.shot_1), np.max(df.shot_1)
                                 ]
    
    feat_new_2.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0] , 
                               np.mean(df.shot_2), np.median(df.shot_2), np.std(df.shot_2), 
                               np.min(df.shot_2), np.max(df.shot_2)
                                 ]
                                  
    feat_new_3.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0] , 
                               np.mean(df.shot_3), np.median(df.shot_3), np.std(df.shot_3), 
                               np.min(df.shot_3), np.max(df.shot_3)
                                 ]
#     curv_eda_scaled.loc[PATIENT_ID] = [df_org.shape[0], df.shape[0],
#                                       np.mean(df.curv_scaled), np.median(df.curv_scaled), np.std(df.curv_scaled), 
#                                       np.min(df.curv_scaled), np.max(df.curv_scaled)
#                          ]
feat_new_1 = feat_new_1.infer_objects()
feat_new_2 = feat_new_2.infer_objects()
feat_new_3 = feat_new_3.infer_objects()

# curv_eda_scaled = curv_eda_scaled.infer_objects()

feat_new_1.head(2)
feat_new_1.describe(percentiles = np.linspace(0.5,1.0,10))
feat_new_2.describe(percentiles = np.linspace(0.5,1.0,10))
feat_new_3.describe(percentiles = np.linspace(0.5,1.0,10))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14730533903277176, 'fraction_numerical': 0.021163355963456542, 'mean_word_length': 3.0072208228379513, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '516114', 'license': 'Apache-2.0', 'n_tokens_mistral': 9998, 'n_tokens_neox': 9545, 'n_words': 1034, 'path': 'aneurysm_segmentation3d/notebooks/EDA 2 - Min_Max.ipynb', 'repo_name': 'abhimandal/aneurysm-segmentation', 'size': '23862'}