<jupyter_start><jupyter_text>**Final: Removing Correlations**Used to clean up the datasets taking into account the correlations(spearman) between variables<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler<jupyter_output><empty_output><jupyter_text>1.- Read the dataframes<jupyter_code>dflist_areas = []
areas = ['ingenieria', 'negocios','estudioscreativos', 'salud', 'cienciassociales', 'ambienteconstruido']

for elem in areas:
    df_area = pd.read_csv("C:/Users/Milara/..."+elem+"_v1.csv",encoding="ISO-8859-1")
    dflist_areas.append(df_area)<jupyter_output><empty_output><jupyter_text>2.- Create the functions to get the subsets of correlations and to remove variables<jupyter_code># Pair of combinations
import itertools


def get_subsetCorr(area_temp):
    """
    Function that computes the correlation between pairs of variables
    Input : dataframe to be analyzed
    Output: highest correlation value and pair 
    
    Note: df with no TARGET VARIABLE  (input)
    """
    columns = area_temp.columns # Get the list of variables
    list_corr = [] #Empty list to store the correlation for each pair of variables
    list_subset = [] #Empty list to store the subsets for each pair of variables

    for subset in itertools.combinations(columns, 2):#create the combinations nC2, where n is the number of variables
        area_in = area_temp[[subset[0],subset[1]]] # Get the df only for the pair of variables
        list_corr.append(np.squeeze(area_in.corr(method = "spearman").loc[[area_in.columns[0]],[area_in.columns[1]]].values))
        list_subset.append(subset) 


    df_results = pd.DataFrame({'CoefCorrelation': list_corr,'Subset':list_subset})
    df_results.sort_values(by='CoefCorrelation', ascending = False, inplace = True)
    
    return(df_results['CoefCorrelation'].iloc[0], df_results['Subset'].iloc[0])
    

def remove_lessCorrVar(area_df,subset):
    """
    Function that deletes the variable less correlated to the target variable
    Input:  dataframe with all the variables, subset with results of the highest correlation pair
    Output: dataframe with the variable removed
    
    Note: df with TARGET VARIABLE (input)
    """
    area_temp = area_df.copy()
    
    a = np.squeeze(area_temp[[subset[0],'For_Prom']].corr(method = "spearman").loc[[subset[0]],['For_Prom']].values)
    b = np.squeeze(area_temp[[subset[1],'For_Prom']].corr(method = "spearman").loc[[subset[1]],['For_Prom']].values)

    if a > b : 
        df_out = area_temp.drop(subset[1], axis = 1)
        removed = subset[1]
    elif b > a : 
        df_out = area_temp.drop(subset[0], axis = 1)
        removed = subset[0]
    else:
        rand = np.random.random()
        if rand > 0.5 :
            df_out = area_temp.drop(subset[1], axis = 1)
            removed = subset[1]
        else : 
            df_out = area_temp.drop(subset[0], axis = 1)
            removed = subset[0]
            
    return df_out, removed<jupyter_output><empty_output><jupyter_text>3.- Create the function to clean correlated variables under same database<jupyter_code>def clean_corrDB(df):
    """
    Function that receives a dataframe and cleans the correlated variables
    Default to remove variable with correlation above 0.5
    in : df to clean, with target variable
    out: df cleaned
    """
    df_temp = df.copy()
    flag = True
    while(flag):

        topCorr, sub_Corr = get_subsetCorr(df_temp.drop('For_Prom', axis = 1)) # compute the correlation of subsets
        #print(topCorr,sub_Corr, "Highest Correlation")

        if topCorr > 0.5: #moderate correlation -> remove correlated variables above 0.5 
            df_temp, removed = remove_lessCorrVar(df_temp,sub_Corr) # Remove the less correlated variable
            #print(removed, "Variable Removed")

            if df_temp.shape[1] <= 2 :
                flag = False
        else:
            flag = False

    return (df_temp.drop('For_Prom', axis = 1))<jupyter_output><empty_output><jupyter_text>5. Function that runs over the 4 datasets of each area<jupyter_code>def keep_var(prefix,lista):
    """
    Function that only keeps the variables with the same prefix
    in: prefix and list of names
    out: cleaned list
    """
    res = []
    #res.append(lista[0])
    for elem in prefix:
        for i in range(len(lista)):
            value = lista[i].find(elem)
            if value >= 0:
                res.append(lista[i])
    #res.append('For_Prom')
    return res
def clean_area(df):
    """
    Function to run over the whole dataset to clean the correlated variables depending on each dataset
    in: dataframe of an area
    out: dataframe of the area but with variable reduction
    """
    area_temp = df.copy()
    
    list_df = []
    list_db = ['_ord','_n','Eva_ini','Puntaje']

    for elem in list_db:
        listRed = keep_var([elem,'For_Prom'],area_temp.columns)
        df_touse = area_temp[listRed]
        df_cleaned = clean_corrDB(df_touse)
        list_df.append(df_cleaned)

    list_df.append(area_temp[['CalPromedioPrepa','For_Prom']])
    
    return(pd.concat(list_df, axis = 1))<jupyter_output><empty_output><jupyter_text>6. Final Loop to clean all the areas<jupyter_code>for i in range(len(dflist_areas)):
    print(dflist_areas[i].shape, "Before ", areas[i])
    df_out = clean_area(dflist_areas[i])
    print(df_out.shape, "After ", areas[i])
    df_out.to_csv("C:/Users/Milara/..._"+areas[i]+"_v1.csv", encoding="ISO-8859-1", index = False) #thus it is necessary to save it as the usual type of coding<jupyter_output>(1862, 43) Before  ingenieria
(1862, 24) After  ingenieria
(1084, 41) Before  negocios
(1084, 28) After  negocios
(220, 43) Before  estudioscreativos
(220, 29) After  estudioscreativos
(117, 38) Before  salud
(117, 25) After  salud
(209, 41) Before  cienciassociales
(209, 23) After  cienciassociales
(120, 43) Before  ambienteconstruido
(120, 25) After  ambienteconstruido
{'fraction_non_alphanumeric': 0.11003989361702128, 'fraction_numerical': 0.020113031914893616, 'mean_word_length': 3.5308734939759034, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '288491', 'license': 'MIT', 'n_tokens_mistral': 2046, 'n_tokens_neox': 1938, 'n_words': 527, 'path': 'F_Correlations.ipynb', 'repo_name': 'milara-ds/master-thesis', 'size': '6016'}