<jupyter_start><jupyter_text>Basic imports<jupyter_code>import pandas as pd
import os
import time
import math
import matplotlib.pyplot as plt
%matplotlib inline 
plt.rcParams.update({'figure.max_open_warning': 0})
current_path = os.getcwd()
root_path=current_path.replace('\\forward_feature_selection','')<jupyter_output><empty_output><jupyter_text>Loading dataframe<jupyter_code>mixed_df=pd.read_csv(root_path+"\\molecules.csv",sep="\t")<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>from sklearn.model_selection import cross_validate
from sklearn.svm import LinearSVC
def save_df_to_disk(df,name:str,separator="\t"):
    df.to_csv(name,sep=separator,index = False, header=True)
def get_data_and_true_prediction(df,not_wanted_features:list):
        temp_df=df.drop(not_wanted_features,axis=1)
        y=temp_df[temp_df.columns[-1]]
        x=temp_df.drop([temp_df.columns[-1]],axis=1)
        
        return x,y   
def get_df_with_name_and_prediction(df,true_prediction,big_df):
    new_df=df
    new_df.insert(0,"m_name",big_df["m_name"].values)
    new_df=new_df.join(true_prediction)
    
    return new_df
def get_roc_auc_score(x,y,model): # gets roc auc average
        cv_results = cross_validate(model, x, y, cv=10,scoring=('roc_auc'))
        roc_auc_avrg=cv_results['test_score'].mean()
        
        return roc_auc_avrg 
def get_best_score_index(score_lst:list):
    max_score = max(score_lst) # best score
    max_score_index=[i for i, j in enumerate(score_lst) if j == max_score] # indx with best score 
        
    return max_score_index[0]
def show_results(df,roc,f_var,s_var):
    print("\n------------Results-------------\n")
    print("Best model when first param is {} and second param is {}".format(f_var,s_var))
    print("Roc auc = {}".format(roc))
    print("\nFeatures\n")
    for cnt,col in enumerate(df.columns):
        print("{}. {}".format(cnt+1,col))
    print("")
def left_partition_is_better(first_score,middle_score,last_score):
    if ((first_score >= middle_score) and (last_score <= middle_score) and (first_score >= last_score)): return True               
    elif ((first_score <= middle_score) and (middle_score >= last_score) and(last_score <= first_score)): return True                   
    elif ((first_score >= middle_score) and (middle_score <= last_score) and (last_score >= first_score)): return False                  
    elif ((first_score >= middle_score) and (middle_score <= last_score) and (first_score >= last_score)): return True                
    else: return False
def get_model(first_prm,second_prm):
    return LinearSVC(random_state=0, C=first_prm, tol=second_prm, dual=False)<jupyter_output><empty_output><jupyter_text>Forward selection<jupyter_code>def forward_selection(x,y,model): # O(n) worst case scenario, where n depends on len(x.columns)
    
    def first_iteration(x,y,model):
        score_lst = []
        for i in range(len(x.columns)):
            k = x.columns[i]
            temp_x = x[[k]]
            score = get_roc_auc_score(temp_x,y,model)
            score_lst.append(score)
    
        max_score = max(score_lst) # best score
        max_score_index = [i for i, j in enumerate(score_lst) if j == max_score] # indx with best score
        top_score_index = max_score_index[0]
        new_feature = x.columns[top_score_index] 
        best_x = x[new_feature]
        new_x = x.drop(new_feature,axis=1)
        
        return best_x,new_x,max_score
    
    def else_iteration(best_x,x,y,model,actual_score):     
        new_x_lenght = len(x.columns)
        if (new_x_lenght > 0):
            score_lst = []
            for i in range(new_x_lenght):
                k = x.columns[i]
                temp_x = x[[k]]
                temp_new_x = pd.concat([best_x,temp_x],axis=1, ignore_index=True)
                score = get_roc_auc_score(temp_new_x,y,model)
                score_lst.append(score)

            max_score = max(score_lst) # best score

            if(max_score<actual_score):
                return best_x,actual_score # break condition, recursive function

            max_score_index = [i for i, j in enumerate(score_lst) if j == max_score] # indx with best score
            top_score_index = max_score_index[0]

            new_feature = x.columns[top_score_index]
            temp_x = x[new_feature]
            best_x = pd.concat([best_x,temp_x],axis=1)
            new_x = x.drop(new_feature,axis=1)

            return else_iteration(best_x,new_x,y,model,max_score)
        
        return best_x,actual_score
    
    f_best_x,new_x,f_score = first_iteration(x,y,model)
    best_x,best_score = else_iteration(f_best_x,new_x,y,model,f_score)
    
    return best_x,best_score
def inner_forward_partition(x,y,variable_prm_lst:list,static_prm): # (training data, prediction, variable list, static var)

    def else_iteration(x,y,variable_prm_lst:list,first_score,last_score,static_prm):       
        print("\ninner variable param list -> {}".format(variable_prm_lst))  
                           
        if (len(variable_prm_lst) == 2):
            print("\n---------inner param last iteration---------\n")
            first = variable_prm_lst[0]
            print("first is {}".format(first))
            last = variable_prm_lst[-1]
            print("last is {}".format(last))
            
            clf_first = get_model(static_prm,first) 
            first_df,first_score = forward_selection(x,y,clf_first)
            clf_last = get_model(static_prm,last) 
            last_df,last_score = forward_selection(x,y,clf_last)

            print("left score: {} | right score: {}".format(first_score,last_score))
            
            if (last_score > first_score):
                print("last score is greater than first score")
                show_results(last_df,last_score,static_prm,last)
                return last_df,last_score,last

            print("first score is greater than last score")
            show_results(first_df,first_score,static_prm,first)
            return first_df,first_score,first
        else:        
            if(len(variable_prm_lst) % 2 == 0): 
                first = variable_prm_lst[0]
                print("first is {}".format(first))
                middle = variable_prm_lst[(len(variable_prm_lst)//2)-1]
                print("middle is {}".format(middle))
                last = variable_prm_lst[-1]
                print("last is {}".format(last))
   
                clf_middle = get_model(static_prm,middle) 
                _,middle_score = forward_selection(x,y,clf_middle)
        
                print("left score: {} | middle score:{} | right score: {}".format(first_score,middle_score,last_score))

                if (left_partition_is_better(first_score,middle_score,last_score)):             
                    print("taken left: {}".format(variable_prm_lst[:(len(variable_prm_lst)//2)]))
                    return else_iteration(x,y,variable_prm_lst[:(len(variable_prm_lst)//2)],first_score,middle_score,static_prm)          
                
                print("taken right: {}".format(variable_prm_lst[(len(variable_prm_lst)//2)-1:]))
                return else_iteration(x,y,variable_prm_lst[(len(variable_prm_lst)//2)-1:],middle_score,last_score,static_prm)
            else:
                first = variable_prm_lst[0]
                print("first is {}".format(first))
                middle = variable_prm_lst[(len(variable_prm_lst)//2)]
                print("middle is {}".format(middle))
                last = variable_prm_lst[-1]
                print("last is {}".format(last))
    
                clf_middle = get_model(static_prm,middle) 
                _,middle_score = forward_selection(x,y,clf_middle)      
                
                print("left score: {} | middle score:{} | right score: {}".format(first_score,middle_score,last_score))
                
                if (left_partition_is_better(first_score,middle_score,last_score)):
                    print("taken left: {}".format(variable_prm_lst[:(len(variable_prm_lst)//2)+1]))
                    return else_iteration(x,y,variable_prm_lst[:(len(variable_prm_lst)//2)+1],first_score,middle_score,static_prm)
            
                print("taken right: {}".format(variable_prm_lst[(len(variable_prm_lst)//2):]))
                return else_iteration(x,y,variable_prm_lst[(len(variable_prm_lst)//2):],middle_score,last_score,static_prm)
                   
    def first_iteration(x,y,variable_prm_lst:list,static_prm):
        print("\ninner variable param list -> {}".format(variable_prm_lst)) 
        
        if(len(variable_prm_lst) % 2 == 0):       
            first = variable_prm_lst[0]
            print("first is {}".format(first))
            middle = variable_prm_lst[(len(variable_prm_lst)//2)-1]
            print("middle is {}".format(middle))
            last = variable_prm_lst[-1]
            print("last is {}".format(last))
            
            clf_first = get_model(static_prm,first) 
            _,first_score = forward_selection(x,y,clf_first)       
            clf_middle = get_model(static_prm,middle) 
            _,middle_score = forward_selection(x,y,clf_middle)
            clf_last = get_model(static_prm,last) 
            _,last_score = forward_selection(x,y,clf_last)
            
            if(left_partition_is_better(first_score,middle_score,last_score)):
                print("taken left: {}".format(variable_prm_lst[:(len(variable_prm_lst)//2)]))
                return variable_prm_lst[:(len(variable_prm_lst)//2)],first_score,middle_score
          
            print("taken right: {}".format(variable_prm_lst[(len(variable_prm_lst)//2)-1:]))
            return variable_prm_lst[(len(variable_prm_lst)//2)-1:],middle_score,last_score
        
        else:
            first = variable_prm_lst[0]
            print("first is {}".format(first))
            middle = variable_prm_lst[(len(variable_prm_lst)//2)]
            print("middle is {}".format(middle))
            last = variable_prm_lst[-1]
            print("last is {}".format(last))
            
            clf_first = get_model(static_prm,first)  
            _,first_score = forward_selection(x,y,clf_first)       
            clf_middle = get_model(static_prm,middle) 
            _,middle_score = forward_selection(x,y,clf_middle)
            clf_last = get_model(static_prm,last) 
            _,last_score = forward_selection(x,y,clf_last)
            
            if (left_partition_is_better(first_score,middle_score,last_score)):               
                print("taken left: {}".format(variable_prm_lst[:(len(variable_prm_lst)//2)+1]))
                return variable_prm_lst[:(len(variable_prm_lst)//2)+1],first_score,middle_score
         
            print("taken right: {}".format(variable_prm_lst[(len(variable_prm_lst)//2):]))
            return variable_prm_lst[(len(variable_prm_lst)//2):],middle_score,last_score

    if(len(variable_prm_lst) > 1 and len(variable_prm_lst) <=2):
        print("\nfirst param is {}".format(static_prm))
        best_df,best_score,best_variable_prm = else_iteration(x,y,variable_prm_lst,0,0,static_prm)
        return best_df,best_score,best_variable_prm                          
    else:
        print("\nouter param is {}".format(static_prm))
        new_variable_prm_lst,first_score,last_score = first_iteration(x,y,variable_prm_lst,static_prm)
        print("\nout prms from inner first iteration => variable_prm_lst: {} | left score: {} | right score: {}".format(new_variable_prm_lst,first_score,last_score))
        best_df,best_score,best_variable_prm = else_iteration(x,y,new_variable_prm_lst,first_score,last_score,static_prm)
        return best_df,best_score,best_variable_prm                          
def forward_partition(x,y,K:list,P:list): # (training data, prediction, outer variable list, inner variable list)
    
    def first_iteration(x,y,K:list,P:list):
        print("outer param list -> {}".format(K))
        
        if(len(K) % 2 == 0):
            first = K[0]
            print("first is {}".format(first))
            middle = K[(len(K)//2)-1]
            print("middle is {}".format(middle))
            last = K[-1]
            print("last is {}".format(last))
            
            print("\n**************************")
            _,first_score,first_best_p = inner_forward_partition(x,y,P,first)
            print("\n**************************")
            _,middle_score,middle_best_p = inner_forward_partition(x,y,P,middle)
            print("\n**************************")
            _,last_score,last_best_p = inner_forward_partition(x,y,P,last)
            
            print("left score: {} | middle score:{} | right score: {}".format(first_score,middle_score,last_score))
            
            if (left_partition_is_better(first_score,middle_score,last_score)):          
                print("taken left: {}".format(K[:(len(K)//2)]))
                return K[:(len(K)//2)],first_score,middle_score
          
            print("taken right: {}".format(K[(len(K)//2)-1:]))
            return K[(len(K)//2)-1:],middle_score,last_score
        else:
            first = K[0]
            print("first is {}".format(first))
            middle = K[(len(K)//2)]
            print("middle is {}".format(middle))
            last = K[-1]
            print("last is {}".format(last))
            
            print("\n**************************")
            _,first_score,first_best_p = inner_forward_partition(x,y,P,first)
            print("\n**************************")
            _,middle_score,middle_best_p = inner_forward_partition(x,y,P,middle)
            print("\n**************************")
            _,last_score,last_best_p = inner_forward_partition(x,y,P,last)
            
            print("left score: {} | middle score:{} | right score: {}".format(first_score,middle_score,last_score))
            
            if (left_partition_is_better(first_score,middle_score,last_score)):
                print("taken left: {}".format(K[:(len(K)//2)+1]))
                return K[:(len(K)//2)+1],first_score,middle_score
         
            print("taken right: {}".format(K[(len(K)//2):]))
            return K[(len(K)//2):],middle_score,last_score

    def else_iteration(x,y,K:list,P:list,first_score,last_score):
        
        if(len(K) == 2):          
            print("\n---------outer param last iteration---------\n")
            first = K[0]
            print("first is {}".format(first))
            last = K[-1]
            print("last is {}".format(last))
            print("\n**************************")
            first_df,first_score,first_best_p = inner_forward_partition(x,y,P,first)
            print("\n**************************")
            last_df,last_score,last_best_p = inner_forward_partition(x,y,P,last) 
            print("")
            
            if (last_score > first_score):
                print("last score is greater than first score")
                return last_df,last_score,last,last_best_p

            print("first score is greater than last score")
            return first_df,first_score,first,first_best_p        
        else:
            print("first param list -> {}".format(K))  
        
            if(len(K) % 2 == 0):
                first = K[0]
                print("first is {}".format(first))
                middle = K[(len(K)//2)-1]
                print("middle is {}".format(middle))
                last = K[-1]
                print("last is {}".format(last))
                
                print("\n**************************")
                _,middle_score,middle_best_p = inner_forward_partition(x,y,P,middle)
                
                print("left score: {} | middle score:{} | right score: {}".format(first_score,middle_score,last_score))
                
                if (left_partition_is_better(first_score,middle_score,last_score)):                  
                    print("taken left: {}".format(K[:(len(K)//2)]))
                    return else_iteration(x,y,K[:(len(K)//2)],P,first_score,middle_score)
                
                print("taken right: {}".format(K[(len(K)//2)-1:]))
                return else_iteration(x,y,K[(len(K)//2)-1:],P,middle_score,last_score)
            else:
                first = K[0]
                print("first is {}".format(first))
                middle = K[(len(K)//2)]
                print("middle is {}".format(middle))
                last = K[-1]
                print("last is {}".format(last))
                
                print("\n**************************")
                _,middle_score,middle_best_p = inner_forward_partition(x,y,P,middle)
                
                print("left score: {} | middle score:{} | right score: {}".format(first_score,middle_score,last_score))

                if (left_partition_is_better(first_score,middle_score,last_score)):                    
                    print("taken left: {}".format(K[:(len(K)//2)+1]))
                    return else_iteration(x,y,K[:(len(K)//2)+1],P,first_score,middle_score)
               
                print("taken right: {}".format(K[(len(K)//2):]))
                return else_iteration(x,y,K[(len(K)//2):],P,middle_score,last_score)
    
    if(len(K) > 1 and len(K) <= 2):
        best_df,best_score,best_first_var,best_second_var = else_iteration(x,y,K,P,0,0)
        show_results(best_df,best_score,best_first_var,best_second_var)
        return [best_df,best_score,best_first_var,best_second_var]                          
    else:
        new_lst,first_score,last_score = first_iteration(x,y,K,P)
        print("\nout prms from outer first iteration => lst: {} | left score: {} | right score: {}".format(new_lst,first_score,last_score))
        best_df,best_score,best_first_var,best_second_var = else_iteration(x,y,new_lst,P,first_score,last_score)
        show_results(best_df,best_score,best_first_var,best_second_var)
        return [best_df,best_score,best_first_var,best_second_var]  
def guarantee_results(x,y,K:list,P:list):    
    best_outer_prm_lst = []
    for i in K:
        print("----------------------------\n")
        print("First param is {}\n".format(i))
        best_inner_prm_lst = []
        for j in P:
            print("----------------------------\n")
            print("Second param is {}".format(j))
            clf = get_model(i,j)
            best_df,best_score = forward_selection(x,y,clf)
            print("ROC AUC = {}".format(best_score))
            print("\nFeatures\n")
            for c,d in enumerate(best_df.columns):
                print("{}. {}".format(c+1,d))       
            print("")
            best_inner_prm_lst.append([best_df,i,j,best_score])
            
        score_lst = [d  for [a,b,c,d] in best_inner_prm_lst]
        index = get_best_score_index(score_lst)
        best_outer_prm_lst.append([best_inner_prm_lst[index][0],best_inner_prm_lst[index][1],best_inner_prm_lst[index][2],best_inner_prm_lst[index][3]])
        
    score_lst = [d  for [a,b,c,d] in best_outer_prm_lst]
    index = get_best_score_index(score_lst)
    print("\n------------ Final results ----------------")
    print("Best model when first param = {} ,second param = {} ,ROC AUC = {}".format(best_outer_prm_lst[index][1],best_outer_prm_lst[index][2],best_outer_prm_lst[index][3]))
    
    print("\nFeatures\n")
    for c,d in enumerate(best_outer_prm_lst[index][0].columns):
        print("{}. {}".format(c+1,d))       
            
    return [best_outer_prm_lst[index][0],best_outer_prm_lst[index][1],best_outer_prm_lst[index][2],best_outer_prm_lst[index][3]]
def for_TUKEY_TEST(x,y,f,s):
    model = get_model(f,s)
    cv_results = cross_validate(model, x, y, cv=10,scoring=('roc_auc'))
    rslts = cv_results['test_score']
    print(rslts)
    return list(rslts)<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>unnecesary_features=["m_name"]
x,y = get_data_and_true_prediction(mixed_df,unnecesary_features)
C = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
tol = [0.000001,0.00001,0.0001,0.001,0.01,0.1,1,5,10,15]
start = time.time()
best_data = forward_partition(x,y,C,tol)
end = time.time()

time.strftime('%H:%M:%S', time.gmtime(end-start))
fp_rslts = for_TUKEY_TEST(best_data[0],y,best_data[2],best_data[-1])
start = time.time()
g_data = guarantee_results(x,y,C,tol)
end = time.time()

time.strftime('%H:%M:%S', time.gmtime(end-start))
g_rslts = for_TUKEY_TEST(g_data[0],y,g_data[1],g_data[2])<jupyter_output>[0.89632937 0.89980159 0.87450397 0.86954365 0.8891129  0.89465726
 0.85848075 0.9037461  0.91831426 0.91050989]<jupyter_text>Comparison<jupyter_code>print("forward partiton | normal way")
for fp,g  in zip (fp_rslts,g_rslts):
    print("{} | {}".format(fp,g))<jupyter_output>forward partiton | normal way
0.8938492063492063 | 0.8963293650793651
0.8973214285714285 | 0.8998015873015872
0.8700396825396826 | 0.8745039682539683
0.8754960317460317 | 0.8695436507936508
0.8936491935483871 | 0.8891129032258065
0.887600806451613 | 0.8946572580645161
0.8610822060353799 | 0.858480749219563
0.9001040582726327 | 0.9037460978147763
0.917273673257024 | 0.9183142559833507
0.9131113423517169 | 0.9105098855359001<jupyter_text>Selected features<jupyter_code># nested binary search
best_data[0].columns
# normal way
g_data[0].columns<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1484695782008212, 'fraction_numerical': 0.030281821575214633, 'mean_word_length': 2.604608139926001, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '221192', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 7525, 'n_tokens_neox': 7047, 'n_words': 991, 'path': 'forward_feature_selection/Forward_feature_selection_Lineal.ipynb', 'repo_name': 'PaolaRondon/Biomol_classifier', 'size': '21432'}