<jupyter_start><jupyter_text>FoNS Datathon 2021   Looping through all training data to get the best outcome for a given model.<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import *
train_descriptors = pd.read_csv("train_descriptors.csv")
train_mord3d = pd.read_csv("train_mord3d.csv")
train_morgan = pd.read_csv("train_morgan.csv")
train_rdk = pd.read_csv("train_rdk.csv")

train_crystals = pd.read_csv("train_crystals.csv")
train_distances = pd.read_csv("train_distances.csv")
train_centroid_distances = pd.read_csv("train_centroid_distances.csv")
test_descriptors = pd.read_csv("test_descriptors.csv")
test_mord3d = pd.read_csv("test_mord3d.csv")
test_morgan = pd.read_csv("test_morgan.csv")
test_rdk = pd.read_csv("test_rdk.csv")
def preprocess_rdk(train_data, test_data, target):
    
    # Drop first column - no columns should have NA values
    train_data_full = train_data.iloc[:, 1:]
    
    # Make test set have same columns as training set
    test_data_full = test_data[train_data_full.columns]
    
    # Create training and validation sets
    X_train, X_valid, y_train, y_valid = model_selection.train_test_split(train_data_full,
                                                                              target,
                                                                              random_state = 0)
    
    return X_train, X_valid, y_train, y_valid, train_data_full, test_data_full
def preprocess_morgan(train_data, test_data, target): 
    
    # Drop first column - no columns should have NA values
    train_data_full = train_data.iloc[:, 1:]
    
    # Make test set have same columns as training set
    test_data_full = test_data[train_data_full.columns]
    
    # Create training and validation sets
    X_train, X_valid, y_train, y_valid = model_selection.train_test_split(train_data_full,
                                                                              target,
                                                                              random_state = 0)
    
    return X_train, X_valid, y_train, y_valid, train_data_full, test_data_full
def preprocess_mord3d(train_data, test_data, target, pca=False):
    
    
    # Are there any redundant columns (all values the same)? These wouldn't contribute to the model at all
    for col in train_data.columns:
        if len(train_data[col].unique()) == 1:
            # If so, drop redundant columns:
            train_data.drop(col,inplace=True,axis=1)
                               
    # Drop all columns with NA - and any non-numerical data
    train_data_full = train_data.iloc[:, 2:-4].dropna(axis= 1, how="any")
    
    # Make test set have same columns as training set
    test_data_full = test_data[train_data_full.columns]
    
     # Perform PCA
    if pca == True:
        train_PCA = decomposition.PCA(n_components=.95)
        scaler_for_PCA = preprocessing.StandardScaler()
        train_data_full = train_PCA.fit_transform(scaler_for_PCA.fit_transform(train_data_full))
        test_data_full = train_PCA.transform(scaler_for_PCA.transform(test_data_full))
        
        # Create training and validation sets
        X_train, X_valid, y_train, y_valid = model_selection.train_test_split(train_data_full,
                                                                              target,
                                                                              random_state = 0)
    
    else:
        # Create training and validation sets
        X_train, X_valid, y_train, y_valid = model_selection.train_test_split(train_data_full,
                                                                              target,
                                                                              random_state = 0)

    
    return X_train, X_valid, y_train, y_valid, train_data_full, test_data_full
    
def preprocess_descriptors(train_data, test_data, target, pca=False):
    
    # Are there any redundant columns (all values the same)? These wouldn't contribute to the model at all
    for col in train_data.columns:
        if len(train_data[col].unique()) == 1:
            # If so, drop redundant columns:
            train_data.drop(col,inplace=True,axis=1)
                               
    # Drop all columns with NA - Last 2 columns are InchiKey and SMILES
    train_data_full = train_data.iloc[:, 3:-2].dropna(axis= 1, how="any")
    
    # Make test set have same columns as training set
    test_data_full = test_data[train_data_full.columns]
    
    # Perform PCA
    if pca == True:
        train_PCA = decomposition.PCA(n_components=.95)
        scaler_for_PCA = preprocessing.StandardScaler()
        train_data_full = train_PCA.fit_transform(scaler_for_PCA.fit_transform(train_data_full))
        test_data_full = train_PCA.transform(scaler_for_PCA.transform(test_data_full))
        
        # Create training and validation sets
        X_train, X_valid, y_train, y_valid = model_selection.train_test_split(train_data_full,
                                                                              target,
                                                                              random_state = 0)
    
    else:
        # Create training and validation sets
        X_train, X_valid, y_train, y_valid = model_selection.train_test_split(train_data_full,
                                                                              target,
                                                                              random_state = 0)

    
    return X_train, X_valid, y_train, y_valid, train_data_full, test_data_full
    
# Define the prediction target
prediction_target = "calculated_density"
target= train_crystals[prediction_target]
# Preprocess all datasets
train_descriptors_list = preprocess_descriptors(train_descriptors,
                                                test_descriptors,
                                                target,
                                                pca=True)

train_mord3d_list = preprocess_mord3d(train_mord3d,
                                     test_mord3d,
                                     target, 
                                     pca=True)

train_morgan_list = preprocess_morgan(train_morgan,
                                     test_morgan,
                                     target)

train_rdk_list = preprocess_rdk(train_rdk,
                               test_rdk,
                               target)
# make a list of all the processed datasets
to_loop = [train_descriptors_list, train_mord3d_list, train_morgan_list,  train_rdk_list]
def loop_descriptors(list_of_descriptor_sets, cv_folds):
    
    '''This function loops through the list of processed datasets and outputs
    the CV score for each one in a list.'''
    
    # Set up an algorithm 
    clf = ensemble.RandomForestRegressor(n_estimators=190, max_depth=27.3442, n_jobs=-1)
    
    # Loop through the datasets and calculate CV score
    cv_scores = []
    for dataset in list_of_descriptor_sets:
            
            clf.fit(dataset[0], dataset[2])
            score = model_selection.cross_val_score(clf, dataset[0],
                                            dataset[2], 
                                            n_jobs=-1,
                                            scoring="neg_mean_absolute_error",
                                            cv=cv_folds).mean()
            cv_scores.append(score)
            
    return cv_scores
loop_descriptors(to_loop, 3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.06472919418758256, 'fraction_numerical': 0.007001321003963012, 'mean_word_length': 1.7391461649782922, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 55, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '283976', 'license': 'MIT', 'n_tokens_mistral': 2087, 'n_tokens_neox': 2030, 'n_words': 464, 'path': 'Loop_through_training_data_GH.ipynb', 'repo_name': 'JackS7806/DA_Summer_School_Hackathon', 'size': '7570'}