<jupyter_start><jupyter_text>Predicting Missing ValuesThis notebook is the beginning process for a planned blog post testing different algorithms for effectiveness on filling missing values by building predictive models. To-do:- Collect datasets with missing values- Try missing at random/missing completely at random- Compare against actual values that were artificially removed- Add additional algorithms    - Do classification in addition to just regression- Generate plots of results<jupyter_code>import sys
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import datasets

print(time.strftime('%Y/%m/%d %H:%M'))
print('OS:', sys.platform)
print('Python:', sys.version)
print('NumPy:', np.__version__)
print('Pandas:', pd.__version__)

# Formatting for seaborn plots
sns.set_context('notebook', font_scale=1.1)
sns.set_style('ticks')

# Displays all dataframe columns
pd.set_option('display.max_columns', None)

%matplotlib inline
dataset = datasets.load_breast_cancer()

df = pd.DataFrame(dataset.data, columns=dataset.feature_names)

df['label'] = dataset.target

df.head()
df = df.mask(np.random.random(df.shape) < .1)

# Plotting missing values
import missingno as msno  # Visualizes missing values
msno.matrix(df)
def percent_missing(dataframe):
    """
    Prints the percentage of missing values for each column in a dataframe
    """
    # Summing the number of missing values per column and then dividing by the total
    sumMissing = df.isnull().values.sum(axis=0)
    pctMissing = sumMissing / df.shape[0]
    
    # Looping through and printing out each columns missing value percentage
    print('Percent Missing Values:', '\n')
    for idx, col in enumerate(df.columns):
        print('{0}: {1:.2f}%'.format(col, pctMissing[idx]*100))

        
percent_missing(df)
def predict_missing_summary(data, model, correlationThresh=0.5, cross_validations=3, modelName='model'):
    """
    Takes a dataframe and model and gathers summary statistics on effectiveness of predicting
    missing values
    """
    from sklearn.model_selection import cross_val_score
    
    # Stats to gather
    timeSeconds = []
    crossValScore = []
    r2orAccuracy = []
    
    # Actually predicting per column
    def predict_missing_values(data, column, model, correlationThresh, cross_validations):
        """
        Performs the predictions on individual columns
        """
        
        start_time = time.time()
        
        # Skipping classification features
        num_unique_values = len(np.unique(data[column]))
        if num_unique_values < 25 or data[column].dtype.name == 'category':
            return
        
        # Calculating the highly correlated columns to use for the model
        highlyCorrelated = abs(data.corr()[column]) >= correlationThresh
    
        # Exiting the function if there are not any highly correlated columns found
        if highlyCorrelated.sum() < 2:  # Will always be 1 because of correlation with self
            print('Error: No correlated variables found. Re-try with less a lower correlation threshold')
            return  # Exits the function
        highlyCorrelated = data[data.columns[highlyCorrelated]]
        highlyCorrelated = highlyCorrelated.dropna(how='any')  # Drops any missing records

        # Creating the X/y objects to use for the
        y = highlyCorrelated[column]
        X = highlyCorrelated.drop(column, axis=1)
    
        # Evaluating the effectiveness of the model
        cvScore = np.mean(cross_val_score(model, X, y, cv=cross_validations))
        crossValScore.append(cvScore)

        # Fitting the model for predictions
        model.fit(X, y)
        r2orAccuracy.append(model.score(X, y))
        
        end_time = time.time()
        timeSeconds.append(end_time - start_time)

        
    data.apply(lambda col: predict_missing_values(data, col.name, model, correlationThresh, cross_validations))
    
    results = pd.DataFrame({'Cross Val Mean': np.mean(crossValScore),
                            'Cross Val Std': np.std(crossValScore),  
                            'R2 Mean': np.mean(r2orAccuracy),
                            'R2 Std': np.std(r2orAccuracy),
                            'Execution Time (s) Mean': np.mean(timeSeconds),
                            'Execution Time (s) Std': np.std(timeSeconds)},
                           index=[modelName])
    return results


from sklearn import neighbors
knn = neighbors.KNeighborsRegressor()
predict_missing_summary(df, knn, correlationThresh=0.3, modelName='k-NN')
def compile_results(data, correlationThresh=0.4, cross_validations=3):
    """
    Outputs a data frame of the results for testing predicting 
    missing values using multiple algorithms
    """
    from sklearn import linear_model
    from sklearn import ensemble
    from sklearn import neighbors
    
    linear_regression = linear_model.LinearRegression()
    linearResults = predict_missing_summary(data, linear_regression, correlationThresh,
                                            cross_validations, modelName='Linear Regression')
    
    knn = neighbors.KNeighborsRegressor()
    knnResults = predict_missing_summary(data, knn, correlationThresh,
                                         cross_validations, modelName='k-NN')
    
    rf = ensemble.RandomForestRegressor()
    rfResults = predict_missing_summary(data, rf, correlationThresh,
                                         cross_validations, modelName='Random Forest')
    
    gbt = ensemble.GradientBoostingRegressor()
    gbtResults = predict_missing_summary(data, gbt, correlationThresh,
                                         cross_validations, modelName='Gradient Boosted Trees')
    
    results = linearResults.append(knnResults).append(rfResults).append(gbtResults)
    
    return results
    
    
compile_results(df, correlationThresh=0.3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08446683459277918, 'fraction_numerical': 0.005373635600335852, 'mean_word_length': 2.9839464882943143, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '547667', 'license': 'MIT', 'n_tokens_mistral': 1649, 'n_tokens_neox': 1593, 'n_words': 465, 'path': 'PredictingMissingValues.ipynb', 'repo_name': 'JeffMacaluso/Misc', 'size': '5955'}