<jupyter_start><jupyter_text>PreambleThis notebook is used for exploring Decision Trees and Naive Bayes models to find trends in the data. Import libraries<jupyter_code>import sys, subprocess, collections

# Import Pandas
import pandas as pd

# Import SciKit decision tree
from sklearn.tree import DecisionTreeClassifier, export_graphviz

# Import Scikit cross-validation function
from sklearn.cross_validation import cross_val_score
from sklearn.grid_search import GridSearchCV

# Import naive bayes
from sklearn.naive_bayes import BernoulliNB<jupyter_output><empty_output><jupyter_text>Step 1: Import CSV containing photovoltaic performance of solar cells into Pandas Data Frame object<jupyter_code># Import module to read in secure data
sys.path.append('../data/NREL')
import retrieve_data as rd
solar = rd.retrieve_dirks_sheet()<jupyter_output><empty_output><jupyter_text>Step 2: Clean the data for inconsistencies<jupyter_code>sys.path.append('utils')
import process_data as prd
prd.clean_data(solar)<jupyter_output><empty_output><jupyter_text>Step 3: Defining general functions for all classification models<jupyter_code>def get_classification_model_data(df, modes):
    """
    Build a DataFrame that holds all of the information for building naive bayes models
    
    Args:
        df (DataFrame): Pandas DataFrame that has been cleaned using the clean_data function
        modes (list of string): List of degradation modes to add as dummy variable columns
    
    Returns:
        DataFrame: Holds the features and target necessary for building naive bayes
    """
    # Create binary dummy data for each of the categorical variables
    naive_df = pd.DataFrame(df.loc[:, 'Mounting'])
    naive_df = naive_df.join(pd.get_dummies(df['Mounting']))
    naive_df = naive_df.join(df.loc[:, 'Climate3'])
    naive_df = naive_df.join(pd.get_dummies(df['Climate3']))

    # Bin the installation year data into binary values (0: <2000, 1: >=2000)
    naive_df = naive_df.join(df.loc[:, 'Begin.Year'])
    naive_df['Begin.Year'].fillna(naive_df['Begin.Year'].mean(), inplace=True)
    bins = [0, 2000, 9999]
    group_names = [0, 1]
    naive_df['After 2000'] = pd.cut(naive_df['Begin.Year'], bins, labels=group_names)

    # Add the cleaned Cause column for visual reference
    naive_df = naive_df.join(df.loc[:, 'Cause (Cleaned)'])

    # Add the dummy variable columns of the degradation modes
    for m in modes:
        naive_df = pd.concat([naive_df, df[m]], axis=1)
    
    return naive_df<jupyter_output><empty_output><jupyter_text>Step 4: Decision trees - Setup data and build models Defining functions specific to decision tree models<jupyter_code>def visualize_tree(tree, class_name, feature_names, dot_name, png_name):
    """Create tree png using graphviz
    NOTE: Will export trees into an output folder located on the same level as this notebook

    Args:
        tree -- scikit-learn DecsisionTree
        feature_names -- list of feature names
    """
    with open("output/" + dot_name, 'w') as f:
        export_graphviz(tree, out_file=f,
                        filled=True,
                        class_names=['None', class_name],
                        feature_names=feature_names)

    command = ["dot", "-Tpng", "output/" + dot_name, "-o", "output/" + png_name]
    try:
        subprocess.check_call(command)
    except:
        exit("Could not run dot, ie graphviz, to "
             "produce visualization")
def generate_decision_tree_models(df, modes, features, params_dict):
    """
    Generate dot and png files for decision tree models for each specified degradation mode
    
    Args:
        df (DataFrame): DataFrame returned by the get_classification_model_data function
        modes (list of string): List of degradation modes to build models for
        features (list of string): List of features (columns in the dataframe) to use for the model
        params_dict (dict): A dictionary containing the min_samples_leaf and max_depth parameters of each degradation mode
                            Format {Key: Degradation mode,Value:{Key:min_samples_leaf,Value:int}
                                                                {Key:max_depth,Value:int}}
    """
    for m in modes:
        # Fitting the decision tree with scikit-learn
        y = df[m]
        x = df.loc[:, feature_names]
        msl = params_dict[m]['min_samples_leaf']
        md = params_dict[m]['max_depth']
        dt = DecisionTreeClassifier(min_samples_split=1000, min_samples_leaf=msl, max_depth=md, random_state=99)
        dt.fit(x, y)
        scores = cross_val_score(dt, x, y, cv=10, scoring='accuracy')
        
        # Draw the decision tree and save the output file
        if m == 'Diode/J-box problem':
            m = 'Diode_J-box problem'
        dot_name = 'Cause ' + m + '.dot'
        png_name = 'Cause ' + m + '.png'
        print('Exported ' + png_name + ' to output/')
        print('Score: ' + str(scores.mean()))
        visualize_tree(dt, m, feature_names, dot_name, png_name)<jupyter_output><empty_output><jupyter_text>Build decision tree model data and export resultsPrepare the pandas DataFrame that will be used for building both classification models, this involves:- Creating one hot encoding variables for the categorical data, climate and mounting- Bin installation year data into 'Before 2000 and After 2000' categories- Include the previously created one hot encoding variables of degradation modes presence into this model<jupyter_code>feature_names = ('Snow', 'Hot & Humid', 'Desert', 'Moderate',
                'roof', 'roof rack', '1-axis tracker', 'rack')
best_params_dict = {'Major delamination': {'max_depth': 2, 'min_samples_leaf': 2000},
                    'Hot spots': {'max_depth': 1, 'min_samples_leaf': 2600},
                    'Encapsulant discoloration': {'max_depth': 1, 'min_samples_leaf': 2800},
                    'Internal circuitry discoloration':{'max_depth': 1, 'min_samples_leaf': 2000},
                    'Fractured cells': {'max_depth': 1, 'min_samples_leaf': 2000},
                    'Glass breakage': {'max_depth': 1, 'min_samples_leaf': 2000},
                    'Permanent soiling': {'max_depth': 1, 'min_samples_leaf': 2000},
                    'Diode/J-box problem': {'max_depth': 1, 'min_samples_leaf': 2000}}
modes = ['Hot spots', 'Encapsulant discoloration', 'Major delamination', 'Internal circuitry discoloration',
         'Fractured cells', 'Glass breakage', 'Permanent soiling', 'Diode/J-box problem']

dt_df = get_classification_model_data(solar, modes)
dt_df<jupyter_output><empty_output><jupyter_text>Next, build decision trees using mounting type and climate as features and each degradation mode as a target (show Scikit accuracy score for each decision tree model built). Additionally, export them to the output/ directory as dot and png files.<jupyter_code>generate_decision_tree_models(dt_df, modes, feature_names, best_params_dict)<jupyter_output>Exported Cause Hot spots.png to output/
Score: 0.657144521523
Exported Cause Encapsulant discoloration.png to output/
Score: 0.851125373798
Exported Cause Major delamination.png to output/
Score: 0.693859457864
Exported Cause Internal circuitry discoloration.png to output/
Score: 0.73429310679
Exported Cause Fractured cells.png to output/
Score: 0.958627772182
Exported Cause Glass breakage.png to output/
Score: 0.941789186781
Exported Cause Permanent soiling.png to output/
Score: 0.97555675782
Exported Cause Diode_J-box problem.png to output/
Score: 0.889009771133<jupyter_text>Optimizing parameters for building the decision tree models and model cross validationFunctions useful for finding the parameters that create the best performance from each degradation mode's classification modelThe results from previous findings have been saved in the 'best_params_dict' variable above, however the code will be left here in case of further experimentation<jupyter_code>def dt_grid_search(df, mode, feature_names, lr, dr):
    """
    Perform grid search on the given DataFrame to find the optimal parameters for each decision tree
    NOTE: Leaf range is searched in a range of 2000 to 3000 at intervals of 100
          Depth range is searched in a range of 1 and 9 at intervals of 1
          
    Args:
        df (DataFrame): DataFrame returned by the get_classification_model_data function
        mode (string): Degradation mode to run grid search for
        feature_names (list of string): List of features (columns in the dataframe) to use for the model
        lr (list of integers or None): Integers to try for the grid search for the min_samples_leaf param
                                       Defaulted to 2000-3000 for every 100
        dr (list of integers or None): Integers to try for the grid search for the max_depth param
                                       Defaulted to 1-9 for every 1
    """
    y = df[m]
    x = df.loc[:, feature_names]
    leaf_range = lr
    depth_range = dr
    if lr == None:
        leaf_range = [a*100 for a in range(20, 31)]
    if dr == None:
        depth_range = [b for b in range(1, 10)]

    dt = DecisionTreeClassifier(min_samples_split=0.01, min_samples_leaf=6000, max_depth=6, random_state=99)
    param_grid = dict(min_samples_leaf=leaf_range, max_depth=depth_range)

    grid = GridSearchCV(dt, param_grid, cv=10, scoring='accuracy')
    grid.fit(x, y)

    print(m)
    print(grid.best_params_)
    print(grid.best_score_)
# Example of how to use the dt_grid_search function
modes = ['Hot spots', 'Encapsulant discoloration', 'Major delamination', 'Internal circuitry discoloration',
     'Fractured cells', 'Glass breakage', 'Permanent soiling', 'Diode/J-box problem']
feature_names = ('Snow', 'Hot & Humid', 'Desert', 'Moderate',
                'roof', 'roof rack', '1-axis tracker', 'rack')    
    
for m in modes:
    dt_grid_search(dt_df, m, feature_names, None, None)
def dt_cross_val_score(df, m, feature_names, msl, md):
    """
    Performs a 10-fold cross validation on a decisition tree built with the specified parameters
    
    Args:
        df (DataFrame): DataFrame returned by the get_classification_model_data function
        m (string): Degradation mode to build the model on
        msl (int): min_samples_leaf parameter for the model
        md (int): max_depth pramater for the model
    Returns:
        float: The average of accuracies from each of the 10 folds
    """
    y = df[m]
    x = df.loc[:, feature_names]
    
    dt = DecisionTreeClassifier(min_samples_split=0.01, min_samples_leaf=msl, max_depth=md, random_state=99)
    scores = cross_val_score(dt, x, y, cv=10, scoring='accuracy')
    return scores.mean()
# Example of how to use the dt_cross_val_score function
feature_names = ('Snow', 'Hot & Humid', 'Desert', 'Moderate',
                'roof', 'roof rack', '1-axis tracker', 'rack')
dt_cross_val_score(dt_df, 'Hot spots', feature_names, 2600, 1)<jupyter_output><empty_output><jupyter_text>Step 5: Naive Bayes - Setup data and build models Defining functions specific to Naive Bayes models<jupyter_code>def generate_naive_bayes_models(df, features, modes):
    """
    Build a dictionary that holds all Bernoulli Naive Bayes models for specified degradation modes
    NOTE: This function will print the score provided by Scikit of each model
    
    Args:
        df (DataFrame): DataFrame returned by the get_classification_model_data function
        features (list of string): List of desired columns to include in the model as features
        modes (list of string): List of degradation modes to build models for
    Returns:
        dict: Dictionary to hold all Bernoulli Naive Bayes models
              Format is {Key:Degradation mode, Value:respective Naive Bayes model}
    """
    nb_dict = {}

    # Columns to fit against the target for the Naive Bayes models
    X = df.loc[:, features]

    # Find score of Bernoulli Naive Bayes models for each degradation mode
    for m in modes:
        y = df[m]
        clf = BernoulliNB()
        clf.fit(X, y)
        print('Cause: ' + m)
        print('Score: ' + str(clf.score(X, y)))
        nb_dict[m] = clf
        
    return nb_dict<jupyter_output><empty_output><jupyter_text>Build decision naive bayes models and show scoreBuild Naive Bayes models using the same DataFrame from get_classification_model_data() with mounting type, climate and installation year as features and each degradation mode as a target. Scikit-learn score will also be printed for each model built.<jupyter_code>modes = ['Hot spots', 'Encapsulant discoloration', 'Major delamination', 'Internal circuitry discoloration',
         'Fractured cells', 'Glass breakage', 'Permanent soiling', 'Diode/J-box problem']

nb_df = get_classification_model_data(solar, modes)
nb_df.head()
features = ['1-axis tracker', '2-axis tracker', 'rack', 'roof', 'roof rack',
                     'single-axis', 'Desert', 'Hot & Humid', 'Moderate', 'Snow', 'After 2000']

nb_dict = generate_naive_bayes_models(nb_df, features, modes)
nb_dict<jupyter_output>Cause: Hot spots
Score: 0.646659424226
Cause: Encapsulant discoloration
Score: 0.84990041644
Cause: Major delamination
Score: 0.769328263625
Cause: Internal circuitry discoloration
Score: 0.75484338222
Cause: Fractured cells
Score: 0.958627557487
Cause: Glass breakage
Score: 0.963154082926
Cause: Permanent soiling
Score: 0.983885569437
Cause: Diode/J-box problem
Score: 0.761271048343<jupyter_text>Analysis area for Naive Bayes modelsExamples below<jupyter_code># Scikit score of individual naive bayes model
X = nb_df.loc[:, features]
nb_dict['Hot spots'].score(X,nb_df['Hot spots'])
# Posterior probability of hot spots with 1-axis tracker and Desert and BEFORE 2000
nb_dict['Hot spots'].predict_proba([[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0]])
# Posterior probability of hot spots with 1-axis tracker and Desert and AFTER 2000
nb_dict['Hot spots'].predict_proba([[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1]])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08960881069870558, 'fraction_numerical': 0.028606164628477437, 'mean_word_length': 3.670674682698731, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '400954', 'license': 'BSD-3-Clause-LBNL', 'n_tokens_mistral': 4571, 'n_tokens_neox': 4108, 'n_words': 1433, 'path': 'degradation_modes/classification_models_analysis.ipynb', 'repo_name': 'hackingmaterials/duramat', 'size': '13983'}