<jupyter_start><jupyter_text>Preamble Import libraries<jupyter_code>import os, sys

# Import Pandas
import pandas as pd

# Import Plotly and Cufflinks
# Plotly username and API key should be set in environment variables
import plotly
plotly.tools.set_credentials_file(username=os.environ['PLOTLY_USERNAME'], api_key=os.environ['PLOTLY_KEY'])
import plotly.graph_objs as go
import cufflinks as cf

# Import numpy
import numpy as np<jupyter_output><empty_output><jupyter_text>Step 1: Import CSV containing photovoltaic performance of solar cells into Pandas Data Frame object<jupyter_code># Import module to read in secure data
sys.path.append('../data/NREL')
import retrieve_data as rd
solar = rd.retrieve_dirks_sheet()<jupyter_output><empty_output><jupyter_text>Step 2: Clean the data for inconsistencies<jupyter_code>sys.path.append('utils')
import process_data as prd
prd.clean_data(solar)<jupyter_output><empty_output><jupyter_text>Step 3: Import functions from utils and define notebook-specific functions<jupyter_code>import degradation_utils as du
def get_mode_correlation_percent(df, mode_1, mode_2, weighted):
    """
    Return the percent of rows where two modes are seen together
    
    Args:
        df (DataFrame): Pandas DataFrame that has been cleaned using the clean_data function
        mode_1 (string): Degradation mode to find in the DataFrame in pairing with mode_2
        mode_2 (string): Degradation mode to find in the DataFrame in pairing with mode_1
        weighted (bool): If true, count all modules in a system as degrading
                         If false, count a system as one degrading module
    Returns:
        float: The percentage of modules with both specified degradation modes
    """
    # Calculate total number of modules
    total_modules = du.get_total_modules(df, weighted)
    if total_modules == 0:
        return 0
        
    if weighted:
        single_modules = len(df[(df['System/module'] == 'Module') & (df[mode_1] == 1) & (df[mode_2] == 1)])
        
        specified = df[(df['System/module'] != 'System') | (df['No.modules'].notnull())]
        systems = specified[(specified['System/module'] != 'Module') &
                            (specified[mode_1] == 1) & (specified[mode_2] == 1)]['No.modules'].sum()
        
        total = single_modules + systems
        return float(total) / total_modules
    else:
        return float(len((df[(df[mode_1] == 1) & (df[mode_2] == 1)]))) / total_modules
def get_heatmap_data(df, modes, weighted):
    """
    Returns a DataFrame used to construct a heatmap based on frequency of two degradation modes appearing together
    
    Args:
        df (DataFrame): A *cleaned* DataFrame containing the data entries to check modes from
        modes (List of String): A list of all modes to check for in the DataFrame
        weighted (bool): If true, count all modules in a system as degrading
                         If false, count a system as one degrading module
    Returns:
        heatmap (DataFrame): DataFrame containing all of degradation modes correlation frequency results
    """
    # Initialize DataFrame to hold heatmap data
    heatmap = pd.DataFrame(data=None, columns=modes, index=modes)

    # Calculate all single mode percentages
    mode_percentages = {}
    for mode in modes:
        mode_percentages[mode] = du.get_mode_percentage(df, mode, weighted)
    
    # Iterate through every pair of modes
    for mode_1 in modes:
        for mode_2 in modes:
            if mode_1 == mode_2:
                heatmap.set_value(mode_1, mode_2, np.nan)
            else:
                print(mode_1 + " & " + mode_2)
                
                heatmap_reflection = heatmap.at[mode_2, mode_1]
                
                # If already calculated the reflection, save and skip
                if (not pd.isnull(heatmap_reflection)):
                    heatmap.set_value(mode_1, mode_2, heatmap_reflection)
                    print('Skip, already calculated')
                    continue
                
                percentage_1 = mode_percentages[mode_1]
                percentage_2 = mode_percentages[mode_2]
                
                print('Percentage 1: ' + str(percentage_1))
                print('Percentage 2: ' + str(percentage_2))
                
                if (percentage_1 == 0 or percentage_2 == 0):
                    heatmap.set_value(mode_1, mode_2, 0)
                    continue

                percentage_both = get_mode_correlation_percent(df, mode_1, mode_2, weighted)
                print('Percentage Both: ' + str(percentage_both))

                result = float(percentage_both) / (percentage_1 * percentage_2)
                print('Result: ' + str(result))

                heatmap.set_value(mode_1, mode_2, result)
                
    return heatmap<jupyter_output><empty_output><jupyter_text>Step 4: Generate heatmaps of correlation frequency between degradation modes CalculationFind the correlation strength of all pairs of degradation modes by using the following formula:P(Degradation mode A & Degradation mode B) / P(Degradation mode A)P(Degradation mode B) Weighted: Multiply data entries for module systems by number of modulesNumber of degrading modules =  of degrading single modules + ( of degrading systems ·  of modules per degrading system)Total number of modules =  of single modules + ( of systems ·  of modules per system)P(Degradation mode X) = Number of degrading modules / Total number of modules  Generate heatmap for the entire dataset, regardless of time<jupyter_code>modes = ['Encapsulant discoloration', 'Major delamination', 'Minor delamination',
         'Backsheet other', 'Internal circuitry discoloration', 'Hot spots', 'Fractured cells',
         'Diode/J-box problem', 'Glass breakage', 'Permanent soiling', 'Frame deformation']

sys_heatmap_all = get_heatmap_data(solar, modes, True)
sys_heatmap_all
sys_heatmap_all.iplot(kind='heatmap',colorscale='spectral',
                                filename='sys-heatmap-all', margin=(200,150,120,30))<jupyter_output><empty_output><jupyter_text>Generate heatmap for the dataset of all modules installed before 2000<jupyter_code>modes = ['Encapsulant discoloration', 'Major delamination', 'Minor delamination',
         'Backsheet other', 'Internal circuitry discoloration', 'Hot spots', 'Fractured cells',
         'Diode/J-box problem', 'Glass breakage', 'Permanent soiling', 'Frame deformation']

specified = solar[solar['Begin.Year'] < 2000]

sys_heatmap_pre_2000 = get_heatmap_data(specified, modes, True)
sys_heatmap_pre_2000
sys_heatmap_pre_2000.iplot(kind='heatmap',colorscale='spectral',
                                filename='sys-heatmap-pre-2000', margin=(200,150,120,30))<jupyter_output><empty_output><jupyter_text>Generate heatmap for the dataset of all modules installed post 2000<jupyter_code>modes = ['Encapsulant discoloration', 'Major delamination', 'Minor delamination',
         'Backsheet other', 'Internal circuitry discoloration', 'Hot spots', 'Fractured cells',
         'Diode/J-box problem', 'Glass breakage', 'Permanent soiling', 'Frame deformation']

specified = solar[solar['Begin.Year'] >= 2000]

sys_heatmap_post_2000 = get_heatmap_data(specified, modes, True)
sys_heatmap_post_2000
sys_heatmap_post_2000.iplot(kind='heatmap',colorscale='spectral',
                                filename='sys-heatmap-post-2000', margin=(200,150,120,30))<jupyter_output><empty_output><jupyter_text>Unweighted: Count module systems as single moduleNumber of degrading modules =  of degrading single modules +  of degrading systemsTotal number of modules =  of single modules +  of systemsP(Degradation mode X) = Number of degrading modules / Total number of modules<jupyter_code>modes = ['Encapsulant discoloration', 'Major delamination', 'Minor delamination',
         'Backsheet other', 'Internal circuitry discoloration', 'Hot spots', 'Fractured cells',
         'Diode/J-box problem', 'Glass breakage', 'Permanent soiling', 'Frame deformation']

modes_heatmap_all = get_heatmap_data(solar, modes, False)
modes_heatmap_all
modes_heatmap_all.iplot(kind='heatmap',colorscale='spectral',
                         filename='modes-heatmap-all', margin=(200,150,120,30))<jupyter_output><empty_output><jupyter_text>Generate heatmap for the dataset of all modules installed before 2000<jupyter_code>modes = ['Encapsulant discoloration', 'Major delamination', 'Minor delamination',
         'Backsheet other', 'Internal circuitry discoloration', 'Hot spots', 'Fractured cells',
         'Diode/J-box problem', 'Glass breakage', 'Permanent soiling', 'Frame deformation']

specified = solar[solar['Begin.Year'] < 2000]

modes_heatmap_pre_2000 = get_heatmap_data(specified, modes, False)
modes_heatmap_pre_2000
modes_heatmap_pre_2000.iplot(kind='heatmap',colorscale='spectral',
                                filename='modes-heatmap-pre-2000', margin=(200,150,120,30))<jupyter_output><empty_output><jupyter_text>Generate heatmap for the dataset of all modules installed post 2000<jupyter_code>modes = ['Encapsulant discoloration', 'Major delamination', 'Minor delamination',
         'Backsheet other', 'Internal circuitry discoloration', 'Hot spots', 'Fractured cells',
         'Diode/J-box problem', 'Glass breakage', 'Permanent soiling', 'Frame deformation']

specified = solar[solar['Begin.Year'] >= 2000]

modes_heatmap_post_2000 = get_heatmap_data(specified, modes, False)
modes_heatmap_post_2000
modes_heatmap_post_2000.iplot(kind='heatmap',colorscale='spectral',
                                filename='modes-heatmap-post-2000', margin=(200,150,120,30))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09531201490220428, 'fraction_numerical': 0.02266376901583359, 'mean_word_length': 3.5975261655566126, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '15191', 'license': 'BSD-3-Clause-LBNL', 'n_tokens_mistral': 3066, 'n_tokens_neox': 2744, 'n_words': 783, 'path': 'degradation_modes/correlating_modes_analysis.ipynb', 'repo_name': 'hackingmaterials/duramat', 'size': '9663'}