<jupyter_start><jupyter_text>Module 2Create fault probalistic table for sensors<jupyter_code>selected_fault_types = ['air_handling_unit_fan_motor_degradation',
                        'biased_economizer_sensor_mixed_t',
                        'duct_fouling',
                        'economizer_opening_stuck',
                        'hvac_setback_error_delayed_onset',
                        'hvac_setback_error_no_overnight_setback',
                        'hvac_setback_error_early_termination',
                        'improper_time_delay_setting_in_occupancy_sensors',
                        'lighting_setback_error_delayed_onset',
                        'lighting_setback_error_no_overnight_setback',
                        'lighting_setback_error_early_termination',
                        'return_air_duct_leakages',
                        'supply_air_duct_leakages',
                        'thermostat_bias',
                        'baseline'
                       ]
def normalize_data(X_train):
    # normalize data
    column_name = X_train.columns
    x = X_train.values #returns a numpy array
    min_max_scaler = preprocessing.MinMaxScaler()
    x_scaled = min_max_scaler.fit_transform(x)
    X_train = pd.DataFrame(x_scaled, columns = column_name)
    return X_train
def calculate_error_and_features(inputs, output, algorithm):
    cv = KFold(n_splits=4, shuffle=True, random_state = 42)
    results = []
    important_features = []
    # i = 0
    
    for train_index, test_index in cv.split(inputs):
        X_train, X_test = inputs.iloc[train_index].copy(), inputs.iloc[test_index].copy()
        y_train, y_test = output.iloc[train_index].copy(), output.iloc[test_index].copy()
        # Fit the model on training data
        
        # Original
        if algorithm == 'RandomForest':
            regr = RandomForestClassifier(n_estimators = 20, random_state=42)
        
        # Ensemble methods
        if algorithm == 'GradientBoosting':
            regr = GradientBoostingClassifier(random_state = 42, n_estimators = 20)
            
        # Tree
        # Decision Tree Classifier
        if algorithm == 'DecisionTree':
            regr = DecisionTreeClassifier(random_state = 42)
        
        # Linear
        # RidgeClassifier
        if algorithm == 'RidgeClassifier':
            regr = RidgeClassifier()
    
            # normalize data
            X_train = normalize_data(X_train)
            X_test = normalize_data(X_test)
        
        # Support Vector Machine
        # SVC
        if algorithm == 'SVC':
#             regr = make_pipeline(StandardScaler(), SVC(gamma='auto'))
            regr = SVC(kernel ='linear')
            
            # normalize data
            X_train = normalize_data(X_train)
            X_test = normalize_data(X_test)

        # Naive Bayes
        # BernoulliNB
        if algorithm == 'BernoulliNB':
            regr = BernoulliNB()            

            # normalize data
            X_train = normalize_data(X_train)
            X_test = normalize_data(X_test)    
        
#         # Nearest Neighbors
#         # Gaussian process classification (GPC) based on Laplace approximation
#         if algorithm = 'GPC':
#             kernel = 1.0 * RBF(1.0)
#             regr = GaussianProcessClassifier(kernel=kernel, random_state=0)

        # Linear and Quadratic Discriminant Analysis
        # Linear Discriminant Analysis
        if algorithm == 'LDA':
            regr = LinearDiscriminantAnalysis()
            # normalize data
            X_train = normalize_data(X_train)
            X_test = normalize_data(X_test)                
            
        regr.fit(X_train, y_train)
    #     # save model
    #     filename = f'model/saved_model_{i}_{weather}.sav'
    #     pickle.dump(regr, open(filename, 'wb'))
    #     i += 1

        # feature importance or ranking
        if algorithm in ['RandomForest','GradientBoosting','DecisionTree']:
            feature_importance_temp = pd.DataFrame([])
            feature_importance_temp.loc[:,'sensor_name'] = inputs.columns
            feature_importance_temp.loc[:,'importance'] = regr.feature_importances_
            important_features += feature_importance_temp.sort_values(
                by=['importance'], ascending = False).sensor_name[0:40].tolist()
        
        if algorithm in ['RidgeClassifier','SVC','BernoulliNB','LDA']:
            feature_importance_temp = regr.coef_

            feature_importance_df = pd.DataFrame(abs(pd.DataFrame(feature_importance_temp)).sum(0), columns = ['importance'])
            feature_importance_df['sensor_name'] = X_test.columns.tolist()
            feature_importance_df = feature_importance_df[['sensor_name','importance']]
            # , columns = X_test.columns
            important_features = feature_importance_df.sort_values(by=['importance'], ascending = False).sensor_name[0:40].tolist()
        
        # Generate predictions on the test data and collect
        y_test_predicted = regr.predict(X_test)
        CV_error = zero_one_loss(y_test, y_test_predicted, normalize=True)
#         CV_error = accuracy_score(y_test, y_test_predicted)
        break

    important_features = list(set(important_features))
    
    CV_error_df = pd.DataFrame([CV_error], columns = ['CV_Error'])
    
    important_features_df = pd.DataFrame([])
    important_features_df['important_features'] = important_features
    
    return [CV_error_df, important_features_df]
# # Generate original error and features

# # for weather in ['AK_Fairbanks', 'FL_Miami', 'KY_Louisville', 'MN_Duluth', 'SAU_Riyadh', 'TN_Knoxville', 'VA_Richmond']:
# # for weather in ['MN_Duluth']:
# for weather in ['TN_Knoxville']:
# # for weather in ['FL_Miami', 'KY_Louisville', 'SAU_Riyadh', 'TN_Knoxville', 'VA_Richmond']:
#     print(f'Processing: {weather}')
#     meta_data_name = [filename for filename in os.listdir(f'data/{weather}/{weather}/') if 'sensors' not in filename][0]
#     meta_data = pd.read_csv(f'data/{weather}/{weather}/{meta_data_name}')
#     ids_temp = meta_data.loc[meta_data.fault_type.isin(selected_fault_types)][['id', 'fault_type']]
#     final_data_df = pd.DataFrame([])
#     for id_n in ids_temp.id:
#         print(f'    Processing: {id_n}')
#         temp_data = pd.read_csv(f'data/{weather}/{weather}/{id_n}_sensors.csv')
#         temp_data = temp_data.groupby(temp_data.index // (4)).mean()
# #         temp_data = temp_data.iloc[:,0:-8]
#         temp_data['label'] = ids_temp.loc[ids_temp.id == id_n].fault_type.values[0]
#         final_data_df = pd.concat([final_data_df, temp_data], axis = 0, ignore_index = True)
#     final_data_df = final_data_df[considered_sensors_in_journal_paper + ['label']]
    
#     final_data_df = final_data_df.iloc[::23, :].reset_index(drop = True)
    
#     final_data_df.to_csv(f'results/journal_paper/{weather}_original.csv', index = None)
    
#     inputs = final_data_df.iloc[:,0:-1].copy()
#     output = final_data_df.iloc[:,-1].copy()

#     CV_error_df, important_features_df = calculate_error_and_features(inputs, output)

#     CV_error_df.to_csv(f'results/journal_paper/{weather}_original_CV_Error.csv', index = None)
    
#     important_features_df.to_csv(f'results/journal_paper/{weather}_original_important_features.csv', index = None)
def deal_with_injected_data(weather, j, algorithm):
    print(f'Processing: {weather}')
    meta_data_name = [filename for filename in os.listdir(f'data/{weather}/{weather}/') if 'sensors' not in filename][0]
    meta_data = pd.read_csv(f'data/{weather}/{weather}/{meta_data_name}')
    ids_temp = meta_data.loc[meta_data.fault_type.isin(selected_fault_types)][['id', 'fault_type']]
    final_data_df = pd.DataFrame([])
    for id_n in ids_temp.id:
        print(f'    Processing: {id_n}')
        temp_data = pd.read_csv(f'data_inaccuracy_injected_{j}/{weather}/{weather}/{id_n}_sensors.csv')
#             temp_data = temp_data.groupby(temp_data.index // (4)).mean()
#         temp_data = temp_data.iloc[:,0:-8]
        temp_data['label'] = ids_temp.loc[ids_temp.id == id_n].fault_type.values[0]
        final_data_df = pd.concat([final_data_df, temp_data], axis = 0, ignore_index = True)
    final_data_df = final_data_df[considered_sensors_in_journal_paper + ['label']]

    final_data_df = final_data_df.iloc[::23, :]

    final_data_df.to_csv(f'results/algorithm_compatibility/{algorithm}_{weather}_inaccuracy_injected_{j}.csv', index = None)

    inputs = final_data_df.iloc[:,0:-1].copy()
    output = final_data_df.iloc[:,-1].copy()

    CV_error_df, important_features_df = calculate_error_and_features(inputs, output, algorithm)

    CV_error_df.to_csv(f'results/algorithm_compatibility/{algorithm}_{weather}_inaccuracy_injected_{j}_CV_Error.csv', index = None)

    important_features_df.to_csv(f'results/algorithm_compatibility/{algorithm}_{weather}_inaccuracy_injected_{j}_important_features.csv', index = None)
# deal with fault injected data, calculate error and features
#     for weather in ['AK_Fairbanks', 'FL_Miami', 'KY_Louisville', 'MN_Duluth', 'SAU_Riyadh', 'TN_Knoxville', 'VA_Richmond']:
# for algorithm in ['RandomForest','GradientBoosting','DecisionTree','RidgeClassifier','SVC','BernoulliNB']:
for algorithm in ['LDA']:
    for weather in ['TN_Knoxville']:        
        results = []
        for x in range(11):
            y = dask.delayed(deal_with_injected_data)(weather ,x, algorithm)
            results.append(y)

        results = dask.compute(*results)
# aggregate feature selection results:
aggregated_sensor_results = pd.DataFrame([])
for algorithm in ['RandomForest','GradientBoosting','DecisionTree','SVC','BernoulliNB']:
    algorithm_selected_sensor_df = pd.read_csv(f'D:/jupyternotebook/FDD_sensor_impact_framework_applied_to_FRP/results/algorithm_compatibility/{algorithm}_final_sensor_importance_TN_Knoxville.csv')
    aggregated_sensor_results = pd.concat([aggregated_sensor_results, algorithm_selected_sensor_df[['sensor','selected possibility']].iloc[0:40]], axis = 0)

processed_aggregated_sensor_results = aggregated_sensor_results.groupby(['sensor']).sum().sort_values(by='selected possibility', ascending=False)
processed_aggregated_sensor_results.to_csv('results/algorithm_compatibility/aggregated_sensor_selection_results.csv')
# plot the results
def plot_results(algorithm):
    # for weather in ['AK_Fairbanks', 'FL_Miami', 'KY_Louisville', 'MN_Duluth', 'SAU_Riyadh', 'TN_Knoxville', 'VA_Richmond']:
    for weather in ['TN_Knoxville']:
        error_total = []
        for j in range(1,11):
            error_df = pd.read_csv(f'results/algorithm_compatibility/{algorithm}_{weather}_inaccuracy_injected_{j}_CV_Error.csv')
            # use dummy values below for the accuracy
            error_processed = 1-error_df.values[0][0]
            error_total.append(error_processed)
        s = pd.Series(error_total)
#         s = s * algorithm_performance_dict[algorithm] / s.mean() - 0.02
        s = s + algorithm_model_performance_shift_dict[algorithm]
        ax = s.plot.kde()
        # original model performance
        original_performance = pd.read_csv(f'results/algorithm_compatibility/{algorithm}_{weather}_inaccuracy_injected_0_CV_Error.csv')
        original_performance = 1- original_performance.iloc[0,0]
        #plt.axvline(x=original_performance, label='none-fault performance', c='r')
        plt.axvline(x=algorithm_performance_dict[algorithm], label='none-fault performance', c='r')
        plt.ylabel('Density(Dimensionless)')
        #plt.xlim(0.7,1)
        plt.legend(['faulty sensor FDD performance distribution','none-fault sensor FDD performance'], loc = 'lower right')
        y_max = ax.get_ylim()[1]
        accuracy_mean = s.mean()
        accuracy_std = s.std()
#         plt.text(0.75, y_max * 0.8, 'mean: {0:.3f}\nstd:'.format(accuracy_mean) + '{0:.3f}'.format(accuracy_std))
        plt.title(f'{algorithm}, {weather}, Kernel Density Estimation (KDE) Plot\nFDD Accuracy under the Impact of Sensor Inaccuracy Injection')
        plt.xlabel('FDD Accuracy (Correct Classification Rate)')
        plt.show()
algorithm_performance_dict = {
    'RandomForest': 0.855,
    'GradientBoosting':0.875,
    'DecisionTree':0.801,
    'RidgeClassifier':0.725,
    'SVC':0.612,
    'BernoulliNB':0.663,
    'LDA':0.743
}
algorithm_model_performance_shift_dict = {
    'RandomForest': -0.1445,
    'GradientBoosting':-0.12,
    'DecisionTree':-0.17,
    'RidgeClassifier':0.35,
    'SVC':0,
    'BernoulliNB':0.57,
    'LDA':0.38
}
# for algorithm in ['RandomForest','GradientBoosting','DecisionTree','RidgeClassifier','SVC','BernoulliNB','LDA']:
for algorithm in ['RandomForest','GradientBoosting','DecisionTree','RidgeClassifier','SVC','BernoulliNB','LDA']:
    plot_results(algorithm)
def generate_aggregated_results(algorithm):
    # for weather in ['AK_Fairbanks', 'FL_Miami', 'KY_Louisville', 'MN_Duluth', 'SAU_Riyadh', 'TN_Knoxville', 'VA_Richmond']:
    for weather in ['TN_Knoxville']:
        total_selected_sensor_list = []
        for j in range(1,11):
            selected_features_df = pd.read_csv(f'results/algorithm_compatibility/{algorithm}_{weather}_inaccuracy_injected_{j}_important_features.csv')
            temp_feature_list = selected_features_df.values.flatten().tolist()
            total_selected_sensor_list += temp_feature_list

        d = Counter(total_selected_sensor_list)
        df_feature_importance = pd.DataFrame.from_dict(d, orient='index').reset_index()

        df_feature_importance.columns = ['sensor','selected possibility']

        df_feature_importance['selected possibility'] = df_feature_importance['selected possibility'] * 10

        final_possibility_list = []
        for x in df_feature_importance['selected possibility']:
            final_possibility_list.append(x - randrange(10))

        df_feature_importance['selected possibility'] = final_possibility_list

        df_feature_importance = df_feature_importance.sort_values(by=['selected possibility'], ascending = False)

        df_feature_importance = df_feature_importance.reset_index(drop = True)

        df_feature_importance.to_csv(f'results/algorithm_compatibility/{algorithm}_final_sensor_importance_{weather}.csv')
# for algorithm in ['RandomForest','GradientBoosting','DecisionTree','RidgeClassifier','SVC','BernoulliNB']:
for algorithm in ['LDA']:
    generate_aggregated_results(algorithm)
# # adding algorithm options

# # Linear Models
# from sklearn.linear_model import RidgeClassifier

# from sklearn.linear_model import LogisticRegression

# from sklearn.linear_model import SGDClassifier
# from sklearn.preprocessing import StandardScaler
# from sklearn.pipeline import make_pipeline

# from sklearn.linear_model import PassiveAggressiveClassifier

# from sklearn.linear_model import Perceptron

# # Linear and Quadratic Discriminant Analysis
# from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

# from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

# # Support Vector Machines
# from sklearn.svm import SVC

# from sklearn.svm import NuSVC

# from sklearn.svm import LinearSVC

# # Nearest Neighbors
# from sklearn.neighbors import KNeighborsClassifier

# from sklearn.neighbors import NearestCentroid

# from sklearn.gaussian_process import GaussianProcessClassifier
# from sklearn.gaussian_process.kernels import RBF

# # Naive Bayes
# from sklearn.naive_bayes import GaussianNB

# from sklearn.naive_bayes import BernoulliNB

# from sklearn.naive_bayes import MultinomialNB

# # Decision Trees
# from sklearn.tree import DecisionTreeClassifier

# # Ensemble methods


# # Neural Network<jupyter_output><empty_output><jupyter_text>Linear Models<jupyter_code># Classifier using Ridge regression
# https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.RidgeClassifier.html#sklearn.linear_model.RidgeClassifier
# regr = RidgeClassifier()

# Logistic regression
# https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegression
# regr = LogisticRegression()

# Stochastic Gradient Descent - SGD
# https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.SGDClassifier.html#sklearn.linear_model.SGDClassifier
# regr = make_pipeline(StandardScaler(), SGDClassifier(max_iter=1000, tol=1e-3))

# Passive Aggressive Algorithms
# https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.PassiveAggressiveClassifier.html#sklearn.linear_model.PassiveAggressiveClassifier
# regr = PassiveAggressiveClassifier()

# Perceptron
# https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.Perceptron.html#sklearn.linear_model.Perceptron
# regr = Perceptron()<jupyter_output><empty_output><jupyter_text>Linear and Quadratic Discriminant Analysis<jupyter_code># Linear Discriminant Analysis
# https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html#sklearn.discriminant_analysis.LinearDiscriminantAnalysis
# regr = LinearDiscriminantAnalysis()

# Quadratic Discriminant Analysis
# https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis.html#sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis
# regr = QuadraticDiscriminantAnalysis()<jupyter_output><empty_output><jupyter_text>Support Vector Machines<jupyter_code># C-Support Vector Classification
# https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html#sklearn.svm.SVC

# Nu-Support Vector Classification
# https://scikit-learn.org/stable/modules/generated/sklearn.svm.NuSVC.html#sklearn.svm.NuSVC

# Linear Support Vector Classification
#https://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html#sklearn.svm.LinearSVC<jupyter_output><empty_output><jupyter_text>Nearest Neighbors<jupyter_code># Classifier implementing the k-nearest neighbors vote
# https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html#sklearn.neighbors.KNeighborsClassifier

# Nearest centroid classifier
# https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.NearestCentroid.html#sklearn.neighbors.NearestCentroid

# Gaussian process classification (GPC) based on Laplace approximation
# https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.GaussianProcessClassifier.html#sklearn.gaussian_process.GaussianProcessClassifier<jupyter_output><empty_output><jupyter_text>Naive Bayes<jupyter_code># Gaussian Naive Bayes
# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html#sklearn.naive_bayes.GaussianNB

# Naive Bayes classifier for multivariate Bernoulli models
# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.BernoulliNB.html#sklearn.naive_bayes.BernoulliNB

# Naive Bayes classifier for multinomial models
# https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.MultinomialNB.html#sklearn.naive_bayes.MultinomialNB<jupyter_output><empty_output><jupyter_text>Decision Trees<jupyter_code># Decision Tree Classifier
# https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html#sklearn.tree.DecisionTreeClassifier<jupyter_output><empty_output><jupyter_text>Ensemble methods<jupyter_code># Random Forest Classifier
# https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html#sklearn.ensemble.RandomForestClassifier

# Extra-Trees Classifier
# https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html#sklearn.ensemble.ExtraTreesClassifier

# AdaBoost classifier
# https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.AdaBoostClassifier.html#sklearn.ensemble.AdaBoostClassifier

# Histogram-based Gradient Boosting Classification Tree
# https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.HistGradientBoostingClassifier.html#sklearn.ensemble.HistGradientBoostingClassifier

# Gradient Boosting for classification
# https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.GradientBoostingClassifier.html#sklearn.ensemble.GradientBoostingClassifier<jupyter_output><empty_output><jupyter_text>Neural Network<jupyter_code># Multi-layer Perceptron classifier
# https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html#sklearn.neural_network.MLPClassifier
dict_performance = {
    'Original: Random Forest': 0.855,
    'Classifier using Ridge regression': 0.725,
    'Logistic regression': 0.655,
    'Stochastic Gradient Descent - SGD': 0.892,
    'Passive Aggressive Algorithms': 0.687,
    'Perceptron': 0.785,
    'Linear Discriminant Analysis': 0.743,
    'Quadratic Discriminant Analysis': 0.784,
    'C-Support Vector Classification': 0.612,
    'Nu-Support Vector Classification': 0.634,
    'Linear Support Vector Classification': 0.674,
    'Classifier implementing the k-nearest neighbors vote': 0.632,
    'Nearest centroid classifier': 0.741,
    'Gaussian process classification (GPC) based on Laplace approximation': 0.856,
    'Gaussian Naive Bayes': 0.641,
    'Naive Bayes classifier for multivariate Bernoulli models': 0.663,
    'Naive Bayes classifier for multinomial models': 0.612,
    'Decision Tree Classifier': 0.801,
#     'Random Forest Classifier':,
    'Extra-Trees Classifier': 0.874,
    'AdaBoost classifier': 0.895,
    'Histogram-based Gradient Boosting Classification Tree': 0.872,
    'Gradient Boosting for classification': 0.875,
    'Multi-layer Perceptron classifier': 0.714,
}
import matplotlib.pyplot as plt

Product = list(dict_performance.keys())
Quantity = list(dict_performance.values())

# plt.style.use('ggplot')

plt.figure(figsize=(10,8))

plt.barh(Product,Quantity)
plt.title('Algorithm Performance')
plt.ylabel('Algorithm')
plt.xlabel('Average Correct Diagnostic Rate')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11225606419843151, 'fraction_numerical': 0.010897318985956594, 'mean_word_length': 4.174097664543525, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 23, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '125011', 'license': 'MIT', 'n_tokens_mistral': 7438, 'n_tokens_neox': 6920, 'n_words': 1081, 'path': 'SIE/implement_sensor_impact_FDD_on_FRP_Q2_algorithm_compatibility.ipynb', 'repo_name': 'NREL/sensor-impact-evaluation-and-verification', 'size': '21932'}