<jupyter_start><jupyter_text>import data<jupyter_code>data = pd.read_csv('../../Data/WeatherOutagesAllJerry.csv')
# clean up data
data = data.dropna(how = 'all')
data = data.fillna(0)
data = data.round(4)
# # of outages apart from equipment outage
natural = data['natural'] = data.Trees + data.Animals + data.Lightning
pd.set_option('display.max_columns', 500)
data.head(5)
data['Total_outages'].max()
data['natural'].max()
data['Equipment'].max()
# look at the distribution of power outages

for array in data[['Total_outages','natural','Equipment']].hist(figsize=(10,3),layout=(1,3),bins=50):
    for subplot in array:
        subplot.set_xlim((20,50))
        subplot.set_ylim((0,10))
        
for array in data[['Total_outages','natural','Equipment']].hist(figsize=(10,3),layout=(1,3),bins=50):
    for subplot in array:
        subplot.set_xlim((0,20))
        subplot.set_ylim((0,3000))
        
#0-2, 3-8, >9
plt.boxplot(data.Equipment)
#data[['Max_temp_F','Avg_Temp_F','Min_temp_F']].plot()
#data[['Max_humidity_percent','Avg_humidity_percent','Min_humidity_percent']].plot()
#data[['Max_visibility_mi']].plot()
#data[['Avg_visibility_mi']].plot()
# convert to classification problem
def f(row):
    """function that categories days with more than 8 outages as extreme, 
                                3-8 outages as bad, and 0-2 as normal"""
    if row['Total_outages'] > 8:
        val = 'extreme'
    elif row['Total_outages'] > 2:
        val = 'bad'
    else:
        val = 'normal'
    return val

data['category'] = data.apply(f, axis=1)
data.head(2)
data.category.value_counts().plot(kind='bar')
feature = data[['Day_length_hr','Avg_Temp_F','Avg_humidity_percent',
             'Max_windspeed_mph','Precipitation_in']]
scaler = preprocessing.StandardScaler()
data_std = np.hstack((scaler.fit_transform(feature),np.atleast_2d(pd.factorize(data.category)[0]).T))
joblib.dump(scaler, 'scaler.pkl')
train,test=train_test_split(data_std,test_size=0.2,random_state=567)
np.shape(train[:,0:-1])
x_train = train[:,0:-1]
y_train = train[:,-1]

x_test = test[:,0:-1]
y_test = test[:,-1]
x_train.size
y_test.size<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code>clf = dt.fit(x_train,y_train)

y_train_pred = dt.predict(x_train)
y_test_pred = dt.predict(x_test)

print("Train error for standardized data",mean_squared_error(y_train,y_train_pred))
print("Test error for standardized data",mean_squared_error(y_test,y_test_pred))
print("training score for standardized data",dt.score(x_train,y_train))

tree_dot_std = export_graphviz(clf,out_file = 'tree_all_std.dot')
# probability of each event
print(dt.predict_proba(x_train)[0:20], y_test[0:20])
# features importance
list(zip(dt.feature_importances_))
graph = pydotplus.graphviz.graph_from_dot_file('tree_all_std.dot')
Image(graph.create_png())
#plt.savefig('DecisionTree.png')
# plot the results
plt.figure(figsize=(5,5))
#plt.scatter(y_test,y_test_pred_nor,alpha=0.2,color='red',label='normalized data')
plt.scatter(y_test,y_test_pred,alpha=0.2,color='green',label='standardardized data')
plt.xlabel('real test value')
plt.ylabel('predicted test value')
plt.plot(np.arange(-5,30),np.arange(-5,30))
plt.xlim([-1,4])
plt.ylim([-1,4])
plt.legend()
sns.set_style("white")
sns.set_color_codes()
plot_colors = sns.cubehelix_palette(n_colors=3)
cmap = sns.cubehelix_palette(dark=0.4, as_cmap=True)
plot_step = 0.02  # fine step width for decision surface contours
plot_step_coarser = 0.5  # step widths for coarse classifier guesses
RANDOM_SEED = 13  # fix the seed on each iteration
n_classes = 3

#for pairidx, pair in enumerate([[0, 3], [0, 4], [3, 4]]):
for pairidx, pair in enumerate([[0, 3]]):
# We only take the two corresponding features
    X = x_train[:,pair]
    y = y_train

    # Train
    clf = DecisionTreeClassifier(min_samples_leaf=5,max_depth=2,class_weight='balanced').fit(X, y)
    print("training score using %s features is %f" % (pair, clf.score(X,y)))
    # Plot the decision boundary
    #plt.subplot(1, 3, pairidx+1)
    x_min, x_max = X[:, 0].min() - 0.5, X[:, 0].max() + 0.5
    y_min, y_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5
    xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),
                         np.arange(y_min, y_max, plot_step))

    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    cs = plt.contourf(xx, yy, Z, cmap=cmap)

    plt.axis("tight")

    # Plot the training points
    for i, color,label_ in zip(range(n_classes), plot_colors, ["Normal",'Bad','Extreme']):
        idx = np.where(y == i)
        plt.scatter(X[idx[0].tolist(), 0], X[idx[0].tolist(), 1], c=color,label=label_,
                    cmap=cmap)

plt.axis("tight")
plt.xlabel("Standardized day length",fontsize=12)
plt.ylabel("Standardized max wind speed",fontsize=12)
plt.suptitle("Decision Surface of A 2-layer Decision Tree",fontsize=14, fontweight='bold')
plt.text(-0.7, 10, 'training score = 0.679', style='italic',
        bbox={'facecolor':'yellow', 'alpha':0.5, 'pad':10})

leg = plt.legend()
frame = leg.get_frame()
frame.set_facecolor('w')
frame.set_edgecolor('w')
for text in leg.get_texts():
    plt.setp(text, color = 'k')
plt.savefig('DecisionBoundary.png')
plt.show()<jupyter_output>training score using [0, 3] features is 0.678879<jupyter_text>Ensemble Tree Methods  without weights on class<jupyter_code>n_estimators = 30 # number of trees used, default=10 for rf, et, 50 for ab, 100 for gb
#n_jobs=-1, the number of jobs is set to the number of cores

dt = DecisionTreeClassifier(max_leaf_nodes=6,min_samples_leaf=5, presort=True)
rf = RandomForestClassifier(n_estimators=n_estimators,min_samples_leaf=5,n_jobs=-1)
et = ExtraTreesClassifier(n_estimators=n_estimators,min_samples_leaf=5,n_jobs=-1)
ab = AdaBoostClassifier(DecisionTreeClassifier(max_depth=3),learning_rate=0.1)
bg = BaggingClassifier(max_samples=3,n_jobs=-1)
gb = GradientBoostingClassifier(n_estimators=300, min_samples_leaf=5)

plot_idx = 1
for method, color_, label_ in [(dt,'m','DecisionTree'),(rf,'g','RandomForest'), (et,'r','ExtraTrees'),
                               (ab,'y','AdaBoost'),(gb,'k','GradientBoost')]:
    clf_std = method.fit(x_train,y_train)

    y_train_pred_std = method.predict(x_train_std)
    y_test_pred_std = method.predict(x_test_std)

    print("Train error for standardized data using %s is" % (label_),mean_squared_error(y_train,y_train_pred_std))
    print("Test error for standardized data using %s is" % (label_),mean_squared_error(y_test,y_test_pred_std))
    print("Feature importance using %s is" % (label_),'\n',list(zip(x_train_std, method.feature_importances_)),'\n')
    plt.subplot(2, 3, plot_idx)
    plt.scatter(y_test,y_test_pred_std,alpha=0.2,color=color_,label=label_)
    plt.plot(np.arange(-1,3),np.arange(-1,3))
    plt.xlabel('real test value')
    plt.ylabel('predicted test value')
    plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.05),
          ncol=3, fancybox=True, shadow=True)
    plt.xlim([-1,3])
    plt.ylim([-1,5])
    plt.tight_layout()
    plot_idx += 1
    
clf_std = bg.fit(x_train_std,y_train)
y_train_pred_std = bg.predict(x_train_std)
y_test_pred_std = bg.predict(x_test_std)
print("Train error for standardized data using %s is" % 'Bagging',mean_squared_error(y_train,y_train_pred_std))
print("Test error for standardized data using %s is" % 'Bagging',mean_squared_error(y_test,y_test_pred_std))

plt.subplot(2, 3, 6)
plt.scatter(y_test,y_test_pred_std,alpha=0.2,color='b',label='Bagging')    

plt.xlabel('real test value')
plt.ylabel('predicted test value')
plt.plot(np.arange(-5,30),np.arange(-5,30))
plt.xlim([-1,3])
plt.ylim([-1,5])
plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.05),
          ncol=3, fancybox=True, shadow=True)
plt.tight_layout()<jupyter_output>Train error for standardized data using DecisionTree is 0.124917236813
Test error for standardized data using DecisionTree is 0.128861429832
Feature importance using DecisionTree is 
 [(0, 0.35530288466855692), (1, 0.2064649666750612), (2, 0.0), (3, 0.35833916407601485), (4, 0.079892984580367005)] 

Train error for standardized data using RandomForest is 0.104391966453
Test error for standardized data using RandomForest is 0.130626654898
Feature importance using RandomForest is 
 [(0, 0.26804638892182614), (1, 0.21297146234466471), (2, 0.16023507580606275), (3, 0.23541387665341462), (4, 0.12333319627403179)] 

Train error for standardized data using ExtraTrees is 0.139483557714
Test error for standardized data using ExtraTrees is 0.137687555163
Feature importance using ExtraTrees is 
 [(0, 0.26970126411865764), (1, 0.22604873657003166), (2, 0.11445242172534974), (3, 0.27887174928223135), (4, 0.1109258283037296)] 

Train error for standardized data using AdaBoost is 0.112337232399
Test [...]<jupyter_text>with weights on class<jupyter_code>n_estimators = 30 # number of trees used, default=10 for rf, et, 50 for ab, 100 for gb
#n_jobs=-1, the number of jobs is set to the number of cores

dt = DecisionTreeClassifier(max_leaf_nodes=6,min_samples_leaf=5, 
                            class_weight='balanced', presort=True)
rf = RandomForestClassifier(n_estimators=n_estimators,min_samples_leaf=5,n_jobs=-1,
                            class_weight='balanced')
et = ExtraTreesClassifier(n_estimators=n_estimators,min_samples_leaf=5,n_jobs=-1,
                          class_weight='balanced')
ab = AdaBoostClassifier(DecisionTreeClassifier(max_depth=3),learning_rate=0.1)
bg = BaggingClassifier(max_samples=3,n_jobs=-1)
gb = GradientBoostingClassifier(n_estimators=300, min_samples_leaf=5)
plot_idx = 1
for method, color_, label_ in [(dt,'m','DecisionTree'),(rf,'g','RandomForest'), (et,'r','ExtraTrees'),
                               (ab,'y','AdaBoost'),(gb,'k','GradientBoost')]:
    clf = method.fit(x_train_std,y_train)

    y_train_pred_std = method.predict(x_train_std)
    y_test_pred_std = method.predict(x_test_std)

    print("Train error for standardized data using %s is" % (label_),mean_squared_error(y_train,y_train_pred_std))
    print("Test error for standardized data using %s is" % (label_),mean_squared_error(y_test,y_test_pred_std))
    print("Feature importance using %s is" % (label_),'\n',list(zip(x_train_std, method.feature_importances_)),'\n')
    plt.subplot(2, 3, plot_idx)
    plt.scatter(y_test,y_test_pred_std,alpha=0.2,color=color_,label=label_)
    plt.plot(np.arange(-1,3),np.arange(-1,3))
    plt.xlabel('real test value')
    plt.ylabel('predicted test value')
    plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.05),
          ncol=3, fancybox=True, shadow=True)
    plt.xlim([-1,3])
    plt.ylim([-1,5])
    plt.tight_layout()
    plot_idx += 1
    
clf_std = bg.fit(x_train_std,y_train)
y_train_pred_std = bg.predict(x_train_std)
y_test_pred_std = bg.predict(x_test_std)
print("Train error for standardized data using %s is" % 'Bagging',mean_squared_error(y_train,y_train_pred_std))
print("Test error for standardized data using %s is" % 'Bagging',mean_squared_error(y_test,y_test_pred_std))

plt.subplot(2, 3, 6)
plt.scatter(y_test,y_test_pred_std,alpha=0.2,color='b',label='Bagging')    

plt.xlabel('real test value')
plt.ylabel('predicted test value')
plt.plot(np.arange(-5,30),np.arange(-5,30))
plt.xlim([-1,3])
plt.ylim([-1,4])
plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.05),
          ncol=3, fancybox=True, shadow=True)
plt.tight_layout()
dt.fit(x_train_std,y_train)
rf.fit(x_train_std,y_train)
et.fit(x_train_std,y_train)
ab.fit(x_train_std,y_train)
bg.fit(x_train_std,y_train)
gb.fit(x_train_std,y_train)

#joblib.dump(dt, 'dt.pkl')
#joblib.dump(rf, 'rf.pkl')
#joblib.dump(et, 'et.pkl')
#joblib.dump(ab, 'ab.pkl')
#joblib.dump(bg, 'bg.pkl')
#joblib.dump(gb, 'gb.pkl')
weather = pd.DataFrame()
weatherData = weather.append({"Day_length_hr": 11, 
                "Avg_Temp_F": 38,
                "Avg_humidity_percent": 85,
                "Max_windspeed_mph": 16,
                "Avg_windspeed_mph": 7,
                "Max_windgust_mph": 20,
                "Precipitation_in": 0.33},
                ignore_index=True)
weatherData
weather = pd.DataFrame(preprocessing.scale(weatherData[['Day_length_hr','Avg_Temp_F','Avg_humidity_percent','Max_windspeed_mph','Precipitation_in']]))
weather = weather.fillna(0)
weather
print(dt.predict(weather))<jupyter_output>[0]<jupyter_text>count correctness for difference classes<jupyter_code>df = pd.DataFrame()
for method, label_ in [(dt,'DecisionTree'),(rf,'RandomForest'), (et,'ExtraTrees'),
                               (ab,'AdaBoost'),(gb,'GradientBoost'),(bg,'Bagging')]:
    clf = method.fit(x_train_std,y_train)
    y_pred = method.predict(x_test_std)
    y_test = pd.DataFrame(y_test)
    #initialize counters 
    overall_correctness = 0
    under_predict_normal = 0
    right_predict_normal = 0
    over_predict_normal = 0
    under_predict_bad = 0
    right_predict_bad = 0
    over_predict_bad = 0
    under_predict_extreme = 0
    right_predict_extreme = 0
    over_predict_extreme = 0
    
    for index, row in y_test.iterrows():
        if y_pred[index] == row.values: # predict correctly
            overall_correctness += 1
            if row.values == 0:
                right_predict_normal += 1
            elif row.values == 1:
                right_predict_bad += 1
            else:
                right_predict_extreme += 1
        elif row.values == 0:
            over_predict_normal += 1
        elif row.values == 1:
            if y_pred[index] == 0:
                under_predict_bad += 1
            else:
                over_predict_bad += 1
        else:
            under_predict_extreme += 1
    
    df = df.append({"Method": label_, "Overall_Correctness": overall_correctness/index,
                    "right_predict_normal": right_predict_normal/(right_predict_normal
                                            + under_predict_normal + over_predict_normal),
                    "right_predict_bad": right_predict_bad/(right_predict_bad
                                            + under_predict_bad + over_predict_bad),
                    "right_predict_extreme": right_predict_extreme/(right_predict_extreme
                                            + under_predict_extreme + over_predict_extreme),
                    "under_predict_normal": under_predict_normal/(right_predict_normal
                                            + under_predict_normal + over_predict_normal),
                    "under_predict_bad": under_predict_bad/(right_predict_bad
                                            + under_predict_bad + over_predict_bad),
                    "under_predict_extreme": under_predict_extreme/(right_predict_extreme
                                            + under_predict_extreme + over_predict_extreme),
                    "over_predict_normal": over_predict_normal/(right_predict_normal
                                            + under_predict_normal + over_predict_normal),
                    "over_predict_bad": over_predict_bad/(right_predict_bad
                                            + under_predict_bad + over_predict_bad),
                    "over_predict_extreme": over_predict_extreme/(right_predict_extreme
                                            + under_predict_extreme + over_predict_extreme)},
                  ignore_index=True)
df.to_csv('TreeMethodsResults_NoGust.csv')
df
                
        
df.iloc[:,0].tolist()
sns.set_style("darkgrid")
plt.plot(x,df.over_predict_normal,label='Normal Case, Over Predict',color='r')
plt.plot(x,df.over_predict_bad,label='Bad Case, Over Predict',color='y')

plt.plot(x,df.under_predict_bad,label='Bad Case, Under Predict',color='g')
plt.plot(x,df.under_predict_extreme,label='Extreme Case, Under Predict',color='b')

x = range(6)
my_xticks = df.iloc[:,0].tolist()
plt.xticks(x, my_xticks,fontweight='bold')
plt.yticks(fontweight='bold')

plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.03),ncol=2, fancybox=True, shadow=True)
plt.ylim(0,1.2)
plt.title("Prediction Breakdown",fontsize=14, fontweight='bold')
plt.tight_layout()
plt.savefig('PredictionBreakdown.png')
sns.set_style("darkgrid")
plt.plot(x,df.iloc[:,1],label='Overall',color='crimson')
plt.plot(x,df.right_predict_normal,label='Normal Case',color='k')
plt.plot(x,df.right_predict_bad,label='Bad Case',color='c')
plt.plot(x,df.right_predict_extreme,label='Extreme Case',color='gold')
x = range(6)
my_xticks = df.iloc[:,0].tolist()
plt.xticks(x, my_xticks,fontweight='bold')
plt.yticks(fontweight='bold')

plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.03),ncol=4, fancybox=True, shadow=True)
plt.ylim(0,1.2)
plt.title("Prediction Correctness",fontsize=14, fontweight='bold')
plt.tight_layout()
plt.savefig('PredictionCorrectness.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1425421607770693, 'fraction_numerical': 0.042428937488826646, 'mean_word_length': 4.141544117647059, 'pattern_counts': {'":': 18, '<': 18, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '415464', 'license': 'MIT', 'n_tokens_mistral': 6576, 'n_tokens_neox': 5993, 'n_words': 1025, 'path': 'PowerOutagePredictor/Tree/history/BuildingTheTree-AllData-classification-lessFeature-count.ipynb', 'repo_name': 'rkastilani/PowerOutage', 'size': '16781'}