<jupyter_start><jupyter_text>MODELLING  Loading the data<jupyter_code>import pandas as pd 
import numpy as np 
import matplotlib.pyplot as plt 
import seaborn as sns
import random
# sns.set()
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold, cross_validate, GridSearchCV, ShuffleSplit, learning_curve
from sklearn.tree import DecisionTreeClassifier, plot_tree
from sklearn.metrics import classification_report, plot_roc_curve, plot_precision_recall_curve, \
precision_recall_curve, plot_confusion_matrix, confusion_matrix, ConfusionMatrixDisplay, precision_score, f1_score, accuracy_score
from imblearn.pipeline import Pipeline
from sklearn.dummy import DummyClassifier
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.svm import SVC
from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, RandomForestClassifier
from imblearn.over_sampling import SMOTE, RandomOverSampler
from imblearn.under_sampling import RandomUnderSampler, TomekLinks, EditedNearestNeighbours
from imblearn.combine import SMOTETomek, SMOTEENN

from sklearn.preprocessing import MinMaxScaler, RobustScaler, OneHotEncoder, OrdinalEncoder
from xgboost.sklearn import XGBClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.feature_selection import RFE
import category_encoders as ce
from sklearn.compose import ColumnTransformer
import pickle
%%time
filename = "df_comp_pluto_merged.csv"
df = pd.read_csv(filename,index_col=False)
df = df[df.Borough_x == 'BRONX']
df['Heat'] = np.where(df['Complaint Type'] == 'HEAT/HOT WATER', 1, 0)
# df = 
def check_values(df):
    col_desc = []
    data = {
        'features': [col for col in df.columns],
        'data_type': [df[col].dtype for col in df.columns],
        'nan_total': [df[col].isna().sum() for col in df.columns],
        'nan_pct': [round(df[col].isna().sum()/len(df)*100,2) for col in df.columns],
        'unique': [df[col].nunique() for col in df.columns],
        'values_ex': [df[col].drop_duplicates().sample(df[col].nunique()).values if df[col].nunique() <= 5 else df[col].drop_duplicates().sample(2).values for col in df.columns]
    }
    return pd.DataFrame(data)
check_values(df)
df.columns<jupyter_output><empty_output><jupyter_text>Drop unneeded columns for modelling<jupyter_code>df_train = df.drop(columns=['Unnamed: 0', 'Unique Key', 'Created Date', 'Complaint Type',
       'Incident Zip', 'Incident Address', 'Street Name', 'City', 'Status',
       'Borough_x', 'Latitude', 'Longitude','ZipCode', 'Borough_y', 'Address', 'YearBuilt', 'YearAlter1', 'BldgFront', 'BldgDepth',
        'HEAT/HOT WATER_count', 'UNSANITARY CONDITION_count',
       'PAINT/PLASTER_count', 'PLUMBING_count', 'DOOR/WINDOW_count',
       'WATER LEAK_count', 'ELECTRIC_count', 'GENERAL_count',
       'FLOORING/STAIRS_count', 'APPLIANCE_count'])
df_train.shape
df_train.dtypes<jupyter_output><empty_output><jupyter_text>Checking Categorical Features  LandUseLandUse column is already encoded according to the following:| CODES | DECODES ||--- | --- || 1 | One & Two Family Buildings || 2 | Multi-Family Walk-Up Buildings || 3 | Multi-Family Elevator Buildings || 4 | Mixed Residential & Commercial Buildings || 5 | Commercial & Office Buildings || 6 | Industrial & Manufacturing || 7 | Transportation & Utility || 8 | Public Facilities & Institutions || 9 | Open Space & Outdoor Recreation || 10 | Parking Facilities || 11 | Vacant Land |The bar chart below shows no difference in LandUse distribution for heat and non-heat complaints, thus this column will not be used<jupyter_code>fig, ax = plt.subplots(1,2,figsize=(12,6))
df[df.Heat ==1].groupby('LandUse')['Heat'].count().plot.barh(ax=ax[0], label='Yes')
ax[0].set_title('Heat Complaints')
ax[0].legend()
df[df.Heat ==0].groupby('LandUse')['Heat'].count().plot.barh(ax=ax[1], label='No')
ax[1].set_title('Non-Heat Complaints')
ax[1].legend()
plt.show()<jupyter_output><empty_output><jupyter_text>LotTypeLotType column is already encoded according to the following:    | Codes | Decodes || ---|---|| 0 | Mixed or Unknown || 1 | Block assemblage || 2 | Waterfront || 3 | Corner || 4 | Through || 5 | Inside || 6 | Interior lot || 7 | Island lot || 8 | Alley lot || 9 | Submerged land lot |The bar chart below shows no difference in LotType distribution for heat and non-heat complaints, so this column will be removed<jupyter_code>fig, ax = plt.subplots(1,2,figsize=(12,6))
df[df.Heat ==1].groupby('LotType')['Heat'].count().plot.barh(ax=ax[0], label='Yes')
ax[0].set_title('Heat Complaints')
ax[0].legend()
df[df.Heat ==0].groupby('LotType')['Heat'].count().plot.barh(ax=ax[1], label='No')
ax[1].set_title('Non-Heat Complaints')
ax[1].legend()
plt.show()<jupyter_output><empty_output><jupyter_text>BsmtCodeBsmtCode column is already encoded according to the following:| Codes | Decodes || --- | --- || 0 | No basement || 1 | Full basement that is above grade || 2 | Full basement that is below grade || 3 | Partial basement that is above grade | | 4 | Partial basement that is below grade || 5 | Unknown |The bar chart below shows no difference in BsmtCode distribution for heat and non-heat complaints, so this column will be removed<jupyter_code>fig, ax = plt.subplots(1,2,figsize=(12,6))
df[df.Heat ==1].groupby('BsmtCode')['Heat'].count().plot.barh(ax=ax[0], label='Yes')
ax[0].set_title('Heat Complaints')
ax[0].legend()
df[df.Heat ==0].groupby('BsmtCode')['Heat'].count().plot.barh(ax=ax[1], label='No')
ax[1].set_title('Non-Heat Complaints')
ax[1].legend()
plt.show()
df_train.columns<jupyter_output><empty_output><jupyter_text>Remove outliers<jupyter_code>clip_cols = ['LotArea', 'BldgArea', 'ComArea', 'ResArea', 'OfficeArea', 'RetailArea', 'GarageArea', 'StrgeArea', 'FactryArea', 'OtherArea',
       'NumBldgs', 'NumFloors', 'UnitsTotal', 'AssessTot', 'AlterAge']
df_clipped = df_train[clip_cols].quantile([0.05, 0.95])
df_clipped
np.sum([df_train.OfficeArea > 0])<jupyter_output><empty_output><jupyter_text>drop columnsdrop columns that have the same value for 5% and 95% percentile as they do not provide information<jupyter_code>dropped_cols = ['OfficeArea', 'GarageArea', 'StrgeArea', 'FactryArea', 'OtherArea']
clip_cols = ['LotArea', 'BldgArea', 'ComArea', 'ResArea', 'RetailArea', 'NumBldgs', 'NumFloors', 'UnitsTotal', 'AssessTot', 'AlterAge']
df_clipped = df_train[clip_cols].quantile([0.05, 0.95])
df_clipped<jupyter_output><empty_output><jupyter_text>Filter values based on the computed percentiles<jupyter_code>df_train_clipped = df_train[clip_cols].apply(lambda x: x[(x>df_clipped.loc[0.05,x.name]) & 
                                    (x < df_clipped.loc[0.95,x.name])], axis=0)
df_train_clipped
df_train_fin = pd.concat([df_train.drop(columns=clip_cols), df_train_clipped], axis=1)
df_train_fin
df_train_fin.isna().sum()/len(df_train_fin)*100<jupyter_output><empty_output><jupyter_text>Drop columns with high unknown values<jupyter_code>df_train_fin.drop(columns=['NumBldgs', 'ComArea', 'RetailArea', 'AlterAge', 'year', 'FactryArea'], inplace=True)
df_train_fin.dropna(inplace=True)
df_train_fin.shape
plt.figure(figsize=(20,10))
sns.heatmap(df_train_fin.corr().abs(),annot=False)
plt.title('Correlation of Datapoints')
plt.show()<jupyter_output><empty_output><jupyter_text>Bin hour columnmake a new feature, hourbin, to categorize the hour into 'early morning', 'morning', 'afternoon', and 'evening'.<jupyter_code>%%time
fig, ax = plt.subplots(1,2, figsize=(12,6))
df_train_fin[df_train_fin.Heat == 1].groupby('hour')['hour'].count().plot.bar(label='Yes', ax=ax[0])
ax[0].legend()
df_train_fin[df_train_fin.Heat == 0].groupby('hour')['hour'].count().plot.bar(label='No', ax=ax[1])
ax[1].legend()
plt.show()
def hour_binning(value):
    if value < 6:
        result = 'early morning'
    elif value < 11:
        result = 'morning'
    elif value < 16:
        result = 'afternoon'
    else: result = 'evening'
    return result

df_train_fin['hourbin'] = df_train_fin['hour'].apply(hour_binning)
df_train_fin[['hour','hourbin']]
df_train_fin['bldgperlot'] = df_train_fin['BldgArea'] / df_train_fin['LotArea']
df_train_fin['bldgperlot'].head()<jupyter_output><empty_output><jupyter_text>modelling  Preparing train and test set<jupyter_code>df_train_fin.columns
X = df_train_fin.drop(columns=['Heat', 'hour', 'LotArea', 'BsmtCode', 'LotType', 'LandUse'])
y = df_train_fin['Heat']

X_train_val, X_test, y_train_val, y_test = train_test_split(X,y, stratify=y, test_size=0.33, random_state=26)
transformer = ColumnTransformer([
            ('onehot', OneHotEncoder(drop='first'), ['hourbin'])
#             ('poly', PolynomialFeatures(degree=2), ['OfficeArea','GarageArea','StrgeArea','OtherArea','BldgArea','NumFloors','UnitsTotal'])
        ], remainder='passthrough', n_jobs=4, verbose=1)
X_train_val_prep = transformer.fit_transform(X_train_val)
X_test_prep = transformer.transform(X_test)
X.columns<jupyter_output><empty_output><jupyter_text>Baseline model Majority classifieralways predict the majority class, in this case 0 or "No" (not heat complaint), for the target value.<jupyter_code>df_train_fin.Heat.value_counts()
print(classification_report(y_test, np.zeros(y_test.shape)))<jupyter_output>precision    recall  f1-score   support

           0       0.62      1.00      0.76      9285
           1       0.00      0.00      0.00      5745

    accuracy                           0.62     15030
   macro avg       0.31      0.50      0.38     15030
weighted avg       0.38      0.62      0.47     15030<jupyter_text>Single-feature predictive modellingin this classification problem, the class is predicted by only one variable. we will make a decision tree with only one decision level with the most-informative feature (decision stump).<jupyter_code>dtc_baseline = DecisionTreeClassifier(max_depth=1)
dtc_baseline.fit(X_train_val_prep, y_train_val)
print(classification_report(y_test, dtc_baseline.predict(X_test_prep)))
plot_confusion_matrix(dtc_baseline, X_test_prep, y_test)
plt.show()
feature_names = list(transformer.transformers_[0][1].get_feature_names())
# feature_names.extend(list(transformer.transformers_[1][1].get_feature_names()))
old_col = list(X_train_val.drop(columns=['hourbin']))#,'OfficeArea','GarageArea','StrgeArea','OtherArea','BldgArea','NumFloors','UnitsTotal']).columns)
feature_names.extend(old_col)

feature_names
df_feature = pd.DataFrame({'name': feature_names, 'tree':dtc_baseline.feature_importances_})
df_feature.set_index('name').sort_values('tree')['tree'].plot.barh()
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>according to this model, 'month' column is the most important feature to differentiate between heat and non-heat complaints. based on the decision tree plot below, a decision stump with month <= 10.5 separates the sample into two leaves with high impurity as shown by the high gini ratio (0.439 and 0.492)<jupyter_code>plot_tree(dtc_baseline, feature_names=feature_names, class_names=['No', 'Yes'], filled=True)
plt.show()
plot_precision_recall_curve(dtc_baseline, X_test_prep, y_test)
plt.show()<jupyter_output><empty_output><jupyter_text>Benchmark modelling  Make pipelines with different stepsthe models used are:1. Logistic Regression1. Decision Tree Classifier1. Stochastic Gradient Descent1. Naive Bayesthe strategy used for these pipelines are:1. Oversampling with SMOTE1. Scaling with MinMaxScaler1. Feature selection with RFE1. Oversampling with RandomOverSampler1. Combined oversampling (SMOTE) and undersampling (TomekLinks) with SMOTETomek1. Combined oversampling (SMOTE) and undersampling (ENN (Edited Nearest Neighbours))with SMOTEENN<jupyter_code>def pipe_model(models):
    pipelines = []
    for model in models:
        pl = Pipeline([
        # ('transfr', transformer),
#         ('scaler', MinMaxScaler()),
#         ('sampling', SMOTE()),
        ('model', model[1])
])
        pipelines.append(pl)
    return pipelines
def pipe_smote_model(models):
    pipelines = []
    for model in models:
        pl = Pipeline([
        # ('transfr', transformer),
#         ('scaler', MinMaxScaler()),
        ('sampling', SMOTE()),
        ('model', model[1])
])
        pipelines.append(pl)
    return pipelines
def pipe_scale_smote_model(models):
    pipelines = []
    for model in models:
        pl = Pipeline([
        # ('transfr', transformer),
        ('scaler', MinMaxScaler()),
        ('sampling', SMOTE()),
        ('model', model[1])
])
        pipelines.append(pl)
    return pipelines
def pipe_scale_rfe_smote_model(models):
    pipelines = []
    for model in models:
        pl = Pipeline([
        # ('transfr', transformer),
        ('scaler', MinMaxScaler()),
        ('rfe', RFE(model[1])),
        ('sampling', SMOTE()),
        ('model', model[1])
])
        pipelines.append(pl)
    return pipelines
def pipe_scale_overs_model(models):
    pipelines = []
    for model in models:
        pl = Pipeline([
        # ('transfr', transformer),
        ('scaler', MinMaxScaler()),
#         ('rfe', RFE(model[1])),
        ('sampling', RandomOverSampler(sampling_strategy='minority')),
        ('model', model[1])
])
        pipelines.append(pl)
    return pipelines
def pipe_scale_overs_unders_model(models):
    pipelines = []
    for model in models:
        pl = Pipeline([
        # ('transfr', transformer),
        ('scaler', MinMaxScaler()),
#         ('rfe', RFE(model[1])),
#         ('sampling', RandomOverSampler(sampling_strategy=0.1)),
#         ('sampling2', RandomUnderSampler(sampling_strategy=0.5)),
        ('sampling', SMOTETomek(tomek=TomekLinks(sampling_strategy='majority'))),
        ('model', model[1])
])
        pipelines.append(pl)
    return pipelines
def pipe_scale_overs_unders_enn_model(models):
    pipelines = []
    for model in models:
        pl = Pipeline([
        # ('transfr', transformer),
        ('scaler', MinMaxScaler()),
#         ('rfe', RFE(model[1])),
#         ('sampling', RandomOverSampler(sampling_strategy=0.1)),
#         ('sampling2', RandomUnderSampler(sampling_strategy=0.5)),
        ('sampling', SMOTEENN(enn=EditedNearestNeighbours(sampling_strategy='majority'))),
        ('model', model[1])
])
        pipelines.append(pl)
    return pipelines<jupyter_output><empty_output><jupyter_text>Make a function for cross validationuse cross_validate to return several metrics at once<jupyter_code>def model_evaluation(pipes, metric):
    skfold = StratifiedKFold(n_splits=5, random_state=26, shuffle=True)
    model_cvs = []
    for pipe in pipes:
        model_cv = cross_validate(pipe, X_train_val_prep, y_train_val, cv=skfold, scoring=metric, return_train_score=True, n_jobs=4, verbose=1)
        model_cvs.append(model_cv)
    return model_cvs<jupyter_output><empty_output><jupyter_text>Make a function to summarize the cross validation results<jupyter_code>def results_dataframe(models, model_cvs, metric):
    scores = np.zeros((len(models),len(metric)*2))
    j = 0
    for model, model_cv in zip(models, model_cvs):
        
        model_score = []
        for i in model_cv:
            
            if i == 'fit_time' or i == 'score_time':
                continue
            model_score.append(model_cv[i].mean())
        scores[j] = model_score
        j+=1
    return pd.DataFrame(scores, index=[i[0] for i in models], columns = [i for i in list(model_cv_1[0].keys()) if i not in ('fit_time', 'score_time')])
    # return scores<jupyter_output><empty_output><jupyter_text>Cross validatetry several strategies and cross validate the model. use the f1 score for test set to compare the models on our cross validation set (X_train_val, y_train_val).1. Step: Model onlyonly DecisionTreeClassifier gives good performance while the others are severely underfitted. LogisticRegression and NaiveBayes perform only slightly better than random classifier, as shown by the AUC score of 0.500 and 0.525.despite having the best performance the DecisionTreeClassifier is overfitted since it scores very high (all metrics above 0.95) but only on the training set.<jupyter_code>models_1 = [('logreg', LogisticRegression()),('tree', DecisionTreeClassifier(random_state=26)), ('sgdc', SGDClassifier(random_state=26)), ('nb', GaussianNB())]
metric = ('f1', 'accuracy', 'roc_auc', 'recall', 'precision')
pl_1 = pipe_model(models_1)
model_cv_1 = model_evaluation(pl_1, metric)
df_model = results_dataframe(models_1, model_cv_1, metric).T
df_model<jupyter_output><empty_output><jupyter_text>2. Steps: Oversampling with SMOTE, modeladding SMOTE for oversampling step improves LogisticRegression and SGBC greatly<jupyter_code>models_2 = [('logreg', LogisticRegression(solver='saga')),('tree', DecisionTreeClassifier(random_state=26)), ('sgdc', SGDClassifier(random_state=26)), ('nb', GaussianNB())]
metric = ('f1', 'accuracy', 'roc_auc', 'recall', 'precision')
pl_2 = pipe_smote_model(models_2)
model_cv_2 = model_evaluation(pl_2, metric)
df_smote_model =  results_dataframe(models_2, model_cv_2, metric).T
df_smote_model<jupyter_output><empty_output><jupyter_text>3. Steps: Scaling, oversampling with SMOTE, modelScaling greatly improves LogisticRegression, StochasticGradientDescent and NaiveBayes, although Decision TreeClassifier still has the best performance of all.<jupyter_code>models_3 = [('logreg', LogisticRegression(solver='saga')), ('tree', DecisionTreeClassifier(random_state=26)), ('sgdc', SGDClassifier(random_state=26)), ('nb', GaussianNB())]
metric = ('f1', 'accuracy', 'roc_auc', 'recall', 'precision')
pl_3 = pipe_scale_smote_model(models_3)
model_cv_3 = model_evaluation(pl_3, metric)
df_scale_smote_model = results_dataframe(models_3, model_cv_3, metric).T
df_scale_smote_model<jupyter_output><empty_output><jupyter_text>4. Steps: Scaling, feature selection with RFE, oversampling with SMOTE, model    Feature selection with RFE was done only on LogisticRegression and DecisionTreeClassifier. no improvement compared with only scale+SMOTE+model.<jupyter_code>models_4 = [('logreg', LogisticRegression(solver='saga', max_iter=500)), ('tree', DecisionTreeClassifier(random_state=26))]
metric = ('f1', 'accuracy', 'roc_auc', 'recall', 'precision')
pl_4 = pipe_scale_rfe_smote_model(models_4)
model_cv_4 = model_evaluation(pl_4, metric)
df_rfe_smote_model = results_dataframe(models_4, model_cv_4, metric).T
df_rfe_smote_model<jupyter_output><empty_output><jupyter_text>5. Steps: Scaling, oversampling with RandomOverSampler, model    Oversampling with RandomOverSampler doesn't have much difference with SMOTE for any of the models. DecisionTreeClassifier still has the highest performance.<jupyter_code>models_5 = [('logreg', LogisticRegression(solver='saga')), ('tree', DecisionTreeClassifier(random_state=26)), ('sgdc', SGDClassifier(random_state=26)), ('nb', GaussianNB())]
metric = ('f1', 'accuracy', 'roc_auc', 'recall', 'precision')
pl_5 = pipe_scale_overs_model(models_5)
model_cv_5 = model_evaluation(pl_5, metric)
df_overs_model = results_dataframe(models_5, model_cv_5, metric).T
df_overs_model<jupyter_output><empty_output><jupyter_text>6. Steps: Scaling, combined oversampling and undersampling with SMOTETomek, modelin this step, we use the combination of oversampling and undersampling with SMOTETomek. not much difference with only oversampling with RandomOverSampler or SMOTE.<jupyter_code>models_6 = [('logreg', LogisticRegression(solver='saga')), ('tree', DecisionTreeClassifier(random_state=26)), ('sgdc', SGDClassifier(random_state=26)), ('nb', GaussianNB())]
metric = ('f1', 'accuracy', 'roc_auc', 'recall', 'precision')
pl_6 = pipe_scale_overs_unders_model(models_6)
model_cv_6 = model_evaluation(pl_6, metric)
df_overs_unders_model = results_dataframe(models_6, model_cv_6, metric).T
df_overs_unders_model<jupyter_output><empty_output><jupyter_text>7. Steps: Combined oversampling and undersampling with SMOTEENN, modelSMOTEENN combined oversampling with SMOTE and undersampling with ENN (EditedNearestNeighbours). DecisionTreeClassifier performance increased with this method compared to SMOTETomek. NaiveBayes improved greatly while LogisticRegression only slightly.DecisionTreeClassfier score reaches 0.638, which is the best of all steps and models we have tried. we will use this strategy for further model improvement<jupyter_code>models_7 = [('logreg', LogisticRegression(solver='saga')), ('tree', DecisionTreeClassifier(random_state=26)), ('sgdc', SGDClassifier(random_state=26)), ('nb', GaussianNB())]
metric = ('f1', 'accuracy', 'roc_auc', 'recall', 'precision')
pl_7 = pipe_scale_overs_unders_enn_model(models_7)
model_cv_7 = model_evaluation(pl_7, metric)
df_overs_unders_enn_model = results_dataframe(models_7, model_cv_7, metric).T
df_overs_unders_enn_model<jupyter_output><empty_output><jupyter_text>Summary of cross validation for DecisionTreeClassifierbest strategy: combined oversampling and undersampling with SMOTEENN + modeltest f1 score = 0.638<jupyter_code>def summary_df(name):
    return pd.concat([df_model[[name]].rename(columns={name:'model'}),
           df_smote_model[[name]].rename(columns={name:'SMOTE+model'}),
           df_scale_smote_model[[name]].rename(columns={name:'Scaling+SMOTE+model'}),
           df_rfe_smote_model[[name]].rename(columns={name:'RFE+SMOTE+model'}),
           df_overs_model[[name]].rename(columns={name:'Scaling+RandOverSamp+model'}),
           df_overs_unders_model[[name]].rename(columns={name:'Scaling+SMOTETomek+model'}), 
          df_overs_unders_enn_model[[name]].rename(columns={name:'Scaling+SMOTEENN+model'})], axis=1)
summary_df_dtc = summary_df('tree')
summary_df_dtc
summary_df_logreg = summary_df('logreg')
summary_df_logreg
scores = summary_df_dtc.loc['test_f1'].T.to_frame()
scores<jupyter_output><empty_output><jupyter_text>Model Improvementstrategies:1. Boosting with AdaBoostClassifier1. Bagging with RandomForestClassifier1. Hyperparameter tuning for RandomForestClassifierall the strategies are fitted with X_train_val, y_train_val and then evaluated on the test set (X_test, y_test)<jupyter_code>scaler = MinMaxScaler()
X_train_val_scaled = scaler.fit_transform(X_train_val_prep)
X_test_scaled = scaler.transform(X_test_prep)<jupyter_output><empty_output><jupyter_text>Evaluate the best DecisionTreeClassifier  on test setthe best model from previous section, which is DecisionTreeClassifier with scaling and SMOTEENN, is evaluated on test set. result: F1 score = 0.64<jupyter_code>pl_7[1]
dtc_smoteENN = pl_7[1]
dtc_smoteENN.fit(X_train_val_prep, y_train_val)
print('DTC with SMOTEENN')
print(classification_report(y_test, dtc_smoteENN.predict(X_test_prep)))
plot_confusion_matrix(dtc_smoteENN, X_test_prep, y_test)
plt.show()
def plot_learning_curve_scoring(estimator, title, X, y, axes=None, ylim=None, cv=None, scoring=None,
                        n_jobs=None, train_sizes=np.linspace(.1, 1.0, 5)):
    """
    Generate 3 plots: the test and training learning curve, the training
    samples vs fit times curve, the fit times vs score curve.

    Parameters
    ----------
    estimator : estimator instance
        An estimator instance implementing `fit` and `predict` methods which
        will be cloned for each validation.

    title : str
        Title for the chart.

    X : array-like of shape (n_samples, n_features)
        Training vector, where ``n_samples`` is the number of samples and
        ``n_features`` is the number of features.

    y : array-like of shape (n_samples) or (n_samples, n_features)
        Target relative to ``X`` for classification or regression;
        None for unsupervised learning.

    axes : array-like of shape (3,), default=None
        Axes to use for plotting the curves.

    ylim : tuple of shape (2,), default=None
        Defines minimum and maximum y-values plotted, e.g. (ymin, ymax).

    cv : int, cross-validation generator or an iterable, default=None
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

          - None, to use the default 5-fold cross-validation,
          - integer, to specify the number of folds.
          - :term:`CV splitter`,
          - An iterable yielding (train, test) splits as arrays of indices.

        For integer/None inputs, if ``y`` is binary or multiclass,
        :class:`StratifiedKFold` used. If the estimator is not a classifier
        or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validators that can be used here.

    n_jobs : int or None, default=None
        Number of jobs to run in parallel.
        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
        ``-1`` means using all processors. See :term:`Glossary <n_jobs>`
        for more details.

    train_sizes : array-like of shape (n_ticks,)
        Relative or absolute numbers of training examples that will be used to
        generate the learning curve. If the ``dtype`` is float, it is regarded
        as a fraction of the maximum size of the training set (that is
        determined by the selected validation method), i.e. it has to be within
        (0, 1]. Otherwise it is interpreted as absolute sizes of the training
        sets. Note that for classification the number of samples usually have
        to be big enough to contain at least one sample from each class.
        (default: np.linspace(0.1, 1.0, 5))
    """
#     if axes is None:
#         _, axes = plt.subplots(1, 3, figsize=(20, 5))
    _, axes = plt.subplots()

    axes.set_title(title)
    if ylim is not None:
        axes.set_ylim(*ylim)
    axes.set_xlabel("Training examples")
    axes.set_ylabel("Score")

    train_sizes, train_scores, test_scores, fit_times, _ = \
        learning_curve(estimator, X, y, cv=cv, n_jobs=n_jobs,
                       train_sizes=train_sizes, scoring=scoring,
                       return_times=True)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)

    # Plot learning curve
    axes.grid()
    axes.fill_between(train_sizes, train_scores_mean - train_scores_std,
                         train_scores_mean + train_scores_std, alpha=0.1,
                         color="r")
    axes.fill_between(train_sizes, test_scores_mean - test_scores_std,
                         test_scores_mean + test_scores_std, alpha=0.1,
                         color="g")
    axes.plot(train_sizes, train_scores_mean, 'o-', color="r",
                 label="Training score")
    axes.plot(train_sizes, test_scores_mean, 'o-', color="g",
                 label="Cross-validation score")
    axes.legend(loc="best")

    return plt<jupyter_output><empty_output><jupyter_text>both the cross-validation score and the training score are increasing over increasing training examples, which suggests that it might be beneficial to add more training example. on the other hand, judging by the small f1 score increase over wide range (0 to ~25000 rows) of data, the score might increase but only very slightly.<jupyter_code>%%time
title = r"Learning Curves (Decision Tree Classifier)"
cv = ShuffleSplit(n_splits=10, test_size=0.25, random_state=0)
plot_learning_curve_scoring(dtc_smoteENN, title, X_train_val_prep, y_train_val, #axes=axes[:, 2], ylim=(0.7, 1.01),
                    cv=cv, scoring='f1', n_jobs=4)
plt.show()<jupyter_output><empty_output><jupyter_text>1. Boosting with AdaboostClassifierno difference in performance between 50 estimators and 500 estimators, both give f1 score 0.65 on test set<jupyter_code>%%time
# tree = DecisionTreeClassifier()
adaboost = AdaBoostClassifier(DecisionTreeClassifier(random_state=26), n_estimators=50, learning_rate=0.1, random_state=26)

X_train_val_samp, y_train_val_samp = SMOTEENN(enn=EditedNearestNeighbours(sampling_strategy='majority')).fit_resample(X_train_val_prep, y_train_val)
adaboost.fit(X_train_val_samp, y_train_val_samp)
print('adaboost')
print(classification_report(y_test, adaboost.predict(X_test_prep)))
plot_confusion_matrix(adaboost, X_test_prep, y_test)
plt.show()<jupyter_output><empty_output><jupyter_text>2. Bagging with RandomForestClassifierwhen we use scaling, the model performs worse than without scaling, so we stick without scaling to build the ensemble. after bagging, f1 score increases to 0.68, which is the highest of all strategies we have tried. RandomForestClassifier will be used for hyperparameter tuning.<jupyter_code>rfc_ = Pipeline([
        ('sampling', SMOTEENN(enn=EditedNearestNeighbours(sampling_strategy='majority'))),
        ('tree', RandomForestClassifier(max_depth=20, random_state=26, oob_score=True, n_estimators=50, n_jobs=4, verbose=1))
])

rfc_.fit(X_train_val_prep, y_train_val)
print('rfc')
print(classification_report(y_test, rfc_.predict(X_test_prep)))
plot_confusion_matrix(rfc_, X_test_prep, y_test)
plt.show()<jupyter_output>[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.
[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.2s
[Parallel(n_jobs=4)]: Done  50 out of  50 | elapsed:    0.2s finished<jupyter_text>We will use our model prediction to ultimately count the total heat complaints in a certain area and then choose only the area with the highest number of complaints to focus on. Thus, we want to be as confident as possible when predicting whether a building will have a heat complaint or not --> increase threshold, for example, to 0.7 to increase the precision<jupyter_code>y_prob_rfc = rfc_.predict_proba(X_test_prep)
y_pred_rfc = np.where(y_prob_rfc[:,1]>0.7, 1, 0)
confusion_mat = confusion_matrix(y_test,y_pred_rfc)
disp = ConfusionMatrixDisplay(confusion_matrix=confusion_mat, display_labels=[0,1])
disp = disp.plot(include_values=True, cmap='viridis', ax=None, xticks_rotation='vertical')
plt.show()
print(f'precision: {precision_score(y_test,y_pred_rfc)}')
print(f'f1 score: {f1_score(y_test,y_pred_rfc)}')
print(f'accuracy: {accuracy_score(y_test,y_pred_rfc)}')
%%time
title = r"Learning Curves (Random Forest Classifier)"
cv = ShuffleSplit(n_splits=10, test_size=0.25, random_state=0)
plot_learning_curve_scoring(rfc_, title, X_train_val_prep, y_train_val, #axes=axes[:, 2], ylim=(0.7, 1.01),
                    cv=cv, scoring='f1', n_jobs=4)
# plt.savefig('learningcurve.png', bbox_inches = 'tight')
plt.show()
# rfc_[1].feature_importances_
plt.figure(figsize=(10,8))
feature_names = list(transformer.transformers_[0][1].get_feature_names())
feature_names.extend(list(X.drop(columns=['hourbin']).columns))
feat_imp = pd.DataFrame({'features':feature_names, 'importance':rfc_[1].feature_importances_}).set_index('features').sort_values('importance')
feat_imp.plot.barh()
plt.xticks(size=16)
plt.yticks(size=16)
plt.ylabel('features', size=16)
# plt.savefig('featureimp.png', bbox_inches = 'tight')
plt.show()
rfc_final = Pipeline([
        ('transfr', ColumnTransformer([
            ('onehot', OneHotEncoder(drop='first'), ['hourbin'])
        ], remainder='passthrough', n_jobs=4, verbose=1)),
        ('sampling', SMOTEENN(enn=EditedNearestNeighbours(sampling_strategy='majority'))),
        ('tree', RandomForestClassifier(max_depth=20, random_state=26, oob_score=True, n_estimators=50, n_jobs=4, verbose=1))
])
rfc_final.fit(X,y)
filename = 'rfc_final.sav'
pickle.dump(rfc_final, open(filename, 'wb'))
print('Finish')<jupyter_output>[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.
[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    6.3s
[Parallel(n_jobs=4)]: Done  50 out of  50 | elapsed:    7.3s finished<jupyter_text>3. Hyperparameter tuning for RandomForestClassifierthe hyperparameters used are number of estimators, maximum tree depth, maximum tree features, and decision criterion.after an exhaustive process we get the supposedly best estimator and we evaluate the model on the test set. there is no difference in performance when compared with the previous RandomForestClassifier, both have f1 score of 0.68 on test set.<jupyter_code>tree_params = {
    'tree__n_estimators': [50, 150],
    'tree__max_depth': [10,20,30],
    'tree__max_features': ['auto', 'sqrt', 'log2'],
    'tree__criterion': ['gini', 'entropy']
    }

pl_rfc = Pipeline([
        ('sampling', SMOTEENN(enn=EditedNearestNeighbours(sampling_strategy='majority'))),
        ('tree', RandomForestClassifier(max_depth=20, random_state=26, oob_score=True, n_estimators=50, n_jobs=-1, verbose=1))
])

skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=26)
tree_grid = GridSearchCV(pl_rfc, param_grid=tree_params, scoring='f1', cv=skf, verbose=1, n_jobs=-1)
tree_grid.fit(X_train_val_prep, y_train_val)



tree_tuned = tree_grid.best_estimator_
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=26)
tree_tuned_score = cross_val_score(tree_tuned, X_train_val_prep, y_train_val, scoring='f1', cv=skf, n_jobs=-1, verbose=1)
tree_tuned_score
tree_tuned = tree_grid.best_estimator_
tree_tuned.fit(X_train_val_prep, y_train_val)
print(classification_report(y_test, tree_tuned.predict(X_test_prep)))<jupyter_output>[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 4 concurrent workers.
[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:    3.8s
[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:   12.9s finished
[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.
[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.1s<jupyter_text>Predict suppose we want to predict whether some building on a particular location in Bronx will send a heat complaint in the evening on October. the database for the buildings is loaded only 200000 rows.<jupyter_code># %%time
# filename = 'comp_pluto_merged.csv'
# random.seed(26)
# n = sum(1 for line in open(filename)) - 1 #number of records in file (excludes header)
# s = 20000 #desired sample size
# skip = sorted(random.sample(range(1,n+1),n-s)) #the 0-indexed header will not be included in the skip list
# database = pd.read_csv(filename, skiprows=skip, index_col=False)
# # database = pd.read_csv(filename, index_col=False)
# database = database[database.Borough == 'BX']
# database.head()
database = df.sample(20000, random_state=26)
database['ZipCode'].nunique()
df.columns
database.columns<jupyter_output><empty_output><jupyter_text>set the location we want to check. in this case, we set the zipcode<jupyter_code>zipcode = database.sample(1, random_state=26)['ZipCode'].values[0]
zipcode<jupyter_output><empty_output><jupyter_text>in this NYC tax lot database we have loaded (only 200,000 data), there are 150 buildings in the area of zipcode 10465<jupyter_code>zipcodedf = database[database.ZipCode == zipcode].drop_duplicates('Incident Address')
zipcodedf
X.drop(columns=['hourbin', 'month']).columns
zipcodedf.loc[:,'bldgperlot'] = zipcodedf['BldgArea']/zipcodedf['LotArea']
zipcodedf_X = zipcodedf[list(X.drop(columns=['hourbin', 'month']).columns)]
zipcodedf_X.loc[:,'hourbin'] = 'evening'
zipcodedf_X.loc[:,'month'] = 1
zipcodedf_X.shape
used_cols = X.columns
used_cols
zipcodedf_X.columns
filename = 'rfc_final.sav'
loaded_model = pickle.load((open(filename, 'rb')))
loaded_model
zipcodedf_X.shape
y_proba = loaded_model.predict_proba(zipcodedf_X[used_cols])
y_pred = np.where(y_proba[:,1]>0.7, 1, 0)
# y_pred = loaded_model.predict(zipcodedf_X[used_cols])
total_heat = y_pred.sum()
total_heat<jupyter_output>[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.
[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.1s
[Parallel(n_jobs=4)]: Done  50 out of  50 | elapsed:    0.1s finished
{'fraction_non_alphanumeric': 0.12278978388998035, 'fraction_numerical': 0.018692113387594723, 'mean_word_length': 5.340035587188612, 'pattern_counts': {'":': 0, '<': 144, '<?xml version=': 0, '>': 143, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '35', 'dir': 'jupyter-structured-clean-dedup', 'id': '608324', 'license': 'MIT', 'n_tokens_mistral': 12572, 'n_tokens_neox': 11737, 'n_words': 2796, 'path': 'Notebook/5 Modelling.ipynb', 'repo_name': 'azukacchi/nyc311_housing_buildings_analysis', 'size': '35630'}