<jupyter_start><jupyter_text>Automated ML - Tuning<jupyter_code>COLAB = True

DATASET_NAME = '4D.zip'

FEATURE_DATASET_PREFIX = 'feature_matrix_d2_v3'
if COLAB:
  !pip install -U imblearn
  !rm -rf dl-projects
  !git clone https://github.com/mengwangk/dl-projects
if COLAB:
  !cp dl-projects/utils* .
  !cp dl-projects/preprocess* .
  !cp dl-projects/plot* .
%load_ext autoreload
# %reload_ext autoreload
%autoreload 2

%matplotlib inline
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import math 
import matplotlib
import sys

from scipy import stats
from collections import Counter
from pathlib import Path

plt.style.use('fivethirtyeight')

sns.set(style="ticks")

import featuretools as ft

import warnings
warnings.filterwarnings('ignore')

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler
from sklearn.impute import SimpleImputer
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_auc_score, precision_recall_curve, make_scorer, recall_score, roc_curve, mean_squared_error, accuracy_score, average_precision_score, classification_report
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV, RandomizedSearchCV, StratifiedShuffleSplit
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from imblearn.ensemble import BalancedRandomForestClassifier

from sklearn.decomposition import PCA
import pylab as pl
from collections import Counter

# from skopt import BayesSearchCV
# from skopt.space import Real, Categorical, Integer

# from sklearn.ensemble import RandomForestClassifier

# from scikitplot.plotters import plot_precision_recall_curve

from dateutil.relativedelta import relativedelta

from IPython.display import display

from utils import *
from preprocess import *

import xgboost as xgb

np.set_printoptions(threshold=sys.maxsize)

# The Answer to the Ultimate Question of Life, the Universe, and Everything.
np.random.seed(42)

from utils import feature_selection, plot_feature_importances
from plot import plot_correlation_matrix, plot_labeled_scatter


from imblearn.under_sampling import (RandomUnderSampler, 
                                     ClusterCentroids,
                                     TomekLinks,
                                     NeighbourhoodCleaningRule,
                                     AllKNN,
                                     NearMiss,
                                     OneSidedSelection)
from imblearn.combine import SMOTETomek
from imblearn.pipeline import make_pipeline as make_pipeline_imb
from imblearn.metrics import classification_report_imbalanced
from imblearn.over_sampling import SMOTE, SMOTENC, ADASYN
%aimport<jupyter_output><empty_output><jupyter_text>Preparation<jupyter_code>if COLAB:
  from google.colab import drive
  drive.mount('/content/gdrive')
  GDRIVE_DATASET_FOLDER = Path('gdrive/My Drive/datasets/')
if COLAB:
  DATASET_PATH = GDRIVE_DATASET_FOLDER
  ORIGIN_DATASET_PATH = Path('dl-projects/datasets')
else:
  DATASET_PATH = Path("datasets")
  ORIGIN_DATASET_PATH = Path('datasets')

DATASET = DATASET_PATH/f"{FEATURE_DATASET_PREFIX}.ft"
ORIGIN_DATASET = ORIGIN_DATASET_PATH/DATASET_NAME

if COLAB:
  !ls -l gdrive/"My Drive"/datasets/ --block-size=M
  !ls -l dl-projects/datasets --block-size=M
data = pd.read_feather(DATASET)
origin_data = format_tabular(ORIGIN_DATASET)<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis  View data<jupyter_code># Feature matrix
feature_matrix = data.drop(columns=['NumberId', 'month', 'year'])
# Sort data
feature_matrix.sort_values(by=['time', 'MAX(Results.LuckyNo)'], inplace=True)
feature_matrix.info()
print('Distrbution')
print(feature_matrix['Label'].value_counts())
print()
print('Positive: ' + str(feature_matrix['Label'].value_counts()[0]) + ' which is ', round(feature_matrix['Label'].value_counts()[0]/len(feature_matrix) * 100,2), '% of the dataset')
print('Negative: ' + str(feature_matrix['Label'].value_counts()[1]) + ' which is ', round(feature_matrix['Label'].value_counts()[1]/len(feature_matrix) * 100,2), '% of the dataset')
plt.figure(figsize=(8, 8))
sns.countplot('Label', data=feature_matrix)
plt.title("Strike Count", fontsize=18)
plt.xlabel("Strike?", fontsize=15)
plt.ylabel("Count", fontsize=15)
plt.show()
feature_matrix.isna().sum().sort_values(ascending=False)
feature_matrix[feature_matrix.isnull().any(axis=1)].head()<jupyter_output><empty_output><jupyter_text>Data Cleansing<jupyter_code>## Fill all NaN with 0
feature_matrix = feature_matrix.fillna(0)
feature_matrix.isna().sum().sort_values(ascending=False)
feature_matrix[feature_matrix.isnull().any(axis=1)].head()
# Feature Distribution

# plt.figure(figsize=(16,4))
# feature_matrix.iloc[:,:-1].boxplot()
# plt.title('(Raw) Distribution of Features', fontsize=17)
# plt.show()

# plt.figure(figsize=(16,4))
# np.log(feature_matrix.drop(columns=['time']).iloc[:,:-1]).boxplot() # Remove time column
# plt.title('(Log) Distribution of Features', fontsize=17)
# plt.show()<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code># Feature scaling first??
print(feature_matrix.shape)
feature_matrix.columns
feature_matrix_selection = feature_selection(feature_matrix.drop(columns = ['time', 'TotalStrike', 'Label']))
feature_matrix_selection.shape, feature_matrix_selection.columns
feature_matrix_selection['time'] = feature_matrix['time']
feature_matrix_selection['TotalStrike'] = feature_matrix['TotalStrike']
feature_matrix_selection['Label'] = feature_matrix['Label']<jupyter_output><empty_output><jupyter_text>Feature Correlation<jupyter_code># Check without feature selection
# corrs = feature_matrix.corr().sort_values('Label')
# corrs['Label'].tail(100)
# Check with feature selection
corrs = feature_matrix_selection.corr().sort_values('Label')
corrs['Label'].tail(20)<jupyter_output><empty_output><jupyter_text>Feature Distributions<jupyter_code>fig = plt.figure(figsize=(24,10))
fig.tight_layout()
fig.subplots_adjust(wspace=1)
plt.subplot(311)
ax = feature_matrix_selection.iloc[:,:20].boxplot()
ax.set_xticklabels(ax.get_xticklabels(),rotation=30)
plt.title('(Raw) Distribution of Features', fontsize=17)

plt.subplot(312)
ax = feature_matrix_selection.iloc[:,20:40].boxplot()
ax.set_xticklabels(ax.get_xticklabels(),rotation=30)

plt.subplot(313)
ax = feature_matrix_selection.iloc[:,40:-1].boxplot()
ax.set_xticklabels(ax.get_xticklabels(),rotation=30)

fig.subplots_adjust(hspace=10)
plt.show()
plt.figure(figsize=(16,4))
np.log(feature_matrix_selection.drop(columns=['time']).iloc[:,:-1]).boxplot() # Remove time column
plt.title('(Log) Distribution of Features', fontsize=17)
plt.show()
plt.figure(figsize=(11,6))
sns.distplot(feature_matrix_selection['LAST(Results.CUM_MEAN(LuckyNo))'],kde=False)
plt.title('Distribution of Time', fontsize=17)
plt.show()<jupyter_output><empty_output><jupyter_text>Modeling<jupyter_code>## Baseline - https://www.kaggle.com/miguelniblock/optimizing-imbalanced-classification-100-recall
def predict_baseline(dt, feature_matrix): 
    
    feature_matrix['date'] = feature_matrix['time']

    # Subset labels
    test_labels = feature_matrix.loc[feature_matrix['date'] == dt, 'Label']
    train_labels = feature_matrix.loc[feature_matrix['date'] < dt, 'Label']

    # Features
    X_train = feature_matrix[feature_matrix['date'] < dt].drop(columns = ['NumberId', 'time',
                                                                                     'date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
    X_test = feature_matrix[feature_matrix['date'] == dt].drop(columns = ['NumberId', 'time',
                                                                                     'date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
    feature_names = list(X_train.columns)
    
    if type(train_labels) == np.ndarray:
      hit_ratio = float( len(np.where(train_labels == 0)[0]) / len(np.where(train_labels == 1)[0]) ) 
    else:
      hit_ratio = float(train_labels.value_counts()[0]/train_labels.value_counts()[1]) 
    
    print(f"Hit ratio before balancing - {hit_ratio}\n")

    # Labels
    y_train = np.array(train_labels).reshape((-1, ))
    y_test = np.array(test_labels).reshape((-1, ))
    
    print('Training on {} observations.'.format(len(X_train)))
    print('Testing on {} observations.\n'.format(len(X_test)))
  
    pipeline = make_pipeline_imb(
                                SimpleImputer(strategy = 'constant', fill_value=0),
                                StandardScaler()
                                )
    
   
    X_train = pipeline.fit_transform(X_train)
    X_test = pipeline.transform(X_test)

    # Create the classifier
    clf = xgb.XGBClassifier(
                             n_jobs=4, 
                             random_state = 42,
                            #  n_estimators=300, 
                            #  max_depth=5,
                            n_estimators=100, 
                            max_depth=3,
                            # n_estimators=1000, 
                            #  max_depth=5,
                            #  gamma=0,
                            #  subsample=0.8,
                            #  colsample_bytree=0.8,
                            #  learning_rate=0.1,
                             objective='binary:logistic',
                             min_child_weight=1,
                             scale_pos_weight=hit_ratio 
                             )
    
    # Make a scoring callable from recall_score
    recall = make_scorer(recall_score)

    train_cv = cross_val_score(X=X_train,y=y_train, scoring=recall, estimator=clf,cv=3)
    print("TRAIN GROUP")
    print("\nCross-validation recall scores:",train_cv)
    print("Mean recall score:",train_cv.mean())

    # Now predict on the test group
    print("\nTEST GROUP")
    y_pred = clf.fit(X_train, y_train).predict(X_test)
    print("\nRecall:",recall_score(y_test,y_pred))

    # Classification report
    print('\nClassification report:\n')
    print(classification_report(y_test, y_pred))

    # Confusion matrix
    conf_matrix = confusion_matrix(y_test,y_pred)
    sns.heatmap(conf_matrix, annot=True,fmt='d', cmap=plt.cm.copper)
    plt.show()

#predict_baseline(pd.datetime(2019,6,1), feature_matrix_selection)<jupyter_output><empty_output><jupyter_text>Balancing Data<jupyter_code># Make a scoring function that improves specificity while identifying all frauds
def recall_optim(y_true, y_pred):
    
    conf_matrix = confusion_matrix(y_true, y_pred)
    
    # Recall will be worth a greater value than specificity
    rec = recall_score(y_true, y_pred) * 0.8 
    spe = conf_matrix[0,0]/conf_matrix[0,:].sum() * 0.2 
    
    # Imperfect recalls will lose a penalty
    # This means the best results will have perfect recalls and compete for specificity
    if rec < 0.8:
        rec -= 0.2
    return rec + spe 
    
# Create a scoring callable based on the scoring function
optimize = make_scorer(recall_optim)
def predict_imbalanced(dt, feature_matrix): 
    
    feature_matrix['date'] = feature_matrix['time']

    # Subset labels
    test_labels = feature_matrix.loc[feature_matrix['date'] == dt, 'Label']
    train_labels = feature_matrix.loc[feature_matrix['date'] < dt, 'Label']

    # Features
    X_train = feature_matrix[feature_matrix['date'] < dt].drop(columns = ['NumberId', 'time',
                                                                                     'date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
    X_test = feature_matrix[feature_matrix['date'] == dt].drop(columns = ['NumberId', 'time',
                                                                                     'date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
    feature_names = list(X_train.columns)
    
    if type(train_labels) == np.ndarray:
      hit_ratio = float( len(np.where(train_labels == 0)[0]) / len(np.where(train_labels == 1)[0]) ) 
    else:
      hit_ratio = float(train_labels.value_counts()[0]/train_labels.value_counts()[1]) 
    
    print(f"Hit ratio before balancing - {hit_ratio}\n")

    # Labels
    y_train = np.array(train_labels).reshape((-1, ))
    y_test = np.array(test_labels).reshape((-1, ))
    
    print('Training on {} observations.'.format(len(X_train)))
    print('Testing on {} observations.\n'.format(len(X_test)))
  
    # Join the train data
    train = X_train.join(train_labels)

    print('Data shape before balancing:',train.shape)
    print('\nCounts of strikes VS non-strikes in previous data:')
    print(train.Label.value_counts())
    print('-'*40)

    # Balancing the test data
    pipeline = make_pipeline_imb(
                                SimpleImputer(strategy = 'constant', fill_value=0),
                                StandardScaler(),
                                SMOTE(sampling_strategy='minority',random_state=42, n_jobs=4)
                                #OneSidedSelection(n_neighbors=1, n_seeds_S=200, random_state=42, n_jobs=4)
                                )
    X_bal, y_bal = pipeline.fit_resample(X_train, y_train)
    X_bal = pd.DataFrame(X_bal,columns=X_train.columns)
    y_bal = pd.DataFrame(y_bal,columns=['Label'])
    balanced = X_bal.join(y_bal)

    print('-'*40)
    print('Data shape after balancing:',balanced.shape)
    print('\nCounts of strikes VS non-strikes in new data:')
    print(balanced.Label.value_counts())

    print('Distribution of the Classes in the subsample dataset')
    print(balanced.Label.value_counts()/len(train))

    sns.countplot('Label', data=balanced)
    plt.title('Label Distribution', fontsize=14)
    plt.show()

    # Compare correlation of raw train data VS balanced train data
    f, (ax1, ax2) = plt.subplots(2, 1, figsize=(24,20))

    # Imbalanced DataFrame
    corr = train.corr()
    # sns.heatmap(corr, annot_kws={'size':20}, ax=ax1)
    # ax1.set_title("Imbalanced Correlation Matrix \n (Biased)", fontsize=14)

    # Balanced DataFrame
    bal_corr = balanced.corr()
    # sns.heatmap(bal_corr, annot_kws={'size':20}, ax=ax2)
    # ax2.set_title('Balanced Correlation Matrix', fontsize=14)
    # plt.show()

    # Each feature's correlation with Class
    print(bal_corr.Label)

    no_outliers=pd.DataFrame(balanced.copy())

    # Removing Outliers from high-correlation features

    cols = bal_corr.Label.index[:-1]

    # For each feature correlated with Class...
    for col in cols:
        # If absolute correlation value is more than X percent...
        correlation = bal_corr.loc['Label',col]

        if np.absolute(correlation) > 0.1:
          # Separate the classes of the high-correlation column
          nonstrikes = no_outliers.loc[no_outliers.Label==0,col]
          strikes = no_outliers.loc[no_outliers.Label==1,col]

          # Identify the 25th and 75th quartiles
          all_values = no_outliers.loc[:,col]
          q25, q75 = np.percentile(all_values, 25), np.percentile(all_values, 75)
          # Get the inter quartile range
          iqr = q75 - q25
          # Smaller cutoffs will remove more outliers
          cutoff = iqr * 7
          # Set the bounds of the desired portion to keep
          lower, upper = q25 - cutoff, q75 + cutoff
          
          # If positively correlated...
          # Drop nonstrikes above upper bound, and strikes below lower bound
          if correlation > 0: 
              no_outliers.drop(index=nonstrikes[nonstrikes>upper].index,inplace=True)
              no_outliers.drop(index=strikes[strikes<lower].index,inplace=True)
          
          # If negatively correlated...
          # Drop non strikes below lower bound, and strikes above upper bound
          elif correlation < 0: 
              no_outliers.drop(index=nonstrikes[nonstrikes<lower].index,inplace=True)
              no_outliers.drop(index=strikes[strikes>upper].index,inplace=True)
        
    print('\nData shape before removing outliers:', balanced.shape)
    print('\nCounts of strikes VS non-strikes in previous data:')
    print(balanced.Label.value_counts())
    print('-'*40)
    print('-'*40)
    print('\nData shape after removing outliers:', no_outliers.shape)
    print('\nCounts of strikes VS non-strikes in new data:')
    print(no_outliers.Label.value_counts())

    no_outliers.iloc[:,:-1].boxplot(rot=90,figsize=(16,4))
    plt.title('Distributions with Less Outliers', fontsize=17)
    plt.show()


    # Feature selection
    feat_sel =pd.DataFrame(no_outliers.copy())

    # Make a dataframe with the label-correlations before removing outliers
    corr_change = pd.DataFrame()
    corr_change['correlation']= bal_corr.Label
    corr_change['origin']= 'w/outliers'

    # Make a dataframe with label-correlations after removing outliers 
    corr_other = pd.DataFrame()
    corr_other['correlation']= feat_sel.corr().Label
    corr_other['origin']= 'no_outliers'

    # Join them
    corr_change = corr_change.append(corr_other)

    plt.figure(figsize=(14,6))
    plt.xticks(rotation=90)

    # Plot them
    sns.set_style('darkgrid')
    plt.title('Label Correlation per Feature. With VS W/out Outliers', fontsize=17)
    sns.barplot(data=corr_change,x=corr_change.index,y='correlation',hue='origin')
    plt.show()

    # Feature Selection based on correlation with Class
    print('\nData shape before feature selection:', feat_sel.shape)
    print('\nCounts of strikes VS non-strikes before feature selection:')
    print(feat_sel.Label.value_counts())
    print('-'*40)

    X_train = feat_sel.drop('Label',1)
    y_train = feat_sel.Label

    # After feature-selection, X_test needs to include only the same features as X_train
    cols = X_train.columns
    X_test = X_test[cols]

    # Undersample and balance classes
    X_train, y_train = RandomUnderSampler(sampling_strategy={1:100000,0:100000}, random_state=42).fit_resample(X_train,y_train)

    print('\nX_train shape after reduction:', X_train.shape)
    print('\nCounts of strikes VS non-strikes in y_train:')
    print(np.unique(y_train, return_counts=True))

    # First-Run: Predictions on Default Parameters
    performance = pd.DataFrame(columns=['Train_Recall','Test_Recall','Test_Specificity'])

    # Load simple classifiers
    classifiers = [
                   #SVC(max_iter=1000),
                   #LogisticRegression(),
                   DecisionTreeClassifier(),
                   #KNeighborsClassifier(),
                   xgb.XGBClassifier(n_jobs=4, random_state = 42,objective='binary:logistic')
                   ]

    
    # Make a scoring callable from recall_score
    # recall = make_scorer(recall_score)

    # Get a classification report from each algorithm
    for clf in classifiers:    
    
      # Heading
      print('\n','-'*40,'\n',clf.__class__.__name__,'\n','-'*40)
      
      # Cross-validate on the train data
      print("TRAIN GROUP")
      train_cv = cross_val_score(X=X_train, y=y_train, estimator=clf, cv=3, n_jobs=4, scoring=optimize) # scoring=recall
      print("\nCross-validation recall scores:",train_cv)
      print("Mean recall score:",train_cv.mean())

      # Now predict on the test group
      print("\nTEST GROUP")
      y_pred = clf.fit(X_train, y_train).predict(X_test)
      print("\nRecall:",recall_score(y_test,y_pred))
      
      # Print confusion matrix
      conf_matrix = confusion_matrix(y_test,y_pred)
      sns.heatmap(conf_matrix, annot=True,fmt='d', cmap=plt.cm.copper)
      plt.show()
      
      # Store results
      performance.loc[clf.__class__.__name__+'_default',
                      ['Train_Recall','Test_Recall','Test_Specificity']] = [
          train_cv.mean(),
          recall_score(y_test,y_pred),
          conf_matrix[0,0]/conf_matrix[0,:].sum()
      ]


%time predict_imbalanced(pd.datetime(2019,6,1), feature_matrix_selection)
X = feature_matrix_selection.drop(columns = ['NumberId', 'time','date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
y = feature_matrix_selection.Label
counter = Counter(y)
print(counter)
# scatter plot of examples by class label
# for label, _ in counter.items():
# 	row_ix = np.where(y == label)[0]
# 	plt.scatter(X[row_ix, 0], X[row_ix, 1], label=str(label))
# plt.legend()
# plt.show()

# def predict_imb(dt, feature_matrix): 
#     """
#     Reference:
#       - https://xgboost.readthedocs.io/en/latest/parameter.html
#       - https://xgboost.readthedocs.io/en/latest/tutorials/param_tuning.html
#       - https://stats.stackexchange.com/questions/224512/reduce-false-positives-with-xgboost
#       - https://stackoverflow.com/questions/52499788/smotetomek-how-to-set-ratio-as-dictionary-for-fixed-balance
#       - https://github.com/coding-maniacs/over-under-sampling/blob/master/src/main.py
#     """
#     feature_matrix['date'] = feature_matrix['time']

#     # Subset labels
#     test_labels = feature_matrix.loc[feature_matrix['date'] == dt, 'Label']
#     train_labels = feature_matrix.loc[feature_matrix['date'] < dt, 'Label']

#     # Features
#     X_train = feature_matrix[feature_matrix['date'] < dt].drop(columns = ['NumberId', 'time',
#                                                                                      'date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
#     X_test = feature_matrix[feature_matrix['date'] == dt].drop(columns = ['NumberId', 'time',
#                                                                                      'date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
#     feature_names = list(X_train.columns)
    
#     if type(train_labels) == np.ndarray:
#       hit_ratio = float( len(np.where(train_labels == 0)[0]) / len(np.where(train_labels == 1)[0]) ) 
#     else:
#       hit_ratio = float(train_labels.value_counts()[0]/train_labels.value_counts()[1]) 
    
#     print(f"Hit ratio before balancing - {hit_ratio}\n")

#     # Labels
#     y_train = np.array(train_labels).reshape((-1, ))
#     y_test = np.array(test_labels).reshape((-1, ))
    
#     print('Training on {} observations.'.format(len(X_train)))
#     print('Testing on {} observations.\n'.format(len(X_test)))
  
#     # balancer = ClusterCentroids(sampling_strategy='auto', random_state=42, n_jobs=4)
#     # balancer = SMOTE(sampling_strategy='auto', random_state=42, n_jobs=4)
#     # # balancer = make_pipeline_imb(SMOTE(sampling_strategy='auto', random_state=42, n_jobs=4))
#     # X_bal, y_bal = balancer.fit_resample(X_train, train_labels)
#     # counter = Counter(y_bal)
#     # print('Balanced data: {}'.format(Counter(y_bal)))
#     # hit_ratio = float(counter[0] / counter[1])
#     # print(f"Hit ratio after balancing - {hit_ratio}\n")

   
#     # Sampling technique - https://stackoverflow.com/questions/52499788/smotetomek-how-to-set-ratio-as-dictionary-for-fixed-balance
#     pipeline = make_pipeline_imb(
#                                 SimpleImputer(strategy = 'constant', fill_value=0),
#                                 StandardScaler(),
#                                 SMOTE(sampling_strategy='auto', random_state=42, n_jobs=4)
#                                 )
    
#     # Calculate the hit ratio after resampling
#     X_bal, y_bal = pipeline.fit_resample(X_train, train_labels)
#     counter = Counter(y_bal)
#     print('Balanced data: {}'.format(Counter(y_bal)))
#     hit_ratio = float(counter[0] / counter[1])
#     print(f"Hit ratio after balancing - {hit_ratio}\n")
    
#     return 1,1

#     # Create the classifier
#     classifier = xgb.XGBClassifier(
#                              n_jobs=4, 
#                              random_state = 42,
#                             #  n_estimators=300, 
#                             #  max_depth=5,
#                             n_estimators=100, 
#                             max_depth=3,
#                             # n_estimators=1000, 
#                             #  max_depth=5,
#                             #  gamma=0,
#                             #  subsample=0.8,
#                             #  colsample_bytree=0.8,
#                             #  learning_rate=0.1,
#                              objective='binary:logistic',
#                              min_child_weight=1,
#                              scale_pos_weight=hit_ratio 
#                              )
    
#     # Create pipeline with classifier
#     pipeline = make_pipeline_imb(
#                                 SimpleImputer(strategy = 'constant', fill_value=0),
#                                 StandardScaler(),
#                                 SMOTE(sampling_strategy='auto', random_state=42, n_jobs=4),
#                                 classifier
#                                 )
    
#     # Train and predict
#     model = pipeline.fit(X_train, train_labels)

#     # Predict
#     predictions = pipeline.predict(X_test)
#     probs = pipeline.predict_proba(X_test)[:, 1]

#     # Total positive
#     positive = np.where((predictions==1))
#     print(f'Total predicted to be positive: {len(positive[0])} \n')
  
#     # Calculate metrics
#     rpt = classification_report(y_test, predictions)
#     rpt_imb = classification_report_imbalanced(y_test, predictions)
#     cm = confusion_matrix(y_test, predictions)
#     print('Classification report')
#     print(rpt)
#     print('Classification report - imbalanced')
#     print(rpt_imb)
#     print(f'Confusion matrix:\n {cm}\n')

#     # Total predicted matches
#     print('Predicted matches')
#     pred = np.where((predictions==1))
#     print(len(pred[0]), pred[0][0:23])
#     topN = np.argpartition(probs, -23)[-23:]
#     print(f'\n{topN}\n')  # Top N most high probability numbers
  
#     if len(positive[0]) > 0:
    
#       # Matching draws
#       print('Matched draws')
#       md = np.where((predictions==1) & (y_test==1))
#       print(f"Count: {len(md[0])}, Index: {md}")
#       month_data = feature_matrix.loc[feature_matrix['date'] == dt]
#       numbers = month_data.iloc[md[0]][['MAX(Results.LuckyNo)']]

#       print('\n\nTop 23 Possibility')
#       print(origin_data[(origin_data['DrawDate'].dt.year == dt.year) & 
#                           (origin_data['DrawDate'].dt.month == dt.month) &  
#                           (origin_data['LuckyNo'].isin(topN))].head(23))  
      
#       print('\n\nFirst 23 Numbers')
#       print(origin_data[(origin_data['DrawDate'].dt.year == dt.year) & 
#                           (origin_data['DrawDate'].dt.month == dt.month) &  
#                           (origin_data['LuckyNo'].isin(pred[0][0:23]))].head(23))    
             

#       print('\n\nAll matched')
#       print(origin_data[(origin_data['DrawDate'].dt.year == dt.year) & 
#                           (origin_data['DrawDate'].dt.month == dt.month) &  
#                           (origin_data['LuckyNo'].isin(numbers['MAX(Results.LuckyNo)']))].head(100))    
                                                  
#     else:
#       print('No luck this month')                 

#     # Feature importances
#     fi = pd.DataFrame({'feature': feature_names, 'importance': classifier.feature_importances_})
    
#     return fi, probs
#%time fi, probs = predict_imb(pd.datetime(2019,6,1), feature_matrix_selection)
#normalized_fi = plot_feature_importances(fi)
# Loop through from June to Dec
# start_mt = pd.datetime(2019,6,1)
# how_many_mt = 7
# for i in range(how_many_mt):
#   month_to_predict = start_mt + relativedelta(months=i)
#   print(f"\n{month_to_predict}\n-------------------\n")
#   %time predict(month_to_predict, feature_matrix_selection)<jupyter_output><empty_output><jupyter_text>Hyperparameter Tuning<jupyter_code>#feature_matrix_selection.info()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1264688024572306, 'fraction_numerical': 0.014500517875638415, 'mean_word_length': 2.934794828555368, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 46, 'https://': 8, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 35, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '588620', 'license': 'MIT', 'n_tokens_mistral': 9236, 'n_tokens_neox': 8751, 'n_words': 1591, 'path': '04_06_auto_ml_4.ipynb', 'repo_name': 'mengwangk/FortuneNet', 'size': '27999'}