<jupyter_start><jupyter_text>Automated ML - Tuning<jupyter_code>COLAB = True

DATASET_NAME = '4D.zip'

FEATURE_DATASET_PREFIX = 'feature_matrix_d2_v2'
if COLAB:
  !rm -rf dl-projects
  !git clone https://github.com/mengwangk/dl-projects
if COLAB:
  !cp dl-projects/utils* .
  !cp dl-projects/preprocess* .
  !cp dl-projects/plot* .
%load_ext autoreload
# %reload_ext autoreload
%autoreload 2

%matplotlib inline
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import math 
import matplotlib
import sys

from scipy import stats
from collections import Counter
from pathlib import Path

plt.style.use('fivethirtyeight')

sns.set(style="ticks")

import featuretools as ft

import warnings
warnings.filterwarnings('ignore')

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler
from sklearn.impute import SimpleImputer
from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix, roc_auc_score, precision_recall_curve, roc_curve, mean_squared_error, accuracy_score, average_precision_score, classification_report
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV, RandomizedSearchCV
from sklearn.decomposition import PCA

# from skopt import BayesSearchCV
# from skopt.space import Real, Categorical, Integer

# from sklearn.ensemble import RandomForestClassifier

# from scikitplot.plotters import plot_precision_recall_curve

from dateutil.relativedelta import relativedelta

from IPython.display import display

from utils import *
from preprocess import *

import xgboost as xgb

np.set_printoptions(threshold=sys.maxsize)

# The Answer to the Ultimate Question of Life, the Universe, and Everything.
np.random.seed(42)

from utils import feature_selection, plot_feature_importances
from plot import plot_correlation_matrix, plot_labelled_scatter
%aimport<jupyter_output>Modules to reload:
all-except-skipped

Modules to skip:<jupyter_text>Preparation<jupyter_code>if COLAB:
  from google.colab import drive
  drive.mount('/content/gdrive')
  GDRIVE_DATASET_FOLDER = Path('gdrive/My Drive/datasets/')
if COLAB:
  DATASET_PATH = GDRIVE_DATASET_FOLDER
  ORIGIN_DATASET_PATH = Path('dl-projects/datasets')
else:
  DATASET_PATH = Path("datasets")
  ORIGIN_DATASET_PATH = Path('datasets')

DATASET = DATASET_PATH/f"{FEATURE_DATASET_PREFIX}.ft"
ORIGIN_DATASET = ORIGIN_DATASET_PATH/DATASET_NAME

if COLAB:
  !ls -l gdrive/"My Drive"/datasets/ --block-size=M
  !ls -l dl-projects/datasets --block-size=M
data = pd.read_feather(DATASET)
origin_data = format_tabular(ORIGIN_DATASET)
data.shape, data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 959893 entries, 0 to 959892
Columns: 335 entries, NumberId to year
dtypes: datetime64[ns](1), float64(214), int64(118), uint8(2)
memory usage: 2.4 GB<jupyter_text>Exploratory Data Analysis  View data<jupyter_code>feature_matrix = data.drop(columns=['NumberId', 'month', 'year'])
feature_matrix.columns
feature_matrix[feature_matrix['Label'] == 1 ].tail(4)
feature_matrix.sort_values(by=['time', 'MAX(Results.LuckyNo)'], inplace=True)
feature_matrix.tail(4)
origin_data[origin_data.LuckyNo == 9957]
feature_matrix.describe().round(2)
print('Positive: ' + str(feature_matrix['Label'].value_counts()[0]) + ' which is ', round(feature_matrix['Label'].value_counts()[0]/len(feature_matrix) * 100,2), '% of the dataset')
print('Negative: ' + str(feature_matrix['Label'].value_counts()[1]) + ' which is ', round(feature_matrix['Label'].value_counts()[1]/len(feature_matrix) * 100,2), '% of the dataset')
plt.figure(figsize=(8, 8))
sns.countplot('Label', data=feature_matrix)
feature_matrix.isna().sum().sort_values(ascending=False)
feature_matrix.isnull().sum().sort_values(ascending=False)
len(feature_matrix[feature_matrix.isnull().any(axis=1)].time.unique())
feature_matrix[feature_matrix.isnull().any(axis=1)].head()<jupyter_output><empty_output><jupyter_text>Data Cleansing<jupyter_code>## Fill all NaN with 0
feature_matrix = feature_matrix.fillna(0)
feature_matrix.isnull().sum().sort_values(ascending=False)
feature_matrix.isna().sum().sort_values(ascending=False)
feature_matrix[feature_matrix.isnull().any(axis=1)].head()<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code># Feature scaling first??
print(feature_matrix.shape)
feature_matrix.columns
feature_matrix_selection = feature_selection(feature_matrix.drop(columns = ['time', 'TotalStrike', 'Label']))
feature_matrix_selection.shape, feature_matrix_selection.columns
feature_matrix_selection['time'] = feature_matrix['time']
feature_matrix_selection['TotalStrike'] = feature_matrix['TotalStrike']
feature_matrix_selection['Label'] = feature_matrix['Label']<jupyter_output><empty_output><jupyter_text>Feature Correlation<jupyter_code># Check without feature selection
# %time corrs = feature_matrix.corr().sort_values('Label')
# corrs['Label'].tail(100)
# Check with feature selection
%time corrs = feature_matrix_selection.corr().sort_values('Label')
corrs['Label'].tail(20)<jupyter_output><empty_output><jupyter_text>Balancing data<jupyter_code>from imblearn.combine import SMOTETomek
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import TomekLinks, AllKNN

balancer = AllKNN(random_state=42)
X = feature_matrix_selection.drop(columns = ['time', 'Label', 'TotalStrike'])
X.shape
y = feature_matrix_selection['Label']
y.shape, y.value_counts()[0], y.value_counts()[1]
# Normalized X first??

#X_bal, y_bal = balancer.fit_sample(X,y)
#X_bal.shape, y_bal.shape
#np.where(y_bal==0)[0].shape, np.where(y_bal==1)[0].shape, 
#len(np.where(y_bal==1)[0])/len(np.where(y_bal==0)[0]) * 100<jupyter_output><empty_output><jupyter_text>Modeling<jupyter_code>def predict(dt, feature_matrix, return_probs = False): 
   
    feature_matrix['date'] = feature_matrix['time']

    # Subset labels
    test_labels = feature_matrix.loc[feature_matrix['date'] == dt, 'Label']
    train_labels = feature_matrix.loc[feature_matrix['date'] < dt, 'Label']

    print(f"Size of test labels {len(test_labels)}")
    print(f"Size of train labels {len(train_labels)}")
    
    # Features
    X_train = feature_matrix[feature_matrix['date'] < dt].drop(columns = ['NumberId', 'time',
                                                                                     'date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
    X_test = feature_matrix[feature_matrix['date'] == dt].drop(columns = ['NumberId', 'time',
                                                                                     'date', 'Label', 'TotalStrike', 'month', 'year', 'index'], errors='ignore')
    print(f"Size of X train {len(X_train)}")
    print(f"Size of X test  {len(X_test)}")
   
    feature_names = list(X_train.columns)
    
    # Impute and scale features
    pipeline = Pipeline([('imputer', SimpleImputer(strategy = 'constant', fill_value=0)), 
                      ('scaler', StandardScaler())])

    # Fit and transform training data
    X_train = pipeline.fit_transform(X_train)
    X_test = pipeline.transform(X_test)
    
    # Labels
    y_train = np.array(train_labels).reshape((-1, ))
    y_test = np.array(test_labels).reshape((-1, ))
    
    print('Training on {} observations.'.format(len(X_train)))
    print('Testing on {} observations.\n'.format(len(X_test)))
    
    # https://xgboost.readthedocs.io/en/latest/parameter.html
    # https://xgboost.readthedocs.io/en/latest/tutorials/param_tuning.html
    # https://stats.stackexchange.com/questions/224512/reduce-false-positives-with-xgboost
    hit_ratio = float(train_labels.value_counts()[0]/train_labels.value_counts()[1]) 
    print(f"Hit ratio - {hit_ratio}")

    # Evaluation sets
    eval_set = [(X_train, y_train)]

    # Create the classifier
    model = xgb.XGBClassifier(n_jobs=-1, 
                             random_state = 42,
                             n_estimators=300, 
                             max_depth=3,
                             min_child_weight=5,
                             # criterion = 'entropy',
                             scale_pos_weight=hit_ratio,
                             #eval_metric=["error"], 
                             #eval_set=eval_set
                             )

    # Train 
    model.fit(X_train, y_train)
    
    # Make predictions
    predictions = model.predict(X_test)
    probs = model.predict_proba(X_test)[:, 1]
    
    # Total positive
    positive = np.where((predictions==1))
    print('Total predicted to be positive: ', len(positive[0]))
  
    # Calculate metrics
    rpt = classification_report(y_test, predictions)
    cm = confusion_matrix(y_test, predictions)
    print('Classification report')
    print(rpt)
    print('Confusion matrix')
    print(cm)

    # Total predicted matches
    print('Predicted matches')
    # print(predictions)
    pred = np.where((predictions==1))
    print(len(pred[0]), pred)
    #print('Proba of predicted matches')
    #print(probs[pred[0]])

    if len(positive[0]) > 0:
      
      # Matching draws
      
      print('Matched draws')
      md = np.where((predictions==1) & (y_test==1))
      print(f"Count: {len(md[0])}, Index: {md}")
      month_data = feature_matrix.loc[feature_matrix['date'] == dt]
      numbers = month_data.iloc[md[0]][['MIN(Results.LuckyNo)']]

      #print(f"Predicted numbers :\n {numbers}")
      #print(probs[m[0]])
      #print('-----')
      #print(probs)
      
      display(origin_data[(origin_data['DrawDate'].dt.year == dt.year) & 
                          (origin_data['DrawDate'].dt.month == dt.month) & 
                          (origin_data['LuckyNo'].isin(numbers['MIN(Results.LuckyNo)']))].head(len(positive[0])))                                                   
    else:
      print('No luck this month')                 

    # Feature importances
    fi = pd.DataFrame({'feature': feature_names, 'importance': model.feature_importances_})
    
    if return_probs:
        return fi, probs
    
    return fi

%time june_2019 = predict(pd.datetime(2019,6,1), feature_matrix_selection)
# arr = np.array([  15,   19,   33,   36,   38,   42,   44,   52,   66,   72,   75,
#          77,   80,   82,   96,  107])
# feature_matrix[feature_matrix_selection['MAX(Results.LuckyNo)'].isin(arr[0:1])]
normalized_fi = plot_feature_importances(june_2019)
normalized_fi.importance_normalized.sum()
#origin_data[ (origin_data['DrawDate'].dt.year==2019) & (origin_data['DrawDate'].dt.month==6)].DrawDate.unique()
#feature_matrix[ (feature_matrix['time'].dt.year==2019) & (feature_matrix['time'].dt.month==6)]
# Loop through from June to Dec
start_mt = pd.datetime(2019,6,1)
how_many_mt = 7
for i in range(how_many_mt):
  month_to_predict = start_mt + relativedelta(months=i)
  print(f"\n{month_to_predict}\n-------------------\n")
  %time predict(month_to_predict, feature_matrix_selection)
# origin_data[ (origin_data['DrawDate'].dt.year==2019) & (origin_data['DrawDate'].dt.month==12)].DrawDate.unique()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1342233447591557, 'fraction_numerical': 0.017319141259245895, 'mean_word_length': 4.1760037348272645, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 33, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '106028', 'license': 'MIT', 'n_tokens_mistral': 3793, 'n_tokens_neox': 3650, 'n_words': 650, 'path': '04_05_auto_ml_1.ipynb', 'repo_name': 'mengwangk/FortuneNet', 'size': '11086'}