<jupyter_start><jupyter_text>EXP1: Initial Exploration Let's try to work with a subset of data to see a first aproach of data clasification with a couple of algorithmns<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Data Exploration  Data Loading  Let's load a reduced version of the dataset to explore: **cat datasets/train.csv | head -n +100001 > datasets/train_reduced.csv**<jupyter_code>#! cat datasets/train.csv | head -n +100001 > datasets/train_reduced.csv
total_data=100000
df = pd.read_csv('datasets/train_reduced.csv', index_col='MachineIdentifier')<jupyter_output>/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2698: DtypeWarning: Columns (28) have mixed types. Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)<jupyter_text>Data Description<jupyter_code>df.info()
df.head()
df.describe()<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code># Let's remove each column with more than a 40% of nulls
th_perc = 0.3
clean_threshold = total_data*th_perc
df_for_clean = df.isnull()
columns_to_delete=[]
for c in df_for_clean.columns:
    nulls = df_for_clean[c].sum()
    if (nulls > clean_threshold):
        columns_to_delete.append(c)
        add=', REMOVE!!!'
    else:
        add=''
    print(c + ': ' + str(nulls) + add)
print('pre  filter N/A dimension' + str(df.shape))
df.drop(labels=columns_to_delete,axis=1,inplace=True)
print('post 1 filter N/A dimension' + str(df.shape))
df.dropna(inplace=True)
print('post 2 filter N/A dimension' + str(df.shape))<jupyter_output>pre  filter N/A dimension(100000, 82)
post 1 filter N/A dimension(100000, 73)
post 2 filter N/A dimension(80604, 73)<jupyter_text>Splitting label information<jupyter_code>df_y = df['HasDetections']
df.drop(labels=['HasDetections'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Categorical to Numerical  We have 83 columns, 53 numbers and 30 categorical. Let's see the diversity of each of them:<jupyter_code>columns_categorical = df.select_dtypes(include=['object']).columns
total = 0
for c in columns_categorical:
    values = df[c].nunique()
    total += values
    print(str(c) + ': ' + str(values))
print('Total new vars: ' + str(total))<jupyter_output>ProductName: 2
EngineVersion: 35
AppVersion: 73
AvSigVersion: 2323
Platform: 3
Processor: 3
OsVer: 9
OsPlatformSubRelease: 9
OsBuildLab: 273
SkuEdition: 7
Census_MDC2FormFactor: 11
Census_DeviceFamily: 1
Census_PrimaryDiskTypeName: 4
Census_ChassisTypeName: 27
Census_PowerPlatformRoleName: 9
Census_OSVersion: 235
Census_OSArchitecture: 3
Census_OSBranch: 13
Census_OSEdition: 15
Census_OSSkuName: 13
Census_OSInstallTypeName: 9
Census_OSWUAutoUpdateOptionsName: 5
Census_GenuineStateName: 4
Census_ActivationChannel: 5
Census_FlightRing: 7
Total new vars: 3098<jupyter_text>There are some of the categorical values that have a lot of values ....<jupyter_code>df_num=pd.get_dummies(data=df,columns=columns_categorical)<jupyter_output><empty_output><jupyter_text>Scaling data<jupyter_code>from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
data_scaled = scaler.fit_transform(df_num.values)<jupyter_output><empty_output><jupyter_text>Classification problem<jupyter_code>import sys
sys.path.append('../../GitRepos/dsbase/src/main/')

from sklearn.model_selection import train_test_split
from ModelDSBase import ModelDSBaseWrapper<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>from RandomForestClassificationDSBase import RandomForestClassificationDSBaseModel
from RandomForestClassificationDSBase import RandomForestClassificationDSBaseModelParamsToMap
params = RandomForestClassificationDSBaseModelParamsToMap(100,15)
rfc = ModelDSBaseWrapper('RF',data_scaled,df_y.values,[30,65,100],0.3,RandomForestClassificationDSBaseModel,params,splitter=train_test_split)
rfc.train()
lcrfc = rfc.getLearningCurves()
plt.plot(lcrfc[0,:],'b',lcrfc[1,:],'r')
rfc.getScore()<jupyter_output><empty_output><jupyter_text>Ada Boosting<jupyter_code>from AdaBoostClassificationDSBase import AdaBoostClassificationDSBaseModelParamsToMap
from AdaBoostClassificationDSBase import AdaBoostClassificationDSBaseModel
params = AdaBoostClassificationDSBaseModelParamsToMap(100,1.0)
abc = ModelDSBaseWrapper('AB',data_scaled,df_y.values,[30,65,100],0.3,AdaBoostClassificationDSBaseModel,params,splitter=train_test_split)
abc.train()
lcabc = abc.getLearningCurves()
plt.plot(lcabc[0,:],'b',lcabc[1,:],'r')
abc.getScore()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10416666666666667, 'fraction_numerical': 0.030927835051546393, 'mean_word_length': 8.240079365079366, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '172429', 'license': 'MIT', 'n_tokens_mistral': 1766, 'n_tokens_neox': 1618, 'n_words': 323, 'path': 'mmp_EXP_with_100000.ipynb', 'repo_name': 'giorbernal/kagglemmp', 'size': '4656'}