<jupyter_start><jupyter_text>EDA - Malware In, Malware Out_By <NAME>, December 2018_<jupyter_code>from IPython.display import HTML

HTML('''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }
 code_show = !code_show
} 
$( document ).ready(code_toggle);
</script>
The raw code for this IPython notebook is by default hidden for easier reading.
To toggle on/off the raw code, click <a href="javascript:code_toggle()">here</a>.''')
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

# Visualization
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib as mpl
%matplotlib inline
from wordcloud import WordCloud, STOPWORDS
import missingno as mn
from sklearn import preprocessing

import os
print("Datasets Available:/n",os.listdir("../input"))

# Any results you write to the current directory are saved as output.
dtypes = {
        'MachineIdentifier':                                    'category',
        'ProductName':                                          'category',
        'EngineVersion':                                        'category',
        'AppVersion':                                           'category',
        'AvSigVersion':                                         'category',
        'IsBeta':                                               'int8',
        'RtpStateBitfield':                                     'float16',
        'IsSxsPassiveMode':                                     'int8',
        'DefaultBrowsersIdentifier':                            'float16',
        'AVProductStatesIdentifier':                            'float32',
        'AVProductsInstalled':                                  'float16',
        'AVProductsEnabled':                                    'float16',
        'HasTpm':                                               'int8',
        'CountryIdentifier':                                    'int16',
        'CityIdentifier':                                       'float32',
        'OrganizationIdentifier':                               'float16',
        'GeoNameIdentifier':                                    'float16',
        'LocaleEnglishNameIdentifier':                          'int8',
        'Platform':                                             'category',
        'Processor':                                            'category',
        'OsVer':                                                'category',
        'OsBuild':                                              'int16',
        'OsSuite':                                              'int16',
        'OsPlatformSubRelease':                                 'category',
        'OsBuildLab':                                           'category',
        'SkuEdition':                                           'category',
        'IsProtected':                                          'float16',
        'AutoSampleOptIn':                                      'int8',
        'PuaMode':                                              'category',
        'SMode':                                                'float16',
        'IeVerIdentifier':                                      'float16',
        'SmartScreen':                                          'category',
        'Firewall':                                             'float16',
        'UacLuaenable':                                         'float32',
        'Census_MDC2FormFactor':                                'category',
        'Census_DeviceFamily':                                  'category',
        'Census_OEMNameIdentifier':                             'float16',
        'Census_OEMModelIdentifier':                            'float32',
        'Census_ProcessorCoreCount':                            'float16',
        'Census_ProcessorManufacturerIdentifier':               'float16',
        'Census_ProcessorModelIdentifier':                      'float16',
        'Census_ProcessorClass':                                'category',
        'Census_PrimaryDiskTotalCapacity':                      'float32',
        'Census_PrimaryDiskTypeName':                           'category',
        'Census_SystemVolumeTotalCapacity':                     'float32',
        'Census_HasOpticalDiskDrive':                           'int8',
        'Census_TotalPhysicalRAM':                              'float32',
        'Census_ChassisTypeName':                               'category',
        'Census_InternalPrimaryDiagonalDisplaySizeInInches':    'float16',
        'Census_InternalPrimaryDisplayResolutionHorizontal':    'float16',
        'Census_InternalPrimaryDisplayResolutionVertical':      'float16',
        'Census_PowerPlatformRoleName':                         'category',
        'Census_InternalBatteryType':                           'category',
        'Census_InternalBatteryNumberOfCharges':                'float32',
        'Census_OSVersion':                                     'category',
        'Census_OSArchitecture':                                'category',
        'Census_OSBranch':                                      'category',
        'Census_OSBuildNumber':                                 'int16',
        'Census_OSBuildRevision':                               'int32',
        'Census_OSEdition':                                     'category',
        'Census_OSSkuName':                                     'category',
        'Census_OSInstallTypeName':                             'category',
        'Census_OSInstallLanguageIdentifier':                   'float16',
        'Census_OSUILocaleIdentifier':                          'int16',
        'Census_OSWUAutoUpdateOptionsName':                     'category',
        'Census_IsPortableOperatingSystem':                     'int8',
        'Census_GenuineStateName':                              'category',
        'Census_ActivationChannel':                             'category',
        'Census_IsFlightingInternal':                           'float16',
        'Census_IsFlightsDisabled':                             'float16',
        'Census_FlightRing':                                    'category',
        'Census_ThresholdOptIn':                                'float16',
        'Census_FirmwareManufacturerIdentifier':                'float16',
        'Census_FirmwareVersionIdentifier':                     'float32',
        'Census_IsSecureBootEnabled':                           'int8',
        'Census_IsWIMBootEnabled':                              'float16',
        'Census_IsVirtualDevice':                               'float16',
        'Census_IsTouchEnabled':                                'int8',
        'Census_IsPenCapable':                                  'int8',
        'Census_IsAlwaysOnAlwaysConnectedCapable':              'float16',
        'Wdft_IsGamer':                                         'float16',
        'Wdft_RegionIdentifier':                                'float16',
        'HasDetections':                                        'int8'
        }
train_df = pd.read_csv('../input/train.csv', dtype=dtypes, nrows = 1000000)
test_df = pd.read_csv('../input/test.csv', dtype=dtypes, nrows = 10000000)
target = train_df.HasDetections.copy()
mapped_target = target.map({0: "Safe", 1: "Malware"})
print("Normalized Dependent Variable - Has Detection:\n",
      mapped_target.value_counts(normalize=True))
train_df.drop('HasDetections', axis =1, inplace= True)
target_var = "HasDetections"<jupyter_output><empty_output><jupyter_text>Explore Missing Values<jupyter_code>mn.dendrogram(train_df)
plt.show()
mn.matrix(train_df)
plt.show()<jupyter_output><empty_output><jupyter_text>Mass EDA for Continuous, Categorical, and Boolean<jupyter_code># Seperating Variables by Number of Unique Values
df_nnunique = train_df.nunique().reset_index().rename(columns = {"index": "cols",0:"unique_num"})
binary = list(df_nnunique.loc[df_nnunique.unique_num <= 2, "cols"])
few_categories = list(df_nnunique.loc[(df_nnunique.unique_num >= 3)
                                      & (df_nnunique.unique_num <= 10) , "cols"])
continuous = train_df.loc[:,list(df_nnunique.loc[df_nnunique.unique_num > 10, "cols"])].select_dtypes(include=[np.number]).columns
# continuous = train_df.select_dtypes(include=['floating']).columns

print("Number of Binary Variables: ", len(binary)-1)
print("Number of Continous Variables: ", len(continuous)-1)
print("Number of Non-Binary, Categorical Variables: ", len(few_categories))<jupyter_output><empty_output><jupyter_text>Continous<jupyter_code># Melt
melt_df = pd.melt(pd.concat([train_df.loc[:,continuous],mapped_target],axis=1), id_vars=target_var)
grid = sns.FacetGrid(melt_df,col="variable", hue=target_var, col_wrap=4 , size=4.0, aspect=1.3, sharex=False, sharey=False)
grid.map(sns.kdeplot, "value")
grid.set_titles(size=10)
grid.add_legend();
plt.show()<jupyter_output><empty_output><jupyter_text>Binary / Boolean<jupyter_code>for col in train_df.loc[:,binary].columns:
    le = preprocessing.LabelEncoder()
    train_df[col] = le.fit_transform(train_df[col].astype(str))
# Melt
binary_df = pd.concat([train_df.loc[:,binary],mapped_target],axis=1)

melt_df = pd.melt(binary_df, id_vars=target_var)
binary_data = pd.pivot_table(melt_df, values="value", index="variable",columns=["HasDetections"], aggfunc = np.sum)

f, ax = plt.subplots(figsize=[8,6])
sns.heatmap(binary_data, annot=False, fmt=".2f",cbar_kws={'label': 'Occurence'},cmap="magma",ax=ax)
ax.set_title("Binary Variable Positive Occurence Count by Dependent Variable")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10300640709709216, 'fraction_numerical': 0.013799901429275506, 'mean_word_length': 1.5608278647147904, 'pattern_counts': {'":': 1, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 60, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '347365', 'license': 'MIT', 'n_tokens_mistral': 2694, 'n_tokens_neox': 2533, 'n_words': 469, 'path': 'eda-malware-in-malware-out.ipynb', 'repo_name': 'nicapotato/KaggleNotebooks', 'size': '10150'}