<jupyter_start><jupyter_text>Imports<jupyter_code>import seaborn as sns
import numpy as np 
import pandas as pd
import datatable as dt

import matplotlib.pyplot as plt
%matplotlib inline

pd.set_option('display.max_columns', None)<jupyter_output><empty_output><jupyter_text>Load dataset<jupyter_code>%%time
train = dt.fread('../input/tabular-playground-series-oct-2021/train.csv').to_pandas()
test  = dt.fread('../input/tabular-playground-series-oct-2021/test.csv').to_pandas()
sub   = dt.fread('../input/tabular-playground-series-oct-2021/sample_submission.csv').to_pandas()<jupyter_output>CPU times: user 22.5 s, sys: 8.01 s, total: 30.5 s
Wall time: 23.9 s<jupyter_text>General overview<jupyter_code>train.head(5)
train.info(verbose=True)
train.describe()
def nan_counts(df):
    null_counts = df.isna().sum()
    info_df = pd.DataFrame(list(zip(null_counts.index,null_counts.values))\
                                         , columns = ['Column', 'nan_counts'])
    data_types = df.dtypes
    info_df['Dtype'] = data_types.values
    return info_df.T
nan_counts(train)<jupyter_output><empty_output><jupyter_text>Feature distribution<jupyter_code>def plot_feature_distribution(train, test):
    fig = plt.figure(figsize = (15, 71))
    cols = train.columns.tolist()[1:119]
    for i in cols:
        plt.subplot(24,5,cols.index(i)+1)
        sns.set_style("white")
        plt.title(i, size = 12, fontname = 'monospace')
        a = sns.kdeplot(train[i], color = '#f9ba32', linewidth = 1.3)
        sns.kdeplot(test[i], color = '#426e86', linewidth = 1.3)
        plt.ylabel('')
        plt.xlabel('')
        plt.xticks(fontname = 'monospace')
        plt.yticks([])
        for j in ['right', 'left', 'top']:
            a.spines[j].set_visible(False)
            a.spines['bottom'].set_linewidth(1.2)

    fig.tight_layout(h_pad = 3)

    plt.figtext(0.335, 1.02, 'Distribution of features', color = '#2f3131', fontname = 'monospace', size = 25)
    plt.figtext(0.3, 1.01, 'train', color = '#f9ba32', fontname = 'monospace', size = 18)
    plt.figtext(0.66, 1.01, 'test', color = '#426e86', fontname = 'monospace', size = 18)

    plt.show()
plot_feature_distribution(train.select_dtypes(include='float64') , test.select_dtypes(include='float64') )<jupyter_output><empty_output><jupyter_text>Target distribution<jupyter_code>train.target.value_counts().plot(kind='pie', figsize=(10, 10), autopct='%1.1f%%')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16080196399345337, 'fraction_numerical': 0.04132569558101473, 'mean_word_length': 4.13655462184874, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '238192', 'license': 'MIT', 'n_tokens_mistral': 980, 'n_tokens_neox': 911, 'n_words': 147, 'path': '2021/OCT/tps-oct-2021-eda-101.ipynb', 'repo_name': 'aayush26/kaggle_monthly_tabular_playground_series', 'size': '2444'}