<jupyter_start><jupyter_text>Data exploration Summary<jupyter_code>import pandas as pd
from data.paths import file_paths as data_file_paths
from data.load import load_params as data_load_params<jupyter_output><empty_output><jupyter_text>Options<jupyter_code>pd.options.display.max_rows = None
pd.options.display.max_columns = None<jupyter_output><empty_output><jupyter_text>Load<jupyter_code>data_sets = {
    data_key: pd.read_csv(data_path, **data_load_params[data_key])
    for data_key, data_path in data_file_paths.items()
}<jupyter_output><empty_output><jupyter_text>Info<jupyter_code>data_descr = {
    data_key: {
        'rows': data_set.shape[0],
        'columns': data_set.shape[1],
        'nulls': data_set.isnull().values.any(),
        # 'notnull': data_set.notnull().values.all(),
    }
    for data_key, data_set in data_sets.items()
}
data_descr = pd.DataFrame(
    data=data_descr,
    index=['rows', 'columns', 'nulls']
).transpose().loc[['train', 'test'],:]

data_descr
for data_key, data_set in data_sets.items():
    print(f'\n{data_key}:\n')
    data_set.info()<jupyter_output>train:

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 4250 entries, 0 to 4249
Data columns (total 20 columns):
 #   Column                         Non-Null Count  Dtype  
---  ------                         --------------  -----  
 0   state                          4250 non-null   object 
 1   account_length                 4250 non-null   int64  
 2   area_code                      4250 non-null   object 
 3   international_plan             4250 non-null   object 
 4   voice_mail_plan                4250 non-null   object 
 5   number_vmail_messages          4250 non-null   int64  
 6   total_day_minutes              4250 non-null   float64
 7   total_day_calls                4250 non-null   int64  
 8   total_day_charge               4250 non-null   float64
 9   total_eve_minutes              4250 non-null   float64
 10  total_eve_calls                4250 non-null   int64  
 11  total_eve_charge               4250 non-null   float64
 12  total_night_minutes            4250 non[...]<jupyter_text>Train<jupyter_code>data_key = 'train'
data_set = data_sets[data_key]
data_descr = {
    'type': data_set.dtypes,
    # 'count': data_set.count(),
    'notnull': data_set.notnull().sum(),
    'null': data_set.isnull().sum(),
    'unique': data_set.nunique(),
}
data_descr = pd.DataFrame(
    data=data_descr,
    columns=['type', 'notnull', 'null', 'unique']
).sort_values(
    by=['type', 'unique'],
    # ascending=[True, True]
)

data_descr<jupyter_output><empty_output><jupyter_text>Sample<jupyter_code>n = 5
data_sample = data_set.sample(n).sort_index().transpose()

data_sample<jupyter_output><empty_output><jupyter_text>Describe<jupyter_code>data_type = object
data_subset = data_set.select_dtypes(data_type)
data_descr = data_subset.describe().transpose()

data_descr
data_type = int
data_subset = data_set.select_dtypes(data_type)
data_descr = data_subset.describe().transpose()

data_descr
data_type = float
data_subset = data_set.select_dtypes(data_type)
data_descr = data_subset.describe().transpose()

data_descr<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>data_key = 'test'
data_set = data_sets[data_key]
data_descr = {
    'type': data_set.dtypes,
    # 'count': data_set.count(),
    'notnull': data_set.notnull().sum(),
    'null': data_set.isnull().sum(),
    'unique': data_set.nunique(),
}
data_descr = pd.DataFrame(
    data=data_descr,
    columns=['type', 'notnull', 'null', 'unique']
).sort_values(
    by=['type', 'unique'],
    # ascending=[True, True]
)

data_descr
n = 5
data_sample = data_set.sample(n).sort_index().transpose()

data_sample<jupyter_output><empty_output><jupyter_text>Describe<jupyter_code>data_type = object
data_subset = data_set.select_dtypes(data_type)
data_descr = data_subset.describe().transpose()

data_descr
data_type = int
data_subset = data_set.select_dtypes(data_type)
data_descr = data_subset.describe().transpose()

data_descr
data_type = float
data_subset = data_set.select_dtypes(data_type)
data_descr = data_subset.describe().transpose()

data_descr<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12449126167105579, 'fraction_numerical': 0.023701220971989467, 'mean_word_length': 3.69438202247191, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '155387', 'license': 'MIT', 'n_tokens_mistral': 1567, 'n_tokens_neox': 1484, 'n_words': 244, 'path': 'exploration/data/example-1-summary.ipynb', 'repo_name': 'dmitry-s-danilov/kaggle-customer-churn-prediction-2020', 'size': '4177'}