<jupyter_start><jupyter_text>Changelog Version 1* Initial code  Library<jupyter_code>import os
import random
import platform

import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
import sklearn
!pip freeze > requirements.txt
print('Python version:', platform.python_version())
print('Numpy version:', np.__version__)
print('Pandas version:', pd.__version__)
print('Matplotlib version:', matplotlib.__version__)
print('Seaborn version:', sns.__version__)
print('Scikit-Learn version:', sklearn.__version__)
SEED = 42

os.environ['PYTHONHASHSEED']=str(SEED)
random.seed(SEED)
np.random.seed(SEED)
pd.set_option('display.width', None)
pd.set_option('display.max_column', None)<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>!ls /kaggle/input
!ls /kaggle/input/student-shopee-code-league-marketing-analytics
df_train = pd.read_csv('/kaggle/input/student-shopee-code-league-marketing-analytics/train.csv')
df_train
df_test = pd.read_csv('/kaggle/input/student-shopee-code-league-marketing-analytics/test.csv')
df_test
df_users = pd.read_csv('/kaggle/input/student-shopee-code-league-marketing-analytics/users.csv')
df_users<jupyter_output><empty_output><jupyter_text>Check  1. dtype<jupyter_code>df_train.dtypes
df_test.dtypes
df_users.dtypes<jupyter_output><empty_output><jupyter_text>2. Null<jupyter_code>df_train.isnull().sum()
df_test.isnull().sum()
df_users.isnull().sum()<jupyter_output><empty_output><jupyter_text>3. Unique value<jupyter_code>print(np.sort(df_train['country_code'].unique()))
print(np.sort(df_test['country_code'].unique()))
print(np.sort(df_users['attr_1'].unique()))
print(np.sort(df_users['attr_2'].unique()))
print(np.sort(df_users['attr_3'].unique()))
print(np.sort(df_users['domain'].unique()))<jupyter_output>[ 0.  1. nan]
[ 0.  1. nan]
[0. 1. 2. 3. 4.]
['@<EMAIL>' '@gmail.com' '@hotmail.com' '@icloud.com' '@live.com'
 '@outlook.com' '@qq.com' '@rocketmail.com' '@yahoo.com' '@ymail.com'
 'other']<jupyter_text>4. Corelation<jupyter_code>df_train.corr()<jupyter_output><empty_output><jupyter_text>Preprocess  1. Domain<jupyter_code>list_unique = df_users['domain'].unique()
dict_unique = {list_unique[i]: i for i in range(len(list_unique))}
df_users['domain'] = df_users['domain'].apply(lambda d: dict_unique[d])<jupyter_output><empty_output><jupyter_text>2. last_{}_day<jupyter_code>def convert(day):
    try:
        return np.float(day)
    except:
        return np.nan
df_train['last_open_day'] = df_train['last_open_day'].apply(convert)
df_train['last_login_day'] = df_train['last_login_day'].apply(convert)
df_train['last_checkout_day'] = df_train['last_checkout_day'].apply(convert)

df_test['last_open_day'] = df_test['last_open_day'].apply(convert)
df_test['last_login_day'] = df_test['last_login_day'].apply(convert)
df_test['last_checkout_day'] = df_test['last_checkout_day'].apply(convert)<jupyter_output><empty_output><jupyter_text>3. User ID<jupyter_code>df_train = df_train.join(df_users, on='user_id', rsuffix='_unused')
df_test = df_test.join(df_users, on='user_id', rsuffix='_unused')<jupyter_output><empty_output><jupyter_text>4. Drop Column<jupyter_code>del df_train['user_id']
del df_train['user_id_unused']
del df_train['row_id']

del df_test['user_id']
del df_test['user_id_unused']
del df_test['row_id']<jupyter_output><empty_output><jupyter_text>Inspect<jupyter_code>df_train
df_test
df_users<jupyter_output><empty_output><jupyter_text>Save<jupyter_code>df_train.to_csv('train_processed.csv', index=False)
df_test.to_csv('test_processed.csv', index=False)
df_users.to_csv('users_processed.csv', index=False)
df_train.to_parquet('train_processed.parquet', engine='pyarrow')
df_test.to_parquet('test_processed.parquet', engine='pyarrow')
df_users.to_parquet('users_processed.parquet', engine='pyarrow')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1601451153148484, 'fraction_numerical': 0.005960093288416689, 'mean_word_length': 9.633608815426998, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '516507', 'license': 'Unlicense', 'n_tokens_mistral': 1519, 'n_tokens_neox': 1495, 'n_words': 179, 'path': '01_preprocessing.ipynb', 'repo_name': 'ilos-vigil/scl-2020-marketing-analytics', 'size': '3859'}