<jupyter_start><jupyter_text>Preparing data<jupyter_code>import sys
import pandas as pd
import numpy as np
import scipy.sparse as sparse
from scipy.sparse.linalg import spsolve
import random
import os
import scipy.stats as ss
import scipy
from sklearn.preprocessing import MinMaxScaler

import implicit
df_train = pd.read_csv("../input/train.csv", low_memory=None, header=None)
df_train.columns = df_train.iloc[2713723]
df_train.drop(df_train.index[2713723], inplace=True)
df_train.drop(['deleted', 'deleter_fk'], axis=1, inplace=True)
df_train = df_train[~df_train['receipt_mfc'].str.match('^(?![0123456789])')]
df_train = df_train[~df_train['order_number'].str.match('^(?![0123456789])')]
df_train = df_train[~df_train['mfc'].str.match('^(?![0123456789])')]
df_train['service'] = df_train['service'].astype('int64')
df_train['cpgu_user'] = df_train['cpgu_user'].astype('int64')
df_train['mfc'] = df_train['mfc'].astype('int64')
df_train.drop(['receipt_mfc', 'creation_mfc', 'issue_date', 'close_date', 'id_x', 'id_y', 'id', 'cpgu_user'], axis=1, inplace=True)
def get_subdep(x):
    if x['sub_department'] is not np.nan:
        return 1
    return 0
df_train['is_subdep'] = df_train.apply(get_subdep, axis=1)
df_train.drop(['sub_department'], axis=1, inplace=True)
def get_csid(x):
    if x['custom_service_id'] is not np.nan:
        return 1
    return 0
df_train['is_csid'] = df_train.apply(get_csid, axis=1)
df_train['custom_service_id'].fillna(-1, inplace=True)
df_train['win_count'].fillna(df_train['win_count'].mean(), inplace=True)
df_train['order_date'] = df_train['order_date'].apply(pd.to_datetime)
df_train['change_timestamp'] = df_train['change_timestamp'].apply(pd.to_datetime)
df_train['proc_time'] = (df_train['change_timestamp']-df_train['order_date'])/np.timedelta64(1,'h')
def get_time(x):
    if x['proc_time'] < 0:
        return 0
    return x['proc_time']
df_train['proc_time'] = df_train.apply(get_time, axis=1)
df_train['dayofweek'] = df_train['order_date'].dt.dayofweek
def get_hour(x):
    h = x['order_date'].hour
    if h>=5 and h<11:
        return 'morning'
    elif h>=11 and h<17:
        return 'lunch'
    elif h>=17 and h<23:
        return 'evening'
    return 'night'
df_train['day_part'] = df_train.apply(get_hour, axis=1)
df_train.to_csv('clean_train.csv', index=False)
service = pd.read_csv("../input/cpgu_service.csv", sep=';', encoding='latin-1')
df_train = pd.merge(df_train, service[['id', 'person', 'sole', 'legal']], how='left', left_on = 'service', right_on = 'id')
cpgu_user = pd.read_csv("../input/cpgu_user.csv", sep=';')
df_train = pd.merge(df_train, cpgu_user, how='left', left_on = 'cpgu_user', right_on = 'id')
cpgu_mfc = pd.read_csv("../input/cpgu_mfc.csv", sep=';', encoding='latin-1')
df_train = pd.merge(df_train, cpgu_mfc[['id', 'win_count']], how='left', left_on = 'mfc', right_on = 'id')
df_train.to_csv('clean_train.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18430034129692832, 'fraction_numerical': 0.027303754266211604, 'mean_word_length': 6.4961636828644505, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '99159', 'license': 'MIT', 'n_tokens_mistral': 1205, 'n_tokens_neox': 1148, 'n_words': 194, 'path': 'Data preparation.ipynb', 'repo_name': 'Aindstorm/Deep-behavioral-analysis', 'size': '2930'}