<jupyter_start><jupyter_text>Feature extraction<jupyter_code>df_sessions['id'] = df_sessions['user_id']
df_sessions = df_sessions.drop(['user_id'],axis=1)
df_sessions.action = df_sessions.action.fillna('NAN')
df_sessions.action_type = df_sessions.action_type.fillna('NAN')
df_sessions.action_detail = df_sessions.action_detail.fillna('NAN')
# Action values with low frequency are changed to 'other'
act_freq = 100  # Threshold of frequency
act = dict(zip(*np.unique(df_sessions.action, return_counts=True)))
df_sessions.action = df_sessions.action.apply(lambda x: 'other' if act[x] < act_freq else x)

f_act = df_sessions.action.value_counts().argsort()
f_act_detail = df_sessions.action_detail.value_counts().argsort()
f_act_type = df_sessions.action_type.value_counts().argsort()
f_dev_type = df_sessions.device_type.value_counts().argsort()

# group by id
dgr_sess = df_sessions.groupby(['id'])
# Loop on dgr_sess to create all the features.
samples = [] 
ln = len(dgr_sess) # calculate length of grouped df_sessions

for g in dgr_sess:
    gr = g[1]   # data frame that comtains all the data for one user id
    l = []  #store temporary features
    l.append(g[0]) #append id

    # number of total actions
    l.append(len(gr)) # append corresponding data length of data

    # secs_elapsed null value filled with 0
    sev = gr.secs_elapsed.fillna(0).values
    
    # action features
    # The frequency of each action, 
    # the number of each action types, 
    # the mean and the standard deviation
    c_act = [0] * len(f_act)
    for i,v in enumerate(gr.action.values): 
        c_act[f_act[v]] += 1
    _, c_act_uqc = np.unique(gr.action.values, return_counts=True)
    c_act += [len(c_act_uqc), np.mean(c_act_uqc), np.std(c_act_uqc)]
    l = l + c_act

    # action_detail features
    c_act_detail = [0] * len(f_act_detail)
    for i,v in enumerate(gr.action_detail.values):
        c_act_detail[f_act_detail[v]] += 1
    _, c_act_det_uqc = np.unique(gr.action_detail.values, return_counts=True)
    c_act_detail += [len(c_act_det_uqc), np.mean(c_act_det_uqc), np.std(c_act_det_uqc)]
    l = l + c_act_detail

    # action_type features
    # log of the sum of secs_elapsed for each value
    l_act_type = [0] * len(f_act_type)
    c_act_type = [0] * len(f_act_type)
    for i,v in enumerate(gr.action_type.values):
        l_act_type[f_act_type[v]] += sev[i] # Total elapse of each action type
        c_act_type[f_act_type[v]] += 1  
    l_act_type = np.log(1 + np.array(l_act_type)).tolist() # log elapse
    _, c_act_type_uqc = np.unique(gr.action_type.values, return_counts=True)
    c_act_type += [len(c_act_type_uqc), np.mean(c_act_type_uqc), np.std(c_act_type_uqc)]
    l = l + c_act_type + l_act_type    

    # device_type features
    c_dev_type  = [0] * len(f_dev_type)
    for i,v in enumerate(gr.device_type .values):
        c_dev_type[f_dev_type[v]] += 1 
    c_dev_type.append(len(np.unique(gr.device_type.values))) 
    _, c_dev_type_uqc = np.unique(gr.device_type.values, return_counts=True)
    c_dev_type += [len(c_dev_type_uqc), np.mean(c_dev_type_uqc), np.std(c_dev_type_uqc)]        
    l = l + c_dev_type    

    # secs_elapsed features 
    l_secs = [0]*5
    l_log = [0]*15
    if len(sev) > 0:
        # Statistics about the secs_elapsed values.
        l_secs[0] = np.log(1 + np.sum(sev))
        l_secs[1] = np.log(1 + np.mean(sev)) 
        l_secs[2] = np.log(1 + np.std(sev))
        l_secs[3] = np.log(1 + np.median(sev))
        l_secs[4] = l_secs[0] / float(l[1]) #

        # Values are grouped in 15 intervals. Compute the number of values in each interval.
        log_sev = np.log(1 + sev).astype(int) 
        l_log = np.bincount(log_sev, minlength=15).tolist()                    
    l = l + l_secs + l_log

    # The list l has the feature values of one sample.
    samples.append(l)

# preparing objects    
samples = np.array(samples) 
samp_ar = samples[:, 1:].astype(np.float16) # Features except id
samp_id = samples[:, 0]   # The first column is id
    
col_names = []    #name of the columns
for i in range(len(samples[0])-1):  #except id
    col_names.append('c_' + str(i))  #column name    
df_agg_sess = pd.DataFrame(samp_ar, columns=col_names)
df_agg_sess['id'] = samp_id
df_agg_sess.index = df_agg_sess.id #set id as index

df_agg_sess.head()
train_row = train.shape[0] 
labels_full = train['country_destination'].values
train.drop(['date_first_booking'], axis = 1, inplace = True)
test.drop(['date_first_booking'], axis = 1, inplace = True)

#concatenate test and train
df = pd.concat([train, test], axis = 0, ignore_index = True)
tfa = df.timestamp_first_active.astype(str).apply(lambda x: datetime.datetime(int(x[:4]),
                                                                          int(x[4:6]), 
                                                                          int(x[6:8]),
                                                                          int(x[8:10]),
                                                                          int(x[10:12]),
                                                                          int(x[12:])))
# create year, month, day feature
df['tfa_year'] = np.array([x.year for x in tfa])
df['tfa_month'] = np.array([x.month for x in tfa])
df['tfa_day'] = np.array([x.day for x in tfa])
# create weekday feature
df['tfa_wd'] = np.array([x.isoweekday() for x in tfa]) 
df_tfa_wd = pd.get_dummies(df.tfa_wd, prefix = 'tfa_wd')  # one hot encoding 
df = pd.concat((df, df_tfa_wd), axis = 1) 
df.drop(['tfa_wd'], axis = 1, inplace = True)
# create season feature
Y = 2000
seasons = [(0, (date(Y,  1,  1),  date(Y,  3, 1))),  #'winter'
           (1, (date(Y,  3, 2),  date(Y,  6, 1))),  #'spring'
           (2, (date(Y,  6, 2),  date(Y,  9, 1))),  #'summer'
           (3, (date(Y,  9, 2),  date(Y, 12, 1))),  #'autumn'
           (0, (date(Y, 12, 2),  date(Y, 12, 31)))]  #'winter'

def get_season(dt):
    dt = dt.date() 
    dt = dt.replace(year=Y) # set year as 2000, which is useless in this feature
    return next(season for season, (start, end) in seasons if start <= dt <= end)

df['tfa_season'] = np.array([get_season(x) for x in tfa])
df_tfa_season = pd.get_dummies(df.tfa_season, prefix = 'tfa_season') # one hot encoding 
df = pd.concat((df, df_tfa_season), axis = 1)
df.drop(['tfa_season'], axis = 1, inplace = True)
dac = pd.to_datetime(df.date_account_created)
dt_span = dac.subtract(tfa).dt.days 
dt_span.value_counts().head()
# create categorical feature: span = -1; span > -1
def get_span(dt):
    # dt is an integer
    if dt == -1:
        return 'OneDay'
    else:
        return 'other'

df['dt_span'] = np.array([get_span(x) for x in dt_span])
df_dt_span = pd.get_dummies(df.dt_span, prefix = 'dt_span')
df = pd.concat((df, df_dt_span), axis = 1)
df.drop(['dt_span'], axis = 1, inplace = True)
df.drop(['date_account_created','timestamp_first_active'], axis = 1, inplace = True)
# Age
age = df.age
age.fillna(-1, inplace = True)
av = age.values
# This are birthdays instead of age (estimating age by doing 2014 - value)
av = np.where(np.logical_and(av<2000, av>1930), 2014-av, av) 
df['age'] = av
# Age has many abnormal values that we need to deal with. 
def get_age(age):
    if age < 0:
        return 'NAN'
    elif (np.logical_and(age<25, age>=15)):
        return 20
    elif (np.logical_and(age<35, age>=25)):
        return 30 
    elif (np.logical_and(age<45, age>=35)):
        return 40
    elif (np.logical_and(age<55, age>=45)):
        return 50
    elif (np.logical_and(age<65, age>=55)):
        return 60
    elif (np.logical_and(age<75, age>=65)):
        return 70
    elif (np.logical_and(age<85, age>=75)):
        return 80
    else:
        return 'Unphysical' # abnormal age,[0,15][85,]
df['age'] = np.array([get_age(x) for x in age])
df_age = pd.get_dummies(df.age, prefix = 'age')
df = pd.concat((df, df_age), axis = 1)
df.drop(['age'], axis = 1, inplace = True)
feat_toOHE = ['gender', 
             'signup_method', 
             'signup_flow', 
             'language', 
             'affiliate_channel', 
             'affiliate_provider', 
             'first_affiliate_tracked', 
             'signup_app', 
             'first_device_type', 
             'first_browser']
# one-hot-encoding
for f in feat_toOHE:
    df_ohe = pd.get_dummies(df[f], prefix=f, dummy_na=True)
    df.drop([f], axis = 1, inplace = True)
    df = pd.concat((df, df_ohe), axis = 1)
all_row = df.shape[0] 
df.head()
# merge session data and user data
df_all = pd.merge(df, df_agg_sess, how='left')
Xtrain = df_all.iloc[:train_row, :]
Xtest = df_all.iloc[train_row:, :]

Xtrain_temp = []
Xtrain_temp = Xtrain

Xtrain = Xtrain.drop(columns=['id','country_destination']) # delete id and label column
Xtrain = Xtrain.fillna(-1)  # null session data fill -1

# delete rows where all session data is null
Xtrain['all_null'] = np.array([sum(r<0) for r in Xtrain.values]) 
Xtrain = Xtrain_temp[Xtrain['all_null'] == 0]
labels = Xtrain['country_destination'].values
# delete useless feature
Xtrain = Xtrain.drop(['id','country_destination','tfa_year','tfa_month'],axis=1)
Xtest = Xtest.drop(['id','country_destination','tfa_year','tfa_month'],axis=1)


# full dataset
# df_all_full = df_all.drop(columns=['id','country_destination']) 
# df_all_full = df_all_full.fillna(-1)
# df_all_full['all_null'] = np.array([sum(r<0) for r in df_all_full.values]) 
# Xtrain_full = df_all_full.iloc[:train_row, :]
# Xtest_full = df_all_full.iloc[train_row:, :]
Xtrain.head()
Xtrain.to_csv("Airbnb_xtrain_v2.csv")
Xtest.to_csv("Airbnb_xtest_v2.csv")
labels.tofile("Airbnb_ytrain_v2.csv", sep='\n', format='%s') 

# full dataset
# Xtrain_full.to_csv("Airbnb_xtrain_v2_full.csv")
# Xtest_full.to_csv("Airbnb_xtest_v2_full.csv")
# labels_full.tofile("Airbnb_ytrain_v2_full.csv", sep='\n', format='%s')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14461008011357873, 'fraction_numerical': 0.019774870702768482, 'mean_word_length': 3.1611814345991562, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '403066', 'license': 'MIT', 'n_tokens_mistral': 3885, 'n_tokens_neox': 3702, 'n_words': 802, 'path': 'airbnb_2.FeatureEngineering.ipynb', 'repo_name': 'superhen/Prediction-of-Airbnb-new-user-booking', 'size': '9861'}