<jupyter_start><jupyter_text>Expedia Hotel Recommendations Kaggle competition<NAME> (<EMAIL> .at. g<EMAIL>)The latest version of this Jupyter notebook is available at https://github.com/ppik/playdata/tree/master/Kaggle-ExpediaHere I'll try to test some machine learning techniques on this dataset.<jupyter_code>import collections
import itertools
import operator
import random

import heapq
import matplotlib.pyplot as plt
import ml_metrics as metrics
import numpy as np
import pandas as pd
import sklearn
import sklearn.decomposition
import sklearn.linear_model
import sklearn.preprocessing

%matplotlib notebook<jupyter_output><empty_output><jupyter_text>Data importDefining a list of available data columns:<jupyter_code>traincols = ['date_time', 'site_name', 'posa_continent', 'user_location_country',
             'user_location_region', 'user_location_city', 'orig_destination_distance',
             'user_id', 'is_mobile', 'is_package', 'channel', 'srch_ci', 'srch_co',
             'srch_adults_cnt', 'srch_children_cnt', 'srch_rm_cnt', 'srch_destination_id',
             'srch_destination_type_id', 'is_booking', 'cnt', 'hotel_continent',
             'hotel_country', 'hotel_market', 'hotel_cluster']
testcols = ['id', 'date_time', 'site_name', 'posa_continent', 'user_location_country',
            'user_location_region', 'user_location_city', 'orig_destination_distance',
            'user_id', 'is_mobile', 'is_package', 'channel', 'srch_ci', 'srch_co',
            'srch_adults_cnt', 'srch_children_cnt', 'srch_rm_cnt', 'srch_destination_id',
            'srch_destination_type_id', 'hotel_continent', 'hotel_country', 'hotel_market']<jupyter_output><empty_output><jupyter_text>Convenience function for reading the data in:<jupyter_code>def read_csv(filename, cols, nrows=None):
    datecols = ['date_time', 'srch_ci', 'srch_co']
    dateparser = lambda x: pd.to_datetime(x, format='%Y-%m-%d %H:%M:%S', errors='coerce')

    dtypes = {
        'id': np.uint32,
        'site_name': np.uint8,
        'posa_continent': np.uint8,
        'user_location_country': np.uint16,
        'user_location_region': np.uint16,
        'user_location_city': np.uint16,
        'orig_destination_distance': np.float32,
        'user_id': np.uint32,
        'is_mobile': bool,
        'is_package': bool,
        'channel': np.uint8,
        'srch_adults_cnt': np.uint8,
        'srch_children_cnt': np.uint8,
        'srch_rm_cnt': np.uint8,
        'srch_destination_id': np.uint32,
        'srch_destination_type_id': np.uint8,
        'is_booking': bool,
        'cnt': np.uint64,
        'hotel_continent': np.uint8,
        'hotel_country': np.uint16,
        'hotel_market': np.uint16,
        'hotel_cluster': np.uint8,
    }

    df = pd.read_csv(
        filename,
        nrows=nrows,
        usecols=cols,
        dtype=dtypes,
        parse_dates=[col for col in datecols if col in cols],
        date_parser=dateparser,
    )
    
    if 'date_time' in df.columns:
        df['month'] = df['date_time'].dt.month.astype(np.uint8)
        df['year'] = df['date_time'].dt.year.astype(np.uint16)
        
    if 'srch_ci' and 'srch_co' in df.columns:
        df['srch_ngt'] = (df['srch_co'] - df['srch_ci']).astype('timedelta64[h]')
    
    if 'srch_children_cnt' in df.columns:
        df['is_family'] = np.array(df['srch_children_cnt'] > 0)
    
    return df
train = read_csv('data/train.csv.gz', nrows=None, cols=traincols)<jupyter_output><empty_output><jupyter_text>Getting a list of all `user_id`s in the sample.<jupyter_code>train_ids = set(train.user_id.unique())
len(train_ids)<jupyter_output><empty_output><jupyter_text>Pick a subset of users for testing and validation<jupyter_code>sel_user_ids = sorted(random.sample(train_ids, 12000))
sel_train = train[train.user_id.isin(sel_user_ids)]<jupyter_output><empty_output><jupyter_text>Create new test and training sets, using bookings from 2013 as training data and 2014 as test data.<jupyter_code>cv_train = sel_train[sel_train.year == 2013]
cv_test = sel_train[sel_train.year == 2014]<jupyter_output><empty_output><jupyter_text>Remove click events from `cv_test` as in original test data.<jupyter_code>cv_test = cv_test[cv_test.is_booking == True]<jupyter_output><empty_output><jupyter_text>Model 0: Most common clustersPublic solutions to the compedition ([Dataquest tutorial](https://www.dataquest.io/blog/kaggle-tutorial/) by <NAME> and [Leakage solution](https://www.kaggle.com/zfturbo/expedia-hotel-recommendations/leakage-solution) by ZFTurbo) use most common clusters in following groups:* `srch_destination_id`* `user_location_city`, `orig_destination_distance` (data leak)* `srch_destination_id`, `hotel_country`, `hotel_market` (for year 2014)* `srch_destination_id`* `hotel_country` Finding the most common overall clusters<jupyter_code>most_common_clusters = list(cv_train.hotel_cluster.value_counts().head().index)<jupyter_output><empty_output><jupyter_text>Predicting the most common clusters in groups of `srch_destination_id`, `hotel_country`, `hotel_market`.<jupyter_code>match_cols = ['srch_destination_id']
match_cols = ['srch_destination_id', 'hotel_country', 'hotel_market']
groups = cv_train.groupby(match_cols + ['hotel_cluster'])
top_clusters = {}
for name, group in groups:
    bookings = group['is_booking'].sum()
    clicks = len(group) - bookings
    
    score = bookings + .15*clicks
    
    clus_name = name[:len(match_cols)]
    if clus_name not in top_clusters:
        top_clusters[clus_name] = {}
    top_clusters[clus_name][name[-1]] = score<jupyter_output><empty_output><jupyter_text>This dictionary has a key of `srch_destination_id`, `hotel_country`, `hotel_market` and each value is another dictionary, with hotel clusters as keys and scores as values. Finding the top 5 for each destination.<jupyter_code>cluster_dict = {}
for n in top_clusters:
    tc = top_clusters[n]
    top = [l[0] for l in sorted(tc.items(), key=operator.itemgetter(1), reverse=True)[:5]]
    cluster_dict[n] = top<jupyter_output><empty_output><jupyter_text>Making predictions based on destination<jupyter_code>preds = []
for index, row in cv_test.iterrows():
    key = tuple([row[m] for m in match_cols])
    pred = cluster_dict.get(key, most_common_clusters)
    preds.append(pred)
cv_target = [[l] for l in cv_test['hotel_cluster']]
metrics.mapk(cv_target, preds, k=5)<jupyter_output><empty_output><jupyter_text>* `srch_destination_id`, `is_booking`: 0.212* `srch_destination_id`, `hotel_country`, `hotel_market`: 0.214  Model 1: Logistic regressionOne vs all classification using stohastic gradient decent and forward stepwise feature selection.<jupyter_code>clf = sklearn.linear_model.SGDClassifier(loss='log', n_jobs=4)<jupyter_output><empty_output><jupyter_text>Make dummy variables from categorical features. Pandas has `get_dummies()`, but currently this returns only `float64`-s, that thends to be rather memory hungry and slow. See [8725](https://github.com/pydata/pandas/issues/8725).<jupyter_code>cv_train_data = pd.DataFrame()
for elem in cv_train['srch_destination_id'].unique():
    cv_train_data[str(elem)] = cv_train['srch_destination_id'] == elem
cv_test_data = pd.DataFrame()
for elem in cv_train_data.columns:
    cv_test_data[elem] = cv_test['srch_destination_id'] == int(elem)
# cv_train_data['is_booking'] = cv_train['is_booking']
# cv_test_data['is_booking'] = np.ones(len(cv_test_data), dtype=bool)
clf.fit(cv_train_data, cv_train['hotel_cluster'])
result = clf.predict_proba(cv_test_data)
preds = [heapq.nlargest(5, clf.classes_, row.take) for row in result]
metrics.mapk(cv_target, preds, k=5)<jupyter_output><empty_output><jupyter_text>I would say that not that bad at all (comparing the random forrest classifier in the Dataquest tutorial). Using destination latent features form destination description data file.<jupyter_code>dest = pd.read_csv(
    'data/destinations.csv.gz',
    index_col = 'srch_destination_id',
)
pca = sklearn.decomposition.PCA(n_components=10)
dest_small = pca.fit_transform(dest[['d{}'.format(i) for i in range(1,150)]])
dest_small = pd.DataFrame(dest_small, index=dest.index)
cv_train_data = pd.DataFrame({key: cv_train[key] for key in ['srch_destination_id']})
cv_train_data = cv_train_data.join(dest_small, on=['srch_destination_id'], how='left')
cv_train_data = cv_train_data.fillna(dest_small.mean())
cv_test_data = pd.DataFrame({key: cv_test[key] for key in ['srch_destination_id']})
cv_test_data = cv_test_data.join(dest_small, on='srch_destination_id', how='left', rsuffix='dest')
cv_test_data = cv_test_data.fillna(dest_small.mean())
clf = sklearn.linear_model.SGDClassifier(loss='log', n_jobs=4)
clf.fit(cv_train_data, cv_train['hotel_cluster'])
result = clf.predict_proba(cv_test_data)
preds = [heapq.nlargest(5, clf.classes_, row.take) for row in result]
metrics.mapk(cv_target, preds, k=5)<jupyter_output><empty_output><jupyter_text>=> destination latent features seem not to be for any good use?!<jupyter_code>features = [
    'site_name', 'posa_continent', 'user_location_country',
    'user_location_region', 'user_location_city',
    'is_mobile', 'is_package',
    'channel', 'srch_adults_cnt', 'srch_destination_id',
    'srch_destination_type_id', 'is_booking', 'cnt',
    'hotel_continent', 'hotel_country', 'hotel_market',
    'month', 'year', 'is_family',
]
def fit_features(features, train, test):
    # Data manipulation - split categorical features
    train_data = pd.DataFrame()
    test_data = pd.DataFrame()
    for feature in features:
        if train[feature].dtype == np.dtype('bool'):
            train_data[feature] = train[feature]
            test_data[feature] = test[feature]
        else:
            for elem in train[feature].unique():
                train_data['{}_{}'.format(feature, elem)] = train[feature] == elem
                test_data['{}_{}'.format(feature, elem)] = test[feature] == elem
    
    # Fitting
    clf = sklearn.linear_model.SGDClassifier(loss='log', n_jobs=4)
    clf.fit(train_data, train['hotel_cluster'])
    
    # Cross-validate the fit
    result = clf.predict_proba(test_data)
    preds = [heapq.nlargest(5, clf.classes_, row.take) for row in result]
    target = [[l] for l in test['hotel_cluster']]
    
    return metrics.mapk(target, preds, k=5)
cv_results = {}
for feature in features:
    cv_results[feature] = fit_features([feature], cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results[feature]))
sorted(cv_results.items(), key=operator.itemgetter(1), reverse=True)<jupyter_output><empty_output><jupyter_text>The best single predictor of a hotel cluster seems to be `hotel_market`.<jupyter_code>features2 = [['hotel_market'] + [f] for f in features if f not in ['hotel_market']]
cv_results2 = {}
for feature in features2:
    cv_results2[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results2[tuple(feature)]))
sorted(cv_results2.items(), key=operator.itemgetter(1), reverse=True)[:3]
features3 = [['hotel_market', 'srch_destination_id'] + [f] for f in features if f not in ['hotel_market', 'srch_destination_id']]
cv_results3 = {}
for feature in features3:
    cv_results3[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results3[tuple(feature)]))
sorted(cv_results3.items(), key=operator.itemgetter(1), reverse=True)[:3]
features4 = [['hotel_market', 'srch_destination_id', 'hotel_country'] + [f] for f in features if f not in ['hotel_market', 'srch_destination_id', 'hotel_country']]
cv_results4 = {}
for feature in features4:
    cv_results4[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results4[tuple(feature)]))
sorted(cv_results4.items(), key=operator.itemgetter(1), reverse=True)[:3]
sel_features = ['hotel_market', 'srch_destination_id', 'hotel_country', 'is_package']
features5 = [sel_features + [f] for f in features if f not in sel_features]
cv_results5 = {}
for feature in features5:
    cv_results5[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results5[tuple(feature)]))
sorted(cv_results5.items(), key=operator.itemgetter(1), reverse=True)[:3]
sel_features = ['hotel_market', 'srch_destination_id', 'hotel_country', 'is_package', 'is_booking']
features6 = [sel_features + [f] for f in features if f not in sel_features]
cv_results6 = {}
for feature in features6:
    cv_results6[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results6[tuple(feature)]))
sorted(cv_results6.items(), key=operator.itemgetter(1), reverse=True)[:3]
sel_features = ['hotel_market', 'srch_destination_id', 'hotel_country', 'is_package', 'is_booking', 'posa_continent']
features7 = [sel_features + [f] for f in features if f not in sel_features]
cv_results7 = {}
for feature in features7:
    cv_results7[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results7[tuple(feature)]))
sorted(cv_results7.items(), key=operator.itemgetter(1), reverse=True)[:3]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13517840805123513, 'fraction_numerical': 0.011588899054589813, 'mean_word_length': 5.535625311410064, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 71, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '594041', 'license': 'MIT', 'n_tokens_mistral': 4657, 'n_tokens_neox': 4556, 'n_words': 924, 'path': 'Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb', 'repo_name': 'ppik/playdata', 'size': '13132'}