<jupyter_start><jupyter_text>Load data files<jupyter_code>PROJECT_DATA_PATH = '/Users/tanmaymathur/Documents/personal_workspace/recommender/docs/'
USERS_FILE = '{parent_dir}/users.txt'
HOTELS_FILE = '{parent_dir}/hotels.txt'
ACTIVITY_FILE = '{parent_dir}/activity.txt'
SEP = '\t'
users_df = pd.read_csv(USERS_FILE.format(parent_dir=PROJECT_DATA_PATH), sep=SEP)
hotels_df = pd.read_csv(HOTELS_FILE.format(parent_dir=PROJECT_DATA_PATH), sep=SEP)
activity_df = pd.read_csv(ACTIVITY_FILE.format(parent_dir=PROJECT_DATA_PATH), sep=SEP)<jupyter_output><empty_output><jupyter_text>See sample data<jupyter_code>print "Users data:\n", users_df.sample(5)
print "Hotels data :\n", hotels_df.sample(5)
print "Activity data :\n", activity_df.sample(5)<jupyter_output>Activity data :
       user  hotel
14154  3699     56
15681  4111     46
8922   2357     45
14545  3804     52
10784  2809     39<jupyter_text>Data Exploration<jupyter_code>"""Lets count the number of unique home continents"""
print "Unique continents are :",set(users_df['home continent'])
print "Number of unique continents are : ", len(set(users_df['home continent']))
"Distribution of users per continent"
users_df['home continent'].value_counts().plot(kind='bar',colormap="Set3",figsize=(15,5))
#Q1a. What continent is the hotel in?
"""To answer the above question we do the following steps:
1. Merge users and activity data
2. Identify unique rows by user id and hotel id
2. Group and count the hotels by hotel id and home continent id
3. Sort the grouped result and pick the majority home continent id for hotel id

Assumption: Majority users in our dataset dont travel across continents
"""
users_activity_df = pd.merge(users_df, activity_df, how='inner', on='user')
unique_users_activity_df = users_activity_df.drop_duplicates(['user','hotel'])
hotel_home_continent_df = unique_users_activity_df.groupby(['hotel','home continent']).size().reset_index(name='counts').sort_values(['hotel','counts'], ascending=False)
idx = hotel_home_continent_df.groupby(['hotel'])['counts'].transform(max) == hotel_home_continent_df['counts']
hotel_continent = hotel_home_continent_df[idx]
"""The result below shows that the hotel can be visited by users from different continents but based on our assumptions,
we assign hotel to the continent with highest user home continent count"""
hotel_home_continent_df.head(10)
"""Based on our majority hypothesis, we see that all hotels belong to continent 2, which is not very useful information"""
hotel_continent[hotel_continent['home continent'] !=2]
#Q1b. What continent is the hotel in? Try to answer the same question but by changing the our second step
"""To answer the above question we do the following steps:
1. Merge users and activity data
2. We dont identify unique user to hotel visits because the frequency can be a good indicator of home continent
2. Group and count the hotels by hotel id and home continent id
3. Sort the grouped result and pick the majority home continent id for hotel id
Assumption: Majority users in our dataset dont travel across continents
"""
users_activity_df = pd.merge(users_df, activity_df, how='inner', on='user')
hotel_home_continent_df = users_activity_df.groupby(['hotel','home continent']).size().reset_index(name='counts').sort_values(['hotel','counts'], ascending=False)
"""The result below shows that the hotel can be visited by users from different continents but based on our assumptions,
we assign hotel to the continent with highest user home continent count"""
hotel_home_continent_df.head(10)
"""Looking at the data, no user has visited the same hotel twice and hence does not contribute to finding the home continent"""
"""Creating a new column for hotels to tell which continents do the majority users come from (Will be done on training set)"""
hotel_home_continent_df['counts'] = hotel_home_continent_df['counts'].groupby(hotel_home_continent_df['hotel']).transform(lambda x: np.where(x>=np.median(x), x, np.NaN))
hotel_home_continent_df = hotel_home_continent_df.dropna()
hotel_home_continent_df['home continent'] = hotel_home_continent_df['home continent'].astype(str)
hotel_home_continent_df.groupby(['hotel'])['home continent'].apply(','.join).reset_index()
# Q2. What are the most visited hotels?
mean = users_activity_df.groupby(['hotel']).size().mean()
users_activity_df.groupby(['hotel']).agg(['mean','count'])
fig, (axis1) = plt.subplots(1,1,figsize=(15,3))
sns.countplot(x='hotel', data=users_activity_df, palette="Set3")
"Plot hotels that have activity more than mean"
hotel_activity_count = users_activity_df.groupby(['hotel']).size().reset_index(name='counts')
hotel_activity_more_than_mean = hotel_activity_count[hotel_activity_count['counts']>mean]
filtered_hotels = pd.merge(users_activity_df, hotel_activity_more_than_mean, on='hotel')
sns.countplot(x='hotel', data=filtered_hotels, palette="Set3")
"""Count number of hotels visited by every user"""
fig, (axis1) = plt.subplots(1,1,figsize=(15,3))
sns.distplot(users_activity_df['user'], label='Number of hotels visited by every user')
"Distribution of users per continent"
users_df['home continent'].value_counts().plot(kind='bar',colormap="Set3",figsize=(15,5))
"""Add user history information to each user record. Will be used as feature"""
users_activity_df['hotel'] = users_activity_df['hotel'].astype(str)
users_activity_df.groupby(['user'])['hotel'].apply(','.join).reset_index()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11793928242870286, 'fraction_numerical': 0.01766329346826127, 'mean_word_length': 7.350230414746544, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '183401', 'license': 'MIT', 'n_tokens_mistral': 1748, 'n_tokens_neox': 1683, 'n_words': 497, 'path': 'src/Data Exploration.ipynb', 'repo_name': 'mathurtx/hotel-recommender', 'size': '5435'}