<jupyter_start><jupyter_text>Exercises for "Hands-on with Pydata: How to Build a Minimal Recommendation Engine"  Systems check: imports and files<jupyter_code>import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>Pandas questions: Series and DataFrames 1. Adding a column in a DataFrame<jupyter_code># given the following DataFrame, add a new column to it
df = pd.DataFrame({'col1': [1,2,3,4]})
df['col2']=[5,6,7,8]
df<jupyter_output><empty_output><jupyter_text>2. Deleting a row in a DataFrame<jupyter_code># given the following DataFrame, delete row 'd' from it
df = pd.DataFrame({'col1': [1,2,3,4]}, index = ['a','b','c','d'])
df.drop('a',axis=0)<jupyter_output><empty_output><jupyter_text>3. Creating a DataFrame from a few Series<jupyter_code># given the following three Series, create a DataFrame such that it holds them as its columns
ser_1 = pd.Series(np.random.randn(6))
ser_2 = pd.Series(np.random.randn(6))
ser_3 = pd.Series(np.random.randn(6))
df=pd.DataFrame({'1':ser_1,'2':ser_2, '3':ser_3})
df<jupyter_output><empty_output><jupyter_text>Pandas questions: Indexing  1. Indexing into a specific column<jupyter_code># given the following DataFrame, try to index into the 'col_2' column
df = pd.DataFrame(data={'col_1': [0.12, 7, 45, 10], 'col_2': [0.9, 9, 34, 11]},
                  columns=['col_1', 'col_2', 'col_3'],
                  index=['obs1', 'obs2', 'obs3', 'obs4'])
df.col_1<jupyter_output><empty_output><jupyter_text>2. Label-based indexing<jupyter_code># using the same DataFrame, index into the row whose index is 'obs3'
df.loc['obs3']<jupyter_output><empty_output><jupyter_text>2. Position-based indexing<jupyter_code># using the same DataFrame, index into into its first row
df.iloc[0,0]<jupyter_output><empty_output><jupyter_text>Mini-Challenge prep: data loading 1. How to load the `users` and `movies` portions of MovieLens<jupyter_code>import pandas as pd

users = pd.read_table(r'C:\Users\prati\Downloads\recommendation system\ml-1m\users.dat',
                      sep='::', header=None,
                      names=['user_id', 'gender', 'age', 'occupation', 'zip'], engine='python')

movies = pd.read_table( r'C:\Users\prati\Downloads\recommendation system\ml-1m\movies.dat',
                       sep='::', header=None,
                       names=['movie_id', 'title', 'genres'], engine='python') 
    
# subset version (hosted notebook)
movielens_train = pd.read_csv(r'C:\Users\prati\Downloads\recommendation system\movielens_train.csv', index_col=0, encoding='latin')
movielens_test = pd.read_csv(r'C:\Users\prati\Downloads\recommendation system\movielens_test.csv', index_col=0, encoding='latin')
movielens_train.head()
movielens_test.head()<jupyter_output><empty_output><jupyter_text>Mini-Challenge prep: evaluation functionsThese are the two functions that you will need to test your `estimate` method.<jupyter_code>def compute_rmse(y_pred, y_true):
    """ Compute Root Mean Squared Error. """
    
    return np.sqrt(np.mean(np.power(y_pred - y_true, 2)))
def evaluate(estimate_f):
    """ RMSE-based predictive performance evaluation with pandas. """
    
    ids_to_estimate = zip(movielens_test.user_id, movielens_test.movie_id)
    estimated = np.array([estimate_f(u,i) for (u,i) in ids_to_estimate])
    real = movielens_test.rating.values
    return compute_rmse(estimated, real)<jupyter_output><empty_output><jupyter_text>Test a dummy solution!<jupyter_code>def my_estimate_func(user_id, movie_id):
    return 3.0<jupyter_output><empty_output><jupyter_text>You can test for performance with the following line, which assumes that your function is called `my_estimate_func`:<jupyter_code>print ('RMSE for my estimate function: %s' % evaluate(my_estimate_func))<jupyter_output>RMSE for my estimate function: 1.283038170580714<jupyter_text>Reco systems questions: Minimal reco engine v1.0  1. Simple collaborative filtering using mean ratings<jupyter_code># write an 'estimate' function that computes the mean rating of a particular user
def collab_mean(user_id, movie_id):
    # first, index into all ratings of this movie
    # second, compute the mean of those ratings
        
    user_condition = movielens_train.user_id != user_id
    movie_condition = movielens_train.movie_id == movie_id
    if movielens_train.loc[user_condition & movie_condition].empty:
        return(3)
    else:
        a= movielens_train.loc[user_condition & movie_condition]
        b= a.rating
        return b.mean()

    
# try it out for a user_id, movie_id pair
collab_mean(4653, 2648)<jupyter_output><empty_output><jupyter_text>Mini-Challenge: first roundImplement an `estimate` function of your own using other similarity notions, eg.:- collaborative filter based on age similarities- collaborative filter based on zip code similarities- collaborative filter based on occupation similarities- content filter based on movie genre<jupyter_code>user_info = users.set_index('user_id')
user_info.head(5)
user_id = 3
user_info.loc[user_id, 'age']
#pd.__version__ == '1.0.0'
class CollabAgeReco:
    """ Collaborative filtering using an implicit sim(u,u'). """

    def learn(self):
        """ Prepare datastructures for estimation. """
        
        self.means_by_age = movielens_train.pivot_table('rating', index='movie_id', columns='age')

    def estimate(self, user_id, movie_id):
        """ Mean ratings by other users of the same gender. """
        
        if movie_id not in self.means_by_age.index: 
            return 3.0
        
        user_age = user_info.loc[user_id, 'age']
        if ~np.isnan(self.means_by_age.loc[movie_id, user_age]):
            return self.means_by_age.loc[movie_id, user_age]
        else:
            return self.means_by_age.loc[movie_id].mean()

#try .ix in place of .loc if it supports 
reco = CollabAgeReco()
reco.learn()
print('RMSE for CollabGenderReco: %s' % evaluate(reco.estimate))<jupyter_output>RMSE for CollabGenderReco: 1.2297001526594797<jupyter_text>Mini-Challenge: second roundImplement an `estimate` function of your own using other custom similarity notions, eg.:- euclidean- cosine<jupyter_code>def euclidean(s1, s2):
    """Take two pd.Series objects and return their euclidean 'similarity'."""
    diff = s1 - s2
    return 1 / (1 + np.sqrt(np.sum(diff ** 2)))
users = pd.read_table(r'C:\Users\prati\Downloads\recommendation system\ml-1m\users.dat',
                      sep='::', header=None, 
                      names=['user_id', 'gender', 'age', 'occupation', 'zip'], engine='python')

ratings = pd.read_table(r'C:\Users\prati\Downloads\recommendation system\ml-1m\ratings.dat',
                        sep='::', header=None, 
                        names=['user_id', 'movie_id', 'rating', 'timestamp'], engine='python')

movies = pd.read_table(r'C:\Users\prati\Downloads\recommendation system\ml-1m\movies.dat',
                       sep='::', header=None, 
                       names=['movie_id', 'title', 'genres'], engine='python')
movielens = pd.merge(pd.merge(ratings, users), movies)
class CollabeuclideanReco:
    """ Collaborative filtering using a custom sim(u,u'). """

    def learn(self):
        """ Prepare datastructures for estimation. """
        
        self.all_user_profiles = movielens.pivot_table('rating', index='movie_id', columns='user_id')

    def estimate(self, user_id, movie_id):
        """ Ratings weighted by correlation similarity. """
        
        user_condition = movielens_train.user_id != user_id
        movie_condition = movielens_train.movie_id == movie_id
        ratings_by_others = movielens_train.loc[user_condition & movie_condition]
        if ratings_by_others.empty: 
            return 3.0
        
        ratings_by_others.set_index('user_id', inplace=True)
        their_ids = ratings_by_others.index
        their_ratings = ratings_by_others.rating
        their_profiles = self.all_user_profiles[their_ids]
        user_profile = self.all_user_profiles[user_id]
        sims = their_profiles.apply(lambda profile: euclidean(profile, user_profile), axis=0)
        ratings_sims = pd.DataFrame({'sim': sims, 'rating': their_ratings})
        ratings_sims = ratings_sims[ratings_sims.sim > 0]
        if ratings_sims.empty:
            return their_ratings.mean()
        else:
            return np.average(ratings_sims.rating, weights=ratings_sims.sim)
        
reco = CollabeuclideanReco()
reco.learn()
print('RMSE for CollabPearsonReco: %s' % evaluate(reco.estimate))<jupyter_output>RMSE for CollabPearsonReco: 1.1243627596220371
{'fraction_non_alphanumeric': 0.12022498242324818, 'fraction_numerical': 0.018162643543473165, 'mean_word_length': 4.098566308243727, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '473692', 'license': 'MIT', 'n_tokens_mistral': 2998, 'n_tokens_neox': 2838, 'n_words': 647, 'path': 'recommendation_system.ipynb', 'repo_name': 'pratikdaga12/recoomendation_system', 'size': '8534'}