<jupyter_start><jupyter_text>user-movie-ratings data<jupyter_code>user_movie_rating_df = pd.read_csv('data/ml-100k/u.data', sep='\t', header=None)
user_movie_rating_df.drop([3], axis=1, inplace=True)
user_movie_rating_df.rename(columns={0:'user_id', 1:'movie_id', 2:'rating'}, inplace=True)
user_movie_rating_df.head(3)<jupyter_output><empty_output><jupyter_text>Write to csv<jupyter_code>user_movie_rating_df.to_csv('data/user_movie_rating.csv', index=False)<jupyter_output><empty_output><jupyter_text>Read file<jupyter_code>user_movie_rating_df = pd.read_csv('data/user_movie_rating.csv')
user_movie_rating_df.head(3)<jupyter_output><empty_output><jupyter_text>Info<jupyter_code>user_movie_rating_df.info()
user_movie_rating_df.describe()<jupyter_output><empty_output><jupyter_text>Duplicates<jupyter_code>user_movie_rating_df[user_movie_rating_df.duplicated()]<jupyter_output><empty_output><jupyter_text>Most ratings<jupyter_code>user_movie_rating_df.rating.value_counts() / len(user_movie_rating_df)<jupyter_output><empty_output><jupyter_text>Validation data preparation  Complete data<jupyter_code>user_movie_rating_df.head(2)<jupyter_output><empty_output><jupyter_text>Split<jupyter_code>train_user_movie_rating_df, val_user_movie_rating_df = train_test_split(user_movie_rating_df, 
                                                                        test_size=0.015, random_state=2)<jupyter_output><empty_output><jupyter_text>Training user-movie-ratings data<jupyter_code>print(train_user_movie_rating_df.shape)
train_user_movie_rating_df.head(3)<jupyter_output>(98500, 3)<jupyter_text>Validation user-movie-ratings data<jupyter_code>print(val_user_movie_rating_df.shape)
val_user_movie_rating_df.head(2)<jupyter_output>(1500, 3)<jupyter_text>Write to csv<jupyter_code>train_user_movie_rating_df.to_csv('data/train_user_movie_rating.csv', index=False)
val_user_movie_rating_df.to_csv('data/val_user_movie_rating.csv', index=False)<jupyter_output><empty_output><jupyter_text>Number of Users, Movies and ratings<jupyter_code>n_users = user_movie_rating_df.user_id.unique().shape[0]
n_movies = user_movie_rating_df.movie_id.unique().shape[0]
n_ratings = len(user_movie_rating_df)

n_train_users = train_user_movie_rating_df.user_id.unique().shape[0]
n_train_movies = train_user_movie_rating_df.movie_id.unique().shape[0]
n_train_ratings = len(train_user_movie_rating_df)

n_val_users = val_user_movie_rating_df.user_id.unique().shape[0]
n_val_movies = val_user_movie_rating_df.movie_id.unique().shape[0]
n_val_ratings = len(val_user_movie_rating_df)

print(n_users, n_train_users, n_val_users)
print(n_movies, n_train_movies, n_val_movies)
print(n_ratings, n_train_ratings, n_val_ratings)<jupyter_output>943 943 606
1682 1682 683
100000 98500 1500<jupyter_text>Rating table<jupyter_code>rating_df = user_movie_rating_df.pivot(index = 'user_id', columns ='movie_id', values = 'rating').fillna(0)
rating_df.columns = rating_df.columns.map(str)
rating_df.reset_index(inplace=True)
rating_df.head(2)<jupyter_output><empty_output><jupyter_text>Training rating table<jupyter_code>train_rating_df = train_user_movie_rating_df.pivot(index = 'user_id', columns ='movie_id', 
                                                         values = 'rating').fillna(0)
train_rating_df.columns = train_rating_df.columns.map(str)
train_rating_df.reset_index(inplace=True)
train_rating_df.head(2)<jupyter_output><empty_output><jupyter_text>Verify number of movies and users<jupyter_code>print(n_train_movies, n_train_users)
print(len(train_rating_df.columns)-1, len(train_rating_df.index))<jupyter_output>1682 943
1682 943<jupyter_text>Verify validation's ratings (should be zero in train_rating_df)<jupyter_code>val_user_movie_rating_df.head(3)
user_id = 157
movie_id = 273

print(rating_df[str(movie_id)].loc[rating_df.user_id == user_id])
print(train_rating_df[str(movie_id)].loc[rating_df.user_id == user_id])<jupyter_output>156    5.0
Name: 273, dtype: float64
156    0.0
Name: 273, dtype: float64<jupyter_text>Write to csv<jupyter_code>rating_df.to_csv('data/rating.csv', index=False)
train_rating_df.to_csv('data/train_rating.csv', index=False)<jupyter_output><empty_output><jupyter_text>Sparsity<jupyter_code>round(1.0 - n_train_ratings / float(n_train_users * n_train_movies), 3)<jupyter_output><empty_output><jupyter_text>Movies<jupyter_code>m_cols = ["movie_id", "movie_title", "release date","video release date",
              "IMDb URL","unknown", "Action", "Adventure", "Animation", 
              "Children's" ,"Comedy","Crime","Documentary", "Drama","Fantasy",
          "Film-Noir","Horror", "Musical", "Mystery", "Romance", "Sci-Fi","Thriller","War","Western"]
movies_df = pd.read_csv('data/ml-100k/u.item', sep='|', names=m_cols, encoding="iso-8859-1")
movies_df.drop(["release date","video release date","IMDb URL"], axis=1, inplace=True)
movies_df.head()<jupyter_output><empty_output><jupyter_text>Write to csv<jupyter_code>movies_df.to_csv('data/movies.csv', index=False)<jupyter_output><empty_output><jupyter_text>Unknown genre<jupyter_code>movies_df.unknown.unique()
movies_df.loc[movie_df.unknown == 1]
user_movie_rating_df.loc[user_movie_rating_df.movie_id == 1373]<jupyter_output><empty_output><jupyter_text>Popular genres<jupyter_code>movies_df.drop(['movie_id', 'movie_title'], axis=1).sum(axis=0).sort_values(ascending=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12746557499069594, 'fraction_numerical': 0.02512095273539263, 'mean_word_length': 9.316698656429942, 'pattern_counts': {'":': 0, '<': 84, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '211577', 'license': 'MIT', 'n_tokens_mistral': 2243, 'n_tokens_neox': 2072, 'n_words': 194, 'path': 'Data_Preprocessing_and_EDA.ipynb', 'repo_name': 'mohdsaqibhbi/Movie_recommendation', 'size': '5374'}