<jupyter_start><jupyter_text>Add indices to users and movies<jupyter_code>num_movies = len(df.movie_id.unique())
num_users = len(df.user_id.unique())
num_movies, num_users
movie_to_idx = {m:idx for idx,m in enumerate(df.movie_id.unique())}
idx_to_movie = {idx:m for idx,m in enumerate(df.movie_id.unique())}

user_to_idx = {m:idx for idx,m in enumerate(df.user_id.unique())}
idx_to_user = {idx:m for idx,m in enumerate(df.user_id.unique())}
df['movie_idx']=df.movie_id.map(movie_to_idx)
df['user_idx']=df.user_id.map(user_to_idx)
df.shape
df.head()
table = pa.Table.from_pandas(df)
pq.write_table(table, '../data/rating_indexed.parquet')
dictionaries = {'movie_to_idx':movie_to_idx,
                'idx_to_movie':idx_to_movie,
                'user_to_idx':user_to_idx,
                'idx_to_user':idx_to_user}
pickle.dump(dictionaries,open('sl_dictionaries.pkl','wb'))<jupyter_output><empty_output><jupyter_text>Compression with BLOSC<jupyter_code>df.shape
num_users = df.user_idx.max()+1
num_movies = df.movie_idx.max()+1
num_movies, num_users
size=df.shape[0]
size
ratings = df.rating.values.astype(np.int8)
users = df.user_idx.values.astype(np.int32)
movies = df.movie_idx.values.astype(np.int16)
ratings.dtype,users.dtype,movies.dtype
a=ratings
packed = blosc.compress_ptr(a.__array_interface__['data'][0], a.size, a.dtype.itemsize, 9, True, cname='zstd')
with pa.OSFile('ratings.dat', 'wb') as f:
    f.write(packed)
a=users
packed = blosc.compress_ptr(a.__array_interface__['data'][0], a.size, a.dtype.itemsize, 9, True, cname='zstd')
with pa.OSFile('users.dat', 'wb') as f:
    f.write(packed)
a=movies
packed = blosc.compress_ptr(a.__array_interface__['data'][0], a.size, a.dtype.itemsize, 9, True, cname='zstd')
with pa.OSFile('movies.dat', 'wb') as f:
    f.write(packed)<jupyter_output><empty_output><jupyter_text>Testing blosc uncompressing<jupyter_code>size = size
rating_rec = np.empty(size,dtype=np.int8)
movie_rec = np.empty(size,dtype=np.int16)
user_rec = np.empty(size,dtype=np.int32)
rating_file = 'ratings.dat'
users_file = 'users.dat'
movies_file = 'movies.dat'

blosc.decompress_ptr(pa.OSFile(users_file).readall(), user_rec.__array_interface__['data'][0])
blosc.decompress_ptr(pa.OSFile(movies_file).readall(), movie_rec.__array_interface__['data'][0])
blosc.decompress_ptr(pa.OSFile(rating_file).readall(), rating_rec.__array_interface__['data'][0])
rating_rec
ratings
all(rating_rec==ratings)
all(user_rec==users)
all(movie_rec==movies)
df.rating.unique()
np.unique(rating_rec)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1612776025236593, 'fraction_numerical': 0.008675078864353312, 'mean_word_length': 6.977987421383648, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '87348', 'license': 'Apache-2.0', 'n_tokens_mistral': 1046, 'n_tokens_neox': 1026, 'n_words': 136, 'path': 'notebooks/reindexing_blosc.ipynb', 'repo_name': 'MaveriQ/lmu_iml_recc', 'size': '2536'}