<jupyter_start><jupyter_text>Part 1<jupyter_code>set(data_crypt[0]['rating'])
my_ratings_1 = data_clear[0].set_index('mail').loc[mail_greg][clear_cols].values.tolist()
my_ratings_1
my_ratings_1[0]
data_clear[0].set_index(clear_cols).loc[my_ratings_1[0]]
data_crypt[0].set_index(clear_cols).loc[my_ratings_1[0]]<jupyter_output><empty_output><jupyter_text>Just to be sure<jupyter_code>associated_crypt = [data_crypt[0].set_index(clear_cols).loc[r].values.tolist() for r in my_ratings_1]
possible_crypt_mail = {m:0 for l in associated_crypt for m, _ in l}

for l in associated_crypt:
    for m, _ in l:
        possible_crypt_mail[m] += 1
        
        
my_mail_crypt_1 = max(possible_crypt_mail.items(), key=lambda x: x[1])[0]
my_mail_crypt_1
my_movies_crypt = data_crypt[0].set_index(['mail']).loc[my_mail_crypt_1]['movie'].values.tolist()
count_per_movie = {m:dict() for m in my_movies_crypt}
movie_to_name = dict()

for m in my_movies_crypt:
    ratings = data_crypt[0].set_index('movie').loc[m][clear_cols].values.tolist()
    for r in ratings:
        for movie_name in data_clear[0].set_index(clear_cols).loc[r]['movie'].values.tolist():
            count_per_movie[m][movie_name] = count_per_movie[m].get(movie_name, 0) + 1
    movie_to_name[m] = max(count_per_movie[m].items(), key=lambda x: x[1])[0]
movie_to_name
with open('solutions/ex1a.txt', 'w') as file: 
    file.write('\n'.join(movie_to_name.values()))<jupyter_output><empty_output><jupyter_text>Part 2<jupyter_code>set(data_clear[1]['rating'].values)
len(data_clear[1]), len(data_crypt[1])
data_clear[1].set_index('mail').loc[mail_greg]['movie'].values.tolist()
movie_crypt_freq = data_crypt[1]\
    .pivot_table(values='date', index='movie', aggfunc=len)\
    .fillna(0)\
    .sort_values('date')\
    .reset_index()
movie_crypt_freq.head()
movie_clear_freq = data_clear[1]\
    .pivot_table(values='date', index='movie', aggfunc=len)\
    .fillna(0)\
    .sort_values('date')\
    .reset_index()
movie_clear_freq.head()
sha_to_movie = dict(pd.concat((movie_crypt_freq, movie_clear_freq), axis=1)['movie'].values.tolist())
movie_to_sha = {m:s for s,m in sha_to_movie.items()}
movie_to_sha['A Separation']
my_movies_2 = data_clear[1].set_index('mail').loc[mail_greg]['movie'].values.tolist()
len(my_movies_2)
my_movies_2
all_mails = []
for m in my_movies_2:
    mails = data_crypt[1].set_index('movie').loc[movie_to_sha[m]]['mail']
    if type(mails) == str:
        all_mails.append(mails)
    else:
        for mail in mails.values.tolist():
            all_mails.append(mail)
all_mails = [mail for m in my_movies_2 for mail in data_crypt[1].set_index('movie').loc[[movie_to_sha[m]]]['mail'].values.tolist()]
sorted({m:all_mails.count(m) for m in set(all_mails)}.items(), key=lambda x:x[1], reverse=True)[:5]
mail_crypt_2, _ = max({m:all_mails.count(m) for m in set(all_mails)}.items(), key=lambda x:x[1])
my_movies_crypt_2 = data_crypt[1].set_index(['mail']).loc[mail_crypt_2]['movie'].values.tolist()
movie_to_name_2 = {m:sha_to_movie[m] for m in my_movies_crypt_2}
set(movie_to_name_2.values()) - set(my_movies_2)
with open('solutions/ex1b.txt', 'w') as file:
    txt = '\n'.join(movie_to_name_2.values())
    print(txt)
    file.write(txt)<jupyter_output>Psycho
Seven Samurai
Apocalypse Now
Metropolis
Spirited Away
The Shawshank Redemption
Pulp Fiction
City Lights
12 Angry Men
The Godfather
Singin' in the Rain
The Good, the Bad and the Ugly
Bicycle Thieves
Lawrence of Arabia
On the Waterfront
Goodfellas
M
The Third Man
Schindler's List
Raiders of the Lost Ark<jupyter_text>Part 3<jupyter_code>len(data_clear[2]), len(data_crypt[2])
weights = list(range(1,15))+list(range(13, 0, -1))
data_clear[2].head()
a = pd.to_datetime('01/02/03', format='%d/%m/%y')
set_my_movies = list(set(data_clear[2].set_index('mail').loc[mail_greg]['movie'].values.tolist()))
set_movies_clear = list(set(data_clear[2]['movie'].values.tolist()))
set_movies_crypt = list(set(data_crypt[2]['movie'].values.tolist()))
data_clear[2]['date'].head()
from tqdm import tqdm_notebook as tqdm
movie_to_sha = dict()
for movie in tqdm(set_movies_clear):
    movie_score = dict()
    for date in data_clear[2].set_index('movie').loc[movie]['date'].values.tolist():
        mask = data_crypt[2]['date'].apply(lambda x: date - 14 < x < date + 14)
        for movie_crypt, date2 in data_crypt[2][mask][['movie', 'date']].values.tolist():
            movie_score[movie_crypt] = movie_score.get(movie_crypt, 0) + weights[date - date2 + 13]
    movie_to_sha[movie] = max(movie_score.items(), key=lambda x:x[1])[0]
sha_to_movie = {s:m for m,s in movie_to_sha.items()}
len(movie_to_sha), len(sha_to_movie)
all_mails = []
for m in set_my_movies:
    mails = data_crypt[2].set_index('movie').loc[movie_to_sha[m]]['mail']
    if type(mails) == str:
        all_mails.append(mails)
    else:
        for mail in mails.values.tolist():
            all_mails.append(mail)
sorted({m:all_mails.count(m) for m in set(all_mails)}.items(), key=lambda x:x[1], reverse=True)[:5]
my_mail_crypt_3, _ = max({m:all_mails.count(m) for m in set(all_mails)}.items(), key=lambda x:x[1])
my_mail_crypt_3
my_movies_crypt_3 = data_crypt[2].set_index(['mail']).loc[my_mail_crypt_3]['movie'].values.tolist()
my_movies_3 = [sha_to_movie[m] for m in my_movies_crypt_3]
with open('solutions/ex1c.txt', 'w') as file:
    txt = '\n'.join(my_movies_3)
    print(txt)
    file.write(txt)<jupyter_output>Full Metal Jacket
Once Upon a Time in America
The Iron Giant
A Streetcar Named Desire
Hannah and Her Sisters
Dog Day Afternoon
Ran
The Matrix
Throne of Blood
Toy Story 3
Spirited Away
The Exterminating Angel
The Shawshank Redemption
The Lord of the Rings: The Return of the King
Harold and Maude
The Wages of Fear
Harry Potter and the Deathly Hallows: Part 2
Rosemary's Baby
Blue Velvet
Terminator 2: Judgment Day
In the Mood for Love
Kind Hearts and Coronets
The Great Escape
Frankenstein
The Seventh Seal
Toy Story 2
The Philadelphia Story
The Gold Rush
Lagaan: Once Upon a Time in India
To Be or Not to Be
To Kill a Mockingbird
Blade Runner
Yojimbo
Star Wars: Episode V - The Empire Strikes Back
Saving Private Ryan
Whiplash
Incendies
Rome, Open City
Laura
Cinema Paradiso
Life of Brian
Before Sunset
Persepolis
Trainspotting
The Diving Bell and the Butterfly
It's a Wonderful Life
The Deer Hunter
The Last Picture Show
Fight Club
Citizen Kane
Woman in the Dunes
L.A. Confidential
Anatomy of a Mur[...]
{'fraction_non_alphanumeric': 0.14212168486739468, 'fraction_numerical': 0.01794071762870515, 'mean_word_length': 5.6092783505154635, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '169628', 'license': 'MIT', 'n_tokens_mistral': 2622, 'n_tokens_neox': 2432, 'n_words': 524, 'path': 'HW3/Exercice01/ex1.ipynb', 'repo_name': 'gregunz/ISP2019', 'size': '6410'}