<jupyter_start><jupyter_text>Read all Data<jupyter_code># Movies untill 2016
df1 = pd.read_csv("prep1data.csv")
df1
# Movies of 2017
df2 = pd.read_csv("prep2data.csv")
df2
# Movies of 2018 to 2020
df3 = pd.read_csv("prep3data.csv")
df3
# Movies of 2021 (Until June)
df4 = pd.read_csv("prep4data.csv")
df4<jupyter_output><empty_output><jupyter_text>Combining all Data<jupyter_code>df = df1.append(df2.append(df3.append(df4)))
df
df['comb'] = df['actor_1_name'] + ' ' + df['actor_2_name'] + ' '+ df['actor_3_name'] + ' '+ df['director_name'] +' ' + df['genres']
df
df.drop_duplicates(subset ="movie_title", keep = 'last', inplace = True)
df<jupyter_output><empty_output><jupyter_text>Save the File<jupyter_code>df.to_csv('main_data.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16859716859716858, 'fraction_numerical': 0.05019305019305019, 'mean_word_length': 7.741573033707865, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '202104', 'license': 'Python-2.0', 'n_tokens_mistral': 357, 'n_tokens_neox': 327, 'n_words': 51, 'path': 'Recommendation/5Preprocessing_CombineData.ipynb', 'repo_name': 'MintuJohnsontj/HollywoodRecommenderSystem', 'size': '777'}