<jupyter_start><jupyter_text>Load and merge the dataThe source is Kaggle, [IMDB Spoiler Dataset](https://www.kaggle.com/rmisra/imdb-spoiler-dataset). The downloaded zip includes two files, we will demonstrate some of their contents below.<jupyter_code>df_reviews = pd.read_json('./data/imdb/IMDB_reviews.json', lines=True).\
             drop_duplicates('review_text').sample(frac=1)
df_reviews.review_date = pd.to_datetime(df_reviews.review_date, infer_datetime_format=True)
df_reviews.user_id = df_reviews.user_id.astype('category')
print(df_reviews.shape)
review_fields = [field for field in df_reviews]
df_reviews.tail()
# the total number of spoilers in dataset
num_spoilers = sum(df_reviews.is_spoiler)
print('num of spoilers in all: %d\nration of spoilers in all: %.4f\na dummy classifier can achieve acc of %.4f' \
      % (num_spoilers, num_spoilers / df_reviews.shape[0], \
         1 - num_spoilers / df_reviews.shape[0]))
# only less than 20% of the user comments' explicitly say that they contain spoilers
num_notice = df_reviews.review_text.str.contains('spoiler', case=False).sum() / \
             df_reviews.is_spoiler.sum()
print('a rough estimation of how many users declare their reviews as spoilers: %.4f' % num_notice)
tokenizer = nlp.data.SpacyTokenizer('en')
def get_word_count(x):
    return len(tokenizer(x))

with mp.Pool() as pool:
    length_list = pool.map(get_word_count, df_reviews.review_text)
print('the average number of words in a review is:', round(sum(length_list) / df_reviews.shape[0]))
# imdb information about the movies
df_movies = pd.read_json('./data/imdb/IMDB_movie_details.json', lines=True)
df_movies.release_date = pd.to_datetime(df_movies.release_date, infer_datetime_format=True)
print(df_movies.shape)
df_movies.tail()
with mp.Pool() as pool:
    length_list = pool.map(get_word_count, df_movies.plot_synopsis)
print('the average number of words in a plot synopsis is %d' % \
      round(sum(length_list) / df_movies.shape[0]))
# join the two dataframe according to movie id
df_reviews = df_reviews.merge(df_movies, on="movie_id", how="left", suffixes=('_review','_movie'))
df_train = pd.DataFrame(columns=['movie_id', 'sentence1', 'sentence2', 'label'])

df_train['movie_id'] = df_reviews['movie_id']

summaries = pd.Series([re.split('\s*Written by\s*\n', str(plot))[0] for plot in df_reviews['plot_summary']])
synopses = pd.Series([str(synopsis) for synopsis in df_reviews['plot_synopsis']])
'''
num=0
for synopsis in synopses:
    if len(synopsis)>10:
        num+=1
print(num)
-> output: 538318, most of them has synopses
'''
df_train['sentence1'] = pd.Series([synopses[i] if len(synopses[i])>len(summaries[i]) else summaries[i] \
                                  for i in range(len(synopses))])
df_train['sentence2'] = df_reviews['review_summary'] + ' ' + df_reviews['review_text']
df_train['label'] = df_reviews['is_spoiler']
df_train.tail()
df_train['movie_id'][233], df_train['sentence1'][233]<jupyter_output><empty_output><jupyter_text>Save the processed data for future use<jupyter_code>print('# of rows before dropping incomplete entries:', df_train.shape[0])
dt_train=df_train.dropna()
print('# of rows after dropping incomplete entries:', df_train.shape[0])
df_train.to_csv('./data/imdb/train.csv', index=False)
df_train.head()<jupyter_output># of rows before dropping incomplete entries: 573385
# of rows after dropping incomplete entries: 573385
{'fraction_non_alphanumeric': 0.1333138001757984, 'fraction_numerical': 0.01377087606211544, 'mean_word_length': 5.603481624758221, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '340257', 'license': 'Apache-2.0', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1187, 'n_words': 270, 'path': 'imdb_loader.ipynb', 'repo_name': 'StevenBirdBrown/spoiler-detector', 'size': '3413'}