<jupyter_start><jupyter_text>Merging all JSON Files Collected<jupyter_code>import pandas as pd
import numpy as np
import json
import os

# --------- Panda Settings ------------ #
pd.set_option('display.max_rows', 1000)
pd.set_option('display.max_columns', 1000)
pd.set_option('display.max_colwidth', -1)
DATA_DIR = '../data'
BASE_MEMES = 'memefly_data.json'
BLEACHED_DATA = 'bleached_data.json'

DATA_BASE_PATH = os.path.join(DATA_DIR, BASE_MEMES)
DATA_BLEACH_PATH = os.path.join(DATA_DIR, BLEACHED_DATA)


print(DATA_BASE_PATH)
print(DATA_BLEACH_PATH)<jupyter_output>../data/memefly_data.json
../data/bleached_data.json<jupyter_text>Load base meme names<jupyter_code>base_memes = pd.read_json(DATA_BASE_PATH, orient='records', typ='frame')
base_memes.head()
base_memes.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 829 entries, 0 to 828
Data columns (total 3 columns):
meme_name            829 non-null object
meme_bounding_box    829 non-null object
meme_url             829 non-null object
dtypes: object(3)
memory usage: 19.6+ KB<jupyter_text>Load bleach data<jupyter_code>bleach_memes = pd.read_json(DATA_BLEACH_PATH, orient='records', typ='frame')
bleach_memes.head()
# Remove any straggling punctuation
import string
bleach_memes['base_meme_name'] = bleach_memes['base_meme_name'].str.translate(str.maketrans('', '', string.punctuation))
bleach_memes.sample(15)
# Edit base_meme_name column
# add - where there are spaces
bleach_memes['base_meme_name'] = bleach_memes['base_meme_name'].str.strip().str.lower().str.replace(' ', '-')
bleach_memes.sample(15)
# How many memes in each dataframe?
base_memes['meme_name'].value_counts().sum(), bleach_memes['base_meme_name'].value_counts().sum()
# Merge merged & bleach_memes
# left on meme_name, right on base_meme_name
merged = base_memes.merge(bleach_memes, 
                      left_on='meme_name', 
                      right_on='base_meme_name',
                      )
merged = merged.drop(columns='base_meme_name')
merged = merged.rename(columns={'base_meme_text':'meme_text'})
print(f'Shape of DF: {merged.shape}')
print(merged.isnull().sum())
merged.head(15)
grouped = merged.groupby('meme_name')
meme_txt_grouped = grouped['meme_text'].apply(list).to_frame().reset_index()
meme_txt_grouped.head(1)
merged_de_dups = merged.drop_duplicates(subset='meme_name', keep='first').drop(columns='meme_text')
merged_de_dups.head()
new_merged = merged_de_dups.merge(meme_txt_grouped, on='meme_name')
new_merged.head()
new_merged.info()
new_merged = new_merged.reset_index()
new_merged = new_merged.rename(columns={'index':'meme_id'})
new_merged.head()
full_path = os.path.join(DATA_DIR, 'combined_data.json')
new_merged.to_json(full_path, orient='records')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14141048824593128, 'fraction_numerical': 0.013381555153707052, 'mean_word_length': 6.020304568527918, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '574306', 'license': 'MIT', 'n_tokens_mistral': 1079, 'n_tokens_neox': 1022, 'n_words': 159, 'path': 'memefly-de/notebooks/02_merging_json_files.ipynb', 'repo_name': 'Nburkhal/memefly-ds', 'size': '2765'}