<jupyter_start><jupyter_text>Preprocessing  Imports<jupyter_code>import pandas as pd
import numpy as np
import re
import spacy

nlp = spacy.load('en_core_web_md')<jupyter_output><empty_output><jupyter_text>Load Datasets<jupyter_code>scripts_df = pd.read_csv('Data/Raw/simpsons_script_lines.csv', dtype = 'unicode')
episods_df = pd.read_csv('Data/Raw/simpsons_episodes.csv', dtype = 'unicode')

scripts_df.head()<jupyter_output><empty_output><jupyter_text>Cleanup  Drop Irrelevant Columns<jupyter_code># episods_df
episods_df.drop(columns = ['image_url', 'original_air_year', 'video_url', 'production_code', 'views'], inplace = True)

# scripts_df
scripts_df.drop(columns = ['character_id', 'location_id', 'timestamp_in_ms', 'normalized_text', 'raw_text', 'number', 'id'], inplace = True)<jupyter_output><empty_output><jupyter_text>Fix Types<jupyter_code># episodes_df
episodes_types = {
    'id': int,
    'imdb_rating': float,
    'imdb_votes': float,
    'number_in_season': int,
    'number_in_series': int,
    'season': int,
    'us_viewers_in_millions': float,    
}
episods_df = episods_df.astype(episodes_types)
episods_df['original_air_date'] = pd.to_datetime(episods_df['original_air_date'], format = '%Y-%m-%d')

# scripts_df
scripts_types = {
    'id': int,
    'episode_id': int,
    'character_id': int,
    'location_id': int,    
}
episods_df = episods_df.astype(episodes_types)<jupyter_output><empty_output><jupyter_text>More Filtering<jupyter_code># Rename columns of episods_df
episods_df.drop(columns = ['id'], inplace = True)
episods_df.rename(columns = {'imdb_rating': 'imdb', 'original_air_date': 'datetime', 'us_viewers_in_millions': 'us_viewers'}, inplace = True)

# Convert 'us_viewers' from episods_df to original numeric format
episods_df['us_viewers'] = episods_df['us_viewers'].apply(lambda x: x * (10 ** 6))

# Drop unspokon scripts from scripts_df
scripts_df = scripts_df[scripts_df['speaking_line'] == 'true']
scripts_df.drop(columns = ['speaking_line'], inplace = True)
scripts_df.rename(columns = {'spoken_words': 'raw_text', 'raw_character_text': 'character', 'raw_location_text': 'location'}, inplace = True)<jupyter_output><empty_output><jupyter_text>Text Processing<jupyter_code>def normalize_text(text):
    return ' '.join([token.lemma_ for token in nlp(text.lower()) if (not token.is_stop and not token.is_punct)])    

scripts_df['normalized_text'] = scripts_df['raw_text'].apply(normalize_text)<jupyter_output><empty_output><jupyter_text>Save Datasets<jupyter_code>scripts_df.to_csv('Data/Processed/scripts.csv', index = False)
episods_df.to_csv('Data/Processed/episodes.csv', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14333958724202628, 'fraction_numerical': 0.001125703564727955, 'mean_word_length': 7.827814569536423, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '616644', 'license': 'Apache-2.0', 'n_tokens_mistral': 1029, 'n_tokens_neox': 999, 'n_words': 157, 'path': '1.Preprocessing.ipynb', 'repo_name': 'keivanipchihagh/Fun-Text-Mining-with-Simpsons', 'size': '2665'}