<jupyter_start><jupyter_text>DMML TEAM BLANCPAIN<jupyter_code>%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import collections  as mc
%load_ext autoreload
%autoreload 2
import pandas as pd 
import seaborn as sns
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
sns.set_style("white")<jupyter_output><empty_output><jupyter_text>**Data Importation and visualization**  **Load Data**<jupyter_code>df_train = pd.read_csv('https://raw.githubusercontent.com/sfrancey/Real-or-Not-NLP-with-Disaster-Tweets_Team_Blancpain/main/Data/training_data.csv')

df_test = pd.read_csv('https://raw.githubusercontent.com/sfrancey/Real-or-Not-NLP-with-Disaster-Tweets_Team_Blancpain/main/Data/test_data.csv')

df_sample = pd.read_csv('https://raw.githubusercontent.com/sfrancey/Real-or-Not-NLP-with-Disaster-Tweets_Team_Blancpain/main/Data/sample_submission.csv')<jupyter_output><empty_output><jupyter_text>Train Model<jupyter_code>df_train.head(10)
print("There are {0} rows and {1} columns in the train dataset.".format(df_train.shape[0],df_train.shape[1]))<jupyter_output>There are 6471 rows and 5 columns in the train dataset.<jupyter_text>Test Model<jupyter_code>df_test.head(10)
print("There are {0} rows and {1} columns in the test dataset.".format(df_test.shape[0],df_test.shape[1]))<jupyter_output>There are 1142 rows and 4 columns in the test dataset.<jupyter_text>Missing values:<jupyter_code>missingTrain = df_train.isnull().sum(axis = 0)
print(missingTrain)
#missing_train.plot.bar(x='lab', y='val', rot=0, color='plum', title='Missing Values in Train set')
missingTest = df_test.isnull().sum(axis = 0)
print(missingTest)
#missing_test.plot.bar(x='lab', y='val', rot=0, color='lightblue', title='Missing Values in Test set')
missingKeywordsTrain = df_train.keyword.isnull().sum(axis = 0)
missingKeywordsTest = df_test.keyword.isnull().sum(axis = 0)

#plot NaN proportion for Keyword
plt.rcParams['figure.figsize'] = (8, 6)
x = np.arange(2)
plt.bar(x,[missingKeywordsTrain, missingKeywordsTest], color=['darkorange','lightblue'])
plt.ylabel('Number of inputs')
plt.title('Missing Keywords')
plt.xticks(x, ('Train_set', 'Test_set'), rotation=20)
plt.show()

missingLocationTrain = df_train.location.isnull().sum(axis = 0)
missingLocatioTtest = df_test.location.isnull().sum(axis = 0)

#plot NaN proportion for Keyword
plt.rcParams['figure.figsize'] = (8, 6)
x = np.arange(2)
plt.bar(x,[missingLocationTrain, missingLocatioTtest], color=['darkorange','lightblue'])
plt.ylabel('Number of inputs')
plt.title('Missing Location')
plt.xticks(x, ('Train_set', 'Test_set'), rotation=20)
plt.show()<jupyter_output><empty_output><jupyter_text>Check the distribution:  Identify the proportion between real disasters and not real disasters in the train set.<jupyter_code>realDisaster = df_train[df_train['target'] == 1].shape[0]
notRealDisaster = df_train[df_train['target'] == 0].shape[0]
#Plot the result
plt.rcParams['figure.figsize'] = (8, 6)
x = np.arange(2)
plt.bar(x,[realDisaster, notRealDisaster], color=['seagreen','tomato'])
plt.ylabel('Number of inputs')
plt.title('Number of Tweets by targets')
plt.xticks(x, ('real_natural_disaster', 'not_natural_disaster'), rotation=20)
plt.show()<jupyter_output><empty_output><jupyter_text>Identify which keywords are most related with a real disaster and which one are not.<jupyter_code># Top 15 for real Natural disasters

keywordRealTop = df_train['keyword'].loc[(df_train['target'] == 1)].value_counts()[0:15]

# Top 15 for not real Natural disasters

keywordNotRealTop = df_train['keyword'].loc[(df_train['target'] == 0)].value_counts()[0:15]
#PLot top 15 real
keywordRealTop.plot.bar(xlabel = 'Keywords', 
                          ylabel = 'Number', 
                          title = 'Top 15 Keywords in Real Disaster', 
                          figsize=(15, 5), 
                          color = 'lightgoldenrodyellow')
#PLot top 15 not real
keywordNotRealTop.plot.bar(xlabel = 'Keywords', 
                          ylabel = 'Number', 
                          title = 'Top 15 Keywords in Not Real Disaster',
                          figsize=(15, 5), 
                          color = 'darkseagreen')
#Identify which Keywords are mainly related to a specific target and which ones are ambigous 

df_train['targetM'] = df_train.groupby('keyword')['target'].transform('mean')

figKey = plt.figure(figsize=(8, 60), dpi=100)

sns.countplot(y=df_train.sort_values(by='targetM', ascending=False)['keyword'],
              hue=df_train.sort_values(by='targetM', ascending=False)['target'],palette="Set3")

plt.legend()
plt.title('Target Distribution regarding Keywords')

plt.show()
df_train_selected = df_train[(df_train['targetM'] > 0.9) | (df_train['targetM'] < 0.1)]

figKey = plt.figure(figsize=(8, 60), dpi=100)

sns.countplot(y=df_train_selected.sort_values(by='targetM', ascending=False)['keyword'],
              hue=df_train_selected.sort_values(by='targetM', ascending=False)['target'],palette="Set3")

plt.legend()
plt.title('Target Distribution regarding Keywords')

plt.show()
keywords_list = df_train_selected['keyword'].unique()
file_name = 'keywords_list.pkl'

import pickle

def save_file_pkl(file, file_name):
  # Store data (serialize)
  with open(file_name, 'wb') as handle:
      pickle.dump(file, handle, protocol=pickle.HIGHEST_PROTOCOL)

def load_file_pkl(file_name):
  # Load data (deserialize)
  with open(file_name, 'rb') as handle:
    return pickle.load(handle)

save_file_pkl(keywords_list, file_name)

df_train.drop(columns=['targetM'], inplace=True)
list(keywords_list)<jupyter_output><empty_output><jupyter_text>Analyzing the "location" feature<jupyter_code>df_train.location.value_counts()[0:20]
# Top 15 for real disaster
locationRealTop = df_train['location'].loc[(df_train['target'] == 1)].value_counts()[0:15]
# Top 15 for not real disaster
locationNotRealTop = df_train['location'].loc[(df_train['target'] == 0)].value_counts()[0:15]

locationRealTop.plot.bar(xlabel = 'Location', 
                          ylabel = 'Number', 
                          title = 'Top 15 Location in Real Disaster', 
                          figsize=(15, 5), 
                          color = 'lightblue')
locationNotRealTop.plot.bar(xlabel = 'Location', 
                          ylabel = 'Number', 
                          title = 'Top 15 Location in Not Real Disaster', 
                          figsize=(15, 5), 
                          color = 'lightpink')<jupyter_output><empty_output><jupyter_text>EDA on "text" feature<jupyter_code>#number of characters in a Tweet 
fig,(ax1,ax2) = plt.subplots(1,2,figsize=(10,5))

tweetRealTrain = df_train[df_train['target']==1]['text'].str.len()
ax1.hist(tweetRealTrain,color='plum')
ax1.set_title('Natural Disaster tweets')

tweetNotRealTrain = df_train[df_train['target']==0]['text'].str.len()
ax2.hist(tweetNotRealTrain,color='lightblue')
ax2.set_title('Not natural disaster tweets')
fig.suptitle('Number of Characters in tweets')
plt.show()
#number of words in a Tweet 
fig,(ax1,ax2)=plt.subplots(1,2,figsize=(10,5))

tweetReal = df_train[df_train['target']==1]['text'].str.split().map(lambda x: len(x))
ax1.hist(tweetReal,color='plum')
ax1.set_title('Disaster tweets')

tweetNotReal = df_train[df_train['target']==0]['text'].str.split().map(lambda x: len(x))
ax2.hist(tweetNotReal,color='lightblue')
ax2.set_title('Not disaster tweets')
fig.suptitle('Number of Words in a tweet')

plt.show()
#Find most common words on Tweets 
import nltk
from nltk import tokenize

tokenSpace = tokenize.WhitespaceTokenizer()

def counter_common_words(text, columnText, quantity, title):
    allWords = ' '.join([text for text in text[columnText]])
    tokenizedPhrase = tokenSpace.tokenize(allWords)
    frequency = nltk.FreqDist(tokenizedPhrase) 
    dfFrequency = pd.DataFrame({"Word": list(frequency.keys()), "Frequency": list(frequency.values())}) 
    dfFrequency = dfFrequency.nlargest(columns = "Frequency", n = quantity)
    
    plt.figure(figsize=(12,8))
    ax = sns.barplot(data = dfFrequency, x = "Word", y = "Frequency", palette="Set3")
    ax.set(ylabel = "Count")
    plt.title(title)
    plt.xticks(rotation='horizontal')
    plt.show()
counter_common_words(df_train[df_train['target'] == 1], 'text', 20, 'Most common words on real natural disaster')
counter_common_words(df_train[df_train['target'] == 0], 'text', 20, 'Most common words on not real natural disaster')<jupyter_output><empty_output><jupyter_text>Bigrame<jupyter_code>import spacy
import string

# Load English language model
sp = spacy.load('en_core_web_sm')

# Create a list of punctuation marks
punctuations = string.punctuation

# Create a list of stopwords
stop_words = spacy.lang.en.stop_words.STOP_WORDS

def tokenize_function(text):
    # Create token object, which is used to create documents with linguistic annotations.
    sp_obj = sp(text)

    # Lemmatize each token and convert each token into lowercase
    mytokens = [ word.lemma_.lower().strip() if word.lemma_ != "-PRON-" else word.lower_ for word in sp_obj ]

    ## Remove stop words and punctuation
    #mytokens = [ word for word in mytokens if word not in stop_words and word not in punctuations ]
    
    return mytokens
from nltk.util import ngrams
import collections

corpus = []
for text in [list(df_train['text']), list(df_test['text'])]:
  for string in text:
    corpus.extend(tokenize_function(string))

collections.Counter(ngrams(corpus, 1)).most_common(30)

Bigrams = ngrams(corpus, 2)

# get the frequency of each bigram in our corpus
esBigramFreq = collections.Counter(Bigrams)

# what are the ten most popular ngrams in this Spanish corpus?
esBigramFreq.most_common(30)<jupyter_output><empty_output><jupyter_text>Data cleaning  What we want to do: On Keywords: - Removing spelling errors (%20, spelling...)- RegroupmentsOn Tweets: - Spelling correction - Remove Http - Remove HTML - Remove punctuation - Remove - Remove @- Remove stopwords On Locations: - Harmonize<jupyter_code># Cleaning locations 

df_train["location"].replace(["Adelaide, Australia"],"Adelaide, AU", inplace = True)
df_train["location"].replace(["Alaska"],"AK, USA", inplace = True)
df_train["location"].replace(["Ashburn, VA"],"Ashburn, VA, USA", inplace = True)
df_train["location"].replace(["Atlanta, GA","Atlanta", "Atlanta Georgia"],"Atlanta, GA, USA", inplace = True)
df_train["location"].replace(["Austin, TX"],"Austin, TX, USA", inplace = True)
df_train["location"].replace(["Bakersfield, California"],"Bakersfield, CA, USA", inplace = True)
df_train["location"].replace(["Barbados"],"BB", inplace = True)
df_train["location"].replace(["Bend, Oregon"],"Bend, OR, USA", inplace = True)
df_train["location"].replace(["Berlin - Germany"],"Berlin, GER", inplace = True)
df_train["location"].replace(["Birmingham"],"Birmingham, UK", inplace = True)
df_train["location"].replace(["ÌÏT: 10.614817868480726,12.195582811791382"],"Biu, NG", inplace = True)
df_train["location"].replace(["Boise, Idaho"],"Boise, ID, USA", inplace = True)
df_train["location"].replace(["Boston, MA"],"Boston, MA, USA", inplace = True)
df_train["location"].replace(["Brazil"],"Brazil, BA", inplace = True)
df_train["location"].replace(["British Columbia, Canada"],"British_Columbia, CAN", inplace = True)
df_train["location"].replace(["Brooklyn, NY"],"Brooklyn, NY, USA", inplace = True)
df_train["location"].replace(["Budapest, Hungary"],"Budapest, HUN", inplace = True)
df_train["location"].replace(["Bukittinggi  ?? Sumatera Barat"],"Bukittinggi, INA", inplace = True)
df_train["location"].replace(["Canada"],"Canada, CAN", inplace = True)
df_train["location"].replace(["California", "California, USA", "California, United States"],"CA, USA", inplace = True)
df_train["location"].replace(["Calcary, AB, Canada", "Calgary", 'Calgary, Alberta', 'Calgary, AB, Canada'],"Calcary, CAN", inplace = True)
df_train["location"].replace(["Cape Town"],"Cape Town, SA", inplace = True)
df_train["location"].replace(["Charlotte, NC"],"Charlotte, NC, USA", inplace = True)
df_train["location"].replace(["Chicago, IL, USA", 'Chicago'],"Chicago, IL, USA", inplace = True)
df_train["location"].replace(["china", 'China'],"China, CN", inplace = True)
df_train["location"].replace(["Colorado"],"CO, USA", inplace = True)
df_train["location"].replace(["Coventry"],"Coventry, UK", inplace = True)
df_train["location"].replace(["Dallas, TX", 'Street of Dallas'],"Dallas, TX, USA", inplace = True)
df_train["location"].replace(["Decatur, GA"],"Decatur, GA, USA", inplace = True)
df_train["location"].replace(["Cherry Creek Denver CO", 'Denver, Colorado', 'Denver, CO'],"Denver, CO, USA", inplace = True)
df_train["location"].replace(["Derby"],"Derby, UK", inplace = True)
df_train["location"].replace(["DOMINICANA REPUBLICA"],"Dominican Republic, DO", inplace = True)
df_train["location"].replace(["Edinburgh"],"Edinburgh, UK", inplace = True)
df_train["location"].replace(["El Dorado, Arkansas"],"El Dorado, AR, USA", inplace = True)
df_train["location"].replace(["Erie, PA"],"Erie, PA, USA", inplace = True)
df_train["location"].replace(["Florida"],"FL, USA", inplace = True)
df_train["location"].replace(["Geneva"],"Geneva, CH", inplace = True)
df_train["location"].replace(["Germany"],"Germany, GER", inplace = True)
df_train["location"].replace(["Glasgow"],"Glasgow, UK", inplace = True)
df_train["location"].replace(["Haddonfield, NJ"],"Haddonfield, NJ, USA", inplace = True)
df_train["location"].replace(["Hawaii, USA", 'Ewa Beach, HI'],"Hawaii, HI, USA", inplace = True)
df_train["location"].replace(["Hawaii, USA","Ewa Beach, HI"],"Hawaii, HI, USA", inplace = True)
df_train["location"].replace(["Hong Kong"],"Hong Kong, CN", inplace = True)
df_train["location"].replace(["Houston, TX","Houston TX"],"Houston, TX, USA", inplace = True)
df_train["location"].replace(["Ireland"],"Ireland, IE", inplace = True)
df_train["location"].replace(["Indiana"],"IN, USA", inplace = True)
df_train["location"].replace(["India"],"India, IND", inplace = True)
df_train["location"].replace(["Indonesia"],"Indonesia, INA", inplace = True)
df_train["location"].replace(["Inexpressible Island"],"Inexpressible Island, ANT", inplace = True)
df_train["location"].replace(["Italy"],"Italy, IT", inplace = True)
df_train["location"].replace(["Kansas City"],"Kansas City, MO, USA", inplace = True)
df_train["location"].replace(["Karachi, Pakistan","Karachi"],"Karachi, PK", inplace = True)
df_train["location"].replace(["Kashmir!"],"Kashmir, IND", inplace = True)
df_train["location"].replace(["Kenya"],"Kenya, KY", inplace = True)
df_train["location"].replace(["Kingston, Jamaica"],"Kingston, JAM", inplace = True)
df_train["location"].replace(["Korea"],"Korea, KR", inplace = True)
df_train["location"].replace(["Louisiana"],"LA, USA", inplace = True)
df_train["location"].replace(["Lagos, Nigeria","Lagos"],"Lagos, NG", inplace = True)
df_train["location"].replace(["Leeds, England"],"Leeds, UK", inplace = True)
df_train["location"].replace(["London","London, England","London, UK","london"],"London, UK", inplace = True)
df_train["location"].replace(["Los Angeles","Los Angeles, CA","Subconscious LA"],"Los Angeles, CA, USA", inplace = True)
df_train["location"].replace(["Massachusetts"],"MA, USA", inplace = True)
df_train["location"].replace(["Manchester, NH"],"Manchester, NH, USA", inplace = True)
df_train["location"].replace(["Manchester","Manchester, England"],"Manchester, UK", inplace = True)
df_train["location"].replace(["Maryland"],"MD, USA", inplace = True)
df_train["location"].replace(["Melbourne","Melbourne, Australia"],"Melbourne, AU", inplace = True)
df_train["location"].replace(["Memphis, TN"],"Memphis, TN, USA", inplace = True)
df_train["location"].replace(["19.600858, -99.047821"],"Mexico City, MX", inplace = True)
df_train["location"].replace(["Michigan"],"MI, USA", inplace = True)
df_train["location"].replace(["Miami, FL"],"Miami, FL, USA", inplace = True)
df_train["location"].replace(["swmo"],"MO, USA", inplace = True)
df_train["location"].replace(["Morioh, Japan"],"Morioh, JP", inplace = True)
df_train["location"].replace(["Mumbai"],"Mumbai, IND", inplace = True)
df_train["location"].replace(["Muntinlupa City, Philippines"],"Muntinlupa City, PH", inplace = True)
df_train["location"].replace(["Nairobi"],"Nairobi, KY", inplace = True)
df_train["location"].replace(["Nashville, TN"],"Nashville, TN, USA", inplace = True)
df_train["location"].replace(["North Carolina","North Carolina, USA","NC"],"NC, USA", inplace = True)
df_train["location"].replace(["New Delhi, India"],"New Delhi, IND", inplace = True)
df_train["location"].replace(["New Hanover County, NC"],"New Hanover County, NC, USA ", inplace = True)
df_train["location"].replace(["New Jersey"],"NJ, USA", inplace = True)
df_train["location"].replace(["New Orleans, LA"],"New Orleans, LA, USA", inplace = True)
df_train["location"].replace(["New York","NYC","New York City","New York, NY","New York, USA"],"New York, NY, USA", inplace = True)
df_train["location"].replace(["Newark, NJ"],"Newark, NJ, USA", inplace = True)
df_train["location"].replace(["New Hampshire"],"NH, USA", inplace = True)
df_train["location"].replace(["Nigeria"],"Nigeria, NG", inplace = True)
df_train["location"].replace(["NJ"],"NJ, USA", inplace = True)
df_train["location"].replace(["107-18 79TH STREET"],"NY, USA", inplace = True)
df_train["location"].replace(["Oakland, CA"],"Oakland, CA, USA", inplace = True)
df_train["location"].replace(["Oklahoma City, OK","Oklahoma City"],"Oklahoma City, OK, USA", inplace = True)
df_train["location"].replace(["Oregon","Oregon, USA"],"OR, USA", inplace = True)
df_train["location"].replace(["Orlando, FL"],"Orlando, FL, USA", inplace = True)
df_train["location"].replace(["Paterson, New Jersey"],"Paterson, NJ, USA", inplace = True)
df_train["location"].replace(["Pennsylvania, USA"],"Pennsylvania, PA, USA", inplace = True)
df_train["location"].replace(["Phoenix, AZ"],"Phoenix, AZ, USA", inplace = True)
df_train["location"].replace(["Portland, Oregon"],"Portland, OR, USA", inplace = True)
df_train["location"].replace(["Raleigh, NC"],"Raleigh, NC, USA", inplace = True)
df_train["location"].replace(["Pakistan"],"PK", inplace = True)
df_train["location"].replace(["Redding, California, USA"],"Redding, CA, USA", inplace = True)
df_train["location"].replace(["Renfrew, Scotland"],"Renfrew, UK", inplace = True)
df_train["location"].replace(["Rio de Janeiro"],"Rio de Janeiro, BR", inplace = True)
df_train["location"].replace(["Sacramento, CA","Sacramento"],"Sacramento, CA, USA", inplace = True)
df_train["location"].replace(["San Diego, CA"],"San Diego, CA, USA", inplace = True)
df_train["location"].replace(["San Francisco","San Francisco, CA","San Francisco Bay Area"],"San Francisco, CA, USA", inplace = True)
df_train["location"].replace(["Seattle, WA","Seattle","Seattle, Washington"],"Seattle, WA, USA", inplace = True)
df_train["location"].replace(["Singapore"],"Singapore, SG	Singapore", inplace = True)
df_train["location"].replace(["St Paul, MN"],"St Paul, MN, USA", inplace = True)
df_train["location"].replace(["State College, PA"],"State College, PA, USA", inplace = True)
df_train["location"].replace(["Stockholm, Sweden"],"Stockholm, SWE", inplace = True)
df_train["location"].replace(["Sweden"],"SWE", inplace = True)
df_train["location"].replace(["Sydney, Australia","Sydney"],"Sydney, AU", inplace = True)
df_train["location"].replace(["The Netherlands"],"NL", inplace = True)
df_train["location"].replace(["Tennessee"],"TN, USA", inplace = True)
df_train["location"].replace(["Tokyo"],"Tokyo, JP", inplace = True)
df_train["location"].replace(["Toronto","Toronto, Ontario"],"Toronto, CA", inplace = True)
df_train["location"].replace(["Texas","Republic of Texas"],"TX, USA", inplace = True)
df_train["location"].replace(["United Kingdom","#FLIGHTCITY UK"],"UK", inplace = True)
df_train["location"].replace(["United States","United States, USA","US","South, USA","Cleveland, OH - San Diego, CA","USA (Formerly @usNOAAgov)"],"USA", inplace = True)
df_train["location"].replace(["In the potters hands","Jupiter","Heaven","?????","Breaking News","??","???  Dreamz","Instagram - @heyimginog","Somewhere Only We Know ?","Gotham City","South Stand","?????","???"],"unknown", inplace = True)
df_train["location"].replace(["Virginia"],"VA, USA", inplace = True)
df_train["location"].replace(["Victoria, Australia, Earth"],"Victoria, AUS", inplace = True)
df_train["location"].replace(["Victoria, BC"],"Victoria, CAN", inplace = True)
df_train["location"].replace(["Washington","Washington DC","Washington D.C.","Washington, D.C","Washington, DC"],"Washington DC, USA", inplace = True)
df_train["location"].replace(["Wisconsin"],"WI, USA", inplace = True)
df_train["location"].replace(["Earth","WorldWide","Everywhere","World","world","Global-NoLocation","World Wide","in the Word of God"],"Worldwide", inplace = True)
df_train["location"].replace(["Yogya Berhati Nyaman"],"Yogya Berhati Nyaman, ID", inplace = True)
# Cleaning Keywords on train set

df_train["keyword"].replace(["bush%20fires","forest%20fires","wild%20fires","hellfire","forest%20fire"],"natural_fires", inplace = True)
df_train["keyword"].replace(["suicide%20bomber","suicide%20bombing","suicide%20bomb"],"suicide_bombing", inplace = True)
df_train["keyword"].replace(["building%20fires","buildings%20burning","buildings%20on%20fire","burning%20buildings"],"building_fires", inplace = True)
df_train["keyword"].replace(["blew%20up","blown%20up"],"blew_up", inplace = True)
df_train["keyword"].replace(["structural%20failure"],"structural_failure", inplace = True)
df_train["keyword"].replace(["natural%20disaster"],"natural_disaster", inplace = True)
df_train["keyword"].replace(["body%20bags","body%20bagging","body%20bag"],"body_bags", inplace = True)
df_train["keyword"].replace(["emergency%20services"],"emergency_services", inplace = True)
df_train["keyword"].replace(["dust%20storm"],"dust_storm", inplace = True)
df_train["keyword"].replace(["heat%20wave"],"heat_wave", inplace = True)
df_train["keyword"].replace(["mass%20murder","mass%20murderer"],"mass_murder", inplace = True)
df_train["keyword"].replace(["war%20zone"],"war_zone", inplace = True)
df_train["keyword"].replace(["emergency%20plan"],"emergency_plan", inplace = True)
df_train["keyword"].replace(["nuclear%20reactor","nuclear%20disaster"],"nuclear", inplace = True)
df_train["keyword"].replace(["airplane%20accident"],"airplane_accident", inplace = True)
df_train["keyword"].replace(["chemical%20emergency"],"chemical_emergency", inplace = True)
df_train["keyword"].replace(["radiation%20emergency"],"radiation_emergency", inplace = True)
df_train["keyword"].replace(["oil%20spill"],"oil_spill", inplace = True)
df_train["keyword"].replace(["loud%20bang"],"loud_bang", inplace = True)
df_train["keyword"].replace(["cliff%20fall"],"cliff_fall", inplace = True)
df_train["keyword"].replace(["bridge%20collapse"],"bridge_collapse", inplace = True)
df_train["keyword"].replace(["fire%20truck"],"fire_truck", inplace = True)
df_train["keyword"].replace(["violent%20storm"],"violent_storm", inplace = True)
df_train["keyword"].replace(["first%20responders"],"first_responders", inplace = True)
# Cleaning Keywords on test set

df_test["keyword"].replace(["bush%20fires","forest%20fires","wild%20fires","hellfire","forest%20fire"],"natural_fires", inplace = True)
df_test["keyword"].replace(["suicide%20bomber","suicide%20bombing","suicide%20bomb"],"suicide_bombing", inplace = True)
df_test["keyword"].replace(["building%20fires","buildings%20burning","buildings%20on%20fire","burning%20buildings"],"building_fires", inplace = True)
df_test["keyword"].replace(["blew%20up","blown%20up"],"blew_up", inplace = True)
df_test["keyword"].replace(["structural%20failure"],"structural_failure", inplace = True)
df_test["keyword"].replace(["natural%20disaster"],"natural_disaster", inplace = True)
df_test["keyword"].replace(["body%20bags","body%20bagging","body%20bag"],"body_bags", inplace = True)
df_test["keyword"].replace(["emergency%20services"],"emergency_services", inplace = True)
df_test["keyword"].replace(["dust%20storm"],"dust_storm", inplace = True)
df_test["keyword"].replace(["heat%20wave"],"heat_wave", inplace = True)
df_test["keyword"].replace(["mass%20murder","mass%20murderer"],"mass_murder", inplace = True)
df_test["keyword"].replace(["war%20zone"],"war_zone", inplace = True)
df_test["keyword"].replace(["emergency%20plan"],"emergency_plan", inplace = True)
df_test["keyword"].replace(["nuclear%20reactor","nuclear%20disaster"],"nuclear", inplace = True)
df_test["keyword"].replace(["airplane%20accident"],"airplane_accident", inplace = True)
df_test["keyword"].replace(["chemical%20emergency"],"chemical_emergency", inplace = True)
df_test["keyword"].replace(["radiation%20emergency"],"radiation_emergency", inplace = True)
df_test["keyword"].replace(["oil%20spill"],"oil_spill", inplace = True)
df_test["keyword"].replace(["loud%20bang"],"loud_bang", inplace = True)
df_test["keyword"].replace(["cliff%20fall"],"cliff_fall", inplace = True)
df_test["keyword"].replace(["bridge%20collapse"],"bridge_collapse", inplace = True)
df_test["keyword"].replace(["fire%20truck"],"fire_truck", inplace = True)
df_test["keyword"].replace(["violent%20storm"],"violent_storm", inplace = True)
df_test["keyword"].replace(["first%20responders"],"first_responders", inplace = True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1884684968142846, 'fraction_numerical': 0.0146306929914261, 'mean_word_length': 7.604737732656514, 'pattern_counts': {'":': 2, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '51828', 'license': 'MIT', 'n_tokens_mistral': 9000, 'n_tokens_neox': 8421, 'n_words': 1623, 'path': 'Code/EDA_team_blancpainTweets.ipynb', 'repo_name': 'sfrancey/Real-or-Not-NLP-with-Disaster-Tweets_Team_Blancpain', 'size': '25426'}