<jupyter_start><jupyter_text>DSCI 592 Team project based on a Kaggle Prediction Competition

 The site details are: https://www.kaggle.com/c/nlp-getting-started/data

Natural Language Processing with Disaster Tweets
Predict which Tweets are about real disasters and which ones are not  Initial Exploration of the Data  Data Grab<jupyter_code>from google.colab import drive
from google.colab import files

 
# This will prompt for authorization.
drive.mount('/content/drive', force_remount=True)
#Joe's location
DSCI_592 = '/content/drive/My Drive/DSCI_592'
%cd {DSCI_592}
!ls
# https://stackoverflow.com/questions/54351852/accessing-shared-with-me-with-colab<jupyter_output>/content/drive/.shortcut-targets-by-id/10-42_ifui6hlObPWvMMivdQwEFf6Vh43/DSCI 592
 Code			       'DSCI 592 Syllabus 2021.pdf'
 Data			        Notes.gdoc
 DSCI_592_deliverables.gdoc     Papers
'DSCI 592 Syllabus 2021.gdoc'  'Project proposal.gdoc'<jupyter_text>Data Check  Checking the train data set and displaying the 50 first rows<jupyter_code>#Joe's way of getting data
import pandas as pd
df_train = pd.read_csv ('Data/train.csv')
df_train.head(50)
# #Jenni's way since the locations are named differently...
# import pandas as pd
# df_train = pd.read_csv("/content/drive/MyDrive/DSCI 592/Data/train.csv", encoding="UTF-8")
# df_test = pd.read_csv("/content/drive/MyDrive/DSCI 592/Data/test.csv")<jupyter_output><empty_output><jupyter_text>The shape of the train data set is:<jupyter_code>df_train.shape<jupyter_output><empty_output><jupyter_text>The train data set column headers are:<jupyter_code>df_train.columns<jupyter_output><empty_output><jupyter_text>The info of train data set displays 5 columns and the count of data included<jupyter_code>df_train.info()
df_train.describe()<jupyter_output><empty_output><jupyter_text>The count of nulls by column is:<jupyter_code>df_train.isnull().sum()<jupyter_output><empty_output><jupyter_text>The count of keywords by targets 

This is to see if any of the keywords have unbalanced targets. For example Aftershock has 0 true disasters and Wreckage has 0 false disasters<jupyter_code>pd.crosstab(columns = df_train.keyword, index = df_train.target,
                     margins=True, margins_name="Total")<jupyter_output><empty_output><jupyter_text>The count of real and non real tweets is listed below, 1 is real and 0 is non real disasters<jupyter_code>df_train.target.value_counts()<jupyter_output><empty_output><jupyter_text>Plot of the real disasters vs non-real disasters in the data:<jupyter_code>import seaborn as sns
g = sns.catplot(x="target",
                #row="keyword", 
                data=df_train, kind="count")<jupyter_output><empty_output><jupyter_text>Data Cleaning The first thing to do is to make any adjustments such as cleaning up emojis and spelling errors before computing any counts.<jupyter_code>pip install emoji
import emoji

demoji_text = []

for row in df_train['text']:
  row2 = emoji.demojize(row, delimiters=(" ", " "))
  demoji_text.append(row2)

df_train['demoji_text'] = demoji_text
df_train['text_charter_count'] = df_train['text'].str.len()
print(df_train['text_charter_count'].mean())
print(df_train['text_charter_count'])
df_train[df_train.apply(lambda x: x['text'] != x['demoji_text'], axis = 1)]
pip install pyspellchecker
from spellchecker import SpellChecker

spell = SpellChecker()

# find those words that may be misspelled
misspelled = spell.unknown(['something', 'is', 'hapenning', 'here'])

for word in misspelled:
    # Get the one `most likely` answer
    print(spell.correction(word))

    # Get a list of `likely` options
    print(spell.candidates(word))<jupyter_output>happenning
{'hapening', 'happenning'}<jupyter_text>tweet analysis<jupyter_code>import numpy as np
# word_count
df_train['word_count'] = df_train['text'].apply(lambda x: len(str(x).split()))

# unique_word_count
df_train['unique_word_count'] = df_train['text'].apply(lambda x: len(set(str(x).split())))

# char_count
df_train['char_count'] = df_train['text'].apply(lambda x: len(str(x)))

import string
# punctuation_count
df_train['punctuation_count'] = df_train['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))

# hashtag_count
df_train['hashtag_count'] = df_train['text'].apply(lambda x: len([c for c in str(x) if c == '#']))

# mention_count
df_train['mention_count'] = df_train['text'].apply(lambda x: len([c for c in str(x) if c == '@']))

# url_count
df_train['url_count'] = df_train['text'].apply(lambda x: len([w for w in str(x).lower().split() if 'http' in w or 'https' in w]))

# mean_word_length
df_train['mean_word_length'] = df_train['text'].apply(lambda x: np.mean([len(w) for w in str(x).split()]))<jupyter_output><empty_output><jupyter_text>Leaving only characters and spaces<jupyter_code>#Tokenize the text 
#Note from Jenni: Moving this lower so we can make sure to make any adjustments needed first.
import nltk
nltk.download('punkt')
df_train['unigrams'] = df_train['text'].apply(nltk.word_tokenize)
df_train['unigrams_two'] = df_train.apply(lambda row: nltk.word_tokenize(row['text']), axis=1).astype(str)

# remove anything but characters and spaces
df_train['ascii_sentences'] = df_train['unigrams_two'].str.replace('[^A-z ]','').str.replace(' +',' ').str.strip()
#print(df_train.ascii_sentences)

df_train['ascii_splitwords'] = [ nltk.word_tokenize( str(sentence) ) for sentence in df_train.ascii_sentences ]
#print(df_train.ascii_splitwords)

df_train['ascii_word_charter_counts'] = [ len(words) for words in df_train.ascii_splitwords ]
#print(df_train.ascii_word_charter_counts)

df_train['ascii_word_counts'] = [ sentence.count(' ') + 1 for sentence in df_train.ascii_sentences ]
#print(df_train.ascii_word_counts)

df_train.head()

#print(max(df_train.ascii_word_charter_counts))
sns.histplot(df_train['ascii_word_and_space__counts'].astype(int),
             bins=max(df_train.ascii_word_charter_counts)).set_title('ASCII Word and Spaces Counts')

print(max(df_train.ascii_word_counts))
sns.histplot(df_train['ascii_word_counts'].astype(int),
             bins=max(df_train.ascii_word_counts)).set_title('ASCII Word Counts')<jupyter_output>33<jupyter_text>location<jupyter_code>from collections import Counter
import matplotlib.pyplot as plt

df_train['location_list'] = df_train['location'].values.tolist()

df_train['ascii_location_list'] = df_train['location_list'].str.replace('[^A-z ]','').str.replace(' +',' ').str.strip()
word_lists = df_train['ascii_location_list']
word_lists = [word_list for word_list in word_lists if str(word_list) != 'nan']
counts = Counter(word_lists)

labels, values = zip(*counts.items())

# # sort your values in descending order
idx = np.argsort(values)[3072:3102:] # top 30 locations

labels = np.array(labels)[idx]
values = np.array(values)[idx]

indexes = np.arange(len(labels))

bar_width = 0.35

plt.bar(indexes, values)
# add labels
plt.title('Location_Frequency')
plt.xticks(indexes + bar_width, labels, rotation=90)
plt.show()<jupyter_output><empty_output><jupyter_text>keyword analysis<jupyter_code>df_train['keyword_charter_count'] = df_train['keyword'].str.len()
print(df_train['keyword_charter_count'].mean())
#print(df_train['keyword_charter_count'])

# word_count
df_train['word_count_kw'] = df_train['keyword'].apply(lambda x: len(str(x).split()))

# unique_word_count
df_train['unique_word_count_kw'] = df_train['keyword'].apply(lambda x: len(set(str(x).split())))

# char_count
df_train['char_count_kw'] = df_train['keyword'].apply(lambda x: len(str(x)))

import string
# punctuation_count
df_train['punctuation_count_kw'] = df_train['keyword'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))

# hashtag_count
df_train['hashtag_count_kw'] = df_train['keyword'].apply(lambda x: len([c for c in str(x) if c == '#']))

# mention_count
df_train['mention_count_kw'] = df_train['keyword'].apply(lambda x: len([c for c in str(x) if c == '@']))

# url_count
df_train['url_count_kw'] = df_train['keyword'].apply(lambda x: len([w for w in str(x).lower().split() if 'http' in w or 'https' in w]))

# mean_word_length
df_train['mean_word_length_kw'] = df_train['keyword'].apply(lambda x: np.mean([len(w) for w in str(x).split()]))
df_train['keyword'] = df_train['keyword'].astype(str)
df_train['unigrams_kw'] = df_train['keyword'].apply(nltk.word_tokenize)


df_train['unigrams_two_kw'] = df_train.apply(lambda row: nltk.word_tokenize(row['keyword']), axis=1).astype(str)

# remove anything but characters and spaces
df_train['ascii_sentences_kw'] = df_train['unigrams_two_kw'].str.replace('[^A-z ]','').str.replace(' +',' ').str.strip()
#print(df_train.ascii_sentences_kw)

df_train['ascii_splitwords_kw'] = [ nltk.word_tokenize( str(sentence) ) for sentence in df_train.ascii_sentences_kw ]
#print(df_train.ascii_splitwords_kw)

df_train['ascii_word_charter_counts_kw'] = [ len(words) for words in df_train.ascii_splitwords_kw ]
#print(df_train.ascii_word_charter_counts_kw)

df_train['ascii_word_counts_kw'] = [ sentence.count(' ') + 1 for sentence in df_train.ascii_sentences_kw ]
#print(df_train.ascii_word_counts_kw)

#print(max(df_train.ascii_word_charter_counts_kw))
sns.histplot(df_train['ascii_word_and_spaces_counts_kw'].astype(int),
             bins=max(df_train.ascii_word_and_spaces_counts_kw)).set_title('ASCII Word and Spaces Counts Keyword')
print(max(df_train.ascii_word_counts_kw))
sns.histplot(df_train['ascii_word_counts_kw'].astype(int),
             bins=max(df_train.ascii_word_counts_kw)).set_title('ASCII Word Counts Keyword')
df_train['keyword'].value_counts(ascending=False).iloc[1:21][::-1].plot(kind='barh')
mean_by_keyword = df_train.groupby('keyword')['target'].mean().sort_values(ascending=False)[1:21][::-1]
mean_by_keyword.plot.barh()
from wordcloud import WordCloud, ImageColorGenerator
from PIL import Image
import urllib
import requests
useless_text = ''.join(text for text in df_train['text'][df_train['target']==0])
uselessful_text = ''.join(text for text in df_train['text'][df_train['target']==1])
# combine the image with dataset
Mask = np.array(Image.open(requests.get('http://clipart-library.com/image_gallery2/Twitter-PNG-Image.png', stream=True).raw))

image_colors = ImageColorGenerator(Mask)

wc1 = WordCloud(background_color='black', height=1500, width=5000, mask=Mask).generate(useless_text)
wc2 = WordCloud(background_color='black', height=1500, width=5000, mask=Mask).generate(uselessful_text)

plt.figure(figsize=(10,20))

plt.imshow(wc1.recolor(color_func=image_colors), interpolation='hamming')

plt.axis('off')
plt.show()
plt.figure(figsize=(10,20))

plt.imshow(wc2.recolor(color_func=image_colors), interpolation='hamming')

plt.axis('off')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1391870813836132, 'fraction_numerical': 0.01165244517845674, 'mean_word_length': 6.185234014502307, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 54, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '376593', 'license': 'MIT', 'n_tokens_mistral': 4135, 'n_tokens_neox': 3903, 'n_words': 735, 'path': 'npl_data_cleaning.ipynb', 'repo_name': 'jlarson200/DSCI_592_NPL', 'size': '10899'}