<jupyter_start><jupyter_text>Dataset used: Disaster TweetsReal or Not? NLP with Disaster Tweets challenge add-onLink to dataset: https://www.kaggle.com/vstepanenko/disaster-tweets<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
! pip install kaggle
! mkdir ~/.kaggle
! cp /content/drive/MyDrive/Kaggle/kaggle.json ~/.kaggle/
! chmod 600 ~/.kaggle/kaggle.json
! kaggle datasets download vstepanenko/disaster-tweets
! mkdir disaster_tweets
! unzip disaster-tweets.zip -d disaster_tweets
import csv
import pandas as pd
import re
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>**Module 1 - Tweet Preprocessing***   Preprocessing and cleaning tweets (removing numbers, hashtags, links, punctuation) and finally tokenizing them<jupyter_code>#optional execution
with open('disaster_tweets/tweets.csv', 'r') as file:
    reader = csv.reader(file)
#converting csv file into pandas dataframe
total_data = pd.read_csv('disaster_tweets/tweets.csv')
#\w stands for "word character", usually [A-Za-z0-9_]
total_data['hashtag'] = total_data['text'].apply(lambda x: re.findall(r"#(\w+)", x))
display(total_data)
#installing ekphrasis
!pip install ekphrasis
#installing tweet-preprocessor
!pip install tweet-preprocessor
import preprocessor as p
#forming a separate feature for cleaned tweets
for i,v in enumerate(total_data['text']):
    total_data.loc[i,'cleaned'] = p.clean(v) #code was wrong, corrected now
display(total_data)
#important libraries for preprocessing using NLTK
import nltk
from nltk import word_tokenize, FreqDist
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
nltk.download
nltk.download('wordnet')
nltk.download('stopwords')
from nltk.tokenize import TweetTokenizer
def preprocess_data(data):
 #removes numbers
 data = data.astype(str).str.replace('\d+', '')
 lemmatizer = nltk.stem.WordNetLemmatizer()
 w_tokenizer =  TweetTokenizer()
 
 def lemmatize_text(text):
  return [(lemmatizer.lemmatize(w)) for w \
                       in w_tokenizer.tokenize((text))]

 def remove_punctuation(words):
  new_words = []
  for word in words:
    new_word = re.sub(r'[^\w\s]', '', (word))
    if new_word != '':
        new_words.append(new_word)
  return new_words

 words = data.apply(lemmatize_text)
 words = words.apply(remove_punctuation)
 return pd.DataFrame(words)
pre_tweets = preprocess_data(total_data['cleaned'])
total_data['cleaned'] = pre_tweets #tokenized tweets
l = [] #list containing tweets with all tokens joined
for lists in total_data['cleaned']:
  l.append(' '.join(lists))
display(total_data)<jupyter_output><empty_output><jupyter_text>**Module 2 - Automated Tweets Classifier***   Classifying which tweets are disaster relevant and which aren't*   Using Support Vector Machines Classifier (discriminative classifier)<jupyter_code>#converts a collection of text documents to a matrix of token counts
#used to transform a given text into a vector on the basis of the frequency (count) of each word that occurs in the entire text 
from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer()
x = cv.fit_transform(l).toarray()
display(x)
display(type(x))
#splitting current dataset into a train and test dataset (80%-20%)
from sklearn.model_selection import train_test_split
train_text , test_text , train_target , test_target = train_test_split(x, total_data['target'], test_size=0.2, random_state=2020)
print(type(train_text))
from sklearn import svm
model = svm.SVC(kernel='linear')

hist = model.fit(train_text, train_target)
from sklearn import metrics
#get predictions
model_pred = model.predict(test_text)

#calculate the accuracy 
model_score = metrics.accuracy_score(test_target, model_pred)

print('SVM Model Score: ', model_score)
#generate confusion matrix
from sklearn.metrics import plot_confusion_matrix

matrix = plot_confusion_matrix(hist, test_text, test_target, cmap=plt.cm.Blues, normalize='true')
plt.title('Confusion matrix for our classifier')
plt.show(matrix)
plt.show()<jupyter_output>/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)<jupyter_text>**Module 3 - Retrieval of Relevant Tweets***   Creating a list of relevant tweets based on the predictions made by the machine learning model *   Relevant tweets are those with target value = 1<jupyter_code>train_text1 , test_text1 , train_target1 , test_target1 = train_test_split(total_data, total_data['target'], test_size=0.2, random_state=2020) #split without CountVectorizing
 print(test_text1.columns.values.tolist())
 test_text_ncv = test_text1.values.tolist()
 print(type(test_text_ncv))
 print(test_text_ncv[0])
#display number of relevant predictions
count = 0
for i in model_pred :
  if i == 1 :
    count += 1
print(count)
#retrieve corresponding relevant tweets
index = 0
count = 0
rel_tweets = []
for i in model_pred :
  index += 1
  if i == 1 :
    rel_tweets.append(test_text_ncv[index][6])
    if test_text_ncv[index][4] == i:
      count = count + 1
for tweet in rel_tweets :
  tweet = ' '.join(tweet)<jupyter_output><empty_output><jupyter_text>**Module 4 - Location Based Tweets - GeoTag***   Parsing tweet text for locations (cities and countries)*   Creating a dataframe with the affected places and their corresponding tweets*   Displaying a histogram depicting the areas most prone to disasters*   Displaying a heat map depicting the areas most prone to disasters<jupyter_code>! pip install geotext
from geotext import GeoText
list_of_places = [] 
for tweet in rel_tweets :
  list_of_places.append(GeoText(' '.join(tweet)))
places = []
total = []
corresponding_tweets = []
i = 0
for place in list_of_places:
  if (len(place.countries) > 0):
    places.append(place.countries)
    corresponding_tweets.append(' '.join(rel_tweets[i]))
    for p in place.countries:
      total.append(p)

  if (len(place.cities) > 0):
    places.append(place.cities)
    corresponding_tweets.append(' '.join(rel_tweets[i]))
    for p in place.cities:
      total.append(p)
  
  i = i + 1
print(places)
print(corresponding_tweets)
df = pd.DataFrame(list(zip(places, corresponding_tweets)),
               columns =['Countries/Cities', 'Corresponding Tweets'])
display(df)
countries = []
for places in list_of_places :
  for country in places.countries :
    countries.append(country)

cities = []
for places in list_of_places :
  for city in places.cities :
    cities.append(city)
print("COUNTRIES WITH DISASTERS:")
for country in countries :
    print(country)
print("\nCITIES WITH DISASTERS:")
for city in cities :
    print(city)
df = pd.DataFrame(list(zip(countries, cities)),
               columns =['Countries', 'Cities'])
fig, ax = plt.subplots()
df['Countries'].value_counts().plot(ax=ax, kind='bar')
plt.show() #country-wise
df1 = pd.DataFrame(list(zip(total)),
               columns =['Countries/Cities'])
fig, ax = plt.subplots()
df1['Countries/Cities'].value_counts().plot(ax=ax, kind='bar')
plt.show() #countries and cities together
! pip install geopy 
! pip install Nominatim
from geopy.geocoders import Nominatim
#finding latitudes and longitudes
tot_lat = []
tot_long = []
occ_count = []
for place in total:
  address = place
  geolocator = Nominatim(user_agent="Your_Name")
  location = geolocator.geocode(address)
  tot_lat.append(location.latitude)
  tot_long.append(location.longitude)
  occ_count.append(total.count(place))
df2 = pd.DataFrame(list(zip(total, tot_lat, tot_long, occ_count)),
               columns =['Countries/Cities', 'Latitude', 'Longitude', 'Occurrence Count'])
display(df2)
! pip install basemap #drawing world map
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from itertools import chain

def draw_map(m, scale=0.2):
    #draw a shaded-relief image
    m.shadedrelief(scale=scale)
    
    #lats and longs are returned as a dictionary
    lats = m.drawparallels(np.linspace(-90, 90, 13))
    lons = m.drawmeridians(np.linspace(-180, 180, 13))

    #keys contain the plt.Line2D instances
    lat_lines = chain(*(tup[1][0] for tup in lats.items()))
    lon_lines = chain(*(tup[1][0] for tup in lons.items()))
    all_lines = chain(lat_lines, lon_lines)
    
    #cycle through these lines and set the desired style
    for line in all_lines:
        line.set(linestyle='-', alpha=0.3, color='w')
#1. draw the map background
fig = plt.figure(figsize=(20, 20), edgecolor='w')
m = Basemap(projection='cyl', resolution=None,
            llcrnrlat=-90, urcrnrlat=90,
            llcrnrlon=-180, urcrnrlon=180, )
draw_map(m)

#2. scatter city data, with color reflecting frequency of occurrence
m.scatter(df2['Longitude'].values, df2['Latitude'].values, latlon=True, 
          c = df2['Occurrence Count'].values, s = 500,
          cmap = 'Reds', alpha = 0.5)

#3. create colorbar and legend
plt.colorbar(label=r'Frequency of Occurrence')
plt.clim(3, 7)

#make legend with dummy points
for a in [100, 300, 500]:
    plt.scatter([], [], c='k', alpha=0.5, s=a,
                label=str(a) + ' km$^2$')
plt.legend(scatterpoints=1, frameon=False,
           labelspacing=1, loc='lower left');<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11332489985241409, 'fraction_numerical': 0.012966476913345983, 'mean_word_length': 4.977945809703844, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 22, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '40907', 'license': 'Apache-2.0', 'n_tokens_mistral': 3311, 'n_tokens_neox': 3158, 'n_words': 845, 'path': 'Team13_SVM.ipynb', 'repo_name': 'shrujaya/cd-proj', 'size': '9486'}