<jupyter_start><jupyter_text>Data Collection<jupyter_code>import tweepy
import pandas as pd
import csv
import re
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')

from wordcloud import WordCloud
import nltk 
from nltk.tokenize.toktok import ToktokTokenizer
tokenizer = ToktokTokenizer()
from nltk.corpus import stopwords
nltk.download("stopwords")
CONSUMER_KEY = "xxxxxxxxxxxxxxxx"
CONSUMER_SECRET = "<KEY>" 
ACCESS_TOKEN = "<KEY>"
ACCESS_TOKEN_SECRET = "<KEY>"

# Authenticate to Twitter
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)

# Enter your search words in accordance with the basic filtering rules
search_words = "(coronavirus OR covid OR pandemic OR covid19 OR lockdown) AND ( loneliness OR lonely OR depressed OR suicide OR sad) "

# We also want to exclude retweets and replies as this may sway results
my_search = search_words + " -filter:retweets" + " -filter:replies"  
# The Twitter data is stored in a Tweet object which we've called tweets
tweets = api.search(q=my_search,lang="en",tweet_mode="extended",count=100)
# Iterate and print tweets
i = 1
for tweet in tweets[0:20]:
    print(str(i) + ') ' + tweet.full_text + '\n')
    i = i + 1 

# Our new method of collecting the tweets
tweets = tweepy.Cursor(api.search,q=my_search,lang="en",tweet_mode='extended').items(1000)
# Extract the info we need from the tweets object
tweet_info = [[tweet.id_str,tweet.created_at,tweet.user.location,tweet.full_text] for tweet in tweets]
# Put our data into a dataframe 
df = pd.DataFrame(data=tweet_info, columns=['tweet_id_str','date_time','location','tweet_text'])

# Have a quick look at the dataframe
df
#for i,tweet in enumerate(df['tweet_text'].head(20)):
   # print(i+1, tweet, '\n')<jupyter_output><empty_output><jupyter_text>Data Cleaning and Processing<jupyter_code>def clean_text(text):
    
    """
    A function to clean the tweet text
    """
    #Remove hyper links
    text = re.sub(r'https?:\/\/\S+', ' ', text)
    
    #Remove @mentions
    text = re.sub(r'@[A-Za-z0-9]+', ' ', text)
    
    #Remove anything that isn't a letter, number, or one of the punctuation marks listed
    text = re.sub(r"[^A-Za-z0-9#'?!,.]+", ' ', text)   
    
    return text
# Apply the clean_text function to the 'tweet_text' column
df['tweet_text']=df['tweet_text'].apply(clean_text)

#for i,tweet in enumerate(df['tweet_text'].head(20)):
    #print(i+1, tweet, '\n')
df['tweet_text']=df['tweet_text'].str.lower()

# Get the list of NLTK stop words

stopwords = stopwords.words("english")
# Define our own list of stopwords
my_stopwords = ['coronavirus','covid','pandemic','covid19','lockdown','amp','via']

# Extend the nltk stopwords list
stopwords.extend(my_stopwords)
def remove_stopwords(text):
    
    """
    A function to remove stop words
    """
    
    # Tokenize the text
    tokens = tokenizer.tokenize(text)
    tokens = [token.strip() for token in tokens]
    
    # Remove stop words
    filtered_tokens = [token for token in tokens if token not in stopwords]
    filtered_text = ' '.join(filtered_tokens)    
    
    return filtered_text
# Apply the stopword removal function to the text of all tweets
df['tweet_text']=df['tweet_text'].apply(remove_stopwords)
# Plot a word cloud

all_words = ' '.join( [data for data in df['tweet_text']])
word_cloud = WordCloud(width=300, height=200, random_state=21, max_font_size = 300,
                       stopwords=stopwords).generate(all_words)

plt.figure(figsize = (20,10))
plt.imshow(word_cloud, interpolation='bilinear')
plt.axis('off')
#plt.show()
df.to_csv('Simba_tweets.csv')<jupyter_output><empty_output><jupyter_text>Using ML<jupyter_code>from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"

from simpletransformers.classification import ClassificationModel

import pandas as pd
import sklearn
from sklearn.model_selection import train_test_split
import random
import numpy as np
import torch
from sklearn.model_selection import KFold

import logging
from pathlib import Path

from sklearn.metrics import accuracy_score, confusion_matrix, f1_score, classification_report 
df2 = pd.read_csv("labelled_data_preprocessed.csv")
# Display some useful information about the data in the dataframe
df2.info()
# Drop all rows which contain a NaN, or null value in either column
df2= df2.dropna()
df2['label'].value_counts()
# make it binary classification challenge
binary_df = df2[(df2['label']==0) | (df2['label']==2) ] 

# Change the label for positive sentiment from 2 to 1
binary_df['label'] = binary_df['label'].replace(2, 1) 

binary_df['label'].value_counts()
binary_df = df2[(df2.label==0) | (df2.label==2) ] # make it binary classification
binary_df.label.replace(2,1, inplace=True) # make it binary classification

pos_samples =binary_df[binary_df['label']==1]
neg_samples = binary_df[binary_df['label']==0].sample(len(pos_samples), random_state=42)

bal_binary_df = pd.concat([pos_samples, neg_samples])

binary_df['label'].value_counts()
bal_binary_df['label'].value_counts()
binary_df.value_counts()
train_df, val_df = train_test_split(binary_df, test_size=0.2,  random_state=42)
train_df['label'].value_counts()
val_df['label'].value_counts()<jupyter_output><empty_output><jupyter_text>BERT<jupyter_code>%%time
# ___Cell no. 30___

# Build the model

bert_model = ClassificationModel('bert',
                            'bert-base-cased',
                            num_labels=2,
                            use_cuda=False,
                            args={'overwrite_output_dir': True})
%%time

# ___Cell no. 31___

# Train the model 
bert_model.train_model(train_df=train_df, eval_df=val_df)
# wrapper functions

def multi_F1(y_true, y_pred, average='macro'):
    return sklearn.metrics.f1_score(y_true=y_true, y_pred=y_pred, average=average)

def multi_classification_report(y_true, y_pred):
    return sklearn.metrics.classification_report(y_true=y_true, y_pred=y_pred)
%%time

# ___Cell no. 33___

# Calculated and print out the f1 score

result, model_outputs, wrong_predictions = bert_model.eval_model(val_df, f1=multi_F1);
print('f1 score = ',result['f1'])
%%time

# ___Cell no. 34___

# Calculated and print out the results in the classification report

result, model_outputs, wrong_predictions = bert_model.eval_model(val_df, report=multi_classification_report);
print('Classification Report: ', result['report'])
tweets = df['tweet_text']
tweets.shape
Result = bert_model.predict(tweets)
print(Result[0])
df['Bert_Sentiment'] = Result[0]
def get_sentiment_label(score):
    if score < 0:
        return 'Negative'
    elif score == 0:
        return 'Neutral'
    else:
        return 'Positive'    
df
df['Bert_sntment']=df['Bert_Sentiment'].apply(get_sentiment_label)
df
del df['tweet_id_str']
df['Bert_sntment'].value_counts().plot(kind='bar',ylabel='People',title='Mental Health Sentiment Analysis around the World',figsize=(8,8))<jupyter_output><empty_output><jupyter_text>East African Region<jupyter_code>loc = "-0.0236,37.9062,1000km"

# Search dates
date_since = "2021-07-22"
date_until = "2021-07-29"
search_words_2="(coronavirus OR covid OR pandemic OR covid19 OR lockdown OR corona OR mask ) AND ( depression OR loneliness OR lonely OR depressed OR suicide OR sad OR dryspell OR suicidal OR unhappy) "
# We also want to exclude retweets and replies as this may sway results
my_search = search_words_2 + " -filter:retweets" + " -filter:replies"  
#for i,tweet in enumerate(df['tweet_text'].head(20)):
   # print(i+1, tweet, '\n')
    
# Use the tweepy Cursor method to access tweets from a specified region and between certain dates
tweets = tweepy.Cursor(api.search,
                       q=my_search,
                       lang="en",
                       tweet_mode='extended',
                       geocode=loc,
                       since=date_since,
                       until=date_until).items(1000)

tweet_info = [[tweet.id_str,tweet.created_at,tweet.user.location,tweet.full_text] for tweet in tweets]

# Put our data into a dataframe 
df_new = pd.DataFrame(data=tweet_info, columns=['tweet_id_str','date_time','location','tweet_text'])
df_new
df_new['tweet_text']=df_new['tweet_text'].apply(clean_text)
df_new['tweet_text']=df_new['tweet_text'].str.lower()
df_new['tweet_text']=df_new['tweet_text'].apply(remove_stopwords)
all_words = ' '.join( [data for data in df_new['tweet_text']])
word_cloud = WordCloud(width=300, height=200, random_state=21, max_font_size = 300,
                       stopwords=stopwords).generate(all_words)

plt.figure(figsize = (20,10))
plt.imshow(word_cloud, interpolation='bilinear')
plt.axis('off')
plt.show()
tweets = df_new['tweet_text']
Result = bert_model.predict(tweets)
print(Result[0])
df_new['Bert_Sentiment'] = Result[0]
df_new
df_new['Bert_sntment'] = df_new['Bert_Sentiment'].apply(get_sentiment_label)
df_new
df_new['Bert_sntment']
df_new['Bert_sntment'].value_counts().plot(kind='bar',ylabel='Number of tweets',title='Bar chart of East Africa Region');
fig=plt.figure()
ax0=fig.add_subplot(1,2,1)
ax1=fig.add_subplot(1,2,2)

df['Bert_Sentiment'].value_counts().plot(kind='barh',color='blue',figsize=(10,10))
ax0.set_title('Bar chart of the World')
ax0.set_ylabel('Number of tweets')
plt.savefig('simba1', dpi=360)


df_new['Bert_Sentiment'].value_counts().plot(kind='barh',color='blue',figsize=(10,10))
ax1.set_title('Bar chart of East Africa Region')
ax1.set_ylabel('Number of tweets')

df['location'].dropna(inplace=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12540093119503362, 'fraction_numerical': 0.018210036213140195, 'mean_word_length': 5.086901763224182, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '289199', 'license': 'MIT', 'n_tokens_mistral': 3483, 'n_tokens_neox': 3303, 'n_words': 752, 'path': 'Simba Task.ipynb', 'repo_name': 'samkamau81/TEAM-SIMBA-PROJECT-', 'size': '9727'}