<jupyter_start><jupyter_text>Import important libraries<jupyter_code>import pandas as pd
import numpy as np
import os 
import glob
import time 
import re
import nltk
from collections import Counter
from string import punctuation
from nltk import word_tokenize
from nltk.corpus import stopwords
from nltk.tag import StanfordPOSTagger
from nltk.stem import WordNetLemmatizer
nltk.download('punkt')
nltk.download("averaged_perceptron_tagger")<jupyter_output>[nltk_data] Downloading package punkt to /home/kidist/nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package averaged_perceptron_tagger to
[nltk_data]     /home/kidist/nltk_data...
[nltk_data]   Package averaged_perceptron_tagger is already up-to-
[nltk_data]       date!<jupyter_text>load the dataset<jupyter_code># Get all file path 

path = 'Health-Tweets'
all_files = glob.glob(path + "/*.txt")
# read dataset 
df_all = []
for filename in all_files:
    df= pd.read_csv(filename,sep='|',header=None,error_bad_lines=False,encoding="iso-8859-15",warn_bad_lines=False)
    df_all.append(df)

# merge all dataframes into a single dataframe
merged_df = pd.concat(df_all, axis=0, ignore_index=True)
merged_df.columns
# rename the columns name
merged_df.columns=["user_id","date","tweet"]
merged_df.columns
merged_df.head()
merged_df.shape<jupyter_output><empty_output><jupyter_text>The total number of tweets in the dataset is 62817  Split the texts into monthly  intervals<jupyter_code>#  Convert string date to Datetime objects
merged_df['date'] = pd.to_datetime(merged_df['date'])

# Create new column month-year. This will be used to split the data into monthly interval
def get_month_and_year(x):
    return "{}-{}".format(x.month,x.year)
merged_df["month-year"] = merged_df["date"].apply(get_month_and_year)
merged_df["tweet"][0]
# remove the tweet URL , number ,and username, because this items will not provide useful information. Specially
# numbers increase the number of unique words without any  siginificant gain in the information
def remove_url_num_uname(text):
    output = re.sub('http[s]?://\S+', '', text) 
    output = re.sub("\.{2,}","",output)
    output = re.sub('@[^\s]+','',output)
    output= re.sub("[^a-zA-Z]+"," ", output)
    return output
         

merged_df["tweet"]= merged_df["tweet"].apply(remove_url_num_uname)
merged_df["tweet"][0]

# group the dataframe into monthly interval
months_df = list(merged_df.groupby("month-year"))
months_df[21]
len(months_df)<jupyter_output><empty_output><jupyter_text>There are 47 months in the five years of data. This shows that some of the months are missing.   Text preprocessing<jupyter_code># sentence tokenization function 
def get_tokens(text):
    return word_tokenize(text)

# apply word toknization 
for month,month_df in months_df:
    month_df["tokens"] =  month_df["tweet"].apply(get_tokens)
months_df[0][1].iloc[0]["tokens"]
# function to remove stop words, panctuation , and empty string and lowercasing
stoplist = set(stopwords.words('english') + list(punctuation))
def remove_stopwords(tokens):
    output = []
    for word in tokens:
        if word not in stoplist and word.strip() != "":
            output.append(word.strip().lower())
    return output

# remove stop words, panctuation , and empty string and lowercasing
for month,month_df in months_df:
    month_df["tokens"] =  month_df["tokens"].apply(remove_stopwords)
months_df[0][1].iloc[0]["tokens"]
#  Building the POS mapper for token tags
from nltk.corpus.reader.wordnet import VERB, NOUN, ADJ, ADV
dict_pos_map = {
    # Look for NN in the POS tag because all nouns begin with NN
    'NN': NOUN,
    'VB':VERB,
    'JJ' : ADJ,
    'RB':ADV,
    'VBG':VERB
}
# getting pos information
nltk.pos_tag(months_df[0][1].iloc[0]["tokens"]) 
 # word lemitization function
def lemmitize_tokens(tokens):
    t = WordNetLemmatizer()
    outputs = []
    for word,pos in nltk.pos_tag(tokens):
        if pos in dict_pos_map:
            lemmatized_word = t.lemmatize(word, pos= dict_pos_map[pos])
            outputs.append(lemmatized_word)
        else:
            outputs.append(t.lemmatize(word))
    return outputs

# applay word lematization 
for month,month_df in months_df:
    month_df["lemm_tokens"] =  month_df["tokens"].apply(lemmitize_tokens)
months_df[0][1].head()
months_df[1][1].iloc[5]["tokens"]
months_df[1][1].iloc[5]["lemm_tokens"]<jupyter_output><empty_output><jupyter_text>Unigram language model for every month<jupyter_code># function to builed word count for each month  and find total number of words occurrence 

def build_word_count(tokens):
    word_count = Counter()
    total_word_count = 0
    for li in tokens:
        total_word_count +=len(li)
        word_count.update(li)
    return dict(word_counts = word_count,total = total_word_count)

months_words_count = dict() # Frequency of words in a month.
for month,month_df in months_df:
    counts = build_word_count(month_df["lemm_tokens"])
    months_words_count[month]= counts
     
# for each month compute the probavlities of each word 
def compute_prob(word_counts,total):
    return { word:count/total for word,count in word_counts.items()}
# get the probablity of each words of the months 
months_words_prob = dict()
for month,month_words_count in months_words_count.items():
    probs = compute_prob(**month_words_count)
    months_words_prob[month]= probs

# unigram model
class UnigramLangugeModel:
    def __init__(self,month_words_prob):
        self.words =list( month_words_prob.keys())
        self.probs = list(month_words_prob.values())
        self.words_prob = month_words_prob
    def generate_word(self):
        return np.random.choice(self.words, p = self.probs)
    def generate_sentence(self,n):
        return " ".join([self.generate_word() for i in range(n)])
    def get_word_prob(self,word):
        if not word in self.words_prob:
            return 0
        return self.words_prob[word]
months_words_prob.keys()
model = UnigramLangugeModel(months_words_prob["1-2015"]) # Unigram model for January 2015 data.

model.generate_word() # Generate single word using the Unigram model
model.generate_sentence(100) # Generate sentence with 100 words using the unigram model
# Building unigram model for each month 
lang_models = dict()
for month,month_words_count in months_words_count.items():
    model = UnigramLangugeModel(months_words_prob[month])
    lang_models[month]= model
    

lang_models.keys()

lang_models["1-2012"].generate_word()<jupyter_output><empty_output><jupyter_text>Calculate KL-divergence for these models and build pairwise representation ![image.png](attachment:image.png)  Comparing two models' probablities<jupyter_code>model_1= lang_models["1-2012"]
model_2 = lang_models["1-2013"]
model_1.get_word_prob("made")
model_2.get_word_prob("made")
### Computing KL divergance between two models
def kl_divergence(p,q):
    dkl = p*np.log(p/q)
    return dkl
    
def dkl(model_1,model_2):
    output  = 0
    for word in model_1.words:
        p = model_1.get_word_prob(word)
        q = model_2.get_word_prob(word)
        if q != 0:
            output += kl_divergence(p,q)     
        
    return output
dkl(model_1,model_2)<jupyter_output><empty_output><jupyter_text>Building pairwise matrix from the KL divergance result<jupyter_code>months = list(lang_models.keys())

pairwise_matrix = np.zeros((len(months),len(months)))
pairwise_matrix
for i in range(len(months)):
    for j in range(len(months)):
        pairwise_matrix[i,j] = dkl(lang_models[months[i]],lang_models[months[j]])
        
print(pairwise_matrix)<jupyter_output>[[0.         0.66141723 0.54392037 ... 0.28811025 0.46907418 0.73865714]
 [0.17912045 0.         0.30464566 ... 0.08876156 0.29059466 0.54169743]
 [0.12466507 0.37233368 0.         ... 0.10370461 0.11522305 0.32437324]
 ...
 [0.21528997 0.47138389 0.42181024 ... 0.         0.34383827 0.64964065]
 [0.16977653 0.47127292 0.22854255 ... 0.12713369 0.         0.43425258]
 [0.04152094 0.31527176 0.07992819 ... 0.15347459 0.07469613 0.        ]]<jupyter_text>Finding most added or removed words On average if a words was most added or removed from  month to month it will have higher variance. Here I will use standard deviation to find most variying words.<jupyter_code># find unique words 
uniq_words = set()
for month in months :
    uniq_words.update(lang_models[month].words)
    

uniq_words = list(uniq_words)
uniq_words.sort()
len(uniq_words)
months_all_words_count = np.zeros((len(uniq_words),len(months)))
months_all_words_count
for i in range(len(uniq_words)):
    for j in range(len(months)):
        if uniq_words[i] in months_words_count[months[j]]["word_counts"]:
            months_all_words_count[i,j] = months_words_count[months[j]]["word_counts"][uniq_words[i]]
    

words_stds = months_all_words_count.std(axis = 1)
topk = words_stds[words_stds.argsort()[-3:]]
topk
words_stds.max()
months_all_words_count
words_stds
uniq_words = np.array(uniq_words)
uniq_words[words_stds.argsort()[-30:]]
words_stds[words_stds.argsort()[-30:]]<jupyter_output><empty_output><jupyter_text>As it can be seen here the top varying words are `ebola`, `rt`, `healthtalk`, etc.<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11335285822757349, 'fraction_numerical': 0.039375203384315, 'mean_word_length': 4.652973635806254, 'pattern_counts': {'":': 1, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '57517', 'license': 'MIT', 'n_tokens_mistral': 3483, 'n_tokens_neox': 3196, 'n_words': 748, 'path': 'text_analysis_(1).ipynb', 'repo_name': 'kidist-amde/Text-analysis', 'size': '9219'}