<jupyter_start><jupyter_text>Questions - For each participant:     - number of messages    - words per message    - most used words    - number of emojis    - most used emojis- Sentiment analysis:    - group sentiment over time    - sentiment index of each participant    - pie chart of message sentiments    - worst and best message and from who    - wordcloud Library Imports<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
from nltk.corpus import stopwords
from textblob import TextBlob
import operator
from collections import Counter
import emoji
import requests
import json
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import asyncio
import concurrent
import re
from functools import partial
from pandas.api.types import CategoricalDtype
from datetime import datetime
from wordcloud import WordCloud
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Some details about the file.<jupyter_code>data = "data/chat.txt"
language = 'portuguese'
language_code = 'pt'
table = pd.DataFrame(columns=('date', 'time', 'sender', 'message'))<jupyter_output><empty_output><jupyter_text>Processing the file and building the dataframe:<jupyter_code>with open(data) as f:
    raw_message = ""
    counter = 0
    for line in tqdm(f):
        if line.startswith('['):
            # date
            datePattern = '(\d+/\d+/\d+)' 
            try:
                date = re.search(datePattern, raw_message).group(0)
            except AttributeError:
                date = "No date"
            
            # time
            timePattern = '(\d+:\d+:\d+)'
            try:
                time = re.search(timePattern, raw_message).group(0)
            except AttributeError:
                time = "No Time"

            # sender
            personPattern = '((?<=]).+?(?=:))'
            try:
                person = re.search(personPattern, raw_message).group(0).replace("] ", "")
            except AttributeError:
                person = "No Person"
            
            # message
            messagePattern = '(:\s).*'    
            try:
                text = re.search(messagePattern, raw_message).group(0).replace(": ", "")
            except AttributeError:
                text = "No message"
            
            table.loc[counter] = [date, time, person, text]
            counter += 1
        
            raw_message = line
        else:
            raw_message += line<jupyter_output>13524it [00:19, 700.64it/s]<jupyter_text>We clean the imports and print the head of the dataframe:<jupyter_code>table = table[~table.message.str.contains("omitted")]
table = table[~table.message.str.contains("No message")]
table = table[~table.message.str.contains("Messages to this group are")]
table = table[~table.date.str.contains("No date")]
table.head()<jupyter_output><empty_output><jupyter_text>Participant comparison<jupyter_code>table.groupby('sender').message.count().plot(kind='barh')
plt.title('Total Messages send by each participant')
table['message_size'] = table.message.str.split().str.len()
table.groupby('sender').message_size.count().plot(kind='barh')
plt.title('Words per message for each participant')
senders = list(set(table.sender))
raw_text = {}

for sender in senders: 
    raw_text[sender] = ""

for idx, row in table.iterrows():
    sender = row.sender
    message = row.message
    raw_text[sender] += message + " "
    
for sender in senders:
    # clean words and extract most used
    bad_words = stopwords.words(language)
    blob = list(TextBlob(raw_text[sender]).lower().words)
    clean_blob = [word for word in blob if word not in bad_words]
    top_words = Counter(clean_blob).most_common()[0:5]
    print(f"\nTop words for {sender}:")
    for element in top_words:
        print(f"{element[0]} with {element[1]} uses.")

    # get most used emojis
    emoji_list = []
    for item in clean_blob:
        if item in emoji.UNICODE_EMOJI:
            emoji_list.append(item)
    top_emoji = Counter(emoji_list).most_common()[0:3]
    print(f"\nTop emojis for {sender}:")
    for element in top_emoji:
        print(f"{element[0]} with {element[1]} uses.")
        
    # create a cloud of words for each participant
    print(f"\nCloud for {sender}:")
    stopwords_set = set(bad_words)
    wc = WordCloud(stopwords=stopwords_set).generate(raw_text[sender])
    plt.figure()
    plt.imshow(wc, interpolation="bilinear")
    plt.axis("off")
    plt.show()
    print('-' * 20)<jupyter_output>Top words for  Pai:
boa with 250 uses.
ló with 137 uses.
docas with 129 uses.
catocas with 116 uses.
é with 110 uses.

Top emojis for  Pai:
❤ with 11 uses.
👍 with 9 uses.
🤙 with 5 uses.

Cloud for  Pai:<jupyter_text>Sentiment Analysis<jupyter_code>def convert_to_weekday(date_string):
    day_names = ['Mon', 'Tue', 'Wed', 'Thur', 'Fri', 'Sat', 'Sun']
    datetime_object = datetime.strptime(date_string, '%d/%m/%Y')
    return day_names[datetime_object.weekday()]

cats = ['Mon', 'Tue', 'Wed', 'Thur', 'Fri', 'Sat', 'Sun']
cat_type = CategoricalDtype(categories=cats, ordered=True)

table['weekday'] = [convert_to_weekday(element) for element in table['date']]
table.groupby('weekday').message.count().reindex(cats).plot(kind='bar')
plt.title('Total messages per weekday')
sender = senders[3]
a = table[(table['sender'] == sender)]
a.groupby('weekday').message.count().reindex(cats).plot(kind='bar')
plt.title(f"Total messages per weekday for {sender}")
def happy_message(message):
    if '!' in message: 
        return True 
    else: return False

table['happy_message'] = [happy_message(message) for message in table['message']]
#table.groupby('happy_message').message.count().plot(kind='bar')
#plt.title(f"Are messages happy?")
happy_percentage = table['happy_message'].sum() / table['message'].count() * 100
sad_percentage = 100 - happy_percentage
data = [happy_percentage, sad_percentage]
labels = ['happy', 'not happy']
plt.pie(data, labels=labels, autopct='%1.1f%%')
happyness_ratios = []
for sender in senders: 
    personal = table[(table['sender']== sender)]
    happyness_ratio = personal.happy_message.sum() / personal.message.count()
    happyness_ratios.append(happyness_ratio)

plt.bar(senders, happyness_ratios)
plt.xticks(rotation=90)
plt.title('Happiness Ratios')
table['date_datetime'] = pd.to_datetime(table['date'], format='%d/%m/%Y')
table['time_datetime'] = pd.to_datetime(table['time'], format='%H:%M:%S')
# I could do mean
table.groupby(table.date_datetime.dt.month).happy_message.count().plot()
plt.title('Happy messages throughout the year')
table.groupby(table.date_datetime.dt.day).happy_message.sum().plot(label='Happy messages')
#table.groupby(table.date_datetime.dt.day).message.count().plot(label='Total messages')
plt.title('Happy messages throughout the month')
plt.legend()
table.groupby(table.date_datetime.dt.dayofweek).happy_message.sum().plot(label='Happy messages')
#table.groupby(table.date_datetime.dt.dayofweek).message.count().plot(label='Total messages')
plt.title('Happy messages throughout the week')
plt.legend()
plt.xticks(range(len(cats)), cats)
table.groupby(table.time_datetime.dt.hour).happy_message.sum().plot(label='Happy messages')
#table.groupby(table.time_datetime.dt.hour).message.count().plot(label='Total messages')
plt.title('Happy messages throughout the day')
plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13770180436847104, 'fraction_numerical': 0.00814000814000814, 'mean_word_length': 3.827766863130321, 'pattern_counts': {'":': 1, '<': 24, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '586347', 'license': 'Apache-2.0', 'n_tokens_mistral': 2432, 'n_tokens_neox': 2308, 'n_words': 538, 'path': 'analysis.ipynb', 'repo_name': 'duarteocarmo/whatsapp-visualizations', 'size': '7371'}