<jupyter_start><jupyter_text>Whatsapp Chat Analysis<jupyter_code># Importing libraries
import re
import jovian
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from wordcloud import WordCloud, STOPWORDS
import emoji
from collections import Counter
from pprint import pprint
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code>with open('WhatsApp Chat with School of AI Amritsar.txt','r') as file:
    x = file.read()
# pprint(x)
def rawToDf(file):
    with open(file, 'r') as raw_data:
        raw_string = ' '.join(raw_data.read().split('\n')) # converting the list split by newline char. as one whole string as there can be multi-line messages
        user_msg = re.split('\d{1,2}/\d{1,2}/\d{2,4},\s\d{1,2}:\d{2}\s[APap][mM]\s-\s', raw_string) [1:] # splits at all the date-time pattern, resulting in list of all the messages with user names
        date_time = re.findall('\d{1,2}/\d{1,2}/\d{2,4},\s\d{1,2}:\d{2}\s[APap][mM]\s-\s', raw_string) # finds all the date-time patterns

        df = pd.DataFrame({'date_time': date_time, 'user_msg': user_msg}) # exporting it to a df

    # converting date-time pattern which is of type String to type datetime, format is to be specified for the whole string where the placeholders are extracted by the method    
    try:
        df['date_time'] = pd.to_datetime(df['date_time'], format='%m/%d/%y, %I:%M %p - ') #10/20/19, 10:24 pm - 
    except:
        df['date_time'] = pd.to_datetime(df['date_time'], format='%d/%m/%Y, %I:%M %p - ') #20/10/2019, 10:24 pm -

    # split user and msg 
    usernames = []
    msgs = []
    for i in df['user_msg']:
        a = re.split('([\w\W]+?):\s', i) # lazy pattern match to first {user_name}: pattern and spliting it aka each msg from a user
        if(a[1:]): # user typed messages
            usernames.append(a[1])
            msgs.append(a[2])
        else: # other notifications in the group(eg: someone was added, some left ...)
            usernames.append("grp_notif")
            msgs.append(a[0])

    # creating new columns         
    df['user'] = usernames
    df['msg'] = msgs

    # dropping the old user_msg col.
    df.drop('user_msg', axis=1, inplace=True)

    return df
df = rawToDf('Group Chat.txt')
# pprint(df['user'].value_counts())
# Abc is the most active person
me = "<NAME>"
df.shape # no. of msgs<jupyter_output><empty_output><jupyter_text>No. of Images, group notifications and dropping them<jupyter_code>images = df[df['msg']=="<Media omitted> "] #no. of images, images are represented by <media omitted>
notifications = df[df['user']=='grp_notif']
print(images.shape)
print(notifications.shape)
df.drop(images.index, inplace=True) #removing images
df.drop(notifications.index, inplace=True) #removing grp_notif
df.reset_index(inplace=True, drop=True)
df.shape<jupyter_output><empty_output><jupyter_text>Who is the most active member of the group. Who is the least active. Is it same on weekday weekend?<jupyter_code># pprint(df.groupby("user")["msg"].count().sort_values(ascending=False))
df['weekday'] = df['date_time'].apply(lambda x: x.day_name()) # can use day_name or weekday from datetime 
df.weekday.value_counts(sort=True)
df['is_weekend'] = df.weekday.isin(['Sunday', 'Saturday'])
print(df['is_weekend'].value_counts())
user_list = list(df["user"].unique())
emoji_ctr = Counter()
emojis_list = map(lambda x: ''.join(x.split()), emoji.UNICODE_EMOJI.keys())
r = re.compile('|'.join(re.escape(p) for p in emojis_list))
for user in user_list:
    for idx, row in df.iterrows():
        if row["user"] == user:
            emojis_found = r.findall(row["msg"])
            for emoji_found in emojis_found:
                emoji_ctr[emoji_found] += 1
    for item in emoji_ctr.most_common(5):
#         print(user,item[0] + " - " + str(item[1]))
        print(item[0] + " - " + str(item[1]))
    print('*'*10,"Next User",'*'*10)<jupyter_output>😂 - 7
🏻 - 3
😇 - 2
🍪 - 2
✋ - 2
********** Next User **********
😂 - 13
♂ - 4
😅 - 3
🏻 - 3
🙇 - 3
********** Next User **********
😂 - 13
♂ - 4
😅 - 3
🏻 - 3
🙇 - 3
********** Next User **********
😂 - 13
🏻 - 5
♂ - 4
🙂 - 4
🙄 - 3
********** Next User **********
😂 - 13
🏻 - 5
♂ - 4
🙂 - 4
🙄 - 3
********** Next User **********
😂 - 13
🏻 - 5
♂ - 4
🙂 - 4
🙄 - 3
********** Next User **********
😂 - 14
🏻 - 9
😅 - 6
👍 - 5
♂ - 4
********** Next User **********
😂 - 14
🏻 - 9
😅 - 6
👍 - 5
♂ - 4
********** Next User **********
😂 - 14
🏻 - 9
😅 - 6
👍 - 5
♂ - 4
********** Next User **********
😂 - 14
🏻 - 11
😅 - 6
👍 - 5
✋ - 5
********** Next User **********
😂 - 14
🏻 - 13
😅 - 7
👍 - 7
✋ - 5
********** Next User **********
😂 - 14
🏻 - 13
😅 - 7
👍 - 7
✋ - 5
********** Next User **********
😂 - 14
🏻 - 13
😅 - 7
👍 - 7
✋ - 5
********** Next User **********
🏻 - 17
😂 - 14
👍 - 8
😅 - 7
✋ - 5
********** Next User **********
🏻 - 17
😂 - 14
🙏 - 10
👍 - 9
😅 - 7
********** Next User **********
🏻 - 17
😂 - 16
🙏 - 10
👍 - 9
😅 - 7
********** Next [...]<jupyter_text>What can my activity say about my sleep cycle?<jupyter_code>def to_hour(val):
    return val.hour
df['hour'] = df['date_time'].apply(to_hour)
df[df['user']==me].groupby(['hour']).size().sort_index().plot(x="hour", kind='bar')<jupyter_output><empty_output><jupyter_text>What is the difference in Weekend vs Weekday usage pattern? How many words do I type on average on weekday vs weekend?<jupyter_code>msgs_per_user = df['user'].value_counts(sort=True)
# pprint(msgs_per_user)
top5_users = msgs_per_user.index.tolist()[:5]
# pprint(top5_users)
df_top5 = df.copy()
df_top5 = df_top5[df_top5.user.isin(top5_users)]
# pprint(df_top5.head())
plt.figure(figsize=(30,10))
sns.countplot(x="user", hue="weekday", data=df_top5)
df_top5['is_weekend'] = df_top5.weekday.isin(['Sunday', 'Saturday'])
plt.figure(figsize=(20,10))
sns.countplot(x="user", hue="is_weekend", data=df_top5)
def word_count(val):
    return len(val.split())
df['no_of_words'] = df['msg'].apply(word_count)
df_top5['no_of_words'] = df_top5['msg'].apply(word_count)
total_words_weekday = df[df['is_weekend']==False]['no_of_words'].sum()
total_words_weekday
total_words_weekend = df[df['is_weekend']]['no_of_words'].sum()
total_words_weekend
total_words_weekday/5
total_words_weekend/2
# pprint(df.groupby('user')['no_of_words'].sum().sort_values(ascending=False))
# pprint(df.groupby('user')['no_of_words'].sum()/df.groupby('user').size()).sort_values(ascending=False)
# ppirnt(df_top5.groupby('user')['no_of_words'].sum()/df_top5.groupby('user').size()).sort_values(ascending=False)
wordPerMsg_weekday_vs_weekend = (df_top5.groupby(['user', 'is_weekend'])['no_of_words'].sum()/df_top5.groupby(['user', 'is_weekend']).size())
# pprint(wordPerMsg_weekday_vs_weekend)
wordPerMsg_weekday_vs_weekend.plot(kind='barh')<jupyter_output><empty_output><jupyter_text>Most Usage - Time of Day<jupyter_code>x = df.groupby(['hour', 'weekday'])['msg'].size().reset_index()
x2 = x.pivot("hour", 'weekday', 'msg')
x2.head()
days = ["Monday", 'Tuesday', "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
sns.heatmap(x2[days].fillna(0), robust=True)<jupyter_output><empty_output><jupyter_text>Which are the most common words?<jupyter_code>my_msgs_index = np.array(df[df['user']==me].index)
print(my_msgs_index, my_msgs_index.shape)
prev_msgs_index = my_msgs_index - 1
print(prev_msgs_index, prev_msgs_index.shape)
comment_words = ' '
stopwords = STOPWORDS.update(['lo', 'ge', 'Lo', 'illa', 'yea', 'ella', 'en', 'na', 'En', 'yeah', 'alli', 'ide', 'okay', 'ok', 'will'])
  
for val in df.msg.values: 
    val = str(val) 
    tokens = val.split() 
        
    for i in range(len(tokens)): 
        tokens[i] = tokens[i].lower() 
          
    for words in tokens: 
        comment_words = comment_words + words + ' '
  
  
wordcloud = WordCloud(width = 800, height = 800, 
                background_color ='black', 
                stopwords = stopwords, 
                min_font_size = 10).generate(comment_words)
plt.figure(figsize = (8, 8), facecolor = None) 
plt.imshow(wordcloud) 
plt.axis("off") 
plt.tight_layout(pad = 0) 
  
plt.show()
wordcloud.to_image()<jupyter_output><empty_output><jupyter_text>In any group, do I have any inclination towards responding to someone?<jupyter_code>my_msgs_index = np.array(df[df['user']==me].index)
print(my_msgs_index, my_msgs_index.shape)
prev_msgs_index = my_msgs_index - 1
print(prev_msgs_index, prev_msgs_index.shape)
prev_msgs_index = np.delete(prev_msgs_index, 0)
prev_msgs_index
df_replies = df.iloc[prev_msgs_index].copy()
df_replies.shape
df_replies.groupby(["user"])["msg"].size().sort_values().plot(kind='barh')
df_replies.groupby(["user"])["msg"].size().sort_values().plot(kind='barh')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2016333103289625, 'fraction_numerical': 0.023809523809523808, 'mean_word_length': 4.215956808638272, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '243213', 'license': 'MIT', 'n_tokens_mistral': 3508, 'n_tokens_neox': 3438, 'n_words': 755, 'path': 'Projects/Whatsapp Chat Analyzer/Whatsapp_chat_analysis.ipynb', 'repo_name': 'Satyam-Bhalla/Machine-Learning-Practice', 'size': '8701'}