<jupyter_start><jupyter_text>Imports<jupyter_code>from __future__ import print_function
from __future__ import division

import copy
import json
import re
import string

import matplotlib
import matplotlib.pyplot as plt
import pandas as pd
import seaborn  # To improve the chart styling.
import wordtree

from IPython.display import display
from IPython.display import HTML
from IPython.display import Javascript
from wordcloud import STOPWORDS
import ipywidgets as widgets
from wordcloud import WordCloud

#import iphone_connector<jupyter_output><empty_output><jupyter_text>Load the data from disk and set up the dataframes<jupyter_code>%matplotlib inline
matplotlib.style.use('ggplot')
pd.set_option('display.max_colwidth', 1000)
df1 = pd.read_csv("Sedus – Insights Content Template - All Data.csv")
df1['Content']=df1['Content'].fillna("")
#print(df1)
#iphone_connector.initialize()

#fully_merged_messages_df, address_book_df = iphone_connector.get_cleaned_fully_merged_messages()
#full_names = set(address_book_df.full_name)  # Handy set to check for misspellings later on.
#fully_merged_messages_df.full_name.replace('nan nan nan', 'Unknown', inplace=True)

WORDS_PER_PAGE = 450  # Based upon http://wordstopages.com/
print('\nTotal pages if all texts were printed: {0:,d} (Arial size 12, single spaced)\n'.format(
    sum(df1.Content.apply(lambda x: len(x.split())))//WORDS_PER_PAGE))<jupyter_output>Total pages if all texts were printed: 24 (Arial size 12, single spaced)<jupyter_text>Diving deeper into the actual text  Visualize a word tree of texts exchanged with a specific contact<jupyter_code># Note this requires an internet connection to load Google's JS library.
def get_json_for_word_tree(contact):
    df = fully_merged_messages_df[(fully_merged_messages_df.full_name == contact)]
    print('Exchanged {0:,} texts with {1}'.format(df.shape[0], contact))
    
    array_for_json = [[text[1]] for text in df.text.iteritems()]
    array_for_json.insert(0, [['Phrases']])
    return json.dumps(array_for_json)
    
CONTACT_NAME = 'Mom'
ROOT_WORD = 'feel'
HTML(wordtree.get_word_tree_html(get_json_for_word_tree('Mom'),
                                 ROOT_WORD.lower(),
                                 lowercase=True,
                                 tree_type='double'))<jupyter_output><empty_output><jupyter_text>Preprocessing and data munging for TFIDF<jupyter_code>punctuation = copy.copy(string.punctuation)
punctuation += u'“”‘’\ufffc\uff0c'  # Include some UTF-8 punctuation that occurred.
punct_regex = re.compile(u'[{0}]'.format(punctuation))
spaces_regex = re.compile(r'\s{2,}')
numbers_regex = re.compile(r'\d+')

def clean_text(input_str):
    processed = input_str.lower()
    processed = punct_regex.sub('', processed)
    # Also try: processed = numbers_regex.sub('_NUMBER_', processed)
    processed = numbers_regex.sub('', processed)
    processed = spaces_regex.sub(' ', processed)
    
    return processed

# The normal stopwords list contains words like "i'll" which is unprocessed.
processed_stopwords = [clean_text(word) for word in STOPWORDS]<jupyter_output><empty_output><jupyter_text>Create TFIDF matrix for all contactsNote the methods below focus on texts received from these contacts, not texts you've sent to them.<jupyter_code>grouped_by_name = df1.groupby('Content_Num')['Content'].apply(lambda x: ' '.join(x)).to_frame()
#grouped_by_name.head()
#print(grouped_by_name.index)
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk import tokenize
import numpy as np

vectorizer = TfidfVectorizer(preprocessor=clean_text,
                             tokenizer=tokenize.WordPunctTokenizer().tokenize,
                             stop_words=processed_stopwords,
                             ngram_range=(1, 2), max_df=.9, max_features=50000)
tfidf_transformed_dataset = vectorizer.fit_transform(grouped_by_name.Content)
word_list = pd.Series(vectorizer.get_feature_names())

print('TFIDF sparse matrix is {0}MB'.format(tfidf_transformed_dataset.data.nbytes / 1024 / 1024))
print('TFIDF matrix has shape: {0}'.format(tfidf_transformed_dataset.shape))<jupyter_output>TFIDF sparse matrix is 0.08097076416015625MB
TFIDF matrix has shape: (142, 7980)<jupyter_text>Helper methods to leverage the TFIDF matrix<jupyter_code>def get_word_summary_for_content(content_num, top_n):
   # content = convert_unicode_to_str_if_needed(content)
    content_num = int(content_num)
    tfidf_record = _get_tfidf_record_for_content(content_num)
    if tfidf_record is None:
        print('"{0}" was not found.'.format(content_num))
        return
    sorted_indices = tfidf_record.argsort()[::-1]
    return pd.DataFrame({'Word': word_list.iloc[sorted_indices[:top_n]]}).reset_index(drop=True)

def get_word_summary_for_diffs(content_num, other_content_num, top_n=25):
    #content = convert_unicode_to_str_if_needed(content)
    content_num = int(content_num)
    #other_content = convert_unicode_to_str_if_needed(other_content)
    other_content_num = int(other_content_num)
    
    tfidf_record_content = _get_tfidf_record_for_content(content_num)
    tfidf_record_other_content = _get_tfidf_record_for_content(other_content_num)
    
    if tfidf_record_content is None or tfidf_record_other_content is None:
        # Print out the first contact not found.
        content_not_found = content if tfidf_record_content is None else other_content
        print('"{0}" was not found.'.format(content_not_found))
        return
    sorted_indices = (tfidf_record_content - tfidf_record_other_content).argsort()[::-1]
    return pd.DataFrame({'Word': word_list.iloc[sorted_indices[:top_n]]}).reset_index(drop=True)

# Returns the row in the TFIDF matrix for a given contact by name.
def _get_tfidf_record_for_content(content_num):
    if content_num not in grouped_by_name.index:
        return None
    row = np.argmax(grouped_by_name.index == content_num)
    return tfidf_transformed_dataset.getrow(row).toarray().squeeze()<jupyter_output><empty_output><jupyter_text>Words that identify a specific contact<jupyter_code>widgets.interact(
    get_word_summary_for_content,
    content_num=widgets.Text(value="1", description='Content_Num:', placeholder='Enter number'),
    top_n=widgets.IntSlider(min=5, max=100, step=1, value=5, description='Max words to show:')
)<jupyter_output><empty_output><jupyter_text>Words that identify the difference between two contacts<jupyter_code>widgets.interact(
    get_word_summary_for_diffs,
    content_num=widgets.Text(value="1", description='Content1:', placeholder='Enter number'),
    other_content_num=widgets.Text(value="2",description='Content2:', placeholder='Enter 2nd number'),
    top_n=widgets.IntSlider(description='Max words to show:', min=5, max=20, step=1, value=5)
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10214895496025905, 'fraction_numerical': 0.013982926111274654, 'mean_word_length': 5.368322399250235, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '408095', 'license': 'MIT', 'n_tokens_mistral': 2320, 'n_tokens_neox': 2223, 'n_words': 470, 'path': 'sms_analysis.ipynb', 'repo_name': 'capricec/sms-analysis', 'size': '6794'}