<jupyter_start><jupyter_text>Extracting value from the payment notes<jupyter_code>import pandas as pd
import numpy as np
import psycopg2
import pymongo
import json
import datetime
import pickle
import functions as fn
import matplotlib.pyplot as plt 
%matplotlib inline
import emoji
import regex
import nltk
from nltk import FreqDist
from nltk import word_tokenize
from nltk.corpus import stopwords, wordnet
import string
from emoji.unicode_codes import UNICODE_EMOJI as ue
from nltk.stem.wordnet import WordNetLemmatizer
import gensim
from gensim.utils import simple_preprocess
from gensim import corpora<jupyter_output><empty_output><jupyter_text>Info to access db<jupyter_code># Retrieve information to connect to the database
keys = fn.get_keys("/Users/jjherranzsarrion/.secret/local_info.json")
username = keys['username']
password = keys['password']

# Select your time ranges and add them in variables
train_window_end = '2018-08-05 23:59:59'
test_window_start = '2018-08-06 00:00:00'
test_window_end = '2018-08-07 23:59:59'
previous_day_start = '2018-08-05 00:00:00'<jupyter_output><empty_output><jupyter_text>Select all payment notes in db for the train period<jupyter_code>user_notes = fn.extract_user_notes(username, password, train_window_end)
notes = user_notes['note']<jupyter_output><empty_output><jupyter_text>Preprocess the text data<jupyter_code>recomposed_note, emoji_dict = fn.get_notes_into_unicode(notes)
recomposed_note_stopped = fn.get_clean_text_pattern(recomposed_note)
lemmatized_notes = fn.lemmatize_notes(recomposed_note_stopped)
recomposed_note_stopped_em = fn.turn_emoji_unicode_to_text(lemmatized_notes, emoji_dict)
fully_recomposed_notes = fn.emojis_to_text(recomposed_note_stopped_em)<jupyter_output><empty_output><jupyter_text>Vectorise the text using Doc2Vec method<jupyter_code># Train the model (time consuming)
tagged_data = fn.train_doc2vec_vectorizer(fully_recomposed_notes)
user_vectrs_df = fn.get_aggregated_user_note_vector(username, password, train_window_end)
user_vectrs_df.head()<jupyter_output><empty_output><jupyter_text>Using k means clustering to determine the optimal number of clusters<jupyter_code># Retrieve the distortion plot for the different number of clusters k
fn.get_distortion_plot(whole_corpus_notes)<jupyter_output><empty_output><jupyter_text>__Optimal number of k's is 6__  Using LDA model to derive the topic of each cluster<jupyter_code># Fit the model and retrieve clusters (time consuming)
lda_model = fn.get_cluster_topics_with_LDA(recomposed_note_stopped_em)<jupyter_output><empty_output><jupyter_text>The topics of the 6 clusters are:- Food- Money_with_wings emoji- Pizza emoji- Uber- Clickling beer mugs emoji- Beer mug<jupyter_code># Infer topic distributions on new, unseen documents, with
doc_lda = lda_model[doc_bow]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0708215297450425, 'fraction_numerical': 0.02160056657223796, 'mean_word_length': 8.707903780068728, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '267898', 'license': 'MIT', 'n_tokens_mistral': 1030, 'n_tokens_neox': 974, 'n_words': 231, 'path': 'feature_engineer_notes.ipynb', 'repo_name': 'fromdatavistodatascience/venmo', 'size': '2824'}