<jupyter_start><jupyter_text>Vectorize and do topic and sentiment analysis<jupyter_code>import pickle
with open('df_4_companies_prep_nostop.pickle', 'rb') as read_file:
    df_4_companies_prep=pickle.load(read_file)
df_4_companies_prep.tail(1)
import pandas as pd
import numpy as np
import re
import string
from pprint import pprint
import nltk
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.decomposition import NMF
from sklearn.metrics.pairwise import cosine_similarity
import matplotlib.pyplot as plt
%matplotlib inline
import sys
sys.path.append("/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages")
#first I need to convert the column to string datatype
df_4_companies_prep['article_words_str']=df_4_companies_prep['article_words_nostop'].astype(str)
df_4_companies_prep.head(1)
#vectorize with tf-idf. Input the string-converted column. May need to adjust min_df
#last time used min_df=0.02
corpus=df_4_companies_prep['article_words_str']
cv_tfidf = TfidfVectorizer(decode_error='ignore', strip_accents='unicode', min_df=0.02)
X_tfidf = cv_tfidf.fit_transform(corpus).toarray() 
df_4_companies_doc_term=pd.DataFrame(X_tfidf, columns=cv_tfidf.get_feature_names())
df_4_companies_doc_term.head(5)
#LSA
#Try Latent Semantic Analysis (LSA) topic modeling. Refer to Topic_Modeling_LSA_NMF notebook from class.
#LSA is just another name for Singular Value Decomposition (SVD) applied to Natural Language Processing (NLP)
#Need to specify the number of topics; I'll start with 20 and tune.
lsa = TruncatedSVD(20)
df_4_companies_doc_topic = lsa.fit_transform(X_tfidf)
lsa.explained_variance_ratio_
#LSA
#from class notebook. change name of vectorizer, round to 4 decimal places. Make for 5 components (topics).
df_4_companies_word = pd.DataFrame(lsa.components_.round(4),
             index = ["component_1","component_2", "component_3", "component_4", "component_5", 
                      "component_6", "component_7", "component_8", "component_9", "component_10",
                      "component_11", "component_12", "component_13", "component_14", "component_15", 
                     "component_16", "component_17", "component_18", "component_19", "component_20"],
             columns = cv_tfidf.get_feature_names())
df_4_companies_word
#LSA
#From notebook
def display_topics(model, feature_names, no_top_words, topic_names=None):
    for ix, topic in enumerate(model.components_):
        if not topic_names or not topic_names[ix]:
            print("\nTopic ", ix)
        else:
            print("\nTopic: '",topic_names[ix],"'")
        print(", ".join([feature_names[i]
                        for i in topic.argsort()[:-no_top_words - 1:-1]]))
#LSA
display_topics(lsa, cv_tfidf.get_feature_names(), 20)
#NMF 
nmf_model = NMF(20)
df_4_companies_NMF_doc_topic = nmf_model.fit_transform(X_tfidf)
#NMF 
df_4_companies_NMF_topic_word = pd.DataFrame(nmf_model.components_.round(4),
             index = ["component_1","component_2", "component_3", "component_4", "component_5", 
                      "component_6", "component_7", "component_8", "component_9", "component_10",
                      "component_11", "component_12", "component_13", "component_14", "component_15", 
                     "component_16", "component_17", "component_18", "component_19", "component_20"],
             columns = cv_tfidf.get_feature_names())
df_4_companies_NMF_topic_word
#NMF 
display_topics(nmf_model, cv_tfidf.get_feature_names(), 20)
#Function to plot top few words in topic based on weights
# Author: <NAME> <<EMAIL>>
#         <NAME>
#         <NAME> <<EMAIL>>
# License: BSD 3 clause

n_samples = 2000
n_features = 1000
n_components = 20
n_top_words = 20
def plot_top_words(model, feature_names, n_top_words, title):
    fig, axes = plt.subplots(4, 5, figsize=(30, 15), sharex=True)
    axes = axes.flatten()
    for topic_idx, topic in enumerate(model.components_):
        top_features_ind = topic.argsort()[:-n_top_words - 1:-1]
        top_features = [feature_names[i] for i in top_features_ind]
        weights = topic[top_features_ind]

        ax = axes[topic_idx]
        ax.barh(top_features, weights, height=0.7)
        ax.set_title(f'Topic {topic_idx +1}',
                     fontdict={'fontsize': 30})
        ax.invert_yaxis()
        ax.tick_params(axis='both', which='major', labelsize=20)
        for i in 'top right left'.split():
            ax.spines[i].set_visible(False)
        fig.suptitle(title, fontsize=40)

    plt.subplots_adjust(top=0.90, bottom=0.05, wspace=0.90, hspace=0.3)
    plt.show()



#NMF
n_top_words = 5
tf_feature_names = cv_tfidf.get_feature_names()
plot_top_words(nmf_model, tf_feature_names, n_top_words, 'Topics in NMF model')
#LSA
n_top_words = 5
tf_feature_names = cv_tfidf.get_feature_names()
plot_top_words(lsa, tf_feature_names, n_top_words, 'Topics in LSA model')
##Edit this to get data to make a word cloud
#Function to plot top few words in topic based on weights
# Author: <NAME> <<EMAIL>>
#         <NAME>
#         <NAME> <<EMAIL>>
# License: BSD 3 clause

n_samples = 2000
n_features = 1000
n_components = 20
n_top_words = 20
def plot_top_words2(model, feature_names, n_top_words, title):
    top_features_list=[]
    weights_list=[]
    fig, axes = plt.subplots(4, 5, figsize=(30, 15), sharex=True)
    axes = axes.flatten()
    for topic_idx, topic in enumerate(model.components_):
        top_features_ind = topic.argsort()[:-n_top_words - 1:-1]
        #print(top_features_ind)
        top_features = [feature_names[i] for i in top_features_ind]
        top_features_list.append(top_features)
        print(top_features)
        print(top_features_list)
        weights = topic[top_features_ind]
        weights_list.append(weights)
        print(weights)
        print(weights_list)

        #ax = axes[topic_idx]
        #ax.barh(top_features, weights, height=0.7)
        #ax.set_title(f'Topic {topic_idx +1}',
         #            fontdict={'fontsize': 30})
        #ax.invert_yaxis()
        #ax.tick_params(axis='both', which='major', labelsize=20)
        #for i in 'top right left'.split():
         #   ax.spines[i].set_visible(False)
        #fig.suptitle(title, fontsize=40)

  #  plt.subplots_adjust(top=0.90, bottom=0.05, wspace=0.90, hspace=0.3)
   # plt.show()


n_top_words = 10
tf_feature_names = cv_tfidf.get_feature_names()
plot_top_words2(lsa, tf_feature_names, n_top_words, 'Topics in LSA model')

df_topic_words_weights=pd.DataFrame(list(zip(top_features_list, weights_list)), 
                                    columns=['top_features', 'weights'])
#df_topic_words_weights.info()
print(weights_list)
#sentiment analysis with TextBlob since I can't get gensim to work
from textblob import TextBlob
#Sentiment analysis using TextBlob
#See https://medium.com/swlh/simple-sentiment-analysis-for-nlp-beginners-and-everyone-else-using-vader-and-textblob-728da3dbe33d
#Load the 'article_words_str' column as a textblob.
article_blob = [TextBlob(article) for article in df_4_companies_prep['article_words_str']]
#add the sentiment metrics to the dataframe
df_4_companies_prep['tb_Sent_Pol'] = [b.sentiment.polarity for b in article_blob]
df_4_companies_prep['tb_Sent_Subj'] = [b.sentiment.subjectivity for b in article_blob]
#show dataframe
df_4_companies_prep.head(3)
#Polarity is a float in the range -1 to 1, where -1 is most negative, and 1 is most positive
#Subjectivity is a float in the range 0 to 1, where, 0 is very objective, and 1 is very subjective
df_4_companies_prep.tail()
#pickle this dataframe. 
with open('df_4_companies_prep_tb_sentiment.pickle', 'wb') as to_write:
    pickle.dump(df_4_companies_prep, to_write)
df_4_companies_prep.to_csv('df_4_companies_prep_sentiment.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10985521971043942, 'fraction_numerical': 0.027686055372110743, 'mean_word_length': 4.28169014084507, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '213771', 'license': 'MIT', 'n_tokens_mistral': 2893, 'n_tokens_neox': 2716, 'n_words': 603, 'path': 'Notebook-2-Project-5-4-companies-vectorize-topic-sentiment.ipynb', 'repo_name': 'sutrofog/Sillman-Metis-Project-5', 'size': '7982'}