<jupyter_start><jupyter_text>Define Functions<jupyter_code>def clean(text):
    from nltk.corpus import stopwords
    from nltk.stem import WordNetLemmatizer
    
    exclude = set(string.punctuation)
    stop = set(stopwords.words('english'))
    lemma = WordNetLemmatizer()
    
    text = re.sub('[^A-z,.-]+', ' ', text)

    stop_free = ' '.join([word for word in text.lower().split() if word not in stop])
    punc_free = ''.join(ch for ch in stop_free if ch not in exclude)
    normalized = ' '.join([lemma.lemmatize(word) for word in punc_free.split()])
    return normalized.split()

def bigrams(words, bi_min=15, tri_min=10):
    bigram = gensim.models.Phrases(words, min_count = bi_min)
    bigram_mod = gensim.models.phrases.Phraser(bigram)
    return bigram_mod
    
def get_corpus(text_clean):
    bigram_mod = bigrams(text_clean)
    bigram = [bigram_mod[review] for review in text_clean]
    id2word = gensim.corpora.Dictionary(bigram)
    id2word.filter_extremes(no_below=10, no_above=0.35)
    id2word.compactify()
    corpus = [id2word.doc2bow(text) for text in bigram]
    return corpus, id2word, bigram


def compute_coherence_values(dictionary, corpus, texts, limit, start=2, step=3):
    from gensim.models.ldamodel import LdaModel
    """
    # From https://datascienceplus.com/evaluation-of-topic-modeling-topic-coherence/
    
    Compute c_v coherence for various number of topics

    Parameters:
    ----------
    dictionary : Gensim dictionary
    corpus : Gensim corpus
    texts : List of input texts
    limit : Max num of topics

    Returns:
    -------
    model_list : List of LDA topic models
    coherence_values : Coherence values corresponding to the LDA model with respective number of topics
    """
    coherence_values = []
    model_list = []
    for num_topics in range(start, limit, step):
        model=LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics, passes = 4)
        model_list.append(model)
        coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')
        coherence_values.append(coherencemodel.get_coherence())

    return model_list, coherence_values<jupyter_output><empty_output><jupyter_text>Load Preprocessed Data<jupyter_code># Load nontext data
file = open('../data/preprocessed/nontext_data.pickle', 'rb')
nontext_data = pd.read_pickle(file)
file.close()

print(nontext_data.shape)
nontext_data.head()
# Load text data
file = open('../data/preprocessed/text_no_split.pickle', 'rb')
text_data = pd.read_pickle(file)
file.close()

print(text_data.shape)
text_data.head()<jupyter_output>(7930, 11)<jupyter_text>Review text types<jupyter_code># Frequency of text-data type
text_data.type.value_counts()
# show timing of types -- drop meeting transcripts due to delay in release, press conference transcripts as only began in 2011
plt.hist(text_data[text_data.type == 'minutes'].date, alpha = 0.5, label = 'meeting minutes')
plt.hist(text_data[text_data.type == 'speech'].date, alpha = 0.5, label = 'speech')
plt.hist(text_data[text_data.type == 'statement'].date, alpha = 0.5, label = 'statement')
plt.hist(text_data[text_data.type == 'testimony'].date, alpha = 0.5, label = 'testimony')

plt.legend()<jupyter_output><empty_output><jupyter_text>Keep only Meeting Minutes, Speech, Statement, Testimony<jupyter_code>text_filtered = text_data[text_data.type.isin(['minutes', 'speech', 'statement', 'testimony'])]
text_filtered.shape<jupyter_output><empty_output><jupyter_text>Clean text<jupyter_code>text_filtered['text_clean'] = text_filtered['text'].apply(clean)

text_filtered.text_clean[0]<jupyter_output><empty_output><jupyter_text>Split into Bigrams<jupyter_code>corpus, id2word, bigram_text = get_corpus(text_filtered.text_clean)<jupyter_output><empty_output><jupyter_text>Apply LDA  Choose number of topics<jupyter_code>model_list, coherence_values = compute_coherence_values(dictionary=id2word, corpus=corpus, texts=bigram_text, start=5, limit=30, step=2)
# Coherence score plot
limit=30; start=5; step=2;
x = range(start, limit, step)
plt.plot(x, coherence_values)
plt.xlabel("Num Topics")
plt.ylabel("Coherence score")
plt.legend(("coherence_values"), loc='best')
plt.show()<jupyter_output><empty_output><jupyter_text>Choose 8 topics -- highest coherence score  Model<jupyter_code># Number of Topics
num_topics = 8
# Build LDA model
lda_model_8 = gensim.models.LdaMulticore(corpus=corpus,
                                        id2word=id2word,
                                        num_topics=num_topics, 
                                        passes=50)<jupyter_output><empty_output><jupyter_text>Words in Topics<jupyter_code>from wordcloud import WordCloud, STOPWORDS
import matplotlib.colors as mcolors

cols = [color for name, color in mcolors.TABLEAU_COLORS.items()]  # more colors: 'mcolors.XKCD_COLORS'

cloud = WordCloud(stopwords=stop_words,
                  background_color='white',
                  width=2500,
                  height=1800,
                  max_words=10,
                  colormap='tab10',
                  color_func=lambda *args, **kwargs: cols[i],
                  prefer_horizontal=1.0)

topics = lda_model.show_topics(formatted=False)

fig, axes = plt.subplots(2, 2, figsize=(10,10), sharex=True, sharey=True)

for i, ax in enumerate(axes.flatten()):
    fig.add_subplot(ax)
    topic_words = dict(topics[i][1])
    cloud.generate_from_frequencies(topic_words, max_font_size=300)
    plt.gca().imshow(cloud)
    plt.gca().set_title('Topic ' + str(i), fontdict=dict(size=16))
    plt.gca().axis('off')


plt.subplots_adjust(wspace=0, hspace=0)
plt.axis('off')
plt.margins(x=0, y=0)
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Keywords for each topic<jupyter_code># Print the Keyword in the topics
pprint(lda_model_8.print_topics())
doc_lda = lda_model_8[corpus]<jupyter_output>[(0,
  '0.008*"liquidity" + 0.007*"community" + 0.007*"borrower" + 0.007*"crisis" + '
  '0.006*"fed" + 0.006*"financial_institution" + 0.006*"lending" + '
  '0.006*"housing" + 0.005*"central_bank" + 0.005*"lender"'),
 (1,
  '0.007*"banking" + 0.007*"financial_system" + 0.005*"regulation" + '
  '0.005*"loss" + 0.005*"crisis" + 0.005*"supervisor" + 0.004*"supervisory" + '
  '0.004*"holding_company" + 0.004*"regulatory" + 0.004*"standard"'),
 (2,
  '0.011*"fomc" + 0.006*"fiscal" + 0.006*"recovery" + 0.005*"unemployment" + '
  '0.004*"budget" + 0.004*"projection" + 0.004*"recession" + 0.004*"remains" + '
  '0.004*"policymakers" + 0.003*"central_bank"'),
 (3,
  '0.005*"world" + 0.005*"technology" + 0.004*"education" + '
  '0.004*"productivity" + 0.003*"saving" + 0.003*"trade" + 0.003*"decade" + '
  '0.003*"people" + 0.003*"innovation" + 0.003*"american"'),
 (4,
  '0.022*"facility" + 0.016*"pandemic" + 0.012*"household_business" + '
  '0.012*"flow_credit" + 0.009*"public_health" + 0.009*"mar[...]<jupyter_text>Assess topic coherence<jupyter_code># Compute Perplexity
print('\nPerplexity: ', lda_model_8.log_perplexity(corpus))  # Measure of how good the model is. Lower is better.

# Compute Coherence Score
from gensim.models import CoherenceModel
coherence_model_lda = CoherenceModel(model=lda_model_8, texts=bigram_text, dictionary=id2word, coherence='c_v')
coherence_lda = coherence_model_lda.get_coherence()
print('\nCoherence Score: ', coherence_lda) # Want above .5
# VISUALIZE LDA OUTPUT AS HTML-PAGE
import pyLDAvis.gensim_models
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim_models.prepare(lda_model_8, corpus, dictionary=lda_model_8.id2word)
vis

# output results
pyLDAvis.save_html(vis, '../img/bigram.lda8-50.html')<jupyter_output><empty_output><jupyter_text>Add LDA Topics to dataframe<jupyter_code># ADD LDA TOPICS TO DATAFRAME
# REFERENCE CODE: https://towardsdatascience.com/end-to-end-topic-modeling-in-python-latent-dirichlet-allocation-lda-35ce4ed6b3e0

data = text_filtered['text_clean']
def format_topics_sentences(ldamodel=lda_model_8, corpus=corpus, texts=data):
    # Init output
    sent_topics_df = pd.DataFrame()

    # Get main topic in each document
    for i, row in enumerate(ldamodel[corpus]):
        row = sorted(row, key=lambda x: (x[1]), reverse=True)
        # Get the Dominant topic, Perc Contribution and Keywords for each document
        for j, (topic_num, prop_topic) in enumerate(row):
            if j == 0:  # => dominant topic
                wp = ldamodel.show_topic(topic_num)
                topic_keywords = ", ".join([word for word, prop in wp])
                sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num), round(prop_topic,4), topic_keywords]), ignore_index=True)
            else:
                break
    sent_topics_df.columns = ['Dominant_Topic', 'Perc_Contribution', 'Topic_Keywords']

    # Add original text to the end of the output
    contents = pd.Series(texts)
    sent_topics_df = pd.concat([sent_topics_df, contents], axis=1)
    return(sent_topics_df)

# Run function to create new df
df_topic_sents_keywords = format_topics_sentences(ldamodel=lda_model_8, corpus=corpus, texts=bigram_text)

# Format
df_dominant_topic = df_topic_sents_keywords.reset_index()
df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']

df_dominant_topic.head()

# This command prints out the topics in a text and the percent of the text from each topic

# if you want to make a list for each doc-topic pair
l = [lda_model_8.get_document_topics(item) for item in corpus]

# Create a list of lists (list of topics inside each doc)
topic_distribution = list()
for num in range(len(text_filtered['text_clean'])):
    props = [0] * 18  # create empty list
    doc = num
    for item in lda_model_8[corpus[num]]:
        topic = item[0]
        props[topic] = item[1]
    topic_distribution.append(props)
    
# Turn list of lists into dataframe
topic_dist = pd.DataFrame(topic_distribution)
# combine topics across transcript, minutes, etc. for each meeting date
final_text = text_filtered.join(topic_dist)
final_text = final_text.groupby('next_meeting').mean()

print(final_text.shape)
final_text.head()
# find na's
na_rows = final_text[final_text.isna().any(axis=1)]

print(na_rows.shape)
display(na_rows)
final_text = final_text.fillna(0)
# join LDA and nontext data
joined_data = final_text.join(nontext_data)

print(joined_data.shape)
joined_data.tail()
plt.figure(figsize=(10,6))
plt.plot(joined_data.index, joined_data.next_decision, 'o', label = 'Next Decision')
#plt.plot(joined_data.index, joined_data[0], 'o', label = 'Topic 0')
#plt.plot(joined_data.index, joined_data[1], 'o', label = 'Topic 1')
#plt.plot(joined_data.index, joined_data[2], 'o', label = 'Topic 2')
#plt.plot(joined_data.index, joined_data[3], 'o', label = 'Topic 3')
#plt.plot(joined_data.index, joined_data[4], 'o', label = 'Topic 4')
plt.plot(joined_data.index, joined_data[5], 'o', label = 'Topic 5')
#plt.plot(joined_data.index, joined_data[6], 'o', label = 'Topic 6')
#plt.plot(joined_data.index, joined_data[7], 'o', label = 'Topic 7')
plt.legend()<jupyter_output><empty_output><jupyter_text>Modify rate decision<jupyter_code># change target variable from -1 to 1 to 1 to 3
def convert_class(x):
    if x == 1:
        return 3
    elif x == 0:
        return 2
    elif x == -1:
        return 1

joined_data['RateDecision'] = joined_data.RateDecision.apply(convert_class)
joined_data['next_decision'] = joined_data['RateDecision'].shift(1)

# drop NA caused by shifting
joined_data = joined_data.dropna(subset = ['next_decision'])<jupyter_output><empty_output><jupyter_text>Write data<jupyter_code>def save_data(df, file_name, dir_name='../data/'):
    if not os.path.exists(dir_name):
        os.mkdir(dir_name)
        
    # Save results to a picke file
    file = open(dir_name + file_name + '.pickle', 'wb')
    pickle.dump(df, file)
    file.close()

    # Save results to a csv file
    df.to_csv(dir_name + file_name + '.csv', index=True)
# Final Text Data
save_data(final_text, 'text_w_lda')

# Final joined data
save_data(joined_data, 'final_fed_data')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13358147229114972, 'fraction_numerical': 0.027874276261373034, 'mean_word_length': 4.564196962724345, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 56, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '493513', 'license': 'OLDAP-2.6', 'n_tokens_mistral': 4664, 'n_tokens_neox': 4284, 'n_words': 860, 'path': 'src/5_Fed_Funds_LDA.ipynb', 'repo_name': 'michelleazee/centralbank_analysis', 'size': '12090'}