<jupyter_start><jupyter_text>EXAMPLE: sentiment analysis on earnings call transcripts for ticker SQ.I pulled 8 quarters of transcripts and applied some industry-standard 'NLP' or natural language processing to pull out sentence-level sentiment and phrase-level sentiment.As you'll see below, I think the stock's move in 2020 coincided with management getting more bullish on GPV.<jupyter_code>import os
import nltk
from nltk.tokenize import sent_tokenize
from textblob import TextBlob

from nltk.corpus import stopwords
nltk.download('stopwords')
from nltk.tokenize import word_tokenize

base_dir = os.getcwd()
text_dir = base_dir+"/text/SQ/"
files = os.listdir(text_dir)
# 8 quarters of transcripts
dict_files = {}
for afile in files:
    if '.txt' in afile:
        file_path = text_dir+afile
        print(file_path)
        import pandas as pd
        df = pd.read_csv(file_path,sep='delimiter', header=None, engine='python')
        df.columns = ['text']
        dict_files[afile]=df
        
#helper function
def get_sentiment(df):

    text_str = df['text'].to_string()

    text_blocks = text_str.split('\n')
    all_sentences = []

    for text_block in text_blocks:
        sentences = sent_tokenize(text_block)
        all_sentences.extend(sentences)

    list_phrases = []
    list_sentences = []
    for sentence in all_sentences:
        txt = TextBlob(sentence)
        sentiment = txt.sentiment.polarity
        subjectivity = txt.sentiment.subjectivity
        phrases = txt.noun_phrases
        data = [[sentence,sentiment,subjectivity]] 
        df_sentence = pd.DataFrame(data, columns = ['sentence', 'sentiment','subjectivity']) 
        list_sentences.append(df_sentence)
        
        for phrase in phrases:
            phrase,sentiment,subjectivity
            data = [[phrase,sentiment,subjectivity]] 
            df_phrase = pd.DataFrame(data, columns = ['phrase', 'sentiment','subjectivity']) 
            list_phrases.append(df_phrase)

    df_phrases = pd.concat(list_phrases)
    df_phrases['count']=1
    df_phrases = df_phrases.groupby('phrase').agg({'sentiment':'mean','subjectivity':'mean','count':'sum'}).reset_index()
    df_list = []
    other_stopwords = ['thanks','thank','hi','hey','yeah','well','good evening','question guys']
    for i,row in df_phrases.iterrows():
        if row['phrase'] not in stopwords.words() and row['phrase'] not in other_stopwords:
            df_phrase = pd.DataFrame(row).T
            df_list.append(df_phrase)
    df_phrases_stripped = pd.concat(df_list)
    df_phrases_stripped = df_phrases_stripped[df_phrases_stripped['sentiment']!=0].sort_values('count',ascending = False)
    
    df_sentences = pd.concat(list_sentences)
    return df_sentences,df_phrases_stripped




list_all_sentences = []
list_all_phrases = []
for key in dict_files.keys():
    df = dict_files[key]
    df_sentences,df_phrases_stripped = get_sentiment(df)
    d_date = key.split('.')[0]
    df_sentences['event_date']=d_date
    df_phrases_stripped['event_date']=d_date
    list_all_sentences.append(df_sentences)
    list_all_phrases.append(df_phrases_stripped)<jupyter_output>[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/jed.gore/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/jed.gore/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/jed.gore/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/jed.gore/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/jed.gore/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/jed.gore/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/jed.gore/nltk_data...
[nltk_data]   Package stopwords is alrea[...]<jupyter_text>Overall not much movement in aggregate sentiment and subjectivity.  Sentiment is the top chart, subjectivity on the bottom.<jupyter_code>df_sentences = pd.concat(list_all_sentences)
df_sentences.groupby('event_date').mean().reset_index().plot(x='event_date',y='sentiment',kind='bar')
df_sentences.groupby('event_date').mean().reset_index().plot(x='event_date',y='subjectivity',kind='bar')<jupyter_output><empty_output><jupyter_text>However the picture gets more interesting when we look at individual phrases.  I pulled out 'stopwords' from the phrase list to boil down to just the most common non-stopword phrases:<jupyter_code>df_phrases = pd.concat(list_all_phrases)
df_phrases['count']=df_phrases['count'].astype(float)
df_phrases.groupby('phrase').sum().sort_values('count',ascending=False)

df_phrases['sentiment']=df_phrases['sentiment'].astype(float)
df_phrases['subjectivity']=df_phrases['subjectivity'].astype(float)
df_phrases.groupby('event_date').mean().reset_index().plot(x='event_date',y='sentiment',kind='bar')
df_phrases.groupby('event_date').mean().reset_index().plot(x='event_date',y='subjectivity',kind='bar')

df_phrases['event_date']=df_phrases['event_date'].astype(str)
df_phrases[df_phrases['phrase']=='cash app'].sort_values('event_date').plot(x='event_date',y='sentiment',kind = 'bar')
df_phrases[df_phrases['phrase']=='cash app'].sort_values('event_date').plot(x='event_date',y='subjectivity',kind = 'bar')
#df_phrases[df_phrases['phrase']=='cash app'].plot(x='event_date',y='subjectivity',kind = 'bar')
df_phrases[df_phrases['phrase']=='cash card'].sort_values('event_date').plot(x='event_date',y='sentiment',kind = 'bar')
df_phrases[df_phrases['phrase']=='cash card'].sort_values('event_date').plot(x='event_date',y='subjectivity',kind = 'bar')

df_phrases[df_phrases['phrase']=='gpv'].sort_values('event_date').plot(x='event_date',y='sentiment',kind = 'bar')
df_phrases[df_phrases['phrase']=='gpv'].sort_values('event_date').plot(x='event_date',y='subjectivity',kind = 'bar')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13745148771021992, 'fraction_numerical': 0.0014553686934023287, 'mean_word_length': 5.363168724279835, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '158557', 'license': 'MIT', 'n_tokens_mistral': 2189, 'n_tokens_neox': 2143, 'n_words': 371, 'path': 'sentiment.ipynb', 'repo_name': 'reckoning-machines/steambot-stockworks', 'size': '6184'}