<jupyter_start><jupyter_text>Data PipelineInitial data analysis pipeline including a naive sentiment analysis using TextBlob.<jupyter_code>import re
import spacy
import pandas as pd
import numpy as np
from pathlib import Path
from string import punctuation
import matplotlib.pyplot as plt
%matplotlib inline
import calmap # for making GitHub-style calendar plots of time-series
# Plot using Pandas datatime objects
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
rc_fonts = {'figure.figsize': (15, 8),
            'axes.labelsize': 18,
            'xtick.labelsize': 18,
            'ytick.labelsize': 18,
            'legend.fontsize': 20,
           }
plt.rcParams.update(rc_fonts)
plt.style.use('ggplot')
# Scikit-learn for TF-IDF and similarity detection
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.manifold import MDS<jupyter_output><empty_output><jupyter_text>Use ```spaCy``` for tokenization and sentence segmentation<jupyter_code>import spacy
from spacy import displacy
# Load spaCy language model (blank model to which we add pipeline components)
sentencizer = spacy.blank('en')
sentencizer.add_pipe(sentencizer.create_pipe('sentencizer'))<jupyter_output><empty_output><jupyter_text>Specify named entity of interest<jupyter_code>name = "United Airlines"<jupyter_output><empty_output><jupyter_text>Write data: BooleanSpecify if we want to write the output data to csv or not.<jupyter_code>write_ = True
datafile = 'all_the_news_v2.csv'
datapath = Path('../') / 'data' / datafile 
colnames = ['title', 'author', 'date', 'content', 'year', 'month', 'publication', 'length']

news = pd.read_csv(datapath, usecols=colnames, parse_dates=['date'])
news['author'] = news['author'].str.strip()
news.head()
news = news.dropna(subset=['date', 'title'])
news.shape[0]
news['date'].describe()<jupyter_output><empty_output><jupyter_text>Filter articles based on name matchIn this section we only select those news articles that contain part of or all of the name we input as ```name```.<jupyter_code>def check_name(content, name):
    flag = False
    if name in content:
        flag = True
    return flag

def filter_df(df):
    df['match'] = df['content'].apply(lambda x: check_name(x, name))
    df_relevant = df.loc[df['match'].eq(True)]
    return df_relevant.drop(['match'], axis=1)

news_relevant = filter_df(news)
print(news_relevant.shape[0])
news_relevant.head()<jupyter_output>255<jupyter_text>Perform sentence segmentationStore the sentences in each news articles as a list of sentences, from which we can easily extract per-sentence sentiment.<jupyter_code>def get_relevant(text, name):
    doc = sentencizer(text)
    relevant = []
    for sent in doc.sents:
        for n in name.split():
            if n in sent.text:
                clean = sent.text.replace("\n", " ").replace("\xa0", " ")
                # Strip bad characters at the start of sentences
                clean = clean.strip("[\'").strip("\']").strip('\"').strip("\'\"")
                clean = clean.strip(",\'").strip("\',").strip('\"').strip("\'\"").strip()
                relevant.append(clean)
    # Remove duplicates
    relevant = list(dict.fromkeys(relevant))
    return relevant
news_relevant['relevant'] = news_relevant['content'].apply(lambda x: get_relevant(x, name))
for i in news_relevant['relevant'][:5]:
    print(i, '\n--')<jupyter_output>["The Atlantic)', '-- The United Airlines leggings controversy that took social media by storm. ("] 
--
["American Airlines has agreed to pay $200 million for a stake in China Southern Airlines, one of the country\\'s three major state-owned carriers, and expand commercial cooperation.',", "China Southern Airlines said in an announcement Tuesday through the Hong Kong stock exchange that the purchase will represent 2.76% of its shares.',", "Chinese spending on air travel rose 10% in 2015, compared with 1.7% in the United States, according to the International Air Travel Assn.',", 'Two years ago, Delta Air Lines paid $450 million for 3.55% of China Eastern Airlines.', "The third major U.S. carrier, United Airlines, has a partnership with Air China, the third major Chinese government-owned airline.',", "Regulators in both China and the United States are reluctant to allow large foreign ownership stakes or management control of their airlines.',", 'The partnership with American Airlines "i[...]<jupyter_text>Sentiment scoring using FastText<jupyter_code>import fastText
# Load trained fastText model
model_path = './fasttext_models/pretrained_yelp_review_full.ftz'<jupyter_output><empty_output><jupyter_text>Preprocess text and tokenize as per FastText requirements<jupyter_code>def fasttext_tokenize(string):
    string = string.lower()
    string = re.sub(r"([.!?,'/()])", r" \1 ", string)
    return string
# reviews = [
#     "This restaurant literally changed my life. This is the best food I've ever eaten!",
#     "I hate this place so much. They were mean to me.",
#     "I don't know. It was ok, I guess. Not really sure what to say.",
# ]

# preprocessed_reviews = list(map(strip_formatting, reviews))
# # Load classifier
# classifier = fastText.load_model(model_path)

# # Get fastText to classify each review with the model
# labels, probabilities = classifier.predict(preprocessed_reviews, 1)
# stars = [int(l[0][-1]) - 3 for l in labels]
# stars
classifier = fastText.load_model(model_path)
def get_score_fasttext(text_list):
    # Calculate polarity for each sentence
    preprocessed = list(map(fasttext_tokenize, text_list))
    labels, probabilities = classifier.predict(preprocessed, 1)
    sentiment_list = [(int(l[0][-1]) - 3)/2 for l in labels if l]
    score = np.mean(sentiment_list)
    deviation = np.std(sentiment_list)
    return score, deviation
news_relevant['score'], news_relevant['deviation'] = zip(*news_relevant['relevant'].map(get_score_fasttext))
news_relevant.head(5)<jupyter_output><empty_output><jupyter_text>Lemmatize relevant sentences for comparisonThis is to remove duplicates.<jupyter_code>add_removed_words = {n for n in name.split()}
# Include specific words to be removed
stopwords = sentencizer.Defaults.stop_words
stopwords = stopwords.union(add_removed_words)
# Tokenize and lemmatize text
def lemmatize(text):
    doc = sentencizer(text)
    tokens = [str(tok.lemma_).lower() for tok in doc if tok.text not in stopwords \
              and tok.text not in punctuation]
    return tokens
news_relevant['lemmas'] = news_relevant['relevant'].str.join(' ').apply(lemmatize).str.join(' ')
news_relevant[['relevant', 'lemmas']].head()<jupyter_output><empty_output><jupyter_text>Drop duplicates<jupyter_code>news_relevant = news_relevant.drop_duplicates(subset=['lemmas'])
news_relevant.shape[0]<jupyter_output><empty_output><jupyter_text>Positive sentiment group<jupyter_code>pos = news_relevant[news_relevant['score'] > 0.0].sort_values(by=['score'], ascending=False).reset_index(drop=True)
print("Found {} overall positive articles for {}".format(pos.shape[0], name))
pos.head(3)<jupyter_output>Found 40 overall positive articles for United Airlines<jupyter_text>Write positive results<jupyter_code>if write_:
    out_filename = '_'.join(name.split()).lower() + '_pos.csv'
    out_path = Path('./') / "results/fasttext" / out_filename
    pos.sort_values(by='publication')[['publication', 'title', 'date', 'relevant', 'score', 'deviation']] \
                    .to_csv(out_path, index=False, header=True)<jupyter_output><empty_output><jupyter_text>Negative sentiment group<jupyter_code>neg = news_relevant[news_relevant['score'] < 0.0].sort_values(by=['score']).reset_index(drop=True)
print("Found {} overall negative articles for {}".format(neg.shape[0], name))
neg.head(3)<jupyter_output>Found 190 overall negative articles for United Airlines<jupyter_text>Write negative results<jupyter_code>if write_:
    out_filename = '_'.join(name.split()).lower() + '_neg.csv'
    out_path = Path('./') / "results/fasttext" / out_filename
    neg.sort_values(by='publication')[['publication', 'title', 'date', 'relevant', 'score', 'deviation']] \
                    .to_csv(out_path, index=False, header=True)
mixed = news_relevant[news_relevant['score'] == 0.0].reset_index(drop=True)
print("Found {} overall mixed articles for {}".format(mixed.shape[0], name))
mixed.head(3)<jupyter_output>Found 15 overall mixed articles for United Airlines<jupyter_text>Highlight relevant named entities using ```spaCy```Optional step to observe the key named entities in the positive/negative or mixed sentiment articles.<jupyter_code>from IPython.display import Markdown, display
options = {'ents': ['PERSON', 'ORG', 'GPE', 'EVENT'], 
           'colors': {'PERSON': '#9fafe5', 'ORG': '#d59b9b', 'GPE':'#81cba6'}}
def printmd(string):
    display(Markdown(string))
    
def display_entities(nlp, df, max_entries=5):
    # Set relevant named entities that we want to extract
    for idx, sent in enumerate(df['relevant'].str.join(' ')[:max_entries]):
        doc = nlp(sent)
        printmd('**{}**'.format(df['title'][idx]))
        displacy.render(doc, style='ent', jupyter=True, options=options)
        print('\n')
        
def vis(pos, neg, mixed, spacy_lang='en_core_web_md'):
    nlp = spacy.load(spacy_lang)
    # Visualize positive and negativ groups using markdown
    printmd('<font color=green>**Positive**</font>')
    display_entities(nlp, pos)
    printmd('<font color=red>**Negative**</font>')
    display_entities(nlp, neg)
    printmd('<font color=yellow>**Mixed**</font>')
    display_entities(nlp, mixed)
# vis(pos, neg, mixed, spacy_lang='en_core_web_md')<jupyter_output><empty_output><jupyter_text>Visualization Plot sentiment score and magnitude versus time of publishing of the articleIn this section, sentiment "score" is the median of all polarity values (positive or negative) obtained per-sentence of the article from TextBlob. Sentiment "magnitude" is the standard deviation of sentiment among the per-sentence polarity values.<jupyter_code>news_avg_score = news_relevant.groupby('date')['score'].mean()
news_avg_dev = news_relevant.groupby('date')['deviation'].mean()<jupyter_output><empty_output><jupyter_text>Get article count per day<jupyter_code>news_count = news_relevant.groupby(['date']).count()['title']<jupyter_output><empty_output><jupyter_text>Get peak polar article per day (min negative or max positive score)<jupyter_code>news_relevant['abs'] = news_relevant['score'].abs()
news_relevant[['date', 'score', 'abs']].head(3)
news_relevant[(news_relevant['date'] > '2016-08-14') & (news_relevant['date'] < '2016-08-17')].head(10)
news_peak_polar = news_relevant.groupby('date').max()[['title', 'publication', 'relevant']]
# Extract just the first 3 relevant sentences from the article and convert to single string
news_peak_polar['relevant'] = news_peak_polar['relevant'].apply(lambda x: x[:3]).str.join(' ')
print(news_peak_polar.shape[0])<jupyter_output>133<jupyter_text>Combine scores, magnitudes and article counts per day<jupyter_code>scores = pd.concat((news_avg_score, news_avg_dev, news_count), axis=1).sort_values(by=['date'])
scores.columns = ['mean_score', 'mean_dev', 'count']
scores.head()<jupyter_output><empty_output><jupyter_text>Concatenate scores/counts DataFrame with most polar news content for that day<jupyter_code>data = pd.concat((news_peak_polar, scores), axis=1).sort_index()
data.head()<jupyter_output><empty_output><jupyter_text>Reindex data to show daily scoresSince we have really sparse data (news articles about the target are not written every day, we reindex the time series and fill missing values with zeros.<jupyter_code>idx = pd.date_range('1/1/2014', '7/5/2017')
daily = data.reindex(idx, fill_value=0.0)
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(15, 8))
ax1.fill_between(daily.index, daily['mean_score'], step='mid', color='black', alpha=0.6, linewidth=4);
ax1.set_ylabel('Mean Score', fontdict={'fontsize': 18, 'fontweight': 'bold'});
ax1.set_title('Sentiment scores and deviations with time for "{}"'.format(name), fontdict={'fontsize': 18, 'fontweight': 'bold'});
ax2.fill_between(daily.index, daily['mean_dev'], step='mid', color='black', alpha=0.6, linewidth=4);
ax2.set_ylabel('Mean Deviation', fontdict={'fontsize': 18, 'fontweight': 'bold'});
ax2.set_xlabel('Date', fontdict={'fontsize': 18, 'fontweight': 'bold'});
# Initiate a second y-axis with a shared x-axis for the article counts
ax2_2 = ax2.twinx();
ax2_2.plot(daily.index, daily['count'], 'r--', alpha=0.6, linewidth=2);
ax2_2.grid(False);
ax2_2.set_ylabel('Article Count', fontdict={'fontsize': 18, 'fontweight': 'bold'});
plt.tight_layout()
# plt.savefig("{}_scores".format('_'.join(name.split()).lower()))<jupyter_output><empty_output><jupyter_text>Make calendar plot to show periods of activity<jupyter_code>fig, axes = calmap.calendarplot(daily['mean_score'],
                    vmin = -1.0,
                    vmax=1.0,
                    daylabels='MTWTFSS',
                    dayticks=[0, 2, 4, 6],
                    fig_kws=dict(figsize=(12.5, 9)),
                    linewidth=1,
                    fillcolor='lightgrey',
                    cmap='coolwarm_r',
                   );
fig.suptitle("Calendar map of aggregated sentiment for {}".format(name), fontsize=18);
fig.tight_layout(rect=[0, 0.03, 1, 0.95])

if write_:
    out_filename = '_'.join(name.split()).lower()
    plt.savefig('calmap_{}.png'.format(out_filename))<jupyter_output>/home/pprao/.local/lib/python3.6/site-packages/matplotlib/font_manager.py:1241: UserWarning: findfont: Font family ['Arial'] not found. Falling back to DejaVu Sans.
  (prop.get_family(), self.defaultFamily[fontext]))<jupyter_text>Get counts of positive and negative mentions based on Publication<jupyter_code>grouped = news_relevant.groupby('publication').apply(lambda x: x['score'] >= 0.0)
grouped = grouped.groupby('publication').value_counts().to_frame()
grouped = grouped.unstack().fillna(0.0)
grouped.columns = ['Negative', 'Positive']
grouped = grouped.sort_values(by='Negative')
grouped<jupyter_output><empty_output><jupyter_text>Plot article breakdown<jupyter_code>grouped.plot(kind='barh', figsize=(12, 8));
plt.title('Count of number of articles with Positive/Negative Sentiment for {}'.format(name));
plt.ylabel('');
# plt.savefig("{}_breakdown".format('_'.join(name.split()).lower()))<jupyter_output><empty_output><jupyter_text>Output results to CSV<jupyter_code>if write_:
    out_filename = '_'.join(name.split()).lower() + '_breakdown.csv'
    out_path = Path('./') / "results/fasttext" / out_filename
    grouped.to_csv(out_path, header=True)
if write_: 
    data_filename = '_'.join(name.split()).lower() + '_data.csv'
    data_path = Path('./') / "results/fasttext" / data_filename
    daily[~daily['relevant'].eq(0)].to_csv(data_path, header=True)<jupyter_output><empty_output><jupyter_text>Visualize Cosine Similarity DistancesTo see how similar or different each article is based on publication, we can compute the cosine distances between articles to generate a "distance matrix" and then visualize these distances in two-dimensional space.  Calculate TF-IDF for document similarityWe first define the term frequency-inverse document frequency to vectorize the text for each article into parameters, and generate a ```tf-idf``` matrix. Once we compute the ```tf-idf``` matrix, we can find a "distance matrix" that stores how similar or how different two documents are.<jupyter_code># Define vectorizer parameters
tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=200000,
                                 min_df=0.2)
# Get TF-IDF matrix
tfidf_matrix = tfidf_vectorizer.fit_transform(news_relevant['lemmas'] ) #fit the vectorizer to synopses
print(tfidf_matrix.shape)

# Display some key terms
terms = tfidf_vectorizer.get_feature_names()
print(terms)
# Get cosine distance matrix
dist = 1 - cosine_similarity(tfidf_matrix)<jupyter_output><empty_output><jupyter_text>Multidimensional Scaling (MDS)The computed distances are in multi-dimensional in nature. To visualize the similarity, we "embed" the cosine distances (from the distance matrix) to a two-dimensional space, which we can then plot to see how the articles compare with each other in terms of their content.<jupyter_code>embedding = MDS(n_components=2, dissimilarity="precomputed", random_state=37)
dist_transformed = embedding.fit_transform(dist)
print(dist_transformed.shape)

xs, ys = dist_transformed[:, 0], dist_transformed[:, 1]<jupyter_output>(245, 2)<jupyter_text>Generate an MDS DataFrame for plottingWe combine the x-y distances from the MDS calculation with the original publication labels to see how different the articles are from each other, colored by publication.<jupyter_code>compare = pd.DataFrame(dict(label=news_relevant['publication'], x=xs, y=ys))
compare.head()
L = news_relevant['publication'].nunique()
print("Found {} unique categories for publications".format(L))
groups = compare.groupby('label').agg({'label': 'count', 'x': 'mean', 'y': 'mean'})
groups.columns = ['count', 'x', 'y']
groups = groups.sort_values(by='count')
groups<jupyter_output><empty_output><jupyter_text>Visualize similarities as embedded cosine distances<jupyter_code>fig, ax = plt.subplots(figsize=(12, 9))

colors = [i for i in range(len(groups.index))]

ax.scatter(groups['x'], groups['y'], c=colors, 
            s=groups['count']*100, linewidths=1.5, alpha=0.5,
            edgecolors='k', cmap=plt.cm.gist_rainbow,
            );

for i, txt in enumerate(groups.index):
    ax.annotate(txt, (groups['x'][i], groups['y'][i]),
                fontsize=18, alpha=0.7);
ax.set_xticklabels(['']);
ax.set_yticklabels(['']);
plt.tight_layout()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13708181051447849, 'fraction_numerical': 0.013100927748102334, 'mean_word_length': 5.413992066354129, 'pattern_counts': {'":': 0, '<': 117, '<?xml version=': 0, '>': 118, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '29', 'dir': 'jupyter-structured-clean-dedup', 'id': '628955', 'license': 'MIT', 'n_tokens_mistral': 5867, 'n_tokens_neox': 5542, 'n_words': 1431, 'path': 'assets/notebooks/analysis_workflow_fasttext.ipynb', 'repo_name': 'andrewjwesson/news-sentiment-tracker', 'size': '17785'}