<jupyter_start><jupyter_text>NLP   TF-IDF and Sentiment Analysis on quotes The goal of this notebook is to perform Tf-Idf and Sentiment analysis on the given articles, focusing only on quotes. The original "NLP Manana" notebook analyzes the whole article.<jupyter_code>import seaborn as sns
import csv
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
sns.set()
sns.set_context("talk")
import re

#uploading a small data to work with
short = pd.read_csv('short.csv')

#dropping unnamed column
short = short.drop(["Unnamed: 0"], axis=1)

#short.head()
#BY TITLE
#Check how many articles are relevant to Bitcoin/crypto, and track indices of articles that are not

#Keywords to look for - This can be expanded
keywords = ["Bitcoin", "bitcoin", "Crypto", "crypto"] 

irrelevant_article_indices = [] #To keep track of indices of articles with no keywords

for article_index in range(0, len(short.index)):
    article = short['title'][article_index] #Grab the contents of the current article
    num_keywords = 0 #Initialize keyword counter
    
    for keyword in keywords: #This loop counts the number of keywords from the list that appear in the article
        if keyword in article:
            num_keywords += 1 
            
    if num_keywords > 0:
        print("The title had a keyword in it in article", article_index)
        
    if num_keywords == 0: #If there are no keywords in the article, indicate that and track the index
        print("Did not find any keywords in the title of article", article_index)
        irrelevant_article_indices.append(article_index)
        
        #print("Checking the body of article", article_index)
        #article = short['contents'][article_index] #Grab the contents of the current article
    
        #for keyword in keywords: 
         #   if keyword in article:
          #      print("The word", keyword, "appears in article", article_index, ", \"", short['title'][article_index],
           #           "\"," "in this context: \n")
            #    print(article[(article.find(keyword) - 100):(article.find(keyword) + 100)])
             #   print()
    

print("Out of", len(short.index), "articles,", 
      len(irrelevant_article_indices), "have nothing to do with Bitcoin or cryptocurrency.")
#BY CONTENTS
#Check how many articles are relevant to Bitcoin/crypto, and track indices of articles that are not

keywords = ["Bitcoin", "bitcoin", "Cryptocurrency", "cryptocurrency"] #Keywords to look for - This can be expanded
irrelevant_article_indices = [] #To keep track of indices of articles with no keywords

for article_index in range(0, len(short.index)):
    article = short['contents'][article_index] #Grab the contents of the current article
    num_keywords = 0 #Initialize keyword counter
    
    for keyword in keywords: #This loop counts the number of keywords from the list that appear in the article
        if keyword in article:
            num_keywords += 1 
        
    if num_keywords == 0: #If there are no keywords in the article, indicate that and track the index
        print("Did not find any keywords in article", article_index)
        irrelevant_article_indices.append(article_index)
        
    

print("Out of", len(short.index), "articles,", 
      len(irrelevant_article_indices), "have nothing to do with Bitcoin or cryptocurrency.")
#Remove irrelevant articles from master CSV

for index in irrelevant_article_indices:
    short = short.drop([index], axis=0)
    
short = short.reset_index()
short = short.drop(['index'], axis = 1)

short.head()
#Find the number of times each keyword appears in each article

keywords = ["Bitcoin", "bitcoin", "Cryptocurrency", "cryptocurrency"]
counts = []

for article_index in range(0, len(short.index)):
    keyword_count = 0
    article = short['contents'][article_index]
    
    for keyword in keywords:
        keyword_count += article.split().count(keyword)
        print(keyword, "appears", keyword_count, "times in piece number", article_index)

    if keyword_count == 0:
        print("No keywords in article", article_index)
        #print("Dropping article", article_index)
        #short = short.drop([article_index])
    else:
        counts.append(keyword_count)
    
print("Stats about keyword counts:")
print("Average per article:", np.mean(counts))
print("Median:", np.median(counts))
print("Min:", np.min(counts))
print("Max:", np.max(counts))
print("SD:", np.std(counts))
short['contents'][1]
#Grab all quotes from each article (aligning the indices)
quotes = [] #Initialize array
for article_index in range(0, len(short.index)):
    list = re.findall('"([^"]*)"', short['contents'][article_index])
    quotes.append(list)

print(quotes)
short['contents'][10]
#performing Tf-idf
all_documents= list(removed_punct)
from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer(norm='l2',min_df=0, use_idf=True, smooth_idf=False, sublinear_tf=True)
tfidf_feat = tfidf.fit_transform(all_documents)

tfidf_feat.toarray()<jupyter_output><empty_output><jupyter_text>Sentiment Analysis Uses vader_lexicon for performing lexicon alaysis.<jupyter_code>#using lexicon-sentiment dictionary (not financial though)
!head vader_lexicon.txt
#reading the sentiments
sentiments = pd.read_table('vader_lexicon.txt', names=['token', 'polarity','SD','human ratings'])
sentiments.tail(30)
sent = sentiments.drop(['SD','human ratings'], axis=1).set_index('token')
sent
# lm_sent = pd.read_csv('lm.csv')
# lm_sent.iloc[45]
# Save your regex in punct_re- getting rid of punctuation 
punct_re = r'[^\w\s]'
short['no_punc'] = short['contents'].str.lower().apply(lambda x: re.sub(punct_re,' ', x))
short.head()
#splitting into separate words for sentiment analysis
short['split']= short.no_punc.str.split()
tidy_short = (pd.DataFrame(short.split.apply(pd.Series).stack())
               .reset_index(level=1)
               .rename(columns={'level_1':'num', 0: 'word'}))
short=short.drop('split', axis=1)
tidy_short.head()

tidy_short['id'] = tidy_short.index
tidy_short.head()
tidy_sort = tidy_short.merge(sent, how='left', left_on='word', right_index=True)
#merging polarities with the words we have
polarities = (tidy_short
              .merge(sent, how='left', left_on='word', right_index=True)
              .fillna(value=0)
              .groupby('id')
              .sum()
              .drop('num', axis=1))
polarities.tail()
#putting back the polarities with the dataframe we have 
senti = short.merge(polarities, how='left', left_index=True, right_index=True)
senti<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11542545123615956, 'fraction_numerical': 0.0059153647808281515, 'mean_word_length': 4.072307692307692, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '200124', 'license': 'MIT', 'n_tokens_mistral': 2023, 'n_tokens_neox': 1962, 'n_words': 620, 'path': 'past-team-code/Spring2018Team/Archive/Notebooks (discontinued)/NLP Removing and identifying irrelevant articles.ipynb', 'repo_name': 'shun-lin/project-paradigm-chatbot', 'size': '6593'}