<jupyter_start><jupyter_text>News Headlines SentimentUse the news api to pull the latest news articles for bitcoin and ethereum and create a DataFrame of sentiment scores for each coin. Use descriptive statistics to answer the following questions:1. Which coin had the highest mean positive score?2. Which coin had the highest negative score?3. Which coin had the highest positive score?<jupyter_code># Read your api key environment variable
api_key='8bd0a8ca889b4a2e9626db863dba01ca'
print(api_key)


# Create a newsapi client

newsapi = NewsApiClient(api_key)
# Fetch the Bitcoin news articles

btc_everything = newsapi.get_everything(q="bitcoin AND Bitcoin AND BTC ", language="en",
    page_size=100,sort_by="relevancy"
)
print(f"Total articles about bitcoin: {btc_everything['totalResults']}")
btc_everything["articles"][0]
# Fetch the Ethereum news articles
eth_everything = newsapi.get_everything(q='ethereum AND Ethereum AND ETH', page_size=100,
                                         language='en', sort_by="relevancy"
                                          )
print(f"Total articles about ethereum: {eth_everything['totalResults']}")
eth_everything["articles"][0]
# Create the Bitcoin sentiment scores DataFrame

btc_sentiments = []

for article in btc_everything["articles"]:
    try:
        text = article["content"]
        date = article["publishedAt"][:10]
        sentiment = analyzer.polarity_scores(text)
        compound = sentiment["compound"]
        pos = sentiment["pos"]
        neu = sentiment["neu"]
        neg = sentiment["neg"]
        
    btc_sentiments.append({
            "text": text,
            "date": date,
            "compound": compound,
            "positive": pos,
            "negative": neg,
            "neutral": neu
            
        })
        
    except AttributeError:
        pass
    
# Create DataFrame
btc_df = pd.DataFrame(btc_sentiments)


# Reorder DataFrame columns
cols = ["date", "text", "compound", "positive", "negative", "neutral"]
btc_df = btc_df[cols]

btc_df.head()
# Create the Facebook Libra sentiment scores DataFrame
eth_sentiments = []

for article in eth_everything["articles"]:
    try:
        text = article["content"]
        date = article["publishedAt"][:10]
        sentiment = analyzer.polarity_scores(text)
        compound = sentiment["compound"]
        pos = sentiment["pos"]
        neu = sentiment["neu"]
        neg = sentiment["neg"]
        
        eth_sentiments.append({
            "text": text,
            "date": date,
            "compound": compound,
            "positive": pos,
            "negative": neg,
            "neutral": neu
            
        })
        
    except AttributeError:
        pass
    
# Create DataFrame
eth_df = pd.DataFrame(eth_sentiments)

# Reorder DataFrame columns
cols = ["date", "text", "compound", "positive", "negative", "neutral"]
eth_df = eth_df[cols]

eth_df.head()
# Describe the Bitcoin Sentiment
btc_df.describe()
# Describe the Ethereum Sentiment
eth_df.describe()<jupyter_output><empty_output><jupyter_text>Questions:Q: Which coin had the highest mean positive score?A: BitcoinQ: Which coin had the highest compound score?A: EthereumQ. Which coin had the highest positive score?A: Ethereum ---  TokenizerIn this section, you will use NLTK and Python to tokenize the text for each coin. Be sure to:1. Lowercase each word2. Remove Punctuation3. Remove Stopwords<jupyter_code>from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer, PorterStemmer
from string import punctuation
import re

lemmatizer = WordNetLemmatizer()
# Expand the default stopwords list if necessary
print(stopwords.words('english'))
# btc_everything.get('articles')[1]
# btc_articles=  btc_everything.get('articles')[0]
# btc_articles.get("content")
 # eth_everything.get('articles')[1]
# eth_articles=  eth_everything.get('articles')[0]
# eth_articles.get("content")
# Complete the tokenizer function
def tokenizer(text):
    """Tokenizes text."""
# Complete the tokenizer function
   # Remove the stop words
    sw = set(stopwords.words('english'))
    
    # Remove the punctuation
    regex = re.compile('[^a-zA-Z ]')
    
    re_clean = regex.sub('', text)
    # Create a list of the words
    words = word_tokenize(re_clean)
    
    # Lemmatize Words into root words
    lem = [lemmatizer.lemmatize(word) for word in words]
    
    # Convert the words to lowercase
    tokens = [word.lower() for word in lem if word.lower() not in sw]
    
    return tokens

btc_everything['tokens'] = [tokenizer(t) for t in btc_everything['text']]
btc_everything.head()
# Create a new tokens column for bitcoin
eth_everything['tokens']=[tokenizer(word) for word in eth_everything['text']]
eth_everything.head()
# Create a new tokens column for ethereum
eth_df.iloc[-5:,5]
print<jupyter_output><empty_output><jupyter_text>---  NGrams and Frequency AnalysisIn this section you will look at the ngrams and word frequency for each coin. 1. Use NLTK to produce the n-grams for N = 2. 2. List the top 10 words for each coin.<jupyter_code>from collections import Counter
from nltk import ngrams

processed = tokenizer(text)
print(processed)
word_counts = Counter(processed)
print(dict(word_counts))

# Generate the Bitcoin N-grams where N=2
def bigram_counter(text):
    
    big_string = ''.join(text)
    tokens = tokenizer(big_string)
    bigrams = ngrams(tokens, n=2)
    
    return Counter(bigrams).most_common(10)



# Generate Bitcoin N-grams
bigram_counter(btc_everything['text'])
# Generate the Ethereum N-grams where N=2
bigram_counter(eth_everything['text'])
# Use the token_count function to generate the top 10 words from each coin
def token_count(tokens, N=10):
    print(dict(bigram_counts.most_common(10)))
    return Counter(tokens).most_common(10)
# Get the top 10 words for Bitcoin
btc_everything = []
for row in btc_everything['Tokens']:
    for token in row:
       btc_everything.append(token)
        
token_count(btc_everything)
# Get the top 10 words for Ethereum
eth_everything = []

for row in eth_everything['tokens']:
    for token in row:
        eth_everything.append(token)
        
token_count(eth_everything)<jupyter_output><empty_output><jupyter_text>Word CloudsIn this section, you will generate word clouds for each coin to summarize the news for each coin<jupyter_code>from wordcloud import WordCloud
import matplotlib.pyplot as plt
plt.style.use('seaborn-whitegrid')
import matplotlib as mpl
mpl.rcParams['figure.figsize'] = [20.0, 10.0]
# Generate the Bitcoin word cloud
bitcoin_tokens = ','.join(btc_everything)
wc_b = WordCloud().generate(bitcoin_tokens)
plt.title('Bitcoin Word Cloud', fontsize=40)
plt.imshow(wc_b)
# Generate the Ethereum word cloud
ethereum_tokens = ','.join(eth_everything)
wc_e = WordCloud().generate(eth_everything)
plt.title('Ethereum Word Cloud', fontsize=40)
plt.imshow(wc_e)<jupyter_output><empty_output><jupyter_text>Named Entity RecognitionIn this section, you will build a named entity recognition model for both coins and visualize the tags using SpaCy.<jupyter_code>import spacy
from spacy import displacy
# Optional - download a language model for SpaCy
#!python -m spacy download en_core_web_sm
# Load the spaCy model
nlp = spacy.load('en_core_web_sm')<jupyter_output><empty_output><jupyter_text>Bitcoin NER<jupyter_code># Concatenate all of the bitcoin text together
bitcoin_string = ''
for row in bitcoin_sentiments['Text']:
    bitcoin_string = bitcoin_string + row
    
bitcoin_string[:4000]
# Run the NER processor on all of the text
bit_doc=nlp(bitcoin_string[:4000])

# Add a title to the document
bit_doc.user_data["title"] = "Bitcoin NER"
# Render the visualization
displacy.render(bit_doc, style='ent', jupyter=True)
# List all Entities
bit_entities = [ent.text for ent in bit_doc.ents]
bit_entities[:200]<jupyter_output><empty_output><jupyter_text>---  Ethereum NER<jupyter_code># Concatenate all of the bitcoin text together
ethereum_string = ''
for row in ethereum_sentiments['Text']:
    ethereum_string = ethereum_string + row
    
ethereum_string[:4000]

# Run the NER processor on all of the text
eth_doc = nlp(ethereum_string[:4000])

# Add a title to the document
eth_title = eth_doc.user_data["title"] = "Ethereum NER"
# Render the visualization
displacy.render(eth_doc, style='ent')
# List all Entities
eth_entities = [ent.text for ent in eth_doc.ents]
eth_entities[:200]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09984675232818578, 'fraction_numerical': 0.011198868324885064, 'mean_word_length': 4.1449363250454825, 'pattern_counts': {'":': 12, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '33452', 'license': 'ADSL', 'n_tokens_mistral': 2779, 'n_tokens_neox': 2606, 'n_words': 762, 'path': 'crypto_sentiment.ipynb', 'repo_name': 'sz2739/NLP', 'size': '8483'}