<jupyter_start><jupyter_text>Questions:Q: Which coin had the highest mean positive score?A: Bitcoin has higher average positive score (0.077564>0.068011).Q: Which coin had the highest compound score?A: Maximum of comcpound score is greater in Bitcoin(0.923100>0.851900).Q. Which coin had the highest positive score?A: Maximum of positive score is greater in Bitcoin(0.32600>0.311000).  Tokenizer<jupyter_code>from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer, PorterStemmer
from string import punctuation
import re
# Complete the tokenizer function
lemmatizer = WordNetLemmatizer()
def tokenizer(text):
    """Tokenizes text."""
    sw = set(stopwords.words('english'))
    regex = re.compile("[^a-zA-Z ]")
    re_clean = regex.sub('', text)
    words = word_tokenize(re_clean)
    lem = [lemmatizer.lemmatize(word) for word in words]
    tokens = [word.lower() for word in lem if word.lower() not in sw]
    return tokens
btc_sentiments = []
btc_tokens_corpus = []
for article in btc_headlines["articles"]:
    try:
        text = article["content"]
        sentiment = analyzer.polarity_scores(text)
        compound = sentiment["compound"]
        pos = sentiment["pos"]
        neu = sentiment["neu"]
        neg = sentiment["neg"]
        article_tokens = tokenizer(text)
        
        btc_sentiments.append({
            "text": text,
            "compound": compound,
            "positive": pos,
            "negative": neg,
            "neutral": neu,
            "tokens": article_tokens
        })
        btc_tokens_corpus.append(article_tokens)
        
    except AttributeError:
        pass
btc_df = pd.DataFrame(btc_sentiments)
btc_df.head()
# Create a new tokens column for bitcoin
eth_sentiments = []
eth_tokens_corpus = []
for article in eth_headlines["articles"]:
    try:
        text = article["content"]
        sentiment = analyzer.polarity_scores(text)
        compound = sentiment["compound"]
        pos = sentiment["pos"]
        neu = sentiment["neu"]
        neg = sentiment["neg"]
        article_tokens = tokenizer(text)
        
        eth_sentiments.append({
            "text": text,
            "compound": compound,
            "positive": pos,
            "negative": neg,
            "neutral": neu,
            "tokens": article_tokens
        })
        eth_tokens_corpus.append(article_tokens)
    except AttributeError:
        pass
eth_df = pd.DataFrame(eth_sentiments)
eth_df.head()<jupyter_output><empty_output><jupyter_text>NGRAMS & Frequency Analysis<jupyter_code>from collections import Counter
from nltk import ngrams
# Generate the Bitcoin N-grams where N=2
btc_bigram_counts = ngrams(tokenizer(btc_df.text.str.cat()), n=2)
Counter(btc_bigram_counts).most_common(20)
# Use the token_count function to generate the top 10 words from each coin
def token_count(tokens, N=10):
    """Returns the top N tokens from the frequency count"""
    return Counter(tokens).most_common(N)
# Get the top 10 words for Bitcoin
token_count(tokenizer(btc_df.text.str.cat()))
# Get the top 10 words for Ethereum
token_count(tokenizer(eth_df.text.str.cat()))<jupyter_output><empty_output><jupyter_text>Wordclouds<jupyter_code>from wordcloud import WordCloud
import matplotlib.pyplot as plt
plt.style.use('seaborn-whitegrid')
import matplotlib as mpl
mpl.rcParams['figure.figsize'] = [20.0, 10.0]
# Generate the Bitcoin word cloud
big_string = ' '.join(tokenizer(btc_df.text.str.cat()))
wc = WordCloud().generate(big_string)
plt.imshow(wc)
# Generate the Ethereum word cloud
# YOUR CODE HERE!
big_string = ' '.join(tokenizer(eth_df.text.str.cat()))
wc = WordCloud().generate(big_string)
plt.imshow(wc)<jupyter_output><empty_output><jupyter_text>Named entity recognition<jupyter_code>import spacy
from spacy import displacy
# Load the spaCy model
nlp = spacy.load('en_core_web_sm')<jupyter_output><empty_output><jupyter_text>Bitcoin NER<jupyter_code># Concatenate all of the bitcoin text together
btc_articles = ' '.join(tokenizer(btc_df.text.str.cat()))
# Run the NER processor on all of the text
doc = nlp(btc_articles)
# Render the visualization
displacy.render(doc, style='ent')
# List all Entities
for ent in doc.ents:
    print(ent.text, ent.label_)<jupyter_output>two CARDINAL
singapore GPE
singapore GPE
<NAME> PERSON
us GPE
uk GPE
five year DATE
<NAME> wa extrad PERSON
charsby PERSON
akwagyiram <NAME> PERSON
monthly DATE
africa LOC
year DATE
south africa GPE
kenya GPE
four month ago DATE
reuters ORG
four month ago DATE
satoshi nakaboto ORG
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
today DATE
today DATE
today DATE
bitcoin charsftx cryptocurrency exchange ORG
app ORG
million CARDINAL
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitcoin GPE
bitcoin GPE
today DATE
bitc[...]<jupyter_text>Ethereum NER<jupyter_code># Concatenate all of the bitcoin text together
eth_articles = ' '.join(tokenizer(eth_df.text.str.cat()))
# Run the NER processor on all of the text
doc = nlp(eth_articles)
# Render the visualization
displacy.render(doc, style='ent')
# List all Entities
for ent in doc.ents:
    print(ent.text, ent.label_)
# FIN<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09180965858836017, 'fraction_numerical': 0.0104369361401026, 'mean_word_length': 4.116742081447963, 'pattern_counts': {'":': 12, '<': 27, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '86499', 'license': 'ADSL', 'n_tokens_mistral': 1980, 'n_tokens_neox': 1852, 'n_words': 542, 'path': 'NLP_Homework/Crypto_Sentiment.ipynb', 'repo_name': 'xlspurl0ck/NLP_Homework', 'size': '5669'}