<jupyter_start><jupyter_text>Sentiment Analysis Using TextBlobLet's take a look at polarity ('postive' or 'negative') and the subjectivity of a piece of text. ![](img/Covfefe-Donald-Trump-811379.jpg)<jupyter_code>import pandas as pd
#import numpy as np
import random
from textblob import TextBlob, Word
import requests
from bs4 import BeautifulSoup
import time
import urllib
import sys
import random

from selenium import webdriver
from selenium.webdriver.common.keys import Keys

import os
chromedriver = "/Applications/chromedriver"
os.environ["webdriver.chrome.driver"] = chromedriver

# Use Beautiful Soup to scrape tweets from Donald Trump's twitter page

donald_url = 'https://twitter.com/realDonaldTrump?ref_src=twsrc%5Egoogle%7Ctwcamp%5Eserp%7Ctwgr%5Eauthor'
donald_response = requests.get(donald_url)
donald_page = donald_response.text
soup = BeautifulSoup(donald_page,"lxml")
dt_tweets = []

tweets = soup.find_all(class_="TweetTextSize TweetTextSize--normal js-tweet-text tweet-text")

# This tells me the number of tweets scraped

for t in tweets:
    dt_tweets.append(t.text)

len(dt_tweets)

# Print a random tweet from the set of tweets that were scraped

x = random.randint(1, len(dt_tweets))
print(x)

dt_tweets[x]
#dt_tweets
# TextBlob implementation

blob = []

for d in dt_tweets:
    blob.append(TextBlob(d))
# Sanity check: len(blob) == len(tweets)?
len(blob)
blob[x].sentences
# POS Tagging
# for tag info: https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html

for words, tag in blob[x].tags:
    print(words, tag)
for np in blob[x].noun_phrases:
     print(np)
#n-gram analysis

for ngram in blob[x].ngrams(2):
    print(ngram)
# Sentiment Analysis

dt_sentiments = []
for b in blob:
    print(b, b.sentiment)
    print("\n")
    dt_sentiments.append(b.sentiment)
dt_sentiments[x]
dt_polarity = pd.Series(x[0] for x in dt_sentiments)
dt_subjectivity = pd.Series(x[1] for x in dt_sentiments)
dt_polarity.describe()
dt_subjectivity.describe()
oprah_url = 'https://twitter.com/Oprah?lang=en'
oprah_response = requests.get(oprah_url)
oprah_page = oprah_response.text
o_soup = BeautifulSoup(oprah_page,"lxml")
op_tweets = []

o_tweets = o_soup.find_all(class_="TweetTextSize TweetTextSize--normal js-tweet-text tweet-text")
for t in o_tweets:
    op_tweets.append(t.text)

len(op_tweets)
y = random.randint(1, len(op_tweets))
print(y)

op_tweets[y]
o_blob = []

for o in op_tweets:
    o_blob.append(TextBlob(o))
len(o_blob)
o_blob[y].sentences
for words, tag in o_blob[y].tags:
    print(words, tag)
for np in o_blob[y].noun_phrases:
     print(np)
for ngram in o_blob[y].ngrams(3):
    print(ngram)
ow_sentiments = []
for b in o_blob:
    print(b, b.sentiment)
    print("\n")
    ow_sentiments.append(b.sentiment)
ow_sentiments[y]
ow_polarity = pd.Series(x[0] for x in ow_sentiments)
ow_subjectivity = pd.Series(x[1] for x in ow_sentiments)
ow_polarity.describe()
ow_subjectivity.describe()<jupyter_output><empty_output><jupyter_text>Text Summary<jupyter_code>driver = webdriver.Chrome("/usr/bin/chromedriver")
driver.get("http://www.bbc.com")

first_article = driver.find_element_by_class_name('media__link').click();
txt_url = driver.current_url
driver.close()
bbc_response = requests.get(txt_url)
bbc_page = bbc_response.text
soup = BeautifulSoup(bbc_page,"lxml")
soup.find(class_='story-body__inner').text
news_blob = TextBlob(soup.find(class_='story-body__inner').text)
nouns = list()
for word, tag in news_blob.tags:
    if tag == 'NN':
        nouns.append(word.lemmatize())

print("This text is about...")
for item in random.sample(nouns, 5):
    word = Word(item)
    print(word.pluralize())<jupyter_output>This text is about...
visits
visits
visits
securities
Images<jupyter_text>Language Translation<jupyter_code>en_blob = TextBlob(u'')
langlist = ['ar', 'ga', 'ko', 'ja', 'eu', 'zh-TW', 'ka', 'el', 'tr', 'yi', 'gl', 'fr', 'es' ]
for l in langlist:
    print(en_blob.translate(to=l))
ko_blob = TextBlob('겨울오고있다')
ko_blob.translate(to='en')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1299056135121709, 'fraction_numerical': 0.006954793840039742, 'mean_word_length': 5.580065359477124, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '149681', 'license': 'MIT', 'n_tokens_mistral': 1595, 'n_tokens_neox': 1544, 'n_words': 331, 'path': '_download/textblob.ipynb', 'repo_name': 'Cassini-4B/Cassini-4B.github.io', 'size': '4026'}