<jupyter_start><jupyter_text>Forbes news<jupyter_code>from urllib.request import urlopen
from bs4 import BeautifulSoup
import pandas as pd
import re


search = 'Alphabet'

source = urlopen('https://www.forbes.com/search/?q=' + search)
soup = BeautifulSoup(source,'lxml')

lst = []
for i in soup.findAll('a', {'class': "stream-item__title"}):
    title = i.text
    link = i.get('href')

    title = title.replace('\t','')
    title = title.replace('\n','')
    title = title.strip()

    lst.append({'title':title, 'link':link})

df = pd.DataFrame.from_dict(lst)
df.head()
text_list = []
for link in df['link']:
    source2 = urlopen(link)
    soup2 = BeautifulSoup(source2, 'lxml')
    text = soup2.findAll('p', text=re.compile("."))
    print('---------------------------------')
    print(text)
    text_list.append(text)
import re

def cleanhtml(raw_html):
    cleanr = re.compile('<.*?>')
    cleantext = re.sub(cleanr, '', raw_html)
    return cleantext

new_text_list = []
for text in text_list:
    new_text_list.append(cleanhtml(str(text)))
    
print(new_text_list)
len(new_text_list)
final_text_list = []
for text in new_text_list:
    text = text.replace("[", "")
    text = text.replace("]", "")
    text = text.replace("\\", "")
    text = text.replace("\xa0", " ")
    final_text_list.append(text)
print(final_text_list)
len(final_text_list)
print(lst)
df2 = pd.DataFrame(final_text_list)

df = pd.concat([df.reset_index(drop=True), df2.reset_index(drop=True)], axis=1)


df.columns = ['link','title','text']
df.head()<jupyter_output><empty_output><jupyter_text>Twitter<jupyter_code>import tweepy

TWITTER_AUTH = tweepy.OAuthHandler(config('TWITTER_CONSUMER_KEY'),
                                   config('TWITTER_CONSUMER_SECRET'))
TWITTER_AUTH.set_access_token(config('TWITTER_ACCESS_TOKEN'),
                              config('TWITTER_ACCESS_TOKEN_SECRET'))
TWITTER = tweepy.API(TWITTER_AUTH)


def search_tweets(keyword):
    searched_tweets = [status for status in tweepy.Cursor(api.search, q=str(keyword).items(max_tweets))]

from urllib.request import urlopen
from bs4 import BeautifulSoup
import pandas as pd

search = 'facebook'

source = urlopen('https://www.foxbusiness.com/search?q=' + search + '&ss=fb')
soup = BeautifulSoup(source,'lxml')

lst = []
for i in soup.findAll('div', {'class': "content"}):
    title = i.text
    link = i.get('a')
    link = i.get('href')

    title = title.replace('\t','')
    title = title.replace('\n','')
    title = title.strip()

    lst.append({'title':title, 'link':link})

df2 = pd.DataFrame.from_dict(lst)
print(soup.findAll('div', {'class': "content"}))
df2
import spacy
from spacy.lang.en import English
from spacy.matcher import PhraseMatcher
from spacy.tokens import Doc, Span, Token
nlp = spacy.load("en")

for text in df.text:
    doc = nlp(text)
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import nltk
nltk.download('vader_lexicon')

analyzer = SentimentIntensityAnalyzer()


def sentiment_analyzer_scores(sentence):
    score = analyzer.polarity_scores(sentence)
    print("{}".format(str(score)))
    

for text in df.text:
    sentiment_analyzer_scores(text)
s = analyzer.polarity_scores(df.text[0])

s['neg']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16522544780728846, 'fraction_numerical': 0.004014823965410748, 'mean_word_length': 4.207395498392283, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '224407', 'license': 'MIT', 'n_tokens_mistral': 1187, 'n_tokens_neox': 1162, 'n_words': 215, 'path': 'sentiment-webscraping.ipynb', 'repo_name': 'stock-price-sentiment/data-science', 'size': '3238'}