<jupyter_start><jupyter_text>News Data Extraction<jupyter_code>!pip install tweet-preprocessor
!pip install google-api-python-client
!pip install nltk
import pandas as pd
import preprocessor as p
import networkx as nx
import matplotlib.pyplot as plt
import tweepy
import json
import requests
import re
import sys
import urllib.parse
from datetime import datetime

from sklearn.feature_extraction.text import CountVectorizer
from googleapiclient.discovery import build

import nltk 
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('stopwords')
nltk.download('vader_lexicon')
stop_words = set(stopwords.words('english'))<jupyter_output>[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package wordnet to /root/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!
[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package vader_lexicon to /root/nltk_data...<jupyter_text>Get News Data from News API and Google News<jupyter_code>NEWS_API_KEY = " "
GNEWS_API_KEY = " "

def decode_text(dct, api_data=list()):
    if "title" in dct:
        api_data.append(p.clean(dct["title"]))
    if "description" in dct:
        api_data.append(p.clean(dct["description"]))
    return api_data

def get_news(query, api_source="newsapi", api_key=None):
    if not all([api_source, api_key]):
        return list()
    keywords = urllib.parse.quote(query)
    api_url = "https://newsapi.org"
    url = "{}/v2/everything?q={}&apiKey={}".format(api_url, keywords, api_key)
    if "gnews" in api_source.lower():
        api_url = "https://gnews.io"
        url = "{}/api/v3/search?q={}&token={}".format(api_url, keywords, api_key)
    response = requests.get(url)
    newsApi_json = json.dumps(response.json(), sort_keys=True)
    return json.loads(newsApi_json, object_hook=decode_text)<jupyter_output><empty_output><jupyter_text>Get Twitter Data<jupyter_code>CONSUMER_KEY = " "
CONSUMER_SECRET = " "
TWITTER_TOKEN_KEY = " "
TWITTER_TOKEN_SECRET = " "

def get_twitter_context(topicName):
    auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(TWITTER_TOKEN_KEY, TWITTER_TOKEN_SECRET)
    api = tweepy.API(auth)

    # The search term you want to find
    query = topicName
    language = "en"
    # Calling the user_timeline function with our parameters
    results = api.search(q=query, lang=language)
    corpus = []
    # foreach through all tweets pulled
    for tweet in results:
        # print(tweet.user.screen_name, "Tweeted:", tweet.text)
        corpus.append(p.clean(tweet.text))
    return corpus

# print(get_twitter_context("kobe"))<jupyter_output><empty_output><jupyter_text>Google Search API<jupyter_code>class GoogleSearch(object):
    def __init__(self, api_key, cse_id):
        self.__api_key = api_key
        self.__cse_id = cse_id
        self.service = build("customsearch", "v1", developerKey=api_key)
    
    def search(self, search_term, **kwargs):
        self.__data = self.service.cse().list(q=search_term, cx=self.__cse_id, **kwargs).execute()
    
    def get_results(self):
        return self.__data
    
    def get_search_url(self):
        url_list = list()
        if "items" in self.__data:
            for item in self.__data['items']:
                url_list.append(item['link'])
        return url_list
search_term = "coronavirus"
GOOGLE_API_KEY = " "
GOOGLE_CSE_ID = " "

google_search = GoogleSearch(GOOGLE_API_KEY, GOOGLE_CSE_ID)
google_search.search(search_term)<jupyter_output><empty_output><jupyter_text>Beatiful Soup<jupyter_code>import urllib.request
from bs4 import BeautifulSoup
from bs4.element import Comment

url_list = google_search.get_search_url()

def tag_visible(element):
    if element.parent.name in ['style', 'script', 'input', 'header','head', 'title', 'meta', '[document]']:
        return False
    if isinstance(element, Comment):
        return False
    return True

def text_from_html(body):
    soup = BeautifulSoup(body, 'html.parser')
    texts = soup.findAll(text=True)
    visible_texts = filter(tag_visible, texts)  
    return u" ".join(t.strip() for t in visible_texts)

for url in url_list:
    page = urllib.request.urlopen(url).read()
    result = text_from_html(page)
    print(result)
    print("\n")<jupyter_output><empty_output><jupyter_text>**Data Clearning and Text Preprocessing**<jupyter_code>import re

def cleaning(raw_news):
    import nltk
    
    # 1. Remove non-letters/Special Characters and Punctuations
    news = re.sub("[^a-zA-Z]", " ", raw_news)
    
    # 2. Convert to lower case.
    news =  news.lower()
    
    # 3. Tokenize.
    news_words = nltk.word_tokenize( news)
    
    # 4. Convert the stopwords list to "set" data type.
    stops = set(nltk.corpus.stopwords.words("english"))
    
    # 5. Remove stop words. 
    words = [w for w in  news_words  if not w in stops]
    
    # 6. Lemmentize 
    wordnet_lem = [ WordNetLemmatizer().lemmatize(w) for w in words ]
    
    # 7. Stemming
    stems = [nltk.stem.SnowballStemmer('english').stem(w) for w in wordnet_lem ]
    
    # 8. Join the stemmed words back into one string separated by space, and return the result.
    return " ".join(stems)<jupyter_output><empty_output><jupyter_text>**Visulization of cleaned news content**<jupyter_code>from wordcloud import WordCloud, STOPWORDS
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats

%matplotlib inline
def cloud(data,backgroundcolor = 'white', width = 800, height = 600):
    wordcloud = WordCloud(stopwords = STOPWORDS, background_color = backgroundcolor,
                         width = width, height = height).generate(data)
    plt.figure(figsize = (15, 10))
    plt.imshow(wordcloud)
    plt.axis("off")
    plt.show()<jupyter_output><empty_output><jupyter_text>**Sentiment Analysis**<jupyter_code>import warnings
import nltk.sentiment
warnings.filterwarnings('ignore')

senti = nltk.sentiment.vader.SentimentIntensityAnalyzer()

def print_sentiment_scores(sentence):
    snt = senti.polarity_scores(sentence)
    print("{:-<40} \n{}".format(sentence, str(snt)))
def get_vader_polarity(snt):
    if not snt:
        return None
    elif snt['neg'] > snt['pos'] and snt['neg'] > snt['neu']:
        return -1
    elif snt['pos'] > snt['neg'] and snt['pos'] > snt['neu']:
        return 1
    else:
        return 0
#Function to determine if a text is negative(-1) or postive (1) or neutral (0)
def get_polarity_type(sentence):
    sentimentVector = []
    snt = senti.polarity_scores(sentence)
    sentimentVector.append(get_vader_polarity(snt))
    sentimentVector.append(snt['neg'])
    sentimentVector.append(snt['neu'])
    sentimentVector.append(snt['pos'])
    sentimentVector.append(snt['compound'])
    
    print(sentimentVector)
    return sentimentVector<jupyter_output><empty_output><jupyter_text>__Generate Memory Graph for Visualization__<jupyter_code># news_api_data = []
# keyword = "coronavirus"
# my_api_key = "<KEY>"
# my_cse_id = "014947934928168541572:hgmnooclf3g"
# G=nx.Graph()
# G.add_node(keyword)
 
# corpus_twitter = get_twitter_context(keyword)
# newApi = getNewsAPI(keyword)
# gNews = getGNewsAPI(keyword)
# corpus = corpus_twitter + newApi + gNews

# top5_keyword_twitter = get_top_n_words(corpus,n=10)
# for item in top5_keyword_twitter:
#    edge = (keyword, item[0])
#    G.add_edge(*edge)
#    google_result_list = []
#    google_keyword = item[0]
#    google_result = google_search(google_keyword,my_api_key,my_cse_id)
#    top5_keyword_google = get_top_n_words(google_result,n=10)
#    for result in top5_keyword_google:
#        edge = (item[0], result[0])
#        G.add_edge(*edge)   
 
# nx.draw(G,with_labels=True)
# plt.savefig("plot.png")
# plt.show()<jupyter_output><empty_output><jupyter_text>Named-Entity Recognition (NER)<jupyter_code>import spacy

# Load English tokenizer, tagger, parser, NER and word vectors
ner = spacy.load("en_core_web_sm")
# Working NER - By using the News Corpus directly, it properly identitifies individual entities and their type.
def extract_entities(corpus):
  entities = list()
  for entry in corpus:
    filtered_corpus = "".join(entry)
    news_corpus_entities = ner(filtered_corpus)
    for entity in news_corpus_entities.ents:
      entities.append(entity.text)
      print(entity.text, entity.label_)
  return entities
entity_list = extract_entities(corpus)
print(entity_list)
# Remove duplicates
filtered_entity_list = list(set(entity_list))
print(filtered_entity_list)
'''
# Not Working NER - By using Top Related Words, the NER is unable to identify separate entities.
for word_string in top_related_words:
  filtered_string = "".join(word_string)
  entity_set = ner(filtered_string)
  for entity in entity_set.ents:
    print(entity.text, entity.label_)
'''<jupyter_output><empty_output><jupyter_text>**Based on the results above, it seems that NER only works when the input is a phrase or sentence. If the input is just a list of words, the NER does not properly recognize individual entities in the text.** LDA Topic Modeling<jupyter_code>from sklearn.decomposition import LatentDirichletAllocation as LDA
def print_topics(model, count_vectorizer, n_top_words):
    extracted_words = list()
    words = count_vectorizer.get_feature_names()
    for topic_idx, topic in enumerate(model.components_):
        extracted_words.append(" ".join([words[i] for i in topic.argsort()[:-n_top_words - 1:-1]]))
        print("\nTopic #%d:" % topic_idx)
        print(" ".join([words[i]
                        for i in topic.argsort()[:-n_top_words - 1:-1]]))
    return extracted_words
number_topics = 10
number_words = 10

count_vectorizer = CountVectorizer(stop_words='english')
# Fit and transform the processed titles
#count_data = count_vectorizer.fit_transform(corpus)
# Fit and transform the processed entities
count_data = count_vectorizer.fit_transform(filtered_entity_list)
lda = LDA(n_components=number_topics, n_jobs=-1)
lda.fit(count_data)
# Print the topics found by the LDA model
print("Topics found via LDA:")
top_related_words = print_topics(lda, count_vectorizer, number_words)
print(top_related_words)<jupyter_output>['americans fourty japan coronavirus china weeks annual chinese britons israel', 'delta platinum israeli feb wuhan maga coronavirus americans weeks annual', 'coronavirus initially infectious estimated novel westerdam israel tuesday americans chinese', 'francisco communist party san sunday hundreds cambodia california coronavirus americans', 'tom cotton taiwan barcelona weekend coronavirus americans annual china chinese', 'americans summit mobile reserve britons world delta coronavirus weeks china', 'xi jinping contagious highly annual chinese china weeks coronavirus americans', 'finance ministry kellyanne conway coronavirus americans weeks chinese annual china', 'week earlier 670 coronavirus americans weeks annual china chinese britons', 'world health organization malaysian washington virus coronavirus americans china weeks']<jupyter_text>__Display LDA Topics__<jupyter_code>! pip install pyLDAvis
from pyLDAvis import sklearn as sklearn_lda
import pyLDAvis
LDAvis_prepared = sklearn_lda.prepare(lda, count_data, count_vectorizer)
pyLDAvis.display(LDAvis_prepared)
# pyLDAvis.save_html(LDAvis_prepared, './ldavis_prepared_'+ str(number_topics) +'.html')<jupyter_output>/usr/local/lib/python3.6/dist-packages/pyLDAvis/_prepare.py:257: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  return pd.concat([default_term_info] + list(topic_dfs))<jupyter_text>Neo4j Graph Database Integration<jupyter_code># pip install -U ipython
pip install py2neo
from py2neo import Graph, Node, Relationship
#graph = Graph("bolt://ec2-100-27-23-215.compute-1.amazonaws.com:7687")
graph = Graph("bolt://ec2-100-27-23-215.compute-1.amazonaws.com:7687", user = "kevin", password = "<PASSWORD>" )
graph.delete_all()
news_api_data = []
keyword = "iowacaucus"
my_api_key = "<KEY>"
my_cse_id = "014947934928168541572:hgmnooclf3g"

topic = Node("Keyword", name=keyword)
graph.create(topic)

corpus_twitter = get_twitter_context(keyword)
newApi = getNewsAPI(keyword)
gNews = getGNewsAPI(keyword)
corpus = corpus_twitter + newApi + gNews

top5_keyword_twitter = get_top_n_words(corpus,n=10)
for item in top5_keyword_twitter:
   n = Node("Twitter", name=item[0])
   r = Relationship(topic, "LINKS_TO", n)
   graph.create(n | r)
   google_result_list = []
   google_keyword = item[0]
   google_result = google_search(google_keyword,my_api_key,my_cse_id)
   top5_keyword_google = get_top_n_words(google_result,n=10)
   for result in top5_keyword_google:
       res = Node("Google", name=result[0])
       rel = Relationship(n, "LINKS_TO", res)
       graph.create(res)
       graph.create(rel)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10808555825242719, 'fraction_numerical': 0.01137742718446602, 'mean_word_length': 4.725141120277899, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 57, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '627001', 'license': 'Apache-2.0', 'n_tokens_mistral': 4604, 'n_tokens_neox': 4445, 'n_words': 1036, 'path': 'src/Deep_Context.ipynb', 'repo_name': 'Junten/context-graph', 'size': '13253'}