<jupyter_start><jupyter_text><jupyter_code># install chromium, its driver, and selenium
#!apt install chromium-chromedriver
#!pip install selenium
# set options to be headless, ..
from selenium import webdriver

options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
# open it, go to a website, and get results
wd = webdriver.Chrome('chromedriver',options=options)
wd.get("https://www.google.com")
print(wd.title)  # results
# divs = wd.find_elements_by_css_selector('div')
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
import re
import time
import requests
import random
import os 
import os.path
import csv 
import datetime
import random
from getpass import getpass
import time
import random   
from dateutil import parser
import calendar

from urllib.request import Request, urlopen
from fake_useragent import UserAgent
import pandas as pd
import matplotlib.pyplot as plt
import json
plt.style.use('fivethirtyeight')


import warnings
warnings.filterwarnings("ignore")

idx = datetime.date.today()




t0 = time.time()

#********************************************

topic = "coronavirus"
keyword = "Coronavirus India"

website = "https://www.google.com"
browser = webdriver.Chrome('chromedriver',options=options)
browser.get(website)

browser.maximize_window()



def writerows(rows, filename):
    with open(filename, 'a', encoding='utf-8') as toWrite:
        writer = csv.writer(toWrite)
        writer.writerows([rows])
def scrapePage(keyword,rank):
    soup = BeautifulSoup(browser.page_source,"html.parser")
    result_block = soup.find_all('div', attrs={'class': 'g'})
    for r in result_block:
    
        link = r.find('a', href=True)
        title = r.find('h3')
        description = r.find('span', attrs={'class': 'st'})
        if r.find(class_="f"):
            res = r.find(class_="f").text
        else:
            res = "-1"

        if link and title and description:
            link = link['href']
            title = title.get_text().strip()
            if description:
                description = description.get_text().strip()
            if link != '#':
                rank += 1
           
            row = [keyword,rank,title,res,description,link]
            print(row)
            print(30*"--")
            writerows(row,outputFile)
outputFile = "C:/CART/{}_{}.csv".format(topic.title().replace(" ",""),idx)
print(outputFile)



time.sleep(random.randint(1,100)/50)

search = browser.find_element_by_class_name("gLFyf")
search.clear()
search.send_keys(keyword)
search.send_keys(Keys.RETURN);
time.sleep(random.randint(40,100)/50)
soup = BeautifulSoup(browser.page_source,"html.parser")
time.sleep(random.randint(40,100)/50)
browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")

time.sleep(2)

count = 0

    
search = browser.find_element_by_class_name("gLFyf")
search.clear()
search.send_keys(keyword)
search.send_keys(Keys.RETURN);

time.sleep(random.randint(1,100)/50)
browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    
noPages = int(soup.findAll("a", class_="fl")[-1].text)
for i in range(noPages):
   
    browser.find_element_by_id("pnnext").click()
    time.sleep(random.randint(25,100)/25)
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    count += 1
    scrapePage(keyword,count)
df['Source'] = df['Link'].apply(lambda x: x.split("//")[1].split("/")[0])
excl = ['books.google.com.in']
df= df[~df['Source'].isin(excl)]
print(df.shape)

print(df['Source'].value_counts())
#df[df['Source']=="forum.lowyat.net"]
df.to_csv("C:/CART/{}_prep_{}.csv".format(topic.title().replace(" ",""),idx),encoding="utf-8",index=None)
df.head()
import nltk
from nltk.util import ngrams
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.collocations import *
from nltk.corpus import stopwords
from collections import Counter
# modules for generating the word cloud
from os import path, getcwd
from PIL import Image
from wordcloud import WordCloud, ImageColorGenerator
#tokenizer = nltk.data.load('nltk:tokenizers/punkt/english.pickle')
from nltk.tokenize import RegexpTokenizer
tokenizer = RegexpTokenizer(r'\w+')
field ="Description"

df[field] = df[field].astype(str)
df[field] = df[field].apply(lambda x: x.lower())
print(len(df[field].str.cat(sep=', ')))

#All property Description
print("concat desc")
df[field] = df[field].astype(str)
df[field] = df[field].apply(lambda x: x.lower())
tw = df[field].str.cat(sep=', ')
print('concat length',len(tw))


tw  = re.sub(r'^https?:\/\/.*[\r\n]*', '', tw, flags=re.MULTILINE)
tw = re.sub(r'[^a-zA-Z\s]', ' ', tw)
tw = tw.replace("netflix","").replace("hahaha","").replace("hahah","").replace("haha","")

#tw = re.sub(r'[^a-zA-Z0-9\s]', ' ', tw)
#tw = re.sub(r'[\W_]', ' ', tw)


tok_text = tokenizer.tokenize(tw)
print('tokenize length',len(tok_text))

#remove stopwords
stopwords = nltk.corpus.stopwords.words('english')
malayStopWords =['di','pada','kat',"ke","ko","ye"]
word_set = set(w for w in tok_text if w.lower() not in stopwords)
print('length after english stop words',len(word_set))
word_set = set(w for w in word_set if w.lower() not in malayStopWords)
print('length after malay stop words',len(word_set))
print(len(word_set))

#get topwords and leastwords
word_count_dict = Counter(w.lower() for w in word_set)
w = word_count_dict.most_common()
print(len(w))
w = pd.DataFrame(w)
w = w.rename(columns={0:'word',1:'count'})
desc = w[~w.word.str.match('^\-?(\d*\.?\d+|\d+\.?\d*)$')]
#remove top words
topWord = list(desc[:10]["word"])
print(list(desc[:100]["word"]))
#leastWord = list(desc[desc['count']==1]["word"])

#word_set = [x for x in word_set if x not in leastWord]


text = ' '.join(word_set).lower()

print("done.....................")
from wordcloud import WordCloud, STOPWORDS
from scipy.misc import imread

wordcloud = WordCloud(
                       width = 1500, height=1500,
                      stopwords=STOPWORDS,
                      background_color='black',
                      #mask=imread('C:/Users/Faizal/Anaconda3/my_map2/lib/images/train.png'),
                     ).generate(text)

plt.figure(figsize=(14,14))
plt.title(keyword, color="grey", size=25, y=1.01)
plt.imshow(wordcloud)
plt.axis('off')
#plt.text(x = 0, y = 1450, fontsize = 15, alpha = 1,color = 'white',  s = " www.redangpow.com ", backgroundcolor="grey")
plt.savefig('F:/RAP Cares/{}_{}.png'.format(topic.title().replace(" ",""),idx), transparent=True, bbox_inches='tight',papertype = 'a4')
plt.show()
import collections

ngr = ngrams(tok_text, 5)
ng = collections.Counter(ngr).most_common()
ng = pd.DataFrame(ng)
ng.columns = ["Phrase"," Count"]
ng["Phrase"] = ng["Phrase"].apply(lambda x: " ".join(x))
ng[:20]
ngr = ngrams(tok_text, 4)
ng = collections.Counter(ngr).most_common()
ng = pd.DataFrame(ng)
ng.columns = ["Phrase"," Count"]
ng["Phrase"] = ng["Phrase"].apply(lambda x: " ".join(x))
ng[:20]
ngr = ngrams(tok_text, 3)
ng = collections.Counter(ngr).most_common()
ng = pd.DataFrame(ng)
ng.columns = ["Phrase"," Count"]

ng["Phrase"] = ng["Phrase"].apply(lambda x: " ".join(x))
ng[:20]
ngr = ngrams(tok_text, 2)
ng = collections.Counter(ngr).most_common()
ng = pd.DataFrame(ng)
ng.columns = ["Phrase"," Count"]
ng["Phrase"] = ng["Phrase"].apply(lambda x: " ".join(x))
ng[:20]
ngr = ngrams(tok_text, 1)
ng = collections.Counter(ngr).most_common()
ng = pd.DataFrame(ng)
ng.columns = ["Phrase"," Count"]
ng["Phrase"] = ng["Phrase"].apply(lambda x: " ".join(x))
ng[:20]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17937940412906417, 'fraction_numerical': 0.012856966250463592, 'mean_word_length': 4.610263522884882, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '32390', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2952, 'n_tokens_neox': 2786, 'n_words': 568, 'path': 'corona.ipynb', 'repo_name': 'HappyplaceAI/jupyter', 'size': '8089'}