<jupyter_start><jupyter_text>Machine Learning Web Scraping  Task 1:Collect the first 100 quotes from the website http://quotes.toscrape.com/The output should store in three column namely- Quote, Author and Tags.  Google CoLablink:https://colab.research.google.com/drive/1RgQ-vINmT46JK4U6oPgqSjA4-SUy5vze?usp=sharing<jupyter_code>#to check wheter we are use colab or not 

try:
    from google.colab import drive 
    
    print("Note: not using Google CoLab")
    COLAB = False
# To print current data and time     
from datetime import datetime
now = datetime.now()

current_time = now.strftime("%H:%M:%S")
print("Current Time =", current_time)
today = datetime.today()
print("Today's date:", today)
###install requred packages
!pip install selenium
!apt-get update
!apt install chromium-chromedriver
!cp /usr/lib/chromium-browser/chromedriver /usr/bin
# importing socket for establishment of connection
from socket import socket
!pip install webdriver_manager
from flask import request

import os
#This is the webpage we are use for scraping
webpage="http://quotes.toscrape.com/"
hdr={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
       'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
       'Accept-Encoding': 'none',
       'Accept-Language': 'en-US,en;q=0.8',
       'Connection': 'keep-alive'}
def extract_soup_object(link):
  request=Request(link,headers=hdr)
  html=urlopen(request).read().decode()
  soup=BeautifulSoup(html,'html.parser')
  return soup
import sys
sys.path.insert(0,'/usr/lib/chromium-browser/chromedriver')
from selenium import webdriver
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')

driver = webdriver.Chrome('chromedriver',chrome_options=chrome_options)
driver.get(webpage)
html = driver.page_source.encode('utf-8')
#url="http://quotes.toscrape.com/page/2/"
driver.get("http://quotes.toscrape.com/")
html = driver.page_source.encode('utf-8')
!pip install BeautifulSoup4
import urllib
!pip install tqdm
from bs4 import BeautifulSoup
import time
from urllib.parse import quote_plus
from selenium.webdriver.common.action_chains import ActionChains

import json
import pandas as pd
from datetime import date
from tqdm import tqdm_notebook as tqdm
from urllib.request import urlopen, Request
from pprint import pprint
import requests
from flask import request

req=requests.get('http://quotes.toscrape.com/')
content=req.text
#print(content)
soup=BeautifulSoup(content)
#soup print the HTML document
print(soup)
extract_soup_object(webpage)

soup=extract_soup_object(webpage)
data=soup.find_all('div',{'class':"quote"})
data[0:9]
def extract_links(page):
  extract_soup_object(page)
  data=soup.find_all('div',{'class':"quote"})
  return ['http://quotes.toscrape.com/'+item.a.get('href') for item in data]




count=0
for item in data:
  print(item.a.get('href'))
  count+=1
count
author_names=extract_links(webpage)
author_names[0:9]
soup=extract_soup_object(webpage)
quotes=soup.find_all('span',{'class':"text"})
quotes[0:9]
soup=extract_soup_object(webpage)
author_name=soup.find_all('small',{'class':"author"})
author_name[0:9]
soup=extract_soup_object(webpage)
tagss=soup.find_all('meta',{'class':"keywords"})
tagss[0:10]
soup=extract_soup_object(webpage)
tagss=soup.find_all('a',{'class':"tag"})
tagss[7]
import pandas as pd
from pandas import DataFrame
import numpy as np
#from goto import goto, comefrom, label

r1 = requests.get("http://quotes.toscrape.com/")
coverpage = r1.content

soup1 = BeautifulSoup(coverpage, 'html5lib')

quote_text = soup1.find_all('span', class_='text')

n=0
for t in quote_text:
  if n<=11:
   p=quote_text[n].get_text()
   print(p)
   n+=1
   continue
  else:
    break


q = []
qu=[]
n=0
for q in quote_text:
  if n<=11:
   p=quote_text[n].get_text()
   #row = [q.text for q in p]
   qu.append(p)
   n+=1
   continue
  else:
    break

f1=pd.DataFrame(qu, columns=["Quotes"])
f1


names=soup.find_all('span')
count=0

for n in names:
  count+=1
  if count%2!=0:
    continue
  else:
    p = n.find_all("small")
    row = [n.text for n in p]
    print(row)  

  if count==20:
    break



l = []
count=0
for n in names:
  count+=1
  if count%2!=0:
    continue
  else:
    p = n.find_all('small')
    row = [n.text for n in p]
    l.append(row)
    if count==20:
      break
f2=pd.DataFrame(l, columns=["Authors"])
f2
quotestext=soup.find_all('div')
count=0

for q in quotestext:
  count+=1

  if count<7:
    continue
  else:
    if count%2!=0:

      r = q.find_all('span')
      row = [q.text for q in r]
      print(row) 
    else:
      continue 
    if count==25:
      break
count = 0


filter_data=[]
while driver.find_elements_by_css_selector('.next a'):
  
  driver.implicitly_wait(5)

  driver.find_element_by_css_selector('.next a').click()
  count += 1
  print("page number "+str(count))
  
  extracted_data = BeautifulSoup(driver.page_source, features="html.parser")
  filter_data=filter_data+extracted_data.find_all('div',{'class':"quote"})
  # print(f)
  
  if count ==12:
    break
def getQuote(filter_data):
    list1=[]
    i=0
    for q in filter_data:
      quotes=q.find('span',{'class':'text'}).text
      au=q.find('small',{'class':'author'}).text
      strtags=''
      
      for div in q.findAll('a', attrs={'class':'tag'}):
         if strtags=='':
           comma=''
         else:
           comma=' , '
         strtags= strtags +comma+div.contents[0]
      
      df.loc[i] = [quotes]+[au]+[strtags] 
      i+=1
     
    return 

df = pd.DataFrame(columns=['Quote', 'Author', 'Tags'])
#print(getQuote(filter_data))
getQuote(filter_data)
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14604635310156783, 'fraction_numerical': 0.020279481935923653, 'mean_word_length': 4.389348025711662, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '328560', 'license': 'MIT', 'n_tokens_mistral': 2260, 'n_tokens_neox': 2150, 'n_words': 459, 'path': 'web_scraping.ipynb', 'repo_name': 'Sanu3286/Web_scraping', 'size': '5868'}