<jupyter_start><jupyter_text>Solution: WEB SCRAPING ASSIGNMENT 4<jupyter_code>#importing libraries
import pandas as pd
import re
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys<jupyter_output><empty_output><jupyter_text>Q1. Scrape the details of most viewed videos on YouTube from Wikipedia: Url = https://en.wikipedia.org/wiki/List_of_most-viewed_YouTube_videos**You need to find following details:****A) Rank****B) Name****C) Artist****D) Upload date****E) Views**<jupyter_code>#initializing webdriver with chromedriver
driver = webdriver.Chrome('chromedriver.exe')

#initializing driver with url https://en.wikipedia.org/wiki/List_of_most-viewed_YouTube_videos
driver.get('https://en.wikipedia.org/wiki/List_of_most-viewed_YouTube_videos')

#initializing most viewed videos on YouTube container
views_container = WebDriverWait(driver,20).until(ec.visibility_of_element_located((By.XPATH,'//table[.//caption[contains(text(),"most-viewed")]]')))

#Scraping required details
ranks = []
names = []
artists = []
upload_dates = []
views = []
record_rows = views_container.find_elements_by_xpath('.//tbody/tr')
for row in record_rows:
    cols = row.find_elements_by_xpath('.//td')
    ranks.append(re.search('\d+',cols[0].text).group(0))
    names.append(re.search('"(.*?)"',cols[1].text).group(1))
    artists.append(cols[2].text)
    upload_dates.append(cols[4].text)
    views.append(cols[3].text)
    

#closing driver
driver.close()

#storing scraped details into dataframe
df_most_viewed_yt_videos = pd.DataFrame({
    "Rank": ranks,
    "Name": names,
    "Artist": artists,
    "Upload Date": upload_dates,
    "Views (billions)": views,
})

#Interpreting scraped details
df_most_viewed_yt_videos<jupyter_output><empty_output><jupyter_text>Q2. Scrape the details team India’s international fixtures from bcci.tv. Url = https://www.bcci.tv/.**You need to find following details:****A) Match title (I.e. 1st ODI)****B) Series****C) Place****D) Date****E) Time*****Note: - From bcci.tv home page you have reach to the international fixture page through code.***<jupyter_code>#initializing webdriver with chromedriver
driver = webdriver.Chrome('chromedriver.exe')

#initializing driver with url https://www.bcci.tv/
driver.get('https://www.bcci.tv')

#Clicking on menu International->Fixtures
international = WebDriverWait(driver,10).until(ec.element_to_be_clickable((By.XPATH,'//li[@data-nav-index="0"]')))
international.click()
international.find_element_by_xpath('.//a[contains(text(),"Fixtures")]').click()

#Intializing infomation container which needs to be scraped
fixture_containers = WebDriverWait(driver,10).until(ec.visibility_of_all_elements_located((By.XPATH,'//div[@class="js-list"]/a')))

#scraping required details
match_titles = []
serieses = []
places = []
dates = []
times = []

for fixture in fixture_containers:
    #scraping match title
    title = fixture.find_element_by_xpath('.//p[@class="fixture__additional-info"]/strong')
    match_titles.append(title.text)
    
    #scraping series
    series = fixture.find_element_by_xpath('.//div[@class="fixture__format-strip"]')
    serieses.append(series.text.replace('\n',' - '))
    
    #scraping place
    place = fixture.find_element_by_xpath('.//p[@class="fixture__additional-info"]/span')
    places.append(place.text)
    
    #scraping date
    day = fixture.find_element_by_xpath('.//span[@class="fixture__day"]')
    month = fixture.find_element_by_xpath('.//span[@class="fixture__month"]')
    date = fixture.find_element_by_xpath('.//span[@class="fixture__date"]')
    dt = f'{day.text}, {date.text} {month.text}'
    dates.append(dt)
    
    #scraping time
    time = fixture.find_element_by_xpath('.//span[@class="fixture__time"]')
    times.append(time.text)
    

#closing driver
driver.close()

#storing scraped data into dataframe
df_fixtures_international = pd.DataFrame({
    "Match Title": match_titles,
    "Series": serieses,
    "Place": places,
    "Date": dates,
    "Time": times,
})

#Interpreting scraped data
df_fixtures_international<jupyter_output><empty_output><jupyter_text>Q3. Scrape the details of selenium exception from guru99.com. Url = https://www.guru99.com/**You need to find following details:****A) Name****B) Description*****Note: - From guru99 home page you have to reach to selenium exception handling page through code.***<jupyter_code>#initializing webdriver with chromedriver
driver = webdriver.Chrome('chromedriver.exe')

#initializing driver with url https://www.guru99.com
driver.get('https://www.guru99.com')

#clicking on selenium exception handling page ie. testing->selenium->selenium exception handling
#WebDriverWait(driver,10).until(ec.element_to_be_clickable((By.XPATH,'//span[@class="g-menu-item-title" and contains(text(),"Testing")]'))).click()
elem = driver.find_element_by_xpath('//nav[@class="g-main-nav"]/ul/li[2]')
ActionChains(driver).move_to_element(elem).perform()
WebDriverWait(driver,20).until(ec.element_to_be_clickable((By.XPATH,'//span[@class="g-menu-item-title" and contains(text(),"Selenium")]'))).click()

#searching for selenium exception handling and clicking on it
WebDriverWait(driver,10).until(ec.element_to_be_clickable((By.XPATH,'//td[contains(text(),"Selenium Exception Handling")]/preceding-sibling::td/a'))).click()

#scraping required details
names = []
descriptions = []

exception_containers = driver.find_elements_by_xpath('//table[@class="table table-striped"]/tbody/tr')

for exception in exception_containers[1:]:
    cols = exception.find_elements_by_xpath('.//td')
    names.append(cols[0].text)
    descriptions.append(cols[1].text)
    

#closing driver
driver.close()

#storing records into dataframe
df_exceptions = pd.DataFrame({
    "Name": names,
    "Description": descriptions,
})

#Interpreting exceptions details
df_exceptions<jupyter_output><empty_output><jupyter_text>Q4. Scrape the details of State-wise GDP of India from statisticstime.com. Url = http://statisticstimes.com/**You have to find following details:****A) Rank****B) State****C) GSDP(18-19)****D) GSDP(17-18)****E) Share(2017)****F) GDP($ billion)*****Note: - From statisticstimes home page you have to reach to economy page through code.***<jupyter_code>#initializing webdriver with chromedriver
driver = webdriver.Chrome('chromedriver.exe')

#initializing driver with url http://statisticstimes.com
driver.get('http://statisticstimes.com')

#clickig on economy -> india
WebDriverWait(driver,3).until(ec.element_to_be_clickable((By.XPATH,'//button[@class="dropbtn" and contains(text(),"Economy")]'))).click()
WebDriverWait(driver,3).until(ec.element_to_be_clickable((By.XPATH,'//button[@class="dropbtn" and contains(text(),"Economy")]/following-sibling::div/a[contains(text(),"India")]'))).click()

#manually close the ad pop-up

#clicking on link GDP of Indian States
WebDriverWait(driver,10).until(ec.element_to_be_clickable((By.XPATH,'//a[contains(text(),"GDP of Indian states")]'))).click()

#initializing information container
gdp_containers = driver.find_elements_by_xpath('//table[@id="table_id"]/tbody/tr')

#scraping required data
ranks = []
states = []
gsdp_19_20 = []
gsdp_18_19 = []
shares = []
gdp_billions = []

for gdp in gdp_containers:
    cols = gdp.find_elements_by_xpath('.//td')
    
    #scraping rank
    ranks.append(cols[0].text)
    
    #scraping state
    states.append(cols[1].text)
    
    #scraping GSDP (19-20)
    gsdp_19_20.append(cols[2].text)
    
    #scraping GSDP (18-19)
    gsdp_18_19.append(cols[3].text)
    
    #scraping Share
    shares.append(cols[4].text)
    
    #scraping GDP ($billion)
    gdp_billions.append(cols[5].text)
    
    
#closing driver
driver.close()

#storing scraped data into dataframe
df_gdp_indian_states = pd.DataFrame({
    "Rank": ranks,
    "State": states,
    "GSDP (19-20)": gsdp_19_20,
    "GSDP (18-19)": gsdp_18_19,
    "Share (18-19)": shares,
    "GDP ($billion)": gdp_billions,
})

#interpreting scraped gdp data
df_gdp_indian_states<jupyter_output><empty_output><jupyter_text>Q5. Scrape the details of trending repositories on Github.com. Url = https://github.com/**You have to find the following details:**    **A) Repository title****B) Repository description****C) Contributors count****D) Language used*****Note: - From the home page you have to click on the trending option from Explore menu through code.***<jupyter_code>#initializing webdriver with chromedriver
driver = webdriver.Chrome('chromedriver.exe')

#initializing driver with url https://github.com
driver.get('https://github.com')

#clicking on explore
explore = WebDriverWait(driver,5).until(ec.element_to_be_clickable((By.XPATH,'//summary[contains(text(),"Explore")]')))
ActionChains(driver).move_to_element(explore).perform() #hovering over explore menu
explore.click()
#clicking on trending
WebDriverWait(driver,5).until(ec.element_to_be_clickable((By.XPATH,'//li/a[contains(text(),"Trending")]'))).click()

#initializing trending repository containers
repository_containers = driver.find_elements_by_xpath('//article[@class="Box-row"]')

#storing original window handle
original_window = driver.current_window_handle

#scraping required details
repository_titles = []
repository_descriptions = []
contributor_counts = []
languages_used = []
for repo in repository_containers:
    #scraping repository title
    try:
        title = repo.find_element_by_xpath('.//h1/a')
        repository_titles.append(title.text)
    except NoSuchElementException:
        repository_titles.append('-')
        
    #scraping repository description
    try:
        description = repo.find_element_by_xpath('.//p')
        repository_descriptions.append(description.text)
    except NoSuchElementException:
        repository_descriptions.append('-')
        
    #scraping languages used
    try:
        language = repo.find_element_by_xpath('.//span[@itemprop="programmingLanguage"]')
        languages_used.append(language.text)
    except NoSuchElementException:
        languages_used.append('-')
        
    #scraping contributors count
    try:
        title.send_keys(Keys.CONTROL + Keys.ENTER)
        driver.switch_to.window(driver.window_handles[-1])
        contributor = WebDriverWait(driver,10).until(ec.element_to_be_clickable((By.XPATH,'//a[contains(text(),"Contributors")]/span[@class="Counter "]')))
        contributor_counts.append(contributor.text)
        
        driver.close()
        driver.switch_to.window(original_window)
    except TimeoutException:
        contributor_counts.append('-')
        driver.close()
        driver.switch_to.window(original_window)
        
        
#closing driver
driver.close()

#storing scraped data into dataframe
df_trending_repositories = pd.DataFrame({
    "Repository Title": repository_titles,
    "Repository Description": repository_descriptions,
    "Contributor Counts": contributor_counts,
    "Language Used": languages_used,
})

#interpreting scraped data
df_trending_repositories<jupyter_output><empty_output><jupyter_text>Q6. Scrape the details of top 100 songs on billiboard.com. Url = https://www.billboard.com/**You have to find the following details:****A) Song name****B) Artist name****C) Last week rank****D) Peak rank****E) Weeks on board*****Note: - From the home page you have to click on the charts option then hot 100-page link through code***<jupyter_code>#intializing webdriver with chromedriver
driver = webdriver.Chrome('chromedriver.exe')

#initializing driver with url https://www.billboard.com/
driver.get('https://www.billboard.com')

#intializing the CHARTS menu
containers = driver.find_elements_by_xpath("//li/a[normalize-space(text()) = 'Charts']")
for x in containers:
    if x.text == "CHARTS":
        chart = x
        break
        
#hovering over chart menu
ActionChains(driver).move_to_element(chart).perform()

#initializing and clicking on hot 100 link
hot_100 = driver.find_elements_by_xpath('//a[@data-track-action="main-nav" and text()="Hot 100"]')
hot_100[1].click()

#initializing top 100 songs containers
song_containers = WebDriverWait(driver,10).until(ec.presence_of_all_elements_located((By.XPATH,'//ol[@class="chart-list__elements"]/li')))

#scraping song details
song_names = []
artist_names = []
last_week_ranks = []
peak_ranks = []
weeks_on_boards = []
for song in song_containers:
    #scraping song name
    try:
        song_name = song.find_element_by_xpath('.//span[@class="chart-element__information__song text--truncate color--primary"]')
        song_names.append(song_name.text)
    except NoSuchElementException:
        song_names.append('-')
        
    #scraping artist name
    try:
        artist_name = song.find_element_by_xpath('.//span[@class="chart-element__information__artist text--truncate color--secondary"]')
        artist_names.append(artist_name.text)
    except NoSuchElementException:
        artist_names.append('-')
        
    #scraping last week rank
    try:
        last_week_rank = song.find_element_by_xpath('.//div[@class="chart-element__meta text--center color--secondary text--last"]')
        last_week_ranks.append(last_week_rank.text)
    except NoSuchElementException:
        last_week_ranks.append(last_week_rank.text)
        
    #scraping peak rank
    try:
        peak_rank = song.find_element_by_xpath('.//div[@class="chart-element__meta text--center color--secondary text--peak"]')
        peak_ranks.append(peak_rank.text)
    except NoSuchElementException:
        peak_ranks.append('-')
        
    #scraping weeks on board
    try:
        weeks_on_board = song.find_element_by_xpath('.//div[@class="chart-element__meta text--center color--secondary text--week"]')
        weeks_on_boards.append(weeks_on_board.text)
    except:
        weeks_on_boards.append('-')
        
        
#closing driver
driver.close()

#storing scraped data into dataframe
df_top_100_songs = pd.DataFrame({
    "Song Name": song_names,
    "Artist Name": artist_names,
    "Last Week Rank": last_week_ranks,
    "Peak Rank": peak_ranks,
    "Weeks on Board": weeks_on_boards,
})

#interpreting scraped data
df_top_100_songs<jupyter_output><empty_output><jupyter_text>Q7. Scrape the details of Data science recruiters from naukri.com. Url = https://www.naukri.com/**You have to find the following details:**    **A) Name****B) Designation****C) Company****D) Skills they hire for****E) Location*****Note: - From naukri.com homepage click on the recruiters option and the on the search pane type Data science and click on search. All this should be done through code***<jupyter_code>#initializing webdriver with chromedriver
driver = webdriver.Chrome('chromedriver.exe')

#initializing driver with url https://www.naukri.com
driver.get('https://www.naukri.com')

#clicking on recruiters option
WebDriverWait(driver,10).until(ec.element_to_be_clickable((By.XPATH,'//li/a[@title="Search Recruiters"]'))).click()

#storing current window handle and switching to new opened window
original_window = driver.current_window_handle
driver.switch_to.window(driver.window_handles[-1])

#initializing search box with Data science
driver.find_element_by_xpath('//input[@name="qp"]').send_keys('Data science')

#clicking on search button
driver.find_element_by_xpath('//button[@id="qsbFormBtn"]').click()

#initializing recruiter containers
recruiter_containers = WebDriverWait(driver,10).until(ec.presence_of_all_elements_located((By.XPATH,'//div[@id="tabP-1"]/div[@class="outerRecSec"]')))

#scraping the required details
names = []
designations = []
companies = []
skills_they_hires = []
locations = []
for recruiter in recruiter_containers:
    cols = recruiter.find_elements_by_class_name('recSec')
    for col in cols:
        #scraping name
        try:
            name = col.find_element_by_xpath('.//span[@class="fl ellipsis"]')
            names.append(name.text)
        except NoSuchElementException:
            names.append('-')
            
        #scraping designation
        try:
            designation = col.find_element_by_xpath('.//span[@class="ellipsis clr"]')
            designations.append(designation.text)
        except NoSuchElementException:
            designations.append('-')
            
        #scraping company
        try:
            company = col.find_element_by_xpath('.//a[@class="ellipsis"][2]')
            companies.append(company.text)
        except NoSuchElementException:
            companies.append('-')
            
        #scraping skills they hire for
        try:
            skills = col.find_element_by_xpath('.//div[@class="hireSec highlightable"]')
            skills_they_hires.append(skills.text)
        except NoSuchElementException:
            skills_they_hires.append('-')
            
        #scraping location
        try:
            location = col.find_element_by_xpath('.//small[@class="ellipsis"]')
            locations.append(location.text)
        except NoSuchElementException:
            locations.append('-')
            
#closing driver
driver.close()
driver.switch_to.window(original_window)
driver.close()

#storing scraped data into dataframe
df_recruiters = pd.DataFrame({
    "Name": names,
    "Designation": designations,
    "Company": companies,
    "Skills They Hire For": skills_they_hires,
    "Location": locations,
})

#interpreting scraped data
df_recruiters<jupyter_output><empty_output><jupyter_text>Q8. Scrape the details of Highest selling novels. Url = https://www.theguardian.com/news/datablog/2012/aug/09/best-selling-books-all-time-fifty-shades-grey-compare/**You have to find the following details:**    **A) Book name****B) Author name****C) Volumes sold****D) Publisher****E) Genre**<jupyter_code>#initializing webdriver with chromedriver
driver = webdriver.Chrome('chromedriver.exe')

#initializing driver with url https://www.theguardian.com/news/datablog/2012/aug/09/best-selling-books-all-time-fifty-shades-grey-compare
driver.get('https://www.theguardian.com/news/datablog/2012/aug/09/best-selling-books-all-time-fifty-shades-grey-compare')

#initializing novel containers
novel_containers = WebDriverWait(driver,10).until(ec.presence_of_all_elements_located((By.XPATH,'//table[@class="in-article sortable"]/tbody/tr')))

#scraping required details
book_names = []
author_names = []
volume_solds = []
publishers = []
genres = []

for novel in novel_containers:
    cols = novel.find_elements_by_xpath('.//td')
    
    #scraping book name
    try:
        book_names.append(cols[1].text)
    except NoSuchElementException:
        book_names.append('-')
        
    #scraping author name
    try:
        author_names.append(cols[2].text)
    except NoSuchElementException:
        author_names.append('-')
        
    #scraping volumes sold
    try:
        volume_solds.append(cols[3].text)
    except NoSuchElementException:
        volume_solds.append('-')
        
    #scraping publishers
    try:
        publishers.append(cols[4].text)
    except NoSuchElementException:
        publishers.append('-')
        
    #scraping genres
    try:
        genres.append(cols[5].text)
    except NoSuchElementException:
        genres.append('-')
        
#closing driver
driver.close()

#storing scraped data into dataframe
df_highest_selling_novels = pd.DataFrame({
    "Book Name": book_names,
    "Author Name": author_names,
    "Volumes Sold": volume_solds,
    "Publisher": publishers,
    "Genre": genres,
})

#interpreting scraped data
df_highest_selling_novels<jupyter_output><empty_output><jupyter_text>Q9. Scrape the details most watched tv series of all time from imdb.com. Url = https://www.imdb.com/list/ls095964455/**You have to find the following details:**    **A) Name****B) Year span****C) Genre****D) Run time****E) Ratings****F) Votes**<jupyter_code>#initializing webdriver with chromedriver
driver = webdriver.Chrome('chromedriver.exe')

#initializing driver with url https://www.imdb.com/list/ls095964455/
driver.get('https://www.imdb.com/list/ls095964455/')

#initializing most watched tv series containers
series_containers = WebDriverWait(driver,10).until(ec.presence_of_all_elements_located((By.XPATH,'//div[@class="lister-item-content"]')))

#scraping required details
names = []
year_spans = []
genres = []
run_times = []
ratings = []
votes = []

for series in series_containers:
    #scraping name
    try:
        name = series.find_element_by_xpath('.//h3[@class="lister-item-header"]/a')
        names.append(name.text)
    except NoSuchElementException:
        names.append('-')
        
    #scraping year span
    try:
        year_span = series.find_element_by_xpath('.//span[@class="lister-item-year text-muted unbold"]')
        year_spans.append(year_span.text)
    except NoSuchElementException:
        year_spans.append('-')
        
    #scraping genre
    try:
        genre = series.find_element_by_xpath('.//span[@class="genre"]')
        genres.append(genre.text)
    except NoSuchElementException:
        genres.append('-')
        
    #scraping run time
    try:
        run_time = series.find_element_by_xpath('.//span[@class="runtime"]')
        run_times.append(run_time.text)
    except NoSuchElementException:
        run_times.append('-')
        
    #scraping rating
    try:
        rating = series.find_element_by_xpath('.//span[@class="ipl-rating-star__rating"]')
        ratings.append(rating.text)
    except NoSuchElementException:
        ratings.append('-')
        
    #scraping vote
    try:
        vote = series.find_element_by_xpath('.//span[@name="nv"]')
        votes.append(vote.text)
    except NoSuchElementException:
        votes.append('-')
        
#closing driver
driver.close()

#storing scraped data into dataframe
df_most_watched_tv_series = pd.DataFrame({
    "Name": names,
    "Year Span": year_spans,
    "Genre": genres,
    "Run Time": run_times,
    "Ratings": ratings,
    "Votes": votes,
})

#interpreting scraped data
df_most_watched_tv_series<jupyter_output><empty_output><jupyter_text>Q10. Details of Datasets from UCI machine learning repositories. Url = https://archive.ics.uci.edu/**You have to find the following details:**    **A) Dataset name****B) Data type****C) Task****D) Attribute type****E) No of instances****F) No of attribute****G) Year****Note: - from the home page you have to go to the Show All Dataset page through code**<jupyter_code>#initializing webdriver with chromedriver
driver = webdriver.Chrome('chromedriver.exe')

#initializing driver with url https://archive.ics.uci.edu/
driver.get('https://archive.ics.uci.edu')

#click on link view all data sets
WebDriverWait(driver,10).until(ec.element_to_be_clickable((By.XPATH,'//b[contains(text(),"View ALL Data Sets")]'))).click()

#initializing dataset containers
dataset_containers = WebDriverWait(driver,10).until(ec.presence_of_all_elements_located((By.XPATH,'//body/table[2]/tbody/tr/td[2]/table[2]/tbody/tr')))

#scraping required details
names = []
data_types = []
tasks = []
attribute_types = []
instances = []
attributes = []
years = []

for dataset in dataset_containers[1:]:
    cols = dataset.find_elements_by_xpath('./td')
    
    #scraping dataset name
    try:
        names.append(cols[0].text.strip())
    except NoSuchElementException:
        names.append('-')
        
    #scraping data types
    try:
        data_types.append(cols[1].text.strip())
    except NoSuchElementException:
        data_types.append('-')
        
    #scraping task
    try:
        tasks.append(cols[2].text.strip())
    except NoSuchElementException:
        tasks.append('-')
        
    #scraping attribute types
    try:
        attribute_types.append(cols[3].text.strip())
    except NoSuchElementException:
        attribute_types.append('-')
        
    #scraping number of instances
    try:
        instances.append(cols[4].text.strip())
    except NoSuchElementException:
        instances.append('-')
        
    #scraping number of attributes
    try:
        attributes.append(cols[5].text.strip())
    except NoSuchElementException:
        attributes.append('-')
        
    #scraping year
    try:
        years.append(cols[6].text.strip())
    except NoSuchElementException:
        years.append('-')

#closing driver
driver.close()

#storing scraped data in dataframe
df_datasets = pd.DataFrame({
    "Dataset Name": names,
    "Data Types": data_types,
    "Task": tasks,
    "Atrribute Types": attribute_types,
    "No. of Instances": instances,
    "No. of Attributes": attributes,
    "Year": years,
})

#interpreting scraped data
df_datasets<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13620100502512564, 'fraction_numerical': 0.009085427135678392, 'mean_word_length': 4.210724759111856, 'pattern_counts': {'":': 51, '<': 45, '<?xml version=': 0, '>': 49, 'https://': 27, 'lorem ipsum': 0, 'www.': 18, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '542667', 'license': 'MIT', 'n_tokens_mistral': 8228, 'n_tokens_neox': 7984, 'n_words': 1546, 'path': 'solution-web-scarping-assignment-4.ipynb', 'repo_name': 'ashok-kumar-sharma/ds-internship', 'size': '24875'}