<jupyter_start><jupyter_text>MISSON TO MARS<jupyter_code># Dependencies
import pandas as pd
import os
import requests
import time

from bs4 import BeautifulSoup
from bs4 import BeautifulSoup as bs

from splinter import Browser
from webdriver_manager.chrome import ChromeDriverManager
## NASA Mars News
## Setup splinter, executable path, and Chrome web driver browser
executable_path = {'executable_path': ChromeDriverManager().install()}
browser = Browser('chrome', **executable_path, headless=False)

## URL of page to scrape
url = 'https://redplanetscience.com/'
browser.visit(url)

## HTML setup
NewsHTML = browser.html
NewsSoup = BeautifulSoup(NewsHTML, 'html.parser')

## Scrape article headline title and paragraph text/blerb
NewsTitle = NewsSoup.find('div', class_ = 'col-md-12').find(class_='content_title').text
NewsParagraph = NewsSoup.find('div', class_ = 'col-md-12').find(class_='article_teaser_body').text

NewsParagraph = str(NewsParagraph)
NewsTitle = str(NewsTitle)
print(f"News Title: {NewsTitle}")
print(f"Paragraph Blerb: '{NewsParagraph}'")
browser.quit()<jupyter_output>News Title: Sensors on Mars 2020 Spacecraft Answer Long-Distance Call From Earth
Paragraph Blerb: 'Instruments tailored to collect data during the descent of NASA's next rover through the Red Planet's atmosphere have been checked in flight.'<jupyter_text>JPL Mars Space Images<jupyter_code>## Setup splinter, executable path, and Chrome web driver browser
executable_path = {'executable_path': ChromeDriverManager().install()}
browser = Browser('chrome', **executable_path, headless=False)

## URL of page to scrape
url = 'https://spaceimages-mars.com/'
browser.visit(url)

## HTML setup
JPLMarsHTML = browser.html
JPLMarsSoup = BeautifulSoup(JPLMarsHTML, 'html.parser')
## Scrape website image path
JPLMarsImageHTML = JPLMarsSoup.find('div', class_ = 'header').find(class_='floating_text_area')
JPLMarsImageSRC = JPLMarsImageHTML.find('a')['href']

## Create image url by combining base URL with returned image path
featured_image_url = f"{url}{JPLMarsImageSRC}" 
print(featured_image_url)
browser.quit()<jupyter_output>https://spaceimages-mars.com/image/featured/mars2.jpg<jupyter_text>Mars Facts<jupyter_code>## URL of page to scrape
url = 'https://galaxyfacts-mars.com/'

GalaxyTable_DF = pd.read_html(url)
GalaxyTable_DF = GalaxyTable_DF[1]
#GalaxyTable_DF.head()

GalaxyTable_DF.rename(columns = {0:'MARS DESCRIPTION', 1:'VALUE'}, inplace=True)
#GalaxyTable_DF = GalaxyTable_DF.set_index('MARS DESCRIPTION')
GalaxyTable_DF
## Converting dataframe back into HTML and removing line breaks ('\n')
htmlTable = GalaxyTable_DF.to_html().replace('\n',"")
htmlTable<jupyter_output><empty_output><jupyter_text>Mars Hemispheres<jupyter_code>## Setup splinter, executable path, and Chrome web driver browser
executable_path = {'executable_path': ChromeDriverManager().install()}
browser = Browser('chrome', **executable_path, headless=False)

## URL of page to scrape
#Hemisphere_url = 'https://marshemispheres.com/'
MarsHemis_url = 'https://marshemispheres.com/'
browser.visit(MarsHemis_url)

## HTML setup
MarsHemisHTML = browser.html
MarsHemisSoup = BeautifulSoup(MarsHemisHTML, 'html.parser')

## Returns the HTML code containing ALL page links/div class
MarsHemisphereHTML = MarsHemisSoup.find_all('div', {'class' : 'item'})



## For Loop and Dictionary Setup
links = MarsHemisphereHTML
Hemisphere_Image_URLS = []

for link in range(len(links)):

    ## Code that select a SPECIFIC Hemisphere page based on index (Cerberus, Schiaparelli, Syrtis, Valles)
    NextMarsHemis = MarsHemisphereHTML[link]


    ## Scrape for the Hemisphere/image titles on homepage to add to dictionary
    MarsHemisImageTitle = NextMarsHemis.find('h3').text
    title = MarsHemisImageTitle
    #print(title)


    ## WebScraper command to go down the html page list by clicking the specific/current index "h3" link
    browser.find_by_tag("h3")[link].click()
    
    ## HTML Beautiful soup re-initialization for new hemisphere page
    MarsHemisHTML = browser.html
    MarsHemisSoup = BeautifulSoup(MarsHemisHTML, 'html.parser')


    ## Find Hemisphere partial image path
    HemisphereImageSRC = MarsHemisSoup.find('img', class_ = 'wide-image')['src']
    #HemisphereImageSRC


    ## Create full image url by combining base URL with returned image path
    FullHemisphereImageURL = f"{MarsHemis_url}{HemisphereImageSRC}" 
    img_url = FullHemisphereImageURL
    #print(img_url)

    ## Return back to homepage for next loop step
    browser.back()

    ## HTML Beautiful soup re-initialization for homepage to setup next loop
    MarsHemisHTML = browser.html
    MarsHemisSoup = BeautifulSoup(MarsHemisHTML, 'html.parser')


    ### Append scraped Title/Image Url data to dictionary
    Hemisphere_Image_URLS.append({ "Title": title, "IMG URL": img_url })
    ## End of page scrape, next loop iteration
    

## Close webscrape session
browser.quit()

## Check/print dictionary scrape results
print("Scrape Complete")
print()
#Hemisphere_Image_URLS

for x in Hemisphere_Image_URLS:
    for key, value in x.items():
        print(f"{key} : {value}")<jupyter_output>[WDM] - ====== WebDriver manager ======
[WDM] - Current google-chrome version is 90.0.4430
[WDM] - Get LATEST driver version for 90.0.4430
[WDM] - Driver [C:\Users\ithmo\.wdm\drivers\chromedriver\win32\90.0.4430.24\chromedriver.exe] found in cache


Scrape Complete

Title : Cerberus Hemisphere Enhanced
IMG URL : https://marshemispheres.com/images/f5e372a36edfa389625da6d0cc25d905_cerberus_enhanced.tif_full.jpg
Title : Schiaparelli Hemisphere Enhanced
IMG URL : https://marshemispheres.com/images/3778f7b43bbbc89d6e3cfabb3613ba93_schiaparelli_enhanced.tif_full.jpg
Title : Syrtis Major Hemisphere Enhanced
IMG URL : https://marshemispheres.com/images/555e6403a6ddd7ba16ddb0e471cadcf7_syrtis_major_enhanced.tif_full.jpg
Title : Valles Marineris Hemisphere Enhanced
IMG URL : https://marshemispheres.com/images/b3c7c6c9138f57b4756be9b9c43e3a48_valles_marineris_enhanced.tif_full.jpg<jupyter_text>End<jupyter_code>## Test of all variables/outputs into dictionary 
ScrapeData = {}

ScrapeData["News Title"] = NewsTitle
ScrapeData["News Paragraph"] = NewsParagraph
ScrapeData["Featured Image"] = featured_image_url
ScrapeData["Mars Hemispheres"] = Hemisphere_Image_URLS
ScrapeData["Mars Facts"] = GalaxyTable_DF
ScrapeData["Mars Table HMTL"] = htmlTable

ScrapeData
## --------------------------------------------------------------- <NAME> ------------------------------------------------- ##<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12210365853658536, 'fraction_numerical': 0.017378048780487806, 'mean_word_length': 5.302593659942363, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '495362', 'license': 'ADSL', 'n_tokens_mistral': 2316, 'n_tokens_neox': 2150, 'n_words': 518, 'path': 'Mission_to_Mars.ipynb', 'repo_name': 'IthamarFrancois/WebScraping-Mars-Challenge', 'size': '6570'}