<jupyter_start><jupyter_text>MISSION TO MARS...  A Web Scrapping adventure...<jupyter_code>import time
import pandas as pd
import numpy as np
import requests as req
import os

from splinter import Browser
from bs4 import BeautifulSoup as bs
from selenium import webdriver
from urllib.parse import urlsplit<jupyter_output><empty_output><jupyter_text>NASA News<jupyter_code>#scrape NASA Mars News Site to collect the information
executable_path = {'executable_path': '/usr/local/bin/chromedriver'}
browser = Browser('chrome', **executable_path, headless=False)

nasa_url = 'https://mars.nasa.gov/news/'
browser.visit(nasa_url)
time.sleep(1)
nasa_html = browser.html
nasa_soup = bs(nasa_html, 'html.parser')

news_list = nasa_soup.find('ul', class_='item_list')
first_item = news_list.find('li', class_='slide')
nasa_headline = first_item.find('div', class_='content_title').text
nasa_teaser = first_item.find('div', class_='article_teaser_body').text
print(nasa_headline)
print(nasa_teaser)<jupyter_output>Six Things About Opportunity's Recovery Efforts
The global dust storm on Mars could soon let in enough sunlight for the Opportunity rover to recharge.<jupyter_text>NASA - Featured Mars Image<jupyter_code>#Getting the Space images from the provided URL - Use Splinter to scrap the site and look for for the 
#current Featured Mars Image
url = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
browser.visit(url) 
html = browser.html
soup = bs(html, "html.parser")
browser.click_link_by_partial_text('FULL IMAGE')
time.sleep(5)
browser.click_link_by_partial_text('more info')
new_html = browser.html
new_soup = bs(new_html, 'html.parser')
temp_img_url = new_soup.find('img', class_='main_image')
back_half_img_url = temp_img_url.get('src')

recent_mars_image_url = "https://www.jpl.nasa.gov" + back_half_img_url

print(recent_mars_image_url)<jupyter_output>https://www.jpl.nasa.gov/spaceimages/images/largesize/PIA20464_hires.jpg<jupyter_text>Mars Weather - Via Twitter<jupyter_code>mars_weather_url = 'https://twitter.com/marswxreport?lang=en'
browser.visit(mars_weather_url)
time.sleep(1)
mars_weather_html = browser.html
mars_weather_soup = bs(mars_weather_html, 'html.parser')

tweets = mars_weather_soup.find('ol', class_='stream-items')
mars_weather = tweets.find('p', class_="tweet-text").text
print(mars_weather)<jupyter_output>A view of this morning’s launch of the Parker #SolarProbe launch from about 3 miles awayhttps://youtu.be/hDhIane1Rso<jupyter_text>MARS Facts --><jupyter_code>#Mars Facts....scrape the page to get the facts
request_mars_space_facts = req.get("https://space-facts.com/mars/")
mars_space_table_read = pd.read_html(request_mars_space_facts.text)
mars_space_table_read
df = mars_space_table_read[0]
df.columns = ["Facet", "Value"]
df.set_index("Facet", inplace=True)
mars_facts = df
mars_facts
mars_html = mars_facts.to_html()
mars_html
mars_html.replace('\n', '')
mars_facts.to_html('fact_table.html')<jupyter_output><empty_output><jupyter_text>Mars Hemispheres<jupyter_code>#..Visit the USGS Astrogeology site to obtain hgih resolution images for 
#....each of Mar's hemispheres
url_hemisphere = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
browser.visit(url_hemisphere)
# Get base URL 
hemisphere_base_url = "{0.scheme}://{0.netloc}/".format(urlsplit(url_hemisphere))
print(hemisphere_base_url)<jupyter_output>https://astrogeology.usgs.gov/<jupyter_text>Cerberus Hemisfere - IMG<jupyter_code>hemisphere_img_urls = []
results = browser.find_by_xpath( "//*[@id='product-section']/div[2]/div[1]/a/img").click()
time.sleep(2)
cerberus_open_click = browser.find_by_xpath( "//*[@id='wide-image-toggle']").click()
time.sleep(1)
cerberus_image = browser.html
soup = bs(cerberus_image, "html.parser")
cerberus_url = soup.find("img", class_="wide-image")["src"]
cerberus_img_url = hemisphere_base_url + cerberus_url
print(cerberus_img_url)
cerberus_title = soup.find("h2",class_="title").text
print(cerberus_title)
back_button = browser.find_by_xpath("//*[@id='splashy']/div[1]/div[1]/div[3]/section/a").click()
cerberus = {"image title":cerberus_title, "image url": cerberus_img_url}
hemisphere_img_urls.append(cerberus)<jupyter_output>https://astrogeology.usgs.gov//cache/images/cfa62af2557222a02478f1fcd781d445_cerberus_enhanced.tif_full.jpg
Cerberus Hemisphere Enhanced<jupyter_text>Schiaparelli Hemisphere - IMG<jupyter_code>results1 = browser.find_by_xpath( "//*[@id='product-section']/div[2]/div[2]/a/img").click()
time.sleep(2)
schiaparelli_open_click = browser.find_by_xpath( "//*[@id='wide-image-toggle']").click()
time.sleep(1)
schiaparelli_image = browser.html
soup = bs(schiaparelli_image, "html.parser")
schiaparelli_url = soup.find("img", class_="wide-image")["src"]
schiaparelli_img_url = hemisphere_base_url + schiaparelli_url
print(schiaparelli_img_url)
schiaparelli_title = soup.find("h2",class_="title").text
print(schiaparelli_title)
back_button = browser.find_by_xpath("//*[@id='splashy']/div[1]/div[1]/div[3]/section/a").click()
schiaparelli = {"image title":schiaparelli_title, "image url": schiaparelli_img_url}
hemisphere_img_urls.append(schiaparelli)<jupyter_output>https://astrogeology.usgs.gov//cache/images/3cdd1cbf5e0813bba925c9030d13b62e_schiaparelli_enhanced.tif_full.jpg
Schiaparelli Hemisphere Enhanced<jupyter_text>Syrtis Major Hemisphere - IMG<jupyter_code>results1 = browser.find_by_xpath( "//*[@id='product-section']/div[2]/div[3]/a/img").click()
time.sleep(2)
syrtis_major_open_click = browser.find_by_xpath( "//*[@id='wide-image-toggle']").click()
time.sleep(1)
syrtis_major_image = browser.html
soup = bs(syrtis_major_image, "html.parser")
syrtis_major_url = soup.find("img", class_="wide-image")["src"]
syrtis_major_img_url = hemisphere_base_url + syrtis_major_url
print(syrtis_major_img_url)
syrtis_major_title = soup.find("h2",class_="title").text
print(syrtis_major_title)
back_button = browser.find_by_xpath("//*[@id='splashy']/div[1]/div[1]/div[3]/section/a").click()
syrtis_major = {"image title":syrtis_major_title, "image url": syrtis_major_img_url}
hemisphere_img_urls.append(syrtis_major)<jupyter_output>https://astrogeology.usgs.gov//cache/images/ae209b4e408bb6c3e67b6af38168cf28_syrtis_major_enhanced.tif_full.jpg
Syrtis Major Hemisphere Enhanced<jupyter_text>Valles Marineris Hemisphere - IMG<jupyter_code>results1 = browser.find_by_xpath( "//*[@id='product-section']/div[2]/div[4]/a/img").click()
time.sleep(2)
valles_marineris_open_click = browser.find_by_xpath( "//*[@id='wide-image-toggle']").click()
time.sleep(1)
valles_marineris_image = browser.html
soup = bs(valles_marineris_image, "html.parser")
valles_marineris_url = soup.find("img", class_="wide-image")["src"]
valles_marineris_img_url = hemisphere_base_url + syrtis_major_url
print(valles_marineris_img_url)
valles_marineris_title = soup.find("h2",class_="title").text
print(valles_marineris_title)
back_button = browser.find_by_xpath("//*[@id='splashy']/div[1]/div[1]/div[3]/section/a").click()
valles_marineris = {"image title":valles_marineris_title, "image url": valles_marineris_img_url}
hemisphere_img_urls.append(valles_marineris)
hemisphere_img_urls<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1427577570613608, 'fraction_numerical': 0.015166272436343399, 'mean_word_length': 8.32295719844358, 'pattern_counts': {'":': 8, '<': 34, '<?xml version=': 0, '>': 35, 'https://': 12, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '277927', 'license': 'ADSL', 'n_tokens_mistral': 2909, 'n_tokens_neox': 2788, 'n_words': 386, 'path': 'mars_scrape.ipynb', 'repo_name': 'hconstancio/Web_Scrapping_Mission2Mars', 'size': '7187'}