<jupyter_start><jupyter_text>Basic Setup<jupyter_code>#Dependicies
import time
from splinter import Browser
from bs4 import BeautifulSoup as bs
import pandas as pd
import requests
from config import path
#setup chromedriver
executable_path = {"executable_path":f"{path}"}
browser=Browser("chrome", **executable_path, headless=True)<jupyter_output><empty_output><jupyter_text>Scraping Mars News Site<jupyter_code>#Mars news URL to be scraped
mars_news_page='https://mars.nasa.gov/news/'
browser.visit(mars_news_page)
#wait a moment for the page to be redirected
time.sleep(1)

html=browser.html
news_soup=bs(html,'html.parser')
#print(html)
#Retrieve Title and teaser paragraph

news_title=news_soup.find_all('div',class_='content_title')[1].text
news_p=news_soup.find_all('div',class_="article_teaser_body")[0].text

print(news_title)
print('-----------------------------')
print(news_p)<jupyter_output>NASA's Perseverance Rover Is Midway to Mars 
-----------------------------
Sometimes half measures can be a good thing – especially on a journey this long. The agency's latest rover only has about 146 million miles left to reach its destination.<jupyter_text>Scrap JPL Featured Mars Space Images<jupyter_code>#JPL image URL to be scraped
jpl_url='https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'
browser.visit(jpl_url)
html=browser.html
image_soup=bs(html,'html.parser')
#Retrieve Image URL
relative_img_path=image_soup.find_all('img')[3]['src']
featured_image=f"https://www.jpl.nasa.gov{relative_img_path}"
print(featured_image)<jupyter_output>https://www.jpl.nasa.gov/spaceimages/images/wallpaper/PIA24189-640x350.jpg<jupyter_text>Scrap Mars Facts<jupyter_code>facts_url='https://space-facts.com/mars/'
tables = pd.read_html(facts_url)
tables
mars_facts_df=tables[0]
mars_facts_df.columns =['Description','Value']
mars_facts_df
mars_fact_html=mars_facts_df.to_html()
mars_fact_html.replace('\n','')
print(mars_fact_html)<jupyter_output><table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Description</th>
      <th>Value</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Equatorial Diameter:</td>
      <td>6,792 km</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Polar Diameter:</td>
      <td>6,752 km</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Mass:</td>
      <td>6.39 × 10^23 kg (0.11 Earths)</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Moons:</td>
      <td>2 (Phobos &amp; Deimos)</td>
    </tr>
    <tr>
      <th>4</th>
      <td>Orbit Distance:</td>
      <td>227,943,824 km (1.38 AU)</td>
    </tr>
    <tr>
      <th>5</th>
      <td>Orbit Period:</td>
      <td>687 days (1.9 years)</td>
    </tr>
    <tr>
      <th>6</th>
      <td>Surface Temperature:</td>
      <td>-87 to -5 °C</td>
    </tr>
    <tr>
      <th>7</th>
      <td>First Record:</td>
      <td>2nd millennium BC</td>
    </tr>
    <tr>
      <th>8</th>
    [...]<jupyter_text>Scraping Mars Hemisphere Images<jupyter_code>hemi_url='https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
browser.visit(hemi_url)
html=browser.html
hemi_soup=bs(html,'html.parser')
hemi_url=[]

items = hemi_soup.find_all('div', class_='item')

for i in items:
    hemisphere = i.find('div',class_='description')
    title = hemisphere.h3.text
    
    hemisphere_link=hemisphere.a['href']
    browser.visit(f'https://astrogeology.usgs.gov{hemisphere_link}')
    
    time.sleep(1)
    
    image_html=browser.html
    image_soup=bs(image_html,'html.parser')
    
    image_url=image_soup.find('div',class_='downloads').find('li').a['href']
    
    hemi_url.append({'title':title,'img_url':image_url})
    
print(hemi_url)<jupyter_output>[{'title': 'Cerberus Hemisphere Enhanced', 'img_url': 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/cerberus_enhanced.tif/full.jpg'}, {'title': 'Schiaparelli Hemisphere Enhanced', 'img_url': 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/schiaparelli_enhanced.tif/full.jpg'}, {'title': 'Syrtis Major Hemisphere Enhanced', 'img_url': 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/syrtis_major_enhanced.tif/full.jpg'}, {'title': 'Valles Marineris Hemisphere Enhanced', 'img_url': 'https://astropedia.astrogeology.usgs.gov/download/Mars/Viking/valles_marineris_enhanced.tif/full.jpg'}]<jupyter_text>Putting it all together<jupyter_code>mars_dict={
    "news_title":news_title,
    "news_p":news_p,
    "featured_image":featured_image,
    "fact_table":str(mars_fact_html),
    "hemisphere_images":hemi_url
}
mars_dict
browser.quit()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17168027503223035, 'fraction_numerical': 0.015900300816501935, 'mean_word_length': 4.3939745075318655, 'pattern_counts': {'":': 6, '<': 100, '<?xml version=': 0, '>': 100, 'https://': 11, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '653221', 'license': 'ADSL', 'n_tokens_mistral': 1872, 'n_tokens_neox': 1796, 'n_words': 275, 'path': 'ScrapingMars.ipynb', 'repo_name': 'JordanSchafer/web_scrapping_challenge', 'size': '4654'}