<jupyter_start><jupyter_text>Web Scraping<jupyter_code># Set the executable path and initialize the chrome browser in splinter
executable_path = {'executable_path': ChromeDriverManager().install()}
browser = Browser('chrome', **executable_path, headless=False)<jupyter_output>[WDM] - ====== WebDriver manager ======
[WDM] - Current google-chrome version is 89.0.4389
[WDM] - Get LATEST driver version for 89.0.4389
[WDM] - Driver [C:\Users\jabuk\.wdm\drivers\chromedriver\win32\89.0.4389.23\chromedriver.exe] found in cache<jupyter_text>Visit the NASA mars news site<jupyter_code># Visit the mars nasa news site
url_nasa = 'https://mars.nasa.gov/news/'
browser.visit(url_nasa)
# Optional delay for loading the page
time.sleep(5)
# Convert the browser html to a soup object and then quit the browser
html = browser.html
soup = BeautifulSoup(html, 'html.parser')
# print(soup.prettify())
# Scrape latest news title
# Latest news title as of today, 4/9/2021: "NASA's Mars helicopter to make first flight attempt Sunday"
latest_news = soup.findAll('div', class_="content_title")
news_title= latest_news[1].text
news_title
# Scrape Latest news description text
descriptions = soup.findAll('div', class_="article_teaser_body")
news_desc = descriptions[0].text
news_desc<jupyter_output><empty_output><jupyter_text>JPL Space Images Featured Image<jupyter_code># Visit JPL space images Mars URL to find the featured 
url_jpl = 'https://data-class-jpl-space.s3.amazonaws.com/JPL_Space/index.html'
browser.visit(url_jpl)
# Convert the browser html to a soup object and then quit the browser
html = browser.html
soup = BeautifulSoup(html, 'html.parser')
#Scrape relative image URL of the featured image from the jpl site using soup
images = soup.findAll('img', class_="headerimage fade-in")
featured_img= images[0].attrs['src']
featured_img
# Featured image URL
featured_img_url = 'https://data-class-jpl-space.s3.amazonaws.com/JPL_Space/'+featured_img
featured_img_url<jupyter_output><empty_output><jupyter_text>Mars Facts<jupyter_code># Visit space-facts.com 
url_mars = 'https://space-facts.com/mars/'
browser.visit(url_mars)
# Reading html
rawdata_mars = pd.read_html(url_mars)[1]
rawdata_mars<jupyter_output><empty_output><jupyter_text>Data Cleaning Stage<jupyter_code># Rename column Mars - Earth Comparison to Attributes
mars_df = rawdata_mars.rename(columns= {'Mars - Earth Comparison': 'Attributes'}).drop(columns = ["Earth"])
mars_df
# Export data to HTML
mars_html_table = mars_df.to_html("mars_facts.html", header=False, index=False)<jupyter_output><empty_output><jupyter_text>Mars Hemispheres<jupyter_code># Visit the USGS astrogeology page for hemisphere data from Mars
base_url ="https://astrogeology.usgs.gov"
url_hemisphere = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
browser.visit(url_hemisphere)
# Extract the .html out of the browser & create beautifulsoup object
hem_html = browser.html
hem_soup = BeautifulSoup(hem_html,'html.parser')
# print(hem_soup.prettify())
# Finds images in divs with class=item
results = hem_soup.findAll('div', class_='item')
results
hem_img_urls = []
image_titles = hem_soup.findAll('h3')

for i in range(len(image_titles)):
    hemisphere = {}
    
    img_title = hem_soup.findAll('h3')[i].text
    
    img_url = base_url + hem_soup.findAll('a', class_='itemLink product-item')[i]['href']
    browser.visit(img_url)
    
    time.sleep(1)
    
    html = browser.html
    soup = BeautifulSoup(html, 'html.parser')
    
    final_img_url = soup.find('div', class_='downloads').find('a')['href']
    
    hemisphere["title"] = img_title
    hemisphere["img_url"] = final_img_url
    
    hem_img_urls.append(hemisphere)
    
hem_img_urls
# Quit browser when finished
browser.quit()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12229836584080127, 'fraction_numerical': 0.011333684765419082, 'mean_word_length': 5.9, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '279976', 'license': 'Apache-2.0', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1328, 'n_words': 293, 'path': 'mission_to_mars.ipynb', 'repo_name': 'malnaimi1/web-scraping-challenge', 'size': '3794'}