<jupyter_start><jupyter_text><NAME> - Climate Change Data Scraping and Cleaning<jupyter_code>from splinter import Browser
from bs4 import BeautifulSoup
import pandas as pd
import requests
import pymongo
from flask import Flask, render_template, redirect
from flask_pymongo import PyMongo
from webdriver_manager.chrome import ChromeDriverManager
import pathlib
!pip install pytrends
from pytrends.request import TrendReq
pytrend = TrendReq()<jupyter_output><empty_output><jupyter_text>Import and clean CO2 Emission & Population data from The World Bank. The result will be used for visualization.<jupyter_code># Name the CO2 Emission CSV file
CO2_emission_file = pathlib.Path('CO2_emissions_per_capita.csv')

# Read and display the CO2 Emission CSV with Pandas
CO2_emission_df = pd.read_csv(CO2_emission_file)
CO2_emission_df_2 = CO2_emission_df[['Country Name','Country Code', 'Indicator Name', '2006', '2007','2008','2009','2010','2011','2012','2013','2014','2015','2016']]
CO2_emission_df_2
# Name the population CSV file
population_file = pathlib.Path('population_total_by_country.csv')

# Read and display the population CSV with Pandas
population_df = pd.read_csv(population_file)
population_df_2 = population_df[['Country Name', 'Country Code', 'Indicator Name', '2006', '2007','2008','2009','2010','2011','2012','2013','2014','2015','2016']]
population_df_2
# Merge the above data frames to a single data frame
merged_df = pd.concat([CO2_emission_df_2, population_df_2], keys=['CO2_emission', 'population'])
#           names=['Country Name', 'Row ID'])
merged_df
merged_df.to_csv('output.csv')
# merged_table = merged_df.to_html().replace('\n','')
# merged_table<jupyter_output><empty_output><jupyter_text>Find Goolge Trends for Keywords searches for "Sustainability", "Environment", "Climate Change" and "Clean Energy". The result will be used for visualization.<jupyter_code># pytrend.build_payload(kw_list=['Environment'])
# kw_list=['Environment']
kw_list = ["Sustainability", "Environment", "Climate Change", "Clean Energy"] 
pytrend.build_payload(kw_list, timeframe='2000-01-01 2017-01-01')
# Interest by Region
interest_by_region = pytrend.interest_by_region()
interest_by_region
# Save the interest_by_region data frame to a CSV file
interest_by_region.to_csv('pytrend_keywords_by_country.csv')<jupyter_output><empty_output><jupyter_text>Find Google query trends for "Sustainability", "Environment", "Climate Change" and "Clean Energy". The result will be converted to HTML table strings for final web applications.<jupyter_code>pytrend.build_payload(kw_list = ["Sustainability", "Environment", "Climate Change", "Clean Energy"] )
# Related Queries, returns a dictionary of dataframes
related_queries = pytrend.related_queries()
# related_queries.values()
print(related_queries)
# # build list of keywords
# kw_list = ["Sustainability", "Environment", "Climate Change", "Clean Energy"] 
# # build the payload
# pytrend.build_payload(kw_list, timeframe='2000-01-01 2017-01-01')
# interest_over_time = pytrend.interest_over_time()
# interest_over_time_html = interest_over_time.to_html().replace('\n','')
# # print(interest_over_time_html)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11718504555450833, 'fraction_numerical': 0.043669494187873074, 'mean_word_length': 8.07122507122507, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '113873', 'license': 'Apache-2.0', 'n_tokens_mistral': 1120, 'n_tokens_neox': 996, 'n_words': 254, 'path': 'data/CO2_emission_vs_population_and_google_trend.ipynb', 'repo_name': 'Grace-Bijun-Li/climate-change-dashbaord', 'size': '3191'}