<jupyter_start><jupyter_text>Lambda School Data Science, Unit 2: Predictive Modeling Applied Modeling, Module 3You will use your portfolio project dataset for all assignments this sprint. AssignmentComplete these tasks for your project, and document your work.- [ ] Continue to iterate on your project: data cleaning, exploration, feature engineering, modeling.- [ ] Make at least 1 partial dependence plot to explain your model.- [ ] Share at least 1 visualization on Slack.(If you have not yet completed an initial model yet for your portfolio project, then do today's assignment using your Tanzania Waterpumps model.) Stretch Goals- [ ] Make multiple PDPs with 1 feature in isolation.- [ ] Make multiple PDPs with 2 features in interaction. - [ ] Use Plotly to make a 3D PDP.- [ ] Make PDPs with categorical feature(s). Use Ordinal Encoder, outside of a pipeline, to encode your data first. If there is a natural ordering, then take the time to encode it that way, instead of random integers. Then use the encoded data with pdpbox.I Get readable category names on your plot, instead of integer category codes. Links- [<NAME>: Interpretable Machine Learning — Partial Dependence Plots](https://christophm.github.io/interpretable-ml-book/pdp.html) + [animated explanation](https://twitter.com/ChristophMolnar/status/1066398522608635904)- [Kaggle / <NAME>: Machine Learning Explainability — Partial Dependence Plots](https://www.kaggle.com/dansbecker/partial-plots)- [Plotly: 3D PDP example](https://plot.ly/scikit-learn/plot-partial-dependence/partial-dependence-of-house-value-on-median-age-and-average-occupancy)<jupyter_code>lat, lon = 26.4332,-81.401
year = 2010
api_key = '<KEY>'
attributes = 'ghi,dhi,dni,wind_speed,air_temperature,solar_zenith_angle,wind_speed,cloud_type,fill_flag,relative_humidity,surface_albedo,surface_pressure,wind_direction,dew_point'

leap_year = 'false'
interval = '30'
utc = 'true'
your_name = 'John+Smith'
reason_for_use = 'beta+testing'
your_affiliation = 'my+institution'

your_email = '<EMAIL>'
mailing_list = 'true'

import pandas as pd

year = '2010'

df1 = pd.read_csv('http://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv?wkt=POINT({lon}%20{lat})&names={year}&leap_day={leap}&interval={interval}&utc={utc}&full_name={name}&email={email}&affiliation={affiliation}&mailing_list={mailing_list}&reason={reason}&api_key={api}&attributes={attr}'.format(year=year, lat=lat, lon=lon, leap=leap_year, interval=interval, utc=utc, name=your_name, email=your_email, mailing_list=mailing_list, affiliation=your_affiliation, reason=reason_for_use, api=api_key, attr=attributes), skiprows=2)


df1 = df1.set_index(pd.date_range('1/1/{yr}'.format(yr=year), freq=interval+'Min', periods=525600/int(interval)))



year = '2011'

df2 = pd.read_csv('http://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv?wkt=POINT({lon}%20{lat})&names={year}&leap_day={leap}&interval={interval}&utc={utc}&full_name={name}&email={email}&affiliation={affiliation}&mailing_list={mailing_list}&reason={reason}&api_key={api}&attributes={attr}'.format(year=year, lat=lat, lon=lon, leap=leap_year, interval=interval, utc=utc, name=your_name, email=your_email, mailing_list=mailing_list, affiliation=your_affiliation, reason=reason_for_use, api=api_key, attr=attributes), skiprows=2)


df2 = df2.set_index(pd.date_range('1/1/{yr}'.format(yr=year), freq=interval+'Min', periods=525600/int(interval)))

df1 = pd.concat([df1,df2])
year = '2012'
leap_year = 'true'
dfl = pd.read_csv('http://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv?wkt=POINT({lon}%20{lat})&names={year}&leap_day={leap}&interval={interval}&utc={utc}&full_name={name}&email={email}&affiliation={affiliation}&mailing_list={mailing_list}&reason={reason}&api_key={api}&attributes={attr}'.format(year=year, lat=lat, lon=lon, leap=leap_year, interval=interval, utc=utc, name=your_name, email=your_email, mailing_list=mailing_list, affiliation=your_affiliation, reason=reason_for_use, api=api_key, attr=attributes), skiprows=2)


dfl = dfl.set_index(pd.date_range('1/1/{yr}'.format(yr=year), freq=interval+'Min', periods=527040/int(interval)))

df1 = pd.concat([df1,dfl])
year = '2013'

leap_year = 'false'
df3 = pd.read_csv('http://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv?wkt=POINT({lon}%20{lat})&names={year}&leap_day={leap}&interval={interval}&utc={utc}&full_name={name}&email={email}&affiliation={affiliation}&mailing_list={mailing_list}&reason={reason}&api_key={api}&attributes={attr}'.format(year=year, lat=lat, lon=lon, leap=leap_year, interval=interval, utc=utc, name=your_name, email=your_email, mailing_list=mailing_list, affiliation=your_affiliation, reason=reason_for_use, api=api_key, attr=attributes), skiprows=2)


df3 = df3.set_index(pd.date_range('1/1/{yr}'.format(yr=year), freq=interval+'Min', periods=525600/int(interval)))

df1 = pd.concat([df1,df3])
year = '2014'

df4 = pd.read_csv('http://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv?wkt=POINT({lon}%20{lat})&names={year}&leap_day={leap}&interval={interval}&utc={utc}&full_name={name}&email={email}&affiliation={affiliation}&mailing_list={mailing_list}&reason={reason}&api_key={api}&attributes={attr}'.format(year=year, lat=lat, lon=lon, leap=leap_year, interval=interval, utc=utc, name=your_name, email=your_email, mailing_list=mailing_list, affiliation=your_affiliation, reason=reason_for_use, api=api_key, attr=attributes), skiprows=2)


df4 = df4.set_index(pd.date_range('1/1/{yr}'.format(yr=year), freq=interval+'Min', periods=525600/int(interval)))

df1 = pd.concat([df1,df4])
year = 2015

df5 = pd.read_csv('http://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv?wkt=POINT({lon}%20{lat})&names={year}&leap_day={leap}&interval={interval}&utc={utc}&full_name={name}&email={email}&affiliation={affiliation}&mailing_list={mailing_list}&reason={reason}&api_key={api}&attributes={attr}'.format(year=year, lat=lat, lon=lon, leap=leap_year, interval=interval, utc=utc, name=your_name, email=your_email, mailing_list=mailing_list, affiliation=your_affiliation, reason=reason_for_use, api=api_key, attr=attributes), skiprows=2)


df5 = df5.set_index(pd.date_range('1/1/{yr}'.format(yr=year), freq=interval+'Min', periods=525600/int(interval)))

df1 = pd.concat([df1,df5])
df1.tail()
year = '2016'
leap_year = 'true'
test = pd.read_csv('http://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv?wkt=POINT({lon}%20{lat})&names={year}&leap_day={leap}&interval={interval}&utc={utc}&full_name={name}&email={email}&affiliation={affiliation}&mailing_list={mailing_list}&reason={reason}&api_key={api}&attributes={attr}'.format(year=year, lat=lat, lon=lon, leap=leap_year, interval=interval, utc=utc, name=your_name, email=your_email, mailing_list=mailing_list, affiliation=your_affiliation, reason=reason_for_use, api=api_key, attr=attributes), skiprows=2)


test = test.set_index(pd.date_range('1/1/{yr}'.format(yr=year), freq=interval+'Min', periods=527040/int(interval)))
features = ['Wind Speed','Temperature','Solar Zenith Angle','Pressure','Surface Albedo']
target = 'GHI'

from xgboost import XGBRegressor

boost = XGBRegressor(n_estimators=100, random_state=42, n_jobs=-1)

boost.fit(df1[features],df1[target])

from sklearn.metrics import r2_score
from sklearn.metrics import mean_absolute_error

y_pred_train = boost.predict(df1[features])

print(r2_score(df1[target],y_pred_train))
print(mean_absolute_error(df1[target],y_pred_train))

y_pred_test = boost.predict(test[features])

print(r2_score(test[target],y_pred_test))
print(mean_absolute_error(test[target],y_pred_test))
!pip install pdpbox

from pdpbox.pdp import pdp_isolate, pdp_plot
feature = 'Solar Zenith Angle'

isolated = pdp_isolate(
    model=boost,
    dataset=test,
    model_features=features,
    feature=feature,
    num_grid_points=60
)
import matplotlib.pyplot as plt


pdp_plot(isolated, feature_name=feature, plot_lines=True, frac_to_plot=200) 
plt.xlim(0,120);
import requests

a = requests.get('https://api.darksky.net/forecast/dc757f87dbdcb50907cdcecf02328582/42.3601,-71.0589?extend=hourly')
a= a.json()
a
from datetime import datetime
ts = 1262318400
print(datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S'))
lat, lon = 26.4332,-81.401
a = requests.get('https://api.darksky.net/forecast/dc757f87dbdcb50907cdcecf02328582/42.3601,-71.0589,2010-05-02T00:00:00Z?extend=hourly')
a = a.json()
a['hourly']['data']
ddark = pd.DataFrame(a['hourly']['data'])
ddark.head()
!pip install pysolar
from pysolar.solar import *

date = datetime.datetime(2015, 2, 1, 2, 30, tzinfo=datetime.timezone.utc)
float(90) - get_altitude(6.75189646, -75.19042969, date)
print(date)
def tc(li):
  return(datetime.utcfromtimestamp(li).strftime('%Y-%m-%d %H:%M:%S'))

ddark['date'] = ddark.time.apply(tc)
ddark.head()
'2010-05-01 08:00:00'.split('-')
def sol(li):
  li = str(li)
  date = li.split('-')
  year = int(date[0])
  month = int(date[1])
  day = int(date[2].split(' ')[0])
  hour = int(date[2].split(' ')[1].split(':')[0])
  date2 = datetime.datetime(year, month, day, hour, 0, tzinfo=datetime.timezone.utc)
  zen = float(90) - get_altitude(lat, lon, date2)  
  return(zen)
ddark['zen'] = ddark.date.apply(sol)
ddark['cel'] = (ddark.temperature	-32) / 1.8
lat, lon = 42.3601,-71.0589

year = '2010'
leap_year = 'false'
test = pd.read_csv('http://developer.nrel.gov/api/solar/nsrdb_psm3_download.csv?wkt=POINT({lon}%20{lat})&names={year}&leap_day={leap}&interval={interval}&utc={utc}&full_name={name}&email={email}&affiliation={affiliation}&mailing_list={mailing_list}&reason={reason}&api_key={api}&attributes={attr}'.format(year=year, lat=lat, lon=lon, leap=leap_year, interval=interval, utc=utc, name=your_name, email=your_email, mailing_list=mailing_list, affiliation=your_affiliation, reason=reason_for_use, api=api_key, attr=attributes), skiprows=2)


test = test.set_index(pd.date_range('1/1/{yr}'.format(yr=year), freq=interval+'Min', periods=525600/int(interval)))
test = test.reset_index()
test.head()
test.dtypes
ddark['date'] = pd.to_datetime(ddark.date)
ddark.dtypes
date = pd.merge(test,ddark,left_on='index',right_on='date')
date.head()
date.shape
date[['zen','Solar Zenith Angle']]
lat = 42.3601

lon = -71.0589

years = [2010,2011]
months = ['01','02','03','04','05','06','07','08','09','10','11','12']

dark = pd.DataFrame([])


for year in years:
	for month in months:
		a = requests.get('https://api.darksky.net/forecast/dc757f87dbdcb50907cdcecf02328582/' + str(lat) +','+ str(lon) +','+ str(year)+'-'+ month + '-05T00:00:00Z?extend=hourly')
		print(a)
		a = a.json()
		dic = a['hourly']['data']
		dark = dark.append(pd.DataFrame(dic),ignore_index=True)

dark.to_csv('first.csv')
from google.colab import files
uploaded = files.upload()
first = pd.read_csv('first.csv')
first.head()
first.shape
def tc(li):
  ts = li
  return(datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S'))
first['date'] = first.time.apply(tc)
first.tail(50)
from datetime import datetime
ts = 	1323057600
print(datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S'))

date.head()
datetime.fromtimestamp(1272686400)
print(datetime.fromtimestamp(1272686400).strftime('%Y-%m-%d %H:%M:%S'))
from google.colab import files
uploaded = files.upload()

fir = pd.read_csv('first (1).csv')
fir.head()
from google.colab import files
uploaded = files.upload()

testdata = pd.read_csv('data (4).csv')
testdata.shape
testdata.head(
)
testdata.tail()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17249233464739377, 'fraction_numerical': 0.043451598773543584, 'mean_word_length': 8.213882163034706, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 7, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '109244', 'license': 'MIT', 'n_tokens_mistral': 4544, 'n_tokens_neox': 4207, 'n_words': 656, 'path': 'module3/assignment_applied_modeling_3.ipynb', 'repo_name': 'matthew-sessions/DS-Unit-2-Applied-Modeling', 'size': '11483'}