<jupyter_start><jupyter_text>The Rain in Spain - the last 100 yearsData source: The data comes from a subset of The National Centers for Environmental Information (NCEI) [Daily Global Historical Climatology Network](https://www1.ncdc.noaa.gov/pub/data/ghcn/daily/readme.txt) (GHCN-Daily). The GHCN-Daily is comprised of daily climate records from thousands of land surface stations across the globe.<jupyter_code>import pandas as pd
import ipywidgets as widgets
import matplotlib.pyplot as plt
import mplleaflet
# countries and their codes
path='https://rain-in-spain-data.s3.us-east-2.amazonaws.com/'
ctry_df=pd.read_fwf(path+'ghcnd-countries.txt', widths=[2,1,46], header=None, encoding='utf8')
ctry_df.columns=['CODE','NO1','NAME']
ctry_df=ctry_df.drop(columns='NO1')
# metadata for all stations
meta_df = pd.read_fwf(path+'ghcnd-stations.txt', widths=[11,1,8,1,9,1,6,1,2,1,30,1,3,1,3,1,5], header=None, encoding='utf8')
meta_df.columns = ['ID','NO1','LATITUDE','NO2','LONGITUDE','NO3','ELEVATION','NO4','STATE','NO5','NAME',
            'NO6','GSN FLAG','NO7','HCN/CRN FLAG','NO8','WMO ID']
meta_df = meta_df.drop(columns=['NO1','NO2','NO3','NO4','NO5','NO6','NO7','NO8'])
meta_df['COUNTRY']=[row[:2] for row in meta_df.ID]
# only stations for Spain
ctry_code='SP'
meta_df = meta_df[meta_df.COUNTRY==ctry_code]
print(f'Number of stations in {ctry_df[ctry_df.CODE==ctry_code].NAME.item()}: {len(meta_df)}')
meta_df.head()
def leaflet_plot_stations(df):
    "Map of stations in Spain"
    lats, lons = df.LATITUDE.tolist(), df.LONGITUDE.tolist()
    plt.figure(figsize=(8,8))
    plt.scatter(lons, lats, c='r', alpha=0.7, s=20)
    return mplleaflet.display()
leaflet_plot_stations(meta_df)
# select weather station
stations=[*zip(meta_df.NAME,meta_df.ID)]
w=widgets.Dropdown(
    options=stations[:500],
    value=stations[0][1],
    description='Station:',
    disabled=False,
)
def on_change_stn(change):
    "Code changed from default"
    if change['type'] == 'change' and change['name'] == 'value':
        print (f"station code {change['new']}")
w.observe(on_change_stn)
display(w)
def gen_col_names(lst):
    "select columns of interest"
    for i in range(31):
        lst=lst+['VALUE'+str(i+1),'MFLAG'+str(i+1),'QFLAG'+str(i+1),'SFLAG'+str(i+1)]
    return lst
def gen_drop_col_names(lst):
    "drop columns of no interest"
    for i in range(31):
        lst=lst+['MFLAG'+str(i+1),'QFLAG'+str(i+1),'SFLAG'+str(i+1)]
    return lst
# rainfall data for a single station
# PRCP = Precipitation (tenths of mm)
station=w.value
print('Daily',meta_df[meta_df.ID==station].NAME.item())
df=pd.read_fwf(path+'SP_dly/'+station+'.dly', widths=[11,4,2,4,]+[5,1,1,1]*31, header=None)
df.columns=gen_col_names(['ID','YEAR','MONTH','ELEMENT'])
df=df[df.ELEMENT=='PRCP'].drop(columns=gen_drop_col_names(['ID','ELEMENT']))
df=df.melt(['YEAR','MONTH']).sort_values(['YEAR','MONTH']).reset_index(drop=True)
df=df[df.value>-9999]
df['variable'] = [row[5:] for row in df.variable]
df['date']=pd.to_datetime(df[['YEAR', 'MONTH', 'variable']].rename(columns={'YEAR': 'year', 'MONTH': 'month', 'variable': 'day'}))
df=df.drop(columns=['YEAR','MONTH','variable']).rename(columns={'value':'PRECP'})
df.set_index('date', inplace=True)
df.tail()
mthly_df=df.resample('MS').sum()
print ('Monthly')
mthly_df.head()
plt.scatter(mthly_df.index[-60:],mthly_df.PRECP[-60:])
plt.title('monthly rainfall last 5 years');
plt.scatter(mthly_df.index[-120:-60],mthly_df.PRECP[-120:-60])
plt.title('monthly rainfall previous 5 years');
yrly_df=df.resample('YS').sum()
print ('Yearly')
yrly_df.head()
plt.scatter(yrly_df.index,yrly_df.PRECP);
plt.scatter(yrly_df.index[-60:],yrly_df.PRECP[-60:]);
plt.scatter(mthly_df.index[-60*12:],mthly_df.PRECP[-60*12:]);
qtly_df=df.resample('QS').sum()
qtly_df.head()
plt.scatter(qtly_df.index,qtly_df.PRECP);
%load_ext watermark
%watermark --iversions -p matplotlib,mplleaflet,watermark,pylint
#!git add .
#!git commit -m 'updated notebook'
!jupyter nbconvert --to=script --output-dir=/tmp/converted-notebooks/ ./rain_in_spain.ipynb
!pylint ./tmp/converted-notebooks/rain_in_spain.py --disable=C,E0602,W0301,W0621<jupyter_output>[NbConvertApp] Converting notebook ./rain_in_spain.ipynb to script
[NbConvertApp] Writing 4573 bytes to /tmp/converted-notebooks/rain_in_spain.py

--------------------------------------------------------------------
Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00)
{'fraction_non_alphanumeric': 0.20669380370872908, 'fraction_numerical': 0.03324287652645862, 'mean_word_length': 5.95440251572327, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '97990', 'license': 'CC0-1.0', 'n_tokens_mistral': 1830, 'n_tokens_neox': 1750, 'n_words': 287, 'path': 'notebooks/rain_in_spain.ipynb', 'repo_name': 'AlisonDavey/Capstone_Udacity_DevOps_Engineer', 'size': '4422'}