<jupyter_start><jupyter_text>Weather Derivatites  ENSO Probabilistic Forecast Downloader Developed by [<NAME>](mailto:<EMAIL>)   22 September 2018<jupyter_code># Import needed libraries. 
import numpy as np
import pandas as pd
import random as rand
import matplotlib.pyplot as plt
import time
import datetime<jupyter_output><empty_output><jupyter_text>One Data Downloader<jupyter_code>def oneYearMonthDonwloader(date):
    
    url = ''
    
    if date <= datetime.datetime(2014,1,1):
        month = date.strftime('%m')
        year = date.year
        # Build complete URL.
        url = 'https://iri.columbia.edu/our-expertise/climate/forecasts/enso/archive/'+ str(year)+str(month)+'/figure3.html'
        
        # Download Data
        yearMonthENSOForecast = pd.read_html(url, header= 0)
        return yearMonthENSOForecast[1]
    else: 
        month = date.strftime('%B')
        year = date.year
        
        # Build complete URL.
        url = 'https://iri.columbia.edu/our-expertise/climate/forecasts/enso/'+str(year)+'-' +str(month)+'-quick-look/?enso_tab=enso-iri_plume'
        # Download Data
        yearMonthENSOForecast = pd.read_html(url, header= 0)
        return yearMonthENSOForecast[0]

# Define Month and year of forecast.

date = datetime.datetime(2017,7,1)  # April is set as 04. 


# Extract information
yearMonthENSOForecast = oneYearMonthDonwloader(date)

yearMonthENSOForecast<jupyter_output><empty_output><jupyter_text>Historical Data Downloader<jupyter_code># Generate a timestamp with all months to download data.
startDate = '2004-04-01'
endDate = '2018-06-01'
dates = pd.date_range(startDate, end = endDate, freq='MS')
def totalENSOforecastDownloader(startDate,endDate):
    
    dates = pd.date_range(startDate, end = endDate, freq='MS')
    ensoForecast = {}
    
    print('Downloading data from: ...')
    
    for date in dates:
        month = date.strftime('%m')
        year = date.year
        
        storeStr = str(year)+'-'+str(month)
        print(storeStr)
        
        # Store in dictionary --> '2002-04'
        ensoForecast[storeStr] = oneYearMonthDonwloader(date)
    
    return ensoForecast
ensoForecast= totalENSOforecastDownloader(startDate,endDate)<jupyter_output>2004-04
2004-05
2004-06
2004-07
2004-08
2004-09
2004-10
2004-11
2004-12
2005-01
2005-02
2005-03
2005-04
2005-05
2005-06
2005-07
2005-08
2005-09
2005-10
2005-11
2005-12
2006-01
2006-02
2006-03
2006-04
2006-05
2006-06
2006-07
2006-08
2006-09
2006-10
2006-11
2006-12
2007-01
2007-02
2007-03
2007-04
2007-05
2007-06
2007-07
2007-08
2007-09
2007-10
2007-11
2007-12
2008-01
2008-02
2008-03
2008-04
2008-05
2008-06
2008-07
2008-08
2008-09
2008-10
2008-11
2008-12
2009-01
2009-02
2009-03
2009-04
2009-05
2009-06
2009-07
2009-08
2009-09
2009-10
2009-11
2009-12
2010-01
2010-02
2010-03
2010-04
2010-05
2010-06
2010-07
2010-08
2010-09
2010-10
2010-11
2010-12
2011-01
2011-02
2011-03
2011-04
2011-05
2011-06
2011-07
2011-08
2011-09
2011-10
2011-11
2011-12
2012-01
2012-02
2012-03
2012-04
2012-05
2012-06
2012-07
2012-08
2012-09
2012-10
2012-11
2012-12
2013-01
2013-02
2013-03
2013-04
2013-05
2013-06
2013-07
2013-08
2013-09
2013-10
2013-11
2013-12
2014-01
2014-02
2014-03
2014-04
2014-05
2014-06
2014-07
2014-08
[...]<jupyter_text>Data export<jupyter_code># Save data in pickle format. 

import pickle 

pickle.dump(ensoForecast, open("../datasets/ensoForecastProb/ensoForecastProbabilities.pickle", "wb"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11740890688259109, 'fraction_numerical': 0.23048004626951996, 'mean_word_length': 3.3895939086294415, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '348295', 'license': 'Apache-2.0', 'n_tokens_mistral': 1862, 'n_tokens_neox': 1307, 'n_words': 285, 'path': 'code/.ipynb_checkpoints/ensoForecastDownloader-checkpoint.ipynb', 'repo_name': 'jesugome/WeatherDerivates', 'size': '3485'}