<jupyter_start><jupyter_text>This notebook summarize the prediction results<jupyter_code>import numpy as np
import pandas as pd
from pandas.tseries.offsets import MonthEnd
from scipy.stats.stats import pearsonr
from tools import MBOX, OUTBOX, save_hdf
# Load prediction output files
mp1=[]; mp2=[]; mp3=[]; mp4=[]; mp5=[]; mp6=[]; mp7=[];
for i in range(1593):
    # Multiple file results
    filn = './data/chtc_out/dfFlowDams{}_manu.npz'.format(i)
    mp1.extend(np.load(filn,allow_pickle=True)['mp1'].tolist())
    mp2.extend(np.load(filn,allow_pickle=True)['mp2'].tolist())
    mp3.extend(np.load(filn,allow_pickle=True)['mp3'].tolist())
    mp4.extend(np.load(filn,allow_pickle=True)['mp4'].tolist())
    mp5.extend(np.load(filn,allow_pickle=True)['mp5'].tolist())
    mp6.extend(np.load(filn,allow_pickle=True)['mp6'].tolist())
    mp7.extend(np.load(filn,allow_pickle=True)['mp7'].tolist())
print('All results are loaded.')

# Load Dam Inflow data from SUTD
dfFlowDams = pd.read_hdf('./data/dfFlowDams.hdf')
ind_dams = np.load('/Users/dlee/gdrive/gpower/data/ind_dams.npz')['ind_dams']
damList = ind_dams[0,:]
ndam = len(damList)
assert np.all([ybox['m01']['point_no'] for ybox in mp1] == damList)<jupyter_output>/Users/dlee/opt/anaconda3/envs/ghp/lib/python3.8/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.linear_model.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.linear_model. Anything that cannot be imported from sklearn.linear_model is now part of the private API.
  warnings.warn(message, FutureWarning)
/Users/dlee/opt/anaconda3/envs/ghp/lib/python3.8/site-packages/sklearn/base.py:313: UserWarning: Trying to unpickle estimator LinearRegression from version 0.19.1 when using version 0.22.2. This might lead to breaking code or invalid results. Use at your own risk.
  warnings.warn(<jupyter_text>Align prediction results<jupyter_code># Summary table
arrMP1 = np.zeros(dfFlowDams.shape)     # MP1
arrMP2 = arrMP1.copy()                  # MP2
arrMP3 = arrMP1.copy()                  # MP3
arrMP4 = arrMP1.copy()                  # MP4
arrMP5 = arrMP1.copy()                  # MP5
arrMP6 = arrMP1.copy()                  # MP6
arrMP7 = arrMP1.copy()                  # MP7
signMP1 = np.zeros([12, ndam])
signMP2 = signMP1.copy()
signMP3 = signMP1.copy()
signMP4 = signMP1.copy()
signMP5 = signMP1.copy()
signMP6 = signMP1.copy()
signMP7 = signMP1.copy()

for i in range(ndam):
    # Data preparation
    point_no = mp1[i]['m01']['point_no']
    y = dfFlowDams[point_no]
    # Results tables
    data = np.zeros([516, 8])       # Flow [obs, mp1, mp2, mp3, mp4, mp5, mp6, mp7]
    sign = np.zeros([7, 13])        # isFcst [7mp, 12 months + sum] ("not forecasted" as default)
    for j in range(1, 13):
        # Monthly Prediction
        # - Observed Inflow
        yObs = y[y.index.month == j]
        yClm = yObs.mean()                                      # All-year mean
        data[range(j-1,516,12), 0] = yObs
        data[np.ix_(range(j-1,516,12), [1,2,3,4,5,6,7])] = yClm # Assign climatology first
        # - M1 prediction
        mbox = mp1[i]['m%02d' % j]
        if 'gss' in mbox:
            gss, msess = mbox['gss'], mbox['msess']
            if (gss >= 0) & (msess >= 0):
                # If prediction is better than climatology
                yHat = np.exp(np.hstack((mbox['yTranHat'], mbox['yTestHat'])))
                data[range(j-1,516,12), 1] = yHat       # prediction
                sign[0,j-1] = 1                         # isFcst
        # - M2 prediction
        mbox = mp2[i]['m%02d' % j]
        if 'gss' in mbox:
            gss, msess = mbox['gss'], mbox['msess']
            if (gss >= 0) & (msess >= 0):
                # If prediction is better than climatology
                yHat = np.exp(np.hstack((mbox['yTranHat'], mbox['yTestHat'])))
                data[range(j-1,516,12), 2] = yHat       # prediction
                sign[1,j-1] = 1                         # isFcst
        # - M3 prediction
        mbox = mp3[i]['m%02d' % j]
        if 'gss' in mbox:
            gss, msess = mbox['gss'], mbox['msess']
            if (gss >= 0) & (msess >= 0):
                # If prediction is better than climatology
                yHat = np.exp(np.hstack((mbox['yTranHat'], mbox['yTestHat'])))
                data[range(j-1,516,12), 3] = yHat       # prediction
                sign[2,j-1] = 1                         # isFcst        
        # - M4 prediction
        mbox = mp4[i]['m%02d' % j]
        if 'gss' in mbox:
            gss, msess = mbox['gss'], mbox['msess']
            if (gss >= 0) & (msess >= 0):
                # If prediction is better than climatology
                yHat = np.exp(np.hstack((mbox['yTranHat'], mbox['yTestHat'])))
                data[range(j-1,516,12), 4] = yHat       # prediction
                sign[3,j-1] = 1                         # isFcst
        # - M5 prediction
        mbox = mp5[i]['m%02d' % j]
        if 'gss' in mbox:
            gss, msess = mbox['gss'], mbox['msess']
            if (gss >= 0) & (msess >= 0):
                # If prediction is better than climatology
                yHat = np.exp(np.hstack((mbox['yTranHat'], mbox['yTestHat'])))
                data[range(j-1,516,12), 5] = yHat       # prediction
                sign[4,j-1] = 1                         # isFcst
        # - M6 prediction
        mbox = mp6[i]['m%02d' % j]
        if 'gss' in mbox:
            gss, msess = mbox['gss'], mbox['msess']
            if (gss >= 0) & (msess >= 0):
                # If prediction is better than climatology
                yHat = np.exp(np.hstack((mbox['yTranHat'], mbox['yTestHat'])))
                data[range(j-1,516,12), 6] = yHat       # prediction
                sign[5,j-1] = 1                         # isFcst
        # - M7 prediction
        mbox = mp7[i]['m%02d' % j]
        if 'gss' in mbox:
            gss, msess = mbox['gss'], mbox['msess']
            if (gss >= 0) & (msess >= 0):
                # If prediction is better than climatology
                yHat = np.exp(np.hstack((mbox['yTranHat'], mbox['yTestHat'])))
                data[range(j-1,516,12), 7] = yHat       # prediction
                sign[6,j-1] = 1                         # isFcst
        
    
    # Finish result tables
    sign[:,-1] = np.sum(sign, axis=1)

    # Copy time-series and signs(isFcst) of monthly prediction to Numpy array
    arrMP1[:,i] = data[:,1]
    arrMP2[:,i] = data[:,2]
    arrMP3[:,i] = data[:,3]
    arrMP4[:,i] = data[:,4]
    arrMP5[:,i] = data[:,5]
    arrMP6[:,i] = data[:,6]
    arrMP7[:,i] = data[:,7]
    signMP1[:,i] = sign[0,:-1]
    signMP2[:,i] = sign[1,:-1]
    signMP3[:,i] = sign[2,:-1]
    signMP4[:,i] = sign[3,:-1]
    signMP5[:,i] = sign[4,:-1]
    signMP6[:,i] = sign[5,:-1]
    signMP7[:,i] = sign[6,:-1]
    
    # Export prediction series as csv format
    if False:
        filn = './data/prediction/%s_series.xlsx' % point_no
        index = dfFlowDams.index
        dfFcst = pd.concat([pd.Series(data[:,0],index), 
                            pd.Series(data[:,1],index - MonthEnd(1)),
                            pd.Series(data[:,2],index - MonthEnd(2)), 
                            pd.Series(data[:,3],index - MonthEnd(3)),
                            pd.Series(data[:,4],index - MonthEnd(4)),
                            pd.Series(data[:,5],index - MonthEnd(5)),
                            pd.Series(data[:,6],index - MonthEnd(6)),
                            pd.Series(data[:,7],index - MonthEnd(7)),
                           ],
                           axis=1)
        dfFcst.columns = ['OBS','MP1','MP2','MP3','MP4','MP5','MP6','MP7']
        dfFcst.to_excel(filn)
        
# Export time-series and signs of monthly forecasts
dfMP1 = pd.DataFrame(arrMP1, dfFlowDams.index, dfFlowDams.columns)
dfMP2 = pd.DataFrame(arrMP2, dfFlowDams.index, dfFlowDams.columns)
dfMP3 = pd.DataFrame(arrMP3, dfFlowDams.index, dfFlowDams.columns)
dfMP4 = pd.DataFrame(arrMP4, dfFlowDams.index, dfFlowDams.columns)
dfMP5 = pd.DataFrame(arrMP5, dfFlowDams.index, dfFlowDams.columns)
dfMP6 = pd.DataFrame(arrMP6, dfFlowDams.index, dfFlowDams.columns)
dfMP7 = pd.DataFrame(arrMP7, dfFlowDams.index, dfFlowDams.columns)
dfMP1_sign = pd.DataFrame(signMP1, range(1,13), dfFlowDams.columns)
dfMP2_sign = pd.DataFrame(signMP2, range(1,13), dfFlowDams.columns)
dfMP3_sign = pd.DataFrame(signMP3, range(1,13), dfFlowDams.columns)
dfMP4_sign = pd.DataFrame(signMP4, range(1,13), dfFlowDams.columns)
dfMP5_sign = pd.DataFrame(signMP5, range(1,13), dfFlowDams.columns)
dfMP6_sign = pd.DataFrame(signMP6, range(1,13), dfFlowDams.columns)
dfMP7_sign = pd.DataFrame(signMP7, range(1,13), dfFlowDams.columns)
if False:
    # Save time-series and plot
    save_hdf('./data/prediction/dfMP1.hdf', dfMP1)
    save_hdf('./data/prediction/dfMP2.hdf', dfMP2)
    save_hdf('./data/prediction/dfMP3.hdf', dfMP3)
    save_hdf('./data/prediction/dfMP4.hdf', dfMP4)
    save_hdf('./data/prediction/dfMP5.hdf', dfMP5)
    save_hdf('./data/prediction/dfMP6.hdf', dfMP6)
    save_hdf('./data/prediction/dfMP7.hdf', dfMP7)
    save_hdf('./data/prediction/dfMP1_sign.hdf', dfMP1_sign)
    save_hdf('./data/prediction/dfMP2_sign.hdf', dfMP2_sign)
    save_hdf('./data/prediction/dfMP3_sign.hdf', dfMP3_sign)
    save_hdf('./data/prediction/dfMP4_sign.hdf', dfMP4_sign)
    save_hdf('./data/prediction/dfMP5_sign.hdf', dfMP5_sign)
    save_hdf('./data/prediction/dfMP6_sign.hdf', dfMP6_sign)
    save_hdf('./data/prediction/dfMP7_sign.hdf', dfMP7_sign)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15665147119767925, 'fraction_numerical': 0.039577289680895154, 'mean_word_length': 2.343609282992726, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '255557', 'license': 'MIT', 'n_tokens_mistral': 3712, 'n_tokens_neox': 3563, 'n_words': 664, 'path': '(04) Summarize_Results.ipynb', 'repo_name': 'gnodnooh/SeasonalFlowForecast', 'size': '9652'}