# -*- coding: utf-8 -*-
"""timeseries_m4_mlmodels.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1bSLgqJczgvw26QocAewH3hG8pPxww-nT
"""

#### Install MLMODELS in Colab
# %%capture
from time import time; t0 =time()
!  bash <(wget -qO- https://cutt.ly/mlmodels) &>  log.txt
!pip install pydantic==1.4 --force
print(time() - t0, flush=True) 
import os, time
os.kill(os.getpid(), 9)

from google.colab import drive
drive.mount('/content/drive')

from mlmodels import util
print(util)

# Commented out IPython magic to ensure Python compatibility.
# %matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# %matplotlib inline
import os
from tqdm.autonotebook import tqdm
from pathlib import Path





###############  Working directory  ######################################################
import os
dirdrive = "/content/drive/My Drive/Colab Notebooks/shared_session/timeseries_example/"

os.chdir( dirdrive )
print( os.getcwd())



""" naming

dir_raw :  all initial data from kaggle, ...
dir_input : all data for data_pars and model

dir_inter :  intermediate data


"""









!ls

from mlmodels.preprocess.timeseries import save_to_file
print(save_to_file)



"""
Manual Pre-processing

"""
import json
import pandas as pd, numpy as np


#load data  from SHARED FOLDER
data_folder = "kaggle_data/m4_dataset"


train_df   = pd.read_csv(data_folder + '/Daily-train.csv',index_col=0)
test_df    = pd.read_csv(data_folder + '/Daily-test.csv',index_col=0)
print(train_df.shape, train_df.head(5))
print(test_df.head(5))

print( train_df.columns )
print( train_df.index)



#### PARAMS ####################################################################
freq = "D"
prediction_length = 14
start_date = ""

######################## Parameters
single_pred_length = 14
submission_pred_length = single_pred_length * 2

submission=True

if submission:
    pred_length = submission_pred_length
else:
    pred_length = single_pred_length


#### Cardinality count #########################################################
cardinality_train = len(train_df) # 4227
cardinality_test  = len(test_df) # 4227
print(cardinality_train, cardinality_test)




#### output JSON Files  ########################################################
#data_folder = "kaggle_data/m4_daily_test"
gluonts_datafolder = "gluonts_data/m4_dataset"

train_file = gluonts_datafolder+"/m4_daily/train/data.json"
test_file  = gluonts_datafolder+"/m4_daily/test/data.json"
meta_file  = gluonts_datafolder+'/m4_daily/metadata.json'


#### For mlmodel data_pars
dataset_path  = Path( gluonts_datafolder+"/m4_daily" )



#### Output Metadata JSON
with open(meta_file, 'w') as f:
      f.write( json.dumps(
              {"freq": "D", 
               "prediction_length": 14, 
               "feat_static_cat": [{"name": "feat_static_cat", "cardinality":  str(cardinality_train)}]
               }
              )
          )


#### Output JSON  ##############################################################
train_target_values = [ts[~np.isnan(ts)] for ts in train_df.values]
print(train_target_values[:3])

save_to_file(train_file,
      [{ "start": "1750-01-01 00:00:00",
         "target": list(target),
         "feat_static_cat": [cat]
  }
          for cat, target in enumerate(train_target_values)
      ],
  )


#### Output JSON  ##############################################################
test_target_values = [
        np.hstack([train_ts, test_ts])
        for train_ts, test_ts in zip(train_target_values, test_df.values)
    ]
print(test_target_values[:3])

save_to_file(test_file,
    [
      {
    "start": "1750-01-01 00:00:00",
    "target": list(target),
    "feat_static_cat": [cat]
}      
        for cat, target in enumerate(test_target_values)
    ],
)





### Check the GluonTS Dataset
from gluonts.dataset.common import ListDataset, load_datasets

TD            = load_datasets(  metadata=dataset_path,
                train=dataset_path / "train",
                test=dataset_path / "test",)
print(TD)







# MLMODELS setup
import mlmodels
from mlmodels.models import module_load
from mlmodels.util import path_norm_dict, path_norm, params_json_load
import json

from pprint import pprint as print2




#### Model URI and Config JSON
#model_uri   = "model_keras.gluonts_model"
#config_path = path_norm( 'model_keras/ardmn.json'  )
#config_mode = "test"  ### test/prod



#module=None

#### Load JSON File (no python is included)
## DO NOT INCLUDE PYTHON variables, only string or json values
jpars = {
    "deepar": {
         "model_pars": {
             "model_uri"  : "model_gluon.gluonts_model",
             "model_name" : "deepar",
             "model_pars" : {
                 "prediction_length": 14, 
                 "freq": "D",
                 "distr_output" :  "gluonts.distribution.student_t.StudentTOutput", 
                

                
                 "use_feat_static_cat": true, 
                 "use_feat_static_real":false,
                 
                  "num_layers": 2, 
                  "num_cells": 40, 
                  "cell_type": "lstm", 
                  "dropout_rate": 0.1,

                 
                  "scaling": true, 
                  "num_parallel_samples": 100,
                  "cardinality":  [4227]
             }
             
             
             },
        "data_pars": {
            "train": true, 
            "dt_source": "",
            "data_type":"gluonts",

                
                 "use_feat_static_cat": true, 
                 "use_feat_static_real":false,

           
            "submission": false , 
            "dataset_name": "m4_daily" , 
            
            "data_path":  "gluonts_data/m4_dataset/m4_daily" , 
           
            "single_pred_length":28,
            "freq": "1D",
            "start" : "1750-01-01 00:00:00",        
            "startdate" : "1750-01-01",  
            "col_date"   : "timestamp",                
            "col_ytarget" : ["target"],
            "num_series" : 1,

            "cols_cat": [],   "cols_num" : []
                    
            },
            
            
        "compute_pars": {
            "num_samples": 100,
            
            "learning_rate"         : 1e-3,
            "epochs"                : 100,
            "num_batches_per_epoch" : 50,
            "batch_size"            : 32,   
            
            "compute_pars" : {
                
                "batch_size": 32, "clip_gradient": 100, "epochs": 100, "init": "xavier", "learning_rate": 1e-3, 
                "learning_rate_decay_factor": 0.5, 
                "hybridize": false,
                "num_batches_per_epoch": 50,
                
                "minimum_learning_rate": 5e-05, "patience": 10, "weight_decay": 1e-08
            }
        },
        
      "out_pars": {
         "path": "ztest/model_gluon/gluonts_deepar/",
         "plot_prob": true, "quantiles": [0.5]
      }
    }
}

#### Load JSON
json_path="m4_daily_gluonts.json"
jpars = json.load(open(json_path))
print2(jpars)

for x in [ 'model_pars', 'data_pars', 'compute_pars', 'out_pars' ] :
  globals()[x] = jpars['deepar'][x]
print( model_pars, data_pars, compute_pars, out_pars)





### https://github.com/arita37/mlmodels/blob/dev/mlmodels/model_gluon/gluonts_model.py

#### Setup Model from repo
from mlmodels.models import module_load
model_uri="model_gluon.gluonts_model"
#module         = module_load( model_uri)
#model          = module.Model(model_pars, data_pars, compute_pars) 


#### Local load model
import gluonts_model as module
model_uri="gluonts_model"
# import imp
# imp.reload(module)


model = module.Model(model_pars=model_pars, data_pars=data_pars, compute_pars=compute_pars)
print(model)

#### Fit          >>>>USE module.fit
model = module.fit(model=model,data_pars= data_pars,compute_pars= compute_pars, out_pars=out_pars)           #### fit model

y_pred = module.predict(model=model,data_pars= data_pars,compute_pars= compute_pars, out_pars=out_pars)

metrics_val = module.fit_metrics(y_pred, data_pars, compute_pars, out_pars)   #### Check fit metrics
print(metrics_val)









!ls

module.plot_prob_forecasts(y_pred)







#### Save/Load
module.save(model, save_pars = { 'path': out_pars['path'] +"/model/"})

model2 = module.load(load_pars = { 'path': out_pars['path'] +"/model/"})



!ls 'kaggle_data'

















