<jupyter_start><jupyter_text>Load libraries and Data<jupyter_code># import libraries
import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
from datetime import datetime
import json # import json for formatting data
import os # and os for saving
import boto3
import sagemaker
from sagemaker import get_execution_role
import pickle
features_file = 'data/features.csv'
features_df = pd.read_csv(features_file)

train_file = 'data/train.csv'
train_df = pd.read_csv(train_file)

train_df = train_df.merge(features_df, how='left')
train_df.head()
# Save as Dataframe the train file and use Date as Index

train_df['Date'] = [datetime.strptime(date, '%Y-%m-%d').date() for date in train_df['Date'].astype(str).values.tolist()]
train_df.index = train_df['Date']
train_df = train_df.drop(['Date'], axis=1)
train_df.sort_values(["Store","Dept"], ascending = (False,False))

train_df.head()<jupyter_output><empty_output><jupyter_text>Convert features to use as dynamical features in DeepAR<jupyter_code>def convert_features(train_df):
    
    train_df['IsHoliday'] = train_df['IsHoliday'].astype(int)

    # MarkDown Nan = O, Others = 1
    train_df.fillna(0, inplace=True)
    list=['MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5']
    for i in list:
        train_df.loc[train_df[i] != 0, i] = 1
        
    train_df.replace('NaN', 0, inplace = True)
convert_features(train_df)
train_df.head()
# Look at the Store list to see if there are missing numbers
print(train_df.Store.unique())
print(train_df.Store.nunique())
# Look at the Dept list to see if there are missing numbers
print(train_df.Dept.unique())
print(train_df.Dept.nunique())<jupyter_output>[ 1  2  3  4  5  6  7  8  9 10 11 12 13 14 16 17 18 19 20 21 22 23 24 25
 26 27 28 29 30 31 32 33 34 35 36 37 38 40 41 42 44 45 46 47 48 49 51 52
 54 55 56 58 59 60 67 71 72 74 77 78 79 80 81 82 83 85 87 90 91 92 93 94
 95 96 97 98 99 39 50 43 65]
81<jupyter_text>In DeepAR, Categorical features must be encoded as a 0-based sequence of positive integers.  https://docs.aws.amazon.com/sagemaker/latest/dg/deepar.html  We want a list of 45 Stores and another of 81 Departments, each starting by 0<jupyter_code># Add Cat_Store columns to convert Store to a "Category" field appropriate for DeepAR
train_df['Cat_Store'] = train_df['Store'] - 1

# Add Cat_Dept columns to convert Dept to a "Category" field appropriate for DeepAR
liste = range(81)
for i in liste:
    department = train_df.Dept.unique()[i]
    train_df.loc[train_df['Dept'] == department, 'Cat_Dept'] = int(i)  

train_df.head()
# Verify Cat_Store
print(train_df.Cat_Store.unique())
print(train_df.Cat_Store.nunique())
# Verify Cat_Dept
print(train_df.Cat_Dept.unique())
print(train_df.Cat_Dept.nunique())
# take only 2 stores to stay within AWS Sagemaker size limit when using the deployed model to predict
train_df = train_df[train_df.Store < 3]
train_df.head()<jupyter_output><empty_output><jupyter_text>Plot Sales Evolution<jupyter_code># Define a function to plot sales evolution depending of the store and department number
def sales_evolution(store_number, dept_number):
    example_store_dept_sales = train_df[(train_df.Store == store_number) & (train_df.Dept == dept_number)]['Weekly_Sales']
    plt.figure(figsize=(8,3))
    example_store_dept_sales.plot(title='Store {} Dpt {} Sales evolution'.format(store_number, dept_number), color='blue') 
    return plt.show()
# Plot some examples
sales_evolution(2,3)
sales_evolution(1,10)
sales_evolution(1,4)<jupyter_output><empty_output><jupyter_text>Create and display complete time series and dynamic features<jupyter_code>def make_time_series(df, freq='W-FRI'):
    '''Creates as many time series as they are stores x departments
      :param df: dataframe with train data. One 3 columns : Store, Dept, Weekly_Sales.
          This dataframe should also be indexed by a datetime.
      :param freq: The frequency of data recording (W-FRI = Weekly, starting on Friday)
      :return: A list of pd.Series(), time series data.
      '''
    
    # store time series
    time_series = []
    
    # create time series for each store and each department
    for store in df.Cat_Store.unique().tolist():
        for dept in df.Cat_Dept.unique().tolist():
            
            # create DataFrame with weekly_sales for the store and department studied
            sample_df = df[(df.Cat_Store == store) & (df.Cat_Dept == dept)]['Weekly_Sales']   
            
            # initialise start_idx
            start_idx = 0
            end_idx = start_idx+143

            # create start and datetime
            t_start = '2010-02-05' # start with first date 

            # get wanted sample_df data
            data = sample_df[start_idx:end_idx]

            # create time series
            index = pd.date_range(start=t_start, periods=143, freq=freq)
            Serie_df = pd.Series(data=data, index=index)

            # include info of the store and dept as last values to use as category in DeepAR
            Info_df = pd.Series(data=[int(store), int(dept)], index=['store','dept'])
            time_series.append(pd.concat([Serie_df, Info_df]))               
    
    # return list of time series
    return time_series
# Create time series for all store and departments
freq='W-FRI' # weekly recordings

# make time series
time_series = make_time_series(train_df, freq=freq)
def make_dyn_feat(df, features, prediction_length, freq='W-FRI'):
    '''Create dynamic_features for each stores, each years and each selected features
      :param df: dataframe with train data. Contain at least the selected features in columns and Store number
          This dataframe should also be indexed by a datetime.
      :param features : list of choosen dynamical features, evolving over time
      :param prediction_length : length of dyn_features = 52 - prediction_length (usefull for training series)
      :param freq: The frequency of data recording (W-FRI = Weekly, starting on Friday)
      :return: A 3 dimensional list : Store / year / features
      '''

    # instantiate dynamic features list
    dynamic_feat = []
    
    # create dynamic feature for each store 
    for store in df.Cat_Store.unique().tolist():
        
        # instantite store list
        store_feat=[]
        
        # select features columns and filter on store number.
        # choose one Dept to have only one line for each date
        # Dynamic features are linked to the store, not the departments
        dyn_df = df.loc[(df.Cat_Store == store) & (df.Cat_Dept == 1), features]
        
        # initialise start_idx
        start_idx = 0
        # we get 144 weeks in our dataset, starting at index 0
        end_idx = start_idx + 143 - prediction_length

        # get wanted sample_df data
        dyn_temp_df = dyn_df[start_idx:end_idx]

        for j in features:
            store_feat.append(dyn_temp_df[j].tolist())
            #store_feat = dyn_temp_df[j].tolist()
            
        if df['Cat_Store'].nunique() > 1:
            dynamic_feat.append(store_feat)
        else:
            dynamic_feat = store_feat
    return dynamic_feat
# Create dynamic features for all data
freq='W-FRI' # weekly recordings
features = ['IsHoliday','MarkDown1']
# prediction length for test set
prediction_test = 0

# make dynamic features test series
test_dynamic_features = make_dyn_feat(train_df, features,prediction_test, freq=freq)

#print(test_dynamic_features)
# display first time series
time_series_idx = 0

# plot time series except the last two values corresponding to store and department
plt.figure(figsize=(15,4))
time_series[time_series_idx][:-2].plot(title='Serie n°{} - sales evolution'.format(time_series_idx))
plt.show()<jupyter_output><empty_output><jupyter_text>Splitting in Time<jupyter_code># create truncated, training time series
def create_training_series(complete_time_series, prediction_length):
    '''Given a complete list of time series data, create training time series.
       :param complete_time_series: A list of all complete time series.
       :param prediction_length: The number of points we want to predict.
       :return: A list of training time series.
       '''
    # get training series
    time_series_training = []
    
    for ts in complete_time_series:
        
        # save store and departement number for the serie as Info
        Info_df = ts[-2:]
        
        # truncate "prediction_length" weeks + 2 corresponding to the store and departement number
        truncate = prediction_length + 2
        Train_Serie_df = ts[:-truncate]
        
        # save Train_Serie with its Info
        time_series_training.append(pd.concat([Train_Serie_df, Info_df]))
        
    return time_series_training
# set prediction length to 36 weeks ~ 9 months that we will have to forecast with the trained model
prediction_length = 36 
time_series_training = create_training_series(time_series, prediction_length)
# make dynamic features for train series
train_dynamic_features = make_dyn_feat(train_df, features, prediction_length, freq=freq)
# display train/test time series
time_series_idx = 0

plt.figure(figsize=(15,4))
# test data is the whole time series
time_series[time_series_idx][:-2].plot(label='test', lw=3)
# train data is all but the last prediction pts
time_series_training[time_series_idx][:-2].plot(label='train', ls=':', lw=3)

plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Convert to JSON<jupyter_code>def series_to_json_obj(ts, dynamic_features):
    '''Returns a dictionary of values in DeepAR, JSON format.
       :param ts: A single time series.
       :param dynamic_features : dynamical featres corresponding to ts
       :return: A dictionary of values with "start", "target", "cat" and "dynamic_feat" keys.
       '''
    # get start time and target and cat from the time series, ts
    # with 2 lasts values being store and department number, if len(ts)<3, the list is empty
    if len(ts)< 3:
        target = []
    else:
        target = list(ts[:-2])
    no_nan_target = pd.Series(target).fillna('NaN').tolist()
    
    # get dynamic_feat related to the store (same dynamic_feat for all department of one store)
    store = int(ts[-2])
    
    json_obj = {"start": str(ts.index[0]),
                "target": no_nan_target,
                "cat": list(map(int, ts[-2:])),
                "dynamic_feat": dynamic_features[store]
               }
    return json_obj
# test out the code
ts = time_series_training[100]

json_obj = series_to_json_obj(ts,train_dynamic_features)

print(json_obj)<jupyter_output>{'start': '2010-02-05 00:00:00', 'target': [9731.05, 11838.72, 9620.64, 8785.76, 8678.56, 7650.12, 6416.62, 5995.61, 8839.61, 8731.12, 7025.53, 6527.13, 5279.71, 6856.03, 5971.1, 7352.65, 7605.71, 7865.05, 6938.34, 6542.43, 6341.13, 7864.03, 5930.88, 5785.82, 7873.69, 10894.4, 15464.74, 14371.26, 24084.77, 19731.92, 6059.35, 5180.93, 4892.42, 4884.66, 7490.25, 8597.51, 6924.63, 5685.51, 7699.94, 10758.45, 9882.2, 9555.0, 18633.55, 13103.91, 24949.2, 18880.73, 18032.75, 3824.32, 5832.43, 5901.66, 5521.55, 5425.96, 8343.52, 9763.95, 7927.01, 6366.97, 7311.89, 6599.46, 6543.77, 6116.47, 6230.74, 5621.35, 6145.48, 7159.79, 5623.76, 4563.31, 5268.62, 5166.66, 5831.95, 6584.94, 8384.43, 6689.47, 5908.7, 5482.89, 6364.86, 5985.18, 6791.2, 7592.65, 11799.02, 14356.6, 24585.84, 18995.15, 5215.29, 4326.63, 4790.21, 4792.96, 4662.72, 5764.43, 5332.8, 8525.54, 7251.87, 9947.73, 8135.39, 6915.09, 14529.15, 10526.0, 26097.66, 14642.07, 17986.73, 5483.91, 4236.56, 3750.01, 4193.36, 4313.9, 6176.76, 8[...]<jupyter_text>Saving Data, Locally<jupyter_code>def write_json_dataset(time_series, dynamic_features, filename): 
    with open(filename, 'wb') as f:
        # for each of our times series, there is one JSON line
        for ts in time_series:
            json_line = json.dumps(series_to_json_obj(ts,dynamic_features)) + '\n'
            json_line = json_line.encode('utf-8')
            f.write(json_line)
    print(filename + ' saved.')
# save this data to a local directory
data_dir = 'json_sales_data_dyn'

# make data dir, if it does not exist
if not os.path.exists(data_dir):
    os.makedirs(data_dir)
# directories to save train/test data
train_key = os.path.join(data_dir, 'train.json')
test_key = os.path.join(data_dir, 'test.json')

# write train/test JSON files
write_json_dataset(time_series_training, train_dynamic_features, train_key)        
write_json_dataset(time_series, test_dynamic_features, test_key)<jupyter_output>json_sales_data_dyn/train.json saved.
json_sales_data_dyn/test.json saved.<jupyter_text>Uploading data to S3<jupyter_code># session, role, bucket
sagemaker_session = sagemaker.Session()
role = get_execution_role()
bucket = sagemaker_session.default_bucket()
# general prefix
prefix='deepar-weekly-sales'

# *unique* train/test prefixes
train_prefix   = '{}/{}'.format(prefix, 'train')
test_prefix    = '{}/{}'.format(prefix, 'test')

# uploading data to S3, and saving locations
train_path  = sagemaker_session.upload_data(train_key, bucket=bucket, key_prefix=train_prefix)
test_path   = sagemaker_session.upload_data(test_key,  bucket=bucket, key_prefix=test_prefix)
# check locations
print('Training data is stored in: '+ train_path)
print('Test data is stored in: '+ test_path)<jupyter_output>Training data is stored in: s3://sagemaker-eu-central-1-293973958717/deepar-weekly-sales/train/train.json
Test data is stored in: s3://sagemaker-eu-central-1-293973958717/deepar-weekly-sales/test/test.json<jupyter_text>Training a DeepAR Estimator  Instantiate an Estimator<jupyter_code>from sagemaker.amazon.amazon_estimator import get_image_uri

image_name = get_image_uri(boto3.Session().region_name, # get the region
                           'forecasting-deepar') # specify image
from sagemaker.estimator import Estimator

# dir to save model artifacts
s3_output_path = "s3://{}/{}/output".format(bucket, prefix)

# instantiate a DeepAR estimator
deepar = Estimator(sagemaker_session=sagemaker_session,
                      image_name=image_name,
                      role=role,
                      train_instance_count=1,
                      train_instance_type='ml.c4.xlarge',
                      output_path=s3_output_path
                      )<jupyter_output><empty_output><jupyter_text>Setting Hyperparameters<jupyter_code>freq='W'
context_length=36 # same as prediction_length

hyperparameters = {
    "epochs": "50",
    "time_freq": freq,
    "prediction_length": str(prediction_length),
    "context_length": str(context_length),
    "num_cells": "50",
    "num_layers": "2",
    "mini_batch_size": "128",
    "learning_rate": "0.001",
    "early_stopping_patience": "10"
}
# set the hyperparams
deepar.set_hyperparameters(**hyperparameters)<jupyter_output><empty_output><jupyter_text>Training job<jupyter_code>%%time
# train and test channels
data_channels = {
    "train": train_path,
    "test": test_path
}

# fit the estimator
deepar.fit(inputs=data_channels)<jupyter_output>2020-07-09 12:52:51 Starting - Starting the training job...
2020-07-09 12:52:54 Starting - Launching requested ML instances......
2020-07-09 12:53:59 Starting - Preparing the instances for training...
2020-07-09 12:54:40 Downloading - Downloading input data...
2020-07-09 12:54:51 Training - Downloading the training image..[34mArguments: train[0m
[34m[07/09/2020 12:55:35 INFO 140675871012672] Reading default configuration from /opt/amazon/lib/python2.7/site-packages/algorithm/resources/default-input.json: {u'num_dynamic_feat': u'auto', u'dropout_rate': u'0.10', u'mini_batch_size': u'128', u'test_quantiles': u'[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]', u'_tuning_objective_metric': u'', u'_num_gpus': u'auto', u'num_eval_samples': u'100', u'learning_rate': u'0.001', u'num_cells': u'40', u'num_layers': u'2', u'embedding_dimension': u'10', u'_kvstore': u'auto', u'_num_kv_servers': u'auto', u'cardinality': u'auto', u'likelihood': u'student-t', u'early_stopping_patience': u''}[0m
[34[...]<jupyter_text>Deploy and Create a Predictor¶<jupyter_code>%%time

# create a predictor
predictor = deepar.deploy(
    initial_instance_count=1,
    instance_type='ml.t2.medium',
    content_type="application/json" # specify that it will accept/produce JSON
)<jupyter_output>-----------------!CPU times: user 285 ms, sys: 1.58 ms, total: 287 ms
Wall time: 8min 32s<jupyter_text>JSON Prediction RequestThe code below accepts a list of time series as input and some configuration parameters. It then formats that series into a JSON instance and converts the input into an appropriately formatted JSON_input.<jupyter_code>def json_predictor_input(input_ts, dynamic_feat, num_samples=50, quantiles=['0.1', '0.5', '0.9']):
    '''Accepts a list of input time series and produces a formatted input.
       :input_ts: An list of input time series.
       :dynamic_feat: dynamic features linked with input_ts
       :num_samples: Number of samples to calculate metrics with.
       :quantiles: A list of quantiles to return in the predicted output.
       :return: The JSON-formatted input.
       '''
    # request data is made of JSON objects (instances)
    # and an output configuration that details the type of data/quantiles we want
    
    instances = []
    
    for k in range(len(input_ts)):
        
        # get JSON objects for input time series providing all dynamic features
        instances.append(series_to_json_obj(input_ts[k],dynamic_feat))

    # specify the output quantiles and samples
    configuration = {"num_samples": num_samples, 
                     "output_types": ["quantiles"], 
                     "quantiles": quantiles}

    request_data = {"instances": instances, 
                    "configuration": configuration}

    json_request = json.dumps(request_data).encode('utf-8')
    
    return json_request<jupyter_output><empty_output><jupyter_text>Get a PredictionWe can then use this function to get a prediction for a formatted time series  In the next cell, I'm getting an input time series and known target, and passing the formatted input into the predictor endpoint to get a resultant prediction.<jupyter_code># get all input and target (test) time series
input_ts = time_series_training
target_ts = time_series

# get formatted input time series
json_input_ts = json_predictor_input(input_ts, test_dynamic_features)
# get the prediction from the predictor
json_prediction = predictor.predict(json_input_ts)

# print(json_prediction)
# helper function to decode JSON prediction
def decode_prediction(prediction, encoding='utf-8'):
    '''Accepts a JSON prediction and returns a list of prediction data.
    '''
    prediction_data = json.loads(prediction.decode(encoding))
    prediction_list = []
    for k in range(len(prediction_data['predictions'])):
        prediction_list.append(pd.DataFrame(data=prediction_data['predictions'][k]['quantiles']))
    return prediction_list
# get quantiles/predictions
prediction_list = decode_prediction(json_prediction)

# should get a list of 36 predictions 
# with corresponding quantile values
print(prediction_list[0])
# save output to a local directory
output_dir = 'DeepAR_Output'

# make data dir, if it does not exist
if not os.path.exists(output_dir):
    os.makedirs(output_dir)
# directories to save train/test data
output_key = os.path.join(output_dir, 'Dynfeattotal_prediction_list.txt')
with open(output_key, 'wb') as fp:
    pickle.dump(prediction_list, fp)
with open (output_key, 'rb') as fp:
    prediction_list = pickle.load(fp)<jupyter_output><empty_output><jupyter_text>Display the ResultsThe quantile data will give us all we need to see the results of our prediction.  Quantiles 0.1 and 0.9 represent higher and lower bounds for the predicted values.  Quantile 0.5 represents the median of all sample predictions.<jupyter_code># display train/test time series and prediction median

for time_series_idx in range(0,5):
    plt.figure(figsize=(10,4))
    
    # test data is the whole time series
    time_series[time_series_idx][:-2].plot(label='test', lw=3)
    
    # train data is all but the last prediction pts
    time_series_training[time_series_idx][:-2].plot(label='train', ls=':', lw=3)

    # prediction median is predication for quantile 0.5
    pred = prediction_list[time_series_idx]['0.5']
    # we want our 36 forecasted weeks , 2 last values being store and department number
    pred.index = time_series[time_series_idx][-38:-2].index
    pred.plot(label='prediction median', ls="--", lw=3)
    
    plt.legend()
    plt.show()
# instantiate empty lists to compute WMAE for our series
total_sum_weights=[]
total_sum_weighted_error=[]

for i in range(len(prediction_list)):  
    # merge prediction and train_df to create a dataframe with info to compute WMAE metric 
    prediction_df = pd.DataFrame(prediction_list[i]['0.5'])
    prediction_df.index = time_series[time_series_idx][-38:-2].index
    train_df.index = pd.to_datetime(train_df.index)
    store = time_series[i][-2]
    department = time_series[i][-1]
    data_df = train_df[(train_df.Cat_Store == store)&(train_df.Cat_Dept == department)]
    metric_df = data_df.merge(prediction_df, right_index=True, left_index=True)
    
    # Drop columns we will not use to compute WMAE
    metric_df = metric_df.drop(columns=['Temperature','Fuel_Price','MarkDown1','MarkDown2','MarkDown3','MarkDown4','MarkDown5','CPI','Unemployment'])
    
    # In competition evaluation, weights in WMAE are 5 if week is holiday, 1 otherwise
    metric_df.IsHoliday.replace([0,1], [1,5], inplace = True)
    
    # Compute and print WMAE for each series
    metric_df['weighted_error']=metric_df['IsHoliday']*abs(metric_df['Weekly_Sales']-metric_df['0.5'])
    sum_weights = metric_df.IsHoliday.sum()
    sum_weighted_error = metric_df.weighted_error.sum()
    wmae = (1/sum_weights) * sum_weighted_error
    print('Weighted mean absolute error (WMAE) for Cat_Store {} Cat_Dept {} = {}'.format(str(store),str(department),str(wmae)))
    
    if str(wmae) != 'nan':
        total_sum_weights.append(sum_weights)
        total_sum_weighted_error.append(sum_weighted_error)
        
global_wmae = (1/sum(total_sum_weights)) * sum(total_sum_weighted_error)
print('Global Weighted mean absolute error (WMAE) for these predicted series = {}'.format(str(global_wmae)))
def display_quantiles(prediction_list, target_ts=None):
    # show 5 firsts predictions
    
    for k in range(5):
        plt.figure(figsize=(12,6))
        # get the target month of data
        if target_ts is not None:
            target = target_ts[k][-prediction_length-2:-2]
            plt.plot(range(len(target)), target, label='target')
        # get the quantile values at 10 and 90%
        p10 = prediction_list[k]['0.1']
        p90 = prediction_list[k]['0.9']
        # fill the 80% confidence interval
        plt.fill_between(p10.index, p10, p90, color='y', alpha=0.5, label='80% confidence interval')
        
        # plot the median prediction line
        p50 = prediction_list[k]['0.5']
        plt.plot(p10.index, p50 , label='prediction median')
        
        plt.legend()
        plt.show()
# display predictions
display_quantiles(prediction_list, target_ts)
# delete the endpoint
predictor.delete_endpoint()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10771669737349501, 'fraction_numerical': 0.05210163246068812, 'mean_word_length': 3.866555462885738, 'pattern_counts': {'":': 20, '<': 62, '<?xml version=': 0, '>': 60, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '218735', 'license': 'MIT', 'n_tokens_mistral': 8407, 'n_tokens_neox': 7459, 'n_words': 2101, 'path': 'DeepAR_2stores_train_2features.ipynb', 'repo_name': 'Corentin-M/Sales_forecasting', 'size': '23339'}