<jupyter_start><jupyter_text>Performing the Train-Test Split on a Time Series Dataset This notebook contains the code to help readers work through one of the recipes of the book [Machine Learning with Amazon SageMaker Cookbook: 80 proven recipes for data scientists and developers to perform ML experiments and deployments](https://www.amazon.com/Machine-Learning-Amazon-SageMaker-Cookbook/dp/1800567030)  How to do it...<jupyter_code>import json
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

%matplotlib inline
def load_data_from_json(filename):
    tmp = {}
    
    with open(filename) as file:
        tmp = json.load(file)
        
    index = pd.date_range(
        start=tmp["t0"], 
        freq=tmp["freq"], 
        periods=tmp["length"])
    tmp["data"] = pd.Series(
        data=tmp["data"], 
        index=index)
        
    return tmp
time_series_data = load_data_from_json(
    "tmp/all.json"
)
time_series_data
def train_test_split(data, ratio=0.9):
    train_length = int(len(data) * ratio)
    pl = int(len(data)) - train_length
    prediction_length = pl
    training_dataset = data[:-prediction_length]
    target_dataset = data[train_length-1:]
    test_dataset = data
    
    return {
        "prediction_length": prediction_length,
        "training_dataset": training_dataset,
        "target_dataset": target_dataset,
        "test_dataset": test_dataset
    }
results = train_test_split(
    time_series_data["data"]
)
print(results["prediction_length"])
training_dataset = results["training_dataset"]
target_dataset = results["target_dataset"]

plt.figure(figsize=(14,6))
plt.plot(training_dataset.index, 
         training_dataset, label="training")
plt.plot(target_dataset.index, 
         target_dataset, 
         label="target")
plt.grid(True)
plt.xlabel("DATE")
plt.ylabel("VALUE")    
plt.legend()
plt.show()
def series_to_object(data):
    return {"start": str(data.index[0]), 
            "target": list(data)}
def series_to_jsonline(data):
    return json.dumps(series_to_object(data))
def save_data_to_jsonlines(data, filename):
    tmp = {}
    
    with open(filename, 'wb') as file:
        t = series_to_jsonline(data)
        t = t.encode("utf-8")
        file.write(t)
        file.write("\n".encode("utf-8"))
save_data_to_jsonlines(
    results["training_dataset"], 
    "tmp/training.jsonl"
)
save_data_to_jsonlines(
    results["test_dataset"], 
    "tmp/test.jsonl"
)
s3_bucket = 'sagemaker-cookbook-bucket'
prefix = 'chapter08'
!aws s3 cp tmp/training.jsonl s3://{s3_bucket}/{prefix}/input/training.jsonl
!aws s3 cp tmp/test.jsonl s3://{s3_bucket}/{prefix}/input/test.jsonl

prediction_length = results["prediction_length"]
%store prediction_length
freq = time_series_data["freq"]
%store freq
training_dataset = results["training_dataset"]
%store training_dataset
training_dataset
target_dataset = results["target_dataset"]
%store target_dataset
target_dataset
%store s3_bucket
%store prefix<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11745078411745079, 'fraction_numerical': 0.010677344010677344, 'mean_word_length': 4.038655462184874, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '409250', 'license': 'MIT', 'n_tokens_mistral': 992, 'n_tokens_neox': 957, 'n_words': 203, 'path': 'Chapter08/08 - Performing the Train-Test Split on a Time Series Dataset.ipynb', 'repo_name': 'MARKOM/sagemaker-cookbook', 'size': '2997'}