<jupyter_start><jupyter_text>Create some maps<jupyter_code>import click
import os
import joblib
import numpy as np
import pandas as pd
from collections import defaultdict
from copy import deepcopy
from darts import TimeSeries

from pyprocessta.model.tcn import (
    TCNModelDropout,
    parallelized_inference,
    summarize_results,
)

THIS_DIR = os.path.abspath('/home/kjablonk/documents/timeseries_analysis/pyprocessta/examples')


def load_pickle(filename):
    with open(filename, "rb") as handle:
        res = pickle.load(handle)
    return res


def dump_pickle(object, filename):
    with open(filename, "wb") as handle:
        pickle.dump(object, handle)


MEAS_COLUMNS = [
    "TI-19",
    #      "FI-16",
    #     "TI-33",
    #     "FI-2",
    #     "FI-151",
    #     "TI-8",
    #     "FI-241",
    #  "valve-position-12",  # dry-bed
    #     "FI-38",  # strippera
    #     "PI-28",  # stripper
    #     "TI-28",  # stripper
    #      "FI-20",
    #     "FI-30",
    "TI-3",
    "FI-19",
    #     "FI-211",
    "FI-11",
    #     "TI-30",
    #     "PI-30",
    "TI-1213",
    #     "TI-4",
   "FI-23",
    "FI-20",
    #"FI-20/FI-23",
    #    "TI-22",
   # "delta_t",
    "TI-35",
    #"delta_t_2",
]

# First, we train a model on *all* data
# Then, we do a partial-denpendency plot approach and change one variable and see how the model predictions change

# load the trained model

model_cov1 = TCNModelDropout(
    input_chunk_length=8,
    output_chunk_length=1,
    num_layers=5,
    num_filters=16,
    kernel_size=6,
    dropout=0.3,
    weight_norm=True,
    batch_size=32,
    n_epochs=100,
    log_tensorboard=True,
    optimizer_kwargs={"lr": 2e-4},
)


model_cov2 = TCNModelDropout(
    input_chunk_length=8,
    output_chunk_length=1,
    num_layers=5,
    num_filters=16,
    kernel_size=6,
    dropout=0.3,
    weight_norm=True,
    batch_size=32,
    n_epochs=100,
    log_tensorboard=True,
    optimizer_kwargs={"lr": 2e-4},
)


FEAT_NUM_MAPPING = dict(zip(MEAS_COLUMNS, [str(i) for i in range(len(MEAS_COLUMNS))]))
UPDATE_MAPPING = {
    "amine": {
        "scaler": joblib.load("20210814_y_transformer__reduced_feature_set"),
        "model": model_cov1.load_from_checkpoint(
            os.path.join(THIS_DIR, "20210814_2amp_pip_model_reduced_feature_set_darts")
        ),
        "name": ["2-Amino-2-methylpropanol C4H11NO", "Piperazine C4H10N2"],
    },
    "co2": {
        "scaler": joblib.load("20210814_y_transformer_co2_ammonia_reduced_feature_set"),
        "model": model_cov2.load_from_checkpoint(
            os.path.join(
                THIS_DIR, "20210814_co2_ammonia_model_reduced_feature_set_darts"
            )
        ),
        "name": ["Carbon dioxide CO2", "Ammonia NH3"],
    },
}
SCALER = joblib.load("20210814_x_scaler_reduced_feature_set")


# making the input one item longer as "safety margin"
def calculate_initialization_percentage(
    timeseries_length: int, input_sequence_length: int = 61
):
    fraction_of_input = input_sequence_length / timeseries_length
    res = max([fraction_of_input, 0.1])
    return res


def run_update(df, x, target="amine"):
    model_dict = UPDATE_MAPPING[target]
    y = model_dict["scaler"].transform(
        TimeSeries.from_dataframe(df, value_cols=model_dict["name"])
    )
    df = parallelized_inference(
        model_dict["model"],
        x,
        y,
        repeats=1,
        start=calculate_initialization_percentage(len(y)),
        horizon=2,
    )
    means, stds = summarize_results(df)
    return {"means": means, "stds": stds}


def run_targets(df, feature_levels, target: str = "amine"):

    df = deepcopy(df)

    for k, v in feature_levels.items():
        if k == "valve-position-12":
            df[k] = v
        else:
            df[k] = df[k] + v / 100 * np.abs(df[k])

    X_ = TimeSeries.from_dataframe(df, value_cols=MEAS_COLUMNS)
    X_ = SCALER.transform(X_)

    res = run_update(df, X_, target)
    return res


def run_grid(
    df,
    feature_a: str = "TI-19",
    feature_b: str = "TI-1213",
    lower: float = -20,
    upper: float = 20,
    num_points: int = 5,
    objectives: str = "amine",
):
    grid = np.linspace(lower, upper, num_points)
    results_double_new = defaultdict(dict)

    if feature_a == "valve-position-12":
        grid_a = [0, 1]
    else:
        grid_a = grid
    if feature_b == "valve-position-12":
        grid_b = [0, 1]
    else:
        grid_b = grid

    for point_a in grid_a:
        for point_b in grid_b:
            print(f"Running point {feature_a}: {point_a} {feature_b}: {point_b}")
            results_double_new[point_a][point_b] = run_targets(
                df, {feature_a: point_a, feature_b: point_b}, objectives
            )
    return results_double_new

import pickle
mymap = run_grid(df)
def get_grids(d): 
    outer_keys = sorted(d.keys())
    inner_keys = sorted(d[outer_keys[0]].keys())
    
    return outer_keys, inner_keys
def make_image(res, objective="1"): 
    outer, inner = get_grids(res)

    image_m = np.zeros((len(outer), len(inner)))
    image_l = np.zeros((len(outer), len(inner)))
    image_t = np.zeros((len(outer), len(inner)))

    for i, point_x in enumerate(outer):
        for j, point_y in enumerate(inner):
            image_m[i][j] = np.sum(res[point_x][point_y]['means'][objective].values-res[0][0]['means'][objective].values)
    
   
    return  image_m, outer, inner
im, _, _ = make_image(mymap)
im2, _, _ = make_image(mymap, objective='0')
import matplotlib.pyplot as plt 
plt.imshow(im,cmap='coolwarm')
plt.imshow(im,cmap='coolwarm')
plt.imshow(im,cmap='coolwarm')
plt.imshow(im2, cmap='coolwarm')
plt.imshow(im2, cmap='coolwarm')
plt.imshow(im2, cmap='coolwarm')
with open('20210811_ti3_ti1212_amine.pkl', 'wb') as handle:
    pickle.dump(im, handle)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13502834564507815, 'fraction_numerical': 0.0338429823054458, 'mean_word_length': 2.9578518014955812, 'pattern_counts': {'":': 15, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '280998', 'license': 'MIT', 'n_tokens_mistral': 2332, 'n_tokens_neox': 2182, 'n_words': 407, 'path': 'paper/export_models.ipynb', 'repo_name': 'kjappelbaum/aeml', 'size': '5821'}