<jupyter_start><jupyter_text>Simulator to run Markov Prediction over some datasets<jupyter_code>import os
import numpy as np
import pandas as pd
from main.utils.predictor import TransitionMatrix, MarkovChainPredictor, min_max_scaler, States
from predict.settings import DATASET_DIR
import matplotlib.pyplot as plt
def read_data(data):
    return pd.read_csv(os.path.join(DATASET_DIR, f"{data}.csv"))
data = read_data("flood_test")    
data.describe()
data["WaterLevel"] = data["waterlevel"]
data["WaterLevel"].values.shape
plt.plot(data["WaterLevel"])
max_height=2.0
def split_test_train(data):
    length = data.shape[0]
    test = int(length * 0.8)
    test_data = data[:test]
    train_data = data[test:]
    return pd.DataFrame({'WaterLevel': test_data}), pd.DataFrame({'WaterLevel': train_data})
train_data, test_data = split_test_train(data["WaterLevel"].values)
transition_matrix = TransitionMatrix(train_data, max_height=max_height)
transition_matrix.generate()
pd.DataFrame(transition_matrix.transitions)
current_state = States.get_state(train_data["WaterLevel"].values[-1])
predictor = MarkovChainPredictor(transition_matrix.values, transition_matrix.states)
no_predictions = len(test_data)
predictor.next_state(current_state)
predictions = predictor.generate_states(current_state, no_predictions=no_predictions)
test_data["WaterLevel"] = min_max_scaler(test_data["WaterLevel"].values, max_height)
test_data['state'] = States.get_states(test_data["WaterLevel"], type="names")
difference = pd.DataFrame({"Prediction": predictions, "Actual_state": test_data["state"][:no_predictions]})
difference.query('Prediction != Actual_state').shape
difference
accuracy = (difference.shape[0] - difference.query('Prediction != Actual_state').shape[0])/difference.shape[0]
accuracy
fig, ax = plt.subplots()
ax.plot(difference['Prediction'], label="predicted state")
ax.plot(difference['Actual_state'], label="actual state")
ax.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13387423935091278, 'fraction_numerical': 0.004563894523326572, 'mean_word_length': 8.719211822660098, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '445810', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 647, 'n_tokens_neox': 627, 'n_words': 110, 'path': 'simulator.ipynb', 'repo_name': 'NengakDakup/manas', 'size': '1972'}