new pipeline lstm

.gitignore

@@ -4,4 +4,5 @@ venv
 __pycache__/
 *.tf
 data
-*.keras
+*.keras
+*.csv

MA_temp_model.h5

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ed0c2c96cde21763e2a60597ab8abf0a50bd75ab954525845702014d219027c4
-size 533080

buildingdata.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:62e6cb23b7dfca34ca511b0ff99c68009f860968cd0c5688ac97ffa50a6aea8b
-size 24061812

energy_model.h5

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ad1ad8b65c5030e0dc971b3c488335f2f1d93b0b8acc2bfb6af10145237cda87
-size 780616

energy_model_01.h5

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:58e7e4823df5c20cd2fb4df23d2399d8b0fcf710147f3e52c9c1f3fcbeaa7cc1
-size 535480

kmeans_model.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5bddef9e8a8d413136daf6f99dd236fb6c97700d2fa85567de17aacb4a32bce1
-size 1927225

kmeans_model1.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aaa404e937bf8fe9d52b38aa1e11f87d544530a2d4789b46d20b5a1665e15846
-size 2064297

kmeans_model2.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9cf710504c910255e59d0609617fecf74ee8ad6710f4ba4109d9de5aae0cd879
-size 2064297

kmeans_model3.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1fb375913725bf309d514f55b71799daead266db7caad09f02ab9cfdd56bdc0a
-size 2064297

kmeans_model4.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cfbe4b1a5c1aa4b00e8e932c77dd8ed5ce99fbe5efd4960079d14e380206f9f2
-size 2064297

pca_model.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:225896826ab8bfb9ea32f0f3057c1292ca5bc238d2ea87858c25fa8da5960b46
-size 1179

pipeline.ipynb

@@ -0,0 +1,132 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "def initialize_lists(size=30):\n",
+    "    initial_values = [0] * size\n",
+    "    return initial_values.copy(), initial_values.copy(), initial_values.copy()\n",
+    "\n",
+    "def get_window(df, len_df):\n",
+    "    if len_df > 30:\n",
+    "        return df[len_df-31:len_df].astype('float32')\n",
+    "    else:\n",
+    "        return None\n",
+    "\n",
+    "def transform_window(df_window, scaler):\n",
+    "    return scaler.transform(df_window)\n",
+    "\n",
+    "def prepare_input(df_trans):\n",
+    "    return df_trans[:30,:].reshape((1,30,30))\n",
+    "\n",
+    "def predict(model, df_new):\n",
+    "    return model.predict(df_new)\n",
+    "\n",
+    "def calculate_residuals(df_trans, pred):\n",
+    "    actual = df_trans[30,:25]\n",
+    "    resid = actual - pred\n",
+    "    return actual, resid\n",
+    "\n",
+    "def resize_prediction(pred, df_trans):\n",
+    "    pred.resize((pred.shape[0], pred.shape[1] + len(df_trans[30,25:])))\n",
+    "    pred[:, -len(df_trans[30,25:]):] = df_trans[30,25:]\n",
+    "    return pred\n",
+    "\n",
+    "def inverse_transform(scaler, pred, df_trans):\n",
+    "    pred = scaler.inverse_transform(np.array(pred))\n",
+    "    actual = scaler.inverse_transform(np.array([df_trans[30,:]]))\n",
+    "    return actual, pred\n",
+    "\n",
+    "def update_lists(actual_list, pred_list, resid_list, actual, pred, resid):\n",
+    "    actual_list.pop(0)\n",
+    "    pred_list.pop(0)\n",
+    "    resid_list.pop(0)\n",
+    "    actual_list.append(actual[0,1])\n",
+    "    pred_list.append(pred[0,1])\n",
+    "    resid_list.append(resid[0,1])\n",
+    "    return actual_list, pred_list, resid_list\n",
+    "\n",
+    "def calculate_distances(resid, kmeans1, kmeans2, kmeans3, kmeans4):\n",
+    "    dist = []\n",
+    "    dist.append(np.linalg.norm(resid[:,1:8]-kmeans1.cluster_centers_[0], ord=2, axis=1))\n",
+    "    dist.append(np.linalg.norm(resid[:,8:15]-kmeans2.cluster_centers_[0], ord=2, axis=1))\n",
+    "    dist.append(np.linalg.norm(resid[:,15:22]-kmeans3.cluster_centers_[0], ord=2, axis=1))\n",
+    "    dist.append(np.linalg.norm(resid[:,22:29]-kmeans4.cluster_centers_[0], ord=2, axis=1))\n",
+    "    return np.array(dist)\n",
+    "\n",
+    "def pipeline(df, scaler, model, kmeans1, kmeans2, kmeans3, kmeans4):\n",
+    "    actual_list, pred_list, resid_list = initialize_lists()\n",
+    "    len_df = np.len(df)\n",
+    "    df_window = get_window(df, len_df)\n",
+    "    if df_window is not None:\n",
+    "        df_trans = transform_window(df_window, scaler)\n",
+    "        df_new = prepare_input(df_trans)\n",
+    "        pred = predict(model, df_new)\n",
+    "        actual, resid = calculate_residuals(df_trans, pred)\n",
+    "        pred = resize_prediction(pred, df_trans)\n",
+    "        actual, pred = inverse_transform(scaler, pred, df_trans)\n",
+    "        actual_list, pred_list, resid_list = update_lists(actual_list, pred_list, resid_list, actual, pred, resid)\n",
+    "        dist = calculate_distances(resid, kmeans1, kmeans2, kmeans3, kmeans4)\n",
+    "        return actual_list, pred_list, resid_list, dist\n",
+    "    else:\n",
+    "        return actual_list, pred_list, resid_list, None\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['a', 'b', 'c', 'd']"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "inp = ['a', 'b']\n",
+    "out = ['c','d']\n",
+    "\n",
+    "inp+out"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "smartbuilding",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

sample_data.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aa13789e2460e1b575a63d32ca46615d5df237c62de5da2f4da22c1e8edc945c
-size 144290729

sample_data_01.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8a00542784412e885f2ef707e3dd86b58023e678b35221fefbce5af9ec6cb10d
-size 59769388

sample_data_02.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3fe51d16cfeffdc7df991e0f27da089da5e9f1260e2ab9c316cd6786999d8d29
-size 119517162

sample_data_smooth_01.csv

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:efbfdef5e85cb88d20e7e50ac2351f60d20f07a31b2a40cf4c925596097c5cc4
-size 299846687

scaler.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:63d98018c0b4da0c42820e9017c91944d1486e85471cd44a9a6ab121e7d2c64a
-size 1101

scaler_1.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:778df19577ac7a34c16dfda1b32f26800c62ff687130e22e58715b8d495dcb37
-size 1437