Add LSTM code for VAV analysis

physLSTM/full_lstm.ipynb

@@ -43,9 +43,423 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>zone_047_hw_valve</th>\n",
+       "      <th>rtu_004_sat_sp_tn</th>\n",
+       "      <th>zone_047_temp</th>\n",
+       "      <th>zone_047_fan_spd</th>\n",
+       "      <th>rtu_004_fltrd_sa_flow_tn</th>\n",
+       "      <th>rtu_004_sa_temp</th>\n",
+       "      <th>rtu_004_pa_static_stpt_tn</th>\n",
+       "      <th>rtu_004_oa_flow_tn</th>\n",
+       "      <th>rtu_004_oadmpr_pct</th>\n",
+       "      <th>...</th>\n",
+       "      <th>zone_047_heating_sp</th>\n",
+       "      <th>Unnamed: 47_y</th>\n",
+       "      <th>hvac_S</th>\n",
+       "      <th>hp_hws_temp</th>\n",
+       "      <th>aru_001_cwr_temp</th>\n",
+       "      <th>aru_001_cws_fr_gpm</th>\n",
+       "      <th>aru_001_cws_temp</th>\n",
+       "      <th>aru_001_hwr_temp</th>\n",
+       "      <th>aru_001_hws_fr_gpm</th>\n",
+       "      <th>aru_001_hws_temp</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2018-01-01 00:00:00</td>\n",
+       "      <td>100.0</td>\n",
+       "      <td>69.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>9265.604</td>\n",
+       "      <td>66.1</td>\n",
+       "      <td>0.06</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>28.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>75.3</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2018-01-01 00:01:00</td>\n",
+       "      <td>100.0</td>\n",
+       "      <td>69.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>9265.604</td>\n",
+       "      <td>66.0</td>\n",
+       "      <td>0.06</td>\n",
+       "      <td>6572.099162</td>\n",
+       "      <td>28.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>75.3</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2018-01-01 00:02:00</td>\n",
+       "      <td>100.0</td>\n",
+       "      <td>69.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>9708.240</td>\n",
+       "      <td>66.1</td>\n",
+       "      <td>0.06</td>\n",
+       "      <td>7628.832542</td>\n",
+       "      <td>28.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>75.3</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>2018-01-01 00:03:00</td>\n",
+       "      <td>100.0</td>\n",
+       "      <td>69.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>9611.638</td>\n",
+       "      <td>66.1</td>\n",
+       "      <td>0.06</td>\n",
+       "      <td>7710.294617</td>\n",
+       "      <td>28.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>75.3</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>2018-01-01 00:04:00</td>\n",
+       "      <td>100.0</td>\n",
+       "      <td>69.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>9215.110</td>\n",
+       "      <td>66.0</td>\n",
+       "      <td>0.06</td>\n",
+       "      <td>7139.184090</td>\n",
+       "      <td>28.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>75.3</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072149</th>\n",
+       "      <td>2020-12-31 23:58:00</td>\n",
+       "      <td>100.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>63.2</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>18884.834</td>\n",
+       "      <td>64.4</td>\n",
+       "      <td>0.06</td>\n",
+       "      <td>2938.320000</td>\n",
+       "      <td>23.4</td>\n",
+       "      <td>...</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>69.0</td>\n",
+       "      <td>23.145000</td>\n",
+       "      <td>123.8</td>\n",
+       "      <td>56.25</td>\n",
+       "      <td>54.71</td>\n",
+       "      <td>56.4</td>\n",
+       "      <td>123.42</td>\n",
+       "      <td>61.6</td>\n",
+       "      <td>122.36</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072150</th>\n",
+       "      <td>2020-12-31 23:58:00</td>\n",
+       "      <td>100.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>63.2</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>18884.834</td>\n",
+       "      <td>64.4</td>\n",
+       "      <td>0.06</td>\n",
+       "      <td>2938.320000</td>\n",
+       "      <td>23.4</td>\n",
+       "      <td>...</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>69.0</td>\n",
+       "      <td>23.145000</td>\n",
+       "      <td>123.8</td>\n",
+       "      <td>56.25</td>\n",
+       "      <td>54.71</td>\n",
+       "      <td>56.4</td>\n",
+       "      <td>123.42</td>\n",
+       "      <td>61.6</td>\n",
+       "      <td>122.36</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072151</th>\n",
+       "      <td>2020-12-31 23:59:00</td>\n",
+       "      <td>100.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>63.2</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>19345.508</td>\n",
+       "      <td>64.3</td>\n",
+       "      <td>0.06</td>\n",
+       "      <td>3154.390000</td>\n",
+       "      <td>23.4</td>\n",
+       "      <td>...</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>69.0</td>\n",
+       "      <td>23.145000</td>\n",
+       "      <td>123.8</td>\n",
+       "      <td>56.25</td>\n",
+       "      <td>54.71</td>\n",
+       "      <td>56.4</td>\n",
+       "      <td>123.42</td>\n",
+       "      <td>61.6</td>\n",
+       "      <td>122.36</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072152</th>\n",
+       "      <td>2020-12-31 23:59:00</td>\n",
+       "      <td>100.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>63.2</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>19345.508</td>\n",
+       "      <td>64.3</td>\n",
+       "      <td>0.06</td>\n",
+       "      <td>3154.390000</td>\n",
+       "      <td>23.4</td>\n",
+       "      <td>...</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>69.0</td>\n",
+       "      <td>23.145000</td>\n",
+       "      <td>123.8</td>\n",
+       "      <td>56.25</td>\n",
+       "      <td>54.71</td>\n",
+       "      <td>56.4</td>\n",
+       "      <td>123.42</td>\n",
+       "      <td>61.6</td>\n",
+       "      <td>122.36</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072153</th>\n",
+       "      <td>2021-01-01 00:00:00</td>\n",
+       "      <td>100.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>63.2</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>18650.232</td>\n",
+       "      <td>64.1</td>\n",
+       "      <td>0.06</td>\n",
+       "      <td>3076.270000</td>\n",
+       "      <td>22.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>69.0</td>\n",
+       "      <td>23.788947</td>\n",
+       "      <td>123.8</td>\n",
+       "      <td>56.25</td>\n",
+       "      <td>54.71</td>\n",
+       "      <td>56.4</td>\n",
+       "      <td>123.42</td>\n",
+       "      <td>61.6</td>\n",
+       "      <td>122.36</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>2072154 rows × 30 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                        date  zone_047_hw_valve  rtu_004_sat_sp_tn  \\\n",
+       "0        2018-01-01 00:00:00              100.0               69.0   \n",
+       "1        2018-01-01 00:01:00              100.0               69.0   \n",
+       "2        2018-01-01 00:02:00              100.0               69.0   \n",
+       "3        2018-01-01 00:03:00              100.0               69.0   \n",
+       "4        2018-01-01 00:04:00              100.0               69.0   \n",
+       "...                      ...                ...                ...   \n",
+       "2072149  2020-12-31 23:58:00              100.0               68.0   \n",
+       "2072150  2020-12-31 23:58:00              100.0               68.0   \n",
+       "2072151  2020-12-31 23:59:00              100.0               68.0   \n",
+       "2072152  2020-12-31 23:59:00              100.0               68.0   \n",
+       "2072153  2021-01-01 00:00:00              100.0               68.0   \n",
+       "\n",
+       "         zone_047_temp  zone_047_fan_spd  rtu_004_fltrd_sa_flow_tn  \\\n",
+       "0                 67.5              20.0                  9265.604   \n",
+       "1                 67.5              20.0                  9265.604   \n",
+       "2                 67.5              20.0                  9708.240   \n",
+       "3                 67.5              20.0                  9611.638   \n",
+       "4                 67.5              20.0                  9215.110   \n",
+       "...                ...               ...                       ...   \n",
+       "2072149           63.2              20.0                 18884.834   \n",
+       "2072150           63.2              20.0                 18884.834   \n",
+       "2072151           63.2              20.0                 19345.508   \n",
+       "2072152           63.2              20.0                 19345.508   \n",
+       "2072153           63.2              20.0                 18650.232   \n",
+       "\n",
+       "         rtu_004_sa_temp  rtu_004_pa_static_stpt_tn  rtu_004_oa_flow_tn  \\\n",
+       "0                   66.1                       0.06            0.000000   \n",
+       "1                   66.0                       0.06         6572.099162   \n",
+       "2                   66.1                       0.06         7628.832542   \n",
+       "3                   66.1                       0.06         7710.294617   \n",
+       "4                   66.0                       0.06         7139.184090   \n",
+       "...                  ...                        ...                 ...   \n",
+       "2072149             64.4                       0.06         2938.320000   \n",
+       "2072150             64.4                       0.06         2938.320000   \n",
+       "2072151             64.3                       0.06         3154.390000   \n",
+       "2072152             64.3                       0.06         3154.390000   \n",
+       "2072153             64.1                       0.06         3076.270000   \n",
+       "\n",
+       "         rtu_004_oadmpr_pct  ...  zone_047_heating_sp  Unnamed: 47_y  \\\n",
+       "0                      28.0  ...                  NaN            NaN   \n",
+       "1                      28.0  ...                  NaN            NaN   \n",
+       "2                      28.0  ...                  NaN            NaN   \n",
+       "3                      28.0  ...                  NaN            NaN   \n",
+       "4                      28.0  ...                  NaN            NaN   \n",
+       "...                     ...  ...                  ...            ...   \n",
+       "2072149                23.4  ...                 71.0           69.0   \n",
+       "2072150                23.4  ...                 71.0           69.0   \n",
+       "2072151                23.4  ...                 71.0           69.0   \n",
+       "2072152                23.4  ...                 71.0           69.0   \n",
+       "2072153                22.9  ...                 71.0           69.0   \n",
+       "\n",
+       "            hvac_S  hp_hws_temp  aru_001_cwr_temp  aru_001_cws_fr_gpm  \\\n",
+       "0              NaN         75.3               NaN                 NaN   \n",
+       "1              NaN         75.3               NaN                 NaN   \n",
+       "2              NaN         75.3               NaN                 NaN   \n",
+       "3              NaN         75.3               NaN                 NaN   \n",
+       "4              NaN         75.3               NaN                 NaN   \n",
+       "...            ...          ...               ...                 ...   \n",
+       "2072149  23.145000        123.8             56.25               54.71   \n",
+       "2072150  23.145000        123.8             56.25               54.71   \n",
+       "2072151  23.145000        123.8             56.25               54.71   \n",
+       "2072152  23.145000        123.8             56.25               54.71   \n",
+       "2072153  23.788947        123.8             56.25               54.71   \n",
+       "\n",
+       "         aru_001_cws_temp  aru_001_hwr_temp  aru_001_hws_fr_gpm  \\\n",
+       "0                     NaN               NaN                 NaN   \n",
+       "1                     NaN               NaN                 NaN   \n",
+       "2                     NaN               NaN                 NaN   \n",
+       "3                     NaN               NaN                 NaN   \n",
+       "4                     NaN               NaN                 NaN   \n",
+       "...                   ...               ...                 ...   \n",
+       "2072149              56.4            123.42                61.6   \n",
+       "2072150              56.4            123.42                61.6   \n",
+       "2072151              56.4            123.42                61.6   \n",
+       "2072152              56.4            123.42                61.6   \n",
+       "2072153              56.4            123.42                61.6   \n",
+       "\n",
+       "         aru_001_hws_temp  \n",
+       "0                     NaN  \n",
+       "1                     NaN  \n",
+       "2                     NaN  \n",
+       "3                     NaN  \n",
+       "4                     NaN  \n",
+       "...                   ...  \n",
+       "2072149            122.36  \n",
+       "2072150            122.36  \n",
+       "2072151            122.36  \n",
+       "2072152            122.36  \n",
+       "2072153            122.36  \n",
+       "\n",
+       "[2072154 rows x 30 columns]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "merged = pd.read_csv(r'../data/long_merge.csv')\n",
     "\n",
@@ -70,7 +484,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -207,7 +621,7 @@
        "[2 rows x 23 columns]"
       ]
      },
-     "execution_count": 81,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -230,7 +644,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -245,7 +659,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -267,7 +681,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -276,7 +690,7 @@
        "[]"
       ]
      },
-     "execution_count": 98,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -297,7 +711,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -306,7 +720,7 @@
        "2072154"
       ]
      },
-     "execution_count": 110,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -317,7 +731,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -326,7 +740,7 @@
        "(1157787, 909910)"
       ]
      },
-     "execution_count": 99,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -337,7 +751,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -351,102 +765,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Epoch 1/5\n",
-      "9045/9045 [==============================] - ETA: 0s - loss: 0.0405\n",
-      "Epoch 1: val_loss improved from inf to 0.03129, saving model to lstm_smooth_01.tf\n",
-      "INFO:tensorflow:Assets written to: lstm_smooth_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:tensorflow:Assets written to: lstm_smooth_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "9045/9045 [==============================] - 346s 38ms/step - loss: 0.0405 - val_loss: 0.0313\n",
-      "Epoch 2/5\n",
-      "9045/9045 [==============================] - ETA: 0s - loss: 0.0228\n",
-      "Epoch 2: val_loss improved from 0.03129 to 0.02697, saving model to lstm_smooth_01.tf\n",
-      "INFO:tensorflow:Assets written to: lstm_smooth_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:tensorflow:Assets written to: lstm_smooth_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "9045/9045 [==============================] - 500s 55ms/step - loss: 0.0228 - val_loss: 0.0270\n",
-      "Epoch 3/5\n",
-      "9044/9045 [============================>.] - ETA: 0s - loss: 0.0211\n",
-      "Epoch 3: val_loss improved from 0.02697 to 0.02597, saving model to lstm_smooth_01.tf\n",
-      "INFO:tensorflow:Assets written to: lstm_smooth_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:tensorflow:Assets written to: lstm_smooth_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "9045/9045 [==============================] - 389s 43ms/step - loss: 0.0211 - val_loss: 0.0260\n",
-      "Epoch 4/5\n",
-      "9044/9045 [============================>.] - ETA: 0s - loss: 0.0203\n",
-      "Epoch 4: val_loss improved from 0.02597 to 0.02452, saving model to lstm_smooth_01.tf\n",
-      "INFO:tensorflow:Assets written to: lstm_smooth_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:tensorflow:Assets written to: lstm_smooth_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "9045/9045 [==============================] - 433s 48ms/step - loss: 0.0203 - val_loss: 0.0245\n",
-      "Epoch 5/5\n",
-      "9044/9045 [============================>.] - ETA: 0s - loss: 0.0198\n",
-      "Epoch 5: val_loss did not improve from 0.02452\n",
-      "9045/9045 [==============================] - 420s 46ms/step - loss: 0.0198 - val_loss: 0.0251\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "<keras.src.callbacks.History at 0x1b4590f0250>"
-      ]
-     },
-     "execution_count": 101,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "train,test = traindataset,testdataset\n",
     "\n",
@@ -480,6 +801,46 @@
     "# model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=5, batch_size=128, verbose=1, callbacks=[checkpoint_callback])"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1157756, 30, 15)"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "X_train.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1157756, 8)"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "y_train.shape"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

physLSTM/{lstm_vav.ipynb → lstm_vav_rtu1.ipynb}

@@ -31,32 +31,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 3,
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "MemoryError",
-     "evalue": "Unable to allocate 8.15 GiB for an array with shape (528, 2072154) and data type float64",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
-      "Cell \u001b[1;32mIn[10], line 23\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[38;5;66;03m# for rtu in rtus:\u001b[39;00m\n\u001b[0;32m     15\u001b[0m \u001b[38;5;66;03m#     for column in merged.columns:\u001b[39;00m\n\u001b[0;32m     16\u001b[0m \u001b[38;5;66;03m#         if f\"rtu_00{rtu}_fltrd_sa\" in column:\u001b[39;00m\n\u001b[0;32m     17\u001b[0m \u001b[38;5;66;03m#                 cols.append(column)\u001b[39;00m\n\u001b[0;32m     18\u001b[0m cols \u001b[38;5;241m=\u001b[39m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdate\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m+\u001b[39m cols \u001b[38;5;241m+\u001b[39m [\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mair_temp_set_1\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m     19\u001b[0m  \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mair_temp_set_2\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m     20\u001b[0m  \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdew_point_temperature_set_1d\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m     21\u001b[0m  \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrelative_humidity_set_1\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m     22\u001b[0m  \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msolar_radiation_set_1\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m---> 23\u001b[0m input_dataset \u001b[38;5;241m=\u001b[39m \u001b[43mmerged\u001b[49m\u001b[43m[\u001b[49m\u001b[43mcols\u001b[49m\u001b[43m]\u001b[49m\n\u001b[0;32m     24\u001b[0m input_dataset\u001b[38;5;241m.\u001b[39mcolumns\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\frame.py:4105\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   4102\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(indexer, \u001b[38;5;28mslice\u001b[39m):\n\u001b[0;32m   4103\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_slice(indexer, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m-> 4105\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_take_with_is_copy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m   4107\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_single_key:\n\u001b[0;32m   4108\u001b[0m     \u001b[38;5;66;03m# What does looking for a single key in a non-unique index return?\u001b[39;00m\n\u001b[0;32m   4109\u001b[0m     \u001b[38;5;66;03m# The behavior is inconsistent. It returns a Series, except when\u001b[39;00m\n\u001b[0;32m   4110\u001b[0m     \u001b[38;5;66;03m# - the key itself is repeated (test on data.shape, #9519), or\u001b[39;00m\n\u001b[0;32m   4111\u001b[0m     \u001b[38;5;66;03m# - we have a MultiIndex on columns (test on self.columns, #21309)\u001b[39;00m\n\u001b[0;32m   4112\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m data\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns, MultiIndex):\n\u001b[0;32m   4113\u001b[0m         \u001b[38;5;66;03m# GH#26490 using data[key] can cause RecursionError\u001b[39;00m\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\generic.py:4150\u001b[0m, in \u001b[0;36mNDFrame._take_with_is_copy\u001b[1;34m(self, indices, axis)\u001b[0m\n\u001b[0;32m   4139\u001b[0m \u001b[38;5;129m@final\u001b[39m\n\u001b[0;32m   4140\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_take_with_is_copy\u001b[39m(\u001b[38;5;28mself\u001b[39m, indices, axis: Axis \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Self:\n\u001b[0;32m   4141\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m   4142\u001b[0m \u001b[38;5;124;03m    Internal version of the `take` method that sets the `_is_copy`\u001b[39;00m\n\u001b[0;32m   4143\u001b[0m \u001b[38;5;124;03m    attribute to keep track of the parent dataframe (using in indexing\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   4148\u001b[0m \u001b[38;5;124;03m    See the docstring of `take` for full explanation of the parameters.\u001b[39;00m\n\u001b[0;32m   4149\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m-> 4150\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtake\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindices\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindices\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maxis\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   4151\u001b[0m     \u001b[38;5;66;03m# Maybe set copy if we didn't actually change the index.\u001b[39;00m\n\u001b[0;32m   4152\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m result\u001b[38;5;241m.\u001b[39m_get_axis(axis)\u001b[38;5;241m.\u001b[39mequals(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_axis(axis)):\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\generic.py:4130\u001b[0m, in \u001b[0;36mNDFrame.take\u001b[1;34m(self, indices, axis, **kwargs)\u001b[0m\n\u001b[0;32m   4125\u001b[0m     \u001b[38;5;66;03m# We can get here with a slice via DataFrame.__getitem__\u001b[39;00m\n\u001b[0;32m   4126\u001b[0m     indices \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39marange(\n\u001b[0;32m   4127\u001b[0m         indices\u001b[38;5;241m.\u001b[39mstart, indices\u001b[38;5;241m.\u001b[39mstop, indices\u001b[38;5;241m.\u001b[39mstep, dtype\u001b[38;5;241m=\u001b[39mnp\u001b[38;5;241m.\u001b[39mintp\n\u001b[0;32m   4128\u001b[0m     )\n\u001b[1;32m-> 4130\u001b[0m new_data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_mgr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtake\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   4131\u001b[0m \u001b[43m    \u001b[49m\u001b[43mindices\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   4132\u001b[0m \u001b[43m    \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_block_manager_axis\u001b[49m\u001b[43m(\u001b[49m\u001b[43maxis\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   4133\u001b[0m \u001b[43m    \u001b[49m\u001b[43mverify\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[0;32m   4134\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   4135\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_constructor_from_mgr(new_data, axes\u001b[38;5;241m=\u001b[39mnew_data\u001b[38;5;241m.\u001b[39maxes)\u001b[38;5;241m.\u001b[39m__finalize__(\n\u001b[0;32m   4136\u001b[0m     \u001b[38;5;28mself\u001b[39m, method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtake\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   4137\u001b[0m )\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\internals\\managers.py:894\u001b[0m, in \u001b[0;36mBaseBlockManager.take\u001b[1;34m(self, indexer, axis, verify)\u001b[0m\n\u001b[0;32m    891\u001b[0m indexer \u001b[38;5;241m=\u001b[39m maybe_convert_indices(indexer, n, verify\u001b[38;5;241m=\u001b[39mverify)\n\u001b[0;32m    893\u001b[0m new_labels \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maxes[axis]\u001b[38;5;241m.\u001b[39mtake(indexer)\n\u001b[1;32m--> 894\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreindex_indexer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    895\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnew_axis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnew_labels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    896\u001b[0m \u001b[43m    \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    897\u001b[0m \u001b[43m    \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maxis\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    898\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_dups\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[0;32m    899\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcopy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[0;32m    900\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\internals\\managers.py:680\u001b[0m, in \u001b[0;36mBaseBlockManager.reindex_indexer\u001b[1;34m(self, new_axis, indexer, axis, fill_value, allow_dups, copy, only_slice, use_na_proxy)\u001b[0m\n\u001b[0;32m    677\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mIndexError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRequested axis not found in manager\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    679\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m axis \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m--> 680\u001b[0m     new_blocks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_slice_take_blocks_ax0\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    681\u001b[0m \u001b[43m        \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    682\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfill_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    683\u001b[0m \u001b[43m        \u001b[49m\u001b[43monly_slice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43monly_slice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    684\u001b[0m \u001b[43m        \u001b[49m\u001b[43muse_na_proxy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_na_proxy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    685\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    686\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    687\u001b[0m     new_blocks \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m    688\u001b[0m         blk\u001b[38;5;241m.\u001b[39mtake_nd(\n\u001b[0;32m    689\u001b[0m             indexer,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    695\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m blk \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mblocks\n\u001b[0;32m    696\u001b[0m     ]\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\internals\\managers.py:843\u001b[0m, in \u001b[0;36mBaseBlockManager._slice_take_blocks_ax0\u001b[1;34m(self, slice_or_indexer, fill_value, only_slice, use_na_proxy, ref_inplace_op)\u001b[0m\n\u001b[0;32m    841\u001b[0m                     blocks\u001b[38;5;241m.\u001b[39mappend(nb)\n\u001b[0;32m    842\u001b[0m             \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 843\u001b[0m                 nb \u001b[38;5;241m=\u001b[39m \u001b[43mblk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtake_nd\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtaker\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnew_mgr_locs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmgr_locs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    844\u001b[0m                 blocks\u001b[38;5;241m.\u001b[39mappend(nb)\n\u001b[0;32m    846\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m blocks\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\internals\\blocks.py:1307\u001b[0m, in \u001b[0;36mBlock.take_nd\u001b[1;34m(self, indexer, axis, new_mgr_locs, fill_value)\u001b[0m\n\u001b[0;32m   1304\u001b[0m     allow_fill \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m   1306\u001b[0m \u001b[38;5;66;03m# Note: algos.take_nd has upcast logic similar to coerce_to_target_dtype\u001b[39;00m\n\u001b[1;32m-> 1307\u001b[0m new_values \u001b[38;5;241m=\u001b[39m \u001b[43malgos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtake_nd\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1308\u001b[0m \u001b[43m    \u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maxis\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mallow_fill\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mallow_fill\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfill_value\u001b[49m\n\u001b[0;32m   1309\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1311\u001b[0m \u001b[38;5;66;03m# Called from three places in managers, all of which satisfy\u001b[39;00m\n\u001b[0;32m   1312\u001b[0m \u001b[38;5;66;03m#  these assertions\u001b[39;00m\n\u001b[0;32m   1313\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m, ExtensionBlock):\n\u001b[0;32m   1314\u001b[0m     \u001b[38;5;66;03m# NB: in this case, the 'axis' kwarg will be ignored in the\u001b[39;00m\n\u001b[0;32m   1315\u001b[0m     \u001b[38;5;66;03m#  algos.take_nd call above.\u001b[39;00m\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\array_algos\\take.py:117\u001b[0m, in \u001b[0;36mtake_nd\u001b[1;34m(arr, indexer, axis, fill_value, allow_fill)\u001b[0m\n\u001b[0;32m    114\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m arr\u001b[38;5;241m.\u001b[39mtake(indexer, fill_value\u001b[38;5;241m=\u001b[39mfill_value, allow_fill\u001b[38;5;241m=\u001b[39mallow_fill)\n\u001b[0;32m    116\u001b[0m arr \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39masarray(arr)\n\u001b[1;32m--> 117\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_take_nd_ndarray\u001b[49m\u001b[43m(\u001b[49m\u001b[43marr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mallow_fill\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\array_algos\\take.py:157\u001b[0m, in \u001b[0;36m_take_nd_ndarray\u001b[1;34m(arr, indexer, axis, fill_value, allow_fill)\u001b[0m\n\u001b[0;32m    155\u001b[0m     out \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mempty(out_shape, dtype\u001b[38;5;241m=\u001b[39mdtype, order\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mF\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    156\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 157\u001b[0m     out \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mempty(out_shape, dtype\u001b[38;5;241m=\u001b[39mdtype)\n\u001b[0;32m    159\u001b[0m func \u001b[38;5;241m=\u001b[39m _get_take_nd_function(\n\u001b[0;32m    160\u001b[0m     arr\u001b[38;5;241m.\u001b[39mndim, arr\u001b[38;5;241m.\u001b[39mdtype, out\u001b[38;5;241m.\u001b[39mdtype, axis\u001b[38;5;241m=\u001b[39maxis, mask_info\u001b[38;5;241m=\u001b[39mmask_info\n\u001b[0;32m    161\u001b[0m )\n\u001b[0;32m    162\u001b[0m func(arr, indexer, out, fill_value)\n",
-      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 8.15 GiB for an array with shape (528, 2072154) and data type float64"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "zones = [69, 68,67, 66,65.64, 42,41,40,39,38,37,36]\n",
+    "zones = [69, 68,67, 66,65, 64, 42,41,40,39,38,37,36]\n",
     "rtus = [1]\n",
     "cols = []\n",
     "\n",
@@ -67,8 +46,9 @@
     "\n",
     "for zone in zones:\n",
     "        for column in merged.columns:\n",
-    "              if f\"zone_0{zone}\" in column and \"cooling_sp\" in column or \"heating_sp\" in column:\n",
-    "                cols.append(column)\n",
+    "              if f\"zone_0{zone}\" in column: \n",
+    "                if \"cooling_sp\" in column or \"heating_sp\" in column:\n",
+    "                  cols.append(column)\n",
     "# for rtu in rtus:\n",
     "#     for column in merged.columns:\n",
     "#         if f\"rtu_00{rtu}_fltrd_sa\" in column:\n",
@@ -83,14 +63,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "C:\\Users\\arbal\\AppData\\Local\\Temp\\ipykernel_32464\\216607548.py:1: SettingWithCopyWarning: \n",
+      "C:\\Users\\arbal\\AppData\\Local\\Temp\\ipykernel_16740\\4293840618.py:1: SettingWithCopyWarning: \n",
       "A value is trying to be set on a copy of a slice from a DataFrame.\n",
       "Try using .loc[row_indexer,col_indexer] = value instead\n",
       "\n",
@@ -99,24 +79,10 @@
      ]
     },
     {
-     "ename": "MemoryError",
-     "evalue": "Unable to allocate 8.15 GiB for an array with shape (528, 2070713) and data type float64",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
-      "Cell \u001b[1;32mIn[11], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m input_dataset[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdate\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mto_datetime(input_dataset[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdate\u001b[39m\u001b[38;5;124m'\u001b[39m], \u001b[38;5;28mformat\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mY-\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mm-\u001b[39m\u001b[38;5;132;01m%d\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mH:\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mM:\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124mS\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m----> 2\u001b[0m df_filtered \u001b[38;5;241m=\u001b[39m \u001b[43minput_dataset\u001b[49m\u001b[43m[\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_dataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdate\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdate\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m>\u001b[39;49m\u001b[43mdate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m2018\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m&\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_dataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdate\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdate\u001b[49m\u001b[38;5;241;43m<\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m2021\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m df_filtered\u001b[38;5;241m.\u001b[39misna()\u001b[38;5;241m.\u001b[39many()\u001b[38;5;241m.\u001b[39many():\n\u001b[0;32m      5\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThere are NA values in the DataFrame columns.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\frame.py:4081\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   4079\u001b[0m \u001b[38;5;66;03m# Do we have a (boolean) 1d indexer?\u001b[39;00m\n\u001b[0;32m   4080\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m com\u001b[38;5;241m.\u001b[39mis_bool_indexer(key):\n\u001b[1;32m-> 4081\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem_bool_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   4083\u001b[0m \u001b[38;5;66;03m# We are left with two options: a single key, and a collection of keys,\u001b[39;00m\n\u001b[0;32m   4084\u001b[0m \u001b[38;5;66;03m# We interpret tuples as collections only for non-MultiIndex\u001b[39;00m\n\u001b[0;32m   4085\u001b[0m is_single_key \u001b[38;5;241m=\u001b[39m \u001b[38;5;28misinstance\u001b[39m(key, \u001b[38;5;28mtuple\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_list_like(key)\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\frame.py:4143\u001b[0m, in \u001b[0;36mDataFrame._getitem_bool_array\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   4140\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcopy(deep\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m   4142\u001b[0m indexer \u001b[38;5;241m=\u001b[39m key\u001b[38;5;241m.\u001b[39mnonzero()[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m-> 4143\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_take_with_is_copy\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\generic.py:4150\u001b[0m, in \u001b[0;36mNDFrame._take_with_is_copy\u001b[1;34m(self, indices, axis)\u001b[0m\n\u001b[0;32m   4139\u001b[0m \u001b[38;5;129m@final\u001b[39m\n\u001b[0;32m   4140\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_take_with_is_copy\u001b[39m(\u001b[38;5;28mself\u001b[39m, indices, axis: Axis \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Self:\n\u001b[0;32m   4141\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m   4142\u001b[0m \u001b[38;5;124;03m    Internal version of the `take` method that sets the `_is_copy`\u001b[39;00m\n\u001b[0;32m   4143\u001b[0m \u001b[38;5;124;03m    attribute to keep track of the parent dataframe (using in indexing\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   4148\u001b[0m \u001b[38;5;124;03m    See the docstring of `take` for full explanation of the parameters.\u001b[39;00m\n\u001b[0;32m   4149\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m-> 4150\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtake\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindices\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindices\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maxis\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   4151\u001b[0m     \u001b[38;5;66;03m# Maybe set copy if we didn't actually change the index.\u001b[39;00m\n\u001b[0;32m   4152\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m result\u001b[38;5;241m.\u001b[39m_get_axis(axis)\u001b[38;5;241m.\u001b[39mequals(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_axis(axis)):\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\generic.py:4130\u001b[0m, in \u001b[0;36mNDFrame.take\u001b[1;34m(self, indices, axis, **kwargs)\u001b[0m\n\u001b[0;32m   4125\u001b[0m     \u001b[38;5;66;03m# We can get here with a slice via DataFrame.__getitem__\u001b[39;00m\n\u001b[0;32m   4126\u001b[0m     indices \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39marange(\n\u001b[0;32m   4127\u001b[0m         indices\u001b[38;5;241m.\u001b[39mstart, indices\u001b[38;5;241m.\u001b[39mstop, indices\u001b[38;5;241m.\u001b[39mstep, dtype\u001b[38;5;241m=\u001b[39mnp\u001b[38;5;241m.\u001b[39mintp\n\u001b[0;32m   4128\u001b[0m     )\n\u001b[1;32m-> 4130\u001b[0m new_data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_mgr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtake\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   4131\u001b[0m \u001b[43m    \u001b[49m\u001b[43mindices\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   4132\u001b[0m \u001b[43m    \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_block_manager_axis\u001b[49m\u001b[43m(\u001b[49m\u001b[43maxis\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   4133\u001b[0m \u001b[43m    \u001b[49m\u001b[43mverify\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[0;32m   4134\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   4135\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_constructor_from_mgr(new_data, axes\u001b[38;5;241m=\u001b[39mnew_data\u001b[38;5;241m.\u001b[39maxes)\u001b[38;5;241m.\u001b[39m__finalize__(\n\u001b[0;32m   4136\u001b[0m     \u001b[38;5;28mself\u001b[39m, method\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtake\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   4137\u001b[0m )\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\internals\\managers.py:894\u001b[0m, in \u001b[0;36mBaseBlockManager.take\u001b[1;34m(self, indexer, axis, verify)\u001b[0m\n\u001b[0;32m    891\u001b[0m indexer \u001b[38;5;241m=\u001b[39m maybe_convert_indices(indexer, n, verify\u001b[38;5;241m=\u001b[39mverify)\n\u001b[0;32m    893\u001b[0m new_labels \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maxes[axis]\u001b[38;5;241m.\u001b[39mtake(indexer)\n\u001b[1;32m--> 894\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreindex_indexer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    895\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnew_axis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnew_labels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    896\u001b[0m \u001b[43m    \u001b[49m\u001b[43mindexer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    897\u001b[0m \u001b[43m    \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maxis\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    898\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_dups\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[0;32m    899\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcopy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[0;32m    900\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\internals\\managers.py:687\u001b[0m, in \u001b[0;36mBaseBlockManager.reindex_indexer\u001b[1;34m(self, new_axis, indexer, axis, fill_value, allow_dups, copy, only_slice, use_na_proxy)\u001b[0m\n\u001b[0;32m    680\u001b[0m     new_blocks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_slice_take_blocks_ax0(\n\u001b[0;32m    681\u001b[0m         indexer,\n\u001b[0;32m    682\u001b[0m         fill_value\u001b[38;5;241m=\u001b[39mfill_value,\n\u001b[0;32m    683\u001b[0m         only_slice\u001b[38;5;241m=\u001b[39monly_slice,\n\u001b[0;32m    684\u001b[0m         use_na_proxy\u001b[38;5;241m=\u001b[39muse_na_proxy,\n\u001b[0;32m    685\u001b[0m     )\n\u001b[0;32m    686\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 687\u001b[0m     new_blocks \u001b[38;5;241m=\u001b[39m \u001b[43m[\u001b[49m\n\u001b[0;32m    688\u001b[0m \u001b[43m        \u001b[49m\u001b[43mblk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtake_nd\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    689\u001b[0m \u001b[43m            \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    690\u001b[0m \u001b[43m            \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m    691\u001b[0m \u001b[43m            \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[0;32m    692\u001b[0m \u001b[43m                \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mblk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfill_value\u001b[49m\n\u001b[0;32m    693\u001b[0m \u001b[43m            \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    694\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    695\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mblk\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mblocks\u001b[49m\n\u001b[0;32m    696\u001b[0m \u001b[43m    \u001b[49m\u001b[43m]\u001b[49m\n\u001b[0;32m    698\u001b[0m new_axes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maxes)\n\u001b[0;32m    699\u001b[0m new_axes[axis] \u001b[38;5;241m=\u001b[39m new_axis\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\internals\\managers.py:688\u001b[0m, in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    680\u001b[0m     new_blocks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_slice_take_blocks_ax0(\n\u001b[0;32m    681\u001b[0m         indexer,\n\u001b[0;32m    682\u001b[0m         fill_value\u001b[38;5;241m=\u001b[39mfill_value,\n\u001b[0;32m    683\u001b[0m         only_slice\u001b[38;5;241m=\u001b[39monly_slice,\n\u001b[0;32m    684\u001b[0m         use_na_proxy\u001b[38;5;241m=\u001b[39muse_na_proxy,\n\u001b[0;32m    685\u001b[0m     )\n\u001b[0;32m    686\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    687\u001b[0m     new_blocks \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m--> 688\u001b[0m         \u001b[43mblk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtake_nd\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    689\u001b[0m \u001b[43m            \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    690\u001b[0m \u001b[43m            \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m    691\u001b[0m \u001b[43m            \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[0;32m    692\u001b[0m \u001b[43m                \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mblk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfill_value\u001b[49m\n\u001b[0;32m    693\u001b[0m \u001b[43m            \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    694\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    695\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m blk \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mblocks\n\u001b[0;32m    696\u001b[0m     ]\n\u001b[0;32m    698\u001b[0m new_axes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maxes)\n\u001b[0;32m    699\u001b[0m new_axes[axis] \u001b[38;5;241m=\u001b[39m new_axis\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\internals\\blocks.py:1307\u001b[0m, in \u001b[0;36mBlock.take_nd\u001b[1;34m(self, indexer, axis, new_mgr_locs, fill_value)\u001b[0m\n\u001b[0;32m   1304\u001b[0m     allow_fill \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m   1306\u001b[0m \u001b[38;5;66;03m# Note: algos.take_nd has upcast logic similar to coerce_to_target_dtype\u001b[39;00m\n\u001b[1;32m-> 1307\u001b[0m new_values \u001b[38;5;241m=\u001b[39m \u001b[43malgos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtake_nd\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1308\u001b[0m \u001b[43m    \u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maxis\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mallow_fill\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mallow_fill\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfill_value\u001b[49m\n\u001b[0;32m   1309\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1311\u001b[0m \u001b[38;5;66;03m# Called from three places in managers, all of which satisfy\u001b[39;00m\n\u001b[0;32m   1312\u001b[0m \u001b[38;5;66;03m#  these assertions\u001b[39;00m\n\u001b[0;32m   1313\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m, ExtensionBlock):\n\u001b[0;32m   1314\u001b[0m     \u001b[38;5;66;03m# NB: in this case, the 'axis' kwarg will be ignored in the\u001b[39;00m\n\u001b[0;32m   1315\u001b[0m     \u001b[38;5;66;03m#  algos.take_nd call above.\u001b[39;00m\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\array_algos\\take.py:117\u001b[0m, in \u001b[0;36mtake_nd\u001b[1;34m(arr, indexer, axis, fill_value, allow_fill)\u001b[0m\n\u001b[0;32m    114\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m arr\u001b[38;5;241m.\u001b[39mtake(indexer, fill_value\u001b[38;5;241m=\u001b[39mfill_value, allow_fill\u001b[38;5;241m=\u001b[39mallow_fill)\n\u001b[0;32m    116\u001b[0m arr \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39masarray(arr)\n\u001b[1;32m--> 117\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_take_nd_ndarray\u001b[49m\u001b[43m(\u001b[49m\u001b[43marr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mallow_fill\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\pandas\\core\\array_algos\\take.py:157\u001b[0m, in \u001b[0;36m_take_nd_ndarray\u001b[1;34m(arr, indexer, axis, fill_value, allow_fill)\u001b[0m\n\u001b[0;32m    155\u001b[0m     out \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mempty(out_shape, dtype\u001b[38;5;241m=\u001b[39mdtype, order\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mF\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    156\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 157\u001b[0m     out \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mempty(out_shape, dtype\u001b[38;5;241m=\u001b[39mdtype)\n\u001b[0;32m    159\u001b[0m func \u001b[38;5;241m=\u001b[39m _get_take_nd_function(\n\u001b[0;32m    160\u001b[0m     arr\u001b[38;5;241m.\u001b[39mndim, arr\u001b[38;5;241m.\u001b[39mdtype, out\u001b[38;5;241m.\u001b[39mdtype, axis\u001b[38;5;241m=\u001b[39maxis, mask_info\u001b[38;5;241m=\u001b[39mmask_info\n\u001b[0;32m    161\u001b[0m )\n\u001b[0;32m    162\u001b[0m func(arr, indexer, out, fill_value)\n",
-      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 8.15 GiB for an array with shape (528, 2070713) and data type float64"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "There are NA values in the DataFrame columns.\n"
      ]
     }
    ],
@@ -130,7 +96,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -165,11 +131,11 @@
        "      <th>zone_066_fan_spd</th>\n",
        "      <th>zone_042_temp</th>\n",
        "      <th>...</th>\n",
-       "      <th>zone_066_heating_sp</th>\n",
-       "      <th>zone_067_heating_sp</th>\n",
-       "      <th>zone_069_heating_sp</th>\n",
-       "      <th>zone_070_heating_sp</th>\n",
-       "      <th>zone_071_heating_sp</th>\n",
+       "      <th>zone_038_heating_sp</th>\n",
+       "      <th>zone_037_cooling_sp</th>\n",
+       "      <th>zone_037_heating_sp</th>\n",
+       "      <th>zone_036_cooling_sp</th>\n",
+       "      <th>zone_036_heating_sp</th>\n",
        "      <th>air_temp_set_1</th>\n",
        "      <th>air_temp_set_2</th>\n",
        "      <th>dew_point_temperature_set_1d</th>\n",
@@ -179,124 +145,124 @@
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
-       "      <th>1440</th>\n",
-       "      <td>2018-01-02 00:00:00</td>\n",
-       "      <td>71.4</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>73.2</td>\n",
-       "      <td>70.0</td>\n",
-       "      <td>71.2</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>70.4</td>\n",
-       "      <td>35.0</td>\n",
-       "      <td>71.6</td>\n",
-       "      <td>...</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
+       "      <th>438785</th>\n",
+       "      <td>2019-01-08 20:55:00</td>\n",
+       "      <td>70.9</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>72.4</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>70.2</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>70.9</td>\n",
        "      <td>NaN</td>\n",
-       "      <td>15.280</td>\n",
-       "      <td>15.100</td>\n",
-       "      <td>6.33</td>\n",
-       "      <td>55.40</td>\n",
-       "      <td>161.9</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>...</td>\n",
+       "      <td>72.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>75.0</td>\n",
+       "      <td>72.0</td>\n",
+       "      <td>12.850</td>\n",
+       "      <td>12.930</td>\n",
+       "      <td>9.10</td>\n",
+       "      <td>78.15</td>\n",
+       "      <td>48.7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>1441</th>\n",
-       "      <td>2018-01-02 00:01:00</td>\n",
-       "      <td>71.4</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>73.2</td>\n",
-       "      <td>70.0</td>\n",
-       "      <td>71.2</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>70.4</td>\n",
-       "      <td>35.0</td>\n",
-       "      <td>71.6</td>\n",
-       "      <td>...</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
+       "      <th>438786</th>\n",
+       "      <td>2019-01-08 20:56:00</td>\n",
+       "      <td>70.9</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>72.4</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>70.2</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>70.9</td>\n",
        "      <td>NaN</td>\n",
-       "      <td>15.280</td>\n",
-       "      <td>15.100</td>\n",
-       "      <td>6.33</td>\n",
-       "      <td>55.40</td>\n",
-       "      <td>161.9</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>...</td>\n",
+       "      <td>72.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>75.0</td>\n",
+       "      <td>72.0</td>\n",
+       "      <td>12.850</td>\n",
+       "      <td>12.930</td>\n",
+       "      <td>9.10</td>\n",
+       "      <td>78.15</td>\n",
+       "      <td>48.7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>1442</th>\n",
-       "      <td>2018-01-02 00:02:00</td>\n",
-       "      <td>71.4</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>73.2</td>\n",
-       "      <td>70.0</td>\n",
-       "      <td>71.2</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>70.4</td>\n",
-       "      <td>35.0</td>\n",
-       "      <td>71.6</td>\n",
-       "      <td>...</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
+       "      <th>438787</th>\n",
+       "      <td>2019-01-08 20:57:00</td>\n",
+       "      <td>70.9</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>72.4</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>70.2</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>70.9</td>\n",
        "      <td>NaN</td>\n",
-       "      <td>15.280</td>\n",
-       "      <td>15.100</td>\n",
-       "      <td>6.33</td>\n",
-       "      <td>55.40</td>\n",
-       "      <td>161.9</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>...</td>\n",
+       "      <td>72.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>75.0</td>\n",
+       "      <td>72.0</td>\n",
+       "      <td>12.850</td>\n",
+       "      <td>12.930</td>\n",
+       "      <td>9.10</td>\n",
+       "      <td>78.15</td>\n",
+       "      <td>48.7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>1443</th>\n",
-       "      <td>2018-01-02 00:03:00</td>\n",
-       "      <td>71.4</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>73.2</td>\n",
-       "      <td>70.0</td>\n",
-       "      <td>71.2</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>70.4</td>\n",
-       "      <td>35.0</td>\n",
-       "      <td>71.6</td>\n",
-       "      <td>...</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
+       "      <th>438788</th>\n",
+       "      <td>2019-01-08 20:58:00</td>\n",
+       "      <td>70.9</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>72.4</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>70.2</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>70.9</td>\n",
        "      <td>NaN</td>\n",
-       "      <td>15.280</td>\n",
-       "      <td>15.100</td>\n",
-       "      <td>6.33</td>\n",
-       "      <td>55.40</td>\n",
-       "      <td>161.9</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>...</td>\n",
+       "      <td>72.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>75.0</td>\n",
+       "      <td>72.0</td>\n",
+       "      <td>12.850</td>\n",
+       "      <td>12.930</td>\n",
+       "      <td>9.10</td>\n",
+       "      <td>78.15</td>\n",
+       "      <td>48.7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>1444</th>\n",
-       "      <td>2018-01-02 00:04:00</td>\n",
-       "      <td>71.4</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>73.2</td>\n",
-       "      <td>70.0</td>\n",
-       "      <td>71.2</td>\n",
-       "      <td>20.0</td>\n",
-       "      <td>70.4</td>\n",
-       "      <td>35.0</td>\n",
-       "      <td>71.6</td>\n",
-       "      <td>...</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
+       "      <th>438789</th>\n",
+       "      <td>2019-01-08 20:59:00</td>\n",
+       "      <td>70.9</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>72.4</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>70.2</td>\n",
        "      <td>NaN</td>\n",
+       "      <td>70.9</td>\n",
        "      <td>NaN</td>\n",
-       "      <td>15.280</td>\n",
-       "      <td>15.100</td>\n",
-       "      <td>6.33</td>\n",
-       "      <td>55.40</td>\n",
-       "      <td>161.9</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>...</td>\n",
+       "      <td>72.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>75.0</td>\n",
+       "      <td>72.0</td>\n",
+       "      <td>12.850</td>\n",
+       "      <td>12.930</td>\n",
+       "      <td>9.10</td>\n",
+       "      <td>78.15</td>\n",
+       "      <td>48.7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>...</th>\n",
@@ -335,11 +301,11 @@
        "      <td>35.0</td>\n",
        "      <td>71.4</td>\n",
        "      <td>...</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>74.0</td>\n",
        "      <td>68.0</td>\n",
+       "      <td>74.0</td>\n",
        "      <td>68.0</td>\n",
-       "      <td>68.0</td>\n",
-       "      <td>65.0</td>\n",
-       "      <td>67.0</td>\n",
        "      <td>13.994</td>\n",
        "      <td>13.528</td>\n",
        "      <td>4.11</td>\n",
@@ -359,11 +325,11 @@
        "      <td>35.0</td>\n",
        "      <td>71.4</td>\n",
        "      <td>...</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>74.0</td>\n",
        "      <td>68.0</td>\n",
+       "      <td>74.0</td>\n",
        "      <td>68.0</td>\n",
-       "      <td>68.0</td>\n",
-       "      <td>65.0</td>\n",
-       "      <td>67.0</td>\n",
        "      <td>13.994</td>\n",
        "      <td>13.528</td>\n",
        "      <td>4.11</td>\n",
@@ -383,11 +349,11 @@
        "      <td>35.0</td>\n",
        "      <td>71.4</td>\n",
        "      <td>...</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>74.0</td>\n",
        "      <td>68.0</td>\n",
+       "      <td>74.0</td>\n",
        "      <td>68.0</td>\n",
-       "      <td>68.0</td>\n",
-       "      <td>65.0</td>\n",
-       "      <td>67.0</td>\n",
        "      <td>13.994</td>\n",
        "      <td>13.528</td>\n",
        "      <td>4.11</td>\n",
@@ -407,11 +373,11 @@
        "      <td>35.0</td>\n",
        "      <td>71.4</td>\n",
        "      <td>...</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>74.0</td>\n",
        "      <td>68.0</td>\n",
+       "      <td>74.0</td>\n",
        "      <td>68.0</td>\n",
-       "      <td>68.0</td>\n",
-       "      <td>65.0</td>\n",
-       "      <td>67.0</td>\n",
        "      <td>13.994</td>\n",
        "      <td>13.528</td>\n",
        "      <td>4.11</td>\n",
@@ -431,11 +397,11 @@
        "      <td>35.0</td>\n",
        "      <td>71.4</td>\n",
        "      <td>...</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>74.0</td>\n",
        "      <td>68.0</td>\n",
+       "      <td>74.0</td>\n",
        "      <td>68.0</td>\n",
-       "      <td>68.0</td>\n",
-       "      <td>65.0</td>\n",
-       "      <td>67.0</td>\n",
        "      <td>13.994</td>\n",
        "      <td>13.528</td>\n",
        "      <td>4.11</td>\n",
@@ -444,16 +410,16 @@
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>2070713 rows × 529 columns</p>\n",
+       "<p>1633368 rows × 46 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
        "                       date  zone_069_temp  zone_069_fan_spd  zone_068_temp  \\\n",
-       "1440    2018-01-02 00:00:00           71.4              20.0           73.2   \n",
-       "1441    2018-01-02 00:01:00           71.4              20.0           73.2   \n",
-       "1442    2018-01-02 00:02:00           71.4              20.0           73.2   \n",
-       "1443    2018-01-02 00:03:00           71.4              20.0           73.2   \n",
-       "1444    2018-01-02 00:04:00           71.4              20.0           73.2   \n",
+       "438785  2019-01-08 20:55:00           70.9               NaN           72.4   \n",
+       "438786  2019-01-08 20:56:00           70.9               NaN           72.4   \n",
+       "438787  2019-01-08 20:57:00           70.9               NaN           72.4   \n",
+       "438788  2019-01-08 20:58:00           70.9               NaN           72.4   \n",
+       "438789  2019-01-08 20:59:00           70.9               NaN           72.4   \n",
        "...                     ...            ...               ...            ...   \n",
        "2072148 2020-12-31 23:57:00           68.8              20.0           71.7   \n",
        "2072149 2020-12-31 23:58:00           68.8              20.0           71.7   \n",
@@ -462,11 +428,11 @@
        "2072152 2020-12-31 23:59:00           68.8              20.0           71.7   \n",
        "\n",
        "         zone_068_fan_spd  zone_067_temp  zone_067_fan_spd  zone_066_temp  \\\n",
-       "1440                 70.0           71.2              20.0           70.4   \n",
-       "1441                 70.0           71.2              20.0           70.4   \n",
-       "1442                 70.0           71.2              20.0           70.4   \n",
-       "1443                 70.0           71.2              20.0           70.4   \n",
-       "1444                 70.0           71.2              20.0           70.4   \n",
+       "438785               20.0           70.2               NaN           70.9   \n",
+       "438786               20.0           70.2               NaN           70.9   \n",
+       "438787               20.0           70.2               NaN           70.9   \n",
+       "438788               20.0           70.2               NaN           70.9   \n",
+       "438789               20.0           70.2               NaN           70.9   \n",
        "...                   ...            ...               ...            ...   \n",
        "2072148              20.0           70.4              20.0           68.6   \n",
        "2072149              20.0           70.4              20.0           68.6   \n",
@@ -474,51 +440,51 @@
        "2072151              20.0           70.4              20.0           68.6   \n",
        "2072152              20.0           70.4              20.0           68.6   \n",
        "\n",
-       "         zone_066_fan_spd  zone_042_temp  ...  zone_066_heating_sp  \\\n",
-       "1440                 35.0           71.6  ...                  NaN   \n",
-       "1441                 35.0           71.6  ...                  NaN   \n",
-       "1442                 35.0           71.6  ...                  NaN   \n",
-       "1443                 35.0           71.6  ...                  NaN   \n",
-       "1444                 35.0           71.6  ...                  NaN   \n",
+       "         zone_066_fan_spd  zone_042_temp  ...  zone_038_heating_sp  \\\n",
+       "438785                NaN           72.3  ...                 72.0   \n",
+       "438786                NaN           72.3  ...                 72.0   \n",
+       "438787                NaN           72.3  ...                 72.0   \n",
+       "438788                NaN           72.3  ...                 72.0   \n",
+       "438789                NaN           72.3  ...                 72.0   \n",
        "...                   ...            ...  ...                  ...   \n",
-       "2072148              35.0           71.4  ...                 68.0   \n",
-       "2072149              35.0           71.4  ...                 68.0   \n",
-       "2072150              35.0           71.4  ...                 68.0   \n",
-       "2072151              35.0           71.4  ...                 68.0   \n",
-       "2072152              35.0           71.4  ...                 68.0   \n",
+       "2072148              35.0           71.4  ...                 71.0   \n",
+       "2072149              35.0           71.4  ...                 71.0   \n",
+       "2072150              35.0           71.4  ...                 71.0   \n",
+       "2072151              35.0           71.4  ...                 71.0   \n",
+       "2072152              35.0           71.4  ...                 71.0   \n",
        "\n",
-       "         zone_067_heating_sp  zone_069_heating_sp  zone_070_heating_sp  \\\n",
-       "1440                     NaN                  NaN                  NaN   \n",
-       "1441                     NaN                  NaN                  NaN   \n",
-       "1442                     NaN                  NaN                  NaN   \n",
-       "1443                     NaN                  NaN                  NaN   \n",
-       "1444                     NaN                  NaN                  NaN   \n",
+       "         zone_037_cooling_sp  zone_037_heating_sp  zone_036_cooling_sp  \\\n",
+       "438785                  73.0                 70.0                 75.0   \n",
+       "438786                  73.0                 70.0                 75.0   \n",
+       "438787                  73.0                 70.0                 75.0   \n",
+       "438788                  73.0                 70.0                 75.0   \n",
+       "438789                  73.0                 70.0                 75.0   \n",
        "...                      ...                  ...                  ...   \n",
-       "2072148                 68.0                 68.0                 65.0   \n",
-       "2072149                 68.0                 68.0                 65.0   \n",
-       "2072150                 68.0                 68.0                 65.0   \n",
-       "2072151                 68.0                 68.0                 65.0   \n",
-       "2072152                 68.0                 68.0                 65.0   \n",
+       "2072148                 74.0                 68.0                 74.0   \n",
+       "2072149                 74.0                 68.0                 74.0   \n",
+       "2072150                 74.0                 68.0                 74.0   \n",
+       "2072151                 74.0                 68.0                 74.0   \n",
+       "2072152                 74.0                 68.0                 74.0   \n",
        "\n",
-       "         zone_071_heating_sp  air_temp_set_1  air_temp_set_2  \\\n",
-       "1440                     NaN          15.280          15.100   \n",
-       "1441                     NaN          15.280          15.100   \n",
-       "1442                     NaN          15.280          15.100   \n",
-       "1443                     NaN          15.280          15.100   \n",
-       "1444                     NaN          15.280          15.100   \n",
+       "         zone_036_heating_sp  air_temp_set_1  air_temp_set_2  \\\n",
+       "438785                  72.0          12.850          12.930   \n",
+       "438786                  72.0          12.850          12.930   \n",
+       "438787                  72.0          12.850          12.930   \n",
+       "438788                  72.0          12.850          12.930   \n",
+       "438789                  72.0          12.850          12.930   \n",
        "...                      ...             ...             ...   \n",
-       "2072148                 67.0          13.994          13.528   \n",
-       "2072149                 67.0          13.994          13.528   \n",
-       "2072150                 67.0          13.994          13.528   \n",
-       "2072151                 67.0          13.994          13.528   \n",
-       "2072152                 67.0          13.994          13.528   \n",
+       "2072148                 68.0          13.994          13.528   \n",
+       "2072149                 68.0          13.994          13.528   \n",
+       "2072150                 68.0          13.994          13.528   \n",
+       "2072151                 68.0          13.994          13.528   \n",
+       "2072152                 68.0          13.994          13.528   \n",
        "\n",
        "         dew_point_temperature_set_1d  relative_humidity_set_1  \\\n",
-       "1440                             6.33                    55.40   \n",
-       "1441                             6.33                    55.40   \n",
-       "1442                             6.33                    55.40   \n",
-       "1443                             6.33                    55.40   \n",
-       "1444                             6.33                    55.40   \n",
+       "438785                           9.10                    78.15   \n",
+       "438786                           9.10                    78.15   \n",
+       "438787                           9.10                    78.15   \n",
+       "438788                           9.10                    78.15   \n",
+       "438789                           9.10                    78.15   \n",
        "...                               ...                      ...   \n",
        "2072148                          4.11                    51.61   \n",
        "2072149                          4.11                    51.61   \n",
@@ -527,11 +493,11 @@
        "2072152                          4.11                    51.61   \n",
        "\n",
        "         solar_radiation_set_1  \n",
-       "1440                     161.9  \n",
-       "1441                     161.9  \n",
-       "1442                     161.9  \n",
-       "1443                     161.9  \n",
-       "1444                     161.9  \n",
+       "438785                    48.7  \n",
+       "438786                    48.7  \n",
+       "438787                    48.7  \n",
+       "438788                    48.7  \n",
+       "438789                    48.7  \n",
        "...                        ...  \n",
        "2072148                  188.8  \n",
        "2072149                  188.8  \n",
@@ -539,10 +505,10 @@
        "2072151                  188.8  \n",
        "2072152                  188.8  \n",
        "\n",
-       "[2070713 rows x 529 columns]"
+       "[1633368 rows x 46 columns]"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -553,39 +519,28 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "['zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp',\n",
-       " 'zone_070_heating_sp']"
+       "[]"
       ]
      },
-     "execution_count": 8,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "testdataset_df = df_filtered[(df_filtered.date.dt.date >date(2019, 5, 1)) & (df_filtered.date.dt.date <date(2019,7, 1))]\n",
+    "testdataset_df = df_filtered[(df_filtered.date.dt.date >date(2020, 3, 1)) & (df_filtered.date.dt.date <date(2020,7, 1))]\n",
     "\n",
     "# traindataset_df = df_filtered[ (df_filtered.date.dt.date >date(2019, 11, 8))]\n",
     "\n",
-    "traindataset_df = df_filtered[(df_filtered.date.dt.date >date(2019, 3, 1)) & (df_filtered.date.dt.date <date(2019, 5, 1))]\n",
-    "testdataset = testdataset_df.drop(columns=[\"date\"]).rolling(window = 5, step = 1, min_periods= 1).mean().values\n",
-    "traindataset = traindataset_df.drop(columns=[\"date\"]).rolling(window = 5, step = 1, min_periods= 1).mean().values\n",
+    "traindataset_df = df_filtered[(df_filtered.date.dt.date >date(2019, 3, 1)) & (df_filtered.date.dt.date <date(2020, 3, 1)) | (df_filtered.date.dt.date >date(2020, 7, 1)) & (df_filtered.date.dt.date <date(2020, 12, 1))]\n",
+    "testdataset = testdataset_df.drop(columns=[\"date\"]).values\n",
+    "traindataset = traindataset_df.drop(columns=[\"date\"]).values\n",
     "\n",
     "columns_with_na = traindataset_df.columns[traindataset_df.isna().any()].tolist()\n",
     "columns_with_na"
@@ -593,35 +548,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Index(['date', 'zone_069_temp', 'zone_069_fan_spd', 'zone_068_temp',\n",
-       "       'zone_068_fan_spd', 'zone_067_temp', 'zone_067_fan_spd',\n",
-       "       'zone_066_temp', 'zone_066_fan_spd', 'zone_042_temp',\n",
-       "       ...\n",
-       "       'zone_066_heating_sp', 'zone_067_heating_sp', 'zone_069_heating_sp',\n",
-       "       'zone_070_heating_sp', 'zone_071_heating_sp', 'air_temp_set_1',\n",
-       "       'air_temp_set_2', 'dew_point_temperature_set_1d',\n",
-       "       'relative_humidity_set_1', 'solar_radiation_set_1'],\n",
-       "      dtype='object', length=529)"
-      ]
-     },
-     "execution_count": 9,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "traindataset_df.columns"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 123,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -638,16 +565,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 124,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "(86400, 86400)"
+       "(1073512, 391818)"
       ]
      },
-     "execution_count": 124,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -658,7 +585,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 125,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -672,16 +599,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 126,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "(86400, 45)"
+       "(1073512, 45)"
       ]
      },
-     "execution_count": 126,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -692,7 +619,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 127,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -703,7 +630,7 @@
     "    Y = []\n",
     "    for i in range(len(dataset) - time_step - 1):\n",
     "        x.append(dataset[i:(i+time_step),:])\n",
-    "        Y.append(dataset[i+time_step,0:-5])\n",
+    "        Y.append(dataset[i+time_step,0:23])\n",
     "    x= np.array(x)\n",
     "    Y = np.array(Y)\n",
     "    return x,Y\n",
@@ -715,16 +642,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 128,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "((86369, 30, 45), (86369, 40))"
+       "((1073481, 30, 45), (1073481, 23))"
       ]
      },
-     "execution_count": 128,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -735,7 +662,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 133,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -743,8 +670,8 @@
      "output_type": "stream",
      "text": [
       "Epoch 1/5\n",
-      "674/675 [============================>.] - ETA: 0s - loss: 0.1090\n",
-      "Epoch 1: val_loss improved from inf to 0.26433, saving model to lstm_vav_01.tf\n",
+      "8387/8387 [==============================] - ETA: 0s - loss: 0.0178\n",
+      "Epoch 1: val_loss improved from inf to 0.42313, saving model to lstm_vav_01.tf\n",
       "INFO:tensorflow:Assets written to: lstm_vav_01.tf\\assets\n"
      ]
     },
@@ -759,10 +686,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "675/675 [==============================] - 61s 84ms/step - loss: 0.1089 - val_loss: 0.2643\n",
+      "8387/8387 [==============================] - 307s 36ms/step - loss: 0.0178 - val_loss: 0.4231\n",
       "Epoch 2/5\n",
-      "675/675 [==============================] - ETA: 0s - loss: 0.0155\n",
-      "Epoch 2: val_loss improved from 0.26433 to 0.21391, saving model to lstm_vav_01.tf\n",
+      "8387/8387 [==============================] - ETA: 0s - loss: 0.0032\n",
+      "Epoch 2: val_loss improved from 0.42313 to 0.40364, saving model to lstm_vav_01.tf\n",
       "INFO:tensorflow:Assets written to: lstm_vav_01.tf\\assets\n"
      ]
     },
@@ -777,72 +704,31 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "675/675 [==============================] - 45s 67ms/step - loss: 0.0155 - val_loss: 0.2139\n",
+      "8387/8387 [==============================] - 274s 33ms/step - loss: 0.0032 - val_loss: 0.4036\n",
       "Epoch 3/5\n",
-      "675/675 [==============================] - ETA: 0s - loss: 0.0081\n",
-      "Epoch 3: val_loss improved from 0.21391 to 0.17155, saving model to lstm_vav_01.tf\n",
-      "INFO:tensorflow:Assets written to: lstm_vav_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:tensorflow:Assets written to: lstm_vav_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "675/675 [==============================] - 58s 86ms/step - loss: 0.0081 - val_loss: 0.1716\n",
-      "Epoch 4/5\n",
-      "675/675 [==============================] - ETA: 0s - loss: 0.0049\n",
-      "Epoch 4: val_loss improved from 0.17155 to 0.14438, saving model to lstm_vav_01.tf\n",
-      "INFO:tensorflow:Assets written to: lstm_vav_01.tf\\assets\n"
+      " 259/8387 [..............................] - ETA: 4:02 - loss: 0.0028"
      ]
     },
     {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:tensorflow:Assets written to: lstm_vav_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "675/675 [==============================] - 54s 80ms/step - loss: 0.0049 - val_loss: 0.1444\n",
-      "Epoch 5/5\n",
-      "675/675 [==============================] - ETA: 0s - loss: 0.0030\n",
-      "Epoch 5: val_loss improved from 0.14438 to 0.12414, saving model to lstm_vav_01.tf\n",
-      "INFO:tensorflow:Assets written to: lstm_vav_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:tensorflow:Assets written to: lstm_vav_01.tf\\assets\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "675/675 [==============================] - 60s 89ms/step - loss: 0.0030 - val_loss: 0.1241\n"
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[13], line 11\u001b[0m\n\u001b[0;32m      9\u001b[0m checkpoint_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlstm_vav_01.tf\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     10\u001b[0m checkpoint_callback \u001b[38;5;241m=\u001b[39m ModelCheckpoint(filepath\u001b[38;5;241m=\u001b[39mcheckpoint_path, monitor\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mval_loss\u001b[39m\u001b[38;5;124m'\u001b[39m, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, save_best_only\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmin\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m---> 11\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalidation_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mX_test\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_test\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m128\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mcheckpoint_callback\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\keras\\src\\utils\\traceback_utils.py:65\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     63\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 65\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     66\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\keras\\src\\engine\\training.py:1742\u001b[0m, in \u001b[0;36mModel.fit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   1734\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tf\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mexperimental\u001b[38;5;241m.\u001b[39mTrace(\n\u001b[0;32m   1735\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   1736\u001b[0m     epoch_num\u001b[38;5;241m=\u001b[39mepoch,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1739\u001b[0m     _r\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m,\n\u001b[0;32m   1740\u001b[0m ):\n\u001b[0;32m   1741\u001b[0m     callbacks\u001b[38;5;241m.\u001b[39mon_train_batch_begin(step)\n\u001b[1;32m-> 1742\u001b[0m     tmp_logs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43miterator\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1743\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39mshould_sync:\n\u001b[0;32m   1744\u001b[0m         context\u001b[38;5;241m.\u001b[39masync_wait()\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\util\\traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    148\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m    149\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 150\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    152\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\polymorphic_function.py:825\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    822\u001b[0m compiler \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxla\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnonXla\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    824\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m OptionalXlaContext(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile):\n\u001b[1;32m--> 825\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    827\u001b[0m new_tracing_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperimental_get_tracing_count()\n\u001b[0;32m    828\u001b[0m without_tracing \u001b[38;5;241m=\u001b[39m (tracing_count \u001b[38;5;241m==\u001b[39m new_tracing_count)\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\polymorphic_function.py:857\u001b[0m, in \u001b[0;36mFunction._call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    854\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[0;32m    855\u001b[0m   \u001b[38;5;66;03m# In this case we have created variables on the first call, so we run the\u001b[39;00m\n\u001b[0;32m    856\u001b[0m   \u001b[38;5;66;03m# defunned version which is guaranteed to never create variables.\u001b[39;00m\n\u001b[1;32m--> 857\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_no_variable_creation_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# pylint: disable=not-callable\u001b[39;00m\n\u001b[0;32m    858\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_variable_creation_fn \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    859\u001b[0m   \u001b[38;5;66;03m# Release the lock early so that multiple threads can perform the call\u001b[39;00m\n\u001b[0;32m    860\u001b[0m   \u001b[38;5;66;03m# in parallel.\u001b[39;00m\n\u001b[0;32m    861\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\tracing_compiler.py:148\u001b[0m, in \u001b[0;36mTracingCompiler.__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    145\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[0;32m    146\u001b[0m   (concrete_function,\n\u001b[0;32m    147\u001b[0m    filtered_flat_args) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_maybe_define_function(args, kwargs)\n\u001b[1;32m--> 148\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mconcrete_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_flat\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    149\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfiltered_flat_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcaptured_inputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcrete_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcaptured_inputs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\monomorphic_function.py:1349\u001b[0m, in \u001b[0;36mConcreteFunction._call_flat\u001b[1;34m(self, args, captured_inputs)\u001b[0m\n\u001b[0;32m   1345\u001b[0m possible_gradient_type \u001b[38;5;241m=\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPossibleTapeGradientTypes(args)\n\u001b[0;32m   1346\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (possible_gradient_type \u001b[38;5;241m==\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPOSSIBLE_GRADIENT_TYPES_NONE\n\u001b[0;32m   1347\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m executing_eagerly):\n\u001b[0;32m   1348\u001b[0m   \u001b[38;5;66;03m# No tape is watching; skip to running the function.\u001b[39;00m\n\u001b[1;32m-> 1349\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_call_outputs(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inference_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m   1350\u001b[0m forward_backward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_select_forward_and_backward_functions(\n\u001b[0;32m   1351\u001b[0m     args,\n\u001b[0;32m   1352\u001b[0m     possible_gradient_type,\n\u001b[0;32m   1353\u001b[0m     executing_eagerly)\n\u001b[0;32m   1354\u001b[0m forward_function, args_with_tangents \u001b[38;5;241m=\u001b[39m forward_backward\u001b[38;5;241m.\u001b[39mforward()\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\atomic_function.py:196\u001b[0m, in \u001b[0;36mAtomicFunction.__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m    194\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m record\u001b[38;5;241m.\u001b[39mstop_recording():\n\u001b[0;32m    195\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bound_context\u001b[38;5;241m.\u001b[39mexecuting_eagerly():\n\u001b[1;32m--> 196\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_bound_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_function\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    197\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    198\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    199\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfunction_type\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mflat_outputs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    200\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    201\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    202\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m make_call_op_in_graph(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mlist\u001b[39m(args))\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\context.py:1457\u001b[0m, in \u001b[0;36mContext.call_function\u001b[1;34m(self, name, tensor_inputs, num_outputs)\u001b[0m\n\u001b[0;32m   1455\u001b[0m cancellation_context \u001b[38;5;241m=\u001b[39m cancellation\u001b[38;5;241m.\u001b[39mcontext()\n\u001b[0;32m   1456\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m cancellation_context \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m-> 1457\u001b[0m   outputs \u001b[38;5;241m=\u001b[39m \u001b[43mexecute\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1458\u001b[0m \u001b[43m      \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mutf-8\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1459\u001b[0m \u001b[43m      \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1460\u001b[0m \u001b[43m      \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtensor_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1461\u001b[0m \u001b[43m      \u001b[49m\u001b[43mattrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1462\u001b[0m \u001b[43m      \u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1463\u001b[0m \u001b[43m  \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1464\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   1465\u001b[0m   outputs \u001b[38;5;241m=\u001b[39m execute\u001b[38;5;241m.\u001b[39mexecute_with_cancellation(\n\u001b[0;32m   1466\u001b[0m       name\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[0;32m   1467\u001b[0m       num_outputs\u001b[38;5;241m=\u001b[39mnum_outputs,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1471\u001b[0m       cancellation_manager\u001b[38;5;241m=\u001b[39mcancellation_context,\n\u001b[0;32m   1472\u001b[0m   )\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\execute.py:53\u001b[0m, in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     51\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m     52\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[1;32m---> 53\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m \u001b[43mpywrap_tfe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTFE_Py_Execute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_handle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mop_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     54\u001b[0m \u001b[43m                                      \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     55\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m     56\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
+      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
      ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "<keras.src.callbacks.History at 0x1d5bf064950>"
-      ]
-     },
-     "execution_count": 133,
-     "metadata": {},
-     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -862,16 +748,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 134,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "<tensorflow.python.checkpoint.checkpoint.CheckpointLoadStatus at 0x1d55c631f10>"
+       "<tensorflow.python.checkpoint.checkpoint.CheckpointLoadStatus at 0x2a4b2344610>"
       ]
      },
-     "execution_count": 134,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -882,14 +768,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 135,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2700/2700 [==============================] - 25s 9ms/step\n"
+      "12244/12244 [==============================] - 58s 5ms/step\n"
      ]
     }
    ],
@@ -899,32 +785,48 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 136,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "[<matplotlib.lines.Line2D at 0x1d5582d61d0>]"
+       "Index(['date', 'zone_069_temp', 'zone_069_fan_spd', 'zone_068_temp',\n",
+       "       'zone_068_fan_spd', 'zone_067_temp', 'zone_067_fan_spd',\n",
+       "       'zone_066_temp', 'zone_066_fan_spd', 'zone_042_temp',\n",
+       "       'zone_042_fan_spd', 'zone_041_temp', 'zone_041_fan_spd',\n",
+       "       'zone_040_temp', 'zone_040_fan_spd', 'zone_039_temp',\n",
+       "       'zone_039_fan_spd', 'zone_038_temp', 'zone_038_fan_spd',\n",
+       "       'zone_037_temp', 'zone_037_fan_spd', 'zone_036_temp',\n",
+       "       'zone_036_fan_spd', 'zone_069_cooling_sp', 'zone_069_heating_sp',\n",
+       "       'zone_067_cooling_sp', 'zone_067_heating_sp', 'zone_066_cooling_sp',\n",
+       "       'zone_066_heating_sp', 'zone_042_cooling_sp', 'zone_042_heating_sp',\n",
+       "       'zone_041_cooling_sp', 'zone_041_heating_sp', 'zone_039_cooling_sp',\n",
+       "       'zone_039_heating_sp', 'zone_038_cooling_sp', 'zone_038_heating_sp',\n",
+       "       'zone_037_cooling_sp', 'zone_037_heating_sp', 'zone_036_cooling_sp',\n",
+       "       'zone_036_heating_sp', 'air_temp_set_1', 'air_temp_set_2',\n",
+       "       'dew_point_temperature_set_1d', 'relative_humidity_set_1',\n",
+       "       'solar_radiation_set_1'],\n",
+       "      dtype='object')"
       ]
      },
-     "execution_count": 136,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "plt.plot(y_test[:,3])\n",
-    "plt.plot(y_train[:,3])"
+    "traindataset_df.columns"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 141,
+   "execution_count": 37,
    "metadata": {},
    "outputs": [],
    "source": [
     "%matplotlib qt\n",
+    "plt.figure()\n",
     "var = 1\n",
     "plt.plot(y_test[:,var], label='Original Testing Data', color='blue')\n",
     "plt.plot(test_predict1[:,var], label='Predicted Testing Data', color='red',alpha=0.8)\n",
@@ -950,6 +852,7 @@
     "import matplotlib.pyplot as plt\n",
     "from sklearn.decomposition import PCA\n",
     "\n",
+    "\n",
     "# Generating random data for demonstration\n",
     "np.random.seed(0)\n",
     "X =  test_predict1 - y_test\n",
@@ -978,25 +881,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "ValueError",
-     "evalue": "operands could not be broadcast together with shapes (199403,51) (8,) ",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
-      "Cell \u001b[1;32mIn[19], line 6\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;66;03m# Generating random data for demonstration\u001b[39;00m\n\u001b[0;32m      5\u001b[0m np\u001b[38;5;241m.\u001b[39mrandom\u001b[38;5;241m.\u001b[39mseed(\u001b[38;5;241m0\u001b[39m)\n\u001b[1;32m----> 6\u001b[0m X \u001b[38;5;241m=\u001b[39m \u001b[43m(\u001b[49m\u001b[43mtest_predict1\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[43my_test\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mscaler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvar_\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m:\u001b[49m\u001b[38;5;241;43m8\u001b[39;49m\u001b[43m]\u001b[49m \u001b[38;5;241m+\u001b[39m scaler\u001b[38;5;241m.\u001b[39mmean_[\u001b[38;5;241m0\u001b[39m:\u001b[38;5;241m8\u001b[39m]\n\u001b[0;32m      8\u001b[0m k \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m6\u001b[39m\n\u001b[0;32m     10\u001b[0m kmeans \u001b[38;5;241m=\u001b[39m KMeans(n_clusters\u001b[38;5;241m=\u001b[39mk)\n",
-      "\u001b[1;31mValueError\u001b[0m: operands could not be broadcast together with shapes (199403,51) (8,) "
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from sklearn.cluster import KMeans\n",
     "import numpy as np\n",
     "import matplotlib.pyplot as plt\n",
+    "from sklearn.decomposition import PCA\n",
+    "\n",
     "# Generating random data for demonstration\n",
     "np.random.seed(0)\n",
     "X = (test_predict1 - y_test)\n",
@@ -1029,7 +922,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1080,7 +973,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1094,7 +987,7 @@
     "X = processed_data\n",
     "\n",
     "# Creating the GMM instance with desired number of clusters\n",
-    "gmm = GaussianMixture(n_components=3, init_params='k-means++')\n",
+    "gmm = GaussianMixture(n_components=2, init_params='k-means++')\n",
     "\n",
     "# Fitting the model to the data\n",
     "gmm.fit(X)\n",
@@ -1118,7 +1011,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 38,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1127,10 +1020,9 @@
     "import matplotlib.pyplot as plt\n",
     "# Generating random data for demonstration\n",
     "np.random.seed(0)\n",
-    "X = (test_predict1 * scaler.var_[0:8] + scaler.mean_[0:8]) - (y_test * scaler.var_[0:8] + scaler.mean_[0:8])\n",
-    "k = 6\n",
+    "X = test_predict1 - y_test \n",
     "\n",
-    "kmeans = KMeans(n_clusters=k)\n",
+    "kmeans = KMeans(n_clusters=2)\n",
     "\n",
     "kmeans.fit(X)\n",
     "\n",
@@ -1146,8 +1038,11 @@
     "labels = kmeans.labels_\n",
     "\n",
     "# Plotting the data points and cluster centers\n",
+    "plt.figure()\n",
     "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
     "plt.scatter(centroids[:, 0], centroids[:, 1], marker='x', c='red', s=200, linewidths=2)\n",
+    "plt.text(centroids[0,0], centroids[0,1], 'Normal', fontsize=12, color='red')\n",
+    "plt.text(centroids[1,0], centroids[1,1], 'Anomaly', fontsize=12, color='red')\n",
     "plt.title('KMeans Clustering')\n",
     "plt.xlabel('Feature 1')\n",
     "plt.ylabel('Feature 2')\n",
@@ -1156,10 +1051,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 29,
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "329810"
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sum(labels==0)"
+   ]
   }
  ],
  "metadata": {

physLSTM/lstm_vav_rtu2.ipynb

@@ -0,0 +1,1075 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd \n",
+    "from datetime import datetime \n",
+    "from datetime import date\n",
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from keras.models import Sequential\n",
+    "from keras.layers import LSTM, Dense\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.preprocessing import MinMaxScaler,StandardScaler\n",
+    "from keras.callbacks import ModelCheckpoint\n",
+    "import tensorflow as tf"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "merged = pd.read_csv(r'../data/long_merge.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "zones = [72, 71, 63, 62, 60, 59, 58,57, 50, 49, 44, 43, 35, 34, 33, 32, 31, 30, 29, 28, ]\n",
+    "rtus = [2]\n",
+    "cols = []\n",
+    "\n",
+    "for zone in zones:\n",
+    "        for column in merged.columns:\n",
+    "            if f\"zone_0{zone}\" in column and 'co2' not in column and \"hw_valve\" not in column and \"cooling_sp\" not in column and \"heating_sp\" not in column:\n",
+    "                cols.append(column)\n",
+    "\n",
+    "for zone in zones:\n",
+    "        for column in merged.columns:\n",
+    "              if f\"zone_0{zone}\" in column: \n",
+    "                if \"cooling_sp\" in column or \"heating_sp\" in column:\n",
+    "                  cols.append(column)\n",
+    "# for rtu in rtus:\n",
+    "#     for column in merged.columns:\n",
+    "#         if f\"rtu_00{rtu}_fltrd_sa\" in column:\n",
+    "#                 cols.append(column)\n",
+    "cols =['date'] + cols + ['air_temp_set_1',\n",
+    " 'air_temp_set_2',\n",
+    " 'dew_point_temperature_set_1d',\n",
+    " 'relative_humidity_set_1',\n",
+    " 'solar_radiation_set_1']\n",
+    "input_dataset = merged[cols]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\arbal\\AppData\\Local\\Temp\\ipykernel_38868\\1855433847.py:1: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  input_dataset['date'] = pd.to_datetime(input_dataset['date'], format = \"%Y-%m-%d %H:%M:%S\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "input_dataset['date'] = pd.to_datetime(input_dataset['date'], format = \"%Y-%m-%d %H:%M:%S\")\n",
+    "df_filtered = input_dataset[ (input_dataset.date.dt.date >date(2019, 3, 1)) & (input_dataset.date.dt.date< date(2021, 1, 1))]\n",
+    "\n",
+    "if df_filtered.isna().any().any():\n",
+    "    print(\"There are NA values in the DataFrame columns.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>zone_072_temp</th>\n",
+       "      <th>zone_072_fan_spd</th>\n",
+       "      <th>zone_071_temp</th>\n",
+       "      <th>zone_071_fan_spd</th>\n",
+       "      <th>zone_063_temp</th>\n",
+       "      <th>zone_063_fan_spd</th>\n",
+       "      <th>zone_062_temp</th>\n",
+       "      <th>zone_062_fan_spd</th>\n",
+       "      <th>zone_059_temp</th>\n",
+       "      <th>...</th>\n",
+       "      <th>zone_035_heating_sp</th>\n",
+       "      <th>zone_032_cooling_sp</th>\n",
+       "      <th>zone_032_heating_sp</th>\n",
+       "      <th>zone_030_cooling_sp</th>\n",
+       "      <th>zone_030_heating_sp</th>\n",
+       "      <th>air_temp_set_1</th>\n",
+       "      <th>air_temp_set_2</th>\n",
+       "      <th>dew_point_temperature_set_1d</th>\n",
+       "      <th>relative_humidity_set_1</th>\n",
+       "      <th>solar_radiation_set_1</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>511205</th>\n",
+       "      <td>2019-03-02 00:00:00</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.9</td>\n",
+       "      <td>55.0</td>\n",
+       "      <td>71.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>74.000000</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>67.0</td>\n",
+       "      <td>11.590</td>\n",
+       "      <td>11.130</td>\n",
+       "      <td>3.00</td>\n",
+       "      <td>55.87</td>\n",
+       "      <td>120.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>511206</th>\n",
+       "      <td>2019-03-02 00:01:00</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.9</td>\n",
+       "      <td>55.0</td>\n",
+       "      <td>71.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>74.000000</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>67.0</td>\n",
+       "      <td>11.590</td>\n",
+       "      <td>11.130</td>\n",
+       "      <td>3.00</td>\n",
+       "      <td>55.87</td>\n",
+       "      <td>120.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>511207</th>\n",
+       "      <td>2019-03-02 00:02:00</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.6</td>\n",
+       "      <td>55.0</td>\n",
+       "      <td>71.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>74.000000</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>67.0</td>\n",
+       "      <td>11.590</td>\n",
+       "      <td>11.130</td>\n",
+       "      <td>3.00</td>\n",
+       "      <td>55.87</td>\n",
+       "      <td>120.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>511208</th>\n",
+       "      <td>2019-03-02 00:03:00</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.9</td>\n",
+       "      <td>55.0</td>\n",
+       "      <td>71.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>74.000000</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>67.0</td>\n",
+       "      <td>11.590</td>\n",
+       "      <td>11.130</td>\n",
+       "      <td>3.00</td>\n",
+       "      <td>55.87</td>\n",
+       "      <td>120.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>511209</th>\n",
+       "      <td>2019-03-02 00:04:00</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.4</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.9</td>\n",
+       "      <td>55.0</td>\n",
+       "      <td>71.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>74.000000</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>67.0</td>\n",
+       "      <td>11.590</td>\n",
+       "      <td>11.130</td>\n",
+       "      <td>3.00</td>\n",
+       "      <td>55.87</td>\n",
+       "      <td>120.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072148</th>\n",
+       "      <td>2020-12-31 23:57:00</td>\n",
+       "      <td>69.5</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>67.6</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>...</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>72.714138</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>13.994</td>\n",
+       "      <td>13.528</td>\n",
+       "      <td>4.11</td>\n",
+       "      <td>51.61</td>\n",
+       "      <td>188.8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072149</th>\n",
+       "      <td>2020-12-31 23:58:00</td>\n",
+       "      <td>69.5</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>67.6</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>...</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>72.714138</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>13.994</td>\n",
+       "      <td>13.528</td>\n",
+       "      <td>4.11</td>\n",
+       "      <td>51.61</td>\n",
+       "      <td>188.8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072150</th>\n",
+       "      <td>2020-12-31 23:58:00</td>\n",
+       "      <td>69.5</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>67.6</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>...</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>72.714138</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>13.994</td>\n",
+       "      <td>13.528</td>\n",
+       "      <td>4.11</td>\n",
+       "      <td>51.61</td>\n",
+       "      <td>188.8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072151</th>\n",
+       "      <td>2020-12-31 23:59:00</td>\n",
+       "      <td>69.5</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>67.6</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>...</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>72.714138</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>13.994</td>\n",
+       "      <td>13.528</td>\n",
+       "      <td>4.11</td>\n",
+       "      <td>51.61</td>\n",
+       "      <td>188.8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072152</th>\n",
+       "      <td>2020-12-31 23:59:00</td>\n",
+       "      <td>69.5</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>67.6</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>...</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>72.714138</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>13.994</td>\n",
+       "      <td>13.528</td>\n",
+       "      <td>4.11</td>\n",
+       "      <td>51.61</td>\n",
+       "      <td>188.8</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1560948 rows × 56 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       date  zone_072_temp  zone_072_fan_spd  zone_071_temp  \\\n",
+       "511205  2019-03-02 00:00:00           71.2              40.0           71.5   \n",
+       "511206  2019-03-02 00:01:00           71.2              40.0           71.5   \n",
+       "511207  2019-03-02 00:02:00           71.2              40.0           71.5   \n",
+       "511208  2019-03-02 00:03:00           71.2              40.0           71.5   \n",
+       "511209  2019-03-02 00:04:00           71.2              40.0           71.4   \n",
+       "...                     ...            ...               ...            ...   \n",
+       "2072148 2020-12-31 23:57:00           69.5              40.0           71.2   \n",
+       "2072149 2020-12-31 23:58:00           69.5              40.0           71.3   \n",
+       "2072150 2020-12-31 23:58:00           69.5              40.0           71.3   \n",
+       "2072151 2020-12-31 23:59:00           69.5              40.0           71.5   \n",
+       "2072152 2020-12-31 23:59:00           69.5              40.0           71.5   \n",
+       "\n",
+       "         zone_071_fan_spd  zone_063_temp  zone_063_fan_spd  zone_062_temp  \\\n",
+       "511205               20.0           72.3              20.0           72.9   \n",
+       "511206               20.0           72.3              20.0           72.9   \n",
+       "511207               20.0           72.3              20.0           72.6   \n",
+       "511208               20.0           72.3              20.0           72.9   \n",
+       "511209               20.0           72.3              20.0           72.9   \n",
+       "...                   ...            ...               ...            ...   \n",
+       "2072148              20.0           68.0              20.0           67.6   \n",
+       "2072149              20.0           68.0              20.0           67.6   \n",
+       "2072150              20.0           68.0              20.0           67.6   \n",
+       "2072151              20.0           68.0              20.0           67.6   \n",
+       "2072152              20.0           68.0              20.0           67.6   \n",
+       "\n",
+       "         zone_062_fan_spd  zone_059_temp  ...  zone_035_heating_sp  \\\n",
+       "511205               55.0           71.9  ...                 70.0   \n",
+       "511206               55.0           71.9  ...                 70.0   \n",
+       "511207               55.0           71.9  ...                 70.0   \n",
+       "511208               55.0           71.9  ...                 70.0   \n",
+       "511209               55.0           71.9  ...                 70.0   \n",
+       "...                   ...            ...  ...                  ...   \n",
+       "2072148              40.0           67.5  ...                 68.0   \n",
+       "2072149              40.0           67.5  ...                 68.0   \n",
+       "2072150              40.0           67.5  ...                 68.0   \n",
+       "2072151              40.0           67.5  ...                 68.0   \n",
+       "2072152              40.0           67.5  ...                 68.0   \n",
+       "\n",
+       "         zone_032_cooling_sp  zone_032_heating_sp  zone_030_cooling_sp  \\\n",
+       "511205             74.000000                 68.0                 73.0   \n",
+       "511206             74.000000                 68.0                 73.0   \n",
+       "511207             74.000000                 68.0                 73.0   \n",
+       "511208             74.000000                 68.0                 73.0   \n",
+       "511209             74.000000                 68.0                 73.0   \n",
+       "...                      ...                  ...                  ...   \n",
+       "2072148            72.714138                 71.0                 71.0   \n",
+       "2072149            72.714138                 71.0                 71.0   \n",
+       "2072150            72.714138                 71.0                 71.0   \n",
+       "2072151            72.714138                 71.0                 71.0   \n",
+       "2072152            72.714138                 71.0                 71.0   \n",
+       "\n",
+       "         zone_030_heating_sp  air_temp_set_1  air_temp_set_2  \\\n",
+       "511205                  67.0          11.590          11.130   \n",
+       "511206                  67.0          11.590          11.130   \n",
+       "511207                  67.0          11.590          11.130   \n",
+       "511208                  67.0          11.590          11.130   \n",
+       "511209                  67.0          11.590          11.130   \n",
+       "...                      ...             ...             ...   \n",
+       "2072148                 70.0          13.994          13.528   \n",
+       "2072149                 70.0          13.994          13.528   \n",
+       "2072150                 70.0          13.994          13.528   \n",
+       "2072151                 70.0          13.994          13.528   \n",
+       "2072152                 70.0          13.994          13.528   \n",
+       "\n",
+       "         dew_point_temperature_set_1d  relative_humidity_set_1  \\\n",
+       "511205                           3.00                    55.87   \n",
+       "511206                           3.00                    55.87   \n",
+       "511207                           3.00                    55.87   \n",
+       "511208                           3.00                    55.87   \n",
+       "511209                           3.00                    55.87   \n",
+       "...                               ...                      ...   \n",
+       "2072148                          4.11                    51.61   \n",
+       "2072149                          4.11                    51.61   \n",
+       "2072150                          4.11                    51.61   \n",
+       "2072151                          4.11                    51.61   \n",
+       "2072152                          4.11                    51.61   \n",
+       "\n",
+       "         solar_radiation_set_1  \n",
+       "511205                   120.3  \n",
+       "511206                   120.3  \n",
+       "511207                   120.3  \n",
+       "511208                   120.3  \n",
+       "511209                   120.3  \n",
+       "...                        ...  \n",
+       "2072148                  188.8  \n",
+       "2072149                  188.8  \n",
+       "2072150                  188.8  \n",
+       "2072151                  188.8  \n",
+       "2072152                  188.8  \n",
+       "\n",
+       "[1560948 rows x 56 columns]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_filtered"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "testdataset_df = df_filtered[(df_filtered.date.dt.date >date(2020, 3, 1)) & (df_filtered.date.dt.date <date(2020,7, 1))]\n",
+    "\n",
+    "# traindataset_df = df_filtered[ (df_filtered.date.dt.date >date(2019, 11, 8))]\n",
+    "\n",
+    "traindataset_df = df_filtered[(df_filtered.date.dt.date >date(2019, 3, 1)) & (df_filtered.date.dt.date <date(2020, 3, 1)) | (df_filtered.date.dt.date >date(2020, 7, 1)) & (df_filtered.date.dt.date <date(2020, 12, 1))]\n",
+    "testdataset = testdataset_df.drop(columns=[\"date\"]).values\n",
+    "traindataset = traindataset_df.drop(columns=[\"date\"]).values\n",
+    "\n",
+    "columns_with_na = traindataset_df.columns[traindataset_df.isna().any()].tolist()\n",
+    "columns_with_na"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['date', 'zone_072_temp', 'zone_072_fan_spd', 'zone_071_temp',\n",
+       "       'zone_071_fan_spd', 'zone_063_temp', 'zone_063_fan_spd',\n",
+       "       'zone_062_temp', 'zone_062_fan_spd', 'zone_059_temp',\n",
+       "       'zone_059_fan_spd', 'zone_058_temp', 'zone_058_fan_spd',\n",
+       "       'zone_057_temp', 'zone_057_fan_spd', 'zone_049_temp',\n",
+       "       'zone_049_fan_spd', 'zone_044_temp', 'zone_044_fan_spd',\n",
+       "       'zone_043_temp', 'zone_043_fan_spd', 'zone_035_temp',\n",
+       "       'zone_035_fan_spd', 'zone_033_temp', 'zone_033_fan_spd',\n",
+       "       'zone_032_temp', 'zone_032_fan_spd', 'zone_030_temp',\n",
+       "       'zone_030_fan_spd', 'zone_028_temp', 'zone_028_fan_spd'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "traindataset_df.columns[0:31]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0 0\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(traindataset_df.isna().sum().sum(), testdataset_df.isna().sum().sum())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1073512, 391818)"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(traindataset), len(testdataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "traindataset = traindataset.astype('float32')\n",
+    "testdataset = testdataset.astype('float32')\n",
+    "\n",
+    "scaler = StandardScaler()\n",
+    "traindataset = scaler.fit_transform(traindataset)\n",
+    "testdataset = scaler.transform(testdataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1073512, 55)"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "traindataset.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train,test = traindataset,testdataset\n",
+    "\n",
+    "def create_dataset(dataset,time_step):\n",
+    "    x = []\n",
+    "    Y = []\n",
+    "    for i in range(len(dataset) - time_step - 1):\n",
+    "        x.append(dataset[i:(i+time_step),:])\n",
+    "        Y.append(dataset[i+time_step,0:31])\n",
+    "    x= np.array(x)\n",
+    "    Y = np.array(Y)\n",
+    "    return x,Y\n",
+    "time_step = 30\n",
+    "X_train, y_train = create_dataset(train, time_step)\n",
+    "X_test, y_test = create_dataset(test, time_step)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "((1073481, 30, 55), (1073481, 31))"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "X_train.shape, y_train.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "model = Sequential()\n",
+    "model.add(LSTM(units=50, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))\n",
+    "model.add(LSTM(units=50, return_sequences=True))\n",
+    "model.add(LSTM(units=30))\n",
+    "model.add(Dense(units=y_train.shape[1]))\n",
+    "\n",
+    "model.compile(optimizer='adam', loss='mean_squared_error')\n",
+    "\n",
+    "checkpoint_path = \"lstm_vav_02.tf\"\n",
+    "checkpoint_callback = ModelCheckpoint(filepath=checkpoint_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
+    "# model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=5, batch_size=128, verbose=1, callbacks=[checkpoint_callback])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<tensorflow.python.checkpoint.checkpoint.CheckpointLoadStatus at 0x2b142b76250>"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.load_weights(checkpoint_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "12244/12244 [==============================] - 61s 5ms/step\n"
+     ]
+    }
+   ],
+   "source": [
+    "test_predict1 = model.predict(X_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%matplotlib qt\n",
+    "plt.figure()\n",
+    "var = 2\n",
+    "plt.plot(y_test[:,var], label='Original Testing Data', color='blue')\n",
+    "plt.plot(test_predict1[:,var], label='Predicted Testing Data', color='red',alpha=0.8)\n",
+    "anomalies = np.where(abs(test_predict1[:,var] - y_test[:,var]) > 0.38)\n",
+    "plt.scatter(anomalies,test_predict1[anomalies,var], color='black',marker =\"o\",s=100 )\n",
+    "\n",
+    "\n",
+    "plt.title('Testing Data - Predicted vs Actual')\n",
+    "plt.xlabel('Time')\n",
+    "plt.ylabel('Value')\n",
+    "plt.legend()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " 1254/33547 [>.............................] - ETA: 3:00"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[18], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m get_ipython()\u001b[38;5;241m.\u001b[39mrun_line_magic(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmatplotlib\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mqt\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m----> 2\u001b[0m test_predict2 \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\keras\\src\\utils\\traceback_utils.py:65\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     63\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 65\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     66\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\keras\\src\\engine\\training.py:2554\u001b[0m, in \u001b[0;36mModel.predict\u001b[1;34m(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m   2552\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m step \u001b[38;5;129;01min\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39msteps():\n\u001b[0;32m   2553\u001b[0m     callbacks\u001b[38;5;241m.\u001b[39mon_predict_batch_begin(step)\n\u001b[1;32m-> 2554\u001b[0m     tmp_batch_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpredict_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43miterator\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   2555\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39mshould_sync:\n\u001b[0;32m   2556\u001b[0m         context\u001b[38;5;241m.\u001b[39masync_wait()\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\util\\traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    148\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m    149\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m--> 150\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m    152\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\polymorphic_function.py:825\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    822\u001b[0m compiler \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxla\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnonXla\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    824\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m OptionalXlaContext(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile):\n\u001b[1;32m--> 825\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    827\u001b[0m new_tracing_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperimental_get_tracing_count()\n\u001b[0;32m    828\u001b[0m without_tracing \u001b[38;5;241m=\u001b[39m (tracing_count \u001b[38;5;241m==\u001b[39m new_tracing_count)\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\polymorphic_function.py:864\u001b[0m, in \u001b[0;36mFunction._call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    861\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[0;32m    862\u001b[0m \u001b[38;5;66;03m# In this case we have not created variables on the first call. So we can\u001b[39;00m\n\u001b[0;32m    863\u001b[0m \u001b[38;5;66;03m# run the first trace but we should fail if variables are created.\u001b[39;00m\n\u001b[1;32m--> 864\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_variable_creation_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    865\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_created_variables \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m ALLOW_DYNAMIC_VARIABLE_CREATION:\n\u001b[0;32m    866\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCreating variables on a non-first call to a function\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    867\u001b[0m                    \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m decorated with tf.function.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\tracing_compiler.py:148\u001b[0m, in \u001b[0;36mTracingCompiler.__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    145\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[0;32m    146\u001b[0m   (concrete_function,\n\u001b[0;32m    147\u001b[0m    filtered_flat_args) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_maybe_define_function(args, kwargs)\n\u001b[1;32m--> 148\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mconcrete_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_flat\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    149\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfiltered_flat_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcaptured_inputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcrete_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcaptured_inputs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\monomorphic_function.py:1349\u001b[0m, in \u001b[0;36mConcreteFunction._call_flat\u001b[1;34m(self, args, captured_inputs)\u001b[0m\n\u001b[0;32m   1345\u001b[0m possible_gradient_type \u001b[38;5;241m=\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPossibleTapeGradientTypes(args)\n\u001b[0;32m   1346\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (possible_gradient_type \u001b[38;5;241m==\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPOSSIBLE_GRADIENT_TYPES_NONE\n\u001b[0;32m   1347\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m executing_eagerly):\n\u001b[0;32m   1348\u001b[0m   \u001b[38;5;66;03m# No tape is watching; skip to running the function.\u001b[39;00m\n\u001b[1;32m-> 1349\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_call_outputs(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inference_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m   1350\u001b[0m forward_backward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_select_forward_and_backward_functions(\n\u001b[0;32m   1351\u001b[0m     args,\n\u001b[0;32m   1352\u001b[0m     possible_gradient_type,\n\u001b[0;32m   1353\u001b[0m     executing_eagerly)\n\u001b[0;32m   1354\u001b[0m forward_function, args_with_tangents \u001b[38;5;241m=\u001b[39m forward_backward\u001b[38;5;241m.\u001b[39mforward()\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\polymorphic_function\\atomic_function.py:196\u001b[0m, in \u001b[0;36mAtomicFunction.__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m    194\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m record\u001b[38;5;241m.\u001b[39mstop_recording():\n\u001b[0;32m    195\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_bound_context\u001b[38;5;241m.\u001b[39mexecuting_eagerly():\n\u001b[1;32m--> 196\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_bound_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_function\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    197\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    198\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    199\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfunction_type\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mflat_outputs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    200\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    201\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    202\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m make_call_op_in_graph(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28mlist\u001b[39m(args))\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\context.py:1457\u001b[0m, in \u001b[0;36mContext.call_function\u001b[1;34m(self, name, tensor_inputs, num_outputs)\u001b[0m\n\u001b[0;32m   1455\u001b[0m cancellation_context \u001b[38;5;241m=\u001b[39m cancellation\u001b[38;5;241m.\u001b[39mcontext()\n\u001b[0;32m   1456\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m cancellation_context \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m-> 1457\u001b[0m   outputs \u001b[38;5;241m=\u001b[39m \u001b[43mexecute\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1458\u001b[0m \u001b[43m      \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mutf-8\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1459\u001b[0m \u001b[43m      \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1460\u001b[0m \u001b[43m      \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtensor_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1461\u001b[0m \u001b[43m      \u001b[49m\u001b[43mattrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1462\u001b[0m \u001b[43m      \u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1463\u001b[0m \u001b[43m  \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1464\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   1465\u001b[0m   outputs \u001b[38;5;241m=\u001b[39m execute\u001b[38;5;241m.\u001b[39mexecute_with_cancellation(\n\u001b[0;32m   1466\u001b[0m       name\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m),\n\u001b[0;32m   1467\u001b[0m       num_outputs\u001b[38;5;241m=\u001b[39mnum_outputs,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1471\u001b[0m       cancellation_manager\u001b[38;5;241m=\u001b[39mcancellation_context,\n\u001b[0;32m   1472\u001b[0m   )\n",
+      "File \u001b[1;32md:\\Programs\\minconda3\\envs\\smartbuildings\\Lib\\site-packages\\tensorflow\\python\\eager\\execute.py:53\u001b[0m, in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     51\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m     52\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[1;32m---> 53\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m \u001b[43mpywrap_tfe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTFE_Py_Execute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_handle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mop_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     54\u001b[0m \u001b[43m                                      \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     55\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m     56\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
+      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "%matplotlib qt\n",
+    "test_predict2 = model.predict(X_train)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.figure()\n",
+    "var = 3\n",
+    "plt.plot(y_train[:,var], label='Original Training Data', color='blue')\n",
+    "plt.plot(test_predict2[:,var], label='Predicted Training Data', color='red',alpha=0.8)\n",
+    "anomalies = np.where(abs(test_predict2[:,var] - y_train[:,var]) > 0.38)\n",
+    "plt.scatter(anomalies,test_predict2[anomalies,var], color='black',marker =\"o\",s=100 )\n",
+    "\n",
+    "\n",
+    "plt.title('Training Data - Predicted vs Actual')\n",
+    "plt.xlabel('Time')\n",
+    "plt.ylabel('Value')\n",
+    "plt.legend()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.mixture import GaussianMixture\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.decomposition import PCA\n",
+    "\n",
+    "\n",
+    "# Generating random data for demonstration\n",
+    "np.random.seed(0)\n",
+    "X =  test_predict1 - y_test\n",
+    "\n",
+    "\n",
+    "pca = PCA(n_components=2)\n",
+    "X = pca.fit_transform(X)\n",
+    "\n",
+    "\n",
+    "# Creating the GMM instance with desired number of clusters\n",
+    "gmm = GaussianMixture(n_components=2)\n",
+    "\n",
+    "# Fitting the model to the data\n",
+    "gmm.fit(X)\n",
+    "\n",
+    "# Getting the cluster labels\n",
+    "labels = gmm.predict(X)\n",
+    "\n",
+    "# Plotting the data points with colors representing different clusters\n",
+    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
+    "plt.title('GMM Clustering')\n",
+    "plt.xlabel('Feature 1')\n",
+    "plt.ylabel('Feature 2')\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.cluster import KMeans\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.decomposition import PCA\n",
+    "\n",
+    "# Generating random data for demonstration\n",
+    "np.random.seed(0)\n",
+    "X = (test_predict1 - y_test)\n",
+    "\n",
+    "k = 6\n",
+    "\n",
+    "kmeans = KMeans(n_clusters=k)\n",
+    "\n",
+    "kmeans.fit(X)\n",
+    "\n",
+    "pca = PCA(n_components=2)\n",
+    "X = pca.fit_transform(X)\n",
+    "\n",
+    "\n",
+    "\n",
+    "# Getting the cluster centers and labels\n",
+    "centroids = kmeans.cluster_centers_\n",
+    "centroids = pca.transform(centroids)\n",
+    "labels = kmeans.labels_\n",
+    "\n",
+    "# Plotting the data points and cluster centers\n",
+    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
+    "plt.scatter(centroids[:, 0], centroids[:, 1], marker='x', c='red', s=200, linewidths=2)\n",
+    "plt.title('KMeans Clustering')\n",
+    "plt.xlabel('Feature 1')\n",
+    "plt.ylabel('Feature 2')\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "k = 60\n",
+    "X= test_predict1 - y_test\n",
+    "processed_data = []\n",
+    "feat_df = pd.DataFrame(columns=[\"mean\",\"std\",])\n",
+    "for i in range(0,len(X), 60):\n",
+    "    mean = X[i:i+k].mean(axis = 0)\n",
+    "    std = X[i:i+k].std(axis = 0)\n",
+    "    max = X[i:i+k].max(axis = 0)\n",
+    "    min = X[i:i+k].min(axis = 0)\n",
+    "    iqr = np.percentile(X[i:i+k], 75, axis=0) - np.percentile(X[i:i+k], 25,axis=0)\n",
+    "    data = np.concatenate([mean, std, max, min, iqr])\n",
+    "    processed_data.append([data])\n",
+    "processed_data = np.concatenate(processed_data,axis=0) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X = processed_data\n",
+    "\n",
+    "kmeans = KMeans(n_clusters=2, algorithm='elkan', max_iter=1000, n_init = 5)\n",
+    "\n",
+    "kmeans.fit(X)\n",
+    "\n",
+    "pca = PCA(n_components=2)\n",
+    "X = pca.fit_transform(X)\n",
+    "\n",
+    "\n",
+    "# Getting the cluster centers and labels\n",
+    "centroids = kmeans.cluster_centers_\n",
+    "centroids = pca.transform(centroids)\n",
+    "labels = kmeans.labels_\n",
+    "\n",
+    "# Plotting the data points and cluster centers\n",
+    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
+    "plt.scatter(centroids[:, 0], centroids[:, 1], marker='x', c='red', s=200, linewidths=2)\n",
+    "plt.title('KMeans Clustering')\n",
+    "plt.xlabel('Feature 1')\n",
+    "plt.ylabel('Feature 2')\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.mixture import GaussianMixture\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.decomposition import PCA\n",
+    "\n",
+    "# Generating random data for demonstration\n",
+    "np.random.seed(0)\n",
+    "X = processed_data\n",
+    "\n",
+    "# Creating the GMM instance with desired number of clusters\n",
+    "gmm = GaussianMixture(n_components=2, init_params='k-means++')\n",
+    "\n",
+    "# Fitting the model to the data\n",
+    "gmm.fit(X)\n",
+    "labels = gmm.predict(X)\n",
+    "\n",
+    "\n",
+    "pca = PCA(n_components=2)\n",
+    "X = pca.fit_transform(X)\n",
+    "\n",
+    "\n",
+    "# Getting the cluster labels\n",
+    "\n",
+    "# Plotting the data points with colors representing different clusters\n",
+    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
+    "plt.title('GMM Clustering')\n",
+    "plt.xlabel('Feature 1')\n",
+    "plt.ylabel('Feature 2')\n",
+    "plt.show()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.cluster import KMeans\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "# Generating random data for demonstration\n",
+    "np.random.seed(0)\n",
+    "X = test_predict1 - y_test \n",
+    "\n",
+    "kmeans = KMeans(n_clusters=2)\n",
+    "\n",
+    "kmeans.fit(X)\n",
+    "\n",
+    "\n",
+    "pca = PCA(n_components=2)\n",
+    "X = pca.fit_transform(X)\n",
+    "\n",
+    "\n",
+    "\n",
+    "# Getting the cluster centers and labels\n",
+    "centroids = kmeans.cluster_centers_\n",
+    "centroids = pca.transform(centroids)\n",
+    "labels = kmeans.labels_\n",
+    "\n",
+    "# Plotting the data points and cluster centers\n",
+    "plt.figure()\n",
+    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
+    "plt.scatter(centroids[:, 0], centroids[:, 1], marker='x', c='red', s=200, linewidths=2)\n",
+    "plt.text(centroids[0,0], centroids[0,1], 'Normal', fontsize=12, color='red')\n",
+    "plt.text(centroids[1,0], centroids[1,1], 'Anomaly', fontsize=12, color='red')\n",
+    "plt.title('KMeans Clustering')\n",
+    "plt.xlabel('Feature 1')\n",
+    "plt.ylabel('Feature 2')\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "329763"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sum(labels==0)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "tensorflow",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

physLSTM/lstm_vav_rtu3.ipynb

@@ -0,0 +1,1131 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd \n",
+    "from datetime import datetime \n",
+    "from datetime import date\n",
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from keras.models import Sequential\n",
+    "from keras.layers import LSTM, Dense\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.preprocessing import MinMaxScaler,StandardScaler\n",
+    "from keras.callbacks import ModelCheckpoint\n",
+    "import tensorflow as tf"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "merged = pd.read_csv(r'../data/long_merge.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "zones = [72, 71, 63, 62, 60, 59, 58,57, 50, 49, 44, 43, 35, 34, 33, 32, 31, 30, 29, 28, ]\n",
+    "rtus = [2]\n",
+    "cols = []\n",
+    "\n",
+    "for zone in zones:\n",
+    "        for column in merged.columns:\n",
+    "            if f\"zone_0{zone}\" in column and 'co2' not in column and \"hw_valve\" not in column and \"cooling_sp\" not in column and \"heating_sp\" not in column:\n",
+    "                cols.append(column)\n",
+    "\n",
+    "for zone in zones:\n",
+    "        for column in merged.columns:\n",
+    "              if f\"zone_0{zone}\" in column: \n",
+    "                if \"cooling_sp\" in column or \"heating_sp\" in column:\n",
+    "                  cols.append(column)\n",
+    "# for rtu in rtus:\n",
+    "#     for column in merged.columns:\n",
+    "#         if f\"rtu_00{rtu}_fltrd_sa\" in column:\n",
+    "#                 cols.append(column)\n",
+    "cols =['date'] + cols + ['air_temp_set_1',\n",
+    " 'air_temp_set_2',\n",
+    " 'dew_point_temperature_set_1d',\n",
+    " 'relative_humidity_set_1',\n",
+    " 'solar_radiation_set_1']\n",
+    "input_dataset = merged[cols]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\arbal\\AppData\\Local\\Temp\\ipykernel_34660\\1855433847.py:1: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  input_dataset['date'] = pd.to_datetime(input_dataset['date'], format = \"%Y-%m-%d %H:%M:%S\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "input_dataset['date'] = pd.to_datetime(input_dataset['date'], format = \"%Y-%m-%d %H:%M:%S\")\n",
+    "df_filtered = input_dataset[ (input_dataset.date.dt.date >date(2019, 3, 1)) & (input_dataset.date.dt.date< date(2021, 1, 1))]\n",
+    "\n",
+    "if df_filtered.isna().any().any():\n",
+    "    print(\"There are NA values in the DataFrame columns.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>date</th>\n",
+       "      <th>zone_072_temp</th>\n",
+       "      <th>zone_072_fan_spd</th>\n",
+       "      <th>zone_071_temp</th>\n",
+       "      <th>zone_071_fan_spd</th>\n",
+       "      <th>zone_063_temp</th>\n",
+       "      <th>zone_063_fan_spd</th>\n",
+       "      <th>zone_062_temp</th>\n",
+       "      <th>zone_062_fan_spd</th>\n",
+       "      <th>zone_059_temp</th>\n",
+       "      <th>...</th>\n",
+       "      <th>zone_035_heating_sp</th>\n",
+       "      <th>zone_032_cooling_sp</th>\n",
+       "      <th>zone_032_heating_sp</th>\n",
+       "      <th>zone_030_cooling_sp</th>\n",
+       "      <th>zone_030_heating_sp</th>\n",
+       "      <th>air_temp_set_1</th>\n",
+       "      <th>air_temp_set_2</th>\n",
+       "      <th>dew_point_temperature_set_1d</th>\n",
+       "      <th>relative_humidity_set_1</th>\n",
+       "      <th>solar_radiation_set_1</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>511205</th>\n",
+       "      <td>2019-03-02 00:00:00</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.9</td>\n",
+       "      <td>55.0</td>\n",
+       "      <td>71.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>74.000000</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>67.0</td>\n",
+       "      <td>11.590</td>\n",
+       "      <td>11.130</td>\n",
+       "      <td>3.00</td>\n",
+       "      <td>55.87</td>\n",
+       "      <td>120.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>511206</th>\n",
+       "      <td>2019-03-02 00:01:00</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.9</td>\n",
+       "      <td>55.0</td>\n",
+       "      <td>71.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>74.000000</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>67.0</td>\n",
+       "      <td>11.590</td>\n",
+       "      <td>11.130</td>\n",
+       "      <td>3.00</td>\n",
+       "      <td>55.87</td>\n",
+       "      <td>120.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>511207</th>\n",
+       "      <td>2019-03-02 00:02:00</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.6</td>\n",
+       "      <td>55.0</td>\n",
+       "      <td>71.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>74.000000</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>67.0</td>\n",
+       "      <td>11.590</td>\n",
+       "      <td>11.130</td>\n",
+       "      <td>3.00</td>\n",
+       "      <td>55.87</td>\n",
+       "      <td>120.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>511208</th>\n",
+       "      <td>2019-03-02 00:03:00</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.9</td>\n",
+       "      <td>55.0</td>\n",
+       "      <td>71.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>74.000000</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>67.0</td>\n",
+       "      <td>11.590</td>\n",
+       "      <td>11.130</td>\n",
+       "      <td>3.00</td>\n",
+       "      <td>55.87</td>\n",
+       "      <td>120.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>511209</th>\n",
+       "      <td>2019-03-02 00:04:00</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.4</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>72.9</td>\n",
+       "      <td>55.0</td>\n",
+       "      <td>71.9</td>\n",
+       "      <td>...</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>74.000000</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>73.0</td>\n",
+       "      <td>67.0</td>\n",
+       "      <td>11.590</td>\n",
+       "      <td>11.130</td>\n",
+       "      <td>3.00</td>\n",
+       "      <td>55.87</td>\n",
+       "      <td>120.3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072148</th>\n",
+       "      <td>2020-12-31 23:57:00</td>\n",
+       "      <td>69.5</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.2</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>67.6</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>...</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>72.714138</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>13.994</td>\n",
+       "      <td>13.528</td>\n",
+       "      <td>4.11</td>\n",
+       "      <td>51.61</td>\n",
+       "      <td>188.8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072149</th>\n",
+       "      <td>2020-12-31 23:58:00</td>\n",
+       "      <td>69.5</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>67.6</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>...</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>72.714138</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>13.994</td>\n",
+       "      <td>13.528</td>\n",
+       "      <td>4.11</td>\n",
+       "      <td>51.61</td>\n",
+       "      <td>188.8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072150</th>\n",
+       "      <td>2020-12-31 23:58:00</td>\n",
+       "      <td>69.5</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.3</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>67.6</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>...</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>72.714138</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>13.994</td>\n",
+       "      <td>13.528</td>\n",
+       "      <td>4.11</td>\n",
+       "      <td>51.61</td>\n",
+       "      <td>188.8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072151</th>\n",
+       "      <td>2020-12-31 23:59:00</td>\n",
+       "      <td>69.5</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>67.6</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>...</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>72.714138</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>13.994</td>\n",
+       "      <td>13.528</td>\n",
+       "      <td>4.11</td>\n",
+       "      <td>51.61</td>\n",
+       "      <td>188.8</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2072152</th>\n",
+       "      <td>2020-12-31 23:59:00</td>\n",
+       "      <td>69.5</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>71.5</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>67.6</td>\n",
+       "      <td>40.0</td>\n",
+       "      <td>67.5</td>\n",
+       "      <td>...</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>72.714138</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>71.0</td>\n",
+       "      <td>70.0</td>\n",
+       "      <td>13.994</td>\n",
+       "      <td>13.528</td>\n",
+       "      <td>4.11</td>\n",
+       "      <td>51.61</td>\n",
+       "      <td>188.8</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1560948 rows × 56 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                       date  zone_072_temp  zone_072_fan_spd  zone_071_temp  \\\n",
+       "511205  2019-03-02 00:00:00           71.2              40.0           71.5   \n",
+       "511206  2019-03-02 00:01:00           71.2              40.0           71.5   \n",
+       "511207  2019-03-02 00:02:00           71.2              40.0           71.5   \n",
+       "511208  2019-03-02 00:03:00           71.2              40.0           71.5   \n",
+       "511209  2019-03-02 00:04:00           71.2              40.0           71.4   \n",
+       "...                     ...            ...               ...            ...   \n",
+       "2072148 2020-12-31 23:57:00           69.5              40.0           71.2   \n",
+       "2072149 2020-12-31 23:58:00           69.5              40.0           71.3   \n",
+       "2072150 2020-12-31 23:58:00           69.5              40.0           71.3   \n",
+       "2072151 2020-12-31 23:59:00           69.5              40.0           71.5   \n",
+       "2072152 2020-12-31 23:59:00           69.5              40.0           71.5   \n",
+       "\n",
+       "         zone_071_fan_spd  zone_063_temp  zone_063_fan_spd  zone_062_temp  \\\n",
+       "511205               20.0           72.3              20.0           72.9   \n",
+       "511206               20.0           72.3              20.0           72.9   \n",
+       "511207               20.0           72.3              20.0           72.6   \n",
+       "511208               20.0           72.3              20.0           72.9   \n",
+       "511209               20.0           72.3              20.0           72.9   \n",
+       "...                   ...            ...               ...            ...   \n",
+       "2072148              20.0           68.0              20.0           67.6   \n",
+       "2072149              20.0           68.0              20.0           67.6   \n",
+       "2072150              20.0           68.0              20.0           67.6   \n",
+       "2072151              20.0           68.0              20.0           67.6   \n",
+       "2072152              20.0           68.0              20.0           67.6   \n",
+       "\n",
+       "         zone_062_fan_spd  zone_059_temp  ...  zone_035_heating_sp  \\\n",
+       "511205               55.0           71.9  ...                 70.0   \n",
+       "511206               55.0           71.9  ...                 70.0   \n",
+       "511207               55.0           71.9  ...                 70.0   \n",
+       "511208               55.0           71.9  ...                 70.0   \n",
+       "511209               55.0           71.9  ...                 70.0   \n",
+       "...                   ...            ...  ...                  ...   \n",
+       "2072148              40.0           67.5  ...                 68.0   \n",
+       "2072149              40.0           67.5  ...                 68.0   \n",
+       "2072150              40.0           67.5  ...                 68.0   \n",
+       "2072151              40.0           67.5  ...                 68.0   \n",
+       "2072152              40.0           67.5  ...                 68.0   \n",
+       "\n",
+       "         zone_032_cooling_sp  zone_032_heating_sp  zone_030_cooling_sp  \\\n",
+       "511205             74.000000                 68.0                 73.0   \n",
+       "511206             74.000000                 68.0                 73.0   \n",
+       "511207             74.000000                 68.0                 73.0   \n",
+       "511208             74.000000                 68.0                 73.0   \n",
+       "511209             74.000000                 68.0                 73.0   \n",
+       "...                      ...                  ...                  ...   \n",
+       "2072148            72.714138                 71.0                 71.0   \n",
+       "2072149            72.714138                 71.0                 71.0   \n",
+       "2072150            72.714138                 71.0                 71.0   \n",
+       "2072151            72.714138                 71.0                 71.0   \n",
+       "2072152            72.714138                 71.0                 71.0   \n",
+       "\n",
+       "         zone_030_heating_sp  air_temp_set_1  air_temp_set_2  \\\n",
+       "511205                  67.0          11.590          11.130   \n",
+       "511206                  67.0          11.590          11.130   \n",
+       "511207                  67.0          11.590          11.130   \n",
+       "511208                  67.0          11.590          11.130   \n",
+       "511209                  67.0          11.590          11.130   \n",
+       "...                      ...             ...             ...   \n",
+       "2072148                 70.0          13.994          13.528   \n",
+       "2072149                 70.0          13.994          13.528   \n",
+       "2072150                 70.0          13.994          13.528   \n",
+       "2072151                 70.0          13.994          13.528   \n",
+       "2072152                 70.0          13.994          13.528   \n",
+       "\n",
+       "         dew_point_temperature_set_1d  relative_humidity_set_1  \\\n",
+       "511205                           3.00                    55.87   \n",
+       "511206                           3.00                    55.87   \n",
+       "511207                           3.00                    55.87   \n",
+       "511208                           3.00                    55.87   \n",
+       "511209                           3.00                    55.87   \n",
+       "...                               ...                      ...   \n",
+       "2072148                          4.11                    51.61   \n",
+       "2072149                          4.11                    51.61   \n",
+       "2072150                          4.11                    51.61   \n",
+       "2072151                          4.11                    51.61   \n",
+       "2072152                          4.11                    51.61   \n",
+       "\n",
+       "         solar_radiation_set_1  \n",
+       "511205                   120.3  \n",
+       "511206                   120.3  \n",
+       "511207                   120.3  \n",
+       "511208                   120.3  \n",
+       "511209                   120.3  \n",
+       "...                        ...  \n",
+       "2072148                  188.8  \n",
+       "2072149                  188.8  \n",
+       "2072150                  188.8  \n",
+       "2072151                  188.8  \n",
+       "2072152                  188.8  \n",
+       "\n",
+       "[1560948 rows x 56 columns]"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_filtered"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[]"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "testdataset_df = df_filtered[(df_filtered.date.dt.date >date(2020, 3, 1)) & (df_filtered.date.dt.date <date(2020,7, 1))]\n",
+    "\n",
+    "# traindataset_df = df_filtered[ (df_filtered.date.dt.date >date(2019, 11, 8))]\n",
+    "\n",
+    "traindataset_df = df_filtered[(df_filtered.date.dt.date >date(2019, 3, 1)) & (df_filtered.date.dt.date <date(2020, 3, 1)) | (df_filtered.date.dt.date >date(2020, 7, 1)) & (df_filtered.date.dt.date <date(2020, 12, 1))]\n",
+    "testdataset = testdataset_df.drop(columns=[\"date\"]).values\n",
+    "traindataset = traindataset_df.drop(columns=[\"date\"]).values\n",
+    "\n",
+    "columns_with_na = traindataset_df.columns[traindataset_df.isna().any()].tolist()\n",
+    "columns_with_na"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['date', 'zone_072_temp', 'zone_072_fan_spd', 'zone_071_temp',\n",
+       "       'zone_071_fan_spd', 'zone_063_temp', 'zone_063_fan_spd',\n",
+       "       'zone_062_temp', 'zone_062_fan_spd', 'zone_059_temp',\n",
+       "       'zone_059_fan_spd', 'zone_058_temp', 'zone_058_fan_spd',\n",
+       "       'zone_057_temp', 'zone_057_fan_spd', 'zone_049_temp',\n",
+       "       'zone_049_fan_spd', 'zone_044_temp', 'zone_044_fan_spd',\n",
+       "       'zone_043_temp', 'zone_043_fan_spd', 'zone_035_temp',\n",
+       "       'zone_035_fan_spd', 'zone_033_temp', 'zone_033_fan_spd',\n",
+       "       'zone_032_temp', 'zone_032_fan_spd', 'zone_030_temp',\n",
+       "       'zone_030_fan_spd', 'zone_028_temp', 'zone_028_fan_spd'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "traindataset_df.columns[0:31]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0 0\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(traindataset_df.isna().sum().sum(), testdataset_df.isna().sum().sum())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1073512, 391818)"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(traindataset), len(testdataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "traindataset = traindataset.astype('float32')\n",
+    "testdataset = testdataset.astype('float32')\n",
+    "\n",
+    "scaler = StandardScaler()\n",
+    "traindataset = scaler.fit_transform(traindataset)\n",
+    "testdataset = scaler.transform(testdataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1073512, 55)"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "traindataset.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train,test = traindataset,testdataset\n",
+    "\n",
+    "def create_dataset(dataset,time_step):\n",
+    "    x = []\n",
+    "    Y = []\n",
+    "    for i in range(len(dataset) - time_step - 1):\n",
+    "        x.append(dataset[i:(i+time_step),:])\n",
+    "        Y.append(dataset[i+time_step,0:31])\n",
+    "    x= np.array(x)\n",
+    "    Y = np.array(Y)\n",
+    "    return x,Y\n",
+    "time_step = 30\n",
+    "X_train, y_train = create_dataset(train, time_step)\n",
+    "X_test, y_test = create_dataset(test, time_step)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "((1073481, 30, 55), (1073481, 31))"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "X_train.shape, y_train.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/5\n",
+      "8386/8387 [============================>.] - ETA: 0s - loss: 0.0512\n",
+      "Epoch 1: val_loss improved from inf to 0.29908, saving model to lstm_vav_02.tf\n",
+      "INFO:tensorflow:Assets written to: lstm_vav_02.tf\\assets\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Assets written to: lstm_vav_02.tf\\assets\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "8387/8387 [==============================] - 337s 40ms/step - loss: 0.0512 - val_loss: 0.2991\n",
+      "Epoch 2/5\n",
+      "8387/8387 [==============================] - ETA: 0s - loss: 0.0213\n",
+      "Epoch 2: val_loss improved from 0.29908 to 0.23285, saving model to lstm_vav_02.tf\n",
+      "INFO:tensorflow:Assets written to: lstm_vav_02.tf\\assets\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:tensorflow:Assets written to: lstm_vav_02.tf\\assets\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "8387/8387 [==============================] - 300s 36ms/step - loss: 0.0213 - val_loss: 0.2328\n",
+      "Epoch 3/5\n",
+      "8387/8387 [==============================] - ETA: 0s - loss: 0.0152\n",
+      "Epoch 3: val_loss did not improve from 0.23285\n",
+      "8387/8387 [==============================] - 359s 43ms/step - loss: 0.0152 - val_loss: 0.2426\n",
+      "Epoch 4/5\n",
+      "8387/8387 [==============================] - ETA: 0s - loss: 0.0106\n",
+      "Epoch 4: val_loss did not improve from 0.23285\n",
+      "8387/8387 [==============================] - 350s 42ms/step - loss: 0.0106 - val_loss: 0.2556\n",
+      "Epoch 5/5\n",
+      "8386/8387 [============================>.] - ETA: 0s - loss: 0.0096\n",
+      "Epoch 5: val_loss did not improve from 0.23285\n",
+      "8387/8387 [==============================] - 349s 42ms/step - loss: 0.0096 - val_loss: 0.2635\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<keras.src.callbacks.History at 0x27dbd115490>"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\n",
+    "model = Sequential()\n",
+    "model.add(LSTM(units=50, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))\n",
+    "model.add(LSTM(units=50, return_sequences=True))\n",
+    "model.add(LSTM(units=30))\n",
+    "model.add(Dense(units=y_train.shape[1]))\n",
+    "\n",
+    "model.compile(optimizer='adam', loss='mean_squared_error')\n",
+    "\n",
+    "checkpoint_path = \"lstm_vav_02.tf\"\n",
+    "checkpoint_callback = ModelCheckpoint(filepath=checkpoint_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
+    "model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=5, batch_size=128, verbose=1, callbacks=[checkpoint_callback])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<tensorflow.python.checkpoint.checkpoint.CheckpointLoadStatus at 0x2a4b2344610>"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.load_weights(checkpoint_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "12244/12244 [==============================] - 60s 5ms/step\n"
+     ]
+    }
+   ],
+   "source": [
+    "test_predict1 = model.predict(X_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%matplotlib qt\n",
+    "plt.figure()\n",
+    "var = 1\n",
+    "plt.plot(y_test[:,var], label='Original Testing Data', color='blue')\n",
+    "plt.plot(test_predict1[:,var], label='Predicted Testing Data', color='red',alpha=0.8)\n",
+    "anomalies = np.where(abs(test_predict1[:,var] - y_test[:,var]) > 0.38)\n",
+    "plt.scatter(anomalies,test_predict1[anomalies,var], color='black',marker =\"o\",s=100 )\n",
+    "\n",
+    "\n",
+    "plt.title('Testing Data - Predicted vs Actual')\n",
+    "plt.xlabel('Time')\n",
+    "plt.ylabel('Value')\n",
+    "plt.legend()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "33547/33547 [==============================] - 201s 6ms/step\n"
+     ]
+    },
+    {
+     "ename": "ValueError",
+     "evalue": "operands could not be broadcast together with shapes (1073481,) (391787,) ",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[36], line 8\u001b[0m\n\u001b[0;32m      6\u001b[0m plt\u001b[38;5;241m.\u001b[39mplot(y_train[:,var], label\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mOriginal Testing Data\u001b[39m\u001b[38;5;124m'\u001b[39m, color\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mblue\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m      7\u001b[0m plt\u001b[38;5;241m.\u001b[39mplot(test_predict2[:,var], label\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mPredicted Testing Data\u001b[39m\u001b[38;5;124m'\u001b[39m, color\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mred\u001b[39m\u001b[38;5;124m'\u001b[39m,alpha\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.8\u001b[39m)\n\u001b[1;32m----> 8\u001b[0m anomalies \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mwhere(\u001b[38;5;28mabs\u001b[39m(\u001b[43mtest_predict2\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[43my_test\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m]\u001b[49m) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0.38\u001b[39m)\n\u001b[0;32m      9\u001b[0m plt\u001b[38;5;241m.\u001b[39mscatter(anomalies,test_predict2[anomalies,var], color\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mblack\u001b[39m\u001b[38;5;124m'\u001b[39m,marker \u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mo\u001b[39m\u001b[38;5;124m\"\u001b[39m,s\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m100\u001b[39m )\n\u001b[0;32m     12\u001b[0m plt\u001b[38;5;241m.\u001b[39mtitle(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTraining Data - Predicted vs Actual\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
+      "\u001b[1;31mValueError\u001b[0m: operands could not be broadcast together with shapes (1073481,) (391787,) "
+     ]
+    }
+   ],
+   "source": [
+    "%matplotlib qt\n",
+    "test_predict2 = model.predict(X_train)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.figure()\n",
+    "var = 3\n",
+    "plt.plot(y_train[:,var], label='Original Training Data', color='blue')\n",
+    "plt.plot(test_predict2[:,var], label='Predicted Training Data', color='red',alpha=0.8)\n",
+    "anomalies = np.where(abs(test_predict2[:,var] - y_train[:,var]) > 0.38)\n",
+    "plt.scatter(anomalies,test_predict2[anomalies,var], color='black',marker =\"o\",s=100 )\n",
+    "\n",
+    "\n",
+    "plt.title('Training Data - Predicted vs Actual')\n",
+    "plt.xlabel('Time')\n",
+    "plt.ylabel('Value')\n",
+    "plt.legend()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.mixture import GaussianMixture\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.decomposition import PCA\n",
+    "\n",
+    "\n",
+    "# Generating random data for demonstration\n",
+    "np.random.seed(0)\n",
+    "X =  test_predict1 - y_test\n",
+    "\n",
+    "\n",
+    "pca = PCA(n_components=2)\n",
+    "X = pca.fit_transform(X)\n",
+    "\n",
+    "\n",
+    "# Creating the GMM instance with desired number of clusters\n",
+    "gmm = GaussianMixture(n_components=2)\n",
+    "\n",
+    "# Fitting the model to the data\n",
+    "gmm.fit(X)\n",
+    "\n",
+    "# Getting the cluster labels\n",
+    "labels = gmm.predict(X)\n",
+    "\n",
+    "# Plotting the data points with colors representing different clusters\n",
+    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
+    "plt.title('GMM Clustering')\n",
+    "plt.xlabel('Feature 1')\n",
+    "plt.ylabel('Feature 2')\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.cluster import KMeans\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.decomposition import PCA\n",
+    "\n",
+    "# Generating random data for demonstration\n",
+    "np.random.seed(0)\n",
+    "X = (test_predict1 - y_test)\n",
+    "\n",
+    "k = 6\n",
+    "\n",
+    "kmeans = KMeans(n_clusters=k)\n",
+    "\n",
+    "kmeans.fit(X)\n",
+    "\n",
+    "\n",
+    "pca = PCA(n_components=2)\n",
+    "X = pca.fit_transform(X)\n",
+    "\n",
+    "\n",
+    "\n",
+    "# Getting the cluster centers and labels\n",
+    "centroids = kmeans.cluster_centers_\n",
+    "centroids = pca.transform(centroids)\n",
+    "labels = kmeans.labels_\n",
+    "\n",
+    "# Plotting the data points and cluster centers\n",
+    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
+    "plt.scatter(centroids[:, 0], centroids[:, 1], marker='x', c='red', s=200, linewidths=2)\n",
+    "plt.title('KMeans Clustering')\n",
+    "plt.xlabel('Feature 1')\n",
+    "plt.ylabel('Feature 2')\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "k = 60\n",
+    "X= test_predict1 - y_test\n",
+    "processed_data = []\n",
+    "feat_df = pd.DataFrame(columns=[\"mean\",\"std\",])\n",
+    "for i in range(0,len(X), 60):\n",
+    "    mean = X[i:i+k].mean(axis = 0)\n",
+    "    std = X[i:i+k].std(axis = 0)\n",
+    "    max = X[i:i+k].max(axis = 0)\n",
+    "    min = X[i:i+k].min(axis = 0)\n",
+    "    iqr = np.percentile(X[i:i+k], 75, axis=0) - np.percentile(X[i:i+k], 25,axis=0)\n",
+    "    data = np.concatenate([mean, std, max, min, iqr])\n",
+    "    processed_data.append([data])\n",
+    "processed_data = np.concatenate(processed_data,axis=0) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X = processed_data\n",
+    "\n",
+    "kmeans = KMeans(n_clusters=2, algorithm='elkan', max_iter=1000, n_init = 5)\n",
+    "\n",
+    "kmeans.fit(X)\n",
+    "\n",
+    "pca = PCA(n_components=2)\n",
+    "X = pca.fit_transform(X)\n",
+    "\n",
+    "\n",
+    "# Getting the cluster centers and labels\n",
+    "centroids = kmeans.cluster_centers_\n",
+    "centroids = pca.transform(centroids)\n",
+    "labels = kmeans.labels_\n",
+    "\n",
+    "# Plotting the data points and cluster centers\n",
+    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
+    "plt.scatter(centroids[:, 0], centroids[:, 1], marker='x', c='red', s=200, linewidths=2)\n",
+    "plt.title('KMeans Clustering')\n",
+    "plt.xlabel('Feature 1')\n",
+    "plt.ylabel('Feature 2')\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.mixture import GaussianMixture\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sklearn.decomposition import PCA\n",
+    "\n",
+    "# Generating random data for demonstration\n",
+    "np.random.seed(0)\n",
+    "X = processed_data\n",
+    "\n",
+    "# Creating the GMM instance with desired number of clusters\n",
+    "gmm = GaussianMixture(n_components=2, init_params='k-means++')\n",
+    "\n",
+    "# Fitting the model to the data\n",
+    "gmm.fit(X)\n",
+    "labels = gmm.predict(X)\n",
+    "\n",
+    "\n",
+    "pca = PCA(n_components=2)\n",
+    "X = pca.fit_transform(X)\n",
+    "\n",
+    "\n",
+    "# Getting the cluster labels\n",
+    "\n",
+    "# Plotting the data points with colors representing different clusters\n",
+    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
+    "plt.title('GMM Clustering')\n",
+    "plt.xlabel('Feature 1')\n",
+    "plt.ylabel('Feature 2')\n",
+    "plt.show()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.cluster import KMeans\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "# Generating random data for demonstration\n",
+    "np.random.seed(0)\n",
+    "X = test_predict1 - y_test \n",
+    "\n",
+    "kmeans = KMeans(n_clusters=2)\n",
+    "\n",
+    "kmeans.fit(X)\n",
+    "\n",
+    "\n",
+    "pca = PCA(n_components=2)\n",
+    "X = pca.fit_transform(X)\n",
+    "\n",
+    "\n",
+    "\n",
+    "# Getting the cluster centers and labels\n",
+    "centroids = kmeans.cluster_centers_\n",
+    "centroids = pca.transform(centroids)\n",
+    "labels = kmeans.labels_\n",
+    "\n",
+    "# Plotting the data points and cluster centers\n",
+    "plt.figure()\n",
+    "plt.scatter(X[:, 0], X[:, 1], c=labels, cmap='viridis', alpha=0.5)\n",
+    "plt.scatter(centroids[:, 0], centroids[:, 1], marker='x', c='red', s=200, linewidths=2)\n",
+    "plt.text(centroids[0,0], centroids[0,1], 'Normal', fontsize=12, color='red')\n",
+    "plt.text(centroids[1,0], centroids[1,1], 'Anomaly', fontsize=12, color='red')\n",
+    "plt.title('KMeans Clustering')\n",
+    "plt.xlabel('Feature 1')\n",
+    "plt.ylabel('Feature 2')\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "329810"
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sum(labels==0)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "tensorflow",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}