293f0b77a3d614e98964fbf0e74513433c1a1643493a79225b1dce7ad11c0a2b

experiment/rwkv-x-exp/multi-size-train/v5-L6+6-D2048-layer-baseline.ipynb

@@ -3,13 +3,13 @@
   {
    "attachments": {},
    "cell_type": "markdown",
-   "id": "c4287c8b",
+   "id": "d3126ef2",
    "metadata": {
     "papermill": {
-     "duration": 0.004479,
-     "end_time": "2023-10-11T07:03:00.576797",
+     "duration": 0.004879,
+     "end_time": "2023-10-11T08:02:23.608034",
      "exception": false,
-     "start_time": "2023-10-11T07:03:00.572318",
+     "start_time": "2023-10-11T08:02:23.603155",
      "status": "completed"
     },
     "tags": []
@@ -23,13 +23,13 @@
   {
    "attachments": {},
    "cell_type": "markdown",
-   "id": "5202003b",
+   "id": "986070aa",
    "metadata": {
     "papermill": {
-     "duration": 0.002433,
-     "end_time": "2023-10-11T07:03:00.582179",
+     "duration": 0.002523,
+     "end_time": "2023-10-11T08:02:23.613605",
      "exception": false,
-     "start_time": "2023-10-11T07:03:00.579746",
+     "start_time": "2023-10-11T08:02:23.611082",
      "status": "completed"
     },
     "tags": []
@@ -41,19 +41,19 @@
   {
    "cell_type": "code",
    "execution_count": 1,
-   "id": "fc416bc2",
+   "id": "dc924c7f",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-11T07:03:00.589288Z",
-     "iopub.status.busy": "2023-10-11T07:03:00.588784Z",
-     "iopub.status.idle": "2023-10-11T07:03:01.346279Z",
-     "shell.execute_reply": "2023-10-11T07:03:01.345465Z"
+     "iopub.execute_input": "2023-10-11T08:02:23.620990Z",
+     "iopub.status.busy": "2023-10-11T08:02:23.620432Z",
+     "iopub.status.idle": "2023-10-11T08:02:24.379549Z",
+     "shell.execute_reply": "2023-10-11T08:02:24.378580Z"
     },
     "papermill": {
-     "duration": 0.763505,
-     "end_time": "2023-10-11T07:03:01.348385",
+     "duration": 0.765369,
+     "end_time": "2023-10-11T08:02:24.381741",
      "exception": false,
-     "start_time": "2023-10-11T07:03:00.584880",
+     "start_time": "2023-10-11T08:02:23.616372",
      "status": "completed"
     },
     "tags": []
@@ -69,19 +69,19 @@
   {
    "cell_type": "code",
    "execution_count": 2,
-   "id": "a9dea07f",
+   "id": "2bbc32ac",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-11T07:03:01.355823Z",
-     "iopub.status.busy": "2023-10-11T07:03:01.355275Z",
-     "iopub.status.idle": "2023-10-11T07:03:01.363208Z",
-     "shell.execute_reply": "2023-10-11T07:03:01.362389Z"
+     "iopub.execute_input": "2023-10-11T08:02:24.389788Z",
+     "iopub.status.busy": "2023-10-11T08:02:24.389227Z",
+     "iopub.status.idle": "2023-10-11T08:02:24.398441Z",
+     "shell.execute_reply": "2023-10-11T08:02:24.397578Z"
     },
     "papermill": {
-     "duration": 0.013626,
-     "end_time": "2023-10-11T07:03:01.364896",
+     "duration": 0.015548,
+     "end_time": "2023-10-11T08:02:24.400362",
      "exception": false,
-     "start_time": "2023-10-11T07:03:01.351270",
+     "start_time": "2023-10-11T08:02:24.384814",
      "status": "completed"
     },
     "tags": []
@@ -139,19 +139,19 @@
   {
    "cell_type": "code",
    "execution_count": 3,
-   "id": "bcb68665",
+   "id": "ffa69634",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-11T07:03:01.372738Z",
-     "iopub.status.busy": "2023-10-11T07:03:01.371949Z",
-     "iopub.status.idle": "2023-10-11T07:03:02.581837Z",
-     "shell.execute_reply": "2023-10-11T07:03:02.580259Z"
+     "iopub.execute_input": "2023-10-11T08:02:24.408311Z",
+     "iopub.status.busy": "2023-10-11T08:02:24.407798Z",
+     "iopub.status.idle": "2023-10-11T08:03:19.634663Z",
+     "shell.execute_reply": "2023-10-11T08:03:19.633765Z"
     },
     "papermill": {
-     "duration": 1.216253,
-     "end_time": "2023-10-11T07:03:02.584117",
+     "duration": 55.233419,
+     "end_time": "2023-10-11T08:03:19.636895",
      "exception": false,
-     "start_time": "2023-10-11T07:03:01.367864",
+     "start_time": "2023-10-11T08:02:24.403476",
      "status": "completed"
     },
     "tags": []
@@ -161,9 +161,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "--2023-10-11 07:03:01--  https://huggingface.co/rwkv-x-dev/rwkv-x-playground/blob/main/experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-E0_01-split-2a.pth\r\n",
-      "Resolving huggingface.co (huggingface.co)... 18.154.227.7, 18.154.227.87, 18.154.227.69, ...\r\n",
-      "Connecting to huggingface.co (huggingface.co)|18.154.227.7|:443... connected.\r\n",
+      "--2023-10-11 08:02:24--  https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/0600b94a58219f658326b4792ef5cd020e9d1a43/experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-E0_01-split-2a.pth\r\n",
+      "Resolving huggingface.co (huggingface.co)... 18.154.227.87, 18.154.227.7, 18.154.227.69, ...\r\n",
+      "Connecting to huggingface.co (huggingface.co)|18.154.227.87|:443... connected.\r\n",
+      "HTTP request sent, awaiting response... 302 Found\r\n",
+      "Location: https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/2f52085cee9c3db4bb079dc44edf50b0a19c170bd92128e918e6203efef83cea?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-L6-D2048-E0_01-split-2a.pth%3B+filename%3D%22v5-L6-D2048-E0_01-split-2a.pth%22%3B&Expires=1697270544&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5NzI3MDU0NH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkLzJmNTIwODVjZWU5YzNkYjRiYjA3OWRjNDRlZGY1MGIwYTE5YzE3MGJkOTIxMjhlOTE4ZTYyMDNlZmVmODNjZWE%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=AW451jyDioqxesXvDVp%7EgfYV3uhgFTDwTn3SlZa-gk-yCDb7c-QR44rTm9sWCGSJjaa%7EvJvj9zLGUK7fvbr%7E%7EGQJgL2L%7Es9vkVPg8qs1k%7EtCh-MX%7E45bxo4CapTIo8fx4xLJ738Tks8uzpx3Sy9hWbfuGQFCUwBHzJXG5uGNRzPv87Zdfy4gIIAt0NytaC3bFmKZl4DbXLF4%7EtVWXED7H3NAlBvGETdhjzK5Qr0FLZB2vqC1LQpPTexdTH-ETkPEIQpXRBV-JctzaKBfI1Da-tGpt4JdPlhyPIu1kaNtX13yTibuBrT-mDOy6OVJZ9Zsj%7EHdVUtDrdp-I01dhylHpQ__&Key-Pair-Id=KVTP0A1DKRTAX [following]\r\n",
+      "--2023-10-11 08:02:24--  https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/2f52085cee9c3db4bb079dc44edf50b0a19c170bd92128e918e6203efef83cea?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-L6-D2048-E0_01-split-2a.pth%3B+filename%3D%22v5-L6-D2048-E0_01-split-2a.pth%22%3B&Expires=1697270544&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5NzI3MDU0NH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkLzJmNTIwODVjZWU5YzNkYjRiYjA3OWRjNDRlZGY1MGIwYTE5YzE3MGJkOTIxMjhlOTE4ZTYyMDNlZmVmODNjZWE%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=AW451jyDioqxesXvDVp%7EgfYV3uhgFTDwTn3SlZa-gk-yCDb7c-QR44rTm9sWCGSJjaa%7EvJvj9zLGUK7fvbr%7E%7EGQJgL2L%7Es9vkVPg8qs1k%7EtCh-MX%7E45bxo4CapTIo8fx4xLJ738Tks8uzpx3Sy9hWbfuGQFCUwBHzJXG5uGNRzPv87Zdfy4gIIAt0NytaC3bFmKZl4DbXLF4%7EtVWXED7H3NAlBvGETdhjzK5Qr0FLZB2vqC1LQpPTexdTH-ETkPEIQpXRBV-JctzaKBfI1Da-tGpt4JdPlhyPIu1kaNtX13yTibuBrT-mDOy6OVJZ9Zsj%7EHdVUtDrdp-I01dhylHpQ__&Key-Pair-Id=KVTP0A1DKRTAX\r\n",
+      "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 3.162.112.69, 3.162.112.2, 3.162.112.100, ...\r\n",
+      "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|3.162.112.69|:443... connected.\r\n",
       "HTTP request sent, awaiting response... "
      ]
     },
@@ -172,14 +177,814 @@
      "output_type": "stream",
      "text": [
       "200 OK\r\n",
-      "Length: 44360 (43K) [text/html]\r\n",
+      "Length: 1066536657 (1017M) [binary/octet-stream]\r\n",
       "Saving to: ‘v5-L6-D2048-E0_01-split-2a.pth’\r\n",
       "\r\n",
       "\r",
-      "          v5-L6-D20   0%[                    ]       0  --.-KB/s               \r",
-      "v5-L6-D2048-E0_01-s 100%[===================>]  43.32K  --.-KB/s    in 0.001s  \r\n",
+      "          v5-L6-D20   0%[                    ]       0  --.-KB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "         v5-L6-D204   1%[                    ]  15.26M  42.9MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "        v5-L6-D2048   3%[                    ]  30.52M  47.7MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "       v5-L6-D2048-   4%[                    ]  45.26M  51.5MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "      v5-L6-D2048-E   5%[>                   ]  59.20M  52.1MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "     v5-L6-D2048-E0   6%[>                   ]  65.20M  48.8MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "    v5-L6-D2048-E0_   7%[>                   ]  76.29M  44.4MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "   v5-L6-D2048-E0_0   8%[>                   ]  91.03M  47.2MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "  v5-L6-D2048-E0_01   9%[>                   ]  91.55M  43.0MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      " v5-L6-D2048-E0_01-  10%[=>                  ] 106.81M  43.2MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "v5-L6-D2048-E0_01-s  11%[=>                  ] 120.25M  43.9MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "5-L6-D2048-E0_01-sp  12%[=>                  ] 122.07M  41.4MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-L6-D2048-E0_01-spl  13%[=>                  ] 136.81M  42.5MB/s    eta 21s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "L6-D2048-E0_01-spli  14%[=>                  ] 152.07M  42.8MB/s    eta 21s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "6-D2048-E0_01-split  15%[==>                 ] 152.72M  40.5MB/s    eta 21s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-D2048-E0_01-split-  16%[==>                 ] 167.85M  41.5MB/s    eta 21s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "D2048-E0_01-split-2  18%[==>                 ] 183.10M  43.0MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "2048-E0_01-split-2a  19%[==>                 ] 198.36M  43.4MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "048-E0_01-split-2a.  20%[===>                ] 213.11M  44.1MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "48-E0_01-split-2a.p  22%[===>                ] 228.36M  43.3MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "8-E0_01-split-2a.pt  22%[===>                ] 228.87M  41.1MB/s    eta 18s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-E0_01-split-2a.pth  24%[===>                ] 244.13M  41.0MB/s    eta 18s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "E0_01-split-2a.pth   25%[====>               ] 259.40M  42.4MB/s    eta 18s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "0_01-split-2a.pth    26%[====>               ] 272.83M  40.4MB/s    eta 18s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "_01-split-2a.pth     28%[====>               ] 289.40M  41.4MB/s    eta 18s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "01-split-2a.pth      28%[====>               ] 289.92M  37.9MB/s    eta 18s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "1-split-2a.pth       29%[====>               ] 304.66M  36.1MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-split-2a.pth        30%[=====>              ] 305.18M  33.4MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "split-2a.pth         31%[=====>              ] 318.60M  33.2MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "plit-2a.pth          31%[=====>              ] 320.29M  33.3MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "lit-2a.pth           31%[=====>              ] 320.57M  30.7MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "it-2a.pth            32%[=====>              ] 335.18M  30.1MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "t-2a.pth             33%[=====>              ] 345.53M  31.2MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-2a.pth              34%[=====>              ] 350.82M  29.7MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "2a.pth               35%[======>             ] 360.98M  31.3MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "a.pth                36%[======>             ] 366.20M  29.6MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      ".pth                 37%[======>             ] 380.96M  30.8MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "pth                  38%[======>             ] 392.79M  32.0MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "th                   39%[======>             ] 396.73M  29.1MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "h                    40%[=======>            ] 411.99M  29.1MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                     41%[=======>            ] 426.73M  28.7MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                  v  42%[=======>            ] 427.25M  29.1MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                 v5  42%[=======>            ] 435.25M  27.9MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                v5-  43%[=======>            ] 438.04M  28.2MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "               v5-L  43%[=======>            ] 442.05M  29.7MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "              v5-L6  43%[=======>            ] 446.00M  31.1MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "             v5-L6-  44%[=======>            ] 457.24M  33.9MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "            v5-L6-D  45%[========>           ] 457.89M  31.6MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "           v5-L6-D2  46%[========>           ] 473.02M  34.8MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "          v5-L6-D20  48%[========>           ] 488.28M  34.1MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "         v5-L6-D204  49%[========>           ] 503.03M  34.6MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "        v5-L6-D2048  50%[=========>          ] 518.29M  37.3MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "       v5-L6-D2048-  51%[=========>          ] 525.10M  35.8MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "      v5-L6-D2048-E  52%[=========>          ] 534.05M  34.4MB/s    eta 13s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "     v5-L6-D2048-E0  53%[=========>          ] 548.80M  34.4MB/s    eta 13s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "    v5-L6-D2048-E0_  55%[==========>         ] 562.75M  33.8MB/s    eta 13s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "   v5-L6-D2048-E0_0  56%[==========>         ] 579.31M  36.0MB/s    eta 13s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "  v5-L6-D2048-E0_01  57%[==========>         ] 581.49M  36.7MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      " v5-L6-D2048-E0_01-  58%[==========>         ] 592.93M  37.4MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "v5-L6-D2048-E0_01-s  58%[==========>         ] 595.09M  37.1MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "5-L6-D2048-E0_01-sp  60%[===========>        ] 610.35M  38.5MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-L6-D2048-E0_01-spl  61%[===========>        ] 625.61M  38.7MB/s    eta 11s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "L6-D2048-E0_01-spli  62%[===========>        ] 640.36M  39.9MB/s    eta 11s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "6-D2048-E0_01-split  64%[===========>        ] 653.30M  39.5MB/s    eta 11s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-D2048-E0_01-split-  64%[===========>        ] 656.13M  38.5MB/s    eta 11s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "D2048-E0_01-split-2  66%[============>       ] 671.38M  38.9MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "2048-E0_01-split-2a  67%[============>       ] 685.57M  39.7MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "048-E0_01-split-2a.  67%[============>       ] 686.64M  37.5MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "48-E0_01-split-2a.p  68%[============>       ] 701.39M  37.9MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "8-E0_01-split-2a.pt  69%[============>       ] 708.59M  38.8MB/s    eta 8s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-E0_01-split-2a.pth  70%[=============>      ] 715.34M  38.2MB/s    eta 8s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "E0_01-split-2a.pth   71%[=============>      ] 731.91M  40.7MB/s    eta 8s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "0_01-split-2a.pth    73%[=============>      ] 747.17M  38.0MB/s    eta 8s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "_01-split-2a.pth     73%[=============>      ] 747.75M  38.0MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "01-split-2a.pth      74%[=============>      ] 762.42M  40.2MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "1-split-2a.pth       75%[==============>     ] 762.94M  37.2MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-split-2a.pth        76%[==============>     ] 776.37M  36.7MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "split-2a.pth         76%[==============>     ] 778.20M  34.9MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "plit-2a.pth          77%[==============>     ] 791.63M  38.1MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "lit-2a.pth           78%[==============>     ] 793.46M  36.0MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "it-2a.pth            79%[==============>     ] 808.20M  38.6MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "t-2a.pth             80%[===============>    ] 816.07M  36.7MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-2a.pth              81%[===============>    ] 823.97M  34.7MB/s    eta 5s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "2a.pth               82%[===============>    ] 837.41M  36.4MB/s    eta 5s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "a.pth                83%[===============>    ] 853.98M  38.3MB/s    eta 5s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      ".pth                 85%[================>   ] 867.67M  38.4MB/s    eta 5s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "pth                  85%[================>   ] 873.17M  39.1MB/s    eta 5s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "th                   87%[================>   ] 885.01M  36.1MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "h                    88%[================>   ] 899.75M  37.6MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                     88%[================>   ] 900.40M  34.6MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                  v  90%[=================>  ] 915.53M  35.4MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                 v5  91%[=================>  ] 930.78M  37.6MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                v5-  92%[=================>  ] 945.53M  40.9MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "               v5-L  93%[=================>  ] 946.04M  37.6MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "              v5-L6  94%[=================>  ] 959.48M  38.0MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "             v5-L6-  94%[=================>  ] 961.30M  33.1MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "            v5-L6-D  95%[==================> ] 976.05M  34.9MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "           v5-L6-D2  97%[==================> ] 991.31M  34.8MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "          v5-L6-D20  97%[==================> ] 992.94M  35.0MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "         v5-L6-D204  98%[==================> ]   1005M  34.7MB/s    eta 0s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "        v5-L6-D2048  99%[==================> ]   1016M  33.7MB/s    eta 0s     \r",
+      "v5-L6-D2048-E0_01-s 100%[===================>]   1017M  33.9MB/s    in 28s     \r\n",
       "\r\n",
-      "2023-10-11 07:03:01 (46.3 MB/s) - ‘v5-L6-D2048-E0_01-split-2a.pth’ saved [44360/44360]\r\n",
+      "2023-10-11 08:02:52 (36.4 MB/s) - ‘v5-L6-D2048-E0_01-split-2a.pth’ saved [1066536657/1066536657]\r\n",
       "\r\n"
      ]
     },
@@ -187,51 +992,814 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "--2023-10-11 07:03:02--  https://huggingface.co/rwkv-x-dev/rwkv-x-playground/blob/main/experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-E0_01-split-2b.pth\r\n",
-      "Resolving huggingface.co (huggingface.co)... 18.154.227.67, 18.154.227.7, 18.154.227.69, ...\r\n",
-      "Connecting to huggingface.co (huggingface.co)|18.154.227.67|:443... connected.\r\n",
-      "HTTP request sent, awaiting response... "
+      "--2023-10-11 08:02:53--  https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/0600b94a58219f658326b4792ef5cd020e9d1a43/experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-E0_01-split-2b.pth\r\n",
+      "Resolving huggingface.co (huggingface.co)... 18.154.227.67, 18.154.227.69, 18.154.227.7, ...\r\n",
+      "Connecting to huggingface.co (huggingface.co)|18.154.227.67|:443... connected.\r\n",
+      "HTTP request sent, awaiting response... 302 Found\r\n",
+      "Location: https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/6b64a1018631b9ddd15a746002bab3eafe956dced78a91af7abcdadaae4a7b25?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-L6-D2048-E0_01-split-2b.pth%3B+filename%3D%22v5-L6-D2048-E0_01-split-2b.pth%22%3B&Expires=1697270573&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5NzI3MDU3M319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkLzZiNjRhMTAxODYzMWI5ZGRkMTVhNzQ2MDAyYmFiM2VhZmU5NTZkY2VkNzhhOTFhZjdhYmNkYWRhYWU0YTdiMjU%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=WkKE1KjbKeVQp4dWdBuAAbOfx2JJs%7EDJaKbx8gRQSGABLfGDhkq2L8Q9KZ1fg1v%7E74c0Mkrbvop33pAwQDh782jzEiogbDb8HXSO7AtIYQqvI6K-fmb%7EpxQPFrmypJwWhQj9ePRZX2KSL6LcqN1X0GAheI-PQENpVH3svxhhib2-fYDmuvnpGX7pc6n36GES6lvwOuCQOxfIhlFnIiuNEU00NaBdDiaXb-uteXhSkKO-1EFCM0fBtwT5hVkdHZQG2m6iMcI2KaN0AHV%7EvF838f4DM%7ERbjVkRgwphRaYZxmJxUKZxGTV7rRJjIQA%7EOlnPllE1dSdwJ7y0ULOIKQHYUQ__&Key-Pair-Id=KVTP0A1DKRTAX [following]\r\n",
+      "--2023-10-11 08:02:53--  https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/6b64a1018631b9ddd15a746002bab3eafe956dced78a91af7abcdadaae4a7b25?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5-L6-D2048-E0_01-split-2b.pth%3B+filename%3D%22v5-L6-D2048-E0_01-split-2b.pth%22%3B&Expires=1697270573&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5NzI3MDU3M319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkLzZiNjRhMTAxODYzMWI5ZGRkMTVhNzQ2MDAyYmFiM2VhZmU5NTZkY2VkNzhhOTFhZjdhYmNkYWRhYWU0YTdiMjU%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=WkKE1KjbKeVQp4dWdBuAAbOfx2JJs%7EDJaKbx8gRQSGABLfGDhkq2L8Q9KZ1fg1v%7E74c0Mkrbvop33pAwQDh782jzEiogbDb8HXSO7AtIYQqvI6K-fmb%7EpxQPFrmypJwWhQj9ePRZX2KSL6LcqN1X0GAheI-PQENpVH3svxhhib2-fYDmuvnpGX7pc6n36GES6lvwOuCQOxfIhlFnIiuNEU00NaBdDiaXb-uteXhSkKO-1EFCM0fBtwT5hVkdHZQG2m6iMcI2KaN0AHV%7EvF838f4DM%7ERbjVkRgwphRaYZxmJxUKZxGTV7rRJjIQA%7EOlnPllE1dSdwJ7y0ULOIKQHYUQ__&Key-Pair-Id=KVTP0A1DKRTAX\r\n",
+      "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 3.162.112.95, 3.162.112.100, 3.162.112.2, ...\r\n",
+      "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|3.162.112.95|:443... connected.\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "HTTP request sent, awaiting response... "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "200 OK\r\n",
+      "Length: 1066536657 (1017M) [binary/octet-stream]\r\n",
+      "Saving to: ‘v5-L6-D2048-E0_01-split-2b.pth’\r\n",
+      "\r\n",
+      "\r",
+      "          v5-L6-D20   0%[                    ]       0  --.-KB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "         v5-L6-D204   1%[                    ]  14.74M  67.8MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "        v5-L6-D2048   2%[                    ]  28.69M  63.1MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "       v5-L6-D2048-   3%[                    ]  30.52M  42.3MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "      v5-L6-D2048-E   4%[                    ]  45.26M  45.2MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "     v5-L6-D2048-E0   4%[                    ]  45.78M  37.9MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "    v5-L6-D2048-E0_   6%[>                   ]  61.03M  41.2MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "   v5-L6-D2048-E0_0   7%[>                   ]  75.78M  45.0MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "  v5-L6-D2048-E0_01   8%[>                   ]  85.94M  45.6MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      " v5-L6-D2048-E0_01-   9%[>                   ]  91.55M  40.8MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "v5-L6-D2048-E0_01-s  10%[=>                  ] 106.81M  40.5MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "5-L6-D2048-E0_01-sp  12%[=>                  ] 122.07M  40.2MB/s    eta 22s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-L6-D2048-E0_01-spl  13%[=>                  ] 137.33M  41.7MB/s    eta 22s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "L6-D2048-E0_01-spli  14%[=>                  ] 152.07M  42.9MB/s    eta 22s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "6-D2048-E0_01-split  16%[==>                 ] 167.33M  43.6MB/s    eta 22s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-D2048-E0_01-split-  17%[==>                 ] 181.32M  44.9MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "D2048-E0_01-split-2  18%[==>                 ] 183.10M  41.7MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "2048-E0_01-split-2b  19%[==>                 ] 196.53M  41.8MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "048-E0_01-split-2b.  19%[==>                 ] 198.36M  39.0MB/s    eta 19s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "48-E0_01-split-2b.p  20%[===>                ] 213.11M  39.0MB/s    eta 20s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "8-E0_01-split-2b.pt  21%[===>                ] 220.29M  40.8MB/s    eta 20s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-E0_01-split-2b.pth  22%[===>                ] 228.36M  39.9MB/s    eta 20s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "E0_01-split-2b.pth   24%[===>                ] 244.13M  40.3MB/s    eta 20s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "0_01-split-2b.pth    25%[====>               ] 259.40M  40.4MB/s    eta 18s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "_01-split-2b.pth     26%[====>               ] 274.14M  42.1MB/s    eta 18s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "01-split-2b.pth      27%[====>               ] 274.66M  38.5MB/s    eta 18s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "1-split-2b.pth       28%[====>               ] 289.92M  41.6MB/s    eta 18s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-split-2b.pth        30%[=====>              ] 305.18M  41.6MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "split-2b.pth         31%[=====>              ] 320.43M  40.9MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "plit-2b.pth          32%[=====>              ] 335.18M  41.2MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "lit-2b.pth           33%[=====>              ] 335.69M  38.5MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "it-2b.pth            34%[=====>              ] 350.95M  38.6MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "t-2b.pth             35%[======>             ] 365.70M  40.9MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-2b.pth              36%[======>             ] 366.20M  38.0MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "2b.pth               37%[======>             ] 381.47M  38.8MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "b.pth                37%[======>             ] 385.65M  39.1MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      ".pth                 39%[======>             ] 396.73M  36.3MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "pth                  39%[======>             ] 406.75M  37.5MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "th                   40%[=======>            ] 411.99M  33.0MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "h                    42%[=======>            ] 427.25M  33.6MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                     43%[=======>            ] 441.98M  32.7MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                  v  43%[=======>            ] 442.51M  32.5MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                 v5  44%[=======>            ] 457.25M  32.2MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                v5-  45%[========>           ] 457.76M  32.2MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "               v5-L  46%[========>           ] 472.50M  31.5MB/s    eta 14s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "              v5-L6  46%[========>           ] 473.02M  31.5MB/s    eta 14s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "             v5-L6-  48%[========>           ] 488.28M  30.9MB/s    eta 14s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "            v5-L6-D  49%[========>           ] 503.54M  33.6MB/s    eta 14s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "           v5-L6-D2  50%[=========>          ] 518.29M  34.2MB/s    eta 14s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "          v5-L6-D20  51%[=========>          ] 518.80M  34.8MB/s    eta 13s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "         v5-L6-D204  52%[=========>          ] 534.05M  34.2MB/s    eta 13s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "        v5-L6-D2048  54%[=========>          ] 549.31M  37.5MB/s    eta 13s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "       v5-L6-D2048-  55%[==========>         ] 564.06M  37.7MB/s    eta 13s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "      v5-L6-D2048-E  55%[==========>         ] 565.78M  37.7MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "     v5-L6-D2048-E0  57%[==========>         ] 579.83M  37.8MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "    v5-L6-D2048-E0_  58%[==========>         ] 595.09M  39.7MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "   v5-L6-D2048-E0_0  60%[===========>        ] 610.35M  40.9MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "  v5-L6-D2048-E0_01  61%[===========>        ] 625.47M  44.1MB/s    eta 10s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      " v5-L6-D2048-E0_01-  61%[===========>        ] 629.82M  42.6MB/s    eta 10s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "v5-L6-D2048-E0_01-s  63%[===========>        ] 640.87M  42.6MB/s    eta 10s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "5-L6-D2048-E0_01-sp  64%[===========>        ] 656.13M  45.6MB/s    eta 10s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-L6-D2048-E0_01-spl  66%[============>       ] 671.38M  45.8MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "L6-D2048-E0_01-spli  67%[============>       ] 686.64M  47.1MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "6-D2048-E0_01-split  69%[============>       ] 701.90M  47.0MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-D2048-E0_01-split-  70%[=============>      ] 717.16M  46.9MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "D2048-E0_01-split-2  71%[=============>      ] 730.60M  47.8MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "2048-E0_01-split-2b  73%[=============>      ] 747.17M  45.9MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "048-E0_01-split-2b.  74%[=============>      ] 755.98M  45.7MB/s    eta 7s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "48-E0_01-split-2b.p  75%[==============>     ] 762.94M  43.2MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "8-E0_01-split-2b.pt  76%[==============>     ] 777.68M  45.4MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-E0_01-split-2b.pth  76%[==============>     ] 778.32M  42.7MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "E0_01-split-2b.pth   78%[==============>     ] 793.46M  42.0MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "0_01-split-2b.pth    79%[==============>     ] 808.20M  41.7MB/s    eta 5s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "_01-split-2b.pth     80%[===============>    ] 814.09M  42.3MB/s    eta 5s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "01-split-2b.pth      80%[===============>    ] 823.46M  41.0MB/s    eta 5s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "1-split-2b.pth       81%[===============>    ] 823.97M  40.5MB/s    eta 5s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-split-2b.pth        82%[===============>    ] 838.71M  38.6MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "split-2b.pth         83%[===============>    ] 853.98M  41.4MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "plit-2b.pth          84%[===============>    ] 854.61M  38.0MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "lit-2b.pth           85%[================>   ] 869.24M  35.7MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "it-2b.pth            85%[================>   ] 869.75M  35.3MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "t-2b.pth             86%[================>   ] 875.74M  34.3MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-2b.pth              87%[================>   ] 885.01M  32.5MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "2b.pth               88%[================>   ] 900.27M  33.8MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "b.pth                89%[================>   ] 913.70M  34.5MB/s    eta 3s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      ".pth                 90%[=================>  ] 924.21M  34.8MB/s    eta 3s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "pth                  91%[=================>  ] 930.27M  35.3MB/s    eta 3s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "th                   93%[=================>  ] 946.04M  34.9MB/s    eta 3s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "h                    94%[=================>  ] 961.30M  37.2MB/s    eta 1s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                     95%[==================> ] 970.14M  35.7MB/s    eta 1s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                  v  96%[==================> ] 976.55M  34.9MB/s    eta 1s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                 v5  97%[==================> ] 991.82M  37.0MB/s    eta 1s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                v5-  98%[==================> ] 998.13M  35.6MB/s    eta 1s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "               v5-L  98%[==================> ]   1007M  37.2MB/s    eta 0s     "
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "200 OK\r\n",
-      "Length: 44360 (43K) [text/html]\r\n",
-      "Saving to: ‘v5-L6-D2048-E0_01-split-2b.pth’\r\n",
-      "\r\n",
       "\r",
-      "          v5-L6-D20   0%[                    ]       0  --.-KB/s               \r",
-      "v5-L6-D2048-E0_01-s 100%[===================>]  43.32K  --.-KB/s    in 0s      \r\n",
+      "              v5-L6  99%[==================> ]   1016M  34.9MB/s    eta 0s     \r",
+      "v5-L6-D2048-E0_01-s 100%[===================>]   1017M  35.1MB/s    in 26s     \r\n",
       "\r\n",
-      "2023-10-11 07:03:02 (215 MB/s) - ‘v5-L6-D2048-E0_01-split-2b.pth’ saved [44360/44360]\r\n",
+      "2023-10-11 08:03:19 (38.9 MB/s) - ‘v5-L6-D2048-E0_01-split-2b.pth’ saved [1066536657/1066536657]\r\n",
       "\r\n"
      ]
     }
    ],
    "source": [
     "# Get the init split model, and finetune from there\n",
-    "!cd \"{PROJECT_DIR}/model/\" && wget -nc \"https://huggingface.co/rwkv-x-dev/rwkv-x-playground/blob/main/experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-E0_01-split-2a.pth\"\n",
-    "!cd \"{PROJECT_DIR}/model/\" && wget -nc \"https://huggingface.co/rwkv-x-dev/rwkv-x-playground/blob/main/experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-E0_01-split-2b.pth\""
+    "!cd \"{PROJECT_DIR}/model/\" && wget -nc \"https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/0600b94a58219f658326b4792ef5cd020e9d1a43/experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-E0_01-split-2a.pth\"\n",
+    "!cd \"{PROJECT_DIR}/model/\" && wget -nc \"https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/0600b94a58219f658326b4792ef5cd020e9d1a43/experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-E0_01-split-2b.pth\""
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 4,
-   "id": "81e32bce",
+   "id": "2a3cd2d1",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-11T07:03:02.594064Z",
-     "iopub.status.busy": "2023-10-11T07:03:02.592869Z",
-     "iopub.status.idle": "2023-10-11T07:03:12.284528Z",
-     "shell.execute_reply": "2023-10-11T07:03:12.283195Z"
+     "iopub.execute_input": "2023-10-11T08:03:19.666619Z",
+     "iopub.status.busy": "2023-10-11T08:03:19.665958Z",
+     "iopub.status.idle": "2023-10-11T08:03:29.305787Z",
+     "shell.execute_reply": "2023-10-11T08:03:29.304873Z"
     },
     "papermill": {
-     "duration": 9.699478,
-     "end_time": "2023-10-11T07:03:12.286978",
+     "duration": 9.658186,
+     "end_time": "2023-10-11T08:03:29.308744",
      "exception": false,
-     "start_time": "2023-10-11T07:03:02.587500",
+     "start_time": "2023-10-11T08:03:19.650558",
      "status": "completed"
     },
     "tags": []
@@ -250,7 +1818,7 @@
      "output_type": "stream",
      "text": [
       "\r",
-      "Saving the dataset (0/2 shards):   7%| | 2000/27200 [00:00<00:01, 16059.79 examp"
+      "Saving the dataset (0/2 shards):   7%| | 2000/27200 [00:00<00:01, 16356.85 examp"
      ]
     },
     {
@@ -258,7 +1826,7 @@
      "output_type": "stream",
      "text": [
       "\r",
-      "Saving the dataset (0/2 shards):  15%|▏| 4000/27200 [00:00<00:01, 17165.62 examp"
+      "Saving the dataset (0/2 shards):  15%|▏| 4000/27200 [00:00<00:01, 17283.77 examp"
      ]
     },
     {
@@ -266,7 +1834,7 @@
      "output_type": "stream",
      "text": [
       "\r",
-      "Saving the dataset (0/2 shards):  22%|▏| 6000/27200 [00:00<00:01, 17873.84 examp"
+      "Saving the dataset (0/2 shards):  22%|▏| 6000/27200 [00:00<00:01, 17873.97 examp"
      ]
     },
     {
@@ -274,7 +1842,7 @@
      "output_type": "stream",
      "text": [
       "\r",
-      "Saving the dataset (0/2 shards):  29%|▎| 8000/27200 [00:00<00:01, 18442.04 examp"
+      "Saving the dataset (0/2 shards):  29%|▎| 8000/27200 [00:00<00:01, 18442.59 examp"
      ]
     },
     {
@@ -282,7 +1850,7 @@
      "output_type": "stream",
      "text": [
       "\r",
-      "Saving the dataset (0/2 shards):  40%|▍| 11000/27200 [00:00<00:00, 19214.27 exam"
+      "Saving the dataset (0/2 shards):  40%|▍| 11000/27200 [00:00<00:00, 19135.78 exam"
      ]
     },
     {
@@ -290,8 +1858,8 @@
      "output_type": "stream",
      "text": [
       "\r",
-      "Saving the dataset (0/2 shards):  50%|▌| 13600/27200 [00:00<00:00, 19584.96 exam\r",
-      "Saving the dataset (1/2 shards):  50%|▌| 13600/27200 [00:00<00:00, 19584.96 exam"
+      "Saving the dataset (0/2 shards):  50%|▌| 13600/27200 [00:00<00:00, 19543.92 exam\r",
+      "Saving the dataset (1/2 shards):  50%|▌| 13600/27200 [00:00<00:00, 19543.92 exam"
      ]
     },
     {
@@ -299,7 +1867,7 @@
      "output_type": "stream",
      "text": [
       "\r",
-      "Saving the dataset (1/2 shards):  65%|▋| 17600/27200 [00:00<00:00, 20456.63 exam"
+      "Saving the dataset (1/2 shards):  65%|▋| 17600/27200 [00:00<00:00, 20515.42 exam"
      ]
     },
     {
@@ -307,7 +1875,7 @@
      "output_type": "stream",
      "text": [
       "\r",
-      "Saving the dataset (1/2 shards):  79%|▊| 21600/27200 [00:01<00:00, 21374.91 exam"
+      "Saving the dataset (1/2 shards):  79%|▊| 21600/27200 [00:01<00:00, 21426.57 exam"
      ]
     },
     {
@@ -315,7 +1883,7 @@
      "output_type": "stream",
      "text": [
       "\r",
-      "Saving the dataset (1/2 shards):  94%|▉| 25600/27200 [00:01<00:00, 21799.50 exam"
+      "Saving the dataset (1/2 shards):  94%|▉| 25600/27200 [00:01<00:00, 22078.81 exam"
      ]
     },
     {
@@ -323,18 +1891,12 @@
      "output_type": "stream",
      "text": [
       "\r",
-      "Saving the dataset (2/2 shards): 100%|█| 27200/27200 [00:01<00:00, 21799.50 exam\r",
-      "Saving the dataset (2/2 shards): 100%|█| 27200/27200 [00:01<00:00, 20461.58 exam\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
+      "Saving the dataset (2/2 shards): 100%|█| 27200/27200 [00:01<00:00, 22078.81 exam\r",
+      "Saving the dataset (2/2 shards): 100%|█| 27200/27200 [00:01<00:00, 20603.99 exam\r\n",
       "\r",
       "Saving the dataset (0/1 shards):   0%|           | 0/109 [00:00<?, ? examples/s]\r",
-      "Saving the dataset (1/1 shards): 100%|█| 109/109 [00:00<00:00, 7981.06 examples/\r",
-      "Saving the dataset (1/1 shards): 100%|█| 109/109 [00:00<00:00, 7683.94 examples/\r\n"
+      "Saving the dataset (1/1 shards): 100%|█| 109/109 [00:00<00:00, 8117.24 examples/\r",
+      "Saving the dataset (1/1 shards): 100%|█| 109/109 [00:00<00:00, 7809.82 examples/\r\n"
      ]
     }
    ],
@@ -346,13 +1908,13 @@
   },
   {
    "cell_type": "markdown",
-   "id": "5bf12f9e",
+   "id": "77d1d3e8",
    "metadata": {
     "papermill": {
-     "duration": 0.00368,
-     "end_time": "2023-10-11T07:03:12.294651",
+     "duration": 0.016656,
+     "end_time": "2023-10-11T08:03:29.342825",
      "exception": false,
-     "start_time": "2023-10-11T07:03:12.290971",
+     "start_time": "2023-10-11T08:03:29.326169",
      "status": "completed"
     },
     "tags": []
@@ -364,19 +1926,19 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "id": "c1566af8",
+   "id": "42cb403e",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-11T07:03:12.305523Z",
-     "iopub.status.busy": "2023-10-11T07:03:12.304583Z",
-     "iopub.status.idle": "2023-10-11T07:03:29.051517Z",
-     "shell.execute_reply": "2023-10-11T07:03:29.049989Z"
+     "iopub.execute_input": "2023-10-11T08:03:29.379159Z",
+     "iopub.status.busy": "2023-10-11T08:03:29.378428Z",
+     "iopub.status.idle": "2023-10-11T08:03:46.935627Z",
+     "shell.execute_reply": "2023-10-11T08:03:46.934802Z"
     },
     "papermill": {
-     "duration": 16.755429,
-     "end_time": "2023-10-11T07:03:29.053992",
+     "duration": 17.577903,
+     "end_time": "2023-10-11T08:03:46.937715",
      "exception": false,
-     "start_time": "2023-10-11T07:03:12.298563",
+     "start_time": "2023-10-11T08:03:29.359812",
      "status": "completed"
     },
     "tags": []
@@ -386,7 +1948,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2023-10-11 07:03:16,508] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+      "[2023-10-11 08:03:33,838] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
      ]
     },
     {
@@ -408,9 +1970,9 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/usr/local/lib/python3.10/dist-packages/lightning/fabric/utilities/seed.py:39: UserWarning: No seed found, seed set to 3015291597\r\n",
+      "/usr/local/lib/python3.10/dist-packages/lightning/fabric/utilities/seed.py:39: UserWarning: No seed found, seed set to 1933922385\r\n",
       "  rank_zero_warn(f\"No seed found, seed set to {seed}\")\r\n",
-      "Global seed set to 3015291597\r\n"
+      "Global seed set to 1933922385\r\n"
      ]
     },
     {
@@ -425,11 +1987,11 @@
      "output_type": "stream",
      "text": [
       "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.15.12\r\n",
-      "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m./wandb/run-20231011_070319-yi5f0p8v\u001b[0m\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m./wandb/run-20231011_080337-5696uouo\u001b[0m\r\n",
       "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\r\n",
       "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33m[Multi-size] v5-L6+6-D2048-E0.01 - layer-expansion A3 (train-ctx=4k, deepspeed_stage_2_offload)\u001b[0m\r\n",
       "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments\u001b[0m\r\n",
-      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/runs/yi5f0p8v\u001b[0m\r\n"
+      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/runs/5696uouo\u001b[0m\r\n"
      ]
     },
     {
@@ -471,10 +2033,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33m[Multi-size] v5-L6+6-D2048-E0.01 - layer-expansion A3 (train-ctx=4k, deepspeed_stage_2_offload)\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/runs/yi5f0p8v\u001b[0m\r\n",
-      "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjk0OTk4MDcy/version_details/v15\u001b[0m\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33m[Multi-size] v5-L6+6-D2048-E0.01 - layer-expansion A3 (train-ctx=4k, deepspeed_stage_2_offload)\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/runs/5696uouo\u001b[0m\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjk0OTk4MDcy/version_details/v16\u001b[0m\r\n",
       "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)\r\n",
-      "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231011_070319-yi5f0p8v/logs\u001b[0m\r\n"
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231011_080337-5696uouo/logs\u001b[0m\r\n"
      ]
     }
    ],
@@ -496,19 +2058,19 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "id": "d56d6a30",
+   "id": "53867c42",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-11T07:03:29.066944Z",
-     "iopub.status.busy": "2023-10-11T07:03:29.065449Z",
-     "iopub.status.idle": "2023-10-11T07:03:32.793675Z",
-     "shell.execute_reply": "2023-10-11T07:03:32.792045Z"
+     "iopub.execute_input": "2023-10-11T08:03:46.969471Z",
+     "iopub.status.busy": "2023-10-11T08:03:46.969019Z",
+     "iopub.status.idle": "2023-10-11T08:03:50.682437Z",
+     "shell.execute_reply": "2023-10-11T08:03:50.680986Z"
     },
     "papermill": {
-     "duration": 3.737581,
-     "end_time": "2023-10-11T07:03:32.796266",
+     "duration": 3.732808,
+     "end_time": "2023-10-11T08:03:50.685581",
      "exception": false,
-     "start_time": "2023-10-11T07:03:29.058685",
+     "start_time": "2023-10-11T08:03:46.952773",
      "status": "completed"
     },
     "tags": []
@@ -518,7 +2080,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2023-10-11 07:03:31,388] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+      "[2023-10-11 08:03:49,278] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
      ]
     },
     {
@@ -553,19 +2115,19 @@
   {
    "cell_type": "code",
    "execution_count": 7,
-   "id": "f4c37d28",
+   "id": "5688e577",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-11T07:03:32.809297Z",
-     "iopub.status.busy": "2023-10-11T07:03:32.808263Z",
-     "iopub.status.idle": "2023-10-11T07:03:39.200791Z",
-     "shell.execute_reply": "2023-10-11T07:03:39.199726Z"
+     "iopub.execute_input": "2023-10-11T08:03:50.806267Z",
+     "iopub.status.busy": "2023-10-11T08:03:50.804997Z",
+     "iopub.status.idle": "2023-10-11T08:03:56.788036Z",
+     "shell.execute_reply": "2023-10-11T08:03:56.786568Z"
     },
     "papermill": {
-     "duration": 6.402526,
-     "end_time": "2023-10-11T07:03:39.203696",
+     "duration": 6.08675,
+     "end_time": "2023-10-11T08:03:56.790510",
      "exception": false,
-     "start_time": "2023-10-11T07:03:32.801170",
+     "start_time": "2023-10-11T08:03:50.703760",
      "status": "completed"
     },
     "tags": []
@@ -575,14 +2137,20 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2023-10-11 07:03:37,133] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+      "[2023-10-11 08:03:54,934] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n",
       "Traceback (most recent call last):\r\n",
       "  File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/dragon_test.py\", line 52, in <module>\r\n",
       "    model = SimpleRWKV(MODEL_PATH, device=DEVICE)\r\n",
@@ -602,13 +2170,13 @@
   },
   {
    "cell_type": "markdown",
-   "id": "39b5c9e4",
+   "id": "b4927e87",
    "metadata": {
     "papermill": {
-     "duration": 0.006891,
-     "end_time": "2023-10-11T07:03:39.217738",
+     "duration": 0.015295,
+     "end_time": "2023-10-11T08:03:56.820640",
      "exception": false,
-     "start_time": "2023-10-11T07:03:39.210847",
+     "start_time": "2023-10-11T08:03:56.805345",
      "status": "completed"
     },
     "tags": []
@@ -620,19 +2188,19 @@
   {
    "cell_type": "code",
    "execution_count": 8,
-   "id": "6a2803dc",
+   "id": "6bdd285a",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-11T07:03:39.235541Z",
-     "iopub.status.busy": "2023-10-11T07:03:39.234393Z",
-     "iopub.status.idle": "2023-10-11T07:03:54.515077Z",
-     "shell.execute_reply": "2023-10-11T07:03:54.513988Z"
+     "iopub.execute_input": "2023-10-11T08:03:56.853495Z",
+     "iopub.status.busy": "2023-10-11T08:03:56.852946Z",
+     "iopub.status.idle": "2023-10-11T08:04:11.500794Z",
+     "shell.execute_reply": "2023-10-11T08:04:11.499336Z"
     },
     "papermill": {
-     "duration": 15.292458,
-     "end_time": "2023-10-11T07:03:54.517573",
+     "duration": 14.668001,
+     "end_time": "2023-10-11T08:04:11.503644",
      "exception": false,
-     "start_time": "2023-10-11T07:03:39.225115",
+     "start_time": "2023-10-11T08:03:56.835643",
      "status": "completed"
     },
     "tags": []
@@ -642,7 +2210,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2023-10-11 07:03:43,703] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+      "[2023-10-11 08:04:01,096] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
      ]
     },
     {
@@ -664,9 +2232,9 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/usr/local/lib/python3.10/dist-packages/lightning/fabric/utilities/seed.py:39: UserWarning: No seed found, seed set to 736269213\r\n",
+      "/usr/local/lib/python3.10/dist-packages/lightning/fabric/utilities/seed.py:39: UserWarning: No seed found, seed set to 1732922148\r\n",
       "  rank_zero_warn(f\"No seed found, seed set to {seed}\")\r\n",
-      "Global seed set to 736269213\r\n"
+      "Global seed set to 1732922148\r\n"
      ]
     },
     {
@@ -681,11 +2249,11 @@
      "output_type": "stream",
      "text": [
       "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.15.12\r\n",
-      "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m./wandb/run-20231011_070346-9urskvqu\u001b[0m\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m./wandb/run-20231011_080403-88lcuk7j\u001b[0m\r\n",
       "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\r\n",
       "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33m[Multi-size] v5-L6+6-D2048-E0.01 - layer-expansion B3 (train-ctx=4k, deepspeed_stage_2_offload)\u001b[0m\r\n",
       "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments\u001b[0m\r\n",
-      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/runs/9urskvqu\u001b[0m\r\n"
+      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/runs/88lcuk7j\u001b[0m\r\n"
      ]
     },
     {
@@ -727,10 +2295,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33m[Multi-size] v5-L6+6-D2048-E0.01 - layer-expansion B3 (train-ctx=4k, deepspeed_stage_2_offload)\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/runs/9urskvqu\u001b[0m\r\n",
-      "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjk0OTk4MDcy/version_details/v15\u001b[0m\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33m[Multi-size] v5-L6+6-D2048-E0.01 - layer-expansion B3 (train-ctx=4k, deepspeed_stage_2_offload)\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/runs/88lcuk7j\u001b[0m\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-5X-Experiments/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjk0OTk4MDcy/version_details/v16\u001b[0m\r\n",
       "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\r\n",
-      "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231011_070346-9urskvqu/logs\u001b[0m\r\n"
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231011_080403-88lcuk7j/logs\u001b[0m\r\n"
      ]
     }
    ],
@@ -752,19 +2320,19 @@
   {
    "cell_type": "code",
    "execution_count": 9,
-   "id": "9f50e589",
+   "id": "ae4623a1",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-11T07:03:54.533944Z",
-     "iopub.status.busy": "2023-10-11T07:03:54.532500Z",
-     "iopub.status.idle": "2023-10-11T07:03:58.532096Z",
-     "shell.execute_reply": "2023-10-11T07:03:58.530934Z"
+     "iopub.execute_input": "2023-10-11T08:04:11.546046Z",
+     "iopub.status.busy": "2023-10-11T08:04:11.544870Z",
+     "iopub.status.idle": "2023-10-11T08:04:15.274349Z",
+     "shell.execute_reply": "2023-10-11T08:04:15.272957Z"
     },
     "papermill": {
-     "duration": 4.010668,
-     "end_time": "2023-10-11T07:03:58.534788",
+     "duration": 3.754115,
+     "end_time": "2023-10-11T08:04:15.277163",
      "exception": false,
-     "start_time": "2023-10-11T07:03:54.524120",
+     "start_time": "2023-10-11T08:04:11.523048",
      "status": "completed"
     },
     "tags": []
@@ -774,7 +2342,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2023-10-11 07:03:57,001] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+      "[2023-10-11 08:04:13,869] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
      ]
     },
     {
@@ -809,19 +2377,19 @@
   {
    "cell_type": "code",
    "execution_count": 10,
-   "id": "777a63eb",
+   "id": "8e1b1152",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-10-11T07:03:58.550672Z",
-     "iopub.status.busy": "2023-10-11T07:03:58.550018Z",
-     "iopub.status.idle": "2023-10-11T07:04:05.053290Z",
-     "shell.execute_reply": "2023-10-11T07:04:05.052230Z"
+     "iopub.execute_input": "2023-10-11T08:04:15.319747Z",
+     "iopub.status.busy": "2023-10-11T08:04:15.318636Z",
+     "iopub.status.idle": "2023-10-11T08:04:21.268526Z",
+     "shell.execute_reply": "2023-10-11T08:04:21.267073Z"
     },
     "papermill": {
-     "duration": 6.514677,
-     "end_time": "2023-10-11T07:04:05.056026",
+     "duration": 5.974644,
+     "end_time": "2023-10-11T08:04:21.271495",
      "exception": false,
-     "start_time": "2023-10-11T07:03:58.541349",
+     "start_time": "2023-10-11T08:04:15.296851",
      "status": "completed"
     },
     "tags": []
@@ -831,7 +2399,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2023-10-11 07:04:02,965] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+      "[2023-10-11 08:04:19,430] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
      ]
     },
     {
@@ -877,14 +2445,14 @@
   },
   "papermill": {
    "default_parameters": {},
-   "duration": 66.115721,
-   "end_time": "2023-10-11T07:04:05.485131",
+   "duration": 119.315066,
+   "end_time": "2023-10-11T08:04:21.714050",
    "environment_variables": {},
    "exception": null,
    "input_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/multi-size-train/v5-L6+6-D2048-layer-baseline.ipynb",
    "output_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/rwkv-x-exp/multi-size-train/v5-L6+6-D2048-layer-baseline.ipynb",
    "parameters": {},
-   "start_time": "2023-10-11T07:02:59.369410",
+   "start_time": "2023-10-11T08:02:22.398984",
    "version": "2.4.0"
   }
  },