{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "YlId6SITh8ph" }, "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "TZHREFURkx9F", "outputId": "3c586d9b-5250-45b4-b205-18cedf2cbb7e" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/\n" ] } ], "source": [] }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "kUv_Qy_dh-Ik", "outputId": "5cdf5f76-702d-4c88-dff2-c781eda3f4ad" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Cloning into 'alpaca-lora'...\n", "remote: Enumerating objects: 299, done.\u001b[K\n", "remote: Counting objects: 100% (142/142), done.\u001b[K\n", "remote: Compressing objects: 100% (45/45), done.\u001b[K\n", "remote: Total 299 (delta 121), reused 101 (delta 97), pack-reused 157\u001b[K\n", "Receiving objects: 100% (299/299), 7.03 MiB | 4.60 MiB/s, done.\n", "Resolving deltas: 100% (183/183), done.\n" ] } ], "source": [ "# %%\n", "!git clone https://github.com/tloen/alpaca-lora.git" ] }, { "cell_type": "markdown", "metadata": { "id": "g5PKCTa_iAUq" }, "source": [ "\n", "\n", "# # Alpaca GPT-2" ] }, { "cell_type": "code", "source": [ "\n", "%cd alpaca-lora/" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "e14PzovIEZ2B", "outputId": "9fc6b6d1-d32f-4b71-9e7d-575ab610f47b" }, "execution_count": 2, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "/content/alpaca-lora\n" ] } ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 761, "referenced_widgets": [ "7835ab6d7dc94fcf827a2db768e42f64", "bd23dedad5fd4558a6dd26c4baef5f60", "747b8914e424424682938e2b773b2424", "e3cb2a03e1894b47aab02c6bdfd2c46c", "2e6b0bc5907b4afb8633360c462db5dc", "be33258d7dc74f718eb972cfe64094c1", "f6880ecdbe464ed8926ee94d2bc03434", "f814014360354c39aa22ee9432784711", "3de3c85aa03d49a0961427eb4d8007d3", "5f50878806544040bf8cedb65625c97f", "743969dd6c5044acb238b612ccda8207", "0b7b93d51bdc4772b05b2a05636bd867", "ab920e7ac8b044bc83bcd90610e82f09", "10fd5b804eeb469a92362b3c7f0a7785", "bb59c5a8e64c4742bfb7167b9348bbf0", "c1dc7e54dc294502bacfd939591e4264", "370f1847ccbb416bad6ee31304bd70ee", "d6fab8e142a94269b62c72e2afcd8d86", "e59dd8bf2fe74d38a0c50c5f54b9b890", "da783e32bdf8499dbc2d33d4141393f0", "f6101f65e621458a822bf2940ac97c53", "91b1c270297d4092bbe07447f73e0c5c", "1c0acefbc4244f65a343836af9f27a77", "bd1853bc45734deba223a5c65884f9ea", "ab4f2cb50603478daf0c196363a6f758", "64a9280d1efd4d26b74ea79077396902", "f86f6773a8304dc2977308c74e77f931", "e8ef75e6cbb243cd96cc4d276ca9db46", "aa1cc35c892741cc9fc9a7fbad3895fa", "5a122f6f4d89469db98ef7a4ef83e777", "f82d4e07f3a0401289cac609d413f5e1", "77d01b46b79c490ba3fed0e0a144716a", "9fae0f6cdb5742f59c871eecb87d793a", "d72af3b7a45844d994636329650dd31d", "35fd6d3ba6f042d2a01e7d3087d9dca4", "3d2b38d307ed45c4bb1472f42434cd63", "90bcc3205c1249acba4bab259c031c80", "4d7e746c93f54e3bac04291b279cafa0", "215fa5ccf15a4a209f96deabb9108945", "5165d7fce34a4edcbcee87803b728f47", "78ded35473614809b16e88fbeb8cd7ad", "f7a8b121c8ea4dc9b41152fdd5c9d82f", "999535754baf47fb872d8fd1d6c5f25f", "f76d0fe6035e4a49b72df55e7f1bbd72", "1cf5db6f3e374114943772e03cd105d3", "c35e4c5b986240e4ae5bacf1a53e0435", "0d29021342fa41bd91a62588241fe46a", "abaffca3dac64c50b92b7b6619e3ca22", "0f61cd3fe41a47e4928893c38731a6bc", "5a25f7bea03e46e394f6732dd52d5bcf", "c05cd3d33a9a41588cf3a1e37150f97d", "871c5bd776fb4cc6a5c5e8068ca64663", "028a95eba62e4abe912704eacd6c03e8", "f473162db812457483bb491551ad1c1a", "2c0f59c2d01c44f8964142a3ec467bf7", "a54a06b520de4915a32461519f6e1cd9", "9a4111beb2ef48f18c19ba517b59fc4f", "a649a8aec94f42b38cc6e0262f533c69", "3fcfe7f7eb0a42c1bb44b99e0cd806bb", "87a19cb9c5ba4af3999f69e827dd5a58", "a0e05140ae3b4859abb903f2af376ea7", "830b9f2ed10e47a68c5014c3e7d3c35f", "2fc3c75571994991b53e592f05160761", "0f589865affc4a3bac3f8c7ba931e250", "f4262dc10430430eb0e6bc923d74f146", "dcf4a76576ab4288a6b5c644f1edbe14", "a21fa7af187a453cb811b88b83befd34", "22790a55219b451996133241727cab53", "e67e75116f9c49ce9337a23faa9dce22", "0689b463665a4f4abbbe3ae3b8fec626", "76c0d1a267304833a45bc8c84112b4fb", "38b742d0a6e54d55ac2b9b5050c36d1b", "6e2f974958084b6e92f058d26c0bde4e", "fa35b7bd2ae845eb9cf423b319c441de", "38949a955f5540889494e8bcac8ccce4", "6cce161febe34394819bba5c4f98c0f5", "11a4facaa5354a79bf6b579e71d5cfe7", "7ef1e0d0e99e4d22858afda2be74f4ca", "05cdf6b1bce7417785beba03f065b703", "13ba97c2740b4842a1ba1699614f3084", "348bdf07221a4720956e2ae8be252582", "1a7b7d8371034679b4de849442533e00", "7998c13b2c5f40fc9796b770df4cf290", "6fa33d85855d4cc8bb1f7cd548bf8104", "e885a6f3ff214c4f986c721f84c12f21", "fe9d1c6888df4c0094abee1308c7baa2", "f3f95c8e9376462b9b59de8c8fae0639", "ae287ac3260f4d4c82a417cfba5bd39b", "4656173c09d148ff89c2e3ed9b54a39a", "2a2a43a9b6584146b4f32152a8ee1e4d", "b8596d6499774706a0068817053bd269", "7aa8dd9240cf47b5918f87a35000c498", "a9b5512ccccd408ca37ccf33a0e44400", "6bc2093529194b4bbeddf4845d6b0781", "5849302c40fa483f934de4bd63042865", "f6b982ceab4c43f0b44b6bfe914a58b2", "7db5982828bd4a80956748f54c5987b2", "29cc7c2437bf439ba6f79b232cc97f1c", "f35bc5cf2ac049e6845022787aeafac6", "830cf960e2d246f0bde513cb8c16c79a", "e4fe0b4d3f83485eaa27600c2e1580bf", "dbc41a6eb86d4059938fe34fac2f7823", "cd951725c55e4d29b3931041f02f8d07", "24383ff3c495487f927f1ecabbbcb46a", "16aba3fd042c46a290637dc17bfba2c0", "459d0ea184c24b7a84f88e87a567ad5f", "97c0898379784bca9ecc7ae9a22dfbec", "1507b30e58c34f57876c3bda6c5fbabc", "a4eb41820ded42bcb0304c0f01191694", "8e7e8bc9bafd4ca4acfc8c394ed4cc24", "3ac03458dcc445469dab9bcf3d852772", "dfdd78dea97c4ada92b2e83ddac50b8a", "6ed50699fb3642fa8c3b235f791266f3", "feb17425ac214c53b5aacb165d7c972e", "0e44a3165c8c488daea79f2f0d4fafa2", "f573b113099749499408d325cddac3e0", "bc24f726c3bb44a694b2d2ee481c5e6d", "cdfebac007d642c7801937d786e8aa68", "393ccca6d8344d2f916a5062f38f0fb3", "474658409ff747cdb64fea6472d8b4a2", "18e3ab8d95524a09a60b801585c79a8c", "97789e00e16d45868c909a5d3d2fa72d", "54a9eb3ada0245278cfbb37c816403a4", "1e325ea493bf4aa0b2dd2e05cff39cac", "3d9391d613754bc18754d95467a94c6e", "b158db54ce75444dbd02624b3a6304bf", "9a1ca2a58e1440009f196dd8ba67114b", "989365f8a17b4c0f9588f65e8f83f41b", "9b1316ac60144ec08e1a2344edb71595", "1d2021f0bca3424f829b6fc2bd47704d", "a25a01beb86845b3bebf69e97861b470", "06bce5bb02fc4361852342f54a519611" ] }, "id": "jTwxXULdh15N", "outputId": "df1ddfbf-c90d-4e21-f6a9-67290eb2fa2c" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m469.0/469.0 KB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m79.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.2/212.2 KB\u001b[0m \u001b[31m24.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m68.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 KB\u001b[0m \u001b[31m21.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.9/132.9 KB\u001b[0m \u001b[31m17.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 KB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m100.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 KB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.6/264.6 KB\u001b[0m \u001b[31m22.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 KB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h" ] }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading (…)olve/main/vocab.json: 0%| | 0.00/1.04M [00:00, ?B/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "7835ab6d7dc94fcf827a2db768e42f64" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading (…)olve/main/merges.txt: 0%| | 0.00/456k [00:00, ?B/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "0b7b93d51bdc4772b05b2a05636bd867" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading (…)lve/main/config.json: 0%| | 0.00/665 [00:00, ?B/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "1c0acefbc4244f65a343836af9f27a77" } }, "metadata": {} }, { "output_type": "stream", "name": "stdout", "text": [ "Downloading and preparing dataset json/default to /root/.cache/huggingface/datasets/json/default-1db1379af983bf10/0.0.0/0f7e3662623656454fcd2b650f34e886a7db4b9104504885bd462096cc7a9f51...\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading data files: 0%| | 0/1 [00:00, ?it/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "d72af3b7a45844d994636329650dd31d" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "Extracting data files: 0%| | 0/1 [00:00, ?it/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "1cf5db6f3e374114943772e03cd105d3" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "Generating train split: 0 examples [00:00, ? examples/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "a54a06b520de4915a32461519f6e1cd9" } }, "metadata": {} }, { "output_type": "stream", "name": "stdout", "text": [ "Dataset json downloaded and prepared to /root/.cache/huggingface/datasets/json/default-1db1379af983bf10/0.0.0/0f7e3662623656454fcd2b650f34e886a7db4b9104504885bd462096cc7a9f51. Subsequent calls will reuse this data.\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ " 0%| | 0/1 [00:00, ?it/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "a21fa7af187a453cb811b88b83befd34" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "Map: 0%| | 0/52002 [00:00, ? examples/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "7ef1e0d0e99e4d22858afda2be74f4ca" } }, "metadata": {} }, { "output_type": "stream", "name": "stderr", "text": [ "Token indices sequence length is longer than the specified maximum sequence length for this model (1044 > 1024). Running this sequence through the model will result in indexing errors\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading pytorch_model.bin: 0%| | 0.00/548M [00:00, ?B/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "4656173c09d148ff89c2e3ed9b54a39a" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading (…)neration_config.json: 0%| | 0.00/124 [00:00, ?B/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "830cf960e2d246f0bde513cb8c16c79a" } }, "metadata": {} }, { "output_type": "stream", "name": "stderr", "text": [ "WARNING:datasets.builder:Found cached dataset json (/root/.cache/huggingface/datasets/json/default-1db1379af983bf10/0.0.0/0f7e3662623656454fcd2b650f34e886a7db4b9104504885bd462096cc7a9f51)\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ " 0%| | 0/1 [00:00, ?it/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "3ac03458dcc445469dab9bcf3d852772" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "Map: 0%| | 0/52002 [00:00, ? examples/s]" ], "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "97789e00e16d45868c909a5d3d2fa72d" } }, "metadata": {} } ], "source": [ "\n", "\n", "\n", "\n", "# %%\n", "!pip install -q datasets transformers\n", "\n", "# %%\n", "from datasets import load_dataset\n", "from transformers import GPT2Tokenizer\n", "\n", "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\", add_special_tokens=True)\n", "tokenizer.pad_token = tokenizer.eos_token\n", "tokenizer.pad_token_id = tokenizer.eos_token_id\n", "\n", "data = load_dataset(\"json\", data_files=\"alpaca_data.json\")\n", "\n", "def generate_prompt(data_point):\n", " if data_point[\"instruction\"]:\n", " return f\"\"\"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "{data_point[\"instruction\"]}\n", "\n", "### Input:\n", "{data_point[\"input\"]}\n", "\n", "### Response:\n", "{data_point[\"output\"]}\"\"\"\n", " else:\n", " return f\"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "{data_point[\"instruction\"]}\n", "\n", "### Response:\n", "{data_point[\"output\"]}\"\"\"\n", "\n", "data = data.map(lambda data_point: {\"prompt\": tokenizer(generate_prompt(data_point))})\n", "\n", "# Fine Tuning\n", "import torch\n", "from datasets import load_dataset\n", "import transformers\n", "from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Config, Trainer, TrainingArguments\n", "\n", "# %%\n", "MODEL_NAME = \"gpt2\"\n", "MICRO_BATCH_SIZE = 8\n", "BATCH_SIZE = 128\n", "GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE\n", "EPOCHS = 1\n", "LEARNING_RATE = 2e-5\n", "CUTOFF_LEN = 256\n", "\n", "# %%\n", "model = GPT2LMHeadModel.from_pretrained(MODEL_NAME)\n", "tokenizer = GPT2Tokenizer.from_pretrained(MODEL_NAME, add_special_tokens=True)\n", "tokenizer.pad_token_id = tokenizer.eos_token_id\n", "\n", "data = load_dataset(\"json\", data_files=\"alpaca_data.json\")\n", "\n", "def generate_prompt(data_point):\n", " if data_point[\"input\"]:\n", " return f\"\"\"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n", "### Instruction:\n", "{data_point[\"instruction\"]}\n", "### Input:\n", "{data_point[\"input\"]}\n", "### Response:\n", "{data_point[\"output\"]}\"\"\"\n", " else:\n", " return f\"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "### Instruction:\n", "{data_point[\"instruction\"]}\n", "### Response:\n", "{data_point[\"output\"]}\"\"\"\n", "\n", "data = data.shuffle().map(\n", " lambda data_point: tokenizer(\n", " generate_prompt(data_point),\n", " truncation=True,\n", " max_length=CUTOFF_LEN,\n", " padding=\"max_length\",\n", " )\n", ")\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000 }, "id": "xqDOmTKMjBdO", "outputId": "50963e5b-21ec-4219-9f8d-dddaae76be05" }, "outputs": [ { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] }, { "data": { "text/html": [ "\n", "
Step | \n", "Training Loss | \n", "
---|---|
1 | \n", "3.505800 | \n", "
2 | \n", "3.589200 | \n", "
3 | \n", "3.581400 | \n", "
4 | \n", "3.499100 | \n", "
5 | \n", "3.446200 | \n", "
6 | \n", "3.463100 | \n", "
7 | \n", "3.523100 | \n", "
8 | \n", "3.504400 | \n", "
9 | \n", "3.536200 | \n", "
10 | \n", "3.503900 | \n", "
11 | \n", "3.407800 | \n", "
12 | \n", "3.393300 | \n", "
13 | \n", "3.435200 | \n", "
14 | \n", "3.501500 | \n", "
15 | \n", "3.379000 | \n", "
16 | \n", "3.408300 | \n", "
17 | \n", "3.385700 | \n", "
18 | \n", "3.351200 | \n", "
19 | \n", "3.319400 | \n", "
20 | \n", "3.318500 | \n", "
21 | \n", "3.174300 | \n", "
22 | \n", "3.284800 | \n", "
23 | \n", "3.165500 | \n", "
24 | \n", "3.162600 | \n", "
25 | \n", "3.222300 | \n", "
26 | \n", "3.156100 | \n", "
27 | \n", "3.112700 | \n", "
28 | \n", "2.980200 | \n", "
29 | \n", "2.966500 | \n", "
30 | \n", "2.978900 | \n", "
31 | \n", "2.943500 | \n", "
32 | \n", "2.979800 | \n", "
33 | \n", "2.897400 | \n", "
34 | \n", "2.844700 | \n", "
35 | \n", "2.779500 | \n", "
36 | \n", "2.761400 | \n", "
37 | \n", "2.734000 | \n", "
38 | \n", "2.656200 | \n", "
39 | \n", "2.685300 | \n", "
40 | \n", "2.639600 | \n", "
41 | \n", "2.586300 | \n", "
42 | \n", "2.522400 | \n", "
43 | \n", "2.478000 | \n", "
44 | \n", "2.511800 | \n", "
45 | \n", "2.516400 | \n", "
46 | \n", "2.412900 | \n", "
47 | \n", "2.380500 | \n", "
48 | \n", "2.263900 | \n", "
49 | \n", "2.313200 | \n", "
50 | \n", "2.256700 | \n", "
51 | \n", "2.231000 | \n", "
52 | \n", "2.109000 | \n", "
53 | \n", "2.242600 | \n", "
54 | \n", "2.147000 | \n", "
55 | \n", "2.127600 | \n", "
56 | \n", "2.075800 | \n", "
57 | \n", "2.053100 | \n", "
58 | \n", "2.060600 | \n", "
59 | \n", "2.026200 | \n", "
60 | \n", "2.050500 | \n", "
61 | \n", "2.001800 | \n", "
62 | \n", "1.939500 | \n", "
63 | \n", "2.017000 | \n", "
64 | \n", "1.955400 | \n", "
65 | \n", "1.983100 | \n", "
66 | \n", "1.888200 | \n", "
67 | \n", "1.942800 | \n", "
68 | \n", "1.955000 | \n", "
69 | \n", "1.830700 | \n", "
70 | \n", "1.889600 | \n", "
71 | \n", "1.918400 | \n", "
72 | \n", "1.819400 | \n", "
73 | \n", "1.848000 | \n", "
74 | \n", "1.874500 | \n", "
75 | \n", "1.846700 | \n", "
76 | \n", "1.925400 | \n", "
77 | \n", "1.741800 | \n", "
78 | \n", "1.873700 | \n", "
79 | \n", "1.889100 | \n", "
80 | \n", "1.823100 | \n", "
81 | \n", "1.782900 | \n", "
82 | \n", "1.819800 | \n", "
83 | \n", "1.854300 | \n", "
84 | \n", "1.802200 | \n", "
85 | \n", "1.778000 | \n", "
86 | \n", "1.755700 | \n", "
87 | \n", "1.844600 | \n", "
88 | \n", "1.778300 | \n", "
89 | \n", "1.803900 | \n", "
90 | \n", "1.841800 | \n", "
91 | \n", "1.763200 | \n", "
92 | \n", "1.800000 | \n", "
93 | \n", "1.804400 | \n", "
94 | \n", "1.804800 | \n", "
95 | \n", "1.754500 | \n", "
96 | \n", "1.796900 | \n", "
97 | \n", "1.848600 | \n", "
98 | \n", "1.746900 | \n", "
99 | \n", "1.703900 | \n", "
100 | \n", "1.775400 | \n", "
101 | \n", "1.773300 | \n", "
102 | \n", "1.748300 | \n", "
103 | \n", "1.740100 | \n", "
104 | \n", "1.795300 | \n", "
105 | \n", "1.850800 | \n", "
106 | \n", "1.828400 | \n", "
107 | \n", "1.732500 | \n", "
108 | \n", "1.841300 | \n", "
109 | \n", "1.774700 | \n", "
110 | \n", "1.814700 | \n", "
111 | \n", "1.702000 | \n", "
112 | \n", "1.734600 | \n", "
113 | \n", "1.790700 | \n", "
114 | \n", "1.635100 | \n", "
115 | \n", "1.805300 | \n", "
116 | \n", "1.776300 | \n", "
117 | \n", "1.855000 | \n", "
118 | \n", "1.778700 | \n", "
119 | \n", "1.743800 | \n", "
120 | \n", "1.793500 | \n", "
121 | \n", "1.743100 | \n", "
122 | \n", "1.752300 | \n", "
123 | \n", "1.797400 | \n", "
124 | \n", "1.755000 | \n", "
125 | \n", "1.679400 | \n", "
126 | \n", "1.724200 | \n", "
127 | \n", "1.759600 | \n", "
128 | \n", "1.793600 | \n", "
129 | \n", "1.723000 | \n", "
130 | \n", "1.817600 | \n", "
131 | \n", "1.759500 | \n", "
132 | \n", "1.719300 | \n", "
133 | \n", "1.709900 | \n", "
134 | \n", "1.696500 | \n", "
135 | \n", "1.736700 | \n", "
136 | \n", "1.713900 | \n", "
137 | \n", "1.689000 | \n", "
138 | \n", "1.691400 | \n", "
139 | \n", "1.707600 | \n", "
140 | \n", "1.714700 | \n", "
141 | \n", "1.778000 | \n", "
142 | \n", "1.759700 | \n", "
143 | \n", "1.746300 | \n", "
144 | \n", "1.669000 | \n", "
145 | \n", "1.716300 | \n", "
146 | \n", "1.769200 | \n", "
147 | \n", "1.669500 | \n", "
148 | \n", "1.747200 | \n", "
149 | \n", "1.730200 | \n", "
150 | \n", "1.714300 | \n", "
151 | \n", "1.665400 | \n", "
152 | \n", "1.822100 | \n", "
153 | \n", "1.733000 | \n", "
154 | \n", "1.740300 | \n", "
155 | \n", "1.650800 | \n", "
156 | \n", "1.637600 | \n", "
157 | \n", "1.729200 | \n", "
158 | \n", "1.719300 | \n", "
159 | \n", "1.689800 | \n", "
160 | \n", "1.681200 | \n", "
161 | \n", "1.750400 | \n", "
162 | \n", "1.706700 | \n", "
163 | \n", "1.762800 | \n", "
164 | \n", "1.646600 | \n", "
165 | \n", "1.660200 | \n", "
166 | \n", "1.701700 | \n", "
167 | \n", "1.765900 | \n", "
168 | \n", "1.605600 | \n", "
169 | \n", "1.772600 | \n", "
170 | \n", "1.814300 | \n", "
171 | \n", "1.759800 | \n", "
172 | \n", "1.718600 | \n", "
173 | \n", "1.639600 | \n", "
174 | \n", "1.713800 | \n", "
175 | \n", "1.755000 | \n", "
176 | \n", "1.771900 | \n", "
177 | \n", "1.714100 | \n", "
178 | \n", "1.737800 | \n", "
179 | \n", "1.724300 | \n", "
180 | \n", "1.674700 | \n", "
181 | \n", "1.639800 | \n", "
182 | \n", "1.661000 | \n", "
183 | \n", "1.736600 | \n", "
184 | \n", "1.655200 | \n", "
185 | \n", "1.765800 | \n", "
186 | \n", "1.707400 | \n", "
187 | \n", "1.705600 | \n", "
188 | \n", "1.686100 | \n", "
189 | \n", "1.716100 | \n", "
190 | \n", "1.691500 | \n", "
191 | \n", "1.640000 | \n", "
192 | \n", "1.716100 | \n", "
193 | \n", "1.680800 | \n", "
194 | \n", "1.698200 | \n", "
195 | \n", "1.714100 | \n", "
196 | \n", "1.775800 | \n", "
197 | \n", "1.679100 | \n", "
198 | \n", "1.676600 | \n", "
199 | \n", "1.710200 | \n", "
200 | \n", "1.721300 | \n", "
201 | \n", "1.699500 | \n", "
202 | \n", "1.688800 | \n", "
203 | \n", "1.750100 | \n", "
204 | \n", "1.687800 | \n", "
205 | \n", "1.703300 | \n", "
206 | \n", "1.679600 | \n", "
207 | \n", "1.611800 | \n", "
208 | \n", "1.687100 | \n", "
209 | \n", "1.663500 | \n", "
210 | \n", "1.738600 | \n", "
211 | \n", "1.674800 | \n", "
212 | \n", "1.742700 | \n", "
213 | \n", "1.701000 | \n", "
214 | \n", "1.727100 | \n", "
215 | \n", "1.639400 | \n", "
216 | \n", "1.662600 | \n", "
217 | \n", "1.738500 | \n", "
218 | \n", "1.622400 | \n", "
219 | \n", "1.718900 | \n", "
220 | \n", "1.668300 | \n", "
221 | \n", "1.686200 | \n", "
222 | \n", "1.684600 | \n", "
223 | \n", "1.694200 | \n", "
224 | \n", "1.667400 | \n", "
225 | \n", "1.679600 | \n", "
226 | \n", "1.652500 | \n", "
227 | \n", "1.747100 | \n", "
228 | \n", "1.713300 | \n", "
229 | \n", "1.689700 | \n", "
230 | \n", "1.735200 | \n", "
231 | \n", "1.629800 | \n", "
232 | \n", "1.638500 | \n", "
233 | \n", "1.737700 | \n", "
234 | \n", "1.673000 | \n", "
235 | \n", "1.753100 | \n", "
236 | \n", "1.729100 | \n", "
237 | \n", "1.713900 | \n", "
238 | \n", "1.811600 | \n", "
239 | \n", "1.632600 | \n", "
240 | \n", "1.666300 | \n", "
241 | \n", "1.667900 | \n", "
242 | \n", "1.659300 | \n", "
243 | \n", "1.738200 | \n", "
244 | \n", "1.711500 | \n", "
245 | \n", "1.629500 | \n", "
246 | \n", "1.741400 | \n", "
247 | \n", "1.674200 | \n", "
248 | \n", "1.685500 | \n", "
249 | \n", "1.641000 | \n", "
250 | \n", "1.646500 | \n", "
251 | \n", "1.651900 | \n", "
252 | \n", "1.704700 | \n", "
253 | \n", "1.668500 | \n", "
254 | \n", "1.641100 | \n", "
255 | \n", "1.618600 | \n", "
256 | \n", "1.678100 | \n", "
257 | \n", "1.645500 | \n", "
258 | \n", "1.666800 | \n", "
259 | \n", "1.672900 | \n", "
260 | \n", "1.708300 | \n", "
261 | \n", "1.698600 | \n", "
262 | \n", "1.777200 | \n", "
263 | \n", "1.685300 | \n", "
264 | \n", "1.688500 | \n", "
265 | \n", "1.716800 | \n", "
266 | \n", "1.691000 | \n", "
267 | \n", "1.621700 | \n", "
268 | \n", "1.654700 | \n", "
269 | \n", "1.663800 | \n", "
270 | \n", "1.608900 | \n", "
271 | \n", "1.597600 | \n", "
272 | \n", "1.772600 | \n", "
273 | \n", "1.685900 | \n", "
274 | \n", "1.710600 | \n", "
275 | \n", "1.673800 | \n", "
276 | \n", "1.638300 | \n", "
277 | \n", "1.689400 | \n", "
278 | \n", "1.698100 | \n", "
279 | \n", "1.655300 | \n", "
280 | \n", "1.647500 | \n", "
281 | \n", "1.702600 | \n", "
282 | \n", "1.685500 | \n", "
283 | \n", "1.671200 | \n", "
284 | \n", "1.693100 | \n", "
285 | \n", "1.740100 | \n", "
286 | \n", "1.759800 | \n", "
287 | \n", "1.718800 | \n", "
288 | \n", "1.693200 | \n", "
289 | \n", "1.656800 | \n", "
290 | \n", "1.697000 | \n", "
291 | \n", "1.683600 | \n", "
292 | \n", "1.688200 | \n", "
293 | \n", "1.681800 | \n", "
294 | \n", "1.695600 | \n", "
295 | \n", "1.747800 | \n", "
296 | \n", "1.632400 | \n", "
297 | \n", "1.627600 | \n", "
298 | \n", "1.658600 | \n", "
299 | \n", "1.652800 | \n", "
300 | \n", "1.699200 | \n", "
301 | \n", "1.669900 | \n", "
302 | \n", "1.625900 | \n", "
303 | \n", "1.615500 | \n", "
304 | \n", "1.641800 | \n", "
305 | \n", "1.648800 | \n", "
306 | \n", "1.782700 | \n", "
307 | \n", "1.677900 | \n", "
308 | \n", "1.636300 | \n", "
309 | \n", "1.626400 | \n", "
310 | \n", "1.634600 | \n", "
311 | \n", "1.745300 | \n", "
312 | \n", "1.771900 | \n", "
313 | \n", "1.682700 | \n", "
314 | \n", "1.695700 | \n", "
315 | \n", "1.674900 | \n", "
316 | \n", "1.623100 | \n", "
317 | \n", "1.740700 | \n", "
318 | \n", "1.676600 | \n", "
319 | \n", "1.664900 | \n", "
320 | \n", "1.682200 | \n", "
321 | \n", "1.756100 | \n", "
322 | \n", "1.687600 | \n", "
323 | \n", "1.628200 | \n", "
324 | \n", "1.643100 | \n", "
325 | \n", "1.763200 | \n", "
326 | \n", "1.682700 | \n", "
327 | \n", "1.688500 | \n", "
328 | \n", "1.709200 | \n", "
329 | \n", "1.717200 | \n", "
330 | \n", "1.651900 | \n", "
331 | \n", "1.659000 | \n", "
332 | \n", "1.704900 | \n", "
333 | \n", "1.655300 | \n", "
334 | \n", "1.630200 | \n", "
335 | \n", "1.719700 | \n", "
336 | \n", "1.654200 | \n", "
337 | \n", "1.679700 | \n", "
338 | \n", "1.655600 | \n", "
339 | \n", "1.645600 | \n", "
340 | \n", "1.676200 | \n", "
341 | \n", "1.735000 | \n", "
342 | \n", "1.766000 | \n", "
343 | \n", "1.622600 | \n", "
344 | \n", "1.663600 | \n", "
345 | \n", "1.640000 | \n", "
346 | \n", "1.660200 | \n", "
347 | \n", "1.661300 | \n", "
348 | \n", "1.714300 | \n", "
349 | \n", "1.646500 | \n", "
350 | \n", "1.698700 | \n", "
351 | \n", "1.748200 | \n", "
352 | \n", "1.606300 | \n", "
353 | \n", "1.666100 | \n", "
354 | \n", "1.698800 | \n", "
355 | \n", "1.756400 | \n", "
356 | \n", "1.671900 | \n", "
357 | \n", "1.663000 | \n", "
358 | \n", "1.643400 | \n", "
359 | \n", "1.709000 | \n", "
360 | \n", "1.648700 | \n", "
361 | \n", "1.652000 | \n", "
362 | \n", "1.721000 | \n", "
363 | \n", "1.653800 | \n", "
364 | \n", "1.694400 | \n", "
365 | \n", "1.747800 | \n", "
366 | \n", "1.647300 | \n", "
367 | \n", "1.685400 | \n", "
368 | \n", "1.697800 | \n", "
369 | \n", "1.668200 | \n", "
370 | \n", "1.622900 | \n", "
371 | \n", "1.669300 | \n", "
372 | \n", "1.634500 | \n", "
373 | \n", "1.668900 | \n", "
374 | \n", "1.662200 | \n", "
"
],
"text/plain": [
" "
]
},
"metadata": {}
}
],
"source": [
"trainer = Trainer(\n",
" model=model,\n",
" train_dataset=data[\"train\"],\n",
" args=TrainingArguments(\n",
" per_device_train_batch_size=MICRO_BATCH_SIZE,\n",
" gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,\n",
" warmup_steps=100,\n",
" num_train_epochs=EPOCHS,\n",
" learning_rate=LEARNING_RATE,\n",
" fp16=True,\n",
" logging_steps=1,\n",
" output_dir=\"gpt2-alpaca\",\n",
" save_total_limit=3,\n",
" ),\n",
" data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
")\n",
"model.config.use_cache = False\n",
"trainer.train(resume_from_checkpoint=False)\n",
"\n",
"model.save_pretrained(\"gpt2-alpaca\")"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 568
},
"id": "8rPUnadrjq7H",
"outputId": "49ea41e1-d216-4e83-acda-d4590f2c1b48"
},
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
]
},
{
"output_type": "error",
"ename": "RuntimeError",
"evalue": "ignored",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m\n",
" \n",
"
\n",
" \n",
" \n",
" \n",
" Step \n",
" Training Loss \n",
" \n",
" \n",
" 1 \n",
" 3.505800 \n",
" \n",
" \n",
" 2 \n",
" 3.589200 \n",
" \n",
" \n",
" 3 \n",
" 3.581400 \n",
" \n",
" \n",
" 4 \n",
" 3.499100 \n",
" \n",
" \n",
" 5 \n",
" 3.446200 \n",
" \n",
" \n",
" 6 \n",
" 3.463100 \n",
" \n",
" \n",
" 7 \n",
" 3.523100 \n",
" \n",
" \n",
" 8 \n",
" 3.504400 \n",
" \n",
" \n",
" 9 \n",
" 3.536200 \n",
" \n",
" \n",
" 10 \n",
" 3.503900 \n",
" \n",
" \n",
" 11 \n",
" 3.407800 \n",
" \n",
" \n",
" 12 \n",
" 3.393300 \n",
" \n",
" \n",
" 13 \n",
" 3.435200 \n",
" \n",
" \n",
" 14 \n",
" 3.501500 \n",
" \n",
" \n",
" 15 \n",
" 3.379000 \n",
" \n",
" \n",
" 16 \n",
" 3.408300 \n",
" \n",
" \n",
" 17 \n",
" 3.385700 \n",
" \n",
" \n",
" 18 \n",
" 3.351200 \n",
" \n",
" \n",
" 19 \n",
" 3.319400 \n",
" \n",
" \n",
" 20 \n",
" 3.318500 \n",
" \n",
" \n",
" 21 \n",
" 3.174300 \n",
" \n",
" \n",
" 22 \n",
" 3.284800 \n",
" \n",
" \n",
" 23 \n",
" 3.165500 \n",
" \n",
" \n",
" 24 \n",
" 3.162600 \n",
" \n",
" \n",
" 25 \n",
" 3.222300 \n",
" \n",
" \n",
" 26 \n",
" 3.156100 \n",
" \n",
" \n",
" 27 \n",
" 3.112700 \n",
" \n",
" \n",
" 28 \n",
" 2.980200 \n",
" \n",
" \n",
" 29 \n",
" 2.966500 \n",
" \n",
" \n",
" 30 \n",
" 2.978900 \n",
" \n",
" \n",
" 31 \n",
" 2.943500 \n",
" \n",
" \n",
" 32 \n",
" 2.979800 \n",
" \n",
" \n",
" 33 \n",
" 2.897400 \n",
" \n",
" \n",
" 34 \n",
" 2.844700 \n",
" \n",
" \n",
" 35 \n",
" 2.779500 \n",
" \n",
" \n",
" 36 \n",
" 2.761400 \n",
" \n",
" \n",
" 37 \n",
" 2.734000 \n",
" \n",
" \n",
" 38 \n",
" 2.656200 \n",
" \n",
" \n",
" 39 \n",
" 2.685300 \n",
" \n",
" \n",
" 40 \n",
" 2.639600 \n",
" \n",
" \n",
" 41 \n",
" 2.586300 \n",
" \n",
" \n",
" 42 \n",
" 2.522400 \n",
" \n",
" \n",
" 43 \n",
" 2.478000 \n",
" \n",
" \n",
" 44 \n",
" 2.511800 \n",
" \n",
" \n",
" 45 \n",
" 2.516400 \n",
" \n",
" \n",
" 46 \n",
" 2.412900 \n",
" \n",
" \n",
" 47 \n",
" 2.380500 \n",
" \n",
" \n",
" 48 \n",
" 2.263900 \n",
" \n",
" \n",
" 49 \n",
" 2.313200 \n",
" \n",
" \n",
" 50 \n",
" 2.256700 \n",
" \n",
" \n",
" 51 \n",
" 2.231000 \n",
" \n",
" \n",
" 52 \n",
" 2.109000 \n",
" \n",
" \n",
" 53 \n",
" 2.242600 \n",
" \n",
" \n",
" 54 \n",
" 2.147000 \n",
" \n",
" \n",
" 55 \n",
" 2.127600 \n",
" \n",
" \n",
" 56 \n",
" 2.075800 \n",
" \n",
" \n",
" 57 \n",
" 2.053100 \n",
" \n",
" \n",
" 58 \n",
" 2.060600 \n",
" \n",
" \n",
" 59 \n",
" 2.026200 \n",
" \n",
" \n",
" 60 \n",
" 2.050500 \n",
" \n",
" \n",
" 61 \n",
" 2.001800 \n",
" \n",
" \n",
" 62 \n",
" 1.939500 \n",
" \n",
" \n",
" 63 \n",
" 2.017000 \n",
" \n",
" \n",
" 64 \n",
" 1.955400 \n",
" \n",
" \n",
" 65 \n",
" 1.983100 \n",
" \n",
" \n",
" 66 \n",
" 1.888200 \n",
" \n",
" \n",
" 67 \n",
" 1.942800 \n",
" \n",
" \n",
" 68 \n",
" 1.955000 \n",
" \n",
" \n",
" 69 \n",
" 1.830700 \n",
" \n",
" \n",
" 70 \n",
" 1.889600 \n",
" \n",
" \n",
" 71 \n",
" 1.918400 \n",
" \n",
" \n",
" 72 \n",
" 1.819400 \n",
" \n",
" \n",
" 73 \n",
" 1.848000 \n",
" \n",
" \n",
" 74 \n",
" 1.874500 \n",
" \n",
" \n",
" 75 \n",
" 1.846700 \n",
" \n",
" \n",
" 76 \n",
" 1.925400 \n",
" \n",
" \n",
" 77 \n",
" 1.741800 \n",
" \n",
" \n",
" 78 \n",
" 1.873700 \n",
" \n",
" \n",
" 79 \n",
" 1.889100 \n",
" \n",
" \n",
" 80 \n",
" 1.823100 \n",
" \n",
" \n",
" 81 \n",
" 1.782900 \n",
" \n",
" \n",
" 82 \n",
" 1.819800 \n",
" \n",
" \n",
" 83 \n",
" 1.854300 \n",
" \n",
" \n",
" 84 \n",
" 1.802200 \n",
" \n",
" \n",
" 85 \n",
" 1.778000 \n",
" \n",
" \n",
" 86 \n",
" 1.755700 \n",
" \n",
" \n",
" 87 \n",
" 1.844600 \n",
" \n",
" \n",
" 88 \n",
" 1.778300 \n",
" \n",
" \n",
" 89 \n",
" 1.803900 \n",
" \n",
" \n",
" 90 \n",
" 1.841800 \n",
" \n",
" \n",
" 91 \n",
" 1.763200 \n",
" \n",
" \n",
" 92 \n",
" 1.800000 \n",
" \n",
" \n",
" 93 \n",
" 1.804400 \n",
" \n",
" \n",
" 94 \n",
" 1.804800 \n",
" \n",
" \n",
" 95 \n",
" 1.754500 \n",
" \n",
" \n",
" 96 \n",
" 1.796900 \n",
" \n",
" \n",
" 97 \n",
" 1.848600 \n",
" \n",
" \n",
" 98 \n",
" 1.746900 \n",
" \n",
" \n",
" 99 \n",
" 1.703900 \n",
" \n",
" \n",
" 100 \n",
" 1.775400 \n",
" \n",
" \n",
" 101 \n",
" 1.773300 \n",
" \n",
" \n",
" 102 \n",
" 1.748300 \n",
" \n",
" \n",
" 103 \n",
" 1.740100 \n",
" \n",
" \n",
" 104 \n",
" 1.795300 \n",
" \n",
" \n",
" 105 \n",
" 1.850800 \n",
" \n",
" \n",
" 106 \n",
" 1.828400 \n",
" \n",
" \n",
" 107 \n",
" 1.732500 \n",
" \n",
" \n",
" 108 \n",
" 1.841300 \n",
" \n",
" \n",
" 109 \n",
" 1.774700 \n",
" \n",
" \n",
" 110 \n",
" 1.814700 \n",
" \n",
" \n",
" 111 \n",
" 1.702000 \n",
" \n",
" \n",
" 112 \n",
" 1.734600 \n",
" \n",
" \n",
" 113 \n",
" 1.790700 \n",
" \n",
" \n",
" 114 \n",
" 1.635100 \n",
" \n",
" \n",
" 115 \n",
" 1.805300 \n",
" \n",
" \n",
" 116 \n",
" 1.776300 \n",
" \n",
" \n",
" 117 \n",
" 1.855000 \n",
" \n",
" \n",
" 118 \n",
" 1.778700 \n",
" \n",
" \n",
" 119 \n",
" 1.743800 \n",
" \n",
" \n",
" 120 \n",
" 1.793500 \n",
" \n",
" \n",
" 121 \n",
" 1.743100 \n",
" \n",
" \n",
" 122 \n",
" 1.752300 \n",
" \n",
" \n",
" 123 \n",
" 1.797400 \n",
" \n",
" \n",
" 124 \n",
" 1.755000 \n",
" \n",
" \n",
" 125 \n",
" 1.679400 \n",
" \n",
" \n",
" 126 \n",
" 1.724200 \n",
" \n",
" \n",
" 127 \n",
" 1.759600 \n",
" \n",
" \n",
" 128 \n",
" 1.793600 \n",
" \n",
" \n",
" 129 \n",
" 1.723000 \n",
" \n",
" \n",
" 130 \n",
" 1.817600 \n",
" \n",
" \n",
" 131 \n",
" 1.759500 \n",
" \n",
" \n",
" 132 \n",
" 1.719300 \n",
" \n",
" \n",
" 133 \n",
" 1.709900 \n",
" \n",
" \n",
" 134 \n",
" 1.696500 \n",
" \n",
" \n",
" 135 \n",
" 1.736700 \n",
" \n",
" \n",
" 136 \n",
" 1.713900 \n",
" \n",
" \n",
" 137 \n",
" 1.689000 \n",
" \n",
" \n",
" 138 \n",
" 1.691400 \n",
" \n",
" \n",
" 139 \n",
" 1.707600 \n",
" \n",
" \n",
" 140 \n",
" 1.714700 \n",
" \n",
" \n",
" 141 \n",
" 1.778000 \n",
" \n",
" \n",
" 142 \n",
" 1.759700 \n",
" \n",
" \n",
" 143 \n",
" 1.746300 \n",
" \n",
" \n",
" 144 \n",
" 1.669000 \n",
" \n",
" \n",
" 145 \n",
" 1.716300 \n",
" \n",
" \n",
" 146 \n",
" 1.769200 \n",
" \n",
" \n",
" 147 \n",
" 1.669500 \n",
" \n",
" \n",
" 148 \n",
" 1.747200 \n",
" \n",
" \n",
" 149 \n",
" 1.730200 \n",
" \n",
" \n",
" 150 \n",
" 1.714300 \n",
" \n",
" \n",
" 151 \n",
" 1.665400 \n",
" \n",
" \n",
" 152 \n",
" 1.822100 \n",
" \n",
" \n",
" 153 \n",
" 1.733000 \n",
" \n",
" \n",
" 154 \n",
" 1.740300 \n",
" \n",
" \n",
" 155 \n",
" 1.650800 \n",
" \n",
" \n",
" 156 \n",
" 1.637600 \n",
" \n",
" \n",
" 157 \n",
" 1.729200 \n",
" \n",
" \n",
" 158 \n",
" 1.719300 \n",
" \n",
" \n",
" 159 \n",
" 1.689800 \n",
" \n",
" \n",
" 160 \n",
" 1.681200 \n",
" \n",
" \n",
" 161 \n",
" 1.750400 \n",
" \n",
" \n",
" 162 \n",
" 1.706700 \n",
" \n",
" \n",
" 163 \n",
" 1.762800 \n",
" \n",
" \n",
" 164 \n",
" 1.646600 \n",
" \n",
" \n",
" 165 \n",
" 1.660200 \n",
" \n",
" \n",
" 166 \n",
" 1.701700 \n",
" \n",
" \n",
" 167 \n",
" 1.765900 \n",
" \n",
" \n",
" 168 \n",
" 1.605600 \n",
" \n",
" \n",
" 169 \n",
" 1.772600 \n",
" \n",
" \n",
" 170 \n",
" 1.814300 \n",
" \n",
" \n",
" 171 \n",
" 1.759800 \n",
" \n",
" \n",
" 172 \n",
" 1.718600 \n",
" \n",
" \n",
" 173 \n",
" 1.639600 \n",
" \n",
" \n",
" 174 \n",
" 1.713800 \n",
" \n",
" \n",
" 175 \n",
" 1.755000 \n",
" \n",
" \n",
" 176 \n",
" 1.771900 \n",
" \n",
" \n",
" 177 \n",
" 1.714100 \n",
" \n",
" \n",
" 178 \n",
" 1.737800 \n",
" \n",
" \n",
" 179 \n",
" 1.724300 \n",
" \n",
" \n",
" 180 \n",
" 1.674700 \n",
" \n",
" \n",
" 181 \n",
" 1.639800 \n",
" \n",
" \n",
" 182 \n",
" 1.661000 \n",
" \n",
" \n",
" 183 \n",
" 1.736600 \n",
" \n",
" \n",
" 184 \n",
" 1.655200 \n",
" \n",
" \n",
" 185 \n",
" 1.765800 \n",
" \n",
" \n",
" 186 \n",
" 1.707400 \n",
" \n",
" \n",
" 187 \n",
" 1.705600 \n",
" \n",
" \n",
" 188 \n",
" 1.686100 \n",
" \n",
" \n",
" 189 \n",
" 1.716100 \n",
" \n",
" \n",
" 190 \n",
" 1.691500 \n",
" \n",
" \n",
" 191 \n",
" 1.640000 \n",
" \n",
" \n",
" 192 \n",
" 1.716100 \n",
" \n",
" \n",
" 193 \n",
" 1.680800 \n",
" \n",
" \n",
" 194 \n",
" 1.698200 \n",
" \n",
" \n",
" 195 \n",
" 1.714100 \n",
" \n",
" \n",
" 196 \n",
" 1.775800 \n",
" \n",
" \n",
" 197 \n",
" 1.679100 \n",
" \n",
" \n",
" 198 \n",
" 1.676600 \n",
" \n",
" \n",
" 199 \n",
" 1.710200 \n",
" \n",
" \n",
" 200 \n",
" 1.721300 \n",
" \n",
" \n",
" 201 \n",
" 1.699500 \n",
" \n",
" \n",
" 202 \n",
" 1.688800 \n",
" \n",
" \n",
" 203 \n",
" 1.750100 \n",
" \n",
" \n",
" 204 \n",
" 1.687800 \n",
" \n",
" \n",
" 205 \n",
" 1.703300 \n",
" \n",
" \n",
" 206 \n",
" 1.679600 \n",
" \n",
" \n",
" 207 \n",
" 1.611800 \n",
" \n",
" \n",
" 208 \n",
" 1.687100 \n",
" \n",
" \n",
" 209 \n",
" 1.663500 \n",
" \n",
" \n",
" 210 \n",
" 1.738600 \n",
" \n",
" \n",
" 211 \n",
" 1.674800 \n",
" \n",
" \n",
" 212 \n",
" 1.742700 \n",
" \n",
" \n",
" 213 \n",
" 1.701000 \n",
" \n",
" \n",
" 214 \n",
" 1.727100 \n",
" \n",
" \n",
" 215 \n",
" 1.639400 \n",
" \n",
" \n",
" 216 \n",
" 1.662600 \n",
" \n",
" \n",
" 217 \n",
" 1.738500 \n",
" \n",
" \n",
" 218 \n",
" 1.622400 \n",
" \n",
" \n",
" 219 \n",
" 1.718900 \n",
" \n",
" \n",
" 220 \n",
" 1.668300 \n",
" \n",
" \n",
" 221 \n",
" 1.686200 \n",
" \n",
" \n",
" 222 \n",
" 1.684600 \n",
" \n",
" \n",
" 223 \n",
" 1.694200 \n",
" \n",
" \n",
" 224 \n",
" 1.667400 \n",
" \n",
" \n",
" 225 \n",
" 1.679600 \n",
" \n",
" \n",
" 226 \n",
" 1.652500 \n",
" \n",
" \n",
" 227 \n",
" 1.747100 \n",
" \n",
" \n",
" 228 \n",
" 1.713300 \n",
" \n",
" \n",
" 229 \n",
" 1.689700 \n",
" \n",
" \n",
" 230 \n",
" 1.735200 \n",
" \n",
" \n",
" 231 \n",
" 1.629800 \n",
" \n",
" \n",
" 232 \n",
" 1.638500 \n",
" \n",
" \n",
" 233 \n",
" 1.737700 \n",
" \n",
" \n",
" 234 \n",
" 1.673000 \n",
" \n",
" \n",
" 235 \n",
" 1.753100 \n",
" \n",
" \n",
" 236 \n",
" 1.729100 \n",
" \n",
" \n",
" 237 \n",
" 1.713900 \n",
" \n",
" \n",
" 238 \n",
" 1.811600 \n",
" \n",
" \n",
" 239 \n",
" 1.632600 \n",
" \n",
" \n",
" 240 \n",
" 1.666300 \n",
" \n",
" \n",
" 241 \n",
" 1.667900 \n",
" \n",
" \n",
" 242 \n",
" 1.659300 \n",
" \n",
" \n",
" 243 \n",
" 1.738200 \n",
" \n",
" \n",
" 244 \n",
" 1.711500 \n",
" \n",
" \n",
" 245 \n",
" 1.629500 \n",
" \n",
" \n",
" 246 \n",
" 1.741400 \n",
" \n",
" \n",
" 247 \n",
" 1.674200 \n",
" \n",
" \n",
" 248 \n",
" 1.685500 \n",
" \n",
" \n",
" 249 \n",
" 1.641000 \n",
" \n",
" \n",
" 250 \n",
" 1.646500 \n",
" \n",
" \n",
" 251 \n",
" 1.651900 \n",
" \n",
" \n",
" 252 \n",
" 1.704700 \n",
" \n",
" \n",
" 253 \n",
" 1.668500 \n",
" \n",
" \n",
" 254 \n",
" 1.641100 \n",
" \n",
" \n",
" 255 \n",
" 1.618600 \n",
" \n",
" \n",
" 256 \n",
" 1.678100 \n",
" \n",
" \n",
" 257 \n",
" 1.645500 \n",
" \n",
" \n",
" 258 \n",
" 1.666800 \n",
" \n",
" \n",
" 259 \n",
" 1.672900 \n",
" \n",
" \n",
" 260 \n",
" 1.708300 \n",
" \n",
" \n",
" 261 \n",
" 1.698600 \n",
" \n",
" \n",
" 262 \n",
" 1.777200 \n",
" \n",
" \n",
" 263 \n",
" 1.685300 \n",
" \n",
" \n",
" 264 \n",
" 1.688500 \n",
" \n",
" \n",
" 265 \n",
" 1.716800 \n",
" \n",
" \n",
" 266 \n",
" 1.691000 \n",
" \n",
" \n",
" 267 \n",
" 1.621700 \n",
" \n",
" \n",
" 268 \n",
" 1.654700 \n",
" \n",
" \n",
" 269 \n",
" 1.663800 \n",
" \n",
" \n",
" 270 \n",
" 1.608900 \n",
" \n",
" \n",
" 271 \n",
" 1.597600 \n",
" \n",
" \n",
" 272 \n",
" 1.772600 \n",
" \n",
" \n",
" 273 \n",
" 1.685900 \n",
" \n",
" \n",
" 274 \n",
" 1.710600 \n",
" \n",
" \n",
" 275 \n",
" 1.673800 \n",
" \n",
" \n",
" 276 \n",
" 1.638300 \n",
" \n",
" \n",
" 277 \n",
" 1.689400 \n",
" \n",
" \n",
" 278 \n",
" 1.698100 \n",
" \n",
" \n",
" 279 \n",
" 1.655300 \n",
" \n",
" \n",
" 280 \n",
" 1.647500 \n",
" \n",
" \n",
" 281 \n",
" 1.702600 \n",
" \n",
" \n",
" 282 \n",
" 1.685500 \n",
" \n",
" \n",
" 283 \n",
" 1.671200 \n",
" \n",
" \n",
" 284 \n",
" 1.693100 \n",
" \n",
" \n",
" 285 \n",
" 1.740100 \n",
" \n",
" \n",
" 286 \n",
" 1.759800 \n",
" \n",
" \n",
" 287 \n",
" 1.718800 \n",
" \n",
" \n",
" 288 \n",
" 1.693200 \n",
" \n",
" \n",
" 289 \n",
" 1.656800 \n",
" \n",
" \n",
" 290 \n",
" 1.697000 \n",
" \n",
" \n",
" 291 \n",
" 1.683600 \n",
" \n",
" \n",
" 292 \n",
" 1.688200 \n",
" \n",
" \n",
" 293 \n",
" 1.681800 \n",
" \n",
" \n",
" 294 \n",
" 1.695600 \n",
" \n",
" \n",
" 295 \n",
" 1.747800 \n",
" \n",
" \n",
" 296 \n",
" 1.632400 \n",
" \n",
" \n",
" 297 \n",
" 1.627600 \n",
" \n",
" \n",
" 298 \n",
" 1.658600 \n",
" \n",
" \n",
" 299 \n",
" 1.652800 \n",
" \n",
" \n",
" 300 \n",
" 1.699200 \n",
" \n",
" \n",
" 301 \n",
" 1.669900 \n",
" \n",
" \n",
" 302 \n",
" 1.625900 \n",
" \n",
" \n",
" 303 \n",
" 1.615500 \n",
" \n",
" \n",
" 304 \n",
" 1.641800 \n",
" \n",
" \n",
" 305 \n",
" 1.648800 \n",
" \n",
" \n",
" 306 \n",
" 1.782700 \n",
" \n",
" \n",
" 307 \n",
" 1.677900 \n",
" \n",
" \n",
" 308 \n",
" 1.636300 \n",
" \n",
" \n",
" 309 \n",
" 1.626400 \n",
" \n",
" \n",
" 310 \n",
" 1.634600 \n",
" \n",
" \n",
" 311 \n",
" 1.745300 \n",
" \n",
" \n",
" 312 \n",
" 1.771900 \n",
" \n",
" \n",
" 313 \n",
" 1.682700 \n",
" \n",
" \n",
" 314 \n",
" 1.695700 \n",
" \n",
" \n",
" 315 \n",
" 1.674900 \n",
" \n",
" \n",
" 316 \n",
" 1.623100 \n",
" \n",
" \n",
" 317 \n",
" 1.740700 \n",
" \n",
" \n",
" 318 \n",
" 1.676600 \n",
" \n",
" \n",
" 319 \n",
" 1.664900 \n",
" \n",
" \n",
" 320 \n",
" 1.682200 \n",
" \n",
" \n",
" 321 \n",
" 1.756100 \n",
" \n",
" \n",
" 322 \n",
" 1.687600 \n",
" \n",
" \n",
" 323 \n",
" 1.628200 \n",
" \n",
" \n",
" 324 \n",
" 1.643100 \n",
" \n",
" \n",
" 325 \n",
" 1.763200 \n",
" \n",
" \n",
" 326 \n",
" 1.682700 \n",
" \n",
" \n",
" 327 \n",
" 1.688500 \n",
" \n",
" \n",
" 328 \n",
" 1.709200 \n",
" \n",
" \n",
" 329 \n",
" 1.717200 \n",
" \n",
" \n",
" 330 \n",
" 1.651900 \n",
" \n",
" \n",
" 331 \n",
" 1.659000 \n",
" \n",
" \n",
" 332 \n",
" 1.704900 \n",
" \n",
" \n",
" 333 \n",
" 1.655300 \n",
" \n",
" \n",
" 334 \n",
" 1.630200 \n",
" \n",
" \n",
" 335 \n",
" 1.719700 \n",
" \n",
" \n",
" 336 \n",
" 1.654200 \n",
" \n",
" \n",
" 337 \n",
" 1.679700 \n",
" \n",
" \n",
" 338 \n",
" 1.655600 \n",
" \n",
" \n",
" 339 \n",
" 1.645600 \n",
" \n",
" \n",
" 340 \n",
" 1.676200 \n",
" \n",
" \n",
" 341 \n",
" 1.735000 \n",
" \n",
" \n",
" 342 \n",
" 1.766000 \n",
" \n",
" \n",
" 343 \n",
" 1.622600 \n",
" \n",
" \n",
" 344 \n",
" 1.663600 \n",
" \n",
" \n",
" 345 \n",
" 1.640000 \n",
" \n",
" \n",
" 346 \n",
" 1.660200 \n",
" \n",
" \n",
" 347 \n",
" 1.661300 \n",
" \n",
" \n",
" 348 \n",
" 1.714300 \n",
" \n",
" \n",
" 349 \n",
" 1.646500 \n",
" \n",
" \n",
" 350 \n",
" 1.698700 \n",
" \n",
" \n",
" 351 \n",
" 1.748200 \n",
" \n",
" \n",
" 352 \n",
" 1.606300 \n",
" \n",
" \n",
" 353 \n",
" 1.666100 \n",
" \n",
" \n",
" 354 \n",
" 1.698800 \n",
" \n",
" \n",
" 355 \n",
" 1.756400 \n",
" \n",
" \n",
" 356 \n",
" 1.671900 \n",
" \n",
" \n",
" 357 \n",
" 1.663000 \n",
" \n",
" \n",
" 358 \n",
" 1.643400 \n",
" \n",
" \n",
" 359 \n",
" 1.709000 \n",
" \n",
" \n",
" 360 \n",
" 1.648700 \n",
" \n",
" \n",
" 361 \n",
" 1.652000 \n",
" \n",
" \n",
" 362 \n",
" 1.721000 \n",
" \n",
" \n",
" 363 \n",
" 1.653800 \n",
" \n",
" \n",
" 364 \n",
" 1.694400 \n",
" \n",
" \n",
" 365 \n",
" 1.747800 \n",
" \n",
" \n",
" 366 \n",
" 1.647300 \n",
" \n",
" \n",
" 367 \n",
" 1.685400 \n",
" \n",
" \n",
" 368 \n",
" 1.697800 \n",
" \n",
" \n",
" 369 \n",
" 1.668200 \n",
" \n",
" \n",
" 370 \n",
" 1.622900 \n",
" \n",
" \n",
" 371 \n",
" 1.669300 \n",
" \n",
" \n",
" 372 \n",
" 1.634500 \n",
" \n",
" \n",
" 373 \n",
" 1.668900 \n",
" \n",
" \n",
" 374 \n",
" 1.662200 \n",
" \n",
" \n",
" 375 \n",
" 1.693100 \n",
" \n",
" \n",
" 376 \n",
" 1.701600 \n",
" \n",
" \n",
" 377 \n",
" 1.670400 \n",
" \n",
" \n",
" 378 \n",
" 1.683800 \n",
" \n",
" \n",
" 379 \n",
" 1.600100 \n",
" \n",
" \n",
" 380 \n",
" 1.638700 \n",
" \n",
" \n",
" 381 \n",
" 1.651100 \n",
" \n",
" \n",
" 382 \n",
" 1.722900 \n",
" \n",
" \n",
" 383 \n",
" 1.620800 \n",
" \n",
" \n",
" 384 \n",
" 1.612600 \n",
" \n",
" \n",
" 385 \n",
" 1.627700 \n",
" \n",
" \n",
" 386 \n",
" 1.642800 \n",
" \n",
" \n",
" 387 \n",
" 1.679900 \n",
" \n",
" \n",
" 388 \n",
" 1.689900 \n",
" \n",
" \n",
" 389 \n",
" 1.625800 \n",
" \n",
" \n",
" 390 \n",
" 1.644800 \n",
" \n",
" \n",
" 391 \n",
" 1.653600 \n",
" \n",
" \n",
" 392 \n",
" 1.601700 \n",
" \n",
" \n",
" 393 \n",
" 1.680400 \n",
" \n",
" \n",
" 394 \n",
" 1.758700 \n",
" \n",
" \n",
" 395 \n",
" 1.635300 \n",
" \n",
" \n",
" 396 \n",
" 1.641300 \n",
" \n",
" \n",
" 397 \n",
" 1.681900 \n",
" \n",
" \n",
" 398 \n",
" 1.627000 \n",
" \n",
" \n",
" 399 \n",
" 1.722600 \n",
" \n",
" \n",
" 400 \n",
" 1.627100 \n",
" \n",
" \n",
" 401 \n",
" 1.659300 \n",
" \n",
" \n",
" 402 \n",
" 1.646500 \n",
" \n",
" \n",
" 403 \n",
" 1.688000 \n",
" \n",
" \n",
" 404 \n",
" 1.648800 \n",
" \n",
" \n",
" 405 \n",
" 1.661900 \n",
" \n",
" \n",
" \n",
"406 \n",
" 1.717200 \n",
"
Copy a token from your Hugging Face\ntokens page and paste it below.
Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file.