{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "138889b92720ce2e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-14T09:02:09.162993Z",
     "start_time": "2024-05-14T09:02:09.134625Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>runname</th>\n",
       "      <th>seed</th>\n",
       "      <th>steps</th>\n",
       "      <th>agg_score</th>\n",
       "      <th>commonsense_qa/acc</th>\n",
       "      <th>commonsense_qa/acc_norm</th>\n",
       "      <th>hellaswag/acc</th>\n",
       "      <th>hellaswag/acc_norm</th>\n",
       "      <th>openbookqa/acc</th>\n",
       "      <th>openbookqa/acc_norm</th>\n",
       "      <th>...</th>\n",
       "      <th>siqa/acc</th>\n",
       "      <th>siqa/acc_norm</th>\n",
       "      <th>winogrande/acc</th>\n",
       "      <th>winogrande/acc_norm</th>\n",
       "      <th>sciq/acc</th>\n",
       "      <th>sciq/acc_norm</th>\n",
       "      <th>arc/acc</th>\n",
       "      <th>arc/acc_norm</th>\n",
       "      <th>mmlu/acc</th>\n",
       "      <th>mmlu/acc_norm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>big-run-sampled-fineweb-c4-filters</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0.330893</td>\n",
       "      <td>0.186</td>\n",
       "      <td>0.233</td>\n",
       "      <td>0.272</td>\n",
       "      <td>0.258</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.286</td>\n",
       "      <td>...</td>\n",
       "      <td>0.367</td>\n",
       "      <td>0.362</td>\n",
       "      <td>0.516</td>\n",
       "      <td>0.497</td>\n",
       "      <td>0.208</td>\n",
       "      <td>0.202</td>\n",
       "      <td>0.2195</td>\n",
       "      <td>0.2510</td>\n",
       "      <td>0.230294</td>\n",
       "      <td>0.250147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>big-run-sampled-fineweb-c4-filters</td>\n",
       "      <td>6</td>\n",
       "      <td>1000</td>\n",
       "      <td>0.359303</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.263</td>\n",
       "      <td>0.293</td>\n",
       "      <td>0.285</td>\n",
       "      <td>0.140</td>\n",
       "      <td>0.276</td>\n",
       "      <td>...</td>\n",
       "      <td>0.376</td>\n",
       "      <td>0.401</td>\n",
       "      <td>0.497</td>\n",
       "      <td>0.479</td>\n",
       "      <td>0.594</td>\n",
       "      <td>0.524</td>\n",
       "      <td>0.2740</td>\n",
       "      <td>0.2985</td>\n",
       "      <td>0.241617</td>\n",
       "      <td>0.251920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>big-run-sampled-fineweb-c4-filters</td>\n",
       "      <td>6</td>\n",
       "      <td>2000</td>\n",
       "      <td>0.375393</td>\n",
       "      <td>0.268</td>\n",
       "      <td>0.277</td>\n",
       "      <td>0.319</td>\n",
       "      <td>0.324</td>\n",
       "      <td>0.150</td>\n",
       "      <td>0.274</td>\n",
       "      <td>...</td>\n",
       "      <td>0.372</td>\n",
       "      <td>0.411</td>\n",
       "      <td>0.507</td>\n",
       "      <td>0.484</td>\n",
       "      <td>0.688</td>\n",
       "      <td>0.606</td>\n",
       "      <td>0.3015</td>\n",
       "      <td>0.3270</td>\n",
       "      <td>0.246577</td>\n",
       "      <td>0.259146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>big-run-sampled-fineweb-c4-filters</td>\n",
       "      <td>6</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.389655</td>\n",
       "      <td>0.303</td>\n",
       "      <td>0.305</td>\n",
       "      <td>0.324</td>\n",
       "      <td>0.358</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.280</td>\n",
       "      <td>...</td>\n",
       "      <td>0.383</td>\n",
       "      <td>0.389</td>\n",
       "      <td>0.520</td>\n",
       "      <td>0.506</td>\n",
       "      <td>0.741</td>\n",
       "      <td>0.647</td>\n",
       "      <td>0.3395</td>\n",
       "      <td>0.3405</td>\n",
       "      <td>0.255001</td>\n",
       "      <td>0.268740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>big-run-sampled-fineweb-c4-filters</td>\n",
       "      <td>6</td>\n",
       "      <td>4000</td>\n",
       "      <td>0.401195</td>\n",
       "      <td>0.309</td>\n",
       "      <td>0.310</td>\n",
       "      <td>0.353</td>\n",
       "      <td>0.393</td>\n",
       "      <td>0.138</td>\n",
       "      <td>0.288</td>\n",
       "      <td>...</td>\n",
       "      <td>0.378</td>\n",
       "      <td>0.402</td>\n",
       "      <td>0.534</td>\n",
       "      <td>0.511</td>\n",
       "      <td>0.766</td>\n",
       "      <td>0.652</td>\n",
       "      <td>0.3395</td>\n",
       "      <td>0.3495</td>\n",
       "      <td>0.256203</td>\n",
       "      <td>0.269056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>667</th>\n",
       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
       "      <td>6</td>\n",
       "      <td>163000</td>\n",
       "      <td>0.466255</td>\n",
       "      <td>0.426</td>\n",
       "      <td>0.372</td>\n",
       "      <td>0.469</td>\n",
       "      <td>0.555</td>\n",
       "      <td>0.242</td>\n",
       "      <td>0.354</td>\n",
       "      <td>...</td>\n",
       "      <td>0.389</td>\n",
       "      <td>0.394</td>\n",
       "      <td>0.563</td>\n",
       "      <td>0.544</td>\n",
       "      <td>0.869</td>\n",
       "      <td>0.808</td>\n",
       "      <td>0.4460</td>\n",
       "      <td>0.4435</td>\n",
       "      <td>0.297125</td>\n",
       "      <td>0.317543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>668</th>\n",
       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
       "      <td>6</td>\n",
       "      <td>164000</td>\n",
       "      <td>0.469743</td>\n",
       "      <td>0.431</td>\n",
       "      <td>0.376</td>\n",
       "      <td>0.467</td>\n",
       "      <td>0.556</td>\n",
       "      <td>0.232</td>\n",
       "      <td>0.356</td>\n",
       "      <td>...</td>\n",
       "      <td>0.391</td>\n",
       "      <td>0.397</td>\n",
       "      <td>0.568</td>\n",
       "      <td>0.552</td>\n",
       "      <td>0.861</td>\n",
       "      <td>0.800</td>\n",
       "      <td>0.4450</td>\n",
       "      <td>0.4515</td>\n",
       "      <td>0.302706</td>\n",
       "      <td>0.318447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>669</th>\n",
       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
       "      <td>6</td>\n",
       "      <td>165000</td>\n",
       "      <td>0.469847</td>\n",
       "      <td>0.426</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.472</td>\n",
       "      <td>0.549</td>\n",
       "      <td>0.234</td>\n",
       "      <td>0.364</td>\n",
       "      <td>...</td>\n",
       "      <td>0.389</td>\n",
       "      <td>0.401</td>\n",
       "      <td>0.562</td>\n",
       "      <td>0.548</td>\n",
       "      <td>0.867</td>\n",
       "      <td>0.795</td>\n",
       "      <td>0.4435</td>\n",
       "      <td>0.4475</td>\n",
       "      <td>0.297586</td>\n",
       "      <td>0.319279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670</th>\n",
       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
       "      <td>6</td>\n",
       "      <td>166000</td>\n",
       "      <td>0.467651</td>\n",
       "      <td>0.423</td>\n",
       "      <td>0.365</td>\n",
       "      <td>0.470</td>\n",
       "      <td>0.555</td>\n",
       "      <td>0.226</td>\n",
       "      <td>0.356</td>\n",
       "      <td>...</td>\n",
       "      <td>0.392</td>\n",
       "      <td>0.399</td>\n",
       "      <td>0.564</td>\n",
       "      <td>0.545</td>\n",
       "      <td>0.872</td>\n",
       "      <td>0.812</td>\n",
       "      <td>0.4365</td>\n",
       "      <td>0.4475</td>\n",
       "      <td>0.297256</td>\n",
       "      <td>0.319704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>big-run-sampled_full_filtered_no_dedup</td>\n",
       "      <td>6</td>\n",
       "      <td>167000</td>\n",
       "      <td>0.469652</td>\n",
       "      <td>0.416</td>\n",
       "      <td>0.373</td>\n",
       "      <td>0.469</td>\n",
       "      <td>0.560</td>\n",
       "      <td>0.234</td>\n",
       "      <td>0.356</td>\n",
       "      <td>...</td>\n",
       "      <td>0.392</td>\n",
       "      <td>0.394</td>\n",
       "      <td>0.565</td>\n",
       "      <td>0.557</td>\n",
       "      <td>0.867</td>\n",
       "      <td>0.803</td>\n",
       "      <td>0.4430</td>\n",
       "      <td>0.4455</td>\n",
       "      <td>0.297409</td>\n",
       "      <td>0.317717</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>672 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    runname  seed   steps  agg_score  \\\n",
       "0        big-run-sampled-fineweb-c4-filters     6       0   0.330893   \n",
       "1        big-run-sampled-fineweb-c4-filters     6    1000   0.359303   \n",
       "2        big-run-sampled-fineweb-c4-filters     6    2000   0.375393   \n",
       "3        big-run-sampled-fineweb-c4-filters     6    3000   0.389655   \n",
       "4        big-run-sampled-fineweb-c4-filters     6    4000   0.401195   \n",
       "..                                      ...   ...     ...        ...   \n",
       "667  big-run-sampled_full_filtered_no_dedup     6  163000   0.466255   \n",
       "668  big-run-sampled_full_filtered_no_dedup     6  164000   0.469743   \n",
       "669  big-run-sampled_full_filtered_no_dedup     6  165000   0.469847   \n",
       "670  big-run-sampled_full_filtered_no_dedup     6  166000   0.467651   \n",
       "671  big-run-sampled_full_filtered_no_dedup     6  167000   0.469652   \n",
       "\n",
       "     commonsense_qa/acc  commonsense_qa/acc_norm  hellaswag/acc  \\\n",
       "0                 0.186                    0.233          0.272   \n",
       "1                 0.250                    0.263          0.293   \n",
       "2                 0.268                    0.277          0.319   \n",
       "3                 0.303                    0.305          0.324   \n",
       "4                 0.309                    0.310          0.353   \n",
       "..                  ...                      ...            ...   \n",
       "667               0.426                    0.372          0.469   \n",
       "668               0.431                    0.376          0.467   \n",
       "669               0.426                    0.375          0.472   \n",
       "670               0.423                    0.365          0.470   \n",
       "671               0.416                    0.373          0.469   \n",
       "\n",
       "     hellaswag/acc_norm  openbookqa/acc  openbookqa/acc_norm  ...  siqa/acc  \\\n",
       "0                 0.258           0.166                0.286  ...     0.367   \n",
       "1                 0.285           0.140                0.276  ...     0.376   \n",
       "2                 0.324           0.150                0.274  ...     0.372   \n",
       "3                 0.358           0.152                0.280  ...     0.383   \n",
       "4                 0.393           0.138                0.288  ...     0.378   \n",
       "..                  ...             ...                  ...  ...       ...   \n",
       "667               0.555           0.242                0.354  ...     0.389   \n",
       "668               0.556           0.232                0.356  ...     0.391   \n",
       "669               0.549           0.234                0.364  ...     0.389   \n",
       "670               0.555           0.226                0.356  ...     0.392   \n",
       "671               0.560           0.234                0.356  ...     0.392   \n",
       "\n",
       "     siqa/acc_norm  winogrande/acc  winogrande/acc_norm  sciq/acc  \\\n",
       "0            0.362           0.516                0.497     0.208   \n",
       "1            0.401           0.497                0.479     0.594   \n",
       "2            0.411           0.507                0.484     0.688   \n",
       "3            0.389           0.520                0.506     0.741   \n",
       "4            0.402           0.534                0.511     0.766   \n",
       "..             ...             ...                  ...       ...   \n",
       "667          0.394           0.563                0.544     0.869   \n",
       "668          0.397           0.568                0.552     0.861   \n",
       "669          0.401           0.562                0.548     0.867   \n",
       "670          0.399           0.564                0.545     0.872   \n",
       "671          0.394           0.565                0.557     0.867   \n",
       "\n",
       "     sciq/acc_norm  arc/acc  arc/acc_norm  mmlu/acc  mmlu/acc_norm  \n",
       "0            0.202   0.2195        0.2510  0.230294       0.250147  \n",
       "1            0.524   0.2740        0.2985  0.241617       0.251920  \n",
       "2            0.606   0.3015        0.3270  0.246577       0.259146  \n",
       "3            0.647   0.3395        0.3405  0.255001       0.268740  \n",
       "4            0.652   0.3395        0.3495  0.256203       0.269056  \n",
       "..             ...      ...           ...       ...            ...  \n",
       "667          0.808   0.4460        0.4435  0.297125       0.317543  \n",
       "668          0.800   0.4450        0.4515  0.302706       0.318447  \n",
       "669          0.795   0.4435        0.4475  0.297586       0.319279  \n",
       "670          0.812   0.4365        0.4475  0.297256       0.319704  \n",
       "671          0.803   0.4430        0.4455  0.297409       0.317717  \n",
       "\n",
       "[672 rows x 22 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from matplotlib.figure import Figure\n",
    "\n",
    "df = pd.read_csv(\"../src_data/all-filters-big-runs.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "839a06a71d9183e5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-14T09:02:10.094329Z",
     "start_time": "2024-05-14T09:02:10.081683Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['big-run-sampled-fineweb-c4-filters',\n",
       " 'big-run-sampled_full_ind_minhash',\n",
       " 'big-run-fineweb-v1-all-dumps',\n",
       " 'big-run-sampled_full_filtered_no_dedup']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.unique(df[\"runname\"]).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b610f43caefdf01",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-14T09:03:06.294766Z",
     "start_time": "2024-05-14T09:03:06.291388Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "runs_mapping = {\n",
    "    # \"big-run-refinedweb\": \"RefinedWeb\",\n",
    "    # \"big-run-c4\": \"C4\",\n",
    "    \"big-run-sampled_full_filtered_no_dedup\": \"FineWeb: base filtering only\",\n",
    "    \"big-run-sampled_full_ind_minhash\": \"FineWeb: independent MinHash (id mh)\",\n",
    "    \"big-run-sampled-fineweb-c4-filters\": \"FineWeb: id mh + C4 filters\",\n",
    "    \"big-run-fineweb-v1-all-dumps\": \"FineWeb: id mh + C4 + custom filters\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-14T09:03:08.298110Z",
     "start_time": "2024-05-14T09:03:08.024839Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import os\n",
    "import json\n",
    "\n",
    "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n",
    "                   'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n",
    "\n",
    "def normalize_runname(runname):\n",
    "    return runname.replace(\"/\", \"_\")\n",
    "\n",
    "grouped = (\n",
    "    df.groupby([\"runname\", \"steps\"])\n",
    "    .agg(\n",
    "        {\n",
    "            key: \"mean\" for key in metrics\n",
    "        }\n",
    "    )\n",
    "    .reset_index()\n",
    ")\n",
    "\n",
    "file_id=\"../assets/data/plots/all_filtering_steps\"\n",
    "files = {}\n",
    "for metric in metrics:\n",
    "    datas = {}\n",
    "    for name, group in grouped.groupby(\"runname\"):\n",
    "        group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n",
    "        group = group.set_index(\"steps\")\n",
    "        rolling_avg = group\n",
    "        # rolling_avg = group.rolling(window=5).mean()\n",
    "        datas[name] = {\n",
    "            \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n",
    "            \"y\": rolling_avg[metric].tolist(),\n",
    "            \"label\": runs_mapping[name],\n",
    "        }\n",
    "    # Sort the datata based on the steps\n",
    "    datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n",
    "    # Create a folder\n",
    "    os.makedirs(f\"{file_id}\", exist_ok=True)\n",
    "    with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n",
    "        json.dump({\n",
    "            \"data\": datas,\n",
    "            \"layout\": {\n",
    "                \"title\": {\n",
    "                    \"text\": \"The different FineWeb processing steps\"\n",
    "                },\n",
    "            }\n",
    "        }, f)\n",
    "    files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n",
    "# Create l\n",
    "with open(f\"{file_id}/index.json\", \"w\") as f:\n",
    "    json.dump({\n",
    "        \"files\": files,\n",
    "        \"settings\": {\n",
    "            \"defaultMetric\": \"agg_score\",\n",
    "            \"slider\":{\"min\":0,\"max\":30,\"default\":5}\n",
    "        }\n",
    "    }, f)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "af28ebbd054cdc33",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-14T08:14:41.132508Z",
     "start_time": "2024-05-14T08:14:41.130025Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b8c428e2fedeb1a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}