<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression with Grid Search (scikit-learn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/VertaAI/modeldb/blob/master/client/workflows/demos/sklearn.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# restart your notebook if prompted on Colab\n",
    "try:\n",
    "    import verta\n",
    "except ImportError:\n",
    "    !pip install verta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "verta.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This example features:\n",
    "- **scikit-learn**'s `LinearRegression` model\n",
    "- **verta**'s Python client logging grid search results\n",
    "- **verta**'s Python client retrieving the best run from the grid search to calculate full training accuracy\n",
    "- predictions against a deployed model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "HOST = \"http://localhost:3000\"\n",
    "\n",
    "PROJECT_NAME = \"Webinar 1 - Census Income Classification\"\n",
    "EXPERIMENT_NAME = \"Logistic Regression\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import warnings\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)\n",
    "\n",
    "import itertools\n",
    "import os\n",
    "import time\n",
    "\n",
    "import six\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import sklearn\n",
    "from sklearn import model_selection\n",
    "from sklearn import linear_model\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    import wget\n",
    "except ImportError:\n",
    "    !pip install wget  # you may need pip3\n",
    "    import wget"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Log Workflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Instantiate Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from verta import Client\n",
    "from verta.utils import ModelAPI\n",
    "\n",
    "client = Client(HOST)\n",
    "proj = client.set_project(PROJECT_NAME)\n",
    "expt = client.set_experiment(EXPERIMENT_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "repo = client.set_repository('Webinar 1 - Census Demo')\n",
    "commit = repo.get_commit(branch='master')\n",
    "commit.branch(\"log-reg\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = client.set_dataset(name=\"Census Income\", type=\"s3\", workspace=\"Demos\")\n",
    "version = dataset.create_version(bucket_name=\"verta-starter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_PATH = \"./\"\n",
    "\n",
    "train_data_filename = DATASET_PATH + \"census-train.csv\"\n",
    "test_data_filename = DATASET_PATH + \"census-test.csv\"\n",
    "\n",
    "def download_starter_dataset(bucket_name):\n",
    "    if not os.path.exists(DATASET_PATH + \"census-train.csv\"):\n",
    "        train_data_url = \"http://s3.amazonaws.com/\" + bucket_name + \"/census-train.csv\"\n",
    "        if not os.path.isfile(train_data_filename):\n",
    "            wget.download(train_data_url)\n",
    "\n",
    "    if not os.path.exists(DATASET_PATH + \"census-test.csv\"):\n",
    "        test_data_url = \"http://s3.amazonaws.com/\" + bucket_name + \"/census-test.csv\"\n",
    "        if not os.path.isfile(test_data_filename):\n",
    "            wget.download(test_data_url)\n",
    "\n",
    "download_starter_dataset(\"verta-starter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(train_data_filename)\n",
    "X_train = df_train.iloc[:,:-1]\n",
    "y_train = df_train.iloc[:, -1]\n",
    "\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyperparam_candidates = {\n",
    "    'C': [1e-6, 1e-4],\n",
    "    'solver': ['lbfgs'],\n",
    "    'max_iter': [15, 28],\n",
    "}\n",
    "hyperparam_sets = [dict(zip(hyperparam_candidates.keys(), values))\n",
    "                   for values\n",
    "                   in itertools.product(*hyperparam_candidates.values())]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def run_experiment(hyperparams, index):\n",
    "    from verta.code import Notebook\n",
    "    from verta.configuration import Hyperparameters\n",
    "    from verta.dataset import S3\n",
    "    from verta.environment import Python\n",
    "\n",
    "    code_ver = Notebook()\n",
    "    config_ver = Hyperparameters(hyperparams)\n",
    "    dataset_ver_train = S3(\"s3://{}/{}\".format(\"verta-starter\", \"census-train.csv\"))\n",
    "    dataset_ver_test = S3(\"s3://{}/{}\".format(\"verta-starter\", \"census-test.csv\"))\n",
    "    env_ver = Python()\n",
    "\n",
    "    commit.update(\"notebooks/census-s3-example\", code_ver)\n",
    "    commit.update(\"config/hyperparams\", config_ver)\n",
    "    commit.update(\"data/train-data\", dataset_ver_train)\n",
    "    commit.update(\"data/test-data\", dataset_ver_test)\n",
    "    commit.update(\"env/python\", env_ver)\n",
    "    commit.save(\"Hyperparameter tuning Run: \" + str(index))\n",
    "\n",
    "    # create object to track experiment run\n",
    "    run = client.set_experiment_run()\n",
    "    \n",
    "    # create validation split\n",
    "    (X_val_train, X_val_test,\n",
    "     y_val_train, y_val_test) = model_selection.train_test_split(X_train, y_train,\n",
    "                                                                 test_size=0.2,\n",
    "                                                                 shuffle=True)\n",
    "\n",
    "    # log hyperparameters\n",
    "    run.log_hyperparameters(hyperparams)\n",
    "    print(hyperparams, end=' ')\n",
    "    \n",
    "    # create and train model\n",
    "    model = linear_model.LogisticRegression(**hyperparams)\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    # calculate and log validation accuracy\n",
    "    val_acc = model.score(X_val_test, y_val_test)\n",
    "    run.log_metric(\"val_acc\", val_acc)\n",
    "    print(\"Validation accuracy: {:.4f}\".format(val_acc))\n",
    "    \n",
    "    # save and log model\n",
    "    run.log_model(model)\n",
    "    \n",
    "    # log dataset snapshot as version\n",
    "    run.log_dataset_version(\"train\", version)\n",
    "    \n",
    "    # log Git information as code version\n",
    "    run.log_code()\n",
    "    \n",
    "    run.log_commit(\n",
    "        commit,\n",
    "        {\n",
    "            'notebook': \"notebooks/census-s3-example\",\n",
    "            'hyperparameters': \"config/hyperparams\",\n",
    "            'training_data': \"data/train-data\",\n",
    "            'test_data': \"data/test-data\",\n",
    "            'python_env': \"env/python\",\n",
    "        },\n",
    "    )\n",
    "    \n",
    "for i, hyperparams in enumerate(hyperparam_sets):\n",
    "    run_experiment(hyperparams, i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for c in commit.log():\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "commit.revert()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "master = repo.get_commit(branch='master')\n",
    "master.merge(commit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Revisit Workflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve Best Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run = expt.expt_runs.sort(\"metrics.val_acc\", descending=True)[0]\n",
    "print(\"Validation Accuracy: {:.4f}\".format(best_run.get_metric(\"val_acc\")))\n",
    "\n",
    "best_hyperparams = best_run.get_hyperparameters()\n",
    "print(\"Hyperparameters: {}\".format(best_hyperparams))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train on Full Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = linear_model.LogisticRegression(multi_class='auto', **best_hyperparams)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate Accuracy on Full Training Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_acc = model.score(X_train, y_train)\n",
    "print(\"Training accuracy: {:.4f}\".format(train_acc))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
<commit_msg>Update webinar notebook with new_branch() (#508)<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression with Grid Search (scikit-learn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/VertaAI/modeldb/blob/master/client/workflows/demos/sklearn.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# restart your notebook if prompted on Colab\n",
    "try:\n",
    "    import verta\n",
    "except ImportError:\n",
    "    !pip install verta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "verta.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This example features:\n",
    "- **scikit-learn**'s `LinearRegression` model\n",
    "- **verta**'s Python client logging grid search results\n",
    "- **verta**'s Python client retrieving the best run from the grid search to calculate full training accuracy\n",
    "- predictions against a deployed model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "HOST = \"http://localhost:3000\"\n",
    "\n",
    "PROJECT_NAME = \"Webinar 1 - Census Income Classification\"\n",
    "EXPERIMENT_NAME = \"Logistic Regression\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import warnings\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)\n",
    "\n",
    "import itertools\n",
    "import os\n",
    "import time\n",
    "\n",
    "import six\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import sklearn\n",
    "from sklearn import model_selection\n",
    "from sklearn import linear_model\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    import wget\n",
    "except ImportError:\n",
    "    !pip install wget  # you may need pip3\n",
    "    import wget"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Log Workflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Instantiate Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from verta import Client\n",
    "from verta.utils import ModelAPI\n",
    "\n",
    "client = Client(HOST)\n",
    "proj = client.set_project(PROJECT_NAME)\n",
    "expt = client.set_experiment(EXPERIMENT_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "repo = client.set_repository('Webinar 1 - Census Demo')\n",
    "commit = repo.get_commit(branch='master').new_branch(\"log-reg\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = client.set_dataset(name=\"Census Income\", type=\"s3\", workspace=\"Demos\")\n",
    "version = dataset.create_version(bucket_name=\"verta-starter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_PATH = \"./\"\n",
    "\n",
    "train_data_filename = DATASET_PATH + \"census-train.csv\"\n",
    "test_data_filename = DATASET_PATH + \"census-test.csv\"\n",
    "\n",
    "def download_starter_dataset(bucket_name):\n",
    "    if not os.path.exists(DATASET_PATH + \"census-train.csv\"):\n",
    "        train_data_url = \"http://s3.amazonaws.com/\" + bucket_name + \"/census-train.csv\"\n",
    "        if not os.path.isfile(train_data_filename):\n",
    "            wget.download(train_data_url)\n",
    "\n",
    "    if not os.path.exists(DATASET_PATH + \"census-test.csv\"):\n",
    "        test_data_url = \"http://s3.amazonaws.com/\" + bucket_name + \"/census-test.csv\"\n",
    "        if not os.path.isfile(test_data_filename):\n",
    "            wget.download(test_data_url)\n",
    "\n",
    "download_starter_dataset(\"verta-starter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(train_data_filename)\n",
    "X_train = df_train.iloc[:,:-1]\n",
    "y_train = df_train.iloc[:, -1]\n",
    "\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyperparam_candidates = {\n",
    "    'C': [1e-6, 1e-4],\n",
    "    'solver': ['lbfgs'],\n",
    "    'max_iter': [15, 28],\n",
    "}\n",
    "hyperparam_sets = [dict(zip(hyperparam_candidates.keys(), values))\n",
    "                   for values\n",
    "                   in itertools.product(*hyperparam_candidates.values())]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def run_experiment(hyperparams, index):\n",
    "    from verta.code import Notebook\n",
    "    from verta.configuration import Hyperparameters\n",
    "    from verta.dataset import S3\n",
    "    from verta.environment import Python\n",
    "\n",
    "    code_ver = Notebook()\n",
    "    config_ver = Hyperparameters(hyperparams)\n",
    "    dataset_ver_train = S3(\"s3://{}/{}\".format(\"verta-starter\", \"census-train.csv\"))\n",
    "    dataset_ver_test = S3(\"s3://{}/{}\".format(\"verta-starter\", \"census-test.csv\"))\n",
    "    env_ver = Python()\n",
    "\n",
    "    commit.update(\"notebooks/census-s3-example\", code_ver)\n",
    "    commit.update(\"config/hyperparams\", config_ver)\n",
    "    commit.update(\"data/train-data\", dataset_ver_train)\n",
    "    commit.update(\"data/test-data\", dataset_ver_test)\n",
    "    commit.update(\"env/python\", env_ver)\n",
    "    commit.save(\"Hyperparameter tuning Run: \" + str(index))\n",
    "\n",
    "    # create object to track experiment run\n",
    "    run = client.set_experiment_run()\n",
    "    \n",
    "    # create validation split\n",
    "    (X_val_train, X_val_test,\n",
    "     y_val_train, y_val_test) = model_selection.train_test_split(X_train, y_train,\n",
    "                                                                 test_size=0.2,\n",
    "                                                                 shuffle=True)\n",
    "\n",
    "    # log hyperparameters\n",
    "    run.log_hyperparameters(hyperparams)\n",
    "    print(hyperparams, end=' ')\n",
    "    \n",
    "    # create and train model\n",
    "    model = linear_model.LogisticRegression(**hyperparams)\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    # calculate and log validation accuracy\n",
    "    val_acc = model.score(X_val_test, y_val_test)\n",
    "    run.log_metric(\"val_acc\", val_acc)\n",
    "    print(\"Validation accuracy: {:.4f}\".format(val_acc))\n",
    "    \n",
    "    # save and log model\n",
    "    run.log_model(model)\n",
    "    \n",
    "    # log dataset snapshot as version\n",
    "    run.log_dataset_version(\"train\", version)\n",
    "    \n",
    "    # log Git information as code version\n",
    "    run.log_code()\n",
    "    \n",
    "    run.log_commit(\n",
    "        commit,\n",
    "        {\n",
    "            'notebook': \"notebooks/census-s3-example\",\n",
    "            'hyperparameters': \"config/hyperparams\",\n",
    "            'training_data': \"data/train-data\",\n",
    "            'test_data': \"data/test-data\",\n",
    "            'python_env': \"env/python\",\n",
    "        },\n",
    "    )\n",
    "    \n",
    "for i, hyperparams in enumerate(hyperparam_sets):\n",
    "    run_experiment(hyperparams, i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for c in commit.log():\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "commit.revert()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "master = repo.get_commit(branch='master')\n",
    "master.merge(commit)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Revisit Workflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve Best Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run = expt.expt_runs.sort(\"metrics.val_acc\", descending=True)[0]\n",
    "print(\"Validation Accuracy: {:.4f}\".format(best_run.get_metric(\"val_acc\")))\n",
    "\n",
    "best_hyperparams = best_run.get_hyperparameters()\n",
    "print(\"Hyperparameters: {}\".format(best_hyperparams))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train on Full Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = linear_model.LogisticRegression(multi_class='auto', **best_hyperparams)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate Accuracy on Full Training Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_acc = model.score(X_train, y_train)\n",
    "print(\"Training accuracy: {:.4f}\".format(train_acc))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

{'fraction_non_alphanumeric': 0.21155137676292815, 'fraction_numerical': 0.006312961719274681, 'mean_word_length': 2.4038402925937215, 'pattern_counts': {'":': 310, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1884349', 'n_tokens_mistral': 8451, 'n_tokens_neox': 8203, 'n_words': 1320}