<commit_before>{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Add last boundary to human annotations\n",
      "import jams2\n",
      "import glob\n",
      "import json\n",
      "\n",
      "annotators = {}\n",
      "annotators[\"Colin\"] = {\n",
      "    \"name\"  : \"Colin\",\n",
      "    \"email\" : \"<EMAIL>\"\n",
      "}\n",
      "annotators[\"Eleni\"] = {\n",
      "    \"name\"  : \"Eleni\",\n",
      "    \"email\" : \"<EMAIL>\"\n",
      "}\n",
      "annotators[\"Evan\"] = {\n",
      "    \"name\"  : \"Evan\",\n",
      "    \"email\" : \"<EMAIL>\"\n",
      "}\n",
      "annotators[\"John\"] = {\n",
      "    \"name\"  : \"John\",\n",
      "    \"email\" : \"<EMAIL>\"\n",
      "}\n",
      "annotators[\"Shuli\"] = {\n",
      "    \"name\"  : \"Shuli\",\n",
      "    \"email\" : \"<EMAIL>\"\n",
      "}\n",
      "\n",
      "def update_last_boundary(jam_file, annotator_name, time, context):\n",
      "    jam = jams2.load(jam_file)\n",
      "    for annotation in jam[\"sections\"]:\n",
      "        if annotation.annotation_metadata.annotator.name == annotator_name:\n",
      "            for i, data in enumerate(annotation.data):\n",
      "                if annotation.data[i+1].label.context == \"small_scale\":\n",
      "                    data.end.value = time\n",
      "                    break\n",
      "            break\n",
      "    json.dump(jam, open(jam_file, \"w\"), indent=2)\n",
      "    \n",
      "def add_last_boundary(jam_file, annotator_name, time, context):\n",
      "    jam = jams2.load(jam_file)\n",
      "    for annotation in jam[\"sections\"]:\n",
      "        if annotation.annotation_metadata.annotator.name == annotator_name:\n",
      "            for i, data in enumerate(annotation.data):\n",
      "                if annotation.data[i+1].label.context == \"small_scale\":\n",
      "                    segment = annotation.create_datapoint()\n",
      "                    segment.start.value = data.end.value\n",
      "                    segment.end.value = time\n",
      "                    segment.label.value = \"END\"\n",
      "                    segment.label.context = context\n",
      "                    break\n",
      "            break\n",
      "    json.dump(jam, open(jam_file, \"w\"), indent=2)\n",
      "\n",
      "jam_files = glob.glob(\"/Users/uri/datasets/SubSegments/annotations/*.jams\")\n",
      "context = \"large_scale\"\n",
      "for jam_file in jam_files:\n",
      "    jam = jams2.load(jam_file)\n",
      "    dur = jam.metadata.duration\n",
      "    for key in annotators.keys():\n",
      "        inters, labels = jams2.converters.load_jams_range(jam_file, \"sections\", \n",
      "                                                          annotator_name=annotators[key][\"name\"], \n",
      "                                                          context=context)\n",
      "        if dur - inters[-1,-1] < -.1:\n",
      "            print \"Warning: the last boundary by %s is placed after the track duration! (%s)\" % \\\n",
      "                (key, jam_file)\n",
      "            print dur, inters[-1, -1]\n",
      "            update_last_boundary(jam_file, key, dur, context)\n",
      "        \n",
      "        if np.abs(dur - inters[-1, -1]) > 0.5:\n",
      "            print \"Warning: the last boundary by %s is not placed at the end of the track! (%s)\" % \\\n",
      "                (key, jam_file) \n",
      "            print dur, inters[-1, -1]\n",
      "            add_last_boundary(jam_file, key, dur, context)\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Warning: the last boundary by Eleni is placed after the track duration! (/Users/uri/datasets/SubSegments/annotations/Epiphyte_0780_letmebereal.jams)\n",
        "182.77805 182.880362812\n"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Move estimations to SubSegments folder\n",
      "import glob\n",
      "import os\n",
      "import shutil\n",
      "\n",
      "jam_files = glob.glob(\"/Users/uri/datasets/SubSegments/annotations/*.jams\")\n",
      "for jam_file in jam_files:\n",
      "    orig = \"/Users/uri/datasets/Segments/estimations/\" + os.path.basename(jam_file)[:-5] + \".json\"\n",
      "    dest = \"/Users/uri/datasets/SubSegments/estimations/\" + os.path.basename(orig)\n",
      "    shutil.copy(orig, dest)\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Experiment 1: Dur x #bounds x score\n",
      "import sys\n",
      "import jams2\n",
      "import os\n",
      "import glob\n",
      "\n",
      "sys.path.append(\"..\")\n",
      "import eval as EV\n",
      "import msaf_io as MSAF\n",
      "\n",
      "# max_dur: 1728.257007 (SALAMI_878)\n",
      "# max #boundaries: 75 (SALAMI_888)\n",
      "N = 76\n",
      "M = 100\n",
      "# max_M = 1730.\n",
      "max_M = 2000.\n",
      "\n",
      "MAX_N = 138\n",
      "MAX_M = 149\n",
      "\n",
      "algos = [\"olda\", \"siplca\", \"serra\", \"levy\", \"foote\"]\n",
      "trim = False\n",
      "params_dict = {\"olda\" : \"\", \"siplca\" : \"\", \"serra\" : \"mix\",\n",
      "               \"levy\" : \"mfcc\" , \"foote\" : \"mfcc\"}\n",
      "bins = 250\n",
      "\n",
      "in_path = \"/Users/uri/datasets/Segments/\"\n",
      "est_files = glob.glob(os.path.join(in_path, \"estimations\", \"*.json\"))\n",
      "jam_files = glob.glob(os.path.join(in_path, \"annotations\", \"*.jams\"))\n",
      "\n",
      "\n",
      "heat_map = np.zeros((9, N, M))\n",
      "counts = np.zeros((N, M))\n",
      "for est_file in est_files:\n",
      "    ds_prefix = os.path.basename(est_file).split(\"_\")[0]\n",
      "\n",
      "    # Get corresponding annotation file\n",
      "    jam_file = EV.get_annotation(est_file, jam_files)\n",
      "\n",
      "    # Get number of bounds\n",
      "    try:\n",
      "        ref_inter, ref_labels = jams2.converters.load_jams_range(jam_file,\n",
      "            \"sections\", annotator=0, context=MSAF.prefix_dict[ds_prefix])\n",
      "    except:\n",
      "        print \"No annotation for %s, skipping.\" % jam_file\n",
      "        continue\n",
      "        \n",
      "    n = np.min([len(ref_inter) + 1, MAX_N])\n",
      "    \n",
      "    # Get duration\n",
      "    jam = jams2.load(jam_file)\n",
      "    dur = jam.metadata.duration\n",
      "\n",
      "    # Place duration into correct bin\n",
      "    m = np.min([int(dur / max_M * M), MAX_M])\n",
      "    \n",
      "    # Compute score\n",
      "    score = []\n",
      "    for algo_id in algos:\n",
      "        params = {\"feature\" : params_dict[algo_id]}\n",
      "        est_inter = MSAF.read_boundaries(est_file, algo_id, False, **params)\n",
      "        res = EV.compute_results(ref_inter, est_inter, trim, bins, est_file)\n",
      "        score.append(res)\n",
      "    score = np.mean(np.asarray(score), axis=0)\n",
      "    \n",
      "    # Add to heat map\n",
      "    heat_map[:, n, m] += score\n",
      "    counts[n, m] += 1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1030.jams\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1030.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1040.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1040.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1052.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1052.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1126.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1126.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1140.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1140.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1178.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1178.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1320.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1320.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1398.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1398.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1410.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1410.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1426.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1426.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1430.jams\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1430.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1440.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1440.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1466.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1466.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1486.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1486.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1500.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1500.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_872.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_872.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_918.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_918.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_966.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_966.jams, skipping.\n"
       ]
      }
     ],
     "prompt_number": 128
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print heat_map\n",
      "\n",
      "metric_dict = {\"F3\" : 2, \"F05\" : 5, \"D\" : 6, \"$\\sigma_{R2E}$\" : 7,\n",
      "               \"$\\sigma_{E2R}$\" : 8}\n",
      "metric = \"$\\sigma_{R2E}$\"\n",
      "\n",
      "mN = 40\n",
      "mM = 40\n",
      "heat_map[np.isnan(heat_map)] = 0\n",
      "heat_map_metric = np.zeros((mN, mM))\n",
      "for i in xrange(heat_map[metric_dict[metric], :, :].shape[0]):\n",
      "    for j in xrange(heat_map[metric_dict[metric], :, :].shape[1]):\n",
      "        heat_map_metric[np.min([i, mN-1]), np.min([j, mM-1])] += heat_map[metric_dict[metric], i, j]\n",
      "idx = np.where(heat_map_metric == 0)\n",
      "heat_map_metric[idx] = np.nan\n",
      "\n",
      "\n",
      "# Plotting\n",
      "figsize = (4, 3)\n",
      "plt.figure(1, figsize=figsize, dpi=120, facecolor='w', edgecolor='k')\n",
      "plt.imshow(heat_map_metric, interpolation=\"nearest\", aspect=\"auto\", cmap=\"hot\")\n",
      "plt.xlabel(\"Duration (seconds)\")\n",
      "plt.ylabel(\"Number of boundaries\")\n",
      "plt.title(\"#boundaries vs duration using %s\" % metric)\n",
      "plt.gcf().subplots_adjust(bottom=0.18)\n",
      "plt.xticks(np.arange(0, mM, 10))\n",
      "plt.gca().set_xticklabels(np.arange(0, mM+1, 10, dtype=int) / float(M) * max_M)\n",
      "plt.show()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
        "\n",
        " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
        "\n",
        " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
        "\n",
        " ..., \n",
        " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
        "\n",
        " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
        "\n",
        " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]]\n"
       ]
      }
     ],
     "prompt_number": 169
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Experiment 2a: Obtain evaluation for all the different subjects\n",
      "import sys\n",
      "sys.path.append(\"..\")\n",
      "import eval as EV\n",
      "\n",
      "reload(EV)\n",
      "\n",
      "def print_results(results):\n",
      "    \"\"\"Print all the results.\n",
      "\n",
      "    Parameters\n",
      "    ----------\n",
      "    results: np.array(9)\n",
      "        Results in the following format:\n",
      "            0   :   Precision 3 seconds\n",
      "            1   :   Recall 3 seconds\n",
      "            2   :   F-measure 3 seconds\n",
      "            3   :   Precision 0.5 seconds\n",
      "            4   :   Recall 0.5 seconds\n",
      "            5   :   F-measure 0.5 seconds\n",
      "            6   :   Information Gain\n",
      "            7   :   Median Deviation from Annotated to Estimated boundary\n",
      "            8   :   Median Deviation from Estimated to Annotated boundary\n",
      "    \"\"\"\n",
      "    results = np.asarray(results)\n",
      "    res = results.mean(axis=0)\n",
      "    print \"F3: %.2f, P3: %.2f, R3: %.2f, F05: %.2f, P05: %.2f, \" \\\n",
      "                 \"R05: %.2f, D: %.4f, Ann2EstDev: %.2f, Est2AnnDev: %.2f\" % \\\n",
      "                 (100 * res[2], 100 * res[0], 100 * res[1], 100 * res[5],\n",
      "                  100 * res[3], 100 * res[4], res[6], res[7], res[8])\n",
      "\n",
      "\n",
      "in_path = \"/Users/uri/datasets/SubSegments/\"\n",
      "algos = [\"olda\", \"siplca\", \"serra\", \"levy\", \"foote\"]\n",
      "trim = False\n",
      "params_dict = {\"olda\" : \"\", \"siplca\" : \"\", \"serra\" : \"mix\",\n",
      "               \"levy\" : \"mfcc\" , \"foote\" : \"mfcc\"}\n",
      "annotators = [\"GT\", \"Colin\", \"Eleni\", \"Evan\", \"John\", \"Shuli\"]\n",
      "\n",
      "for annotator in annotators:\n",
      "    res = []\n",
      "    for algo_id in algos:\n",
      "        params = {\"feature\" : params_dict[algo_id]}\n",
      "        results = EV.process(in_path, algo_id, trim=trim, annotator=annotator, \n",
      "                             **params)\n",
      "        res.append(np.mean(results, axis=0))\n",
      "    print annotator\n",
      "    print_results(res)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "GT\n",
        "F3: 34.66, P3: 37.47, R3: 40.11, F05: 23.17, P05: 24.97, R05: 26.89, D: 0.5258, Ann2EstDev: 6.49, Est2AnnDev: 9.83\n",
        "Colin"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/SALAMI_78.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/SALAMI_78.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/SALAMI_78.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/SALAMI_78.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/SALAMI_78.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "F3: 40.29, P3: 41.27, R3: 48.27, F05: 26.49, P05: 27.29, R05: 31.45, D: 0.5289, Ann2EstDev: 6.24, Est2AnnDev: 8.37\n",
        "Eleni"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "F3: 38.47, P3: 41.98, R3: 42.53, F05: 24.62, P05: 27.18, R05: 26.79, D: 0.5149, Ann2EstDev: 6.04, Est2AnnDev: 7.87\n",
        "Evan"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "F3: 38.64, P3: 39.20, R3: 45.54, F05: 25.03, P05: 25.90, R05: 29.05, D: 0.5351, Ann2EstDev: 5.31, Est2AnnDev: 9.08\n",
        "John"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "F3: 39.02, P3: 36.70, R3: 49.89, F05: 26.59, P05: 25.35, R05: 33.26, D: 0.5471, Ann2EstDev: 6.54, Est2AnnDev: 8.54\n",
        "Shuli"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "F3: 38.39, P3: 36.87, R3: 47.99, F05: 25.94, P05: 25.25, R05: 31.65, D: 0.5358, Ann2EstDev: 5.04, Est2AnnDev: 9.86\n"
       ]
      }
     ],
     "prompt_number": 55
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Experiment 2b: Compare annotations\n",
      "import glob\n",
      "import sys\n",
      "sys.path.append(\"..\")\n",
      "sys.path.append(\"../experiment/\")\n",
      "import eval as EV\n",
      "import analyze_results as AR\n",
      "import json\n",
      "\n",
      "reload(EV)\n",
      "reload(AR)\n",
      "\n",
      "metric_dict = {\"F3\" : 2, \"F05\" : 5, \"D\" : 6, \"$\\sigma_{R2E}$\" : 7,\n",
      "               \"$\\sigma_{E2R}$\" : 8}\n",
      "metric = \"$\\sigma_{E2R}$\"\n",
      "trim = True\n",
      "N = 6 # 5 annotators\n",
      "X = np.empty((0, N, N))\n",
      "jam_files = glob.glob(\"/Users/uri/datasets/SubSegments/annotations/*.jams\")\n",
      "for jam_file in jam_files:\n",
      "    mma_res = AR.compute_mma_results(jam_file, AR.annotators, trim, gt=True)\n",
      "    x = np.zeros((N, N))\n",
      "    idx = np.triu_indices(N, k=1)\n",
      "    x[idx] = mma_res[:, metric_dict[metric]]\n",
      "    X = np.append(X, [x], axis=0)\n",
      "    \n",
      "\n",
      "X_mean = np.mean(X, axis=0)\n",
      "X_mean = X_mean + X_mean.T\n",
      "np.fill_diagonal(X_mean, 1)\n",
      "print X_mean\n",
      "vmax = np.sort(np.unique(X_mean))[-2]\n",
      "\n",
      "figsize = (4, 3)\n",
      "annots = [\"GT\", \"Ann1\", \"Ann2\", \"Ann3\", \"Ann4\", \"Ann5\"]\n",
      "plt.figure(1, figsize=figsize, dpi=120, facecolor='w', edgecolor='k')\n",
      "# plt.imshow(X_mean, interpolation=\"nearest\", aspect=\"auto\", cmap=\"hot\", vmin=X_mean.min(), vmax=vmax)\n",
      "plt.imshow(X_mean, interpolation=\"nearest\", aspect=\"auto\", cmap=\"hot\")\n",
      "print json.dumps(X_mean.tolist())\n",
      "plt.gca().set_xticks(np.arange(0,6))\n",
      "plt.gca().set_yticks(np.arange(0,6))\n",
      "plt.gca().set_xticklabels(annots)\n",
      "plt.gca().set_yticklabels(annots)\n",
      "plt.title(\"Annotators Agreement for %s\" % metric)\n",
      "plt.colorbar()\n",
      "plt.show()\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/SALAMI_78.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/SALAMI_78.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/SALAMI_78.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/SALAMI_78.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/SALAMI_78.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[ 1.          6.51506481  6.28387029  4.5007873   8.1014369   4.83731244]\n",
        " [ 6.51506481  1.          6.54933878  6.95230113  7.00937868  6.18588299]\n",
        " [ 6.28387029  6.54933878  1.          2.53599773  5.1941805   2.17176635]\n",
        " [ 4.5007873   6.95230113  2.53599773  1.          6.57837506  5.50191383]\n",
        " [ 8.1014369   7.00937868  5.1941805   6.57837506  1.          7.52985034]\n",
        " [ 4.83731244  6.18588299  2.17176635  5.50191383  7.52985034  1.        ]]\n",
        "[[1.0, 6.515064811744244, 6.283870293285938, 4.500787299735735, 8.101436895338232, 4.837312437725352], [6.515064811744244, 1.0, 6.549338775510205, 6.952301133786848, 7.009378684807254, 6.1858829931972785], [6.283870293285938, 6.549338775510205, 1.0, 2.535997732426303, 5.194180498866213, 2.171766346938772], [4.500787299735735, 6.952301133786848, 2.535997732426303, 1.0, 6.578375056689343, 5.501913832199545], [8.101436895338232, 7.009378684807254, 5.194180498866213, 6.578375056689343, 1.0, 7.529850340136056], [4.837312437725352, 6.1858829931972785, 2.171766346938772, 5.501913832199545, 7.529850340136056, 1.0]]\n"
       ]
      }
     ],
     "prompt_number": 56
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Experiment 3: compare all the annotated boundaries to come up with a more robust reference\n",
      "import glob\n",
      "import sys\n",
      "sys.path.append(\"..\")\n",
      "sys.path.append(\"../experiment/\")\n",
      "import eval as EV\n",
      "import analyze_results as AR\n",
      "import json\n",
      "\n",
      "reload(EV)\n",
      "reload(AR)\n",
      "\n",
      "metric_dict = {\"F3\" : 2, \"F05\" : 5, \"D\" : 6, \"$\\sigma_{R2E}$\" : 7,\n",
      "               \"$\\sigma_{E2R}$\" : 8}\n",
      "metric = \"$\\sigma_{E2R}$\"\n",
      "trim = True\n",
      "N = 6 # 5 annotators\n",
      "X = np.empty((0, N, N))\n",
      "jam_files = glob.glob(\"/Users/uri/datasets/SubSegments/annotations/*.jams\")"
     ],<commit_msg>Data for experiment 3 written<commit_after>{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Add last boundary to human annotations\n",
      "import jams2\n",
      "import glob\n",
      "import json\n",
      "\n",
      "annotators = {}\n",
      "annotators[\"Colin\"] = {\n",
      "    \"name\"  : \"Colin\",\n",
      "    \"email\" : \"<EMAIL>\"\n",
      "}\n",
      "annotators[\"Eleni\"] = {\n",
      "    \"name\"  : \"Eleni\",\n",
      "    \"email\" : \"<EMAIL>\"\n",
      "}\n",
      "annotators[\"Evan\"] = {\n",
      "    \"name\"  : \"Evan\",\n",
      "    \"email\" : \"<EMAIL>\"\n",
      "}\n",
      "annotators[\"John\"] = {\n",
      "    \"name\"  : \"John\",\n",
      "    \"email\" : \"<EMAIL>\"\n",
      "}\n",
      "annotators[\"Shuli\"] = {\n",
      "    \"name\"  : \"Shuli\",\n",
      "    \"email\" : \"<EMAIL>\"\n",
      "}\n",
      "\n",
      "def update_last_boundary(jam_file, annotator_name, time, context):\n",
      "    jam = jams2.load(jam_file)\n",
      "    for annotation in jam[\"sections\"]:\n",
      "        if annotation.annotation_metadata.annotator.name == annotator_name:\n",
      "            for i, data in enumerate(annotation.data):\n",
      "                if annotation.data[i+1].label.context == \"small_scale\":\n",
      "                    data.end.value = time\n",
      "                    break\n",
      "            break\n",
      "    json.dump(jam, open(jam_file, \"w\"), indent=2)\n",
      "    \n",
      "def add_last_boundary(jam_file, annotator_name, time, context):\n",
      "    jam = jams2.load(jam_file)\n",
      "    for annotation in jam[\"sections\"]:\n",
      "        if annotation.annotation_metadata.annotator.name == annotator_name:\n",
      "            for i, data in enumerate(annotation.data):\n",
      "                if annotation.data[i+1].label.context == \"small_scale\":\n",
      "                    segment = annotation.create_datapoint()\n",
      "                    segment.start.value = data.end.value\n",
      "                    segment.end.value = time\n",
      "                    segment.label.value = \"END\"\n",
      "                    segment.label.context = context\n",
      "                    break\n",
      "            break\n",
      "    json.dump(jam, open(jam_file, \"w\"), indent=2)\n",
      "\n",
      "jam_files = glob.glob(\"/Users/uri/datasets/SubSegments/annotations/*.jams\")\n",
      "context = \"large_scale\"\n",
      "for jam_file in jam_files:\n",
      "    jam = jams2.load(jam_file)\n",
      "    dur = jam.metadata.duration\n",
      "    for key in annotators.keys():\n",
      "        inters, labels = jams2.converters.load_jams_range(jam_file, \"sections\", \n",
      "                                                          annotator_name=annotators[key][\"name\"], \n",
      "                                                          context=context)\n",
      "        if dur - inters[-1,-1] < -.1:\n",
      "            print \"Warning: the last boundary by %s is placed after the track duration! (%s)\" % \\\n",
      "                (key, jam_file)\n",
      "            print dur, inters[-1, -1]\n",
      "            update_last_boundary(jam_file, key, dur, context)\n",
      "        \n",
      "        if np.abs(dur - inters[-1, -1]) > 0.5:\n",
      "            print \"Warning: the last boundary by %s is not placed at the end of the track! (%s)\" % \\\n",
      "                (key, jam_file) \n",
      "            print dur, inters[-1, -1]\n",
      "            add_last_boundary(jam_file, key, dur, context)\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Warning: the last boundary by Eleni is placed after the track duration! (/Users/uri/datasets/SubSegments/annotations/Epiphyte_0780_letmebereal.jams)\n",
        "182.77805 182.880362812\n"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Move estimations to SubSegments folder\n",
      "import glob\n",
      "import os\n",
      "import shutil\n",
      "\n",
      "jam_files = glob.glob(\"/Users/uri/datasets/SubSegments/annotations/*.jams\")\n",
      "for jam_file in jam_files:\n",
      "    orig = \"/Users/uri/datasets/Segments/estimations/\" + os.path.basename(jam_file)[:-5] + \".json\"\n",
      "    dest = \"/Users/uri/datasets/SubSegments/estimations/\" + os.path.basename(orig)\n",
      "    shutil.copy(orig, dest)\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Experiment 1: Dur x #bounds x score\n",
      "import sys\n",
      "import jams2\n",
      "import os\n",
      "import glob\n",
      "\n",
      "sys.path.append(\"..\")\n",
      "import eval as EV\n",
      "import msaf_io as MSAF\n",
      "\n",
      "# max_dur: 1728.257007 (SALAMI_878)\n",
      "# max #boundaries: 75 (SALAMI_888)\n",
      "N = 76\n",
      "M = 100\n",
      "# max_M = 1730.\n",
      "max_M = 2000.\n",
      "\n",
      "MAX_N = 138\n",
      "MAX_M = 149\n",
      "\n",
      "algos = [\"olda\", \"siplca\", \"serra\", \"levy\", \"foote\"]\n",
      "trim = False\n",
      "params_dict = {\"olda\" : \"\", \"siplca\" : \"\", \"serra\" : \"mix\",\n",
      "               \"levy\" : \"mfcc\" , \"foote\" : \"mfcc\"}\n",
      "bins = 250\n",
      "\n",
      "in_path = \"/Users/uri/datasets/Segments/\"\n",
      "est_files = glob.glob(os.path.join(in_path, \"estimations\", \"*.json\"))\n",
      "jam_files = glob.glob(os.path.join(in_path, \"annotations\", \"*.jams\"))\n",
      "\n",
      "\n",
      "heat_map = np.zeros((9, N, M))\n",
      "counts = np.zeros((N, M))\n",
      "for est_file in est_files:\n",
      "    ds_prefix = os.path.basename(est_file).split(\"_\")[0]\n",
      "\n",
      "    # Get corresponding annotation file\n",
      "    jam_file = EV.get_annotation(est_file, jam_files)\n",
      "\n",
      "    # Get number of bounds\n",
      "    try:\n",
      "        ref_inter, ref_labels = jams2.converters.load_jams_range(jam_file,\n",
      "            \"sections\", annotator=0, context=MSAF.prefix_dict[ds_prefix])\n",
      "    except:\n",
      "        print \"No annotation for %s, skipping.\" % jam_file\n",
      "        continue\n",
      "        \n",
      "    n = np.min([len(ref_inter) + 1, MAX_N])\n",
      "    \n",
      "    # Get duration\n",
      "    jam = jams2.load(jam_file)\n",
      "    dur = jam.metadata.duration\n",
      "\n",
      "    # Place duration into correct bin\n",
      "    m = np.min([int(dur / max_M * M), MAX_M])\n",
      "    \n",
      "    # Compute score\n",
      "    score = []\n",
      "    for algo_id in algos:\n",
      "        params = {\"feature\" : params_dict[algo_id]}\n",
      "        est_inter = MSAF.read_boundaries(est_file, algo_id, False, **params)\n",
      "        res = EV.compute_results(ref_inter, est_inter, trim, bins, est_file)\n",
      "        score.append(res)\n",
      "    score = np.mean(np.asarray(score), axis=0)\n",
      "    \n",
      "    # Add to heat map\n",
      "    heat_map[:, n, m] += score\n",
      "    counts[n, m] += 1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1030.jams\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1030.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1040.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1040.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1052.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1052.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1126.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1126.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1140.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1140.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1178.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1178.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1320.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1320.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1398.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1398.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1410.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1410.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1426.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1426.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1430.jams\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1430.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1440.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1440.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1466.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1466.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1486.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1486.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_1500.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_1500.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_872.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_872.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_918.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_918.jams, skipping.\n",
        "Warning: sections empty in file /Users/uri/datasets/Segments/annotations/SALAMI_966.jams"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "No annotation for /Users/uri/datasets/Segments/annotations/SALAMI_966.jams, skipping.\n"
       ]
      }
     ],
     "prompt_number": 128
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Experiment 1 (cont)\n",
      "\n",
      "metric_dict = {\"F3\" : 2, \"F05\" : 5, \"D\" : 6, \"$\\sigma_{R2E}$\" : 7,\n",
      "               \"$\\sigma_{E2R}$\" : 8}\n",
      "metric = \"$\\sigma_{E2R}$\"\n",
      "\n",
      "# Reduce data\n",
      "mN = 40\n",
      "mM = 40\n",
      "heat_map[np.isnan(heat_map)] = 0\n",
      "heat_map_metric = np.zeros((mN, mM))\n",
      "for i in xrange(heat_map[metric_dict[metric], :, :].shape[0]):\n",
      "    for j in xrange(heat_map[metric_dict[metric], :, :].shape[1]):\n",
      "        heat_map_metric[np.min([i, mN-1]), np.min([j, mM-1])] += heat_map[metric_dict[metric], i, j]\n",
      "idx = np.where(heat_map_metric == 0)\n",
      "heat_map_metric[idx] = np.nan\n",
      "\n",
      "# Plotting\n",
      "figsize = (4, 3)\n",
      "plt.figure(1, figsize=figsize, dpi=120, facecolor='w', edgecolor='k')\n",
      "plt.imshow(heat_map_metric, interpolation=\"nearest\", aspect=\"auto\", cmap=\"hot\")\n",
      "plt.xlabel(\"Duration (seconds)\")\n",
      "plt.ylabel(\"Number of boundaries\")\n",
      "plt.title(\"#boundaries vs duration using %s\" % metric)\n",
      "plt.gcf().subplots_adjust(bottom=0.18)\n",
      "plt.xticks(np.arange(0, mM, 10))\n",
      "plt.gca().set_xticklabels(np.arange(0, mM+1, 10, dtype=int) / float(M) * max_M)\n",
      "plt.show()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
        "\n",
        " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
        "\n",
        " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
        "\n",
        " ..., \n",
        " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
        "\n",
        " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
        "\n",
        " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  ..., \n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
        "  [ 0.  0.  0. ...,  0.  0.  0.]]]\n"
       ]
      }
     ],
     "prompt_number": 170
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Experiment 2a: Obtain evaluation for all the different subjects\n",
      "import sys\n",
      "sys.path.append(\"..\")\n",
      "import eval as EV\n",
      "\n",
      "reload(EV)\n",
      "\n",
      "def print_results(results):\n",
      "    \"\"\"Print all the results.\n",
      "\n",
      "    Parameters\n",
      "    ----------\n",
      "    results: np.array(9)\n",
      "        Results in the following format:\n",
      "            0   :   Precision 3 seconds\n",
      "            1   :   Recall 3 seconds\n",
      "            2   :   F-measure 3 seconds\n",
      "            3   :   Precision 0.5 seconds\n",
      "            4   :   Recall 0.5 seconds\n",
      "            5   :   F-measure 0.5 seconds\n",
      "            6   :   Information Gain\n",
      "            7   :   Median Deviation from Annotated to Estimated boundary\n",
      "            8   :   Median Deviation from Estimated to Annotated boundary\n",
      "    \"\"\"\n",
      "    results = np.asarray(results)\n",
      "    res = results.mean(axis=0)\n",
      "    print \"F3: %.2f, P3: %.2f, R3: %.2f, F05: %.2f, P05: %.2f, \" \\\n",
      "                 \"R05: %.2f, D: %.4f, Ann2EstDev: %.2f, Est2AnnDev: %.2f\" % \\\n",
      "                 (100 * res[2], 100 * res[0], 100 * res[1], 100 * res[5],\n",
      "                  100 * res[3], 100 * res[4], res[6], res[7], res[8])\n",
      "\n",
      "\n",
      "in_path = \"/Users/uri/datasets/SubSegments/\"\n",
      "algos = [\"olda\", \"siplca\", \"serra\", \"levy\", \"foote\"]\n",
      "trim = False\n",
      "params_dict = {\"olda\" : \"\", \"siplca\" : \"\", \"serra\" : \"mix\",\n",
      "               \"levy\" : \"mfcc\" , \"foote\" : \"mfcc\"}\n",
      "annotators = [\"GT\", \"Colin\", \"Eleni\", \"Evan\", \"John\", \"Shuli\"]\n",
      "\n",
      "for annotator in annotators:\n",
      "    res = []\n",
      "    for algo_id in algos:\n",
      "        params = {\"feature\" : params_dict[algo_id]}\n",
      "        results = EV.process(in_path, algo_id, trim=trim, annotator=annotator, \n",
      "                             **params)\n",
      "        res.append(np.mean(results, axis=0))\n",
      "    print annotator\n",
      "    print_results(res)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "GT\n",
        "F3: 34.66, P3: 37.47, R3: 40.11, F05: 23.17, P05: 24.97, R05: 26.89, D: 0.5258, Ann2EstDev: 6.49, Est2AnnDev: 9.83\n",
        "Colin"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/SALAMI_78.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/SALAMI_78.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/SALAMI_78.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/SALAMI_78.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/SALAMI_78.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "F3: 40.29, P3: 41.27, R3: 48.27, F05: 26.49, P05: 27.29, R05: 31.45, D: 0.5289, Ann2EstDev: 6.24, Est2AnnDev: 8.37\n",
        "Eleni"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "F3: 38.47, P3: 41.98, R3: 42.53, F05: 24.62, P05: 27.18, R05: 26.79, D: 0.5149, Ann2EstDev: 6.04, Est2AnnDev: 7.87\n",
        "Evan"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "F3: 38.64, P3: 39.20, R3: 45.54, F05: 25.03, P05: 25.90, R05: 29.05, D: 0.5351, Ann2EstDev: 5.31, Est2AnnDev: 9.08\n",
        "John"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/estimations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.json\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "F3: 39.02, P3: 36.70, R3: 49.89, F05: 26.59, P05: 25.35, R05: 33.26, D: 0.5471, Ann2EstDev: 6.54, Est2AnnDev: 8.54\n",
        "Shuli"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "F3: 38.39, P3: 36.87, R3: 47.99, F05: 25.94, P05: 25.25, R05: 31.65, D: 0.5358, Ann2EstDev: 5.04, Est2AnnDev: 9.86\n"
       ]
      }
     ],
     "prompt_number": 55
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Experiment 2b: Compare annotations\n",
      "import glob\n",
      "import sys\n",
      "sys.path.append(\"..\")\n",
      "sys.path.append(\"../experiment/\")\n",
      "import eval as EV\n",
      "import analyze_results as AR\n",
      "import json\n",
      "\n",
      "reload(EV)\n",
      "reload(AR)\n",
      "\n",
      "metric_dict = {\"F3\" : 2, \"F05\" : 5, \"D\" : 6, \"$\\sigma_{R2E}$\" : 7,\n",
      "               \"$\\sigma_{E2R}$\" : 8}\n",
      "metric = \"$\\sigma_{E2R}$\"\n",
      "trim = True\n",
      "N = 6 # 5 annotators\n",
      "X = np.empty((0, N, N))\n",
      "jam_files = glob.glob(\"/Users/uri/datasets/SubSegments/annotations/*.jams\")\n",
      "for jam_file in jam_files:\n",
      "    mma_res = AR.compute_mma_results(jam_file, AR.annotators, trim, gt=True)\n",
      "    x = np.zeros((N, N))\n",
      "    idx = np.triu_indices(N, k=1)\n",
      "    x[idx] = mma_res[:, metric_dict[metric]]\n",
      "    X = np.append(X, [x], axis=0)\n",
      "    \n",
      "\n",
      "X_mean = np.mean(X, axis=0)\n",
      "X_mean = X_mean + X_mean.T\n",
      "np.fill_diagonal(X_mean, 1)\n",
      "print X_mean\n",
      "vmax = np.sort(np.unique(X_mean))[-2]\n",
      "\n",
      "figsize = (4, 3)\n",
      "annots = [\"GT\", \"Ann1\", \"Ann2\", \"Ann3\", \"Ann4\", \"Ann5\"]\n",
      "plt.figure(1, figsize=figsize, dpi=120, facecolor='w', edgecolor='k')\n",
      "# plt.imshow(X_mean, interpolation=\"nearest\", aspect=\"auto\", cmap=\"hot\", vmin=X_mean.min(), vmax=vmax)\n",
      "plt.imshow(X_mean, interpolation=\"nearest\", aspect=\"auto\", cmap=\"hot\")\n",
      "print json.dumps(X_mean.tolist())\n",
      "plt.gca().set_xticks(np.arange(0,6))\n",
      "plt.gca().set_yticks(np.arange(0,6))\n",
      "plt.gca().set_xticklabels(annots)\n",
      "plt.gca().set_yticklabels(annots)\n",
      "plt.title(\"Annotators Agreement for %s\" % metric)\n",
      "plt.colorbar()\n",
      "plt.show()\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/Cerulean_Boston_Symphony_Orchestra_&_Charles_Munch-Sympho.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/SALAMI_78.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/SALAMI_78.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/SALAMI_78.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/SALAMI_78.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:root:Couldn't compute the Information Gain for file /Users/uri/datasets/SubSegments/annotations/SALAMI_78.jams\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[ 1.          6.51506481  6.28387029  4.5007873   8.1014369   4.83731244]\n",
        " [ 6.51506481  1.          6.54933878  6.95230113  7.00937868  6.18588299]\n",
        " [ 6.28387029  6.54933878  1.          2.53599773  5.1941805   2.17176635]\n",
        " [ 4.5007873   6.95230113  2.53599773  1.          6.57837506  5.50191383]\n",
        " [ 8.1014369   7.00937868  5.1941805   6.57837506  1.          7.52985034]\n",
        " [ 4.83731244  6.18588299  2.17176635  5.50191383  7.52985034  1.        ]]\n",
        "[[1.0, 6.515064811744244, 6.283870293285938, 4.500787299735735, 8.101436895338232, 4.837312437725352], [6.515064811744244, 1.0, 6.549338775510205, 6.952301133786848, 7.009378684807254, 6.1858829931972785], [6.283870293285938, 6.549338775510205, 1.0, 2.535997732426303, 5.194180498866213, 2.171766346938772], [4.500787299735735, 6.952301133786848, 2.535997732426303, 1.0, 6.578375056689343, 5.501913832199545], [8.101436895338232, 7.009378684807254, 5.194180498866213, 6.578375056689343, 1.0, 7.529850340136056], [4.837312437725352, 6.1858829931972785, 2.171766346938772, 5.501913832199545, 7.529850340136056, 1.0]]\n"
       ]
      }
     ],
     "prompt_number": 56
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Experiment 3: compare all the annotated boundaries to come up with a more robust reference\n",
      "import glob\n",
      "import sys\n",
      "import os\n",
      "sys.path.append(\"..\")\n",
      "sys.path.append(\"../experiment/\")\n",
      "\n",
      "import eval as EV\n",
      "import analyze_results as AR\n",
      "import msaf_io as MSAF\n",
      "import json\n",
      "\n",
      "reload(EV)\n",
      "reload(AR)\n",
      "\n",
      "def create_weighted_annot(jam, weights, histo_times, jam_file):\n",
      "    # Annot\n",
      "    annot_name = \"Weighted\"\n",
      "    annot = None\n",
      "    for annotation in jam.sections:\n",
      "        if annotation.annotation_metadata.annotator.name == annot_name:\n",
      "            annotation.data = []\n",
      "            annot = annotation\n",
      "            break\n",
      "    if annot is None:\n",
      "        annot = jam.sections.create_annotation()\n",
      "        annot.annotation_metadata.origin = \"Synth\"\n",
      "        annot.annotation_metadata.annotator.name = annot_name\n",
      "        annot.annotation_metadata.annotator.email = \"\"\n",
      "    \n",
      "    bound_times = []\n",
      "    bound_weights = []\n",
      "    for i, weight in enumerate(weights):\n",
      "        if weight != 0:\n",
      "            bound_times.append(histo_times[i])\n",
      "            bound_weights.append(weights[i])\n",
      "    for i, bound in enumerate(bound_times):\n",
      "        if i == 0:\n",
      "            continue\n",
      "        section = annot.create_datapoint()\n",
      "        section.start.value = bound_times[i-1]\n",
      "        section.start.confidence = bound_weights[i-1]\n",
      "        section.end.value = bound_times[i]\n",
      "        section.end.confidence = bound_weights[i]\n",
      "        section.label.context = \"synth\"\n",
      "    \n",
      "    json.dump(jam, open(jam_file, \"w\"), indent=2)\n",
      "    \n",
      "\n",
      "def create_thres_annot(jam, thresh_bounds, thresh_conf, jam_file):\n",
      "    # Annot\n",
      "    annot_name = \"Threshold\"\n",
      "    annot = None\n",
      "    for annotation in jam.sections:\n",
      "        if annotation.annotation_metadata.annotator.name == annot_name:\n",
      "            annotation.data = []\n",
      "            annot = annotation\n",
      "            break\n",
      "    if annot is None:\n",
      "        annot = jam.sections.create_annotation()\n",
      "        annot.annotation_metadata.origin = \"Synth\"\n",
      "        annot.annotation_metadata.annotator.name = annot_name\n",
      "        annot.annotation_metadata.annotator.email = \"\"\n",
      "    \n",
      "    for i, bound in enumerate(thresh_bounds):\n",
      "        if i == 0:\n",
      "            continue\n",
      "        section = annot.create_datapoint()\n",
      "        section.start.value = thresh_bounds[i-1]\n",
      "        section.start.confidence = thresh_conf[i-1]\n",
      "        section.end.value = thresh_bounds[i]\n",
      "        section.end.confidence = thresh_conf[i]\n",
      "        section.label.context = \"synth\"\n",
      "    \n",
      "    json.dump(jam, open(jam_file, \"w\"), indent=2)\n",
      "\n",
      "    \n",
      "annotators = [\"GT\", \"Colin\", \"Eleni\", \"Evan\", \"John\", \"Shuli\"]\n",
      "histo_bins = 500\n",
      "N = len(annotators)\n",
      "X = np.empty((0, N, N))\n",
      "jam_files = glob.glob(\"/Users/uri/datasets/SubSegments/annotations/*.jams\")\n",
      "for jam_file in jam_files:\n",
      "    # Get Duration\n",
      "    jam = jams2.load(jam_file)\n",
      "    dur = jam.metadata.duration\n",
      "    \n",
      "    # Compute the weighted boundaries\n",
      "    boundaries = []\n",
      "    ds_prefix = os.path.basename(jam_file).split(\"_\")[0]\n",
      "    for annot in annotators:\n",
      "        if annot == \"GT\":\n",
      "            ann_inter, ann_labels = jams2.converters.load_jams_range(jam_file,\n",
      "                \"sections\", annotator=0, context=MSAF.prefix_dict[ds_prefix])\n",
      "        else:\n",
      "            ann_inter, ann_labels = jams2.converters.load_jams_range(jam_file,\n",
      "                \"sections\", annotator_name=annot, context=\"large_scale\")\n",
      "        ann_times = EV.intervals_to_times(ann_inter)\n",
      "        histo_bounds, histo_times = np.histogram(ann_times, bins=histo_bins, range=(0, dur))\n",
      "        boundaries.append(histo_bounds)\n",
      "    \n",
      "    weights = np.mean(boundaries, axis=0)\n",
      "    weights /= weights.max()\n",
      "    \n",
      "    # Save weighted boundaries\n",
      "    create_weighted_annot(jam, weights, histo_times, jam_file)\n",
      "    \n",
      "    # Compoute the thresholded boundaries\n",
      "    L = 15\n",
      "    hann = np.hanning(L)\n",
      "    weighted_filt = np.convolve(weights, hann, mode=\"same\")\n",
      "    weighted_filt /= weighted_filt.max()\n",
      "    thresh_bounds = []\n",
      "    thresh_conf = []\n",
      "    for i, weight in enumerate(weighted_filt):\n",
      "        peak_found = False\n",
      "        if i == 0 and weighted_filt[i+1] < weight:\n",
      "            peak_found = True\n",
      "        elif i == len(weighted_filt) - 1 and weighted_filt[i-1] < weight:\n",
      "            peak_found = True\n",
      "        elif i != 0 and i != len(weighted_filt) - 1 and \\\n",
      "                weighted_filt[i-1] < weight and weight > weighted_filt[i+1]:\n",
      "            peak_found = True\n",
      "        if peak_found:\n",
      "            thresh_bounds.append(histo_times[i])\n",
      "            thresh_conf.append(weight)\n",
      "\n",
      "    # Save \n",
      "    create_thres_annot(jam, thresh_bounds, thresh_conf, jam_file)\n",
      "    \n",
      "#     plt.plot(weighted_filt)\n",
      "#     plt.show()\n",
      "#     sys.exit()\n",
      "\n",
      "                \n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 213
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
{'fraction_non_alphanumeric': 0.195607568964207, 'fraction_numerical': 0.05141728094840033, 'mean_word_length': 1.7234570967341363, 'pattern_counts': {'":': 390, '<': 18, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '638326', 'n_tokens_mistral': 26695, 'n_tokens_neox': 24406, 'n_words': 4256}