<commit_before>{
 "metadata": {
  "name": "",
  "signature": "sha256:1a776cdcfc0e13a3b36ced1052dbef63c6d99c6bcb6bdcf8c76d5a5684249d38"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "[Back to the Whir-torial](Whir-torial.ipynb)"
     ]
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "New Features"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "v0.11"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "v0.12"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "v0.13"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "v0.14 (soon to be released)"
     ]
    }
   ],
   "metadata": {}
  }
 ]
}<commit_msg>CLN: correct typos; outline things; add eeg MI example<commit_after>{
 "metadata": {
  "name": "",
  "signature": "sha256:f46935cb2ca7ac53365455bdd4107fd5d2f9fc918fbbd0fa905259dee6c705dc"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "[Back to the Whirl-torial](Whirl-torial.ipynb)"
     ]
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "New Features since v0.11.0"
     ]
    },
    {
     "cell_type": "heading",
     "level": 3,
     "metadata": {},
     "source": [
      "(or Interactive Release Notes)"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "v0.11"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "* indexers ``loc``/``at``, ``iloc``/``iat``\n",
      "* all dtypes allowed\n",
      "* now using ``numexpr`` to evaluate expressions where possible\n",
      "* ``to_series()`` for ``Index`` objects"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "v0.12"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "* ``read_html``\n",
      "* ``read_json``\n",
      "* ``read_stata``\n",
      "* ``read_s3``\n",
      "* ``str`` iteration\n",
      "* ``MultiIndex`` column reading and writing\n",
      "* ``GroupBy.filter()``"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "v0.13"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "* ``DataFrame.isin()``\n",
      "* ``str.extract()``\n",
      "* ``GroupBy.cumcount()``\n",
      "* ``to_csv`` ``date_format`` keyword argument\n",
      "\n",
      "#### Experimental Features\n",
      "* ``query``/``eval``\n",
      "* msgpack IO\n",
      "* Google BigQuery IO\n",
      "\n",
      "#### Non-user-facing-but-absolutely-worth-mentioning\n",
      "<NAME>'s epic refactor of ``Series`` to use composition instead of inheriting from ``numpy.ndarray``. Bravo!"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "v0.14 (soon to be released)"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "* Index(tuples) now makes a ``MultiIndex``\n",
      "* hexbin and pie plotting\n",
      "* table plotting\n",
      "* multiindex slicing\n",
      "* automagic ``MultiIndex`` level selection"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#### `MultiIndex` slicing"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# simulate our EEG data set\n",
      "from itertools import repeat\n",
      "\n",
      "def channel_index(name, nchannels):\n",
      "    return list(zip(repeat(name, nchannels), range(nchannels)))\n",
      "    \n",
      "neeg = 32\n",
      "nex = 8\n",
      "eeg_chan = channel_index('eeg', neeg)\n",
      "ex_chan = channel_index('ex', nex)\n",
      "stimulus = channel_index('block', 1)\n",
      "\n",
      "columns = pd.MultiIndex.from_tuples(eeg_chan + ex_chan, names=['signal', 'channel'])\n",
      "df = pd.DataFrame(np.random.randn(256 * 10, columns.labels[0].size), columns=columns)\n",
      "df['stim', ''] = np.sort(np.random.randint(8, size=len(df)))\n",
      "df.loc[np.random.rand(len(df)) > 0.5, ('stim', '')] = np.nan"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "0     0\n",
        "1     0\n",
        "2     0\n",
        "3   NaN\n",
        "4     0\n",
        "5     0\n",
        "6   NaN\n",
        "7     0\n",
        "8   NaN\n",
        "9     0\n",
        "...\n",
        "2549     7\n",
        "2550   NaN\n",
        "2551   NaN\n",
        "2552     7\n",
        "2553   NaN\n",
        "2554     7\n",
        "2555     7\n",
        "2556   NaN\n",
        "2557   NaN\n",
        "2558     7\n",
        "2559   NaN\n",
        "Name: stim, Length: 2560, dtype: float64"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df.loc[:, np.s_[('ex', 'stim'), :]].head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr>\n",
        "      <th>signal</th>\n",
        "      <th colspan=\"8\" halign=\"left\">ex</th>\n",
        "      <th>stim</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>channel</th>\n",
        "      <th>0</th>\n",
        "      <th>1</th>\n",
        "      <th>2</th>\n",
        "      <th>3</th>\n",
        "      <th>4</th>\n",
        "      <th>5</th>\n",
        "      <th>6</th>\n",
        "      <th>7</th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0.3115</td>\n",
        "      <td>-1.1312</td>\n",
        "      <td> 0.3611</td>\n",
        "      <td>-0.8136</td>\n",
        "      <td>-0.9106</td>\n",
        "      <td> 0.0549</td>\n",
        "      <td>-0.1549</td>\n",
        "      <td>-0.2773</td>\n",
        "      <td>  0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 0.6973</td>\n",
        "      <td>-0.7448</td>\n",
        "      <td>-0.9885</td>\n",
        "      <td> 1.9721</td>\n",
        "      <td> 1.3680</td>\n",
        "      <td>-1.8785</td>\n",
        "      <td>-0.5677</td>\n",
        "      <td>-0.9382</td>\n",
        "      <td>  0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 0.2214</td>\n",
        "      <td>-1.7722</td>\n",
        "      <td> 0.6885</td>\n",
        "      <td> 0.9222</td>\n",
        "      <td>-0.3093</td>\n",
        "      <td> 0.4395</td>\n",
        "      <td> 0.1920</td>\n",
        "      <td>-1.8415</td>\n",
        "      <td>  0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>-0.1822</td>\n",
        "      <td>-1.3126</td>\n",
        "      <td>-1.8273</td>\n",
        "      <td> 0.0052</td>\n",
        "      <td>-0.5071</td>\n",
        "      <td>-0.9493</td>\n",
        "      <td> 1.1841</td>\n",
        "      <td> 0.2509</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 0.8980</td>\n",
        "      <td>-1.1519</td>\n",
        "      <td> 0.1316</td>\n",
        "      <td> 0.4667</td>\n",
        "      <td> 0.4433</td>\n",
        "      <td>-0.0084</td>\n",
        "      <td>-0.2709</td>\n",
        "      <td>-0.6189</td>\n",
        "      <td>  0</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>5 rows \u00d7 9 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 37,
       "text": [
        "signal       ex                                                          stim\n",
        "channel       0       1       2       3       4       5       6       7      \n",
        "0        0.3115 -1.1312  0.3611 -0.8136 -0.9106  0.0549 -0.1549 -0.2773     0\n",
        "1        0.6973 -0.7448 -0.9885  1.9721  1.3680 -1.8785 -0.5677 -0.9382     0\n",
        "2        0.2214 -1.7722  0.6885  0.9222 -0.3093  0.4395  0.1920 -1.8415     0\n",
        "3       -0.1822 -1.3126 -1.8273  0.0052 -0.5071 -0.9493  1.1841  0.2509   NaN\n",
        "4        0.8980 -1.1519  0.1316  0.4667  0.4433 -0.0084 -0.2709 -0.6189     0\n",
        "\n",
        "[5 rows x 9 columns]"
       ]
      }
     ],
     "prompt_number": 37
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "res = df.loc[df.stim.notnull(), np.s_[('ex', 'stim'), :]]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 40
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Let's unpack this:\n",
      "\n",
      "1. `df.stim.notnull()`  gives a `bool` dtyped array used to get the rows of `df` where the `stim` column is not `NaN`\n",
      "2. The `np.s_` object is a special `numpy` object that just wraps slice syntax without having to manually construct slices using `slice` so\n",
      "\n",
      "Let's see it:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df.stim.notnull()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 48,
       "text": [
        "0     True\n",
        "1     True\n",
        "2     True\n",
        "3    False\n",
        "4     True\n",
        "5     True\n",
        "6    False\n",
        "7     True\n",
        "8    False\n",
        "9     True\n",
        "...\n",
        "2549     True\n",
        "2550    False\n",
        "2551    False\n",
        "2552     True\n",
        "2553    False\n",
        "2554     True\n",
        "2555     True\n",
        "2556    False\n",
        "2557    False\n",
        "2558     True\n",
        "2559    False\n",
        "Name: stim, Length: 2560, dtype: bool"
       ]
      }
     ],
     "prompt_number": 48
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "slice(None, 1000) == np.s_[:1000]  # Which one would you prefer?"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 47,
       "text": [
        "True"
       ]
      }
     ],
     "prompt_number": 47
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "slice(999, -1, -1) == np.s_[999:-1:-1]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 46,
       "text": [
        "True"
       ]
      }
     ],
     "prompt_number": 46
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Think of the expression\n",
      "\n",
      "```python\n",
      "np.s_[('ex', 'stim'), :]\n",
      "```\n",
      "\n",
      "as saying select `'ex'` and `'stim'` from the first level of the `MultiIndex` and the `:` as saying give me all columns from the second level"
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Methods you wish you (didn't?) know about"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pandas import DataFrame\n",
      "from numpy.random import randn, rand, randint, choice\n",
      "df = DataFrame(randn(10, 2), columns=list('ab'))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# align\n",
      "\n",
      "# between_time\n",
      "# where\n",
      "# at_time"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>a</th>\n",
        "      <th>b</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0.3423</td>\n",
        "      <td>-2.9745</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1.2673</td>\n",
        "      <td> 0.6345</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>-1.1564</td>\n",
        "      <td>-1.8889</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 0.8854</td>\n",
        "      <td>-3.2326</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>-1.3992</td>\n",
        "      <td> 0.6620</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <td>-0.0080</td>\n",
        "      <td> 2.2159</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
        "      <td> 0.0038</td>\n",
        "      <td> 0.4477</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7</th>\n",
        "      <td>-0.6837</td>\n",
        "      <td>-0.5419</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8</th>\n",
        "      <td> 0.4141</td>\n",
        "      <td>-1.4594</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9</th>\n",
        "      <td> 1.2071</td>\n",
        "      <td> 2.4742</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>10 rows \u00d7 2 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 1,
       "text": [
        "        a       b\n",
        "0  0.3423 -2.9745\n",
        "1  1.2673  0.6345\n",
        "2 -1.1564 -1.8889\n",
        "3  0.8854 -3.2326\n",
        "4 -1.3992  0.6620\n",
        "5 -0.0080  2.2159\n",
        "6  0.0038  0.4477\n",
        "7 -0.6837 -0.5419\n",
        "8  0.4141 -1.4594\n",
        "9  1.2071  2.4742\n",
        "\n",
        "[10 rows x 2 columns]"
       ]
      }
     ],
     "prompt_number": 1
    }
   ],
   "metadata": {}
  }
 ]
}
{'fraction_non_alphanumeric': 0.23011789226861198, 'fraction_numerical': 0.06526411776200092, 'mean_word_length': 1.2895914106770057, 'pattern_counts': {'":': 174, '<': 252, '<?xml version=': 0, '>': 253, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1248463', 'n_tokens_mistral': 6489, 'n_tokens_neox': 5957, 'n_words': 976}