<commit_before><commit_msg>initial exploratory analysis<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-30T16:00:18.454396Z",
     "start_time": "2017-04-30T16:00:16.789229Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-30T16:11:22.182133Z",
     "start_time": "2017-04-30T16:11:19.672639Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('../../data/raw/train.csv', \n",
    "                 index_col=0, \n",
    "                 parse_dates=[1], \n",
    "                 infer_datetime_format=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-30T16:11:23.010833Z",
     "start_time": "2017-04-30T16:11:22.933855Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>full_sq</th>\n",
       "      <th>life_sq</th>\n",
       "      <th>floor</th>\n",
       "      <th>max_floor</th>\n",
       "      <th>material</th>\n",
       "      <th>build_year</th>\n",
       "      <th>num_room</th>\n",
       "      <th>kitch_sq</th>\n",
       "      <th>state</th>\n",
       "      <th>...</th>\n",
       "      <th>cafe_count_5000_price_2500</th>\n",
       "      <th>cafe_count_5000_price_4000</th>\n",
       "      <th>cafe_count_5000_price_high</th>\n",
       "      <th>big_church_count_5000</th>\n",
       "      <th>church_count_5000</th>\n",
       "      <th>mosque_count_5000</th>\n",
       "      <th>leisure_count_5000</th>\n",
       "      <th>sport_count_5000</th>\n",
       "      <th>market_count_5000</th>\n",
       "      <th>price_doc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2011-08-20</td>\n",
       "      <td>43</td>\n",
       "      <td>27.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>4</td>\n",
       "      <td>5850000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011-08-23</td>\n",
       "      <td>34</td>\n",
       "      <td>19.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>66</td>\n",
       "      <td>14</td>\n",
       "      <td>6000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2011-08-27</td>\n",
       "      <td>43</td>\n",
       "      <td>29.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>67</td>\n",
       "      <td>10</td>\n",
       "      <td>5700000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2011-09-01</td>\n",
       "      <td>89</td>\n",
       "      <td>50.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>13100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2011-09-05</td>\n",
       "      <td>77</td>\n",
       "      <td>77.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>319</td>\n",
       "      <td>108</td>\n",
       "      <td>17</td>\n",
       "      <td>135</td>\n",
       "      <td>236</td>\n",
       "      <td>2</td>\n",
       "      <td>91</td>\n",
       "      <td>195</td>\n",
       "      <td>14</td>\n",
       "      <td>16331452</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 291 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    timestamp  full_sq  life_sq  floor  max_floor  material  build_year  \\\n",
       "id                                                                        \n",
       "1  2011-08-20       43     27.0    4.0        NaN       NaN         NaN   \n",
       "2  2011-08-23       34     19.0    3.0        NaN       NaN         NaN   \n",
       "3  2011-08-27       43     29.0    2.0        NaN       NaN         NaN   \n",
       "4  2011-09-01       89     50.0    9.0        NaN       NaN         NaN   \n",
       "5  2011-09-05       77     77.0    4.0        NaN       NaN         NaN   \n",
       "\n",
       "    num_room  kitch_sq  state    ...     cafe_count_5000_price_2500  \\\n",
       "id                               ...                                  \n",
       "1        NaN       NaN    NaN    ...                              9   \n",
       "2        NaN       NaN    NaN    ...                             15   \n",
       "3        NaN       NaN    NaN    ...                             10   \n",
       "4        NaN       NaN    NaN    ...                             11   \n",
       "5        NaN       NaN    NaN    ...                            319   \n",
       "\n",
       "   cafe_count_5000_price_4000  cafe_count_5000_price_high  \\\n",
       "id                                                          \n",
       "1                           4                           0   \n",
       "2                           3                           0   \n",
       "3                           3                           0   \n",
       "4                           2                           1   \n",
       "5                         108                          17   \n",
       "\n",
       "    big_church_count_5000  church_count_5000  mosque_count_5000  \\\n",
       "id                                                                \n",
       "1                      13                 22                  1   \n",
       "2                      15                 29                  1   \n",
       "3                      11                 27                  0   \n",
       "4                       4                  4                  0   \n",
       "5                     135                236                  2   \n",
       "\n",
       "    leisure_count_5000  sport_count_5000  market_count_5000  price_doc  \n",
       "id                                                                      \n",
       "1                    0                52                  4    5850000  \n",
       "2                   10                66                 14    6000000  \n",
       "3                    4                67                 10    5700000  \n",
       "4                    0                26                  3   13100000  \n",
       "5                   91               195                 14   16331452  \n",
       "\n",
       "[5 rows x 291 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-04-30T16:11:38.740577Z",
     "start_time": "2017-04-30T16:11:37.400501Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>full_sq</th>\n",
       "      <th>life_sq</th>\n",
       "      <th>floor</th>\n",
       "      <th>max_floor</th>\n",
       "      <th>material</th>\n",
       "      <th>build_year</th>\n",
       "      <th>num_room</th>\n",
       "      <th>kitch_sq</th>\n",
       "      <th>state</th>\n",
       "      <th>area_m</th>\n",
       "      <th>...</th>\n",
       "      <th>cafe_count_5000_price_2500</th>\n",
       "      <th>cafe_count_5000_price_4000</th>\n",
       "      <th>cafe_count_5000_price_high</th>\n",
       "      <th>big_church_count_5000</th>\n",
       "      <th>church_count_5000</th>\n",
       "      <th>mosque_count_5000</th>\n",
       "      <th>leisure_count_5000</th>\n",
       "      <th>sport_count_5000</th>\n",
       "      <th>market_count_5000</th>\n",
       "      <th>price_doc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>30471.000000</td>\n",
       "      <td>24088.000000</td>\n",
       "      <td>30304.000000</td>\n",
       "      <td>20899.000000</td>\n",
       "      <td>20899.000000</td>\n",
       "      <td>1.686600e+04</td>\n",
       "      <td>20899.000000</td>\n",
       "      <td>20899.000000</td>\n",
       "      <td>16912.000000</td>\n",
       "      <td>3.047100e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>30471.000000</td>\n",
       "      <td>30471.000000</td>\n",
       "      <td>30471.000000</td>\n",
       "      <td>30471.000000</td>\n",
       "      <td>30471.000000</td>\n",
       "      <td>30471.000000</td>\n",
       "      <td>30471.000000</td>\n",
       "      <td>30471.000000</td>\n",
       "      <td>30471.000000</td>\n",
       "      <td>3.047100e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>54.214269</td>\n",
       "      <td>34.403271</td>\n",
       "      <td>7.670803</td>\n",
       "      <td>12.558974</td>\n",
       "      <td>1.827121</td>\n",
       "      <td>3.068057e+03</td>\n",
       "      <td>1.909804</td>\n",
       "      <td>6.399301</td>\n",
       "      <td>2.107025</td>\n",
       "      <td>1.765705e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>32.058318</td>\n",
       "      <td>10.783860</td>\n",
       "      <td>1.771783</td>\n",
       "      <td>15.045552</td>\n",
       "      <td>30.251518</td>\n",
       "      <td>0.442421</td>\n",
       "      <td>8.648814</td>\n",
       "      <td>52.796593</td>\n",
       "      <td>5.987070</td>\n",
       "      <td>7.123035e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>38.031487</td>\n",
       "      <td>52.285733</td>\n",
       "      <td>5.319989</td>\n",
       "      <td>6.756550</td>\n",
       "      <td>1.481154</td>\n",
       "      <td>1.543878e+05</td>\n",
       "      <td>0.851805</td>\n",
       "      <td>28.265979</td>\n",
       "      <td>0.880148</td>\n",
       "      <td>2.064961e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>73.465611</td>\n",
       "      <td>28.385679</td>\n",
       "      <td>5.418807</td>\n",
       "      <td>29.118668</td>\n",
       "      <td>47.347938</td>\n",
       "      <td>0.609269</td>\n",
       "      <td>20.580741</td>\n",
       "      <td>46.292660</td>\n",
       "      <td>4.889219</td>\n",
       "      <td>4.780111e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.081628e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>38.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.967000e+03</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.307411e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.740002e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>49.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>6.500000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.979000e+03</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.050803e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>48.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.274411e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>63.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.005000e+03</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.803644e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>76.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>8.300000e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5326.000000</td>\n",
       "      <td>7478.000000</td>\n",
       "      <td>77.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.005201e+07</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>2014.000000</td>\n",
       "      <td>33.000000</td>\n",
       "      <td>2.060718e+08</td>\n",
       "      <td>...</td>\n",
       "      <td>377.000000</td>\n",
       "      <td>147.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>151.000000</td>\n",
       "      <td>250.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>106.000000</td>\n",
       "      <td>218.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>1.111111e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 275 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            full_sq       life_sq         floor     max_floor      material  \\\n",
       "count  30471.000000  24088.000000  30304.000000  20899.000000  20899.000000   \n",
       "mean      54.214269     34.403271      7.670803     12.558974      1.827121   \n",
       "std       38.031487     52.285733      5.319989      6.756550      1.481154   \n",
       "min        0.000000      0.000000      0.000000      0.000000      1.000000   \n",
       "25%       38.000000     20.000000      3.000000      9.000000      1.000000   \n",
       "50%       49.000000     30.000000      6.500000     12.000000      1.000000   \n",
       "75%       63.000000     43.000000     11.000000     17.000000      2.000000   \n",
       "max     5326.000000   7478.000000     77.000000    117.000000      6.000000   \n",
       "\n",
       "         build_year      num_room      kitch_sq         state        area_m  \\\n",
       "count  1.686600e+04  20899.000000  20899.000000  16912.000000  3.047100e+04   \n",
       "mean   3.068057e+03      1.909804      6.399301      2.107025  1.765705e+07   \n",
       "std    1.543878e+05      0.851805     28.265979      0.880148  2.064961e+07   \n",
       "min    0.000000e+00      0.000000      0.000000      1.000000  2.081628e+06   \n",
       "25%    1.967000e+03      1.000000      1.000000      1.000000  7.307411e+06   \n",
       "50%    1.979000e+03      2.000000      6.000000      2.000000  1.050803e+07   \n",
       "75%    2.005000e+03      2.000000      9.000000      3.000000  1.803644e+07   \n",
       "max    2.005201e+07     19.000000   2014.000000     33.000000  2.060718e+08   \n",
       "\n",
       "           ...       cafe_count_5000_price_2500  cafe_count_5000_price_4000  \\\n",
       "count      ...                     30471.000000                30471.000000   \n",
       "mean       ...                        32.058318                   10.783860   \n",
       "std        ...                        73.465611                   28.385679   \n",
       "min        ...                         0.000000                    0.000000   \n",
       "25%        ...                         2.000000                    1.000000   \n",
       "50%        ...                         8.000000                    2.000000   \n",
       "75%        ...                        21.000000                    5.000000   \n",
       "max        ...                       377.000000                  147.000000   \n",
       "\n",
       "       cafe_count_5000_price_high  big_church_count_5000  church_count_5000  \\\n",
       "count                30471.000000           30471.000000       30471.000000   \n",
       "mean                     1.771783              15.045552          30.251518   \n",
       "std                      5.418807              29.118668          47.347938   \n",
       "min                      0.000000               0.000000           0.000000   \n",
       "25%                      0.000000               2.000000           9.000000   \n",
       "50%                      0.000000               7.000000          16.000000   \n",
       "75%                      1.000000              12.000000          28.000000   \n",
       "max                     30.000000             151.000000         250.000000   \n",
       "\n",
       "       mosque_count_5000  leisure_count_5000  sport_count_5000  \\\n",
       "count       30471.000000        30471.000000      30471.000000   \n",
       "mean            0.442421            8.648814         52.796593   \n",
       "std             0.609269           20.580741         46.292660   \n",
       "min             0.000000            0.000000          0.000000   \n",
       "25%             0.000000            0.000000         11.000000   \n",
       "50%             0.000000            2.000000         48.000000   \n",
       "75%             1.000000            7.000000         76.000000   \n",
       "max             2.000000          106.000000        218.000000   \n",
       "\n",
       "       market_count_5000     price_doc  \n",
       "count       30471.000000  3.047100e+04  \n",
       "mean            5.987070  7.123035e+06  \n",
       "std             4.889219  4.780111e+06  \n",
       "min             0.000000  1.000000e+05  \n",
       "25%             1.000000  4.740002e+06  \n",
       "50%             5.000000  6.274411e+06  \n",
       "75%            10.000000  8.300000e+06  \n",
       "max            21.000000  1.111111e+08  \n",
       "\n",
       "[8 rows x 275 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
{'fraction_non_alphanumeric': 0.21332987999654665, 'fraction_numerical': 0.1474574807908141, 'mean_word_length': 0.9471339720961506, 'pattern_counts': {'":': 64, '<': 759, '<?xml version=': 0, '>': 759, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 56, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3621388', 'n_tokens_mistral': 10936, 'n_tokens_neox': 8713, 'n_words': 1023}