<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "import os\n",
    "import sys\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import sklearn as sk\n",
    "import sklearn.preprocessing\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import statsmodels.formula.api as smapi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sys.path.insert(1, os.path.join(sys.path[0], '..'))  # add parent directory to path\n",
    "import samlib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sale price distribution\n",
    "First step is to look at the target sale price for the training data set, i.e. the column we're trying to predict. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "target = pd.read_csv('../data/train_target.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "target.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The sale price is in hte hundreds of thousands, so let's divide the price by 1000 to get more manageable numbers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "target = target / 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "logtarget = np.log1p(target)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge the training and test datasets for data preparation\n",
    "We're going to explore the training dataset and apply some transformations to it (fixing missing values, transforming columns etc). We'll need to apply the same transformations to the test dataset. To make that easy, let's use a class that maintains the training and test datasets and keeps them in sync (so that when we apply a transformation to the full dataset, it's applied automatically to the training and test datasets)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "raw_train = pd.read_csv('../data/train_prepared_light.csv')\n",
    "raw_test = pd.read_csv('../data/test_prepared_light.csv')\n",
    "ds = samlib.DataSet(raw_train, raw_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ds.df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ncategories = sum(ds.df.dtypes == object)\n",
    "ncategories"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Features\n",
    "The dataset is wide with 78 features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ds.columns, len(ds.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We've got 3 data types: int, float and object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ds.df.dtypes.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split the data between categorical and numerical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "is_categorical = (ds.df.dtypes == object)\n",
    "is_numerical = (~ is_categorical)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Categorical features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dscat = ds.apply(lambda df: df.loc[:, is_categorical])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dscat.columns, len(dscat.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We've got 42 categorical features. We can use the `describe` method to get some statistics:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "desc = dscat.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "desc = desc.append(pd.Series(dscat.df.isnull().sum(), name='nulls'))\n",
    "desc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Number of categories per feature **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12, 10))\n",
    "desc.loc['unique'].sort_values(ascending=False).plot(kind='barh')\n",
    "plt.title('Number of categories per feature')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Number of nulls per feature **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12, 10))\n",
    "ax = desc.loc['nulls'][desc.loc['nulls'] > 0].sort_values(ascending=False).plot(kind='barh')\n",
    "plt.title('Number of nulls per feature')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But that's a lot of numbers to digest. Better get started plotting! To help with plotting, but also to improve linear regression models, we're going to standardize our data. But before that we must deal with the NaN values.\n",
    "http://sebastianraschka.com/Articles/2014_about_feature_scaling.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deal with NaN values "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dfnum = dsnum.df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cols_with_nulls = dfnum.columns[dfnum.isnull().sum() > 0]\n",
    "cols_with_nulls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dfnum.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dfnum[cols_with_nulls].isnull().sum().sort_values(ascending=False)\n",
    "#.plot(kind='bar')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the description, the null values for the `MasVnrArea` should be 0 (no massonry veneer type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# We may want to refine this in the future. Perhaps build a model to predict the missing GarageCars from the other features?\n",
    "median_list = 'LotFrontage', 'BsmtFullBath','BsmtHalfBath', 'GarageCars', 'GarageArea'\n",
    "zero_list = 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'TotalBsmtSF', 'BsmtUnfSF'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for feature in median_list:\n",
    "    dfnum[feature].fillna(dfnum[feature].median(), inplace=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for feature in zero_list:\n",
    "    dfnum[feature].fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the GarageYrBlt, replace by the year the house was built. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dfnum.GarageYrBlt.fillna(dfnum.YearBuilt[dfnum.GarageYrBlt.isnull()], inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dsnum.df = dfnum\n",
    "\n",
    "# Check that everything is in order\n",
    "def has_nulls(df):\n",
    "    return df.isnull().sum().any()\n",
    "\n",
    "assert not has_nulls(dfnum)\n",
    "assert not has_nulls(dsnum.df)\n",
    "assert not has_nulls(dsnum.train)\n",
    "assert not has_nulls(dsnum.test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standardize the data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def standardize(df):\n",
    "    _values = sk.preprocessing.StandardScaler().fit_transform(df)\n",
    "    return pd.DataFrame(data=_values, columns=df.columns)\n",
    "\n",
    "dsnum_t = dsnum.apply(standardize)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot violinplots for each feature \n",
    "The violin plots give us some idea of the distribution of data for each feature. We can look for things like skewness, non-normality, and the presence of outliers. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def violinplot(df, ax=None):\n",
    "    if ax is None:\n",
    "        ax = plt.gca()\n",
    "    sns.violinplot(df, ax=ax)\n",
    "    for xlab in ax.get_xticklabels():\n",
    "        xlab.set_rotation(30)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def featureplot(df, nrows=1, figsize=(12,8), plotfunc=violinplot):\n",
    "    \"\"\"Plot the dataframe features\"\"\"\n",
    "    width, height = figsize\n",
    "    fig, axes = plt.subplots(nrows, 1, figsize=(width, height * nrows));\n",
    "    i = 0\n",
    "    plots_per_figure = df.shape[1] // nrows\n",
    "    if nrows == 1:\n",
    "        axes = [axes]\n",
    "    for j, ax in zip(range(plots_per_figure, df.shape[1] + 1, plots_per_figure), axes):\n",
    "        plotfunc(df.iloc[:, i:j], ax=ax)\n",
    "        i = j\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"<commit_msg>Start work on categorical features<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "import os\n",
    "import sys\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import sklearn as sk\n",
    "import sklearn.preprocessing\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import statsmodels.formula.api as smapi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sys.path.insert(1, os.path.join(sys.path[0], '..'))  # add parent directory to path\n",
    "import samlib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sale price distribution\n",
    "First step is to look at the target sale price for the training data set, i.e. the column we're trying to predict. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "target = pd.read_csv('../data/train_target.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "target.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The sale price is in hte hundreds of thousands, so let's divide the price by 1000 to get more manageable numbers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "target = target / 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "logtarget = np.log1p(target)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge the training and test datasets for data preparation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def read():\n",
    "    \"\"\"Read training and test data and return a dataframe with ['Dataset','Id'] multi-index\n",
    "    \"\"\"\n",
    "    raw_train = pd.read_csv('../data/train_prepared_light.csv')\n",
    "    raw_test = pd.read_csv('../data/test_prepared_light.csv')\n",
    "    df = pd.concat([raw_train, raw_test], keys=['train', 'test'])\n",
    "    df.index.names = 'Dataset', 'Id'\n",
    "    return df\n",
    "    \n",
    "df = read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pp = samlib.Pipeline(df.copy())  \n",
    "assert pp == df  # the pipeline output equals df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def select_categorical_features(df):\n",
    "    return df.loc[:, df.dtypes == object]\n",
    "\n",
    "pp.append(select_categorical_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Categorical features\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We've got 42 categorical features. We can use the `describe` method to get some statistics:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pp().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pp().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pp().isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Number of categories per feature **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12, 10))\n",
    "pp().describe().loc['unique'].sort_values(ascending=False).plot(kind='barh')\n",
    "plt.title('Number of categories per feature')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Number of nulls per feature **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nulls = pp().isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12, 10))\n",
    "ax = nulls[nulls > 0].sort_values(ascending=False).plot(kind='barh')\n",
    "plt.title('Number of nulls per feature')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But that's a lot of numbers to digest. Better get started plotting! To help with plotting, but also to improve linear regression models, we're going to standardize our data. But before that we must deal with the NaN values.\n",
    "http://sebastianraschka.com/Articles/2014_about_feature_scaling.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deal with NaN values \n",
    "Based on the descriptions, most of the null values are not missing values but an extra \"none\" category (e.g. no pool, or no alley etc). In the above graph, this is true for all features starting from 'BsmtFinType1'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def replace_bad_nulls(df):\n",
    "    nulls = df.isnull().sum()\n",
    "    bad_nulls_colz = nulls[nulls > 0].sort_values()['BsmtFinType1':].index\n",
    "    return samlib.fillna(df, lambda x: 'none', bad_nulls_colz)\n",
    "\n",
    "pp.append(replace_bad_nulls)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Replace true nulls with mode **\n",
    "(work in progress)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pp().mode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "true_nulls_colz = pp().columns[pp().isnull().sum() > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pp()\n",
    "for col in true_nulls_colz:\n",
    "    df = samlib.fillna(df, 'mode', col) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.MSZoning.mode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Change dtypes to `category`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def object_to_category(df):\n",
    "    new = df.copy()\n",
    "    for column, ctype in zip(df.columns, df.dtypes):\n",
    "        if ctype == object:\n",
    "            new[column] = df[column].astype('category')\n",
    "    return new\n",
    "\n",
    "pp.append(object_to_category)\n",
    "assert all(pp().dtypes == 'category')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create dummy variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pd.get_dummies(pp())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
{'fraction_non_alphanumeric': 0.200019419361103, 'fraction_numerical': 0.004369356248179435, 'mean_word_length': 2.622120625989098, 'pattern_counts': {'":': 446, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4619027', 'n_tokens_mistral': 7830, 'n_tokens_neox': 7606, 'n_words': 1576}