<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from joblib import Parallel, delayed\n",
    "import multiprocessing\n",
    "\n",
    "from utils.preprocessing import one_hot_encoding\n",
    "from utils.preprocessing import get_weekday\n",
    "from utils.preprocessing import process_user_secs_elapsed\n",
    "from utils.preprocessing import process_user_session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw_data_path = '../data/raw/'\n",
    "processed_data_path = '../data/processed/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load raw data\n",
    "train_users = pd.read_csv(raw_data_path + 'train_users.csv')\n",
    "test_users = pd.read_csv(raw_data_path + 'test_users.csv')\n",
    "sessions = pd.read_csv(raw_data_path + 'sessions.csv', nrows=1000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Join users\n",
    "users = pd.concat((train_users, test_users), axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Drop date_first_booking column (empty since competition's restart)\n",
    "users = users.drop('date_first_booking', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Replace NaNs\n",
    "users['gender'].replace('-unknown-', np.nan, inplace=True)\n",
    "users['language'].replace('-unknown-', np.nan, inplace=True)\n",
    "sessions.replace('-unknown-', np.nan, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remove weird age values\n",
    "users.loc[users['age'] > 100, 'age'] = np.nan\n",
    "users.loc[users['age'] < 14, 'age'] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Change type to date\n",
    "users['date_account_created'] = pd.to_datetime(users['date_account_created'])\n",
    "users['date_first_active'] = pd.to_datetime(users['timestamp_first_active'],\n",
    "                                            format='%Y%m%d%H%M%S')\n",
    "\n",
    "users['weekday_account_created'] = users[\n",
    "    'date_account_created'].apply(get_weekday)\n",
    "users['weekday_first_active'] = users['date_first_active'].apply(get_weekday)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Split dates into day, month, year\n",
    "year_account_created = pd.DatetimeIndex(users['date_account_created']).year\n",
    "users['year_account_created'] = year_account_created\n",
    "month_account_created = pd.DatetimeIndex(users['date_account_created']).month\n",
    "users['month_account_created'] = month_account_created\n",
    "day_account_created = pd.DatetimeIndex(users['date_account_created']).day\n",
    "users['day_account_created'] = day_account_created\n",
    "year_first_active = pd.DatetimeIndex(users['date_first_active']).year\n",
    "users['year_first_active'] = year_first_active\n",
    "month_first_active = pd.DatetimeIndex(users['date_first_active']).month\n",
    "users['month_first_active'] = month_first_active\n",
    "day_first_active = pd.DatetimeIndex(users['date_first_active']).day\n",
    "users['day_first_active'] = day_first_active"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Process session data\n",
    "processed_sessions = Parallel(n_jobs=multiprocessing.cpu_count())(\n",
    "    delayed(process_user_session)(\n",
    "        user, sessions.loc[sessions['user_id'] == user])\n",
    "    for user in sessions['user_id'].unique()\n",
    ")\n",
    "user_sessions = pd.DataFrame(processed_sessions).set_index('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Joint the processed data with each user\n",
    "users = users.set_index('id')\n",
    "users = pd.concat([users, user_sessions], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TODO: Classify by dispositive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get the count of general session information\n",
    "user_sessions = sessions.groupby('user_id').count()\n",
    "user_sessions.rename(columns=lambda x: x + '_count', inplace=True)\n",
    "users = pd.concat([users, user_sessions], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "processed_secs_elapsed = Parallel(n_jobs=multiprocessing.cpu_count())(\n",
    "    delayed(process_user_secs_elapsed)(user, sessions.loc[\n",
    "        sessions['user_id'] == user, 'secs_elapsed'])\n",
    "    for user in sessions['user_id'].unique()\n",
    ")\n",
    "\n",
    "processed_secs_elapsed = pd.DataFrame(processed_secs_elapsed).set_index('id')\n",
    "users = pd.concat([users, processed_secs_elapsed], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_users = train_users.set_index('id')\n",
    "test_users = test_users.set_index('id')\n",
    "\n",
    "processed_train_users = users.loc[train_users.index]\n",
    "processed_test_users = users.loc[test_users.index]\n",
    "processed_test_users.drop(['country_destination'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "processed_train_users.to_csv(processed_data_path + 'processed_train_users.csv')\n",
    "processed_test_users.to_csv(processed_data_path + 'processed_test_users.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drop_list = [\n",
    "    'date_account_created',\n",
    "    'date_first_active',\n",
    "    'timestamp_first_active'\n",
    "]\n",
    "\n",
    "# Drop columns\n",
    "users = users.drop(drop_list, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TODO: Try with StandardScaler\n",
    "# from sklearn.preprocessing import StandardScaler\n",
    "# scaler = StandardScaler()\n",
    "# scaler.fit_transform(users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Encode categorical features\n",
    "categorical_features = [\n",
    "    'gender', 'signup_method', 'signup_flow', 'language', 'affiliate_channel',\n",
    "    'affiliate_provider', 'first_affiliate_tracked', 'signup_app',\n",
    "    'first_device_type', 'first_browser', 'most_used_device'\n",
    "]\n",
    "\n",
    "users = one_hot_encoding(users, categorical_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "users.index.name = 'id'\n",
    "processed_train_users = users.loc[train_users.index]\n",
    "processed_test_users = users.loc[test_users.index]\n",
    "processed_test_users.drop('country_destination', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "processed_train_users.to_csv(processed_data_path + 'encoded_train_users.csv')\n",
    "processed_test_users.to_csv(processed_data_path + 'encoded_test_users.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
<commit_msg>Update with new parallelization method<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from joblib import Parallel, delayed\n",
    "import multiprocessing\n",
    "\n",
    "from utils.preprocessing import one_hot_encoding\n",
    "from utils.preprocessing import get_weekday\n",
    "from utils.preprocessing import process_user_secs_elapsed\n",
    "from utils.preprocessing import process_user_session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw_data_path = '../data/raw/'\n",
    "processed_data_path = '../data/processed/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "global sessions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load raw data\n",
    "train_users = pd.read_csv(raw_data_path + 'train_users.csv')\n",
    "test_users = pd.read_csv(raw_data_path + 'test_users.csv')\n",
    "sessions = pd.read_csv(raw_data_path + 'sessions.csv', nrows=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Join users\n",
    "users = pd.concat((train_users, test_users), axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Drop date_first_booking column (empty since competition's restart)\n",
    "users = users.drop('date_first_booking', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Replace NaNs\n",
    "users['gender'].replace('-unknown-', np.nan, inplace=True)\n",
    "users['language'].replace('-unknown-', np.nan, inplace=True)\n",
    "sessions.replace('-unknown-', np.nan, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remove weird age values\n",
    "users.loc[users['age'] > 100, 'age'] = np.nan\n",
    "users.loc[users['age'] < 14, 'age'] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Change type to date\n",
    "users['date_account_created'] = pd.to_datetime(users['date_account_created'])\n",
    "users['date_first_active'] = pd.to_datetime(users['timestamp_first_active'],\n",
    "                                            format='%Y%m%d%H%M%S')\n",
    "\n",
    "users['weekday_account_created'] = users[\n",
    "    'date_account_created'].apply(get_weekday)\n",
    "users['weekday_first_active'] = users['date_first_active'].apply(get_weekday)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Split dates into day, month, year\n",
    "year_account_created = pd.DatetimeIndex(users['date_account_created']).year\n",
    "users['year_account_created'] = year_account_created\n",
    "month_account_created = pd.DatetimeIndex(users['date_account_created']).month\n",
    "users['month_account_created'] = month_account_created\n",
    "day_account_created = pd.DatetimeIndex(users['date_account_created']).day\n",
    "users['day_account_created'] = day_account_created\n",
    "year_first_active = pd.DatetimeIndex(users['date_first_active']).year\n",
    "users['year_first_active'] = year_first_active\n",
    "month_first_active = pd.DatetimeIndex(users['date_first_active']).month\n",
    "users['month_first_active'] = month_first_active\n",
    "day_first_active = pd.DatetimeIndex(users['date_first_active']).day\n",
    "users['day_first_active'] = day_first_active"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def c_process_user_session(user):\n",
    "    user_session = sessions.loc[sessions['user_id'] == user]\n",
    "    # Get the user session\n",
    "    user_session_data = pd.Series()\n",
    "\n",
    "    # Length of the session\n",
    "    user_session_data['session_lenght'] = len(user_session)\n",
    "    user_session_data['id'] = user\n",
    "\n",
    "    suffix = '_secs_elapsed'\n",
    "\n",
    "    for column in ['action', 'action_type', 'action_detail', 'device_type']:\n",
    "        column_data = user_session.groupby(column).secs_elapsed.sum()\n",
    "        column_data.rename(lambda x: x + suffix, inplace=True)\n",
    "        user_session_data = user_session_data.append(column_data)\n",
    "\n",
    "    # Get the most used device\n",
    "    user_session_data['most_used_device'] = user_session['device_type'].max()\n",
    "\n",
    "    return user_session_data.groupby(level=0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 52 ms, sys: 4 ms, total: 56 ms\n",
      "Wall time: 451 ms\n"
     ]
    }
   ],
   "source": [
    "from multiprocessing import Pool\n",
    "import time\n",
    "\n",
    "p = Pool(8)\n",
    "%time a = p.map(c_process_user_session, sessions['user_id'].unique())\n",
    "a = pd.DataFrame(a).set_index('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 480 ms, sys: 128 ms, total: 608 ms\n",
      "Wall time: 828 ms\n"
     ]
    }
   ],
   "source": [
    "# Process session data\n",
    "%time processed_sessions = Parallel(n_jobs=multiprocessing.cpu_count())(delayed(process_user_session)( user, sessions.loc[sessions['user_id'] == user]) for user in sessions['user_id'].unique())\n",
    "user_sessions = pd.DataFrame(processed_sessions).set_index('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Joint the processed data with each user\n",
    "users = users.set_index('id')\n",
    "users = pd.concat([users, user_sessions], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TODO: Classify by dispositive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get the count of general session information\n",
    "user_sessions = sessions.groupby('user_id').count()\n",
    "user_sessions.rename(columns=lambda x: x + '_count', inplace=True)\n",
    "users = pd.concat([users, user_sessions], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "processed_secs_elapsed = Parallel(n_jobs=multiprocessing.cpu_count())(\n",
    "    delayed(process_user_secs_elapsed)(user, sessions.loc[\n",
    "        sessions['user_id'] == user, 'secs_elapsed'])\n",
    "    for user in sessions['user_id'].unique()\n",
    ")\n",
    "\n",
    "processed_secs_elapsed = pd.DataFrame(processed_secs_elapsed).set_index('id')\n",
    "users = pd.concat([users, processed_secs_elapsed], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_users = train_users.set_index('id')\n",
    "test_users = test_users.set_index('id')\n",
    "\n",
    "processed_train_users = users.loc[train_users.index]\n",
    "processed_test_users = users.loc[test_users.index]\n",
    "processed_test_users.drop(['country_destination'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "processed_train_users.to_csv(processed_data_path + 'processed_train_users.csv')\n",
    "processed_test_users.to_csv(processed_data_path + 'processed_test_users.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drop_list = [\n",
    "    'date_account_created',\n",
    "    'date_first_active',\n",
    "    'timestamp_first_active'\n",
    "]\n",
    "\n",
    "# Drop columns\n",
    "users = users.drop(drop_list, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TODO: Try with StandardScaler\n",
    "# from sklearn.preprocessing import StandardScaler\n",
    "# scaler = StandardScaler()\n",
    "# scaler.fit_transform(users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Encode categorical features\n",
    "categorical_features = [\n",
    "    'gender', 'signup_method', 'signup_flow', 'language', 'affiliate_channel',\n",
    "    'affiliate_provider', 'first_affiliate_tracked', 'signup_app',\n",
    "    'first_device_type', 'first_browser', 'most_used_device'\n",
    "]\n",
    "\n",
    "users = one_hot_encoding(users, categorical_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "users.index.name = 'id'\n",
    "processed_train_users = users.loc[train_users.index]\n",
    "processed_test_users = users.loc[test_users.index]\n",
    "processed_test_users.drop('country_destination', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "processed_train_users.to_csv(processed_data_path + 'encoded_train_users.csv')\n",
    "processed_test_users.to_csv(processed_data_path + 'encoded_test_users.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}

{'fraction_non_alphanumeric': 0.19309105431309903, 'fraction_numerical': 0.00509185303514377, 'mean_word_length': 3.225479856570344, 'pattern_counts': {'":': 312, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4866165', 'n_tokens_mistral': 7495, 'n_tokens_neox': 7335, 'n_words': 1136}