{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_last</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Asian,GreaterEastAsian,EastAsian</th>\n",
       "      <td>2739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Asian,GreaterEastAsian,Japanese</th>\n",
       "      <td>4107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Asian,IndianSubContinent</th>\n",
       "      <td>3802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GreaterAfrican,Africans</th>\n",
       "      <td>3149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GreaterAfrican,Muslim</th>\n",
       "      <td>4537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GreaterEuropean,British</th>\n",
       "      <td>15943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GreaterEuropean,EastEuropean</th>\n",
       "      <td>6576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GreaterEuropean,Jewish</th>\n",
       "      <td>6285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GreaterEuropean,WestEuropean,French</th>\n",
       "      <td>9503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GreaterEuropean,WestEuropean,Germanic</th>\n",
       "      <td>3313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GreaterEuropean,WestEuropean,Hispanic</th>\n",
       "      <td>6229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GreaterEuropean,WestEuropean,Italian</th>\n",
       "      <td>8630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GreaterEuropean,WestEuropean,Nordic</th>\n",
       "      <td>3168</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       name_last\n",
       "race                                            \n",
       "Asian,GreaterEastAsian,EastAsian            2739\n",
       "Asian,GreaterEastAsian,Japanese             4107\n",
       "Asian,IndianSubContinent                    3802\n",
       "GreaterAfrican,Africans                     3149\n",
       "GreaterAfrican,Muslim                       4537\n",
       "GreaterEuropean,British                    15943\n",
       "GreaterEuropean,EastEuropean                6576\n",
       "GreaterEuropean,Jewish                      6285\n",
       "GreaterEuropean,WestEuropean,French         9503\n",
       "GreaterEuropean,WestEuropean,Germanic       3313\n",
       "GreaterEuropean,WestEuropean,Hispanic       6229\n",
       "GreaterEuropean,WestEuropean,Italian        8630\n",
       "GreaterEuropean,WestEuropean,Nordic         3168"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "\n",
    "NGRAMS = 2\n",
    "EPOCHS = 25\n",
    "\n",
    "# Wikilabels\n",
    "df = pd.read_csv('./data/wiki/wiki_name_race.csv')\n",
    "df.dropna(subset=['name_first', 'name_last'], inplace=True)\n",
    "sdf = df\n",
    "\n",
    "# Additional features\n",
    "sdf['name_last'] = sdf.name_last.str.title()\n",
    "\n",
    "sdf.groupby('race').agg({'name_last': 'nunique'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing the input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_words = 1946\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/venv/lib/python2.7/site-packages/ipykernel_launcher.py:28: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max feature len = 71, Avg. feature len = 5\n"
     ]
    }
   ],
   "source": [
    "# only last name will be use to train the model\n",
    "sdf['name_last_name_first'] = sdf['name_last'] \n",
    "\n",
    "# build n-gram list\n",
    "vect = CountVectorizer(analyzer='char', max_df=0.3, min_df=3, ngram_range=(NGRAMS, NGRAMS), lowercase=False) \n",
    "a = vect.fit_transform(sdf.name_last_name_first)\n",
    "vocab = vect.vocabulary_\n",
    "\n",
    "# sort n-gram by freq (highest -> lowest)\n",
    "words = []\n",
    "for b in vocab:\n",
    "    c = vocab[b]\n",
    "    #print(b, c, a[:, c].sum())\n",
    "    words.append((a[:, c].sum(), b))\n",
    "    #break\n",
    "words = sorted(words, reverse=True)\n",
    "words_list = [w[1] for w in words]\n",
    "num_words = len(words_list)\n",
    "print(\"num_words = %d\" % num_words)\n",
    "\n",
    "\n",
    "def find_ngrams(text, n):\n",
    "    a = zip(*[text[i:] for i in range(n)])\n",
    "    wi = []\n",
    "    for i in a:\n",
    "        w = ''.join(i)\n",
    "        try:\n",
    "            idx = words_list.index(w)\n",
    "        except:\n",
    "            idx = 0\n",
    "        wi.append(idx)\n",
    "    return wi\n",
    "\n",
    "# build X from index of n-gram sequence\n",
    "X = np.array(sdf.name_last_name_first.apply(lambda c: find_ngrams(c, NGRAMS)))\n",
    "\n",
    "# check max/avg feature\n",
    "X_len = []\n",
    "for x in X:\n",
    "    X_len.append(len(x))\n",
    "\n",
    "max_feature_len = max(X_len)\n",
    "avg_feature_len = int(np.mean(X_len))\n",
    "\n",
    "print(\"Max feature len = %d, Avg. feature len = %d\" % (max_feature_len, avg_feature_len))\n",
    "y = np.array(sdf.race.astype('category').cat.codes)\n",
    "\n",
    "# Split train and test dataset\n",
    "X_train,  X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=21, stratify=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train a LSTM model\n",
    "\n",
    "ref: http://machinelearningmastery.com/sequence-classification-lstm-recurrent-neural-networks-python-keras/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/venv/local/lib/python2.7/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "107097 train sequences\n",
      "26775 test sequences\n",
      "Pad sequences (samples x time)\n",
      "X_train shape: (107097, 20)\n",
      "X_test shape: (26775, 20)\n",
      "13 classes\n",
      "Convert class vector to binary class matrix (for use with categorical_crossentropy)\n",
      "y_train shape: (107097, 13)\n",
      "y_test shape: (26775, 13)\n"
     ]
    }
   ],
   "source": [
    "'''The dataset is actually too small for LSTM to be of any advantage\n",
    "compared to simpler, much faster methods such as TF-IDF + LogReg.\n",
    "Notes:\n",
    "\n",
    "- RNNs are tricky. Choice of batch size is important,\n",
    "choice of loss and optimizer is critical, etc.\n",
    "Some configurations won't converge.\n",
    "\n",
    "- LSTM loss decrease patterns during training can be quite different\n",
    "from what you see with CNNs/MLPs/etc.\n",
    "'''\n",
    "import keras\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, Dropout, Activation\n",
    "from keras.layers import LSTM\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.layers.convolutional import MaxPooling1D\n",
    "from keras.models import load_model\n",
    "\n",
    "max_features = num_words # 20000\n",
    "feature_len = 20 # avg_feature_len # cut texts after this number of words (among top max_features most common words)\n",
    "batch_size = 32\n",
    "\n",
    "print(len(X_train), 'train sequences')\n",
    "print(len(X_test), 'test sequences')\n",
    "\n",
    "print('Pad sequences (samples x time)')\n",
    "X_train = sequence.pad_sequences(X_train, maxlen=feature_len)\n",
    "X_test = sequence.pad_sequences(X_test, maxlen=feature_len)\n",
    "print('X_train shape:', X_train.shape)\n",
    "print('X_test shape:', X_test.shape)\n",
    "\n",
    "num_classes = np.max(y_train) + 1\n",
    "print(num_classes, 'classes')\n",
    "\n",
    "print('Convert class vector to binary class matrix '\n",
    "      '(for use with categorical_crossentropy)')\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "print('y_train shape:', y_train.shape)\n",
    "print('y_test shape:', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 20, 32)            62272     \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 128)               82432     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 13)                1677      \n",
      "=================================================================\n",
      "Total params: 146,381\n",
      "Trainable params: 146,381\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print('Build model...')\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(num_words, 32, input_length=feature_len))\n",
    "model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "# try using different optimizers and different optimizer configs\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train...\n",
      "Train on 96387 samples, validate on 10710 samples\n",
      "Epoch 1/25\n",
      " - 110s - loss: 1.6296 - acc: 0.5088 - val_loss: 1.4220 - val_acc: 0.5765\n",
      "Epoch 2/25\n",
      " - 108s - loss: 1.3881 - acc: 0.5876 - val_loss: 1.3384 - val_acc: 0.6078\n",
      "Epoch 3/25\n",
      " - 108s - loss: 1.3193 - acc: 0.6109 - val_loss: 1.2914 - val_acc: 0.6237\n",
      "Epoch 4/25\n",
      " - 107s - loss: 1.2757 - acc: 0.6242 - val_loss: 1.2654 - val_acc: 0.6323\n",
      "Epoch 5/25\n",
      " - 108s - loss: 1.2425 - acc: 0.6352 - val_loss: 1.2513 - val_acc: 0.6366\n",
      "Epoch 6/25\n",
      " - 107s - loss: 1.2165 - acc: 0.6425 - val_loss: 1.2410 - val_acc: 0.6402\n",
      "Epoch 7/25\n",
      " - 109s - loss: 1.1955 - acc: 0.6483 - val_loss: 1.2306 - val_acc: 0.6455\n",
      "Epoch 8/25\n",
      " - 108s - loss: 1.1773 - acc: 0.6548 - val_loss: 1.2224 - val_acc: 0.6509\n",
      "Epoch 9/25\n",
      " - 108s - loss: 1.1614 - acc: 0.6580 - val_loss: 1.2176 - val_acc: 0.6514\n",
      "Epoch 10/25\n",
      " - 107s - loss: 1.1461 - acc: 0.6618 - val_loss: 1.2126 - val_acc: 0.6525\n",
      "Epoch 11/25\n",
      " - 107s - loss: 1.1329 - acc: 0.6658 - val_loss: 1.2149 - val_acc: 0.6526\n",
      "Epoch 12/25\n",
      " - 107s - loss: 1.1216 - acc: 0.6691 - val_loss: 1.2082 - val_acc: 0.6568\n",
      "Epoch 13/25\n",
      " - 107s - loss: 1.1139 - acc: 0.6714 - val_loss: 1.2033 - val_acc: 0.6619\n",
      "Epoch 14/25\n",
      " - 109s - loss: 1.0999 - acc: 0.6751 - val_loss: 1.2025 - val_acc: 0.6603\n",
      "Epoch 15/25\n",
      " - 108s - loss: 1.0943 - acc: 0.6755 - val_loss: 1.2057 - val_acc: 0.6596\n",
      "Epoch 16/25\n",
      " - 107s - loss: 1.0887 - acc: 0.6785 - val_loss: 1.2004 - val_acc: 0.6613\n",
      "Epoch 17/25\n",
      " - 108s - loss: 1.0803 - acc: 0.6803 - val_loss: 1.2037 - val_acc: 0.6641\n",
      "Epoch 18/25\n",
      " - 108s - loss: 1.0742 - acc: 0.6830 - val_loss: 1.1995 - val_acc: 0.6643\n",
      "Epoch 19/25\n",
      " - 108s - loss: 1.0678 - acc: 0.6838 - val_loss: 1.2029 - val_acc: 0.6636\n",
      "Epoch 20/25\n",
      " - 108s - loss: 1.0639 - acc: 0.6843 - val_loss: 1.2021 - val_acc: 0.6662\n",
      "Epoch 21/25\n",
      " - 112s - loss: 1.0591 - acc: 0.6871 - val_loss: 1.2026 - val_acc: 0.6657\n",
      "Epoch 22/25\n",
      " - 113s - loss: 1.0547 - acc: 0.6880 - val_loss: 1.2027 - val_acc: 0.6669\n",
      "Epoch 23/25\n",
      " - 112s - loss: 1.0504 - acc: 0.6881 - val_loss: 1.2025 - val_acc: 0.6637\n",
      "Epoch 24/25\n",
      " - 113s - loss: 1.0461 - acc: 0.6889 - val_loss: 1.1999 - val_acc: 0.6669\n",
      "Epoch 25/25\n",
      " - 112s - loss: 1.0435 - acc: 0.6908 - val_loss: 1.2059 - val_acc: 0.6660\n",
      "Test score: 1.1949231106827574\n",
      "Test accuracy: 0.6656209150349058\n"
     ]
    }
   ],
   "source": [
    "print('Train...')\n",
    "model.fit(X_train, y_train, batch_size=batch_size, epochs=EPOCHS,\n",
    "          validation_split=0.1, verbose=2)\n",
    "score, acc = model.evaluate(X_test, y_test,\n",
    "                            batch_size=batch_size, verbose=2)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                       precision    recall  f1-score   support\n",
      "\n",
      "     Asian,GreaterEastAsian,EastAsian       0.82      0.75      0.78      1099\n",
      "      Asian,GreaterEastAsian,Japanese       0.83      0.86      0.85      1467\n",
      "             Asian,IndianSubContinent       0.70      0.67      0.69      1572\n",
      "              GreaterAfrican,Africans       0.49      0.37      0.43       734\n",
      "                GreaterAfrican,Muslim       0.56      0.55      0.55      1248\n",
      "              GreaterEuropean,British       0.72      0.86      0.79      8289\n",
      "         GreaterEuropean,EastEuropean       0.72      0.65      0.68      1666\n",
      "               GreaterEuropean,Jewish       0.44      0.36      0.40      2048\n",
      "  GreaterEuropean,WestEuropean,French       0.55      0.49      0.52      2459\n",
      "GreaterEuropean,WestEuropean,Germanic       0.41      0.27      0.33       774\n",
      "GreaterEuropean,WestEuropean,Hispanic       0.61      0.54      0.57      2082\n",
      " GreaterEuropean,WestEuropean,Italian       0.65      0.70      0.68      2374\n",
      "  GreaterEuropean,WestEuropean,Nordic       0.72      0.54      0.62       963\n",
      "\n",
      "                          avg / total       0.66      0.67      0.66     26775\n",
      "\n",
      "[[ 821   41   10    8   19  142    8    8   17    1    9   10    5]\n",
      " [  29 1262    8   19    6   63   11    5   13    2   40    7    2]\n",
      " [   9   19 1060   34  138  159    9   24   28    8   47   32    5]\n",
      " [  10   57   48  275   75  115   14   18   34    7   44   33    4]\n",
      " [  14   14  116   39  682  111   54   65   60    4   38   37   14]\n",
      " [  52   27   69   52   56 7169   66  255  271   38   74  112   48]\n",
      " [  12   16   27   19   37  135 1082  142   65   36   31   46   18]\n",
      " [   9   16   45   24   98  641  123  746   99   65   79   71   32]\n",
      " [  16   14   48   33   40  565   45  126 1193   42  148  177   12]\n",
      " [   5    4    8    3   11  228   15  157   56  209   15   30   33]\n",
      " [   6   27   44   19   23  243   21   52  132   34 1134  327   20]\n",
      " [   9   18   26   24   20  176   27   43  150   22  180 1672    7]\n",
      " [  11    3    8    8   17  188   23   55   45   43   24   21  517]]\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict_classes(X_test, verbose=2)\n",
    "p = model.predict_proba(X_test, verbose=2) # to predict probability\n",
    "target_names = list(sdf.race.astype('category').cat.categories)\n",
    "print(classification_report(np.argmax(y_test, axis=1), y_pred, target_names=target_names))\n",
    "print(confusion_matrix(np.argmax(y_test, axis=1), y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('./models/wiki/lstm/wiki_ln_lstm.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_df = pd.DataFrame(words_list, columns=['vocab'])\n",
    "words_df.to_csv('./models/wiki/lstm/wiki_ln_vocab.csv', index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
