<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMDB Sentiment Analysis\n",
    "\n",
    "Sentiment Aanlysis with a CNN TensorFlow model. Based on:\n",
    "* [This 2014 paper by Kim](https://arxiv.org/pdf/1408.5882v2.pdf)\n",
    "* [This blog post by WILDML](http://www.wildml.com/2015/12/implementing-a-cnn-for-text-classification-in-tensorflow/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import packages\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from random import shuffle\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.corpus import stopwords\n",
    "from tflearn.data_utils import VocabularyProcessor\n",
    "\n",
    "# Define directories\n",
    "train_pos_dir = 'aclImdb/train/pos/'\n",
    "train_neg_dir = 'aclImdb/train/neg/'\n",
    "test_pos_dir = 'aclImdb/test/pos/'\n",
    "test_neg_dir = 'aclImdb/test/neg/'\n",
    "\n",
    "# Define dataset sizes\n",
    "data_set_size = 12500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000 train reviews:\n",
      "anime underrated still hardly dorky kids movie noted still come back years first saw one better movi\n",
      "\n",
      "25000 test reviews:\n",
      "sure version film saw entertaining know twins gillian chung charlene choi seeing movie think english\n",
      "\n",
      "50000 total reviews:\n",
      "anime underrated still hardly dorky kids movie noted still come back years first saw one better movi\n"
     ]
    }
   ],
   "source": [
    "# Read and process data\n",
    "stop_words = set(stopwords.words(\"english\"))\n",
    "\n",
    "with open('aclImdb/imdb.vocab', 'r') as myfile:\n",
    "    vocab = [x for x in myfile.read().split('\\n') if not x in stop_words]\n",
    "\n",
    "def parse_html(data):\n",
    "    data = BeautifulSoup(data, 'lxml').get_text() # Remove markup\n",
    "    data = re.sub(\"[^a-zA-Z]\",\" \", data) # Remove all non-alphanumeric characters\n",
    "    data = ' '.join([x for x in data.lower().split() if not x in stop_words]) # Remove stopwords\n",
    "    return data\n",
    "\n",
    "train_pos = []\n",
    "for file_name in os.listdir(train_pos_dir):\n",
    "    with open(train_pos_dir + file_name, 'r') as myfile:\n",
    "        train_pos.append(myfile.read())\n",
    "        \n",
    "train_neg = []\n",
    "for file_name in os.listdir(train_neg_dir):\n",
    "    with open(train_neg_dir + file_name, 'r') as myfile:\n",
    "        train_neg.append(myfile.read())\n",
    "\n",
    "test_pos = []\n",
    "for file_name in os.listdir(test_pos_dir):\n",
    "    with open(test_pos_dir + file_name, 'r') as myfile:\n",
    "        test_pos.append(myfile.read())\n",
    "        \n",
    "test_neg = []\n",
    "for file_name in os.listdir(test_neg_dir):\n",
    "    with open(test_neg_dir + file_name, 'r') as myfile:\n",
    "        test_neg.append(myfile.read())                 \n",
    "    \n",
    "for i in xrange(data_set_size):\n",
    "    train_pos[i] = parse_html(train_pos[i])\n",
    "    train_neg[i] = parse_html(train_neg[i])\n",
    "    test_pos[i] = parse_html(test_pos[i])\n",
    "    test_neg[i] = parse_html(test_neg[i])    \n",
    "    \n",
    "train_x = np.concatenate([train_pos, train_neg])\n",
    "train_y = np.concatenate([[[0, 1] for _ in xrange(data_set_size)], [[1, 0] for _ in xrange(data_set_size)]])\n",
    "test_x = np.concatenate([test_pos, test_neg])\n",
    "test_y = np.concatenate([[[0, 1] for _ in xrange(data_set_size)], [[1, 0] for _ in xrange(data_set_size)]])\n",
    "total_x = np.concatenate([train_pos, train_neg, test_pos, test_neg])\n",
    "    \n",
    "print '%i train reviews:' % len(train_x)\n",
    "print train_x[0][:100]\n",
    "print '\\n%i test reviews:' % len(test_x)\n",
    "print test_x[0][:100]\n",
    "print '\\n%i total reviews:' % len(total_x)\n",
    "print total_x[0][:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "<KEY>",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6ce193b690>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Graph distribution of review lengths\n",
    "%matplotlib inline\n",
    "review_lengths = sorted([len(x.split()) for x in train_x] + [len(x.split()) for x in test_x])[:49900]\n",
    "plt.show(plt.bar(range(len(review_lengths)), review_lengths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Convert reviews into padded word indexes\n",
    "num_words = 250 # Based on graph (want to capture most reviews without too much padding)\n",
    "vocab_processor = VocabularyProcessor(num_words).fit_transform(total_x)\n",
    "total_x_vector = np.array(list(vocab_processor))\n",
    "train_x_vector = total_x_vector[:25000]\n",
    "test_x_vector = total_x_vector[25000:]\n",
    "\n",
    "# Shuffle data\n",
    "shuffled_i = np.random.permutation(np.arange(data_set_size * 2))\n",
    "train_x_vector_shuffled = train_x_vector[shuffled_i]\n",
    "train_y_shuffled = train_y[shuffled_i]\n",
    "test_x_vector_shuffled = test_x_vector[shuffled_i]\n",
    "test_y_shuffled = test_y[shuffled_i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train epoch 0: loss = 220.23, train accuracy = 48.44%\n",
      "Train epoch 1: loss = 45.31, train accuracy = 50.00%\n"
     ]<commit_msg>hotfix<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMDB Sentiment Analysis\n",
    "\n",
    "Sentiment Aanlysis with a CNN TensorFlow model. Based on:\n",
    "* [This 2014 paper by Kim](https://arxiv.org/pdf/1408.5882v2.pdf)\n",
    "* [This blog post by WILDML](http://www.wildml.com/2015/12/implementing-a-cnn-for-text-classification-in-tensorflow/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import packages\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from random import shuffle\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.corpus import stopwords\n",
    "from tflearn.data_utils import VocabularyProcessor\n",
    "\n",
    "# Define directories\n",
    "train_pos_dir = 'aclImdb/train/pos/'\n",
    "train_neg_dir = 'aclImdb/train/neg/'\n",
    "test_pos_dir = 'aclImdb/test/pos/'\n",
    "test_neg_dir = 'aclImdb/test/neg/'\n",
    "\n",
    "# Define dataset sizes\n",
    "data_set_size = 12500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000 train reviews:\n",
      "anime underrated still hardly dorky kids movie noted still come back years first saw one better movi\n",
      "\n",
      "25000 test reviews:\n",
      "sure version film saw entertaining know twins gillian chung charlene choi seeing movie think english\n",
      "\n",
      "50000 total reviews:\n",
      "anime underrated still hardly dorky kids movie noted still come back years first saw one better movi\n"
     ]
    }
   ],
   "source": [
    "# Read and process data\n",
    "stop_words = set(stopwords.words(\"english\"))\n",
    "\n",
    "with open('aclImdb/imdb.vocab', 'r') as myfile:\n",
    "    vocab = [x for x in myfile.read().split('\\n') if not x in stop_words]\n",
    "\n",
    "def parse_html(data):\n",
    "    data = BeautifulSoup(data, 'lxml').get_text() # Remove markup\n",
    "    data = re.sub(\"[^a-zA-Z]\",\" \", data) # Remove all non-alphanumeric characters\n",
    "    data = ' '.join([x for x in data.lower().split() if not x in stop_words]) # Remove stopwords\n",
    "    return data\n",
    "\n",
    "train_pos = []\n",
    "for file_name in os.listdir(train_pos_dir):\n",
    "    with open(train_pos_dir + file_name, 'r') as myfile:\n",
    "        train_pos.append(myfile.read())\n",
    "        \n",
    "train_neg = []\n",
    "for file_name in os.listdir(train_neg_dir):\n",
    "    with open(train_neg_dir + file_name, 'r') as myfile:\n",
    "        train_neg.append(myfile.read())\n",
    "\n",
    "test_pos = []\n",
    "for file_name in os.listdir(test_pos_dir):\n",
    "    with open(test_pos_dir + file_name, 'r') as myfile:\n",
    "        test_pos.append(myfile.read())\n",
    "        \n",
    "test_neg = []\n",
    "for file_name in os.listdir(test_neg_dir):\n",
    "    with open(test_neg_dir + file_name, 'r') as myfile:\n",
    "        test_neg.append(myfile.read())                 \n",
    "    \n",
    "for i in xrange(data_set_size):\n",
    "    train_pos[i] = parse_html(train_pos[i])\n",
    "    train_neg[i] = parse_html(train_neg[i])\n",
    "    test_pos[i] = parse_html(test_pos[i])\n",
    "    test_neg[i] = parse_html(test_neg[i])    \n",
    "    \n",
    "train_x = np.concatenate([train_pos, train_neg])\n",
    "train_y = np.concatenate([[[0, 1] for _ in xrange(data_set_size)], [[1, 0] for _ in xrange(data_set_size)]])\n",
    "test_x = np.concatenate([test_pos, test_neg])\n",
    "test_y = np.concatenate([[[0, 1] for _ in xrange(data_set_size)], [[1, 0] for _ in xrange(data_set_size)]])\n",
    "total_x = np.concatenate([train_pos, train_neg, test_pos, test_neg])\n",
    "    \n",
    "print '%i train reviews:' % len(train_x)\n",
    "print train_x[0][:100]\n",
    "print '\\n%i test reviews:' % len(test_x)\n",
    "print test_x[0][:100]\n",
    "print '\\n%i total reviews:' % len(total_x)\n",
    "print total_x[0][:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "<KEY>",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6ce193b690>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Graph distribution of review lengths\n",
    "%matplotlib inline\n",
    "review_lengths = sorted([len(x.split()) for x in train_x] + [len(x.split()) for x in test_x])[:49900]\n",
    "plt.show(plt.bar(range(len(review_lengths)), review_lengths))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Convert reviews into padded word indexes\n",
    "num_words = 250 # Based on graph (want to capture most reviews without too much padding)\n",
    "vocab_processor = VocabularyProcessor(num_words).fit_transform(total_x)\n",
    "total_x_vector = np.array(list(vocab_processor))\n",
    "train_x_vector = total_x_vector[:25000]\n",
    "test_x_vector = total_x_vector[25000:]\n",
    "\n",
    "# Shuffle data\n",
    "shuffled_i = np.random.permutation(np.arange(data_set_size * 2))\n",
    "train_x_vector_shuffled = train_x_vector[shuffled_i]\n",
    "train_y_shuffled = train_y[shuffled_i]\n",
    "test_x_vector_shuffled = test_x_vector[shuffled_i]\n",
    "test_y_shuffled = test_y[shuffled_i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train epoch 0: loss = 220.23, train accuracy = 48.44%\n",
      "Train epoch 1: loss = 45.31, train accuracy = 50.00%\n",
      "Train epoch 2: loss = 10.60, train accuracy = 76.56%\n",
      "Train epoch 3: loss = 6.02, train accuracy = 78.12%\n",
      "Train epoch 4: loss = 4.72, train accuracy = 75.00%\n",
      "Train epoch 5: loss = 3.01, train accuracy = 85.94%\n",
      "Train epoch 6: loss = 2.73, train accuracy = 87.50%\n",
      "Train epoch 7: loss = 2.86, train accuracy = 85.94%\n",
      "Train epoch 8: loss = 2.36, train accuracy = 90.62%\n",
      "Train epoch 9: loss = 2.23, train accuracy = 90.62%\n",
      "Test set accuracy: 62.50%\n"
     ]
{'fraction_non_alphanumeric': 0.1844583366305294, 'fraction_numerical': 0.02318588272533038, 'mean_word_length': 2.766915052160954, 'pattern_counts': {'":': 88, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1991282', 'n_tokens_mistral': 5082, 'n_tokens_neox': 4825, 'n_words': 992}