<commit_before>{
 "metadata": {
  "name": "",
  "signature": "sha256:48913c7b7fc08c7b505d9784618e91c3c1d5af338bb23a3d27e8aa1dcdb349f4"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "import numpy as np\n",
      "import matplotlib\n",
      "from models import sync, Tweets\n",
      "%matplotlib inline"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Get Data from SQLAlchemy query\n",
      "session = sync()\n",
      "q = session.query(Tweets.id, Tweets.timestamp, Tweets.coordinates)\n",
      "df = pd.DataFrame([row for row in q])\n",
      "df.index = df[1]\n",
      "# count non-empty columns per day, throw away everything except id and coordinate counts\n",
      "df = df.resample('D', how='count').iloc[:, 3:]\n",
      "df.rename(columns={\n",
      "    'tweet_id': 'Total Tweets',\n",
      "    'coordinates': 'Geotagged Tweets'},\n",
      "    inplace=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# plot\n",
      "plt.clf()\n",
      "fig = plt.figure(figsize=(10, 8))\n",
      "ax = fig.add_subplot(111, axisbg='none')\n",
      "df.plot(\n",
      "    ax=ax,\n",
      "    kind=\"bar\",\n",
      "    stacked=\"true\",\n",
      "    title=\"Tweets from @brockleycentral's followers (Total Tweets: %s)\" % '{0:,}'.format(len(data)),\n",
      "    # I happen to like teal and orange\n",
      "    color=[\"#ff8300\", \"#009aaf\"],\n",
      "    lw=.5,\n",
      "    ec='#000000')\n",
      "leg = ax.legend(loc='best', title=\"Daily Tweet Count\")\n",
      "frame = leg.get_frame()\n",
      "frame.set_alpha(0.4)\n",
      "# default formatting shows hours and gets cut off\n",
      "ax.set_xticklabels(\n",
      "    [dt.strftime(\"%d-%m-%Y\") for dt in df.index.to_pydatetime()])\n",
      "ax.set_ylabel(\"Number of Tweets\")\n",
      "ax.set_xlabel(\"Date\")\n",
      "ax.grid(b=None)\n",
      "plt.tight_layout()\n",
      "# plt.savefig(\"tweets.png\", format=\"png\", bbox_inches='tight', transparent=True, dpi=100)\n",
      "plt.show()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "<matplotlib.figure.Figure at 0x112b30510>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "<KEY>",
       "text": [
        "<matplotlib.figure.Figure at 0x10f4ef950>"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}<commit_msg>Fix up DB query notebook<commit_after>{
 "metadata": {
  "name": "",
  "signature": "sha256:2dcac9020d5041a3955a08911da519cbc994f09a7fb08ee1ff3c83008a3b96f6"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "import dateutil\n",
      "import numpy as np\n",
      "import matplotlib.pyplot as plt\n",
      "from models import sync, Tweets\n",
      "%matplotlib inline"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 61
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from matplotlib import rc\n",
      "rc('font', **{'family':'sans-serif',\n",
      "    'sans-serif':['Helvetica'],\n",
      "    'monospace': ['Inconsolata'],\n",
      "    'serif': ['Helvetica']})\n",
      "rc('text', **{'usetex': True})\n",
      "rc('text', **{'latex.preamble': '\\usepackage{sfmath}'})"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 62
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Get Data from SQLAlchemy query\n",
      "session = sync()\n",
      "q = session.query(Tweets.id, Tweets.timestamp, Tweets.coordinates)\n",
      "df = pd.DataFrame([row for row in q])\n",
      "df.index = df[1]\n",
      "del df[1]\n",
      "df.rename(columns={\n",
      "    0: 'Total Tweets',\n",
      "    2: 'Geotagged Tweets'},\n",
      "    inplace=True)\n",
      "df = df.replace({'Geotagged Tweets': None}, np.nan)\n",
      "# count non-empty columns per day, throw away everything except id and coordinate counts\n",
      "df = df.resample('D', how='count')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 58
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "plt.clf()\n",
      "fig = plt.figure(figsize=(7.28, 5.25))\n",
      "ax = fig.add_subplot(111, axisbg='none')\n",
      "df.plot(\n",
      "    ax=ax,\n",
      "    kind=\"bar\",\n",
      "    stacked=\"true\",\n",
      "    title=\"Tweets from @brockleycentral's followers (Total Tweets: %s)\" % '{0:,}'.format(df['Total Tweets'].sum()),\n",
      "    # I happen to like teal and orange\n",
      "    color=[\"#009aaf\", \"#ff8300\"],\n",
      "    lw=.5,\n",
      "    ec='#000000')\n",
      "leg = ax.legend(loc='best', title=\"Daily Tweet Count\")\n",
      "frame = leg.get_frame()\n",
      "frame.set_alpha(0.4)\n",
      "# default formatting shows hours and gets cut off\n",
      "ax.set_xticklabels(\n",
      "    [dt.strftime(\"%d-%m-%Y\") for dt in df.index.to_pydatetime()])\n",
      "ax.set_ylabel(\"Number of Tweets\")\n",
      "ax.set_xlabel(\"Date\")\n",
      "ax.grid(b=None)\n",
      "plt.tight_layout()\n",
      "# plt.savefig(\"tweets.png\", format=\"png\", bbox_inches='tight', transparent=True, dpi=100)\n",
      "plt.show()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "<matplotlib.figure.Figure at 0x10e5bd090>"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "<KEY>",
       "text": [
        "<matplotlib.figure.Figure at 0x10d41a1d0>"
       ]
      }
     ],
     "prompt_number": 68
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}
{'fraction_non_alphanumeric': 0.21928251121076234, 'fraction_numerical': 0.028400597907324365, 'mean_word_length': 2.1953199617956063, 'pattern_counts': {'":': 90, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '80592', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2495, 'n_words': 432}