<commit_before>{
 "metadata": {
  "name": "Dewp_freq"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import psycopg2\n",
      "import numpy\n",
      "ASOS = psycopg2.connect(database='asos', user='nobody', host='iemdb')\n",
      "cursor = ASOS.cursor()\n",
      "\n",
      "tmp_hits = numpy.zeros( (121-50,), 'f')\n",
      "tmp_cnts = numpy.zeros( (121-50,), 'f')\n",
      "doy_hits = numpy.zeros( (366,), 'f')\n",
      "doy_cnts = numpy.zeros( (366,), 'f')\n",
      "\n",
      "cursor.execute(\"\"\"SELECT extract(doy from valid), round(tmpf::numeric,0), dwpf from alldata\n",
      "  WHERE station = 'DSM' and extract(hour from valid + '10 minutes'::interval) in (13,14,15,16)\n",
      "  and tmpf is not null and dwpf is not null\"\"\")\n",
      "for row in cursor:\n",
      "    doy_cnts[ row[0] - 1 ] += 1.0\n",
      "    if row[1] >= 50:\n",
      "        tmp_cnts[ row[1] - 50 ] += 1.0\n",
      "        if row[2] < 50:\n",
      "            tmp_hits[ row[1] - 50 ] += 1.0\n",
      "    if row[2] < 50:\n",
      "        doy_hits[ row[0] - 1 ] += 1.0\n",
      "        \n",
      "cursor.close()\n",
      "ASOS.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import matplotlib.pyplot as plt\n",
      "(fig, ax) = plt.subplots(2,1)\n",
      "doy_cnts = numpy.where( doy_cnts < 1, 1, doy_cnts)\n",
      "\n",
      "ax[0].bar( numpy.arange(50,101)-0.4, tmp_hits[:51] / tmp_cnts[:51] * 100.0, width=1.0,\n",
      "  fc='r', ec='r')\n",
      "ax[0].set_xlim(49.5,100.5)\n",
      "ax[0].grid(True)\n",
      "ax[0].set_title(\"1933-2012 Des Moines Afternoon (1-4 PM) Frequency\\nof sub 50$^{\\circ}\\mathrm{F}$ Dew Points\")\n",
      "ax[0].set_ylabel(\"Freq.[%] by Air Temp\")\n",
      "ax[0].set_yticks([0,25,50,75,100])\n",
      "\n",
      "ax[1].bar( numpy.arange(50,321), doy_hits[50:321] / doy_cnts[50:321] * 100.0, width=1.0,\n",
      "  fc='g', ec='g')\n",
      "ax[1].set_xticks( (1,32,60,91,121,152,182,213,244,274,305,335,365) )\n",
      "ax[1].set_xticklabels( ('Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec') )\n",
      "ax[1].set_xlim(50,320)\n",
      "ax[1].grid(True)\n",
      "ax[1].set_ylabel(\"Freq.[%] by Day of Year\")\n",
      "ax[1].set_yticks([0,25,50,75,100])\n",
      "\n",
      "fig.savefig('test.svg')\n",
      "import iemplot\n",
      "iemplot.makefeature('test')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "display_data",
       "png": "<KEY>+++<KEY>//<KEY>8DA1NTWzrSQIcwLulSQQCASCfyeixyAQ\nCAQCP0RgEAgEAoEfIjAIBAKBwA8RGAQCgUDghwgMAoFAIPBDBAaBQCAQ+PE/rljuvmSrsAQAAAAA\nSUVORK5CYII=\n"
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}<commit_msg>18 jun feature<commit_after>{
 "metadata": {
  "name": "Dewp_freq"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import psycopg2\n",
      "import numpy\n",
      "ASOS = psycopg2.connect(database='asos', user='nobody', host='iemdb')\n",
      "cursor = ASOS.cursor()\n",
      "\n",
      "tmp_hits = numpy.zeros( (121-50,), 'f')\n",
      "tmp_cnts = numpy.zeros( (121-50,), 'f')\n",
      "doy_hits = numpy.zeros( (366,), 'f')\n",
      "doy_cnts = numpy.zeros( (366,), 'f')\n",
      "\n",
      "cursor.execute(\"\"\"SELECT extract(doy from valid), round(tmpf::numeric,0), dwpf from alldata\n",
      "  WHERE station = 'DSM' and extract(hour from valid + '10 minutes'::interval) in (13,14,15,16)\n",
      "  and tmpf is not null and dwpf is not null\"\"\")\n",
      "for row in cursor:\n",
      "    doy_cnts[ row[0] - 1 ] += 1.0\n",
      "    if row[1] >= 55:\n",
      "        tmp_cnts[ row[1] - 55 ] += 1.0\n",
      "        if row[2] < 55:\n",
      "            tmp_hits[ row[1] - 55 ] += 1.0\n",
      "    if row[2] < 55:\n",
      "        doy_hits[ row[0] - 1 ] += 1.0\n",
      "        \n",
      "cursor.close()\n",
      "ASOS.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import matplotlib.pyplot as plt\n",
      "(fig, ax) = plt.subplots(2,1)\n",
      "doy_cnts = numpy.where( doy_cnts < 1, 1, doy_cnts)\n",
      "\n",
      "ax[0].bar( numpy.arange(55,101)-0.4, tmp_hits[:46] / tmp_cnts[:46] * 100.0, width=1.0,\n",
      "  fc='r', ec='r')\n",
      "ax[0].set_xlim(54.5,100.5)\n",
      "ax[0].grid(True)\n",
      "ax[0].set_title(\"1933-2012 Des Moines Afternoon (1-4 PM) Frequency\\nof sub 55$^{\\circ}\\mathrm{F}$ Dew Points\")\n",
      "ax[0].set_ylabel(\"Freq.[%] by Air Temp\")\n",
      "ax[0].set_yticks([0,25,50,75,100])\n",
      "\n",
      "ax[1].bar( numpy.arange(50,321), doy_hits[50:321] / doy_cnts[50:321] * 100.0, width=1.0,\n",
      "  fc='g', ec='g')\n",
      "ax[1].set_xticks( (1,32,60,91,121,152,182,213,244,274,305,335,365) )\n",
      "ax[1].set_xticklabels( ('Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec') )\n",
      "ax[1].set_xlim(50,320)\n",
      "ax[1].grid(True)\n",
      "ax[1].set_ylabel(\"Freq.[%] by Day of Year\")\n",
      "ax[1].set_yticks([0,25,50,75,100])\n",
      "\n",
      "fig.savefig('test.svg')\n",
      "import iemplot\n",
      "iemplot.makefeature('test')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "/usr/lib64/python2.6/site-packages/matplotlib/__init__.py:1005: UserWarning:  This call to matplotlib.use() has no effect\n",
        "because the the backend has already been chosen;\n",
        "matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n",
        "or matplotlib.backends is imported for the first time.\n",
        "\n",
        "  warnings.warn(_use_error_msg)\n"
       ]
      },
      {
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEcCAYAAADDfRPAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XlcVFX/B/DPoLgyI4hAbDqomQhYYBmaC2qG4o4obuSS\nmZZZuZTVY6BZZs9juZVZrolrWm4Ylem4b78EzQ1LHQU3wBFZBYHz+wPnysDcucNs9w58368XL71z\nZ+75zpmZe+5Z7jkyxhgDIYQQ8piD2AEQQgiRFioYCCGE6KCCgRBCiA4qGAghhOiggoEQQogOKhgI\nIYTooIKBSMr69esRHh4udhg6zp07B39/fzg5OWHp0qVih2NRGRkZ8Pf3R2FhodihGKWwsBD+/v7I\nzMwUO5RqrVoVDEuXLsXzzz+PevXqYezYsZX2L1myBE2bNoVCoUDv3r1x+/Ztbt9XX30FPz8/yOVy\nNG7cGCNHjoRGo+H2d+3aFW5ubmjYsCFatmyJJUuW8MaRkZGBqKgoeHh4wMnJCe3bt8fhw4d1nrNh\nwwb4+flBoVBg0KBBuH//vlHv49ixY+jWrRucnZ3RqFEjDBgwADdv3uSNJSwsDPXr14dCoYCLiwue\nf/55zJ8/H0VFRfwZaSKlUom6devi3r17Oo8HBwfDwcEBN27cEDzGyJEj8dtvv1k8NmOMGTMGjo6O\nuHPnjs7jX375JSIiIpCbm4vJkydDqVRi3759osRoaV988QXGjh2LunXrAgC2bNmCjh07omHDhujW\nrZvRx9FoNHBzc0Pnzp15n7NmzRrUqlULcrkcDRo0QEBAALZu3QoAUKlUcHBwQGRkpM5rzpw5AwcH\nBy6WunXrYty4cfjiiy9404mLi4OjoyPkcjn397///c/o90KqWcHg7e2NWbNmYdy4cZX2JSYmYu7c\nudi/fz/u37+Pp59+GsOHD+f2Dxo0CGfOnEFOTg6uXLmCO3fuYO7cudz+b7/9Fnfu3EFeXh42bNiA\nGTNm4O+//9YbR15eHl5++WVcunQJOTk5mDhxIvr27Yvs7GwAwPnz5/Hmm29i27ZtuHfvHlxcXPDm\nm28a9T5ycnIwbdo0pKen486dO/Dw8MDIkSN580Qmk+Gbb75BdnY2NBoNli5dil9++QURERHCGVpF\nMpkMzZs3x8aNG7nH/v77bxQUFEAmk1k8PUvKy8vDtm3b0KZNG8THx+vsS0tLQ5s2bbhtmUwGU+8L\nZYyZ/FpLKywsxI8//ohRo0Zxj7m6umLq1KmYOXNmlY71wQcfoE2bNoKf80svvYScnBzk5eVh0qRJ\nGDVqFDIyMgAAbm5uOH78uM4F2dq1a9GqVSud4w4fPhxr167Fo0eP9KYhk8kwfPhw5OTkcH/Tp0+v\n9LzS0tIqvceapFoVDIMGDcKAAQPg6upaaV9CQgKGDBmCFi1aoFatWvjwww9x8OBBXLt2DQC4q3eg\n7Avj4OCAZs2aca8PCAhArVq1uG0nJyc0adJEbxxKpRITJ06Ei4sLZDIZxo0bB0dHR1y8eBFAWXNJ\nZGQkQkJC4OjoiE8//RQ///<KEY>"
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}
{'fraction_non_alphanumeric': 0.204321961879325, 'fraction_numerical': 0.07891222532475416, 'mean_word_length': 3.0146198830409356, 'pattern_counts': {'":': 61, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1984361', 'n_tokens_mistral': 4306, 'n_tokens_neox': 3897, 'n_words': 447}