<commit_before>{
 "metadata": {
  "name": "",
  "signature": "sha256:50f57900231b0aae65ab29e4c01335b5baf354e313224568ce75e22948422767"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%matplotlib inline\n",
      "\n",
      "import datetime\n",
      "import json \n",
      "import math\n",
      "import random\n",
      "import operator\n",
      "import os\n",
      "from calendar import monthrange\n",
      "\n",
      "import pandas as pd\n",
      "import numpy as np\n",
      "\n",
      "import sqlalchemy as sa\n",
      "from sqlalchemy import *\n",
      "\n",
      "import matplotlib \n",
      "import matplotlib.pyplot as pyplt\n",
      "from pylab import *\n",
      "\n",
      "data_directory = '../data/common_appliances/'\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#Most common appliances"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# to find days in a certain month for a certain year:\n",
      "monthrange(2011, 2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "(1, 28)"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# [SENSITIVE] Create database engine\n",
      "user_name = 'stomkins'\n",
      "pw='PASSWORD'\n",
      "host = \"db.wiki-energy.org\"\n",
      "port = \"5432\"\n",
      "db = \"postgres\"\n",
      "url = \"postgresql\"+\"://\"+user_name+\":\"+pw+\"@\"+host+\":\"+port+\"/\"+db\n",
      "\n",
      "eng = create_engine(url)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# runs in < 1s\n",
      "def get_unique_dataids(eng,group,month):\n",
      "    '''gets all dataids for a specifc group(int) for a specific month (int)'''\n",
      "    query = 'select distinct dataid from \"PecanStreet_CuratedSets\".group{0}_disaggregated_2013_{1:02d}'.format(group,month)\n",
      "    eng_object=eng.execute(query)\n",
      "    df = pd.DataFrame.from_records(eng_object.fetchall())\n",
      "    return df[0]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dataids = get_unique_dataids(eng,1,1)\n",
      "dataids[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "5949"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# takes about 5s to run\n",
      "def get_home_month(eng,group,year,month,dataid):\n",
      "    '''Returns a dictionary where the keys are the homes and the values are \n",
      "    dataframe represenations of the tables associated with that home\n",
      "    '''\n",
      "    \n",
      "    query = 'select * from \"PecanStreet_CuratedSets\".group{0}_disaggregated_{1}_{2:02d} where dataid={3}'.format(group,year,month,dataid)\n",
      "    eng_object=eng.execute(query)\n",
      "    df = pd.DataFrame.from_records(eng_object.fetchall())\n",
      "    df.columns = eng_object.keys()\n",
      "    return df\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# get all houses data for january. takes about 2 min\n",
      "houses_data = [get_home_month(eng,1,2013,1,dataid) for dataid in dataids]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",<commit_msg>update most common appliances notebook<commit_after>{
 "metadata": {
  "name": "",
  "signature": "sha256:f2db2d159f81b5b61f8c7d9043fe768a10613eda8431ef51f8d7c41af94099ec"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%matplotlib inline\n",
      "\n",
      "import datetime\n",
      "import json \n",
      "import math\n",
      "import random\n",
      "import operator\n",
      "import os\n",
      "from calendar import monthrange\n",
      "\n",
      "import pandas as pd\n",
      "import numpy as np\n",
      "\n",
      "import sqlalchemy as sa\n",
      "from sqlalchemy import *\n",
      "\n",
      "import matplotlib \n",
      "import matplotlib.pyplot as pyplt\n",
      "from pylab import *\n",
      "\n",
      "data_directory = '../data/common_appliances/'\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#Most common appliances"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# to find days in a certain month for a certain year:\n",
      "monthrange(2011, 2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "(1, 28)"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# [SENSITIVE] Create database engine\n",
      "user_name = 'stomkins'\n",
      "pw='PASSWORD'\n",
      "host = \"db.wiki-energy.org\"\n",
      "port = \"5432\"\n",
      "db = \"postgres\"\n",
      "url = \"postgresql\"+\"://\"+user_name+\":\"+pw+\"@\"+host+\":\"+port+\"/\"+db\n",
      "\n",
      "eng = create_engine(url)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# runs in < 1s\n",
      "def get_unique_dataids(eng,group,month):\n",
      "    '''gets all dataids for a specifc group(int) for a specific month (int)'''\n",
      "    query = 'select distinct dataid from \"PecanStreet_CuratedSets\".group{0}_disaggregated_2013_{1:02d}'.format(group,month)\n",
      "    eng_object=eng.execute(query)\n",
      "    df = pd.DataFrame.from_records(eng_object.fetchall())\n",
      "    return df[0]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dataids = get_unique_dataids(eng,1,1)\n",
      "dataids[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "5949"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# takes about 5s to run\n",
      "def get_home_month(eng,group,year,month,dataid):\n",
      "    '''Returns a dictionary where the keys are the homes and the values are \n",
      "    dataframe represenations oft the tables associated with that home\n",
      "    '''\n",
      "    \n",
      "    query = 'select * from \"PecanStreet_CuratedSets\".group{0}_disaggregated_{1}_{2:02d} where dataid={3}'.format(group,year,month,dataid)\n",
      "    eng_object=eng.execute(query)\n",
      "    df = pd.DataFrame.from_records(eng_object.fetchall())\n",
      "    df.columns = eng_object.keys()\n",
      "    return df\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# get all houses data for january. takes about 2 min\n",
      "houses_data = [get_home_month(eng,1,2013,1,dataid) for dataid in dataids]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
{'fraction_non_alphanumeric': 0.19301557338367153, 'fraction_numerical': 0.023596035865974516, 'mean_word_length': 2.0971867007672635, 'pattern_counts': {'":': 142, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '899654', 'n_tokens_mistral': 3156, 'n_tokens_neox': 2998, 'n_words': 579}