<commit_before>{
 "metadata": {
  "name": "",
  "signature": "sha256:c81944589b6beae2f7433977684666f5c6dc6dc6304daded8ade5eadba5f5930"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas\n",
      "from disaggregator import utils\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 220,
       "text": [
        "<module 'disaggregator.utils' from '../disaggregator/utils.py'>"
       ]
      }
     ],
     "prompt_number": 220
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "reload(utils)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 232,
       "text": [
        "<module 'disaggregator.utils' from '../disaggregator/utils.py'>"
       ]
      }
     ],
     "prompt_number": 232
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sys\n",
      "sys.path.append('../')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import disaggregator.PecanStreet as pecan\n",
      "\n",
      "\n",
      "user_name = 'stomkins'\n",
      "pw='PASSWORD'\n",
      "host = \"db.wiki-energy.org\"\n",
      "port = \"5432\"\n",
      "db = \"postgres\"\n",
      "db_url = \"postgresql\"+\"://\"+user_name+\":\"+pw+\"@\"+host+\":\"+port+\"/\"+db\n",
      "\n",
      "table = {'curated':'\\\"PecanStreet_CuratedSets\\\"','raw':'\\\"PecanStreet_RawData\\\"','shared':'\\\"PecanStreet_SharedData\\\"'}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "reload(pecan)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 227,
       "text": [
        "<module 'disaggregator.PecanStreet' from '../disaggregator/PecanStreet.py'>"
       ]
      }
     ],
     "prompt_number": 227
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pecan.set_url(db_url)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 228
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pecan.eng"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "Engine(postgresql://stomkins:***@db.wiki-energy.org:5432/postgres)"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "schema = 'shared'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 41
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tables= pecan.get_table_names(schema)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 47
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[i,a] = pecan.get_table_metadata(schema,str(tables[0]))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 53
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "query = 'select * from {0}.{1} where dataid={2}'.format(pecan.schema_names[schema], tables[0],i[0])\n",
      "df = pecan.get_dataframe(query).fillna(0)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 66
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "temp = pecan.clean_dataframe(df,schema,[])\n",
      "test = pecan.get_month_traces(schema,tables[0],i[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 120
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test[3].series.name"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 125,
       "text": [
        "u'air3'"
       ]
      }
     ],
     "prompt_number": 125
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test_all = pecan.get_app_traces_all(schema,tables[0],'air1',i)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 229
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "utils.pickle_object(test_all,'air_{}'.format(tables[0]))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test_all[0].series.name"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 226,
       "text": [
        "'air1'"
       ]
      }
     ],
     "prompt_number": 226
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}<commit_msg>I don't think anything really changed<commit_after>{
 "metadata": {
  "name": "",
  "signature": "sha256:17e130a8fd48191d0486235f1690e66bfdec75a13e456639516e8e40c647b336"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas\n",
      "from disaggregator import utils\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 220,
       "text": [
        "<module 'disaggregator.utils' from '../disaggregator/utils.py'>"
       ]
      }
     ],
     "prompt_number": 220
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "reload(utils)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 232,
       "text": [
        "<module 'disaggregator.utils' from '../disaggregator/utils.py'>"
       ]
      }
     ],
     "prompt_number": 232
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sys\n",
      "sys.path.append('../')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import disaggregator.PecanStreet as pecan\n",
      "\n",
      "\n",
      "user_name = 'stomkins'\n",
      "pw='PASSWORD'\n",
      "host = \"db.wiki-energy.org\"\n",
      "port = \"5432\"\n",
      "db = \"postgres\"\n",
      "db_url = \"postgresql\"+\"://\"+user_name+\":\"+pw+\"@\"+host+\":\"+port+\"/\"+db\n",
      "\n",
      "table = {'curated':'\\\"PecanStreet_CuratedSets\\\"','raw':'\\\"PecanStreet_RawData\\\"','shared':'\\\"PecanStreet_SharedData\\\"'}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "reload(pecan)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 227,
       "text": [
        "<module 'disaggregator.PecanStreet' from '../disaggregator/PecanStreet.py'>"
       ]
      }
     ],
     "prompt_number": 227
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pecan.set_url(db_url)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 228
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pecan.eng"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "Engine(postgresql://stomkins:***@db.wiki-energy.org:5432/postgres)"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "schema = 'shared'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 41
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tables= pecan.get_table_names(schema)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 47
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "[i,a] = pecan.get_table_metadata(schema,str(tables[0]))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 53
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "query = 'select * from {0}.{1} where dataid={2}'.format(pecan.schema_names[schema], tables[0],i[0])\n",
      "df = pecan.get_dataframe(query).fillna(0)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 66
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "temp = pecan.clean_dataframe(df,schema,[])\n",
      "test = pecan.get_month_traces(schema,tables[0],i[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 120
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test[3].series.name"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 125,
       "text": [
        "u'air3'"
       ]
      }
     ],
     "prompt_number": 125
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test_all = pecan.get_app_traces_all(schema,tables[0],'air1',i)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 229
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "utils.pickle_object(test_all,'air_{}'.format(tables[0]))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 233
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test_all[0].series.name"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 226,
       "text": [
        "'air1'"
       ]
      }
     ],
     "prompt_number": 226
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}
{'fraction_non_alphanumeric': 0.2170806100217865, 'fraction_numerical': 0.02196078431372549, 'mean_word_length': 1.9569698531306365, 'pattern_counts': {'":': 305, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7135551', 'n_tokens_mistral': 4330, 'n_tokens_neox': 4068, 'n_words': 613}