<commit_before>{
 "metadata": {
  "name": "",
  "signature": "sha256:2e00d34e527bf1d264e72083316bafd193fcf615a1d7a3741537726a1aa72062"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "import numpy as np\n",
      "import os\n",
      "import itertools\n",
      "from collections import OrderedDict\n",
      "from popgen.frequencytable import FrequencyTable\n",
      "import popgen.pseudo_sparse_matrix as ps\n",
      "import popgen.adjusting_sample_joint_distribution as adj\n",
      "import popgen.prepare_data as prep\n",
      "from scipy import sparse\n",
      "from popgen.ipu import ipu"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "data_dir = '/Users/conorhenley/Documents/Synthicity/Albuquerque/POPGEN_E'\n",
      "hh_sample_file = 'hhld_pums_500.csv'\n",
      "per_sample_file = 'pop_pums_500.csv'\n",
      "hh_marginals_file = 'hhld_marginals_500.csv'\n",
      "per_marginals_file = 'pop_marginals_500.csv'\n",
      "geocorr_file = 'Puma_Tract_Crosswalk_500.csv'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Parameters\n",
      "ipf_tolerance = 0.0001\n",
      "ipf_iterations = 25\n",
      "ipu_tolerance = 0.0000001\n",
      "ipu_iterations = 50\n",
      "synthetic_draw_tolerance = 0.9999\n",
      "synthetic_draw_iterations = 25\n",
      "rounding_procedure = \"arithmetic\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "hh_sample, hh_marginals, per_sample, per_marginals, hh_var_list, per_vars, hhld_units, hh_dims, per_dims, hhid_var, pid_var = prep.prepare_data(data_dir, hh_sample_file, hh_marginals_file, per_sample_file, per_marginals_file)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "hh_varcorrdict, per_varcorrdict = prep.variables(hh_var_list, hh_dims, per_vars, per_dims)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "matrix = ps.populate_master_matrix(hh_dims, per_dims, hhld_units, hh_sample, per_sample)"
     ],
     "language": "python",
     "metadata": {},<commit_msg>more updates to demo notebook<commit_after>{
 "metadata": {
  "name": "",
  "signature": "sha256:9e86e3561a3451b568c863399fe380d66cd2b124aeddabc29d42193f400b81dd"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from collections import OrderedDict\n",
      "from popgen.frequencytable import FrequencyTable\n",
      "import popgen.pseudo_sparse_matrix as ps\n",
      "import popgen.adjusting_sample_joint_distribution as adj\n",
      "import popgen.prepare_data as prep\n",
      "from popgen.ipu import ipu"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "data_dir = 'data/directory/goes/here'\n",
      "hh_sample_file = 'hhld_pums_500.csv'\n",
      "per_sample_file = 'pop_pums_500.csv'\n",
      "hh_marginals_file = 'hhld_marginals_500.csv'\n",
      "per_marginals_file = 'pop_marginals_500.csv'\n",
      "geocorr_file = 'Puma_Tract_Crosswalk_500.csv'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Parameters\n",
      "ipf_tolerance = 0.0001\n",
      "ipf_iterations = 25\n",
      "ipu_tolerance = 0.0000001\n",
      "ipu_iterations = 50\n",
      "synthetic_draw_tolerance = 0.9999\n",
      "synthetic_draw_iterations = 25\n",
      "rounding_procedure = \"arithmetic\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "hh_sample, hh_marginals, per_sample, per_marginals, hh_var_list, per_vars, hhld_units, hh_dims, per_dims, hhid_var, pid_var = prep.prepare_data(data_dir, hh_sample_file, hh_marginals_file, per_sample_file, per_marginals_file)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "hh_varcorrdict, per_varcorrdict = prep.variables(hh_var_list, hh_dims, per_vars, per_dims)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "matrix = ps.populate_master_matrix(hh_dims, per_dims, hhld_units, hh_sample, per_sample)"
     ],
     "language": "python",
     "metadata": {},
{'fraction_non_alphanumeric': 0.16350446428571427, 'fraction_numerical': 0.03404017857142857, 'mean_word_length': 2.8270462633451956, 'pattern_counts': {'":': 94, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3366764', 'n_tokens_mistral': 2133, 'n_tokens_neox': 1990, 'n_words': 326}