<jupyter_start><jupyter_text>More troubleshooting of network aggregations<NAME>, July 2018 - Python 3.6Confirm what's causing the -1 aggregation values<jupyter_code>import numpy as np
import orca
import pandana as pdna
import pandas as pd
import os; os.chdir('..')
from urbansim.utils import misc, networks
import warnings;warnings.simplefilter('ignore')
os.getcwd()
pdna.__version__
d = 'data/'
@orca.table(cache=True)
def parcels():
    df = pd.read_csv(
        d + 'mtc_data_platform_format_7-6-18/' + 'parcel_attr.csv',
#        d + 'parcels_with_nodes.csv',
        index_col='primary_id', dtype={'primary_id': int, 'block_id':str})
    return df

@orca.table(cache=True)
def buildings():
    df = pd.read_csv(
        d + 'mtc_data_platform_format_7-6-18/' + 'buildings_v2.csv',
        index_col='building_id', dtype={'building_id': int, 'parcel_id': int})
    df['res_sqft_per_unit'] = df['residential_sqft'] / df['residential_units']
    df['res_sqft_per_unit'][df['res_sqft_per_unit'] == np.inf] = 0
    return df

@orca.table(cache=True)
def units():
    df = pd.read_csv(
        d + 'mtc_data_platform_format_7-6-18/' + 'units_v2.csv',
        index_col='unit_id', dtype={'unit_id': int, 'building_id': int})
    return df

@orca.table(cache=True)
def households():
    df = pd.read_csv(
        d + 'mtc_data_platform_format_7-6-18/' + 'households_v2.csv',
        index_col='household_id', dtype={
            'househould_id': int, 'block_group_id': str, 'state': str, 
            'county': str, 'tract': str, 'block_group': str,
            'building_id': int, 'unit_id': int})
    return df
@orca.step()
def initialize_network_walk():
    """
    This will be turned into a data loading template.
    """

    @orca.injectable('netwalk', cache=True)
    def build_networkwalk():
        nodeswalk = pd.read_csv(d + 'bayarea_walk_nodes.csv') \
            .set_index('osmid')
        edgeswalk = pd.read_csv(d + 'bayarea_walk_edges.csv')
        netwalk = pdna.Network(nodeswalk.x, nodeswalk.y, edgeswalk.u, \
                               edgeswalk.v, edgeswalk[['length']], twoway=True)
        netwalk.precompute(2500)
        return netwalk

    parcels = orca.get_table('parcels').to_frame(columns=['x', 'y'])
    idswalk_parcel = orca.get_injectable('netwalk').get_node_ids(parcels.x, parcels.y)
    orca.add_column('parcels', 'node_id_walk', idswalk_parcel, cache=False)
    orca.broadcast('nodeswalk', 'parcels', cast_index=True, onto_on='node_id_walk')

#     rentals = orca.get_table('rentals').to_frame(columns=['longitude', 'latitude'])
#     idswalk_rentals = orca.get_injectable('netwalk').get_node_ids(rentals.longitude, rentals.latitude)
#     orca.add_column('rentals', 'node_id_walk', idswalk_rentals, cache=False)
#     orca.broadcast('nodeswalk', 'rentals', cast_index=True, onto_on='node_id_walk')

    @orca.column('buildings', 'node_id_walk')
    def node_id(parcels, buildings):
        return misc.reindex(parcels.node_id_walk, buildings.parcel_id)

    @orca.column('units', 'node_id_walk')
    def node_id(buildings, units):
        return misc.reindex(buildings.node_id_walk, units.building_id)

    @orca.column('households', 'node_id_walk')
    def node_id(units, households):
        return misc.reindex(units.node_id_walk, households.unit_id)

    @orca.column('persons', 'node_id_walk')
    def node_id(households, persons):
        return misc.reindex(households.node_id_walk, persons.household_id)

    @orca.column('jobs', 'node_id_walk')
    def node_id(buildings, jobs):
        return misc.reindex(buildings.node_id_walk, jobs.building_id)

    # While we're at it, we can use these node_id columns to define direct broadcasts
    # between the nodes table and lower-level ones, which speeds up merging

    orca.broadcast('nodeswalk', 'units', cast_index=True, onto_on='node_id_walk')
@orca.step()
def network_aggregations_walk_test(netwalk):
    """
    This will be turned into a network aggregation template.
    """

    nodeswalk = networks.from_yaml(netwalk, 'network_aggregations_walk_test.yaml')
    nodeswalk = nodeswalk.fillna(0)
    print(nodeswalk.describe())
    orca.add_table('nodeswalk', nodeswalk)
orca.run(["initialize_network_walk"])
orca.run(["network_aggregations_walk_test"])<jupyter_output>Running step 'network_aggregations_walk_test'
Computing accessibility variables
Computing pop_500_walk
Removed 189769 rows because they contain missing values
Computing sqft_unit_500_walk
Computing singles_500_walk
Removed 52084 rows because they contain missing values
Computing elderly_hh_500_walk
Removed 34553 rows because they contain missing values
Computing children_500_walk
Removed 189769 rows because they contain missing values
        pop_500_walk  sqft_unit_500_walk  singles_500_walk  \
count  619889.000000       619889.000000     619889.000000   
mean      561.022034         1239.638062         67.199196   
std       908.448853         1012.501038        212.378616   
min         0.000000            0.000000          0.000000   
25%        11.000000          449.750000          0.000000   
50%       224.000000         1327.473999         13.000000   
75%       770.000000         1760.372437         61.000000   
max     20222.000000        30000.000000       8822.000000   

  [...]<jupyter_text>Comfirm graph is undirected<jupyter_code>edgeswalk = pd.read_csv(d + 'bayarea_walk_edges.csv')
edgeswalk.head(3)
edgeswalk.loc[(edgeswalk.u==25457938) & (edgeswalk.v==2399878404)]<jupyter_output><empty_output><jupyter_text>Yes, seems undirected  What are the "missing values" that were removed?189769 rows<jupyter_code>hh = orca.get_table('households').to_frame()
sum(hh.node_id_walk.isnull())<jupyter_output><empty_output><jupyter_text>This ultimately comes from the node id on the parcels table, via buildings and units..<jupyter_code>p = orca.get_table('parcels').to_frame()
sum(p.node_id_walk.isnull())
b = orca.get_table('buildings').to_frame()
sum(b.node_id_walk.isnull())
u = orca.get_table('units').to_frame()
sum(u.node_id_walk.isnull())<jupyter_output><empty_output><jupyter_text>So it's a problem linking the households to units, maybe?<jupyter_code>sum(hh.unit_id.isnull())
hh.unit_id.describe()
sum(hh.unit_id==-1)
sum(hh.unit_id==-1)/len(hh)<jupyter_output><empty_output><jupyter_text>Why didn't this show up on Sam B's machine?Don't know! I don't see any discrepancies in the code. Best guess is that maybe the message just wasn't printed?  What about the variation in number of rows dropped?````Computing pop_500_walkRemoved 189769 rows because they contain missing valuesComputing sqft_unit_500_walkComputing singles_500_walkRemoved 52084 rows because they contain missing valuesComputing elderly_hh_500_walkRemoved 34553 rows because they contain missing valuesComputing children_500_walkRemoved 189769 rows because they contain missing values````<jupyter_code>len(hh.loc[hh.node_id_walk.isnull()])
len(b.loc[b.node_id_walk.isnull()])
len(hh.loc[(hh.persons==1) & (hh.node_id_walk.isnull())])
len(hh.loc[(hh.age_of_head>65) & (hh.node_id_walk.isnull())])
len(hh.loc[hh.node_id_walk.isnull()])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12158182583087926, 'fraction_numerical': 0.05230682933669892, 'mean_word_length': 4.306547619047619, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '296896', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2738, 'n_tokens_neox': 2445, 'n_words': 487, 'path': 'summer-2018-model/notebooks-sam/More-aggregation-troubleshooting.ipynb', 'repo_name': 'lmwang9527/ual_model_workspace', 'size': '7135'}