<jupyter_start><jupyter_text>Attach block ID and nearest network node to each parcel<jupyter_code>import geopandas as gpd
import numpy as np
import pandas as pd
from scipy.spatial import cKDTree
from sklearn.neighbors import BallTree

crs = {'init':'epsg:4326'}<jupyter_output><empty_output><jupyter_text>Load the input data<jupyter_code>%%time
# load parcels shapefile, drop any that lack geometry, and project to 4326
parcels = gpd.read_file('data/Parcels/')
parcels = parcels.dropna(subset=['geometry'])
parcels = parcels.to_crs(crs)
assert parcels.index.is_unique
print(len(parcels))
# load parcels-to-blocks lookup table
parcels_blocks = pd.read_csv('data/parcels_joined_blocks.csv', dtype={'block_geoid':str})
parcels_blocks = parcels_blocks.set_index('parcel_id')
assert parcels_blocks.index.is_unique
print(len(parcels_blocks))
# load the network nodes
nodes = pd.read_csv('data/network/bay_area_full_strongly_nodes.csv')
nodes = nodes.set_index('osmid')
assert nodes.index.is_unique
print(len(nodes))<jupyter_output>223328<jupyter_text>Add census block ID to each parcel<jupyter_code># convert integer columns to int
to_int = ['PARCEL_ID', 'DEVELOPMEN', 'COUNTY_ID', 'ZONE_ID', 'PROPORTION', 'TAX_EXEMPT', 'ID']
for col in to_int:
    parcels[col] = parcels[col].astype(int)

# drop unused columns and set index
parcels = parcels.drop(columns=['CENTROID', 'X', 'Y'])
parcels = parcels.set_index('PARCEL_ID')
# merge block IDs into parcels df
parcels = pd.merge(left=parcels, right=parcels_blocks, how='left', left_index=True, right_index=True)<jupyter_output><empty_output><jupyter_text>Find network node nearest to each parcel<jupyter_code># calculate parcel centroids
parcels['centroid'] = parcels.centroid
# extract x and y from centroids
parcels = parcels.set_geometry('centroid')
parcels['x'] = parcels.geometry.x
parcels['y'] = parcels.geometry.y
parcels = parcels.set_geometry('geometry')<jupyter_output><empty_output><jupyter_text>scipy euclidean methodfaster but less accurate with unprojected coordinates<jupyter_code># build the tree for nearest-neighbor search
tree = cKDTree(data=nodes[['x', 'y']], compact_nodes=True, balanced_tree=True)
%%time
# query the tree for node nearest to each parcel centroid
dist, idx = tree.query(parcels[['x', 'y']], k=1)
parcels['node'] = nodes.iloc[idx].index<jupyter_output>Wall time: 2.88 s<jupyter_text>scikit-learn haversine methodslower but more accurate with unprojected coordinates<jupyter_code># haversine requires data in form of [lat, lng] and inputs/outputs in units of radians
nodes_rad = np.deg2rad(nodes[['y', 'x']])
parcels_rad = np.deg2rad(parcels[['y', 'x']])
# build the tree for nearest-neighbor search
tree = BallTree(nodes_rad, metric='haversine')
%%time
# query the tree for node nearest to each parcel centroid
idx = tree.query(parcels_rad, k=1, return_distance=False)
parcels['node'] = nodes.iloc[idx[:,0]].index<jupyter_output>Wall time: 10min<jupyter_text>Save parcels data to disk<jupyter_code># as simple csv
df_save = parcels.reset_index()[['PARCEL_ID', 'block_geoid', 'node']]
df_save = df_save.rename(columns={'PARCEL_ID':'parcel_id',
                                  'block_geoid':'block_id',
                                  'node':'node_id'})
df_save.to_csv('data/parcels_blocks_nodes.csv', index=False, encoding='utf-8')
%%time
# as full shapefile
parcels = parcels.reset_index()
parcels = parcels.drop(columns=['centroid', 'x', 'y'])
parcels['node'] = parcels['node'].astype(str) # to handle ints bigger than 32-bits
parcels.to_file('data/parcels_blocks_nodes')<jupyter_output>Wall time: 27min 32s
{'fraction_non_alphanumeric': 0.12896935933147632, 'fraction_numerical': 0.008635097493036212, 'mean_word_length': 6.739224137931035, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '228935', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1332, 'n_tokens_neox': 1256, 'n_words': 288, 'path': '03-parcels-attach-blocks-network-nodes.ipynb', 'repo_name': 'gboeing/parcels-attach-blocks-nodes', 'size': '3590'}