<jupyter_start><jupyter_text>East Bay Transit data project A project exploring East Bay public transit using publicly available data from AC Transit, BART, and the census. GoalThe goal of this notebook is to create the Bay Area census tracts shapefile ContextUsing IPUMS NHGIS we got some preliminary interesting statistics for census tracts. These, however, span the whole US. We also go a shapefile with the 2018 census tracts for the whole US. Since our analysis only covers the Bay Area we need to create a subset of these datasets.<jupyter_code>import pandas as pd
import geopandas as gpd
from zipfile import ZipFile
from pathlib import Path
from tools import tree
from datetime import datetime as dt
today = dt.today().strftime("%d-%b-%y")

today
RAW_DATA = Path("../data/raw/")
INTERIM_DATA = Path("../data/interim/")
PROCESSED_DATA = Path("../data/processed/")
EXTERNAL_DATA = Path("../data/external/")
tree(EXTERNAL_DATA)
ZipFile(EXTERNAL_DATA / 'nhgis0008_csv.zip').extractall(INTERIM_DATA)
ZipFile(EXTERNAL_DATA / 'nhgis0008_shape.zip').extractall(INTERIM_DATA)
tree(INTERIM_DATA)
data = pd.read_csv(INTERIM_DATA / 'nhgis0008_csv' / 'nhgis0008_ds239_20185_2018_tract.csv', encoding = 'latin1')
data.head().T
zip_shape_file = "zip:///" + str(INTERIM_DATA.absolute().resolve() / 'nhgis0008_shape' / 'nhgis0008_shapefile_tl2018_us_tract_2018.zip')
shapes = gpd.read_file(zip_shape_file)
shapes.shape
shapes.head()
mask_california = shapes['STATEFP'] == "06"
shapes[mask_california].shape<jupyter_output><empty_output><jupyter_text>"Officially" the Bay Area is composed by 9 counties:|     County    |  FIPS  | |:-------------:|:------:||    Alameda    |   001  ||  Contra Costa |   013  ||     Marin     |   041  ||     Napa      |   055  || San Francisco |   075  ||   San Mateo   |   081  ||  Santa Clara  |   085  ||    Solano     |   095  ||    Sonoma     |   097  |<jupyter_code>bay_area_FIPS = ['001', '013', '041', '055', '075', '081', '085', '095', '097']
ac_transit_counties = ['001', '013']
mask_bayarea = shapes['COUNTYFP'].isin(bay_area_FIPS)
mask_actransit = shapes['COUNTYFP'].isin(ac_transit_counties)
shapes[mask_california & mask_bayarea].shape
bay_area_tracts = shapes[mask_california & mask_bayarea].copy()
bay_area_tracts.plot();
shapes[mask_california & mask_actransit].plot();
bay_area_tracts.crs
# to match stop/routes crs
bay_area_tracts = bay_area_tracts.to_crs(epsg = 4326)
bay_area_tracts.crs
bay_area_tracts.head()
bay_area_tracts.to_file(PROCESSED_DATA / 'bay_area_tracts.shp', index=False)
tree(PROCESSED_DATA)
file_paths = PROCESSED_DATA.glob("bay_area_tracts.*")
# writing files to a zipfile 
with ZipFile(PROCESSED_DATA / 'bay_area_2018_tracts.zip','w') as bay_area_zip: 
    # writing each file one by one 
    for file in file_paths: 
        bay_area_zip.write(file) 
ZipFile(PROCESSED_DATA / 'bay_area_2018_tracts.zip').filelist<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12159834653806408, 'fraction_numerical': 0.04271443334481571, 'mean_word_length': 4.866666666666666, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '71773', 'license': 'RSA-MD', 'n_tokens_mistral': 1146, 'n_tokens_neox': 1005, 'n_words': 244, 'path': 'notebooks/00_DataPrep.ipynb', 'repo_name': 'vivaladataviz/east-bay-transit', 'size': '2903'}