<jupyter_start><jupyter_text>Get vaccine coverage by ZIP Codes data from CDPH<jupyter_code>%load_ext lab_black
import pandas as pd
import datetime as dt
import json
import os
import glob
import urllib.request
pd.options.display.max_columns = 50
pd.options.display.max_rows = 1000
pd.set_option("display.max_colwidth", None)
today = dt.datetime.today().strftime("%Y-%m-%d")<jupyter_output><empty_output><jupyter_text>Get the metadata from the API<jupyter_code>endpoint = "https://data.chhs.ca.gov/api/3/action/package_show?id=covid-19-vaccine-progress-dashboard-data-by-zip-code"
jsonurl = urllib.request.urlopen(endpoint)
text = json.loads(jsonurl.read())<jupyter_output><empty_output><jupyter_text>Get the object with the max date<jupyter_code>data = text["result"]["resources"]<jupyter_output><empty_output><jupyter_text>Loop through the dict to get the file names that contain dates<jupyter_code># dates = []
# for obj in data:
#     slug = obj["name"]
#     # find valid dates in the name field and append them to a list
#     try:
#         date = pd.to_datetime(
#             slug.replace("COVID-19 Vaccines by ZIP Code ", "")
#         ).strftime("%Y-%m-%d")
#         dates.append(date)
#     except:
#         pass<jupyter_output><empty_output><jupyter_text>Pick out the latest date from that list<jupyter_code># latest_date_str = max(d for d in dates)<jupyter_output><empty_output><jupyter_text>Loop through the objects again and match the latest one with the latest date<jupyter_code># for obj in data:
#     slug = obj["name"]
#     try:
#         date = pd.to_datetime(
#             slug.replace("COVID-19 Vaccines by ZIP Code ", "")
#         ).strftime("%Y-%m-%d")
#         if date == latest_date_str:
#             latest_obj = obj
#         else:
#             pass
#     except:
#         pass
# latest_date = pd.to_datetime(latest_obj["created"]).strftime("%Y-%m-%d")<jupyter_output><empty_output><jupyter_text>Read that latest file into a dataframe to limit redacted rows ~~and save it in raw~~<jupyter_code>url = "https://data.chhs.ca.gov/dataset/ead44d40-fd63-4f9f-950a-3b0111074de8/resource/ec32eece-7474-4488-87f0-6e91cb577458/download/covid19vaccinesbyzipcode_test.csv"
df = pd.read_csv(url)
df = df[df["vem_source"] != "No VEM Assigned"]
df = df.rename(columns={"as_of_date": "date"})
# df.to_csv("data/raw/" + latest_date + ".csv", index=False)<jupyter_output><empty_output><jupyter_text>---  Concatenate all the weekly updates  Get all files and assign a date to each table based on the file name<jupyter_code># path = ""
# files = glob.glob(os.path.join(path, "data/raw/*.csv"))
# file_df = (
#     pd.read_csv(f, low_memory=False).assign(date=os.path.basename(f)) for f in files
# )<jupyter_output><empty_output><jupyter_text>Concatenate them into one timeseries and clean update date field<jupyter_code># concat_df = pd.concat(
#     file_df,
#     ignore_index=True,
# )
# concat_df["date"] = pd.to_datetime(
#     concat_df["date"].str.replace(".csv", "", regex=False)
# )
# concat_df[concat_df["Zip Code Tabulation Area"] == 91320]<jupyter_output><empty_output><jupyter_text>---  Export  All updates<jupyter_code>df.to_csv("data/timeseries.csv", index=False)<jupyter_output><empty_output><jupyter_text>Latest update<jupyter_code>df[df["date"] == df["date"].max()].to_csv("data/latest.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14277233600474917, 'fraction_numerical': 0.01840308696942713, 'mean_word_length': 4.750853242320819, 'pattern_counts': {'":': 1, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '22064', 'license': 'MIT', 'n_tokens_mistral': 1265, 'n_tokens_neox': 1217, 'n_words': 237, 'path': 'vaccine-zip-codes/scrape.ipynb', 'repo_name': 'datadesk/california-coronavirus-scrapers', 'size': '3369'}