<jupyter_start><jupyter_text>Configuration_Initial steps to get the notebook ready to play nice with our repository. Do not delete this section._ Code formatting with [black](https://pypi.org/project/nb-black/).<jupyter_code>%load_ext lab_black
import os
import pytz
import glob
import pathlib

this_dir = pathlib.Path(os.path.abspath(""))
data_dir = this_dir / "data"
import requests
import pandas as pd
from bs4 import BeautifulSoup
import re
import json
from datetime import datetime<jupyter_output>/Users/slee/.local/share/virtualenvs/california-coronavirus-scrapers-QTClc1WQ/lib/python3.8/site-packages/pandas/compat/__init__.py:124: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.
  warnings.warn(msg)<jupyter_text>Download Retrieve the page<jupyter_code>#url = "https://www.placer.ca.gov/DocumentCenter/View/46267/dashboard"
url = "https://itwebservices.placer.ca.gov/coviddashboard"
page = requests.get(url)<jupyter_output><empty_output><jupyter_text>Parse<jupyter_code>soup = BeautifulSoup(page.content, "html.parser")<jupyter_output><empty_output><jupyter_text>Find script tag with the Infogram data<jupyter_code>for elem in soup(text=re.compile(r"Foresthill")):
    script_content = elem
data = json.loads(script_content)
zip_cities_list = data["x"]["calls"][1]["args"][-3]
dict_list = []
TAG_RE = re.compile(r"<[^>]+>")
for z in zip_cities_list:
    split_zips = z.split("<br/>")
    area = TAG_RE.sub("", str(split_zips[0]))
    zip_code, city = area.split(" - ")
    clean_cases = re.sub("Number of cases: ".lower(), "", str(split_zips[1]).lower())
    d = dict(area=area, city=city, zip_code=zip_code, confirmed_cases=clean_cases)
    dict_list.append(d)
df = pd.DataFrame(dict_list)<jupyter_output><empty_output><jupyter_text>Get timestamp<jupyter_code>time_div = soup.find("div", id="dashboard-data-last-updated")
timestamp = time_div.find("span", class_="value-output").get_text()
latest_date = pd.to_datetime(timestamp).date()
df["county_date"] = latest_date
df.insert(0, "county", "Placer")<jupyter_output><empty_output><jupyter_text>Clean up<jupyter_code>df["area"] = df["area"].str.replace(" - ", ": ")
df = df[["county", "area", "confirmed_cases", "county_date", "zip_code"]].rename(
    columns={"zip_code": "zip"}
)<jupyter_output><empty_output><jupyter_text>Vet<jupyter_code>try:
    assert not len(df) < 32
except AssertionError:
    raise AssertionError("Placer County's scraper is missing rows")
try:
    assert not len(df) > 32
except AssertionError:
    raise AssertionError("Placer County's scraper has more rows than before")<jupyter_output><empty_output><jupyter_text>Export Set date<jupyter_code>tz = pytz.timezone("America/Los_Angeles")
today = datetime.now(tz).date()
slug = "placer"
df
df.to_csv(data_dir / slug / f"{today}.csv", index=False)<jupyter_output><empty_output><jupyter_text>Combine<jupyter_code>csv_list = [
    i
    for i in glob.glob(str(data_dir / slug / "*.csv"))
    if not str(i).endswith("timeseries.csv")
]
df_list = []
for csv in csv_list:
    if "manual" in csv:
        df = pd.read_csv(csv, parse_dates=["date"])
    else:
        file_date = csv.split("/")[-1].replace(".csv", "")
        df = pd.read_csv(csv, parse_dates=["county_date"])
        df["date"] = file_date
    df_list.append(df)
df = pd.concat(df_list).sort_values(["date", "area"])
df.to_csv(data_dir / slug / "timeseries.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15105653912050257, 'fraction_numerical': 0.006282124500285551, 'mean_word_length': 5.9229249011857705, 'pattern_counts': {'":': 2, '<': 39, '<?xml version=': 0, '>': 40, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '499620', 'license': 'MIT', 'n_tokens_mistral': 1293, 'n_tokens_neox': 1267, 'n_words': 234, 'path': 'places/placer.ipynb', 'repo_name': 'datadesk/california-coronavirus-scrapers', 'size': '3502'}