<jupyter_start><jupyter_text>Code formatting with [black](https://pypi.org/project/nb-black/).<jupyter_code>%load_ext lab_black<jupyter_output>The lab_black extension is already loaded. To reload it, use:
  %reload_ext lab_black<jupyter_text>Add our `utils` directory to the system's `$PATH` so we can import Python files from sibling directories.<jupyter_code>import os
import sys
import pathlib
this_dir = pathlib.Path(os.path.abspath(""))
data_dir = this_dir / "data"
import requests
import pandas as pd
from bs4 import BeautifulSoup<jupyter_output><empty_output><jupyter_text>Retrieve the page<jupyter_code>url = "https://www.ice.gov/coronavirus#tab1"
page = requests.get(url)<jupyter_output><empty_output><jupyter_text>Parse it.<jupyter_code>soup = BeautifulSoup(page.content, "html.parser")<jupyter_output><empty_output><jupyter_text>Get the timestamp.<jupyter_code>content = soup.find("caption")
date_string = content.find("div", {"style": "color: #676767; padding:10px;"}).text
date = pd.to_datetime(date_string.split()[-1])<jupyter_output><empty_output><jupyter_text>Focus in on the tables.<jupyter_code>table_list = soup.find_all("table")<jupyter_output><empty_output><jupyter_text>Find the right table.<jupyter_code>tbody = table_list[1].tbody
all_info = tbody.find_all("tr")
def get_text(x):
    return x.text
def get_vals(a):
    return [get_text(a[x]) for x in range(0, len(a))]
ice_list = []
for x in range(0, len(all_info)):
    vals = get_vals(all_info[x].find_all("td"))
    ice_list.append(vals)
ice_df = pd.DataFrame(ice_list)
ice_df = ice_df.rename(
    columns={
        0: "custody_aor_facility",
        1: "confirmed_isolation_or_monitoring",
        2: "deaths",
        3: "total_confirmed",
    }
)<jupyter_output><empty_output><jupyter_text>Add the last updated date.<jupyter_code>ice_df["date"] = pd.to_datetime(date)<jupyter_output><empty_output><jupyter_text>Drop header rows.<jupyter_code>ice_df.dropna(subset=["deaths"], inplace=True)<jupyter_output><empty_output><jupyter_text>Read in exisiting raw data to append new entries<jupyter_code>existing = pd.read_csv(data_dir / "timeseries.csv", parse_dates=["date"])
if ice_df["date"].unique() in existing["date"].unique():
    pass
else:
    existing = pd.concat([existing, ice_df])<jupyter_output><empty_output><jupyter_text>Write out combined - or not depending on statement above<jupyter_code>existing.to_csv(data_dir / "timeseries.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13752535496957405, 'fraction_numerical': 0.007302231237322515, 'mean_word_length': 7.22, 'pattern_counts': {'":': 1, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '642003', 'license': 'MIT', 'n_tokens_mistral': 921, 'n_tokens_neox': 900, 'n_words': 156, 'path': 'ice-detainees/scrape.ipynb', 'repo_name': 'datadesk/california-coronavirus-scrapers', 'size': '2465'}