<jupyter_start><jupyter_text>Combine district CSVs into a timeseries file<jupyter_code>%load_ext lab_black
import pandas as pd
import glob
import os
import json<jupyter_output><empty_output><jupyter_text>Get all the district json files<jupyter_code>files = glob.glob(os.path.join(path, "data/districts-2021*.csv"))
charter_files = glob.glob(os.path.join(path, "data/charters-2021*.csv"))<jupyter_output><empty_output><jupyter_text>def concat_output(name):    path = ""     get files    files = glob.glob(os.path.join(path, f"data/{name}-2021*.csv"))     create date field    file_df = (        pd.read_csv(f, low_memory=False).assign(date=os.path.basename(f)) for f in files    )     concat    concatenated_df = pd.concat(        file_df,        ignore_index=True,    )     create date column    concatenated_df["date"] = (        concatenated_df["date"]        .str.replace(f"{name}-", "", regex=False)        .str.replace(".csv", "", regex=False)    )     export    concatenated_df.to_csv(f"data/{name}-timeseries.csv", index=False) concat_output("districts") concat_output("charters") concat_output("private")<jupyter_code>### Read them and create a date field
file_df = (
    pd.read_csv(f, low_memory=False).assign(date=os.path.basename(f)) for f in files
)
charter_df = (
    pd.read_csv(f, low_memory=False).assign(date=os.path.basename(f))
    for f in charter_files
)
### Concatenate all the files
concatenated_df = pd.concat(
    file_df,
    ignore_index=True,
)
concatenated_charters_df = pd.concat(
    charter_df,
    ignore_index=True,
)
### Clean up the date field and column names
concatenated_df["date"] = (
    concatenated_df["date"]
    .str.replace("districts-", "", regex=False)
    .str.replace(".csv", "", regex=False)
)
concatenated_df.columns = (
    concatenated_df.columns.str.replace(".", "", regex=False)
    .str.lower()
    .str.replace("attributes", "", regex=False)
)
concatenated_charters_df["date"] = (
    concatenated_charters_df["date"]
    .str.replace("charters-", "", regex=False)
    .str.replace(".csv", "", regex=False)
)
concatenated_charters_df.columns = (
    concatenated_charters_df.columns.str.replace(".", "", regex=False)
    .str.lower()
    .str.replace("attributes", "", regex=False)
)
### Cleaning up the unix dates
district_dates = [
    "creationdate",
    "editdate",
    "nearterm_reopening_date",
    "elem_reopen_dt",
    "sec_reopen_dt",
]
charter_dates = [
    "creationdate",
    "editdate",
    "reopening_date",
]
def parse_dates(df, dates):
    for d in dates:
        df[f"{d}"] = pd.to_datetime(df[f"{d}"], unit="ms")
        df[f"{d}_clean"] = df[f"{d}"].dt.date
    return df
df = parse_dates(concatenated_df, district_dates)
charters_final = parse_dates(concatenated_charters_df, charter_dates)
---
## Export
df.to_csv("data/districts-all-timeseries.csv", index=False)
charters_final.to_csv("data/charters-all-timeseries.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1565896739130435, 'fraction_numerical': 0.004076086956521739, 'mean_word_length': 4.453703703703703, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '508406', 'license': 'MIT', 'n_tokens_mistral': 1073, 'n_tokens_neox': 1050, 'n_words': 170, 'path': 'school-reopenings/concatenate_districts.ipynb', 'repo_name': 'datadesk/california-coronavirus-scrapers', 'size': '2944'}