<jupyter_start><jupyter_text>This section is for parsing new data files------------------------------------------<jupyter_code>df = pd.read_csv("sequence_jul_16.csv")

map_prov_names(df, 'region')
df['date'] = f"'{'2021-07-16'}'"
print(df)
df.to_csv('sequence_mapped_jul_16.csv', index = False)
voc = pd.read_csv('voc_jul_26.csv')
map_voc_names(voc, 'voc')
map_prov_names(voc, 'region')
voc['date'] = f"'{'2021-07-26'}'"
print(voc.head())
voc.to_csv('voc_case_mapped_jul_26.csv', index = False)

csv_url = "https://health-infobase.canada.ca/src/data/covidLive/covid19.csv"
df = pd.read_csv(csv_url)

#parse government data so it is usable
df['date'] = pd.to_datetime(df['date'], dayfirst = True)
df = df.drop(['prnameFR', 'pruid'], axis = 1)
#if provinces don't report active cases, assume they have 0, so the data will plot
df['numactive'].fillna(0.0, inplace = True)

provnames = df.prname.unique()
provnames = list(provnames)
provnames.pop()
#function to make province-wise dataset
def make_dataset(pnames):
    
    by_prov = pd.DataFrame()

    # Iterate through all the provinces
    for i in pnames:

        # Subset to the province
        sub = df[df['prname'] == i]

        # Add to the overall dataframe
        by_prov = by_prov.append(sub)
    
    return by_prov


#must pivot datasets for linear plot data so prov names are columns
def format_dataset(df, value):
    df_pivot = pd.pivot_table(df,
    columns = 'prname',
    index = 'date',
    values = value)
    
    df_pivot = df_pivot.rename_axis(None)

    return df_pivot


dataset = format_dataset(make_dataset(provnames), 'numtotal_last7')
dataset = dataset.reset_index()
#print(dataset.head())
#dataset.rename_axis('prname')

monday = '2021-07-26'
mon_date = pd.to_datetime(monday, dayfirst = True)
mon_lst = [mon_date]

mon_data = dataset.loc[dataset['index'].isin(mon_lst)]
mon_data.rename(columns={'index':'date'}, inplace = True)
print(mon_data)
cases_formatted = pd.melt(mon_data, id_vars = ['date']).sort_values(by = ['date','prname'])
cases_formatted.rename(columns = {'prname':'region', 'value':'week_cases'}, inplace = True)

map_prov_names(cases_formatted, 'region')
cases_formatted['date'] = f"'{monday}'"
cases_formatted['week_cases'] = cases_formatted['week_cases'].astype(int)

print(cases_formatted)
cases_formatted.to_csv('cases_formatted_jul26.csv', index = False)<jupyter_output>date  region  week_cases
0   "2021-07-12 00:00:00"       1         265
1   "2021-07-12 00:00:00"       2         364
2   "2021-07-12 00:00:00"      14        3467
3   "2021-07-12 00:00:00"       3         438
4   "2021-07-12 00:00:00"       4           0
5   "2021-07-12 00:00:00"       5           2
6   "2021-07-12 00:00:00"       6           0
7   "2021-07-12 00:00:00"       7          15
8   "2021-07-12 00:00:00"       8           0
9   "2021-07-12 00:00:00"       9        1290
10  "2021-07-12 00:00:00"      10           1
11  "2021-07-12 00:00:00"      11         720
12  "2021-07-12 00:00:00"      12         299
13  "2021-07-12 00:00:00"      13          73
{'fraction_non_alphanumeric': 0.1571334648257725, 'fraction_numerical': 0.10059171597633136, 'mean_word_length': 3.112162162162162, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '461589', 'license': 'Apache-2.0', 'n_tokens_mistral': 1328, 'n_tokens_neox': 1153, 'n_words': 246, 'path': 'main.ipynb', 'repo_name': 'eringill/canadian_covid_sequencing', 'size': '3042'}