<jupyter_start><jupyter_text>Since we only have access to data in Colorado, we'll use this notebook to do what `generate_data.py` was supposed to. However, this last file may come in handy if more data will be released.<jupyter_code>import os
import re
import pandas as pd
os.getcwd()
re.search(pattern=r'colorado_county_covid_(\w*)_results\.csv', string='colorado_county_covid_april_results.csv').group(1)
list_files = [
    'colorado_county_covid_03-2020_results.csv',
    'colorado_county_covid_04-2020_results.csv',
    'colorado_county_covid_05-2020_results.csv',
    'colorado_county_covid_06-2020_results.csv',
    'colorado_county_covid_07-2020_results.csv'
]

list_paths = [os.path.join('../../data/question_answering_newspaper3k', filename) for filename in list_files]
list_paths
# Read Colorado DataFrames:
l_df = []
for filepath in list_paths:
    date = re.search(
        pattern=r'colorado_county_covid_(.*)_results\.csv', 
        string=filepath).group(1)
    
    l_df.append(pd.read_csv(filepath).assign(date=date) )

df = pd.concat(l_df)

df.head()
# Preprocess:
df['publish_datetime'] = pd.to_datetime(df['publish_date'])
df['publish_date'] = df['publish_datetime'].dt.date

df.head()
len(df)
# Only keep rows where text is not NaN:
df = df[df['text'].notna()]
len(df)
df.head()
# Add an extra column for future compatibility
df['county'] = 'colorado'
# Export test file:
df_test = df.iloc[0:2]
df_test.to_csv('../../data/question_answering_newspaper3k/df_all_counties_test.csv')
# Export:
df.to_csv('../../data/question_answering_newspaper3k/df_all_counties.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13498452012383902, 'fraction_numerical': 0.022910216718266253, 'mean_word_length': 5.464, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '118982', 'license': 'MIT', 'n_tokens_mistral': 626, 'n_tokens_neox': 597, 'n_words': 110, 'path': 'Part3-Article-NLP/haystack/scripts/generate_data_colorado.ipynb', 'repo_name': 'tonywu71/Excess-Mortality-Covid', 'size': '1615'}