<jupyter_start><jupyter_text>Extract date<jupyter_code>content = requests.get(url).content
soup =BeautifulSoup(content, 'lxml')
data = soup.findAll(text=re.compile("Updated .+ 202\d"))
data
m = re.search("Updated (\w+ \d+, 202\d)", data[0])
data_date_str = m.group(1)
data_date = datetime.strptime(data_date_str, '%B %d, %Y').strftime('%Y-%m-%d')
data_date<jupyter_output><empty_output><jupyter_text>Extract data and add into Dataframe<jupyter_code>dfs = pd.read_html(url)
if len(dfs) == 0 :
    raise Exception('There is no dataframes detected')
df = dfs[0]
df
col0 = df.columns[0]
df = df.set_index(col0).T
df


# remove non alphanumeric characters from the columns
# also lowercase them
df.columns = df.columns.map(lambda x: re.sub('\W+', '', str.lower(x)))

# add column date to the front
df.insert(0, 'date', data_date)

df<jupyter_output><empty_output><jupyter_text>Save to file<jupyter_code>from pathlib import Path
csv_file = 'data/cdc-us-cases.csv'

if Path(csv_file).exists():
    # read out the old dat
    old_df = pd.read_csv(csv_file)
    df = df.append(old_df)
df.drop_duplicates(subset=['date'], inplace=True)
df.sort_values(by=['date'], inplace=True)
df
df.to_csv(csv_file, index=False)<jupyter_output><empty_output><jupyter_text>Plot graph<jupyter_code>import seaborn as sns
import matplotlib.pyplot as plt
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()

png_file = 'images/cdc-us-cases.png'

# convert to pd.datetime
df['datetime_idx'] = pd.to_datetime(df['date'], format="%Y-%m-%d")

df.set_index('datetime_idx', inplace=True)
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15024630541871922, 'fraction_numerical': 0.008620689655172414, 'mean_word_length': 6.523148148148148, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '265905', 'license': 'Apache-2.0', 'n_tokens_mistral': 643, 'n_tokens_neox': 633, 'n_words': 121, 'path': 'notebooks-scrappers/.ipynb_checkpoints/CDC-cases-in-US-checkpoint.ipynb', 'repo_name': 'OwenAnalytics/coronavirus-stats', 'size': '1624'}