<jupyter_start><jupyter_text>Notebook for getting todays (or a defined days) Corona data from SSI https://covid19.ssi.dk/overvagningsdata/download-fil-med-overvaagningdataData are stored in subfolder : dataDate of current dataset is stored in file: data_date.dat<jupyter_code>import pandas as pd
from pathlib import Path
import requests
import zipfile
from bs4 import BeautifulSoup
from urllib.request import Request, urlopen
from datetime import date, timedelta
import pickle
#Define date of today used format
d=date.today()
date_str=d.strftime("%d%m%Y")

#Define a manual date (e.g. if running on a date where no new data is uploaded)
#date_str="16122020"
# Find link to latest file from the web page
# assumes stabel initial part of filename and webpage location
#req = Request("https://www.ssi.dk/sygdomme-beredskab-og-forskning/sygdomsovervaagning/c/covid19-overvaagning")
req = Request("https://covid19.ssi.dk/overvagningsdata/download-fil-med-overvaagningdata")
html_page = urlopen(req)
soup = BeautifulSoup(html_page, "html")
                     
for link in soup.findAll('a'):
    ref=link.get('href')
    if isinstance(ref, str) and "data-epidemiologiske-rapport" in ref.lower() and date_str in ref.lower(): # NB compare in lower case
        print("url for zip file: " + link.get('href'))
        url=link.get('href')
        #date_str = url[50:58]
    if isinstance(ref, str) and "data-epidemiologisk-rapport-" in ref.lower() and date_str in ref.lower(): # NB compare in lower case
        print("url for zip file: " + link.get('href'))
        url=link.get('href')
        #date_str = url[49:57]

        


#Defin todays URL manualy if above code breaks
#url = 'https://files.ssi.dk/Data-epidemiologisk-rapport-09112020-1puf'
#url='https://files.ssi.dk/covid19/overvagning/data/data-epidemiologiske-rapport-12112020-ql82'


# Direct download data and unpacking
r = requests.get(url, allow_redirects=True)
open('data.zip', 'wb').write(r.content)

with zipfile.ZipFile("data.zip","r") as zip_ref:
    zip_ref.extractall("data/")


# save date_str 
f = open("data_date.dat", 'wb')
pickle.dump(date_str, f)
f.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12378303198887343, 'fraction_numerical': 0.020398701900788133, 'mean_word_length': 4.395, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 5, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '528374', 'license': 'MIT', 'n_tokens_mistral': 787, 'n_tokens_neox': 742, 'n_words': 192, 'path': 'SSI_get_data.ipynb', 'repo_name': 'BoJakobsen/Corona-plot', 'size': '2157'}