<jupyter_start><jupyter_text>Retrieve COVID data from ECDC<jupyter_code>covid_url = "https://opendata.ecdc.europa.eu/covid19/casedistribution/json/"
ssl._create_default_https_context = ssl._create_unverified_context
covid_json_unformated = urllib.request.urlopen(covid_url).read().decode("utf-8")
covid_json = json.loads(covid_json_unformated)

cdf = pd.DataFrame(covid_json['records'])
date = pd.to_datetime(cdf["year"] + "-" + cdf["month"] + "-" + cdf["day"]).dt.date
date = date.to_frame(name = "date_reported")
cdf = date.join(cdf.drop(columns = ["dateRep", "day", "month", "year"]))
cdf.rename(columns = {
    "countriesAndTerritories": "Place",
    "geoId" : "ISO_alpha-2_code",
    "countryterritoryCode" : "ISO_alpha-3_code",
    "popData2019": "Population",
    "continentExp" : "Continent",
    "Cumulative_number_for_14_days_of_COVID-19_cases_per_100000" : "14d-incidence"
    }, inplace = True)
cdf["14d-incidence"] = pd.to_numeric(cdf["14d-incidence"], errors = "coerce")
cdf.dtypes
start_of_recording = min(cdf["date_reported"])
deltaTime = (cdf["date_reported"]-start_of_recording)
cdf["deltaTime_since_start_of_recording"] = deltaTime.dt.days<jupyter_output><empty_output><jupyter_text>Cleanup<jupyter_code>cdf.describe()
for column in cdf.columns:
    print(column, "has NaN:", cdf[column].isna().sum())

cdf.shape
cdf = cdf[cdf["14d-incidence"].notna()]
for column in cdf.columns:
    print(column, "has NaN:", cdf[column].isna().sum())
cdf.shape
for column in ["cases", "deaths", "14d-incidence"]:
    isnegative = cdf[column] < 0
    print(column, "has negatives:", isnegative.sum())
cdf.shape
for column in ["cases", "deaths", "14d-incidence"]:
    ispositive = cdf[column] >= 0
    cdf = cdf.loc[ispositive]
    isnegative = cdf[column] < 0
    print(column, "has negatives:", isnegative.sum())
cdf.shape
cdf.describe()
layout_hist = {
        "template" : "plotly", 
        "yaxis": {"title": {"text": "frequency"}},
        "xaxis": {"title": {"text": "number of counts"}}
        }
for column in ["cases", "deaths"]:
    counts = cdf[column].value_counts()
    data = go.Scatter(
        mode = "markers",
        x = counts.index,
        y = counts
    )
    fig = go.Figure(data = data, layout = layout_hist)
    fig.update_layout(title = column)
    fig.update_xaxes(type = "log")
    fig.show()
cdf.describe()<jupyter_output><empty_output><jupyter_text>Questions  Which country showed the highest/lowest fluctuation in 14d-incidence within a year?<jupyter_code># fluctuation definition: high variance

countries_descr = cdf.groupby(["Place"])
countries_var = cdf[["Place", "14d-incidence"]].groupby(["Place"]).var()
n = 1
fluctuating_countries = countries_var.sort_values(by = "14d-incidence", ascending = False).head(n).index
non_fluctuating_countries = countries_var.sort_values(by = "14d-incidence", ascending = True).head(n).index
fluctuating_countries
for column in fluctuating_countries:
    data = go.Scatter(
        x = countries_descr.get_group(column)["deltaTime_since_start_of_recording"],
        y = countries_descr.get_group(column)["14d-incidence"]
    )
    fig = go.Figure(data = data, layout = layout_hist)
    fig.update_layout(title = column)
    fig.update_xaxes(type = "log")
    fig.show()
for column in non_fluctuating_countries:
    data = go.Scatter(
        x = countries_descr.get_group(column)["deltaTime_since_start_of_recording"],
        y = countries_descr.get_group(column)["14d-incidence"]
    )
    fig = go.Figure(data = data, layout = layout_hist)
    fig.update_layout(title = column)
    fig.update_xaxes(type = "log")
    fig.show()
cdf.sample(2)<jupyter_output><empty_output><jupyter_text>Create a line plot showing the 14-incidence for all European countries. Use groupby operation to generate the data list for the plotly plot.<jupyter_code>continents =cdf[["Continent", "Place", "14d-incidence", "date_reported", "deltaTime_since_start_of_recording"]].groupby(["Continent"])
european_countries = continents.get_group("Europe").drop(columns = "Continent")
european_countries.sample(3)
layout = {"title": "14d-incidence in Europe", 
        "xaxis": {"title": {"text": "number of days since " + str(start_of_recording)}},
        "yaxis": {"title": {"text": "14d-incidence"}}
        }

fig1 = go.Figure(layout = layout)

for place in european_countries["Place"].unique()[0:14]:
    country = european_countries.loc[european_countries["Place"] == place]

    fig1.add_trace(go.Scatter(x = country["deltaTime_since_start_of_recording"],
                        y = country["14d-incidence"],
                        name = place))
fig2 = go.Figure(layout = layout)

for place in european_countries["Place"].unique()[14:28]:
    country = european_countries.loc[european_countries["Place"] == place]

    fig2.add_trace(go.Scatter(x = country["deltaTime_since_start_of_recording"],
                        y = country["14d-incidence"],
                        name = place))

fig3 = go.Figure(layout = layout)

for place in european_countries["Place"].unique()[28:42]:
    country = european_countries.loc[european_countries["Place"] == place]

    fig3.add_trace(go.Scatter(x = country["deltaTime_since_start_of_recording"],
                        y = country["14d-incidence"],
                        name = place))

fig4 = go.Figure(layout = layout)

for place in european_countries["Place"].unique()[42:]:
    country = european_countries.loc[european_countries["Place"] == place]

    fig4.add_trace(go.Scatter(x = country["deltaTime_since_start_of_recording"],
                        y = country["14d-incidence"],
                        name = place))

import ipywidgets
f1 = go.FigureWidget(fig1)
f2 = go.FigureWidget(fig2)
f3 = go.FigureWidget(fig3)
f4 = go.FigureWidget(fig4)

ipywidgets.VBox([f1,f2, f3,f4])<jupyter_output><empty_output><jupyter_text>Create a smoothed version of the 14d-incidence by averaging 3 months.<jupyter_code>def averaging(values, months = 3):
    window_size = 30 * months
    window = collections.deque([], maxlen=window_size)
    average = []
    for pos, aa in enumerate(values):
        value = values[pos]
        window.append(value)
        average.append(sum(window)/len(window))
    return average

fig5 = go.Figure(layout = layout)
for place in european_countries["Place"].unique()[42:]:
    country = european_countries.loc[european_countries["Place"] == place]
    
    fig5.add_trace(go.Scatter(x = country["deltaTime_since_start_of_recording"],
                        y = averaging(country["14d-incidence"].values),
                        name = place))
f5 = go.FigureWidget(fig5)
ipywidgets.VBox([f4, f5])<jupyter_output><empty_output><jupyter_text>Create a radial plot  Create a radial plot of death rate / 100000 people (see popData2019), where one year completes a circle, i.e. 360˚. Visualize the recored years for Italy, Germany, Sweden and Greece. Hint you might need to turn the dateTime into day within the year (%j) and adjust 365 to 360 degrees.<jupyter_code>cdf.head(5)
death_rate = (cdf[["cases"]].values / cdf[["Population"]].values)*100000
cdf["death_rate_per_100k"] = death_rate
cdf.sample(2)
cols = ["Place", "Continent", "date_reported", "deltaTime_since_start_of_recording", "deaths", "Population", "death_rate_per_100k"]

gr = cdf[cols].groupby(["Place"])
gr.get_group("Germany").sample(2)
country = gr.get_group("Germany")
fig = px.line_polar(country, r = "death_rate_per_100k", theta= "deltaTime_since_start_of_recording", line_close = True)
fig.show()
radi_steps = np.arange(0,360, 360/len(gr.get_group("Germany")[["deltaTime_since_start_of_recording"]]) )
data = go.Scatterpolar(r = gr.get_group("Germany")["death_rate_per_100k"],
            theta = radi_steps
            )
fig = go.Figure(data = data)
fig.show()
v = gr.get_group("Germany")["death_rate_per_100k"]
v<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1467702892088164, 'fraction_numerical': 0.020894381449866224, 'mean_word_length': 4.455177206393329, 'pattern_counts': {'":': 15, '<': 27, '<?xml version=': 0, '>': 26, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '644565', 'license': 'CC0-1.0', 'n_tokens_mistral': 2864, 'n_tokens_neox': 2696, 'n_words': 490, 'path': 'exercises/day7/ParsingCOVIDdata.ipynb', 'repo_name': 'AnilesE/advanced_python_2021-22_HD', 'size': '7849'}