<jupyter_start><jupyter_text>Maping the wave of spread of covid-19<jupyter_code>import outbreak
import pandas as pd
import numpy as np
import geopandas as gpd
import copy
import csv
import gzip
import io
from urllib.request import Request, urlopen
import matplotlib
import matplotlib.pyplot as plt
import os
os.environ['MAPBOX_API_KEY'] = '<KEY>'
import pydeck
import ujson
font = {'family' : 'DejaVu Sans',
        'weight' : 'normal',
        'size'   : 12}

matplotlib.rc('font', **font)<jupyter_output><empty_output><jupyter_text>Loading the data<jupyter_code>F = outbreak.read_flow_matrix('../dados/flowmatrix_full_mun.csv.gz', header=0) 
nodes = outbreak.read_nodes('../dados/nodesUF.csv.gz')
# nodes['incidence'] = nodes.I/nodes.tgtpop
mapa = gpd.read_file('../mapas/muni_br.gpkg')
# Fetching cases from Brasil.io
request = Request('https://data.brasil.io/dataset/covid19/caso_full.csv.gz', headers={"User-Agent": "python-urllib"})
response = urlopen(request)
cases = pd.read_csv(io.StringIO(gzip.decompress(response.read()).decode("utf-8")))
cases
nodesc = pd.merge(nodes,cases[(cases.is_last)], left_on='tgt', right_on='city_ibge_code', how='left')
nodesc = nodesc[['tgt','tgtname','tgt_UFCOD','tgtpop','last_available_confirmed']]
nodesc.fillna(0, inplace=True)
cases<jupyter_output><empty_output><jupyter_text>Add to map only the latest count per municipality<jupyter_code>mapa['casos']=0
for i, row in mapa.iterrows():
    if row.code_muni in list(cases.city_ibge_code):
        mapa.loc[mapa.code_muni==row.code_muni, 'casos'] += cases[(cases.city_ibge_code==row.code_muni)&
                                                                  (cases.is_last)].last_available_confirmed.iloc[0]
def get_seeds(nodes, threshold=100):
    '''
    Only consider cities with more than `threshold` cases as seeds
    '''
    seeds = copy.deepcopy(nodes)
    seeds['confirmed'] = [c if c>=100 else 0 for c in nodes.last_available_confirmed]
    # nodes = nodes[nodes.confirmed >=100]
    seeds.fillna(0,inplace=True)
    seeds['incidence'] = (seeds.confirmed/seeds.tgtpop)
    return seeds
attenuation = 1 # 1 means no attenuation
r0=2.5
seeds = get_seeds(nodesc,100)
probs = outbreak.get_outbreaks(F, seeds.incidence, R0=r0, attenuate=attenuation)
plt.hist(probs, bins=50);
nodesc['probs'] = probs
mapap = pd.merge(mapa, nodesc, left_on='code_muni', right_on='tgt')<jupyter_output><empty_output><jupyter_text>Visualizing with Pydeck<jupyter_code>DATA = ujson.loads(mapap[mapap.probs>0.5].to_json())

LAND_COVER = [[[-54.291, -14.139], [-54.39180562, -14.23962137], [-54.39180562, -14.23962137], [-54.291, -14.139]]]

INITIAL_VIEW_STATE = pydeck.ViewState(
  latitude=-14.23962137,
  longitude= -54.39180562,
  zoom=3,
  max_zoom=16,
  pitch=45,
  bearing=0
)

polygon = pydeck.Layer(
    'PolygonLayer',
    LAND_COVER,
    stroked=False,
    # processes the data as a flat longitude-latitude pair
    get_polygon='-',
    get_fill_color=[0, 0, 0, 20]
)

geojson = pydeck.Layer(
    'GeoJsonLayer',
    DATA,
    opacity=0.8,
    stroked=False,
    filled=True,
    extruded=True,
    wireframe=True,
#     mapbox_key='<KEY>',
    get_elevation='properties.casos',
    get_fill_color='[properties.probs *255, 255*(1-properties.probs), properties.probs * 255]',
    get_line_color=[255, 255, 255],
    pickable=True
)

r = pydeck.Deck(
    layers=[polygon,geojson],
    initial_view_state=INITIAL_VIEW_STATE)

r.to_html()
cents = mapap.centroid
data2 = mapap[['casos','probs']]
data2['lon'] = copy.deepcopy([p.x for p in cents])
data2['lat'] = copy.deepcopy([p.y for p in cents])
# data2
layer = pydeck.Layer(
    'HexagonLayer',  # `type` positional argument is here
    data2,
    get_position=['lon', 'lat'],
    auto_highlight=True,
    elevation_scale=50,
    lower_percentile=0.8,
#     elevation_upper_percentile=1,
    get_elevation='points.probs',
    radius=3000,
    pickable=True,
    elevation_range=[0, 3000],
    extruded=True,
    coverage=3)

# Set the viewport location
view_state = pydeck.ViewState(
    longitude=-54.39180562,
    latitude=-14.23962137,
    zoom=3,
    min_zoom=3,
    max_zoom=15,
    pitch=40.5,
    bearing=-27.36)

# Combined all of it and render a viewport
r = pydeck.Deck(layers=[layer], initial_view_state=view_state)
r.to_html('hexagon-probs.html')
pydeck.Layer?<jupyter_output><empty_output><jupyter_text>Creating Animation<jupyter_code>cases.date = pd.to_datetime(cases.date).dt.date
casesm = cases[(cases.place_type=='city')&(~cases.city_ibge_code.isnull())]
casesm['city_ibge_code'] = casesm.city_ibge_code.astype(int)
casesm = casesm[casesm.city_ibge_code!=0]
# casesm.fillna(0, inplace=True)
len(set(casesm.city_ibge_code))<jupyter_output><ipython-input-13-f41215808567>:3: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  casesm['city_ibge_code'] = casesm.city_ibge_code.astype(int)<jupyter_text>Creating a column for each day Here we apply pivot_table to cases in order to create a separate columns for each day since the beginning of the epidemic in Brasil.<jupyter_code>casesw = casesm.pivot_table(index='city_ibge_code',columns='date', values='last_available_confirmed')
casesw.fillna(0, inplace=True)
# casesw.drop([0], inplace=True) #remove line with geocode 0.0
casesw
mapa_serie = pd.merge(mapa,casesw, left_on='code_muni', right_on=casesw.index,how='outer')
mapa_serie.set_index('code_muni', inplace=True)
# mapa_serie.fillna(0, inplace=True)
fig,ax = plt.subplots(1,1, figsize=(10,10))
mapa_serie.plot(column='2020-08-26',ax=ax, cmap='Reds', legend=True,edgecolor='white',
                linewidth=0,
               scheme='user_defined', classification_kwds={'bins':[0, 10, 100, 1000, 10000]},
                legend_kwds={'loc':'upper left','bbox_to_anchor': (0.6, 1)}
               );
ax.set_axis_off()
plt.tight_layout()
plt.savefig('cases_june_21.png', dpi=300)
attenuation = 1 # 1 means no attenuation
r0=1.7
for D in casesw.columns:
    print(f"Processing {D}...")
    nodesC = pd.merge(nodes,casesm[casesm.date==D], left_on='tgt', right_on='city_ibge_code', how='left')
    nodesC = nodesC[['tgt','tgtname','tgt_UFCOD','tgtpop','last_available_confirmed']]
    seeds = get_seeds(nodesC,100)
#     print(len(seeds.confirmed>100))
    try:
        probs = outbreak.get_outbreaks(F, seeds.incidence, R0=r0, attenuate=attenuation)
    except:
        print(seeds)
#     print(probs.max())
    cname = 'p_'+str(D)
    nodesC[cname] = probs
    mapa_serie = pd.merge(mapa_serie, nodesC[['tgt',cname]].set_index('tgt'), left_index=True,right_index=True)
mapa_serie.columns = [str(c) for c in mapa_serie.columns]
mapa_serie.to_file('Probability_and_case_map_daily.gpkg', driver='GPKG')<jupyter_output><empty_output><jupyter_text>Gerando as Figuras<jupyter_code>import matplotlib
font = {'family' : 'DejaVu Sans',
        'weight' : 'normal',
        'size'   : 24}

matplotlib.rc('font', **font)
for i,D in enumerate(casesw.columns):
    if mapa_serie['p_'+str(D)].max()==0:
        continue
    fig,ax = plt.subplots(1,1, figsize=(15,15), constrained_layout=False)
    mapa_serie.plot(ax=ax,column='p_'+str(D), legend=True,
                    legend_kwds={'label': "Probabilidade",'orientation': "horizontal"})
    ax.set_title(f'Probabilidades em {D}')
    ax.set_axis_off()
    plt.tight_layout()
    plt.savefig(f'anim/prob_{i}.png', dpi=200)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1495253164556962, 'fraction_numerical': 0.03731540084388186, 'mean_word_length': 4.821181887950883, 'pattern_counts': {'":': 1, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '386422', 'license': 'MIT', 'n_tokens_mistral': 3068, 'n_tokens_neox': 2870, 'n_words': 492, 'path': 'notebooks/Pydeck Probability.ipynb', 'repo_name': 'NAMD/covidash', 'size': '7754'}