<jupyter_start><jupyter_text>author <NAME> (c) 2020 all rights reserverd license: GPL V 2.0<jupyter_code>!pip install numpy
!pip install beautifultable
!pip install matplotlib
!pip install GitPython
from git import Repo
from os import listdir 
import csv
import os
import sys
import shutil
import argparse 
import numpy as np
from beautifultable import BeautifulTable
from pandas.plotting import register_matplotlib_converters

import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import datetime as dt
import bottleneck as bn
from matplotlib.ticker import MaxNLocator
from IPython.display import Javascript 
display(Javascript("google.colab.output.resizeIframeToContent()")) 
from datetime import datetime

register_matplotlib_converters()
destdir="covid19"
searchdir=destdir+"/csse_covid_19_data/csse_covid_19_daily_reports"
precconfirmed=precrecovered=precdeath=0 

def firstfilelayout(row):
# it will receive a row in argument: with first line (header) skipped. 
    confirmed=deaths=recovered=0
    if len(row[3])<1: row[3]=0
    if len(row[4])<1: row[4]=0
    if len(row[5])<1: row[5]=0
    confirmed +=int(float(row[3]))
    deaths    +=int(float(row[4]))
    recovered +=int(float(row[5]))
    return(confirmed,deaths,recovered)

def secondfilelayout(row):
# it will receive a row in argument: with first line (header) skipped. 
    confirmed=deaths=recovered=0
    if len(row[7])<1: row[7]=0
    if len(row[8])<1: row[8]=0
    if len(row[9])<1: row[9]=0
    confirmed +=int(float(row[7]))
    deaths    +=int(float(row[8]))
    recovered +=int(float(row[9]))
    return(confirmed,deaths,recovered)


def processafile(filename):
    completefile=searchdir+"/"+filename
    confirmed=deaths=recovered=0
    with open(completefile,'r') as fi:
        reader=csv.reader(fi)
        headers=next(reader)
        for row in reader:
            if 'FIPS' in headers[0]:
                res=secondfilelayout(row)
            else:
                res=firstfilelayout(row)
            confirmed +=int(float(res[0]))
            deaths    +=int(float(res[1]))
            recovered +=int(float(res[2]))

    return(confirmed,deaths,recovered)

ratio     =True
save      =False
chart     =True
grid      =True
nicetable =True
win       =5
logyscale =False

if os.path.isdir(destdir):
  shutil.rmtree(destdir)

os.mkdir(destdir)

Repo.clone_from("https://github.com/CSSEGISandData/COVID-19",destdir)

ext='csv' 

filenames=listdir(searchdir)
filenames.remove('README.md')
filenames.remove('.gitignore')
filenames.sort(key = lambda date: datetime.strptime(date[:-4],'%m-%d-%Y'))
orig_stdout=sys.stdout

if nicetable: 
  table=BeautifulTable(max_width=132)
  if ratio:
    header=["date","Confirmed","Death","Recovered","d Confirmed","d Death","d Recovered","Death/Conf", "Recovered/Conf"]
  else:
    header=["date","Confirmed","Death","Recovered","d Confirmed","d Death","d Recovered"]
  table.column_headers=header
  if save:
    fo=open('out.csv','w')
    sys.stdout=fo

content=[]
for filename in filenames:
  if filename.endswith(ext):
    r=list(processafile(filename) )
    tablerow=[]
    deltaconfirmed=r[0]-precconfirmed
    deltadeath    =r[1]-precdeath
    deltarecovered=r[2]-precrecovered
    precconfirmed =r[0]
    precdeath     =r[1]
    precrecovered =r[2]
    if r[1] >0:
      r.append(float(r[1]/r[0]))
    else:
      r.append('NaN')
    if r[2] >0:
      r.append(float(r[2]/r[0]))
    else:
      r.append('NaN')

    if nicetable: 
      if ratio:
        tablerow=[filename[:-4],r[0],r[1],r[2],deltaconfirmed,deltadeath,deltarecovered,r[3],r[4]] 
      else: 
        tablerow=[filename[:-4],r[0],r[1],r[2],deltaconfirmed,deltadeath,deltarecovered] 
      table.append_row(tablerow)
    else:
      if ratio:
        print("{},{},{},{},{},{},{:.3f},{:.3f}".format(filename[:-4],r[0],r[1],r[2],deltaconfirmed,deltadeath,deltarecovered,r[3],r[4]) )
      else: 
        print("{},{},{},{},{},{}".format(filename[:-4],r[0],r[1],r[2],deltaconfirmed,deltadeath,deltarecovered) )
    content.append([filename[:-4],r[0],r[1],r[2],deltaconfirmed,deltadeath,deltarecovered])
    






if nicetable:
  print(table)

if save:
  sys.stdout=orig_stdout
  fo.close()


if chart:
 # plt.gca().xaxis_date()
  
  dummy=np.array(content)
  d=dummy[:,0]
  days=[dt.datetime.strptime(d,'%m-%d-%Y').date() for d in d]
  datefile=dt.datetime.today().strftime('%Y%m%d')
    ###### make chart of confirmed
  
  plt.figure()
  plt.ylabel('Confirmed') 
  plt.title('Confirmed') 
  confirmed=np.array(dummy[:,1],dtype=int)
  if logyscale:
    plt.yscale('log')
  plt.plot(days,confirmed,'r')

  plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%d-%m-%Y'))
  plt.gca().xaxis.set_major_locator( MaxNLocator(nbins = 10) )
  plt.gca().yaxis.set_major_locator( MaxNLocator(nbins = 10) )
  plt.gcf().autofmt_xdate()
  if grid:
    plt.grid()
  plt.savefig('Confirmed'+datefile)
  plt.show()

    ###### make chart of Deaths
  death=np.array(dummy[:,2],dtype=int)
  plt.figure()
  plt.ylabel('Death') 
  plt.title('Death') 
  if logyscale:
    plt.yscale('log')
  plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%d-%m-%Y'))
  plt.plot(days,death,'black')
  plt.gca().xaxis.set_major_locator( MaxNLocator(nbins = 10) )
  plt.gca().yaxis.set_major_locator( MaxNLocator(nbins = 10) )
  plt.gcf().autofmt_xdate()
  if grid:
    plt.grid()
  plt.savefig('Death'+datefile)
#        plt.show()


    ###### make chart of Recoverred
  recovered=np.array(dummy[:,3],dtype=int)
  plt.figure()
  plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%d-%m-%Y'))
  plt.ylabel('Recovered') 
  if logyscale:
    plt.yscale('log')
  plt.plot(days,recovered,'g')
  plt.gca().xaxis.set_major_locator( MaxNLocator(nbins = 10) )
  plt.gca().yaxis.set_major_locator( MaxNLocator(nbins = 10) )
  if grid:
    plt.grid()
  plt.gcf().autofmt_xdate()
  plt.savefig('Recovered'+datefile)
#        plt.show()
    ###### make chart of both Confirmed and Recoverred
  plt.figure()
  plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%d-%m-%Y'))
  plt.title('Recovered and Confirmed')
  plt.ylabel('Recovered') 
  if logyscale:
    plt.yscale('log')
  plt.plot(days,confirmed,'r',label='Confirmed')
  plt.plot(days,recovered,'g',label='Recovered')
  plt.gca().xaxis.set_major_locator( MaxNLocator(nbins = 10) )
  plt.gca().yaxis.set_major_locator( MaxNLocator(nbins = 10) )
  plt.legend() 
  if grid:
    plt.grid()
  plt.gcf().autofmt_xdate()
  plt.savefig('Recovered'+datefile)
#        plt.show()
    ###### make chart of both delta Confirmed Death and Recoverred
  dconfirmed=np.array(dummy[:,4],dtype=int)
  ddeath    =np.array(dummy[:,5],dtype=int)
  drecovered=np.array(dummy[:,6],dtype=int)
  plt.figure()
  plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%d-%m-%Y'))
  plt.title('delta Confirmed Recovered and Confirmed')
  if logyscale:
    plt.yscale('log')
  plt.plot(days,dconfirmed,'r',label='delta Confirmed')
  plt.plot(days,ddeath,'black',label='delta Death')
  plt.plot(days,drecovered,'g',label='delta Recovered')
  plt.gca().xaxis.set_major_locator( MaxNLocator(nbins = 10) )
  plt.gca().yaxis.set_major_locator( MaxNLocator(nbins = 10) )
  plt.legend() 
  if grid:
    plt.grid()
  plt.gcf().autofmt_xdate()
  plt.savefig('dRecDeathConf'+datefile)
#        plt.show()
######################################
  mvadconfirmed = bn.move_mean(dconfirmed, window=win,min_count=1) 
  mvaddeath     = bn.move_mean(ddeath, window=win,min_count=1) 
  mvadrecovered = bn.move_mean(drecovered, window=win,min_count=1) 
  plt.figure() 
  plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%d-%m-%Y')) 
  plt.title('mva delta Confirmed Recovered and Confirmed') 
  if logyscale: 
    plt.yscale('log') 
  plt.plot(days,mvadconfirmed,'r',label='delta Confirmed') 
  plt.plot(days,mvaddeath,'black',label='delta Death') 
  plt.plot(days,mvadrecovered,'g',label='delta Recovered') 
  plt.gca().xaxis.set_major_locator( MaxNLocator(nbins = 10) ) 
  plt.gca().yaxis.set_major_locator( MaxNLocator(nbins = 10) ) 
  plt.legend()  
  if grid: 
    plt.grid() 
  plt.gcf().autofmt_xdate() 
  plt.savefig('dRecDeathConf'+datefile) 
######################################
  plt.figure() 
  plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%d-%m-%Y')) 
  plt.title('both mva delta Confirmed Recovered and Confirmed') 
  if logyscale: 
    plt.yscale('log') 
  plt.plot(days,mvadconfirmed,'r',linestyle='-.',label='mva delta Confirmed') 
  plt.plot(days,mvaddeath,'black',linestyle='-.',label='mva delta Death') 
  plt.plot(days,mvadrecovered,'g',linestyle='-.',label='mva delta Recovered') 
  plt.plot(days,dconfirmed,'r',label='dConfirmed') 
  plt.plot(days,ddeath,'black',label='dDeath') 
  plt.plot(days,drecovered,'g',label='dRecovered') 
  plt.gca().xaxis.set_major_locator( MaxNLocator(nbins = 10) ) 
  plt.gca().yaxis.set_major_locator( MaxNLocator(nbins = 10) ) 
  plt.legend()  
  if grid: 
    plt.grid() 
  plt.gcf().autofmt_xdate() 
  plt.savefig('mixmvadRecDeathConf'+datefile)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17645107794361525, 'fraction_numerical': 0.014925373134328358, 'mean_word_length': 4.515853658536585, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '39063', 'license': 'Apache-2.0', 'n_tokens_mistral': 3585, 'n_tokens_neox': 3409, 'n_words': 508, 'path': 'covid19glodata.ipynb', 'repo_name': 'gvieri/coviutil', 'size': '9059'}