<jupyter_start><jupyter_text>CoronaVirus Prediction<jupyter_code># Get data from Github
import numpy as np
from math import sqrt
from sklearn.metrics import mean_squared_error
import pandas as pd
#url = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv'
url = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv'
confirmed = pd.read_csv(url, error_bad_lines=False)
url = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Deaths.csv'
death = pd.read_csv(url, error_bad_lines=False)
url = 'https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Recovered.csv'
recover = pd.read_csv(url, error_bad_lines=False)

# fix region names
confirmed['Country/Region']= confirmed['Country/Region'].str.replace("Mainland China", "China")
confirmed['Country/Region']= confirmed['Country/Region'].str.replace("US", "Unites States")
death['Country/Region']= death['Country/Region'].str.replace("Mainland China", "China")
death['Country/Region']= death['Country/Region'].str.replace("US", "Unites States")
recover['Country/Region']= recover['Country/Region'].str.replace("Mainland China", "China")
recover['Country/Region']= recover['Country/Region'].str.replace("US", "Unites States")<jupyter_output><empty_output><jupyter_text>Get Population<jupyter_code>population=pd.read_csv('population.csv', sep=',', encoding='latin1') 
confirmed=pd.merge(confirmed, population,how='left' ,on=['Province/State','Country/Region'])
death=pd.merge(death, population,how='left' ,on=['Province/State','Country/Region'])
recover=pd.merge(recover, population,how='left' ,on=['Province/State','Country/Region'])
# !pip install qwikidata
# import qwikidata
# import qwikidata.sparql
# ##################################################### This was already done ####################################################
# def get_city_wikidata(city, country):
#     query = """
#     SELECT ?city ?cityLabel ?country ?countryLabel ?population
#     WHERE
#     {
#       ?city rdfs:label '%s'@en.
#       ?city wdt:P1082 ?population.
#       ?city wdt:P17 ?country.
#       ?city rdfs:label ?cityLabel.
#       ?country rdfs:label ?countryLabel.
#       FILTER(LANG(?cityLabel) = "en").
#       FILTER(LANG(?countryLabel) = "en").
#       FILTER(CONTAINS(?countryLabel, "%s")).
#     }
#     """ % (city, country)

#     res = qwikidata.sparql.return_sparql_query_results(query)
#     out = res['results']['bindings'][0]
#     return out

# # Get the population of the city
# for i in range(0,len(confirmed)):
# #for i in range(10,14):
#   try:
#     t=get_city_wikidata(confirmed.iloc[i,0],confirmed.iloc[i,1])
#     t={key: t[key] for key in t.keys()&{'population'}} 
#     t=list(t.values())
#     print(confirmed.iloc[i,0],t[0]['value'])
#   except Exception:
#     print (confirmed.iloc[i,0],'Error')
#   #confirmed['population']=
# merge region
confirmed['region']=confirmed['Country/Region'].map(str)+'_'+confirmed['Province/State'].map(str)
death['region']=death['Country/Region'].map(str)+'_'+death['Province/State'].map(str)
recover['region']=recover['Country/Region'].map(str)+'_'+recover['Province/State'].map(str)
confirmed.iloc[:20,:]<jupyter_output><empty_output><jupyter_text>Create Time Series + Plots<jupyter_code>def create_ts(df):
  ts=df
  ts=ts.drop(['Province/State', 'Country/Region','Lat', 'Long',' Population '], axis=1)
  ts.set_index('region')
  ts=ts.T
  ts.columns=ts.loc['region']
  ts=ts.drop('region')
  ts=ts.fillna(0)
  ts=ts.reindex(sorted(ts.columns), axis=1)
  return (ts)
ts=create_ts(confirmed)
ts_d=create_ts(death)
ts_rec=create_ts(recover)
import matplotlib.pyplot as plt
p=ts.reindex(ts.max().sort_values(ascending=False).index, axis=1)
p.iloc[:,:1].plot(marker='*',figsize=(20,8)).set_title('Daily Total Confirmed - Wuhan',fontdict={'fontsize': 22})
p.iloc[:,2:75].plot(marker='*',figsize=(20,8)).set_title('Daily Total Confirmed - Major areas',fontdict={'fontsize': 22})

p_d=ts_d.reindex(ts.mean().sort_values(ascending=False).index, axis=1)
p_d.iloc[:,:1].plot(marker='*',figsize=(20,8)).set_title('Daily Total Death - Wuhan',fontdict={'fontsize': 22})
p_d.iloc[:,2:75].plot(marker='*',figsize=(20,8)).set_title('Daily Total Death - Major areas',fontdict={'fontsize': 22})

p_r=ts_rec.reindex(ts.mean().sort_values(ascending=False).index, axis=1)
p_r.iloc[:,:1].plot(marker='*',figsize=(20,8)).set_title('Daily Total Recoverd - Wuhan',fontdict={'fontsize': 22})
p_r.iloc[:,2:75].plot(marker='*',figsize=(20,8)).set_title('Daily Total Recoverd - Major areas',fontdict={'fontsize': 22})<jupyter_output><empty_output><jupyter_text>Kalman Filter With R<jupyter_code># Create data for R script
ts_r=ts.reset_index()
ts_r=ts_r.rename(columns = {'index':'date'})
ts_r['date']=pd.to_datetime(ts_r['date'] ,errors ='coerce')
ts_r.to_csv(r'ts_r.csv')
import rpy2
%load_ext rpy2.ipython
# %%R
# install.packages('pracma')
# install.packages('reshape')
# install.packages('readr')
%%R
require(pracma)
require(Metrics)
require(readr)
all<- read_csv("ts_r.csv")
all$X1<-NULL
date<-all[,1]
date[nrow(date) + 1,1] <-all[nrow(all),1]+1
pred_all<-NULL
for (n in 2:ncol(all)-1) {
  Y<-ts(data = all[n+1], start = 1, end =nrow(all)+1)  
  sig_w<-0.01
  w<-sig_w*randn(1,100) # acceleration which denotes the fluctuation (Q/R) rnorm(100, mean = 0, sd = 1)
  sig_v<-0.01
  v<-sig_v*randn(1,100)   
  t<-0.45
  phi<-matrix(c(1,0,t,1),2,2)
  gama<-matrix(c(0.5*t^2,t),2,1)
  H<-matrix(c(1,0),1,2)
  #Kalman
  x0_0<-p0_0<-matrix(c(0,0),2,1)
  p0_0<-matrix(c(1,0,0,1),2,2)
  Q<-0.01
  R<-0.01
  X<-NULL
  X2<-NULL
  pred<-NULL
  for (i in 0:nrow(all)) {
    namp <-paste("p", i+1,"_",i, sep = "")
    assign(namp, phi%*%(get(paste("p", i,"_",i, sep = "")))%*%t(phi)+gama%*%Q%*%t(gama))
    namk <- paste("k", i+1, sep = "")
    assign(namk,get(paste("p", i+1,"_",i, sep = ""))%*%t(H)%*%(1/(H%*%get(paste("p", i+1,"_",i, sep = ""))%*%t(H)+R)))
    namx <- paste("x", i+1,"_",i, sep = "")
    assign(namx,phi%*%get(paste("x", i,"_",i, sep = "")))
    namE <- paste("E", i+1, sep = "")
    assign(namE,Y[i+1]-H%*%get(paste("x", i+1,"_",i, sep = "")))
    namx2 <- paste("x", i+1,"_",i+1, sep = "")
    assign(namx2,get(paste("x", i+1,"_",i, sep = ""))+get(paste("k", i+1, sep = ""))%*%get(paste("E", i+1, sep = "")))
    namp2 <- paste("p", i+1,"_",i+1, sep = "")
    assign(namp2,(p0_0-get(paste("k", i+1, sep = ""))%*%H)%*%get(paste("p", i+1,"_",i, sep = "")))
    X<-rbind(X,get(paste("x", i+1,"_",i,sep = ""))[1])
    X2<-rbind(X2,get(paste("x", i+1,"_",i,sep = ""))[2])
    if(i>2){
      remove(list=(paste("p", i-1,"_",i-2, sep = "")))
      remove(list=(paste("k", i-1, sep = "")))
      remove(list=(paste("E", i-1, sep = "")))
      remove(list=(paste("p", i-2,"_",i-2, sep = "")))
      remove(list=(paste("x", i-1,"_",i-2, sep = "")))
      remove(list=(paste("x", i-2,"_",i-2, sep = "")))}
  }
  pred<-NULL
  pred<-cbind(Y,X,round(X2,4))
  pred<-as.data.frame(pred)
  pred$region<-colnames(all[,n+1])
  pred$date<-date$date
  pred$actual<-rbind(0,(cbind(pred[2:nrow(pred),1])/pred[1:nrow(pred)-1,1]-1)*100)
  pred$predict<-rbind(0,(cbind(pred[2:nrow(pred),2])/pred[1:nrow(pred)-1,2]-1)*100)
  pred$pred_rate<-(pred$X/pred$Y-1)*100
  pred$X2_change<-rbind(0,(cbind(pred[2:nrow(pred),3]-pred[1:nrow(pred)-1,3])))
  pred_all<-rbind(pred_all,pred)
}
pred_all<-cbind(pred_all[,4:5],pred_all[,1:3])
names(pred_all)[5]<-"X2"
pred_all=pred_all[with( pred_all, order(region, date)), ]
pred_all<-pred_all[,3:5]
p=%R pred_all
############ Merge R output due to package problem
t=ts_d
t=t.stack().reset_index(name='confirmed')
t.columns=['date', 'region','confirmed']
t['date']=pd.to_datetime(t['date'] ,errors ='coerce')
t=t.sort_values(['region', 'date'])

temp=t.iloc[:,:3]
temp=temp.reset_index(drop=True)
for i in range(1,len(t)+1):
  if(temp.iloc[i,1] is not temp.iloc[i-1,1]):
    temp.loc[len(temp)+1] = [temp.iloc[i-1,0]+ pd.DateOffset(1),temp.iloc[i-1,1], 0] 
temp=temp.sort_values(['region', 'date'])
temp=temp.reset_index(drop=True)
temp['Y']=p['Y']
temp['X']=p['X']
temp['X2']=p['X2']<jupyter_output><empty_output><jupyter_text>Pre Proccessing Data for ML Model  Extract Weather Data<jupyter_code>#!pip install pyweatherbit
from weatherbit.api import Api
import json  
import pandas as pd  
from pandas.io.json import json_normalize 
api_key = "c17f5d0759054de9b03e1064f9aefff6"
#api_key ="<KEY>"
#api_key ="<KEY>"
#api_key ="be7aaf4ff7184347aa1ebc2a2db514fe"

api = Api(api_key)
api.set_granularity('daily')

# Set the granularity of the API - Options: ['daily','hourly','3hourly']
# Will only affect forecast requests.
r = api.get_forecast(lat=26.9124, lon=75.7873).json
r<jupyter_output><empty_output><jupyter_text>Weather History<jupyter_code>################## already done since API is limited to 500 call per day

w=pd.DataFrame(columns=['date','region','min','max'])
confirmed = confirmed[:10]
for i in range (61,len(confirmed)):
  start_date=pd.to_datetime('2020-03-13')
  for j in range (4,confirmed.shape[1]-2):
    jas=api.get_history(lat=confirmed.iloc[i,2], lon=confirmed.iloc[i,3], start_date=start_date.strftime('%Y-%m-%d'),end_date=(start_date+ pd.DateOffset(days=1)).strftime('%Y-%m-%d')).json
#     j=json_normalize(j)
#     j=j['data']
#     max_temp=json_normalize(j['data'])['max_temp'].values[0] # max
#     min_temp=json_normalize(j['data'])['min_temp'].values[0]
    try:
      w=w.append({'date':confirmed.columns[j],'region':confirmed.iloc[i,confirmed.shape[1]-1] ,'min':json_normalize(jas['data'])['min_temp'].values[0],'max':json_normalize(jas['data'])['max_temp'].values[0]}, ignore_index=True)
    except Exception:
      w=w.append({'date':confirmed.columns[j],'region':confirmed.iloc[i,confirmed.shape[1]-1] ,'min':None,'max':None}, ignore_index=True)
    #start_date=start_date+ pd.DateOffset(days=1)

################## Update Recent Day Weather

w_update=pd.DataFrame(columns=['date','region','min','max'])
for i in range (28,len(confirmed)):
  start_date=pd.to_datetime('2020-02-17')
  for j in range (confirmed.shape[1]-4,confirmed.shape[1]-2):
    jas=api.get_history(lat=confirmed.iloc[i,2], lon=confirmed.iloc[i,3], start_date=start_date.strftime('%Y-%m-%d'),end_date=(start_date+ pd.DateOffset(days=1)).strftime('%Y-%m-%d')).json
    try:
      w_update=w_update.append({'date':confirmed.columns[j],'region':confirmed.iloc[i,confirmed.shape[1]-1] ,'min':json_normalize(jas['data'])['min_temp'].values[0],'max':json_normalize(jas['data'])['max_temp'].values[0]}, ignore_index=True)
    except Exception:
      w_update=w_update.append({'date':confirmed.columns[j],'region':confirmed.iloc[i,confirmed.shape[1]-1] ,'min':None,'max':None}, ignore_index=True)
    start_date=start_date+ pd.DateOffset(days=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.233996186325252, 'fraction_numerical': 0.033778262053936255, 'mean_word_length': 6.574965612104539, 'pattern_counts': {'":': 0, '<': 70, '<?xml version=': 0, '>': 28, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '154923', 'license': 'MIT', 'n_tokens_mistral': 4741, 'n_tokens_neox': 4452, 'n_words': 571, 'path': 'COVID19 Prediction.ipynb', 'repo_name': 'swarnkarkush21/corona-detection-using-data-science', 'size': '11067'}