<jupyter_start><jupyter_text>Read Datasets<jupyter_code>#read super Covid-19 dataframe. 
USCovid = pd.read_csv("../../data/output/covid.csv")
#read country cases dataset and choose countries with similar pop as US
CountryNewCases = pd.read_csv("https://covid.ourworldindata.org/data/ecdc/new_cases.csv")
CountryNewDeaths = pd.read_csv("https://covid.ourworldindata.org/data/ecdc/new_deaths.csv")
CountryPop = pd.read_csv("https://covid.ourworldindata.org/data/ecdc/locations.csv")

CountryNewCases = CountryNewCases[["date","Bangladesh","Indonesia","Pakistan","Brazil","Nigeria"]]
CountryNewDeaths = CountryNewDeaths[["date","Bangladesh","Indonesia","Pakistan","Brazil","Nigeria"]]
PopArray = ["Bangladesh","Indonesia","Pakistan","Brazil","Nigeria"]
CountryPop = CountryPop.loc[CountryPop["location"].isin(PopArray)]

MyFivePop = CountryPop.drop(["countriesAndTerritories","continent","population_year"],axis = 1) 

#drop unneccesary variables in Super covid-19 dataframe. 
USCovid = USCovid.drop(["Unnamed: 0","countyFIPS","stateFIPS"], axis=1)<jupyter_output><empty_output><jupyter_text>Sum up United States Cases together and get New cases and New deaths Normalized by cases per 1 million people.<jupyter_code>#Removes "Statewide unallocated" value in covid dataframe
covid_df = USCovid.drop(USCovid.loc[USCovid['County Name'] == "Statewide Unallocated"].index)

#sums up every covid cases from every county and state to one row. 
sums = covid_df.select_dtypes(pd.np.number).sum(axis = 0).rename('UnitedStates')
sums = sums.to_frame()


#get Population value of US and all countries to normalize in later cells. 
USPopulation = sums.values[0]
Bang_Pop = MyFivePop.values[0,1]
Brazil_Pop = MyFivePop.values[1,1]
Indon_Pop = MyFivePop.values[2,1]
Nigeria_Pop = MyFivePop.values[3,1]
Pak_Pop = MyFivePop.values[4,1]


#seperate cases and deaths. 
US_cases = sums.filter(regex = "x", axis = 0)
US_deaths = sums.filter(regex = "y",axis = 0)
#calculate only new cases 
NewUScases = US_cases.diff()
NewUSdeaths = US_deaths.diff()

print(NewUScases[260:])
print(NewUSdeaths[260:])
#calculate and Normalize dataset to cases per 50 million people. 
norm_val = 50_000_000
USNormalize = lambda x: (x / USPopulation) * norm_val
Norm_NewUScases = NewUScases.transform(USNormalize).round(0)
Norm_NewUSdeaths = NewUSdeaths.transform(USNormalize).round(0)
print(Norm_NewUScases)
print(Norm_NewUSdeaths)
#changed date format for cases
Norm_NewUScases = Norm_NewUScases.T
Norm_NewUScases=Norm_NewUScases.rename(columns=lambda x: (pd.to_datetime(x.split('_')[0], errors= 'ignore')), errors='raise')
Norm_NewUScases = Norm_NewUScases.T
print(Norm_NewUScases.head())

#changed date format for deaths
Norm_NewUSdeaths = Norm_NewUSdeaths.T
Norm_NewUSdeaths=Norm_NewUSdeaths.rename(columns=lambda x: (pd.to_datetime(x.split('_')[0], errors= 'ignore')), errors='raise')
Norm_NewUSdeaths = Norm_NewUSdeaths.T
print(Norm_NewUSdeaths.head())<jupyter_output>UnitedStates
2020-01-22           NaN
2020-01-23           0.0
2020-01-24           0.0
2020-01-25           0.0
2020-01-26           0.0
            UnitedStates
2020-01-22           NaN
2020-01-23           0.0
2020-01-24           0.0
2020-01-25           0.0
2020-01-26           0.0<jupyter_text>Get weekly average of cases<jupyter_code>WeeklyUS_cases = Norm_NewUScases.resample('W').mean().round(0).astype(int).rename_axis('Week')
WeeklyUS_deaths = Norm_NewUSdeaths.resample('W').mean().round(0).astype(int).rename_axis('Week')
print(WeeklyUS_cases)
print(WeeklyUS_deaths)<jupyter_output>UnitedStates
Week                    
2020-01-26             0
2020-02-02             0
2020-02-09             0
2020-02-16             0
2020-02-23             0
2020-03-01             0
2020-03-08            10
2020-03-15            65
2020-03-22           643
2020-03-29          2257
2020-04-05          4170
2020-04-12          4767
2020-04-19          4299
2020-04-26          4521
2020-05-03          4167
2020-05-10          3668
2020-05-17          3365
2020-05-24          3380
2020-05-31          3155
2020-06-07          3283
2020-06-14          3262
2020-06-21          4003
2020-06-28          5751
2020-07-05          7173
2020-07-12          8979
2020-07-19         10008
2020-07-26          9824
2020-08-02          9254
2020-08-09          8105
2020-08-16          7594
2020-08-23          6459
2020-08-30          6273
2020-09-06          6213
2020-09-13          5224
2020-09-20          6205
2020-09-27          6664
2020-10-04          6472
2020-10-11          7362
[...]<jupyter_text>Find Mean, Median and Mode for all weeks of US.<jupyter_code>#Calculates mean median and mode for cases and deaths in US. 
US_MeanCases = WeeklyUS_cases.mean().round(0)
US_MedianCases = WeeklyUS_cases.median()
US_ModeCases = WeeklyUS_cases.mode()


US_MeanDeaths = WeeklyUS_deaths.mean().round(0)
US_MedianDeaths = WeeklyUS_deaths.median()
US_ModeDeaths = WeeklyUS_deaths.mode()

US_ModeCases = US_ModeCases.values.tolist()
US_ModeDeaths = US_ModeDeaths.values.tolist()<jupyter_output><empty_output><jupyter_text>Calculate Country Mean, Median, and Mode weekly statistics to compare with U.S<jupyter_code>#fill NaN with zero for country dataset.
Country_NewCases = CountryNewCases[22:290].fillna(0)
Country_NewDeaths = CountryNewDeaths[22:290].fillna(0)

Country_NewCases.to_csv('../../data/output/Country_NewCases.csv',index=False)
Country_NewDeaths.to_csv('../../data/output/Country_NewDeaths.csv',index=False)
print(Country_NewCases)
print(Country_NewDeaths)<jupyter_output>date  Bangladesh  Indonesia  Pakistan  Brazil  Nigeria
22   2020-01-22         0.0        0.0       0.0       0      0.0
23   2020-01-23         0.0        0.0       0.0       0      0.0
24   2020-01-24         0.0        0.0       0.0       0      0.0
25   2020-01-25         0.0        0.0       0.0       0      0.0
26   2020-01-26         0.0        0.0       0.0       0      0.0
..          ...         ...        ...       ...     ...      ...
285  2020-10-11      1203.0     4294.0     666.0   26749    111.0
286  2020-10-12      1193.0     4497.0       0.0   12345    163.0
287  2020-10-13      1472.0     3267.0     916.0    8426    164.0
288  2020-10-14      1537.0     3906.0     615.0   10220    225.0
289  2020-10-15      1684.0     4127.0     755.0   27235    179.0

[268 rows x 6 columns]
           date  Bangladesh  Indonesia  Pakistan  Brazil  Nigeria
22   2020-01-22         0.0        0.0       0.0       0      0.0
23   2020-01-23         0.0        0.0       0.0    [...]<jupyter_text>Normalize Cases and Death by 1 million population.<jupyter_code>#calculate and Normalize dataset to cases per 1 million people. 
Norm_CountryCases = Country_NewCases.assign(Bangladesh = lambda x: (x['Bangladesh'] / Bang_Pop) * norm_val).round(0)
Norm_CountryCases = Norm_CountryCases.assign(Indonesia = lambda x: (x['Indonesia'] / Indon_Pop) * norm_val).round(0)
Norm_CountryCases = Norm_CountryCases.assign(Pakistan = lambda x: (x['Pakistan'] / Pak_Pop) * norm_val).round(0)
Norm_CountryCases = Norm_CountryCases.assign(Brazil = lambda x: (x['Brazil'] / Brazil_Pop ) * norm_val).round(0)
Norm_CountryCases = Norm_CountryCases.assign(Nigeria = lambda x: (x['Nigeria'] / Nigeria_Pop ) * norm_val).round(0)

Norm_Country_NewDeaths = Country_NewDeaths.assign(Bangladesh = lambda x: (x['Bangladesh'] / Bang_Pop) * norm_val).round(0)
Norm_Country_NewDeaths = Norm_Country_NewDeaths.assign(Indonesia = lambda x: (x['Indonesia'] / Indon_Pop) * norm_val).round(0)
Norm_Country_NewDeaths = Norm_Country_NewDeaths.assign(Pakistan = lambda x: (x['Pakistan'] / Pak_Pop) * norm_val).round(0)
Norm_Country_NewDeaths = Norm_Country_NewDeaths.assign(Brazil = lambda x: (x['Brazil'] / Brazil_Pop ) * norm_val).round(0)
Norm_Country_NewDeaths = Norm_Country_NewDeaths.assign(Nigeria = lambda x: (x['Nigeria'] / Nigeria_Pop ) * norm_val).round(0)



print(Norm_CountryCases)
print(Norm_Country_NewDeaths)

#set index as a datetime to easily calculate by weekly basis. 
Norm_CountryCases['date'] = pd.to_datetime(Norm_CountryCases['date'])
Norm_CountryCases = Norm_CountryCases.set_index('date')

Norm_Country_NewDeaths['date'] = pd.to_datetime(Norm_Country_NewDeaths['date'])
Norm_Country_NewDeaths = Norm_Country_NewDeaths.set_index('date')<jupyter_output><empty_output><jupyter_text>Find average of Cases and deaths for each week.<jupyter_code>#calculates weekly average of cases and deaths

WeeklyCountries_cases = Norm_CountryCases.resample('W').mean().round(0).astype(int).rename_axis('Week')
WeeklyCountries_deaths = Norm_Country_NewDeaths.resample('W').mean().round(0).astype(int).rename_axis('Week')

print(WeeklyCountries_cases)
print(WeeklyCountries_deaths)<jupyter_output>Bangladesh  Indonesia  Pakistan  Brazil  Nigeria
Week                                                        
2020-01-26           0          0         0       0        0
2020-02-02           0          0         0       0        0
2020-02-09           0          0         0       0        0
2020-02-16           0          0         0       0        0
2020-02-23           0          0         0       0        0
2020-03-01           0          0         0       0        0
2020-03-08           0          0         0       0        0
2020-03-15           0          2         1       4        0
2020-03-22           1          9        20      34        1
2020-03-29           1         19        25      93        3
2020-04-05           1         25        34     214        4
2020-04-12          18         46        84     351        4
2020-04-19          72         63        96     533        8
2020-04-26         124         62       153     736       22
2020-05-03         165  [...]<jupyter_text>Calculate Mean, Median, and Mode of cases and deaths for all 5 countries.<jupyter_code>FiveCountry_MeanCases = WeeklyCountries_cases.mean().round(0)
FiveCountry_MedianCases = WeeklyCountries_cases.median()
FiveCountry_ModeCases = WeeklyCountries_cases.mode()

FiveCountry_MeanDeaths = WeeklyCountries_deaths.mean().round(0)
FiveCountry_MedianDeaths = WeeklyCountries_deaths.median()
FiveCountry_ModeDeaths = WeeklyCountries_deaths.mode()

FiveCountry_ModeCases = FiveCountry_ModeCases.values.tolist()
FiveCountry_ModeDeaths = FiveCountry_ModeDeaths.values.tolist()<jupyter_output><empty_output><jupyter_text>Puts Mean,Median and Mode values in a list to put into a dataframe for easier comparisions.<jupyter_code>ListCase = [["United States",US_MeanCases[0],US_MedianCases[0],US_ModeCases[0][0]],
        ["Bangladesh", FiveCountry_MeanCases[0],FiveCountry_MedianCases[0],FiveCountry_ModeCases[0][0]],
        ["Indonesia", FiveCountry_MeanCases[1],FiveCountry_MedianCases[1],FiveCountry_ModeCases[0][1]],
        ["Pakistan", FiveCountry_MeanCases[2],FiveCountry_MedianCases[2],FiveCountry_ModeCases[0][2]],
        ["Brazil", FiveCountry_MeanCases[3],FiveCountry_MedianCases[3],FiveCountry_ModeCases[0][3]],
       ["Nigeria", FiveCountry_MeanCases[4],FiveCountry_MedianCases[4],FiveCountry_ModeCases[0][4]]]


ListDeaths = [["United States",US_MeanDeaths[0],US_MedianDeaths[0],US_ModeDeaths[0][0]],
        ["Bangladesh", FiveCountry_MeanDeaths[0],FiveCountry_MedianDeaths[0],FiveCountry_ModeDeaths[0][0]],
        ["Indonesia", FiveCountry_MeanDeaths[1],FiveCountry_MedianDeaths[1],FiveCountry_ModeDeaths[0][1]],
        ["Pakistan", FiveCountry_MeanDeaths[2],FiveCountry_MedianDeaths[2],FiveCountry_ModeDeaths[0][2]],
        ["Brazil", FiveCountry_MeanDeaths[3],FiveCountry_MedianDeaths[3],FiveCountry_ModeDeaths[0][3]],
       ["Nigeria", FiveCountry_MeanDeaths[4],FiveCountry_MedianDeaths[4],FiveCountry_ModeDeaths[0][4]]]<jupyter_output><empty_output><jupyter_text>WeeklyStatistics Comparision of Cases per 50 mil people.<jupyter_code>CountryWeeklyStatisticsCase = pd.DataFrame(ListCase, columns = ['Country','Mean', 'Median', 'Mode'])

CountryWeeklyStatisticsCase<jupyter_output><empty_output><jupyter_text>WeeklyStatistics Comparision of Deaths per 50 mil people.<jupyter_code>CountryWeeklyStatisticsDeath = pd.DataFrame(ListDeaths, columns = ['Country','Mean', 'Median', 'Mode'])

CountryWeeklyStatisticsDeath<jupyter_output><empty_output><jupyter_text>Analysis of Weekly Statistics: For United States, For both New Cases and New Death, the Mean is greater than the Median, So we can assume that its distrubution will be Right-Skewed.  - For Number of New Cases, Every country except for Brazil and Bangladesh seems to have a right-skewed distrubition. - For Number of New Deaths, United States, Indonesia and pakistan is assumed to have a  right-skewed distrubtion while Brazil and Bangladesh is assumed to have a left-skewed distrubition. - For Number of New Deaths, Nigeria seems to have a normal distrubtion.  United States has the highest average amount of new cases for every 50 million people, compared to other countries with similar populations, with Brazil not that far behind.  Compared to other countries, United States and Brazil's amount of new cases and deaths are really high.  Brazil has the highest average of new deaths on a weekly basis with United States closely behind.   Plot Daily Trends<jupyter_code>trace0 = go.Scatter(x=Norm_NewUScases.index, y=Norm_NewUScases["UnitedStates"], 
                    name="US", 
                    mode="lines+markers") 

trace1 = go.Scatter(x=Norm_CountryCases.index, y=Norm_CountryCases["Bangladesh"], 
                    name="Bangladesh", 
                    mode="lines+markers")  

trace2 = go.Scatter(x=Norm_CountryCases.index, y=Norm_CountryCases["Indonesia"], 
                    name="Indonesia", 
                    mode="lines+markers") 

trace3 = go.Scatter(x=Norm_CountryCases.index, y=Norm_CountryCases["Pakistan"], 
                    name="Pakistan", 
                    mode="lines+markers") 

trace4 = go.Scatter(x=Norm_CountryCases.index, y=Norm_CountryCases["Brazil"], 
                    name="Brazil", 
                    mode="lines+markers") 

trace5 = go.Scatter(x=Norm_CountryCases.index, y=Norm_CountryCases["Nigeria"], 
                    name="Nigeria", 
                    mode="lines+markers")

mydata = go.Data([trace0, trace1,trace2, trace3,trace4,trace5])

mylayout = go.Layout(
    title="Daily Trend of Cases per 50 million population"
)

fig = go.Figure(data=mydata, layout=mylayout)

plotly.offline.iplot(fig, filename = '')

fig.write_image("images/DailyTrend_NewCases_Countries.png")

trace0 = go.Scatter(x=WeeklyUS_cases.index, y=WeeklyUS_cases["UnitedStates"], 
                    name="US", 
                    mode="lines+markers") 

trace1 = go.Scatter(x=WeeklyCountries_cases.index, y=WeeklyCountries_cases["Bangladesh"], 
                    name="Bangladesh", 
                    mode="lines+markers")  

trace2 = go.Scatter(x=WeeklyCountries_cases.index, y=WeeklyCountries_cases["Indonesia"], 
                    name="Indonesia", 
                    mode="lines+markers") 

trace3 = go.Scatter(x=WeeklyCountries_cases.index, y=WeeklyCountries_cases["Pakistan"], 
                    name="Pakistan", 
                    mode="lines+markers") 

trace4 = go.Scatter(x=WeeklyCountries_cases.index, y=WeeklyCountries_cases["Brazil"], 
                    name="Brazil", 
                    mode="lines+markers") 

trace5 = go.Scatter(x=WeeklyCountries_cases.index, y=WeeklyCountries_cases["Nigeria"], 
                    name="Nigeria", 
                    mode="lines+markers")

mydata = go.Data([trace0, trace1,trace2, trace3,trace4,trace5])

mylayout = go.Layout(
    title="Weekly Trend of Cases per 50 million population"
)

fig2 = go.Figure(data=mydata, layout=mylayout)

plotly.offline.iplot(fig2, filename = '')

fig2.write_image("images/WeeklyTrend_NewCases_Countries,.png")
trace0 = go.Scatter(x=Norm_NewUSdeaths.index, y=Norm_NewUSdeaths["UnitedStates"], 
                    name="US", 
                    mode="lines+markers") 

trace1 = go.Scatter(x=Norm_Country_NewDeaths.index, y=Norm_Country_NewDeaths["Bangladesh"], 
                    name="Bangladesh", 
                    mode="lines+markers")  

trace2 = go.Scatter(x=Norm_Country_NewDeaths.index, y=Norm_Country_NewDeaths["Indonesia"], 
                    name="Indonesia", 
                    mode="lines+markers") 

trace3 = go.Scatter(x=Norm_Country_NewDeaths.index, y=Norm_Country_NewDeaths["Pakistan"], 
                    name="Pakistan", 
                    mode="lines+markers") 

trace4 = go.Scatter(x=Norm_Country_NewDeaths.index, y=Norm_Country_NewDeaths["Brazil"], 
                    name="Brazil", 
                    mode="lines+markers") 

trace5 = go.Scatter(x=Norm_Country_NewDeaths.index, y=Norm_Country_NewDeaths["Nigeria"], 
                    name="Nigeria", 
                    mode="lines+markers")

mydata = go.Data([trace0, trace1,trace2, trace3,trace4,trace5])

mylayout = go.Layout(
    title="Daily Trend of Deaths per 50 million population"
)

fig3 = go.Figure(data=mydata, layout=mylayout)

plotly.offline.iplot(fig3, filename = '')

fig3.write_image("images/DailyTrend_NewDeaths_Countries,.png")

trace0 = go.Scatter(x=WeeklyUS_deaths.index, y=WeeklyUS_deaths["UnitedStates"], 
                    name="US", 
                    mode="lines+markers") 

trace1 = go.Scatter(x=WeeklyCountries_deaths.index, y=WeeklyCountries_deaths["Bangladesh"], 
                    name="Bangladesh", 
                    mode="lines+markers")  

trace2 = go.Scatter(x=WeeklyCountries_deaths.index, y=WeeklyCountries_deaths["Indonesia"], 
                    name="Indonesia", 
                    mode="lines+markers") 

trace3 = go.Scatter(x=WeeklyCountries_deaths.index, y=WeeklyCountries_deaths["Pakistan"], 
                    name="Pakistan", 
                    mode="lines+markers") 

trace4 = go.Scatter(x=WeeklyCountries_deaths.index, y=WeeklyCountries_deaths["Brazil"], 
                    name="Brazil", 
                    mode="lines+markers") 

trace5 = go.Scatter(x=WeeklyCountries_deaths.index, y=WeeklyCountries_deaths["Nigeria"], 
                    name="Nigeria", 
                    mode="lines+markers")

mydata = go.Data([trace0, trace1,trace2, trace3,trace4,trace5])

mylayout = go.Layout(
    title="Weekly Trend of Deaths per 50 million population"
)

fig4 = go.Figure(data=mydata, layout=mylayout)

plotly.offline.iplot(fig4, filename = '')

fig4.write_image("images/WeeklyTrend_NewDeaths_Countries,.png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11372037399434659, 'fraction_numerical': 0.06767775603392041, 'mean_word_length': 3.058460180895654, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '519842', 'license': 'MIT', 'n_tokens_mistral': 7053, 'n_tokens_neox': 6103, 'n_words': 1130, 'path': 'src/Stage_II/Teams_Stage_II.ipynb', 'repo_name': 'HarinB4/Covid-19DataProject', 'size': '18396'}