<jupyter_start><jupyter_text>Introduction of dataBoth data files have the following construction:|Header|	Definition||---|---||year|	Year||month|	Month||date_of_month|	Day number of the month|day_of_week|	Day of week, where 1 is Monday and 7 is Sunday|births|	Number of births|<jupyter_code>import pandas as pd
US_births_1994_2003 = pd.read_csv('US_births_1994-2003_CDC_NCHS.csv')
US_births_2000_2014 = pd.read_csv('US_births_2000-2014_SSA.csv')
US_births_1994_2003.head()
US_births_2000_2014.head()<jupyter_output><empty_output><jupyter_text>Is the numer of births the same in the overlaping period?<jupyter_code>US_births_1994_2003[US_births_1994_2003['year'] == 2000].head()
US_births_2000_2014[US_births_2000_2014['year'] == 2000].head()<jupyter_output><empty_output><jupyter_text>Dealing with overlapThere is a overlap between the datasets and the number of births is not the same. One solution is to make a new dataset with average births for the overlapping period.<jupyter_code>import numpy as np
#select parts where the two df do not overlap
data94_99 = US_births_1994_2003[US_births_1994_2003['year'].between(1994,1999)] 
data04_14 = US_births_2000_2014[US_births_2000_2014['year'].between(2004,2014)]

# select parts where the two df overlap
data00_03_94 = US_births_1994_2003[US_births_1994_2003['year'].between(2000,2003)] 
data00_03_14 = US_births_2000_2014[US_births_2000_2014['year'].between(2000,2003)]

# get the values of birth column
births_94_03 = data00_03_94['births'].values
births_00_14 = data00_03_14['births'].values
# get average biths from 2000-2003
births00_03 = (births_94_03 + births_00_14) / 2

#select first 4 columns and make a df for period 2000-2003
data00_03 = data00_03_94[['year', 'month', 'date_of_month', 'day_of_week']]

#add the new column with average births 
data00_03['births'] = births00_03

#merging all the data into one df
frames = [data94_99,data00_03,data04_14]
data = pd.concat(frames)<jupyter_output><empty_output><jupyter_text>update - simpler way of handling the overlapAdd a new column in the form of datetime<jupyter_code># getting what is need for a timestamp:
date_94_03 = US_births_1994_2003.loc[:,['month','date_of_month','year']]
# rename 'date_of_month' to 'day', so pd.to_datetime recognizes what is what
date_94_03.rename(columns={'date_of_month':'day'},inplace = True)
# new column to original dataframe with datetime
US_births_1994_2003['date'] = pd.to_datetime(date_94_03)
US_births_1994_2003.head()
# getting what is need for a timestamp:
date_00_14 = US_births_2000_2014.loc[:,['month','date_of_month','year']]
# rename 'date_of_month' to 'day', so pd.to_datetime recognizes what is what
date_00_14.rename(columns={'date_of_month':'day'},inplace = True)
# new column to original dataframe with datetime
US_births_2000_2014['date'] = pd.to_datetime(date_00_14)
US_births_2000_2014.head()<jupyter_output><empty_output><jupyter_text>put the dataframes togetherUse concat to combine the dataframes<jupyter_code>dfs = [US_births_1994_2003, US_births_2000_2014]
df = pd.concat(dfs)
print(US_births_1994_2003.shape, US_births_2000_2014.shape, df.shape)<jupyter_output>(3652, 6) (5479, 6) (9131, 6)<jupyter_text>sort the overlap with pivot table<jupyter_code>births = df.pivot_table(index='date')
print(births.shape, data.shape)
births[births['year']==2000].head()<jupyter_output>(7670, 5) (7670, 5)<jupyter_text>Analysis  Number of births per month<jupyter_code>import matplotlib.pyplot as plt

total_births_month = data.pivot_table(index='month', values='births', aggfunc= sum)
total_births_month["months"] = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Agu','Sep','Okt','Nov','Dec']

plt.subplots(figsize=(10,5))
total_births_month["births"].plot.bar()
plt.xticks(np.arange(0,12,1),total_births_month['months'])
plt.show()

print(total_births_month.sort_values('births', ascending=False).reset_index())<jupyter_output><empty_output><jupyter_text>Total number of days per month<jupyter_code>data['month'].value_counts()<jupyter_output><empty_output><jupyter_text>Average births per day in a month<jupyter_code>avr_birth_month = data.pivot_table(index='month', values='births')
avr_birth_month["months"] = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Agu','Sep','Okt','Nov','Dec']

plt.subplots(figsize=(10,5))
avr_birth_month["births"].plot.bar()
plt.xticks(np.arange(0,12,1),avr_birth_month['months'])
plt.show()
print(avr_birth_month.sort_values('births', ascending=False).reset_index())<jupyter_output><empty_output><jupyter_text>Births per year<jupyter_code>births_year = data.pivot_table(index='year', values='births', aggfunc='sum')
plt.subplots(figsize=(10,5))
births_year["births"].plot.bar()
plt.show()
print(births_year.sort_values('births', ascending=False).reset_index())<jupyter_output><empty_output><jupyter_text>Births per weekday<jupyter_code>births_weekday = data.pivot_table(index='day_of_week', values='births', aggfunc='sum')
births_weekday['weekday'] = ['Mon','Tue','Wed','Thu','Fri','Sat','Sun']

plt.subplots(figsize=(10,5))
births_weekday['births'].plot.bar()
plt.xticks(np.arange(0,7,1),births_weekday['weekday'])
plt.show()

print(births_weekday.sort_values('births', ascending=False))
#what is the chance of weekend birth
weekdays = sum(births_weekday['births'][0:5])
weekends = sum(births_weekday['births'][5:7])
weekendchance = (weekends/(weekdays+weekends))
expected = 2/7
s = 'Chance of weekend birth: ' + repr(round(weekendchance*100,2)) +'%, What would be expected in equal distribution of births: ' + repr(round(expected*100,2))+'%'
print(s)
import seaborn as sns
sns.set_style('white')
sns.kdeplot(data["births"], shade=True)
sns.despine(left=True, bottom=True)

g = sns.FacetGrid(births, hue="day_of_week", size=6)
g.map(sns.kdeplot, "births", shade=True).add_legend()
sns.despine(left=True, bottom=True)
plt.show()<jupyter_output><empty_output><jupyter_text>Births 1994-2014<jupyter_code>births.reset_index(inplace=True)
import plotly
import plotly.graph_objs as go
plotly.offline.init_notebook_mode(connected=True)


plotly.offline.iplot({
    "data": [go.Scatter(x=births.date, y=births.births)],
    "layout": go.Layout(title="Births per date 1994-2014")
})<jupyter_output><empty_output><jupyter_text>Aggregated births per date 1994-2014<jupyter_code>year_dist = births.pivot_table(index=['month','date_of_month'], values=['births'],aggfunc='sum')
year_dist.head()
# just to make dates, so it is easier to read of the graph
from datetime import datetime
start = datetime.strptime('1/1/16','%m/%d/%y')
end = datetime.strptime('12/31/16','%m/%d/%y')

daterange = pd.date_range(start, end)
year_dates = [datetime.strftime(i,'%d-%b') for i in daterange]
plotly.offline.iplot({
    "data": [go.Scatter(x=year_dates,y=year_dist.births)],
    "layout": go.Layout(title="Distribution of births")
})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1488862837045721, 'fraction_numerical': 0.07195193434935522, 'mean_word_length': 9.126112759643917, 'pattern_counts': {'":': 4, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '315322', 'license': 'MIT', 'n_tokens_mistral': 3017, 'n_tokens_neox': 2538, 'n_words': 454, 'path': 'birth_analysis.ipynb', 'repo_name': 'bjornkristmanns/US_births', 'size': '6824'}