<jupyter_start><jupyter_text>Exploratory Data Analysis Data Cleaning<jupyter_code>df = pd.read_csv("/content/drive/My Drive/Colab Notebooks/PopulationDataset.csv")

#to know the columns and the header of the dataset
df.head()
#finding the missing values
print(df.isnull().sum())
#missing data heatmap
sns.heatmap(df.isnull(), cbar=False)
#splitting the dataframe based on the countries (Making a dataframe dictionary)

Cnt = df.Country.unique()
DataFrameDict = {elem : pd.DataFrame for elem in Cnt}
for key in DataFrameDict.keys():
  DataFrameDict[key] = df[:][df.Country == key]
#remaning the columns to allow selection
d = {"Urban Pop %": 10, "Urban Population":11 }
df = df.rename(columns=d)

#interpolating the values for urban population and urban population percentage
cols=[10,11]
df[cols] = df[cols].interpolate()

#changing the column names back to the original ones
m = {10:"Urban Pop %",11:"Urban Population"}
df = df.rename(columns=m)

#dropping NaN values that can't be computed
df=df.dropna()
#checking for missing values using a heatmap
sns.heatmap(df.isnull(), cbar=False)
#dropping irrelevant data (country global ranking)
df.drop(['Country Global Rank'], axis = 1)
#Density column is of type object, hence converting to float
def convert_me(val):
  new_val = val.replace(',','')
  return float(new_val)

df['Density (P/Km²)'].apply(convert_me)<jupyter_output><empty_output><jupyter_text>Graph Visualization<jupyter_code>#Fertility Rate Histogram over the years in Senegal

df2 = df[df['Country']=='Senegal'].iloc[::-1]

df2.hist(column = 'Fertility Rate')
plt.xlabel('Fertility Rate')
plt.ylabel('Frequency')
plt.title('Fertility Rate Histogram over the years in Senegal')

plt.show()
#Median Age Histogram over the years in Japan

df3 = df[df['Country']=='Japan'].iloc[::-1]

df3.hist(column = 'Median Age')
plt.xlabel('Median Age')
plt.ylabel('Frequency')
plt.title('Median Age Histogram over the years in Japan')

plt.show()
#Bar Chart for Density (P/Km²) vs Yearly Change over the years in India

x = df[df['Country']=='India']['Density (P/Km²)'].iloc[::-1]
y = df[df['Country']=='India']['Yearly Change']

plt.xlabel('Density (P/Km²)')
plt.ylabel('Yearly Change')
plt.title('Density (P/Km²) vs Yearly Change over the years in India')
plt.gcf().axes[0].yaxis.get_major_formatter().set_scientific(False)
plt.bar(x,y)

plt.show()
#Bar Chart for Population and Urban Population over the years in Ukraine

barWidth = 0.35
r1 = np.arange(len(df[df['Country']=='India']['Population']))
r2 = [x + barWidth for x in r1]

labels = np.arange(1955, 2020, 5)
labels = np.append(labels, [2016, 2017, 2018, 2019, 2020])
x= np.arange(len(labels))

fig, ax = plt.subplots(figsize = (10, 5))

rect1= ax.bar(r1, df[df['Country']=='Ukraine']['Population'], width = barWidth, label='Population')
rec2= ax.bar(r2, df[df['Country']=='Ukraine']['Urban Population'], width = barWidth, label='Urban Population')

plt.gcf().axes[0].yaxis.get_major_formatter().set_scientific(False)
plt.gcf().axes[0].xaxis.get_major_formatter().set_scientific(False)

ax.legend()
ax.set_xticks(x)
ax.set_xticklabels(labels)

ax.set_xlabel('Years')
ax.set_ylabel('Population')
ax.set_title('Population and Urban Population over the years in Ukraine')

plt.show()
#Bar Chart for Population Density over the years in the USA

x = df[df['Country'] == 'United States']['Year'][::-1]
x.drop([37, 38, 39,40], axis = 0,inplace=True)

y = df[df['Country'] == 'United States']['Density (P/Km²)'][::-1]
y.drop([37, 38, 39,40], axis = 0,inplace=True)

plt.bar(x,y, width = 1.5)

plt.xlabel('Years')
plt.ylabel('Population Density')
plt.title('Population Density over the years in the US')

plt.show()
#Bar Chart for Fertility Rate vs Yearly % Change in Sweden

labels = np.arange(1955, 2020, 5)
labels = np.append(labels, [2016, 2017, 2018, 2019, 2020])

x = np.arange(len(labels))
width = 0.35
fig, ax = plt.subplots(figsize = (10, 5))

rect1 = ax.bar(x-width/2, df[df['Country'] == 'Sweden']['Fertility Rate'], width, label = 'Fertility Rate')
rect2 = ax.bar(x+width/2, df[df['Country'] == 'Sweden']['Yearly % Change'], width, label = 'Yearly  % Change')

ax.set_ylabel('Rates')
ax.set_title('Fertility Rates and Yearly % Change in Population over the years')
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.legend()

fig.tight_layout()

plt.show()<jupyter_output><empty_output><jupyter_text>We see a fall in fertility rate as well as yearly % change in the 1990s, this was due to the economic crisis faced by the country during the 90s. The decrease in fertility rate in the 2010s was linked to the rising uncertainty with respect to the future, due to an increase in crime as well as climate threats.<jupyter_code>#Box Plot for Fertility Rate of all countries

plt.boxplot(df['Fertility Rate'])
plt.title('Box Plot for Fertility Rate')
plt.xticks([])
plt.show()
#Box Plot for Median Age of all countries

plt.boxplot(df['Median Age'])
plt.title('Box Plot for Median Age')

plt.show()<jupyter_output><empty_output><jupyter_text>Normalization and Standardization<jupyter_code>df1 = df[df.Country == 'India']
def getZscore(array):
  return (array - array.mean(axis=0)) / array.std(axis=0)

def getPvalue(array):
  return scipy.stats.norm.sf(abs(array)) *2

def normalize(array):
  newArray = (array - array.min()) / (array.max() - array.min())
  return newArray

def getValues(array):
  values = {'mean':array.mean(), 'variance':array.var(), 'normalized': normalize(array), 'z-score': getZscore(array)}
  normArr = values['normalized']
  values['normMean'] = round(normArr.mean(), 3)
  values['normVar'] = round(normArr.var(), 3)
  values['pValue']: getPvalue(values['z-score'])
  zscoreArr =  values['z-score']
  values['stdMean'] = round(zscoreArr.mean(), 3)
  values['stdVar'] = round(zscoreArr.var(), 3)
  return values

pop = getValues(np.array(list(df1['Population'])))
percChange = getValues(np.array(list(df1['Yearly % Change'])))
change = getValues(np.array(list(df1['Yearly Change'])))
migrants = getValues(np.array(list(df1['Migrants (net)'])))
medianAge = getValues(np.array(list(df1['Median Age'])))
fertility = getValues(np.array(list(df1['Fertility Rate'])))
density = getValues(np.array(list(df1['Density (P/Km²)'])).astype(np.float))
urbanPopPerc = getValues(np.array(list(df1['Urban Pop %'])))
urbanPop = getValues(np.array(list(df1['Urban Population'])))
worldPopShare = getValues(np.array(list(df1['Country\'s Share of World Pop %'])))
worldPop = getValues(np.array(list(df1['World Population'])))
print('MEAN')
print('Population:', pop['mean'])
print('Yearly % Change:', percChange['mean'])
print('Yearly Change:', change['mean'])
print('Migrants (net):', migrants['mean'])
print('Median Age:', medianAge['mean'])
print('Fertility Rate:', fertility['mean'])
print('Density (P/Km²):', density['mean'])
print('Urban Pop %:', urbanPopPerc['mean'])
print('Urban Population:', urbanPop['mean'])
print('Country\'s Share of World Pop %:', worldPopShare['mean'])
print('World Population:', worldPop['mean'])

print('\n')

print('VARIANCE')
print('Population:', pop['variance'])
print('Yearly % Change:', percChange['variance'])
print('Yearly Change:', change['variance'])
print('Migrants (net):', migrants['variance'])
print('Median Age:', medianAge['variance'])
print('Fertility Rate:', fertility['variance'])
print('Density (P/Km²):', density['variance'])
print('Urban Pop %:', urbanPopPerc['variance'])
print('Urban Population:', urbanPop['variance'])
print('Country\'s Share of World Pop %:', worldPopShare['variance'])
print('World Population:', worldPop['variance'])
print('NORMALIZED MEAN')
print('Population:', pop['normMean'])
print('Yearly % Change:', percChange['normMean'])
print('Yearly Change:', change['normMean'])
print('Migrants (net):', migrants['normMean'])
print('Median Age:', medianAge['normMean'])
print('Fertility Rate:', fertility['normMean'])
print('Density (P/Km²):', density['normMean'])
print('Urban Pop %:', urbanPopPerc['normMean'])
print('Urban Population:', urbanPop['normMean'])
print('Country\'s Share of World Pop %:', worldPopShare['normMean'])
print('World Population:', worldPop['normMean'])

print('\n')

print('NORMALIZED VARIANCE')
print('Population:', pop['normVar'])
print('Yearly % Change:', percChange['normVar'])
print('Yearly Change:', change['normVar'])
print('Migrants (net):', migrants['normVar'])
print('Median Age:', medianAge['normVar'])
print('Fertility Rate:', fertility['normVar'])
print('Density (P/Km²):', density['normVar'])
print('Urban Pop %:', urbanPopPerc['normVar'])
print('Urban Population:', urbanPop['normVar'])
print('Country\'s Share of World Pop %:', worldPopShare['normVar'])
print('World Population:', worldPop['normVar'])
print('STANDARDIZED MEAN')
print('Population:', pop['stdMean'])
print('Yearly % Change:', percChange['stdMean'])
print('Yearly Change:', change['stdMean'])
print('Migrants (net):', migrants['stdMean'])
print('Median Age:', medianAge['stdMean'])
print('Fertility Rate:', fertility['stdMean'])
print('Density (P/Km²):', density['stdMean'])
print('Urban Pop %:', urbanPopPerc['stdMean'])
print('Urban Population:', urbanPop['stdMean'])
print('Country\'s Share of World Pop %:', worldPopShare['stdMean'])
print('World Population:', worldPop['stdMean'])

print('\n')

print('STANDARDIZED VARIANCE')
print('Population:', pop['stdVar'])
print('Yearly % Change:', percChange['stdVar'])
print('Yearly Change:', change['stdVar'])
print('Migrants (net):', migrants['stdVar'])
print('Median Age:', medianAge['stdVar'])
print('Fertility Rate:', fertility['stdVar'])
print('Density (P/Km²):', density['stdVar'])
print('Urban Pop %:', urbanPopPerc['stdVar'])
print('Urban Population:', urbanPop['stdVar'])
print('Country\'s Share of World Pop %:', worldPopShare['stdVar'])
print('World Population:', worldPop['stdVar'])
labels = np.array(df1['Year'])

x = np.arange(len(labels))
width = 0.35
fig, ax = plt.subplots(figsize = (10, 5))

rect1 = ax.bar(x-width/2, df1['Fertility Rate'], width, label = 'Fertility Rate')
rect2 = ax.bar(x+width/2, df1['Yearly % Change'], width, label = 'Yearly  % Change')

ax.set_ylabel('Rates')
ax.set_xlabel('Year')
ax.set_title('Fertility Rate and Yearly % Change in Population over the years')
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.invert_xaxis()
ax.legend()

plt.show()
labels = np.array(df1['Year'])

x = np.arange(len(labels))
width = 0.35
fig, ax = plt.subplots(figsize = (10, 5))

rect1 = ax.bar(x-width/2, fertility['normalized'], width, label = 'Fertility Rate')
rect2 = ax.bar(x+width/2, percChange['normalized'], width, label = 'Yearly  % Change')

ax.set_ylabel('Rates')
ax.set_xlabel('Year')
ax.set_title('Normalized Fertility Rate and Yearly % Change in Population over the years')
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.invert_xaxis()
ax.legend()

plt.show()
plt.boxplot(fertility['normalized'])
plt.xticks([])
plt.ylabel('Normalized Fertility Rate')
plt.show()
plt.boxplot(percChange['normalized'])
plt.xticks([])
plt.ylabel('Normalized Yearly % Change')
plt.show()<jupyter_output><empty_output><jupyter_text>Correlation Plots<jupyter_code>#Median Age vs Fertility Rate Worldwide

plt.scatter(df['Median Age'], df['Fertility Rate'])
plt.xlabel('Median Age')
plt.ylabel('Fertility Rate')
plt.title('Median Age vs Fertility Rate')
plt.show()
#Fertility Rate and Yearly % Change for 2015 

x = df[df['Year'] == 2015]['Fertility Rate']
y = df[df['Year'] == 2015]['Yearly % Change']

plt.scatter(x, y) 
plt.xlabel('Fertility Rate')
plt.ylabel('Yearly % Change')
plt.title('Fertility Rate vs Yearly % Change')
plt.show()
df_nz = DataFrameDict['New Zealand'].iloc[::-1]

plt.scatter(df_nz['Population'],df_nz['Density (P/Km²)'])
plt.gcf().axes[0].xaxis.get_major_formatter().set_scientific(False)

plt.xlabel('Population')
plt.ylabel('Population Density')
plt.title('Population vs Population Density in New Zealand')
plt.show()
df_india = df[df['Country']=='India']

plt.scatter(df_india['Fertility Rate'],df_india['Median Age'])
plt.xlabel('Fertility Rate')
plt.ylabel('Median Age')
plt.title('Fertility Rate vs Median Age in India')
plt.show()
plt.scatter(df_india['Population'], df_india['Urban Population'])
plt.xlabel('Population')
plt.ylabel('Urban Population')
plt.title('Population vs Urban Population in India')
plt.gcf().axes[0].xaxis.get_major_formatter().set_scientific(False)
plt.gcf().axes[0].yaxis.get_major_formatter().set_scientific(False)
plt.show()
df_mol = df[df['Country']=='Moldova']

plt.scatter(df_mol['Migrants (net)'], df_mol['Density (P/Km²)'].iloc[::-1])
plt.xlabel('Migrants (net)')
plt.ylabel('Density (P/Km²)')
plt.title('Migrants (net) vs Density (P/Km²) in Moldova')
plt.gcf().axes[0].xaxis.get_major_formatter().set_scientific(False)
#plt.gcf().axes[0].yaxis.get_major_formatter().set_scientific(False)
plt.show()
plt.scatter(df_india['Population'], df_india['Fertility Rate'])
plt.xlabel('Population')
plt.ylabel('Fertility Rate')
plt.title('Population vs Fertility Rate in India')
plt.gcf().axes[0].xaxis.get_major_formatter().set_scientific(False)
plt.gcf().axes[0].yaxis.get_major_formatter().set_scientific(False)
plt.show()<jupyter_output><empty_output><jupyter_text>Hypothesis Testing<jupyter_code>df4 = df[df['Country'] == 'Cuba']
df5 = df[df['Country'] == 'Hungary']
print('Cuba\'s Population Mean:', df4['Population'].mean())
print('Hungary\'s Population Mean:', df5['Population'].mean())
print('Cuba\'s Population Density Mean:', df4['Density (P/Km²)'].astype(float).mean())
print('Hungary\'s Population Density Mean:', df5['Density (P/Km²)'].astype(float).mean())
 pd.set_option('display.max_rows', None)
 pd.set_option('display.max_columns', None)
 pd.set_option('display.width', None)
 pd.set_option('display.max_colwidth', -1)

m = df[df['Country'] == 'Hungary']['Fertility Rate'].mean()
print('Hungary\'s Mean Fertility Rate:', m)<jupyter_output>Hungary's Mean Fertility Rate: 1.7072222222222224<jupyter_text>H0: mu <= 1.707H1: mu > 1.707<jupyter_code>m0 = df[df['Country'] == 'Cuba']['Fertility Rate'].mean()
s0 = df[df['Country'] == 'Cuba']['Fertility Rate'].std()

print(m0, s0)
import math

xbar = 1.7072222222222224
s1 = 1.1064634824107376/math.sqrt(18)

print(xbar, s1)
#to compute the t-score

t = (m0 - xbar)/s1
print("THe t-score of the given test statistic is: ",t)<jupyter_output>THe t-score of the given test statistic is:  2.520063052061819
{'fraction_non_alphanumeric': 0.18747381650607456, 'fraction_numerical': 0.020527859237536656, 'mean_word_length': 7.288773148148148, 'pattern_counts': {'":': 2, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '120276', 'license': 'MIT', 'n_tokens_mistral': 5555, 'n_tokens_neox': 5017, 'n_words': 1135, 'path': 'PopulationAnalysis.ipynb', 'repo_name': 'kartika-nair/Population-Analysis', 'size': '14322'}